diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_AdaptiveLogSoftmax b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_AdaptiveLogSoftmax new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv1d_circular_stride2_pad2 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv1d_circular_stride2_pad2 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv1d_reflect_stride2_pad2 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv1d_reflect_stride2_pad2 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv1d_replicate_stride2_pad2 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv1d_replicate_stride2_pad2 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv2d_circular_stride2_pad2 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv2d_circular_stride2_pad2 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv2d_reflect_stride2_pad2 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv2d_reflect_stride2_pad2 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv2d_replicate_stride2_pad2 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv2d_replicate_stride2_pad2 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv3d_circular_stride2_pad2 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv3d_circular_stride2_pad2 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv3d_replicate_stride2_pad2 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Conv3d_replicate_stride2_pad2 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_CosineEmbeddingLoss_no_batch_dim_mean b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_CosineEmbeddingLoss_no_batch_dim_mean new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_CosineEmbeddingLoss_no_batch_dim_none b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_CosineEmbeddingLoss_no_batch_dim_none new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_CosineEmbeddingLoss_no_batch_dim_sum b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_CosineEmbeddingLoss_no_batch_dim_sum new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_EmbeddingBag_sparse b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_EmbeddingBag_sparse new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Embedding_sparse b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Embedding_sparse new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_HingeEmbeddingLoss_margin_no_reduce b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_HingeEmbeddingLoss_margin_no_reduce new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_HingeEmbeddingLoss_no_batch_dim_mean b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_HingeEmbeddingLoss_no_batch_dim_mean new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_HingeEmbeddingLoss_no_batch_dim_none b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_HingeEmbeddingLoss_no_batch_dim_none new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_HingeEmbeddingLoss_no_batch_dim_sum b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_HingeEmbeddingLoss_no_batch_dim_sum new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_HingeEmbeddingLoss_no_reduce b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_HingeEmbeddingLoss_no_reduce new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Linear b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Linear new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Linear_no_batch_dim b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Linear_no_batch_dim new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Linear_no_bias b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Linear_no_bias new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MarginRankingLoss_no_batch_dim_mean b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MarginRankingLoss_no_batch_dim_mean new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MarginRankingLoss_no_batch_dim_none b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MarginRankingLoss_no_batch_dim_none new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MarginRankingLoss_no_batch_dim_sum b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MarginRankingLoss_no_batch_dim_sum new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Mish_no_batch_dim b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Mish_no_batch_dim new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MultiLabelSoftMarginLoss_no_batch_dim_mean b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MultiLabelSoftMarginLoss_no_batch_dim_mean new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MultiLabelSoftMarginLoss_no_batch_dim_none b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MultiLabelSoftMarginLoss_no_batch_dim_none new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MultiLabelSoftMarginLoss_no_batch_dim_sum b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MultiLabelSoftMarginLoss_no_batch_dim_sum new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MultiLabelSoftMarginLoss_no_reduce b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MultiLabelSoftMarginLoss_no_reduce new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MultiLabelSoftMarginLoss_weights_no_reduce b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_MultiLabelSoftMarginLoss_weights_no_reduce new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PairwiseDistance b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PairwiseDistance new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PairwiseDistance_broadcast_lhs b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PairwiseDistance_broadcast_lhs new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PairwiseDistance_broadcast_rhs b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PairwiseDistance_broadcast_rhs new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PairwiseDistance_no_batch_dim b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PairwiseDistance_no_batch_dim new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PairwiseDistance_with_non_default_args b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PairwiseDistance_with_non_default_args new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PoissonNLLLoss_no_batch_dim_mean b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PoissonNLLLoss_no_batch_dim_mean new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PoissonNLLLoss_no_batch_dim_none b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PoissonNLLLoss_no_batch_dim_none new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PoissonNLLLoss_no_batch_dim_sum b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PoissonNLLLoss_no_batch_dim_sum new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PoissonNLLLoss_no_reduce b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_PoissonNLLLoss_no_reduce new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_SiLU_no_batch_dim b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_SiLU_no_batch_dim new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Softsign_no_batch_dim b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Softsign_no_batch_dim new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Tanhshrink_no_batch_dim b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Tanhshrink_no_batch_dim new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TransformerDecoderLayer_gelu_activation b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TransformerDecoderLayer_gelu_activation new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TransformerDecoderLayer_relu_activation b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TransformerDecoderLayer_relu_activation new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TransformerEncoderLayer_gelu_activation b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TransformerEncoderLayer_gelu_activation new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TransformerEncoderLayer_relu_activation b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TransformerEncoderLayer_relu_activation new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Transformer_multilayer_coder b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_Transformer_multilayer_coder new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TripletMarginLoss_no_batch_dim_mean b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TripletMarginLoss_no_batch_dim_mean new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TripletMarginLoss_no_batch_dim_none b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TripletMarginLoss_no_batch_dim_none new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TripletMarginLoss_no_batch_dim_sum b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_TripletMarginLoss_no_batch_dim_sum new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_affine_grid b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_affine_grid new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_affine_grid_3d b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_affine_grid_3d new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_bilinear_no_bias b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_bilinear_no_bias new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_cosine_embedding_loss_margin_no_reduce b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_cosine_embedding_loss_margin_no_reduce new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_cosine_embedding_loss_no_reduce b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_cosine_embedding_loss_no_reduce new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_elu_inplace_on_view b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_elu_inplace_on_view new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_grid_sample b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_grid_sample new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_interpolate b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_interpolate new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_bias_weightCOO b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_bias_weightCOO new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_bias_weightCSC b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_bias_weightCSC new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_bias_weightCSR b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_bias_weightCSR new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_nobias_weightCOO b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_nobias_weightCOO new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_nobias_weightCSC b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_nobias_weightCSC new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_nobias_weightCSR b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_linear_autograd_device_cpu_nobias_weightCSR new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_normalize b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_normalize new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_relu_inplace_on_view b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_relu_inplace_on_view new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_swap_module_params_poisons_acc_grad b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_swap_module_params_poisons_acc_grad new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_triplet_margin_loss b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_triplet_margin_loss new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_triplet_margin_loss_no_reduce b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_triplet_margin_loss_no_reduce new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_triplet_margin_loss_swap b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_triplet_margin_loss_swap new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_triplet_margin_loss_swap_no_reduce b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_triplet_margin_loss_swap_no_reduce new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_upsamplingLinear1d b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_upsamplingLinear1d new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_upsampling_bfloat16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_upsampling_bfloat16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_upsampling_not_recompute_scale_factor b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNN.test_upsampling_not_recompute_scale_factor new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_GRU_grad_and_gradgrad_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_GRU_grad_and_gradgrad_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_GroupNorm_general_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_GroupNorm_general_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_LSTM_differentiable_backward_using_oneDNN_cpu_bfloat16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_LSTM_differentiable_backward_using_oneDNN_cpu_bfloat16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_LSTM_differentiable_backward_using_oneDNN_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_LSTM_differentiable_backward_using_oneDNN_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_LSTM_grad_and_gradgrad_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_LSTM_grad_and_gradgrad_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_elu_inplace_with_neg_alpha_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_elu_inplace_with_neg_alpha_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_groupnorm_nhwc_cpu_bfloat16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_groupnorm_nhwc_cpu_bfloat16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_groupnorm_nhwc_cpu_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_groupnorm_nhwc_cpu_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_groupnorm_nhwc_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_groupnorm_nhwc_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_groupnorm_nhwc_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_groupnorm_nhwc_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_leaky_relu_inplace_with_neg_slope_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_leaky_relu_inplace_with_neg_slope_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_triplet_margin_with_distance_loss_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_triplet_margin_with_distance_loss_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_triplet_margin_with_distance_loss_default_parity_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_triplet_margin_with_distance_loss_default_parity_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_False_mode_bicubic_memory_format0_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_False_mode_bicubic_memory_format0_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_False_mode_bicubic_memory_format1_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_False_mode_bicubic_memory_format1_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_False_mode_bilinear_memory_format0_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_False_mode_bilinear_memory_format0_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_False_mode_bilinear_memory_format1_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_False_mode_bilinear_memory_format1_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_True_mode_bicubic_memory_format0_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_True_mode_bicubic_memory_format0_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_True_mode_bicubic_memory_format1_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_True_mode_bicubic_memory_format1_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_True_mode_bilinear_memory_format0_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_True_mode_bilinear_memory_format0_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_True_mode_bilinear_memory_format1_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_False_align_corners_True_mode_bilinear_memory_format1_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_True_align_corners_False_mode_bicubic_memory_format0_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_True_align_corners_False_mode_bicubic_memory_format0_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_True_align_corners_False_mode_bicubic_memory_format1_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_True_align_corners_False_mode_bicubic_memory_format1_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_True_align_corners_True_mode_bicubic_memory_format0_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_True_align_corners_True_mode_bicubic_memory_format0_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_True_align_corners_True_mode_bicubic_memory_format1_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingBiMode2d_antialias_True_align_corners_True_mode_bicubic_memory_format1_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingNearest1d_mode_nearest-exact_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingNearest1d_mode_nearest-exact_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingNearest1d_mode_nearest_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingNearest1d_mode_nearest_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingTrilinear3d_align_corners_False_memory_format0_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingTrilinear3d_align_corners_False_memory_format0_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingTrilinear3d_align_corners_False_memory_format1_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingTrilinear3d_align_corners_False_memory_format1_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingTrilinear3d_align_corners_True_memory_format0_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingTrilinear3d_align_corners_True_memory_format0_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingTrilinear3d_align_corners_True_memory_format1_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNDeviceTypeCPU.test_upsamplingTrilinear3d_align_corners_True_memory_format1_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNParametrization.test_new_spectral_norm_swap_False b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNNParametrization.test_new_spectral_norm_swap_False new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCPU.test_unbind_transpose_ragged_idx_3_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCPU.test_unbind_transpose_ragged_idx_3_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCPU.test_unbind_transpose_ragged_idx_last_dim_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCPU.test_unbind_transpose_ragged_idx_last_dim_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_False_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_False_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_False_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_False_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_False_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_False_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_jagged_requires_grad_True_contiguous_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_False_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_False_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_False_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_False_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_False_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_False_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_2_layout_strided_requires_grad_True_contiguous_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_False_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_False_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_False_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_False_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_False_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_False_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_jagged_requires_grad_True_contiguous_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_False_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_False_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_False_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_False_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_False_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_False_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_3_layout_strided_requires_grad_True_contiguous_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_False_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_False_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_False_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_False_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_False_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_False_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_jagged_requires_grad_True_contiguous_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_False_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_False_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_False_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_False_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_False_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_False_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_as_nested_tensor_from_tensor_dim_4_layout_strided_requires_grad_True_contiguous_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_autograd_function_with_None_grad_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_autograd_function_with_None_grad_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_binary_pointwise_broadcasting_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_binary_pointwise_broadcasting_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_binary_pointwise_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_binary_pointwise_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_binary_pointwise_transposed_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_binary_pointwise_transposed_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_chunk_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_chunk_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_flex_attention_converts_stacked_seq_indices_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_flex_attention_converts_stacked_seq_indices_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_flex_attention_noncontig_with_holes_False_cross_attention_False_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_flex_attention_noncontig_with_holes_False_cross_attention_False_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_flex_attention_noncontig_with_holes_False_cross_attention_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_flex_attention_noncontig_with_holes_False_cross_attention_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_as_nested_tensor_components_require_grad_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_as_nested_tensor_components_require_grad_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_as_nested_tensor_components_require_grad_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_as_nested_tensor_components_require_grad_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_as_nested_tensor_components_require_grad_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_as_nested_tensor_components_require_grad_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_False_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_False_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_False_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_False_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_False_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_False_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_layout_construction_nested_tensor_requires_grad_True_components_require_grad_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_False_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_False_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_False_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_False_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_False_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_False_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_jagged_view_from_values_offsets_requires_grad_True_values_is_view_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_linear_backward_memory_usage_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_linear_backward_memory_usage_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_linear_nt_dim_3_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_linear_nt_dim_3_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_linear_nt_dim_4_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_linear_nt_dim_4_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_linear_nt_dim_5_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_linear_nt_dim_5_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_njt_cat_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_njt_cat_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_reshape_decomp_requires_grad_True_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_reshape_decomp_requires_grad_True_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_cuda_bfloat16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_cuda_bfloat16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_with_constant_sequence_length_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_with_constant_sequence_length_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_with_constant_sequence_length_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_with_constant_sequence_length_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_with_constant_sequence_length_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_sdpa_with_constant_sequence_length_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_softmax_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_softmax_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_2_requires_grad_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_2_requires_grad_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_2_requires_grad_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_2_requires_grad_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_2_requires_grad_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_2_requires_grad_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_3_requires_grad_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_3_requires_grad_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_3_requires_grad_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_3_requires_grad_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_3_requires_grad_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_3_requires_grad_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_4_requires_grad_True_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_4_requires_grad_True_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_4_requires_grad_True_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_4_requires_grad_True_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_4_requires_grad_True_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_to_padded_tensor_nt_dim_4_requires_grad_True_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_unary_pointwise_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_unary_pointwise_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_unary_pointwise_transposed_inputs_cuda b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_unary_pointwise_transposed_inputs_cuda new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_unbind_backward_cuda_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_unbind_backward_cuda_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_unbind_backward_cuda_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_unbind_backward_cuda_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_unbind_backward_cuda_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestNestedTensorSubclassCUDA.test_unbind_backward_cuda_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestOpenReg.test_autograd_init b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestOpenReg.test_autograd_init new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_adaptive_pooling_empty_output_size_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_adaptive_pooling_empty_output_size_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_adaptive_pooling_empty_output_size_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_adaptive_pooling_empty_output_size_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_max_pool3d_ndhwc_cpu_bfloat16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_max_pool3d_ndhwc_cpu_bfloat16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_max_pool3d_ndhwc_cpu_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_max_pool3d_ndhwc_cpu_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_max_pool3d_ndhwc_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_max_pool3d_ndhwc_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_max_pool3d_ndhwc_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPoolingNNDeviceTypeCPU.test_max_pool3d_ndhwc_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestProfiler.test_profiler_fwd_bwd_link b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestProfiler.test_profiler_fwd_bwd_link new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestProfiler.test_source b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestProfiler.test_source new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPythonDispatch.test_custom_autograd b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPythonDispatch.test_custom_autograd new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPythonDispatch.test_shallow_copy_and_detach b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPythonDispatch.test_shallow_copy_and_detach new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPythonDispatch.test_subclass_autograd_device_check b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestPythonDispatch.test_subclass_autograd_device_check new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_amax_grad b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_amax_grad new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_amin_grad b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_amin_grad new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_backward b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_backward new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_mean_dim_grad b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_mean_dim_grad new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_mean_grad_case_1a b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_mean_grad_case_1a new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_mean_grad_case_1b b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_mean_grad_case_1b new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_mean_grad_case_1e b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_mean_grad_case_1e new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_prod_grad b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_prod_grad new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_sum_grad b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestReductions.test_sum_grad new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScatterGatherCPU.test_gather_backward_with_empty_index_tensor_sparse_grad_True_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScatterGatherCPU.test_gather_backward_with_empty_index_tensor_sparse_grad_True_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScatterGatherCPU.test_gather_backward_with_empty_index_tensor_sparse_grad_True_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScatterGatherCPU.test_gather_backward_with_empty_index_tensor_sparse_grad_True_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScript.test_cat b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScript.test_cat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScript.test_linear_grad b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScript.test_linear_grad new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScript.test_mm_batching b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScript.test_mm_batching new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScript.test_stack b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestScript.test_stack new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSelectiveActivationCheckpoint.test_flops_and_mem b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSelectiveActivationCheckpoint.test_flops_and_mem new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSC_masked_fast_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSC_masked_fast_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSC_masked_slow_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSC_masked_slow_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSC_nonmasked_fast_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSC_nonmasked_fast_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSC_nonmasked_slow_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSC_nonmasked_slow_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSR_masked_fast_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSR_masked_fast_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSR_masked_slow_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSR_masked_slow_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSR_nonmasked_fast_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSR_nonmasked_fast_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSR_nonmasked_slow_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseBSR_nonmasked_slow_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCOO_masked_fast_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCOO_masked_fast_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCOO_masked_slow_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCOO_masked_slow_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCOO_nonmasked_fast_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCOO_nonmasked_fast_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCOO_nonmasked_slow_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCOO_nonmasked_slow_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSC_masked_fast_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSC_masked_fast_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSC_masked_slow_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSC_masked_slow_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSC_nonmasked_fast_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSC_nonmasked_fast_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSC_nonmasked_slow_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSC_nonmasked_slow_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSR_masked_fast_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSR_masked_fast_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSR_masked_slow_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSR_masked_slow_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSR_nonmasked_fast_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSR_nonmasked_fast_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSR_nonmasked_slow_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_as_sparse_gradcheck_SparseCSR_nonmasked_slow_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_masked_fast_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_masked_fast_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_masked_fast_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_masked_fast_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_masked_slow_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_masked_slow_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_masked_slow_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_masked_slow_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_sparse_fast_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_sparse_fast_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_sparse_fast_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_sparse_fast_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_sparse_slow_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_sparse_slow_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_sparse_slow_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseBSR_sparse_slow_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_masked_fast_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_masked_fast_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_masked_fast_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_masked_fast_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_masked_slow_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_masked_slow_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_masked_slow_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_masked_slow_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_sparse_fast_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_sparse_fast_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_sparse_fast_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_sparse_fast_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_sparse_slow_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_sparse_slow_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_sparse_slow_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSC_sparse_slow_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_masked_fast_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_masked_fast_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_masked_fast_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_masked_fast_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_masked_slow_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_masked_slow_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_masked_slow_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_masked_slow_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_sparse_fast_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_sparse_fast_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_sparse_fast_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_sparse_fast_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_sparse_slow_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_sparse_slow_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_sparse_slow_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_mm_SparseCSR_sparse_slow_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSC_int64_masked_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSC_int64_masked_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSC_int64_masked_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSC_int64_masked_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSC_int64_sparse_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSC_int64_sparse_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSC_int64_sparse_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSC_int64_sparse_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSR_int64_masked_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSR_int64_masked_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSR_int64_masked_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSR_int64_masked_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSR_int64_sparse_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSR_int64_sparse_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSR_int64_sparse_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseBSR_int64_sparse_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCOO_int64_masked_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCOO_int64_masked_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCOO_int64_masked_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCOO_int64_masked_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCOO_int64_sparse_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCOO_int64_sparse_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCOO_int64_sparse_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCOO_int64_sparse_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSC_int64_masked_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSC_int64_masked_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSC_int64_masked_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSC_int64_masked_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSC_int64_sparse_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSC_int64_sparse_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSC_int64_sparse_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSC_int64_sparse_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSR_int64_masked_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSR_int64_masked_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSR_int64_masked_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSR_int64_masked_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSR_int64_sparse_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSR_int64_sparse_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSR_int64_sparse_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseAnyCPU.test_gradcheck_to_dense_SparseCSR_int64_sparse_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_Sparse_to_Sparse_copy__cpu_bfloat16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_Sparse_to_Sparse_copy__cpu_bfloat16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_Sparse_to_Sparse_copy__cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_Sparse_to_Sparse_copy__cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_Sparse_to_Sparse_copy__cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_Sparse_to_Sparse_copy__cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_log_softmax_zero_nnz_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_log_softmax_zero_nnz_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_log_softmax_zero_nnz_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_log_softmax_zero_nnz_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_softmax_zero_nnz_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_softmax_zero_nnz_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_softmax_zero_nnz_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_softmax_zero_nnz_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_mask_backward_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_mask_backward_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_mask_backward_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_mask_backward_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_matmul_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_matmul_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_matmul_cpu_complex64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_matmul_cpu_complex64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_matmul_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_matmul_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_matmul_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_matmul_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_mul_masked_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_mul_masked_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_mul_sparse_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCPU.test_sparse_mul_sparse_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_autograd_dense_output_addmm_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_autograd_dense_output_addmm_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_autograd_dense_output_addmv_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_autograd_dense_output_addmv_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_mul_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_mul_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_mul_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_mul_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sampled_addmm_autograd_cpu_complex128 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sampled_addmm_autograd_cpu_complex128 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sampled_addmm_autograd_cpu_complex64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sampled_addmm_autograd_cpu_complex64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sampled_addmm_autograd_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sampled_addmm_autograd_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sampled_addmm_autograd_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sampled_addmm_autograd_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sparse_mm_reduce_sum_cpu_bfloat16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sparse_mm_reduce_sum_cpu_bfloat16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sparse_mm_reduce_sum_cpu_float16 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sparse_mm_reduce_sum_cpu_float16 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sparse_mm_reduce_sum_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sparse_mm_reduce_sum_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sparse_mm_reduce_sum_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sparse_mm_reduce_sum_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sum_cpu_float32 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sum_cpu_float32 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sum_cpu_float64 b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestSparseCSRCPU.test_sum_cpu_float64 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_addcmul b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_addcmul new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_clamp b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_clamp new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_comparison_eq_ne b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_comparison_eq_ne new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_comparison_ge_le b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_comparison_ge_le new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_comparison_gt_lt b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_comparison_gt_lt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_concat_invariant b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_concat_invariant new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_exp b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_exp new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_hardsigmoid_fwd_bwd b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_hardsigmoid_fwd_bwd new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_hardswish_fwd_bwd b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_hardswish_fwd_bwd new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_inlined_optimized_graph b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_inlined_optimized_graph new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_lerp b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_lerp new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_lstm_concat b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_lstm_concat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_lstm_traced b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_lstm_traced new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_relu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_relu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_relu_fwd_bwd b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_relu_fwd_bwd new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_small_constant b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserDynamic.test_small_constant new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_addcmul b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_addcmul new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_chunk_distributes b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_chunk_distributes new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_clamp b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_clamp new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_comparison_eq_ne b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_comparison_eq_ne new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_comparison_ge_le b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_comparison_ge_le new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_comparison_gt_lt b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_comparison_gt_lt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_concat_invariant b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_concat_invariant new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_constant_chunk_shapes b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_constant_chunk_shapes new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_exp b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_exp new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_hardsigmoid_fwd_bwd b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_hardsigmoid_fwd_bwd new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_hardswish_fwd_bwd b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_hardswish_fwd_bwd new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_lerp b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_lerp new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_lstm_concat b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_lstm_concat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_lstm_traced b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_lstm_traced new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_milstm b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_milstm new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_relu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_relu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_relu_fwd_bwd b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_relu_fwd_bwd new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_small_constant b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestTEFuserStatic.test_small_constant new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestViewOpsCPU.test_as_strided_gradients_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestViewOpsCPU.test_as_strided_gradients_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestViewOpsLAZY.test_as_strided_gradients_lazy b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestViewOpsLAZY.test_as_strided_gradients_lazy new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestViewOpsLAZY.test_view_copy_lazy b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestViewOpsLAZY.test_view_copy_lazy new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapAPILegacy.test_batched_gradient_basic b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapAPILegacy.test_batched_gradient_basic new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapAPILegacy.test_fallback_with_undefined_grad b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapAPILegacy.test_fallback_with_undefined_grad new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_add_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_add_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_binary_cross_entropy_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_binary_cross_entropy_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_diagonal_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_diagonal_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_div_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_div_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_expand_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_expand_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_index_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_index_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_inplace_manyview_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_inplace_manyview_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_inplace_on_view_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_inplace_on_view_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_lgamma_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_lgamma_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_log1p_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_log1p_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_log_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_log_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_logsumexp_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_logsumexp_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_max_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_max_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_median_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_median_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_min_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_min_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_mul_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_mul_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_permute_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_permute_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_reshape_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_reshape_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_select_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_select_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_sigmoid_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_sigmoid_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_slice_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_slice_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_stack_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_stack_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_sub_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_sub_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_threshold_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_threshold_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_trace_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_trace_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_unrelated_output_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_unrelated_output_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_unrelated_output_multiple_grad_cpu b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestVmapBatchedGradientLegacyCPU.test_unrelated_output_multiple_grad_cpu new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestWithEffects.test_effectful_custom_op_with_subclasses b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestWithEffects.test_effectful_custom_op_with_subclasses new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestWithEffects.test_effects_and_aliased_outputs b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestWithEffects.test_effects_and_aliased_outputs new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestWithEffects.test_regular_effectful_op_in_forward_and_backward b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestWithEffects.test_regular_effectful_op_in_forward_and_backward new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/compiled_autograd_skips/TestWithEffects.test_regular_effectful_op_only_in_backward b/western_ai/usa/pytorch/test/compiled_autograd_skips/TestWithEffects.test_regular_effectful_op_only_in_backward new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/complex_tensor/test_complex_tensor.py b/western_ai/usa/pytorch/test/complex_tensor/test_complex_tensor.py new file mode 100644 index 0000000000000000000000000000000000000000..747ec9a0948d3ffcec2969b21b9dafae23f86aee --- /dev/null +++ b/western_ai/usa/pytorch/test/complex_tensor/test_complex_tensor.py @@ -0,0 +1,190 @@ +# Owner(s): ["module: complex"] +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +import torch.distributed as dist + + +# Support both when imported from elsewhere or directly as a file +try: + from .utils import ( + COMPLEX_DTYPES, + Descriptor, + force_test_op_db, + implemented_op_db, + TestCase, + Variant, + ) +except ImportError: + from utils import ( + COMPLEX_DTYPES, + Descriptor, + force_test_op_db, + implemented_op_db, + TestCase, + Variant, + ) + +from torch.testing._internal.common_device_type import ( + instantiate_device_type_tests, + OpDTypes, + ops, +) +from torch.testing._internal.common_utils import run_tests, unMarkDynamoStrictTest + + +if TYPE_CHECKING: + from torch.testing._internal.opinfo.core import OpInfo + +aten = torch.ops.aten + +SKIPS = { + Descriptor(op=aten.empty_like, variant=None): "Non-deterministic output", + Descriptor(op=aten.randn_like, variant=None): "Non-deterministic output", + Descriptor( + op=aten.any, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.all, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.allclose, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.conj_physical, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten._conj_physical, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.cumprod, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.index_add, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.diagonal_scatter, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.flip, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.masked_fill, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.masked_scatter, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.rsub, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.ne, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.squeeze, variant=Variant.Distributed + ): "does not have a sharding strategy registered", + Descriptor( + op=aten.index_select, variant=Variant.Distributed + ): "Sharding propagation failed", + Descriptor(op=aten.real, variant=Variant.Distributed): "No scalar support", + Descriptor(op=aten.imag, variant=Variant.Distributed): "No scalar support", + Descriptor(op=aten.isfinite, variant=Variant.Distributed): "No scalar support", + Descriptor(op=aten.transpose, variant=Variant.Distributed): "No scalar support", + Descriptor(op=aten.view_as_real, variant=Variant.Distributed): "No scalar support", +} + +EXTRA_KWARGS = { + Descriptor(op=aten.asinh, dtype=torch.complex64, variant=Variant.Op): { + "rtol": 2e-5, + "atol": 5e-5, + }, + Descriptor(op=aten.tanh, dtype=torch.complex64, variant=Variant.Op): { + "rtol": 1e-4, + "atol": 1e-5, + }, + Descriptor(op=aten.pow, dtype=torch.complex64, variant=Variant.Op): { + "rtol": 2e-2, + "atol": 2e-6, + }, + Descriptor(op=aten.asinh, dtype=torch.complex64, variant=Variant.Distributed): { + "rtol": 2e-5, + "atol": 5e-5, + }, + Descriptor(op=aten.tanh, dtype=torch.complex64, variant=Variant.Distributed): { + "rtol": 1e-4, + "atol": 1e-5, + }, + Descriptor(op=aten.pow, dtype=torch.complex64, variant=Variant.Distributed): { + "rtol": 2e-2, + "atol": 2e-6, + }, + Descriptor(op=aten.tan, dtype=torch.complex64, variant=Variant.Distributed): { + "rtol": 2e-6, + "atol": 1e-2, + }, +} + + +class TestComplexTensor(TestCase): + _default_dtype_check_enabled = True + + @ops( + implemented_op_db, + dtypes=OpDTypes.supported, + allowed_dtypes=list(COMPLEX_DTYPES), + ) + def test_consistency(self, device, dtype, op: OpInfo): + self.check_consistency(device, dtype, op, Variant.Op) + + @ops(force_test_op_db, allowed_dtypes=list(COMPLEX_DTYPES)) + def test_maybe_error(self, device, dtype, op: OpInfo): + self.check_consistency(device, dtype, op, Variant.Op) + + def test_get_set_components(self): + from torch._subclasses.complex_tensor import ComplexTensor + + c = ComplexTensor.from_interleaved( + torch.tensor([1 + 2j, 3 + 4j], dtype=torch.complex64) + ) + self.assertEqual(c.real, torch.tensor([1, 3], dtype=torch.float32)) + self.assertEqual(c.imag, torch.tensor([2, 4], dtype=torch.float32)) + + c.real = torch.tensor([5, 6]) + c.imag = torch.tensor([7, 8]) + self.assertEqual(c.real, torch.tensor([5, 6], dtype=torch.float32)) + self.assertEqual(c.imag, torch.tensor([7, 8], dtype=torch.float32)) + self.assertEqual(c, torch.tensor([5 + 7j, 6 + 8j], dtype=torch.complex64)) + + +@unMarkDynamoStrictTest +class TestComplexBwdGradients(TestCase): + _default_dtype_check_enabled = True + + @ops( + implemented_op_db, + dtypes=OpDTypes.supported_backward, + allowed_dtypes=[torch.complex128], + ) + def test_fn_grad(self, device: str, dtype: torch.dtype, op: OpInfo) -> None: + self.check_grad(device, dtype, op) + + +instantiate_device_type_tests(TestComplexTensor, globals()) +instantiate_device_type_tests(TestComplexBwdGradients, globals()) + + +if dist.is_available(): + from torch.testing._internal.common_distributed import MultiProcessTestCase + + @unMarkDynamoStrictTest + class TestComplexDistributed(TestCase, MultiProcessTestCase): + @ops(implemented_op_db, allowed_dtypes=list(COMPLEX_DTYPES)) + def test_distributed(self, device, dtype, op: OpInfo): + self.check_consistency(device, dtype, op, Variant.Distributed) + + instantiate_device_type_tests(TestComplexDistributed, globals()) + +if __name__ == "__main__": + run_tests() diff --git a/western_ai/usa/pytorch/test/complex_tensor/utils.py b/western_ai/usa/pytorch/test/complex_tensor/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..abd22d4faa5ad3d1ad230f822492a49117a69b72 --- /dev/null +++ b/western_ai/usa/pytorch/test/complex_tensor/utils.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +from dataclasses import dataclass, field, fields +from enum import auto, Enum +from typing import Any, TYPE_CHECKING + +import torch +import torch.distributed as dist +from torch._subclasses.complex_tensor._ops.common import ( + _as_complex_tensor, + _as_interleaved, + _get_op_name, + COMPLEX_OPS_TABLE, + COMPLEX_TO_REAL, + FORCE_TEST_LIST, + OpOverloadPacket, +) +from torch.autograd.gradcheck import gradcheck +from torch.testing._internal.common_methods_invocations import op_db +from torch.testing._internal.common_utils import TestCase as PytorchTestCase +from torch.utils._pytree import tree_flatten, tree_map + + +if TYPE_CHECKING: + from collections.abc import Callable + + from torch.distributed.tensor import DTensor + from torch.testing._internal.opinfo.core import OpInfo + +COMPLEX_DTYPES = set(COMPLEX_TO_REAL) + + +class Variant(Enum): + Op = auto() + GradCheck = auto() + Distributed = auto() + + +def _as_local(arg: DTensor | Any) -> torch.Tensor | Any: + if not (dist.is_available() and isinstance(arg, dist.tensor.DTensor)): + return arg + + return arg.full_tensor() + + +def _as_complex_dtensor(arg: torch.Tensor | Any) -> torch.Tensor | Any: + if not isinstance(arg, torch.Tensor): + return arg + + return dist.tensor.DTensor.from_local(_as_complex_tensor(arg)) + + +TRANSFORM_FUNCS = { + Variant.Op: _as_complex_tensor, + Variant.Distributed: _as_complex_dtensor, +} + + +@dataclass(frozen=True, kw_only=True) +class Descriptor: + op: OpOverloadPacket + variant: Variant | None + device_type: str | None = field(default=None) + dtype: torch.dtype | None = field(default=None) + + def matches(self, other: Descriptor) -> bool: + fields1 = fields(self) + fields2 = fields(other) + if fields1 != fields2: + return False + + for f in fields1: + f1 = getattr(self, f.name) + f2 = getattr(other, f.name) + if f1 is not None and f2 is not None and f1 != f2: + return False + + return True + + +class TestCase(PytorchTestCase): + def assertSameResult( + self, + expected: Callable[[], Any], + actual: Callable[[], Any], + *args, + **kwargs, + ) -> None: + try: + result_e = expected() + exception_e = None + except Exception as e: + result_e = None + exception_e = e + + try: + result_a = actual() + exception_a = None + except Exception as e: + result_a = None + exception_a = e + + if (exception_e is None) != (exception_a is None): + if exception_a is not None and exception_e is None: + raise exception_a + self.assertIs( + type(exception_e), + type(exception_a), + f"\n{exception_e=}\n{exception_a=}", + ) + + if exception_e is None: + flattened_e, spec_e = tree_flatten(result_e) + flattened_a, spec_a = tree_flatten(result_a) + + self.assertEqual( + spec_e, + spec_a, + "Both functions must return a result with the same tree structure.", + ) + for value_e, value_a in zip(flattened_e, flattened_a, strict=True): + value_e = _as_interleaved(_as_local(value_e)) + value_a = _as_interleaved(_as_local(value_a)) + + self.assertEqual(value_e, value_a, *args, **kwargs) + + def conditional_skip(self, desc: Descriptor) -> None: + try: + from .test_complex_tensor import SKIPS + except ImportError: + from test_complex_tensor import SKIPS + for xfail_info, reason in SKIPS.items(): + if xfail_info.matches(desc): + self.skipTest(reason) + + def get_extra_kwargs(self, desc: Descriptor) -> dict[str, Any]: + try: + from .test_complex_tensor import EXTRA_KWARGS + except ImportError: + from test_complex_tensor import EXTRA_KWARGS + + for extra_info, extra_kw in EXTRA_KWARGS.items(): + if extra_info.matches(desc): + return extra_kw + + return {} + + def check_consistency( + self, device: str, dtype: torch.dtype, op: OpInfo, variant: Variant + ) -> None: + if variant not in {Variant.Op, Variant.Distributed}: + raise AssertionError( + f"`check_consistency` called with the wrong `variant`: {variant}" + ) + desc = Descriptor( + op=get_overload_packet_from_name(op.name), + device_type=torch.device(device).type, + dtype=dtype, + variant=variant, + ) + self.conditional_skip(desc) + kwargs = self.get_extra_kwargs(desc) + + transform_fn = TRANSFORM_FUNCS[variant] + + sample_inputs = op.sample_inputs(device, dtype) + + for sample_input in sample_inputs: + + def expected(sample_input=sample_input): + return op( + sample_input.input, + *sample_input.args, + **sample_input.kwargs, + ) + + subclass_sample = sample_input.transform(transform_fn) + + def actual(subclass_sample=subclass_sample): + return op( + subclass_sample.input, + *subclass_sample.args, + **subclass_sample.kwargs, + ) + + self.assertSameResult(expected, actual, **kwargs) + + def check_grad(self, device: str, dtype: torch.dtype, op: OpInfo) -> None: + desc = Descriptor( + op=get_overload_packet_from_name(op.name), + device_type=torch.device(device).type, + dtype=dtype, + variant=Variant.GradCheck, + ) + self.conditional_skip(desc) + + sample_inputs = op.sample_inputs(device, dtype, requires_grad=True) + + for sample_input in sample_inputs: + # We don't use `sample_input.transform` here as it + # uses a `no_grad` context which makes one lose the + # `requires_grad` flag. + args, kwargs = (sample_input.input, *sample_input.args), sample_input.kwargs + args, kwargs = tree_map( + _as_complex_tensor, + (args, kwargs), + is_leaf=lambda x: isinstance(x, torch.Tensor) and x.dtype.is_complex, + ) + + if isinstance(args[0], torch.Tensor): + input_list = list(args) + + def func(*t: torch.Tensor): + return op(*t, **kwargs) + else: + # For ops like `stack` and `cat`, we need to specify + # the list of inputs a bit differently + input_list = list(args[0]) + + def func(*t: torch.Tensor): + return op(list(t), *args[1:], **kwargs) + + # Do the actual gradcheck + self.assertTrue( + gradcheck( + func, + input_list, + raise_exception=True, + fast_mode=True, + check_batched_grad=True, + check_forward_ad=True, + ) + ) + + +aten = torch.ops.aten + +complex_op_db = tuple( + filter(lambda op: any(op.supports_dtype(ct, "cpu") for ct in COMPLEX_DTYPES), op_db) +) + + +def get_overload_packet_from_name(name: str) -> OpOverloadPacket: + for domain_name in torch.ops: + op_namespace = getattr(torch.ops, domain_name) + op: OpOverloadPacket | None = getattr(op_namespace, name, None) + if op is not None: + return op + + raise RuntimeError(f"No op with {name=} found.") + + +force_test_names = set(map(_get_op_name, FORCE_TEST_LIST)) +implemented_op_names = ( + set(map(_get_op_name, COMPLEX_OPS_TABLE.keys())) - force_test_names +) +implemented_op_db = tuple( + filter(lambda op: op.name in implemented_op_names, complex_op_db) +) +force_test_op_db = tuple(filter(lambda op: op.name in force_test_names, op_db)) + +tested_op_names = {op.name for op in implemented_op_db} | { + op.name for op in force_test_op_db +} +non_tested_ops = { + op for op in COMPLEX_OPS_TABLE if _get_op_name(op) not in tested_op_names +} + + +# TODO (hameerabbasi): There are a number of ops that don't have any associated +# OpInfos. We still need to write tests for those ops. +if len(non_tested_ops) != 0: + import textwrap + import warnings + + list_missing_ops = "\n".join(sorted([str(op) for op in non_tested_ops])) + warnings.warn( + "Not all implemented ops are tested. List of ops missing tests:" + f"\n{textwrap.indent(list_missing_ops, ' ')}", + UserWarning, + stacklevel=2, + ) diff --git a/western_ai/usa/pytorch/test/conftest.py b/western_ai/usa/pytorch/test/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..20effe27cbe15769cb9c6a8964fbe9c9b596adf6 --- /dev/null +++ b/western_ai/usa/pytorch/test/conftest.py @@ -0,0 +1,367 @@ +import copy +import functools +import json +import os +import re +import sys +import xml.etree.ElementTree as ET +from collections import defaultdict +from types import MethodType +from typing import Any, TYPE_CHECKING + +import pytest +from _pytest.config import Config, filename_arg +from _pytest.config.argparsing import Parser +from _pytest.junitxml import _NodeReporter, bin_xml_escape, LogXML +from _pytest.python import Module +from _pytest.reports import TestReport +from _pytest.stash import StashKey +from _pytest.terminal import _get_raw_skip_reason + +from pytest_shard_custom import pytest_addoptions as shard_addoptions, PytestShardPlugin + + +try: + from torch.testing._internal.common_utils import parse_cmd_line_args +except ImportError: + # Temporary workaround needed until parse_cmd_line_args makes it into a nightlye because + # main / PR's tests are sometimes run against the previous day's nightly which won't + # have this function. + def parse_cmd_line_args(): + pass + + +if TYPE_CHECKING: + from _pytest._code.code import ReprFileLocation + +# a lot of this file is copied from _pytest.junitxml and modified to get rerun info + +xml_key = StashKey["LogXMLReruns"]() +STEPCURRENT_CACHE_DIR = "cache/stepcurrent" + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("general") + group.addoption("--scs", action="store", default=None, dest="stepcurrent_skip") + group.addoption("--sc", action="store", default=None, dest="stepcurrent") + group.addoption("--rs", action="store", default=None, dest="run_single") + + parser.addoption("--use-main-module", action="store_true") + group = parser.getgroup("terminal reporting") + group.addoption( + "--junit-xml-reruns", + action="store", + dest="xmlpath_reruns", + metavar="path", + type=functools.partial(filename_arg, optname="--junit-xml-reruns"), + default=None, + help="create junit-xml style report file at given path.", + ) + group.addoption( + "--junit-prefix-reruns", + action="store", + metavar="str", + default=None, + help="prepend prefix to classnames in junit-xml output", + ) + parser.addini( + "junit_suite_name_reruns", "Test suite name for JUnit report", default="pytest" + ) + parser.addini( + "junit_logging_reruns", + "Write captured log messages to JUnit report: " + "one of no|log|system-out|system-err|out-err|all", + default="no", + ) + parser.addini( + "junit_log_passing_tests_reruns", + "Capture log information for passing tests to JUnit report: ", + type="bool", + default=True, + ) + parser.addini( + "junit_duration_report_reruns", + "Duration time to report: one of total|call", + default="total", + ) + parser.addini( + "junit_family_reruns", + "Emit XML for schema: one of legacy|xunit1|xunit2", + default="xunit2", + ) + shard_addoptions(parser) + + +def pytest_configure(config: Config) -> None: + parse_cmd_line_args() + xmlpath = config.option.xmlpath_reruns + # Prevent opening xmllog on worker nodes (xdist). + if xmlpath and not hasattr(config, "workerinput"): + junit_family = config.getini("junit_family_reruns") + config.stash[xml_key] = LogXMLReruns( + xmlpath, + config.option.junitprefix, + config.getini("junit_suite_name_reruns"), + config.getini("junit_logging_reruns"), + config.getini("junit_duration_report_reruns"), + junit_family, + config.getini("junit_log_passing_tests_reruns"), + ) + config.pluginmanager.register(config.stash[xml_key]) + if config.getoption("stepcurrent_skip"): + config.option.stepcurrent = config.getoption("stepcurrent_skip") + if config.getoption("run_single"): + config.option.stepcurrent = config.getoption("run_single") + if config.getoption("stepcurrent"): + config.pluginmanager.register(StepcurrentPlugin(config), "stepcurrentplugin") + if config.getoption("num_shards"): + config.pluginmanager.register(PytestShardPlugin(config), "pytestshardplugin") + + +def pytest_unconfigure(config: Config) -> None: + xml = config.stash.get(xml_key, None) + if xml: + del config.stash[xml_key] + config.pluginmanager.unregister(xml) + + +class _NodeReporterReruns(_NodeReporter): + def _prepare_content(self, content: str, header: str) -> str: + return content + + def _write_content(self, report: TestReport, content: str, jheader: str) -> None: + if content == "": + return + tag = ET.Element(jheader) + tag.text = bin_xml_escape(content) + self.append(tag) + + def append_skipped(self, report: TestReport) -> None: + # Referenced from the below + # https://github.com/pytest-dev/pytest/blob/2178ee86d7c1ee93748cfb46540a6e40b4761f2d/src/_pytest/junitxml.py#L236C6-L236C6 + # Modified to escape characters not supported by xml in the skip reason. Everything else should be the same. + if hasattr(report, "wasxfail"): + # Super here instead of the actual code so we can reduce possible divergence + super().append_skipped(report) + else: + if not isinstance(report.longrepr, tuple): + raise AssertionError( + f"Expected report.longrepr to be tuple, got {type(report.longrepr)}" + ) + filename, lineno, skipreason = report.longrepr + skipreason = skipreason.removeprefix("Skipped: ") + details = f"{filename}:{lineno}: {skipreason}" + + skipped = ET.Element( + "skipped", type="pytest.skip", message=bin_xml_escape(skipreason) + ) + skipped.text = bin_xml_escape(details) + self.append(skipped) + self.write_captured_output(report) + + +class LogXMLReruns(LogXML): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def append_rerun(self, reporter: _NodeReporter, report: TestReport) -> None: + if hasattr(report, "wasxfail"): + reporter._add_simple("skipped", "xfail-marked test passes unexpectedly") + else: + if report.longrepr is None: + raise AssertionError("Expected report.longrepr to not be None") + reprcrash: ReprFileLocation | None = getattr( + report.longrepr, "reprcrash", None + ) + if reprcrash is not None: + message = reprcrash.message + else: + message = str(report.longrepr) + message = bin_xml_escape(message) + reporter._add_simple("rerun", message, str(report.longrepr)) + + def pytest_runtest_logreport(self, report: TestReport) -> None: + super().pytest_runtest_logreport(report) + if report.outcome == "rerun": + reporter = self._opentestcase(report) + self.append_rerun(reporter, report) + if report.outcome == "skipped": + if isinstance(report.longrepr, tuple): + fspath, lineno, reason = report.longrepr + reason = f"{report.nodeid}: {_get_raw_skip_reason(report)}" + report.longrepr = (fspath, lineno, reason) + + def node_reporter(self, report: TestReport | str) -> _NodeReporterReruns: + nodeid: str | TestReport = getattr(report, "nodeid", report) + # Local hack to handle xdist report order. + workernode = getattr(report, "node", None) + + key = nodeid, workernode + + if key in self.node_reporters: + # TODO: breaks for --dist=each + return self.node_reporters[key] + + reporter = _NodeReporterReruns(nodeid, self) + + self.node_reporters[key] = reporter + self.node_reporters_ordered.append(reporter) + + return reporter + + +# imitating summary_failures in pytest's terminal.py +# both hookwrapper and tryfirst to make sure this runs before pytest's +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_terminal_summary(terminalreporter, exitstatus, config): + # prints stack traces for reruns + if terminalreporter.config.option.tbstyle != "no": + reports = terminalreporter.getreports("rerun") + if reports: + terminalreporter.write_sep("=", "RERUNS") + if terminalreporter.config.option.tbstyle == "line": + for rep in reports: + line = terminalreporter._getcrashline(rep) + terminalreporter.write_line(line) + else: + for rep in reports: + msg = terminalreporter._getfailureheadline(rep) + terminalreporter.write_sep("_", msg, red=True, bold=True) + terminalreporter._outrep_summary(rep) + terminalreporter._handle_teardown_sections(rep.nodeid) + yield + + +@pytest.hookimpl(tryfirst=True) +def pytest_pycollect_makemodule(module_path, parent) -> Module: + if parent.config.getoption("--use-main-module"): + mod = Module.from_parent(parent, path=module_path) + mod._getobj = MethodType(lambda x: sys.modules["__main__"], mod) + return mod + + +@pytest.hookimpl(hookwrapper=True) +def pytest_report_teststatus(report, config): + # Add the test time to the verbose output, unfortunately I don't think this + # includes setup or teardown + pluggy_result = yield + if not isinstance(report, pytest.TestReport): + return + outcome, letter, verbose = pluggy_result.get_result() + if verbose: + pluggy_result.force_result( + (outcome, letter, f"{verbose} [{report.duration:.4f}s]") + ) + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(items: list[Any]) -> None: + """ + This hook is used when rerunning disabled tests to get rid of all skipped tests + instead of running and skipping them N times. This avoids flooding the console + and XML outputs with junk. So we want this to run last when collecting tests. + """ + rerun_disabled_tests = os.getenv("PYTORCH_TEST_RERUN_DISABLED_TESTS", "0") == "1" + if not rerun_disabled_tests: + return + + disabled_regex = re.compile(r"(?P.+)\s+\([^\.]+\.(?P.+)\)") + disabled_tests = defaultdict(set) + + # This environment has already been set by run_test before it calls pytest + disabled_tests_file = os.getenv("DISABLED_TESTS_FILE", "") + if not disabled_tests_file or not os.path.exists(disabled_tests_file): + return + + with open(disabled_tests_file) as fp: + for disabled_test in json.load(fp): + m = disabled_regex.match(disabled_test) + if m: + test_name = m["test_name"] + test_class = m["test_class"] + disabled_tests[test_class].add(test_name) + + # When rerunning disabled test, ignore all test cases that are not disabled + filtered_items = [] + + for item in items: + test_name = item.name + test_class = item.parent.name + + if ( + test_class not in disabled_tests + or test_name not in disabled_tests[test_class] + ): + continue + + cpy = copy.copy(item) + cpy._initrequest() + + filtered_items.append(cpy) + + items.clear() + # NB: Need to edit items directly here to have the list reflected back to pytest + items.extend(filtered_items) + + +class StepcurrentPlugin: + # Modified fromo _pytest/stepwise.py in order to save the currently running + # test instead of the last failed test + def __init__(self, config: Config) -> None: + self.config = config + self.report_status = "" + if config.cache is None: + raise AssertionError("Expected config.cache to not be None") + self.cache: pytest.Cache = config.cache + directory = f"{STEPCURRENT_CACHE_DIR}/{config.getoption('stepcurrent')}" + self.lastrun_location = f"{directory}/lastrun" + self.lastrun: str | None = self.cache.get(self.lastrun_location, None) + self.initial_val = self.lastrun + self.skip: bool = config.getoption("stepcurrent_skip") + self.run_single: bool = config.getoption("run_single") + + self.made_failing_xml_location = f"{directory}/made_failing_xml" + self.cache.set(self.made_failing_xml_location, False) + + def pytest_collection_modifyitems(self, config: Config, items: list[Any]) -> None: + if not self.lastrun: + self.report_status = "Cannot find last run test, not skipping" + return + + # check all item nodes until we find a match on last run + failed_index = None + for index, item in enumerate(items): + if item.nodeid == self.lastrun: + failed_index = index + if self.skip: + failed_index += 1 + break + + # If the previously failed test was not found among the test items, + # do not skip any tests. + if failed_index is None: + self.report_status = "previously run test not found, not skipping." + else: + self.report_status = f"skipping {failed_index} already run items." + deselected = items[:failed_index] + del items[:failed_index] + if self.run_single: + self.report_status += f" Running only {items[0].nodeid}" + deselected += items[1:] + del items[1:] + config.hook.pytest_deselected(items=deselected) + + def pytest_report_collectionfinish(self) -> str | None: + if self.config.getoption("verbose") >= 0 and self.report_status: + return f"stepcurrent: {self.report_status}" + return None + + def pytest_runtest_protocol(self, item, nextitem) -> None: + self.lastrun = item.nodeid + self.cache.set(self.lastrun_location, self.lastrun) + + def pytest_sessionfinish(self, session, exitstatus): + if exitstatus == 0: + self.cache.set(self.lastrun_location, self.initial_val) + if exitstatus != 0: + self.cache.set(self.made_failing_xml_location, True) diff --git a/western_ai/usa/pytorch/test/cpp/__init__.py b/western_ai/usa/pytorch/test/cpp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..2e9d43acba80bbdaa0fb9b7a1fd7a98fef72ce54 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/CMakeLists.txt @@ -0,0 +1,89 @@ +# Skip on windows +if(WIN32) + return() +endif() + +set(AOTI_ABI_CHECK_TEST_ROOT ${TORCH_ROOT}/test/cpp/aoti_abi_check) + +# Build the cpp gtest binary containing the cpp-only tests. +set(AOTI_ABI_CHECK_TEST_SRCS + ${AOTI_ABI_CHECK_TEST_ROOT}/main.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_accessor.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_cast.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_devicetype.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_dispatch.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_dispatch_v2.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_dtype.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_exception.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_headeronlyarrayref.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_layout.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_macros.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_math.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_memoryformat.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_metaprogramming.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_rand.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_scalartype.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_typelist.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_typetraits.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_vec.cpp + ${AOTI_ABI_CHECK_TEST_ROOT}/test_vec_half.cpp +) + +# The below are tests that require CPU_CAPABILITY setup +# You may think test_vec.cpp needs to be in there, but it does not. +set(AOTI_ABI_CHECK_VEC_TEST_SRCS + ${AOTI_ABI_CHECK_TEST_ROOT}/test_vec_half.cpp +) + +add_executable(test_aoti_abi_check + ${AOTI_ABI_CHECK_TEST_SRCS} +) + +# TODO temporary until we can delete the old gtest polyfills. +target_compile_definitions(test_aoti_abi_check PRIVATE USE_GTEST) + +# WARNING: DO NOT LINK torch!!! +# The purpose is to check if the used aten/c10 headers are written in a header-only way +target_link_libraries(test_aoti_abi_check PRIVATE gtest_main sleef) +target_include_directories(test_aoti_abi_check PRIVATE ${ATen_CPU_INCLUDE}) +if(NOT USE_SYSTEM_SLEEF) + target_include_directories(test_aoti_abi_check PRIVATE ${CMAKE_BINARY_DIR}/include) +endif() + +# Disable unused-variable warnings for variables that are only used to test compilation +target_compile_options_if_supported(test_aoti_abi_check -Wno-unused-variable) +target_compile_options_if_supported(test_aoti_abi_check -Wno-unused-but-set-variable) +# Add -Wno-dangling-pointer for GCC 13 +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13) + target_compile_options_if_supported(test_aoti_abi_check -Wno-dangling-pointer) +endif() + +foreach(test_src ${AOTI_ABI_CHECK_VEC_TEST_SRCS}) + foreach(i RANGE ${NUM_CPU_CAPABILITY_NAMES}) + get_filename_component(test_name ${test_src} NAME_WE) + list(GET CPU_CAPABILITY_NAMES ${i} CPU_CAPABILITY) + list(GET CPU_CAPABILITY_FLAGS ${i} FLAGS) + separate_arguments(FLAGS UNIX_COMMAND "${FLAGS}") + add_executable(${test_name}_${CPU_CAPABILITY} "${test_src}") + + target_link_libraries(${test_name}_${CPU_CAPABILITY} PRIVATE gtest_main sleef) + target_include_directories(${test_name}_${CPU_CAPABILITY} PRIVATE ${ATen_CPU_INCLUDE}) + if(NOT USE_SYSTEM_SLEEF) + target_include_directories(${test_name}_${CPU_CAPABILITY} PRIVATE ${CMAKE_BINARY_DIR}/include) + endif() + + # Define CPU_CAPABILITY and CPU_CAPABILITY_XXX macros for conditional compilation + target_compile_definitions(${test_name}_${CPU_CAPABILITY} PRIVATE CPU_CAPABILITY=${CPU_CAPABILITY} CPU_CAPABILITY_${CPU_CAPABILITY}) + target_compile_options(${test_name}_${CPU_CAPABILITY} PRIVATE ${FLAGS}) + target_compile_options_if_supported(${test_name}_${CPU_CAPABILITY} -Wno-unused-variable) + target_compile_options_if_supported(${test_name}_${CPU_CAPABILITY} -Wno-unused-but-set-variable) + endforeach() +endforeach() + +if(INSTALL_TEST) + install(TARGETS test_aoti_abi_check DESTINATION bin) + # Install PDB files for MSVC builds + if(MSVC AND BUILD_SHARED_LIBS) + install(FILES $ DESTINATION bin OPTIONAL) + endif() +endif() diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/README.md b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/README.md new file mode 100644 index 0000000000000000000000000000000000000000..80c4696dac1601f946e209597830275ef19a6cac --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/README.md @@ -0,0 +1,3 @@ +Tests in this directory are meant to guard certain ATen/c10 util functions and data structures are implemented in a header-only fashion, to make sure AOTInductor generated CPU model code is ABI backward-compatible. + +Tests that test functionality offered by the shims and require linking against torch should go into the `shim` test directory. diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/main.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..04a10046efe488caf859d8228ef833becdd796b2 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/main.cpp @@ -0,0 +1,6 @@ +#include + +int main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_accessor.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_accessor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..337582f06525bf6b07223bb46ef7c5bd18d623bc --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_accessor.cpp @@ -0,0 +1,50 @@ +#include +#include +#include + +TEST(TestAccessor, HeaderOnlyTensorAccessor) { + std::vector v = {11, 12, 13, 21, 22, 23}; + std::vector sizes = {2, 3}; + std::vector strides = {3, 1}; + + auto acc = torch::headeronly::HeaderOnlyTensorAccessor( + v.data(), sizes.data(), strides.data()); + EXPECT_EQ(acc[0][0], 11); + EXPECT_EQ(acc[0][1], 12); + EXPECT_EQ(acc[0][2], 13); + EXPECT_EQ(acc[1][0], 21); + EXPECT_EQ(acc[1][1], 22); + EXPECT_EQ(acc[1][2], 23); +} + +TEST(TestAccessor, HeaderOnlyGenericPackedTensorAccessor) { + std::vector v = {11, 12, 13, 21, 22, 23}; + std::vector sizes = {2, 3}; + std::vector strides = {3, 1}; + + auto acc = + torch::headeronly::HeaderOnlyGenericPackedTensorAccessor( + v.data(), sizes.data(), strides.data()); + EXPECT_EQ(acc[0][0], 11); + EXPECT_EQ(acc[0][1], 12); + EXPECT_EQ(acc[0][2], 13); + EXPECT_EQ(acc[1][0], 21); + EXPECT_EQ(acc[1][1], 22); + EXPECT_EQ(acc[1][2], 23); + + auto tacc = acc.transpose(0, 1); + EXPECT_EQ(tacc[0][0], 11); + EXPECT_EQ(tacc[0][1], 21); + EXPECT_EQ(tacc[1][0], 12); + EXPECT_EQ(tacc[1][1], 22); + EXPECT_EQ(tacc[2][0], 13); + EXPECT_EQ(tacc[2][1], 23); + + try { + acc.transpose(0, 2); + } catch (const std::exception& e) { + EXPECT_TRUE( + std::string(e.what()).find("HeaderOnlyIndexBoundsCheck") != + std::string::npos); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_cast.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_cast.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0b43081b4ed924d6a6ba1698f453c2e17722111e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_cast.cpp @@ -0,0 +1,25 @@ +#include + +#include +#include +namespace torch { +namespace aot_inductor { + +TEST(TestCast, TestConvert) { + c10::BFloat16 a = 3.0f; + c10::Half b = 3.0f; + + EXPECT_EQ(c10::convert(a), b); + EXPECT_EQ(a, c10::convert(b)); +} + +TEST(TestCast, TestBitcast) { + c10::BFloat16 a = 3.0f; + c10::Half b = 3.0f; + + EXPECT_EQ(c10::bit_cast(c10::bit_cast(a)), a); + EXPECT_EQ(c10::bit_cast(c10::bit_cast(b)), b); +} + +} // namespace aot_inductor +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_devicetype.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_devicetype.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b2ead6ab49ae37675410a6a83eaddb795c81e442 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_devicetype.cpp @@ -0,0 +1,35 @@ +#include + +#include + +TEST(TestDeviceType, TestDeviceType) { + using torch::headeronly::DeviceType; + constexpr DeviceType expected_device_types[] = { + torch::headeronly::kCPU, + torch::headeronly::kCUDA, + DeviceType::MKLDNN, + DeviceType::OPENGL, + DeviceType::OPENCL, + DeviceType::IDEEP, + torch::headeronly::kHIP, + torch::headeronly::kFPGA, + torch::headeronly::kMAIA, + torch::headeronly::kXLA, + torch::headeronly::kVulkan, + torch::headeronly::kMetal, + torch::headeronly::kXPU, + torch::headeronly::kMPS, + torch::headeronly::kMeta, + torch::headeronly::kHPU, + torch::headeronly::kVE, + torch::headeronly::kLazy, + torch::headeronly::kIPU, + torch::headeronly::kMTIA, + torch::headeronly::kPrivateUse1, + }; + for (int8_t i = 0; i < + static_cast(torch::headeronly::COMPILE_TIME_MAX_DEVICE_TYPES); + i++) { + EXPECT_EQ(static_cast(i), expected_device_types[i]); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_dispatch.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_dispatch.cpp new file mode 100644 index 0000000000000000000000000000000000000000..725bd1c9e2362e88e1e6047eee4395d9bfc2512c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_dispatch.cpp @@ -0,0 +1,82 @@ +#include + +#include +#include + +// MY_PRIVATE_CHECK_SELECTIVE_BUILD is a prelude to case block. For +// testing, we do nothing: +#define MY_PRIVATE_CHECK_SELECTIVE_BUILD(enum_type) /* empty */ + +#define MY_PRIVATE_CASE_TYPE_USING_HINT(...) \ + THO_PRIVATE_CASE_TYPE_USING_HINT_TMPL( \ + MY_PRIVATE_CHECK_SELECTIVE_BUILD, __VA_ARGS__) + +#define MY_DISPATCH_CASE(...) \ + THO_DISPATCH_CASE_TMPL(MY_PRIVATE_CASE_TYPE_USING_HINT, __VA_ARGS__) + +// MY_RECORD_KERNEL_FUNCTION_DTYPE is a prelude to switch +// statement. For testing, we just avoid unused variable warning: +#define MY_RECORD_KERNEL_FUNCTION_DTYPE(DISPATCHNAME, ENUMTYPE) \ + (void)DISPATCHNAME + +// MY_CHECK_NOT_IMPLEMENTED is called in switch default block. For +// testing, we count case mismatches: +#define MY_CHECK_NOT_IMPLEMENTED(...) default_count++ + +#define MY_DISPATCH_SWITCH(...) \ + THO_DISPATCH_SWITCH_TMPL( \ + MY_RECORD_KERNEL_FUNCTION_DTYPE, MY_CHECK_NOT_IMPLEMENTED, __VA_ARGS__) + +// MY_CASE_FUNCTION is called in a case block. For testing, we count +// case matches and ensure that scalar_t/index_t type is defined: +#define MY_CASE_FUNCTION \ + [&] { \ + count++; \ + scalar_t tmp; \ + (void)tmp; \ + } +#define MY_INDEX_CASE_FUNCTION \ + [&] { \ + count++; \ + index_t tmp; \ + (void)tmp; \ + } + +#define DEFINE_ITEM(TYPE, SCALARTYPE) ScalarType::SCALARTYPE, + +#define MY_DISPATCH_V2(TYPE, NAME, BODY, ...) \ + THO_DISPATCH_V2_TMPL( \ + MY_DISPATCH_SWITCH, \ + MY_DISPATCH_CASE, \ + TYPE, \ + NAME, \ + AT_WRAP(BODY), \ + __VA_ARGS__) + +#define TEST_DISPATCH_V2(NAME, EXPECTEDCOUNT, ...) \ + TEST(TestDispatchV2, NAME) { \ + using torch::headeronly::ScalarType; \ + using torch::headeronly::impl::ScalarTypeToCPPTypeT; \ + int8_t total_count = 0; \ + int8_t count = 0; \ + int8_t default_count = 0; \ + for (ScalarType t : \ + {AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(DEFINE_ITEM)}) { \ + total_count++; \ + MY_DISPATCH_V2(t, "test_my_dispatch_v2", MY_CASE_FUNCTION, __VA_ARGS__); \ + } \ + EXPECT_EQ(count, EXPECTEDCOUNT); \ + EXPECT_EQ(default_count + count, total_count); \ + } + +TEST_DISPATCH_V2(AT_FLOAT8_TYPES_, 5, AT_FLOAT8_TYPES); +TEST_DISPATCH_V2(AT_INTEGRAL_TYPES_, 5, AT_INTEGRAL_TYPES); +TEST_DISPATCH_V2(AT_FLOATING_TYPES_, 2, AT_FLOATING_TYPES); +TEST_DISPATCH_V2(AT_BAREBONES_UNSIGNED_TYPES_, 3, AT_BAREBONES_UNSIGNED_TYPES); +TEST_DISPATCH_V2(AT_INTEGRAL_TYPES_V2_, 8, AT_INTEGRAL_TYPES_V2); +TEST_DISPATCH_V2(AT_COMPLEX_TYPES_, 2, AT_COMPLEX_TYPES); +TEST_DISPATCH_V2(AT_QINT_TYPES_, 3, AT_QINT_TYPES); +TEST_DISPATCH_V2(AT_ALL_TYPES_, 7, AT_ALL_TYPES); +TEST_DISPATCH_V2(AT_ALL_TYPES_AND_COMPLEX_, 9, AT_ALL_TYPES_AND_COMPLEX); + +#undef DEFINE_ITEM diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_dispatch_v2.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_dispatch_v2.cpp new file mode 100644 index 0000000000000000000000000000000000000000..afc4b4c6c0d027f4d209c9f814a95b033a6e0754 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_dispatch_v2.cpp @@ -0,0 +1,45 @@ +#include +#include +#include + +#define DEFINE_ITEM(TYPE, SCALARTYPE) ScalarType::SCALARTYPE, + +#define TEST_DISPATCH_V2(NAME, EXPECTEDCOUNT, ...) \ + TEST(TestThoDispatchV2, NAME) { \ + using torch::headeronly::ScalarType; \ + using torch::headeronly::impl::ScalarTypeToCPPTypeT; \ + int8_t total_count = 0; \ + int8_t count = 0; \ + int8_t default_count = 0; \ + for (ScalarType t : \ + {AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(DEFINE_ITEM)}) { \ + total_count++; \ + try { \ + THO_DISPATCH_V2( \ + t, \ + "test_tho_dispatch_v2", \ + [&] { \ + count++; \ + scalar_t tmp; \ + (void)tmp; \ + }, \ + __VA_ARGS__); \ + } catch (...) { \ + default_count++; /* counts mismatches */ \ + } \ + } \ + EXPECT_EQ(count, EXPECTEDCOUNT); \ + EXPECT_EQ(default_count + count, total_count); \ + } + +TEST_DISPATCH_V2(AT_FLOAT8_TYPES_, 5, AT_FLOAT8_TYPES); +TEST_DISPATCH_V2(AT_INTEGRAL_TYPES_, 5, AT_INTEGRAL_TYPES); +TEST_DISPATCH_V2(AT_FLOATING_TYPES_, 2, AT_FLOATING_TYPES); +TEST_DISPATCH_V2(AT_BAREBONES_UNSIGNED_TYPES_, 3, AT_BAREBONES_UNSIGNED_TYPES); +TEST_DISPATCH_V2(AT_INTEGRAL_TYPES_V2_, 8, AT_INTEGRAL_TYPES_V2); +TEST_DISPATCH_V2(AT_COMPLEX_TYPES_, 2, AT_COMPLEX_TYPES); +TEST_DISPATCH_V2(AT_QINT_TYPES_, 3, AT_QINT_TYPES); +TEST_DISPATCH_V2(AT_ALL_TYPES_, 7, AT_ALL_TYPES); +TEST_DISPATCH_V2(AT_ALL_TYPES_AND_COMPLEX_, 9, AT_ALL_TYPES_AND_COMPLEX); + +#undef DEFINE_ITEM diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_dtype.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_dtype.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9c71acc27f1596f5ffa3c929080f73f1583a0238 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_dtype.cpp @@ -0,0 +1,210 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +TEST(TestDtype, TestBFloat16) { + torch::headeronly::BFloat16 a = 1.0f; + torch::headeronly::BFloat16 b = 2.0f; + torch::headeronly::BFloat16 add = 3.0f; + torch::headeronly::BFloat16 sub = -1.0f; + torch::headeronly::BFloat16 mul = 2.0f; + torch::headeronly::BFloat16 div = 0.5f; + + EXPECT_EQ(a + b, add); + EXPECT_EQ(a - b, sub); + EXPECT_EQ(a * b, mul); + EXPECT_EQ(a / b, div); +} + +TEST(TestDtype, TestFloat8_e4m3fn) { + torch::headeronly::Float8_e4m3fn a = 1.0f; + torch::headeronly::Float8_e4m3fn b = 2.0f; + torch::headeronly::Float8_e4m3fn add = 3.0f; + torch::headeronly::Float8_e4m3fn sub = -1.0f; + torch::headeronly::Float8_e4m3fn mul = 2.0f; + torch::headeronly::Float8_e4m3fn div = 0.5f; + + EXPECT_EQ(a + b, add); + EXPECT_EQ(a - b, sub); + EXPECT_EQ(a * b, mul); + EXPECT_EQ(a / b, div); +} + +TEST(TestDtype, TestFloat8_e4m3fuz) { + torch::headeronly::Float8_e4m3fnuz a = 1.0f; + torch::headeronly::Float8_e4m3fnuz b = 2.0f; + torch::headeronly::Float8_e4m3fnuz add = 3.0f; + torch::headeronly::Float8_e4m3fnuz sub = -1.0f; + torch::headeronly::Float8_e4m3fnuz mul = 2.0f; + torch::headeronly::Float8_e4m3fnuz div = 0.5f; + + EXPECT_EQ(a + b, add); + EXPECT_EQ(a - b, sub); + EXPECT_EQ(a * b, mul); + EXPECT_EQ(a / b, div); +} + +TEST(TestDtype, TestFloat8_e5m2) { + torch::headeronly::Float8_e5m2 a = 1.0f; + torch::headeronly::Float8_e5m2 b = 2.0f; + torch::headeronly::Float8_e5m2 add = 3.0f; + torch::headeronly::Float8_e5m2 sub = -1.0f; + torch::headeronly::Float8_e5m2 mul = 2.0f; + torch::headeronly::Float8_e5m2 div = 0.5f; + + EXPECT_EQ(a + b, add); + EXPECT_EQ(a - b, sub); + EXPECT_EQ(a * b, mul); + EXPECT_EQ(a / b, div); +} + +TEST(TestDtype, TestFloat8_e5m2fnuz) { + torch::headeronly::Float8_e5m2fnuz a = 1.0f; + torch::headeronly::Float8_e5m2fnuz b = 2.0f; + torch::headeronly::Float8_e5m2fnuz add = 3.0f; + torch::headeronly::Float8_e5m2fnuz sub = -1.0f; + torch::headeronly::Float8_e5m2fnuz mul = 2.0f; + torch::headeronly::Float8_e5m2fnuz div = 0.5f; + + EXPECT_EQ(a + b, add); + EXPECT_EQ(a - b, sub); + EXPECT_EQ(a * b, mul); + EXPECT_EQ(a / b, div); +} + +TEST(TestDtype, TestFloat8_e8m0fnu) { + torch::headeronly::Float8_e8m0fnu a = 1.0f; + ASSERT_FALSE(a.isnan()); +} + +TEST(TestDtype, TestFloat4) { + // not much you can do with this type, just make sure it compiles + torch::headeronly::Float4_e2m1fn_x2 a(5); +} + +TEST(TestDtype, TestHalf) { + torch::headeronly::Half a = 1.0f; + torch::headeronly::Half b = 2.0f; + torch::headeronly::Half add = 3.0f; + torch::headeronly::Half sub = -1.0f; + torch::headeronly::Half mul = 2.0f; + torch::headeronly::Half div = 0.5f; + + EXPECT_EQ(a + b, add); + EXPECT_EQ(a - b, sub); + EXPECT_EQ(a * b, mul); + EXPECT_EQ(a / b, div); + EXPECT_EQ(a += b, add); + EXPECT_EQ(a -= b, add - b); + EXPECT_EQ(a *= b, b); + EXPECT_EQ(a /= b, mul * div); + +#if defined(__aarch64__) && !defined(__CUDACC__) + EXPECT_EQ( + torch::headeronly::detail::fp16_to_bits( + torch::headeronly::detail::fp16_from_bits(32)), + 32); +#endif +} + +TEST(TestDtype, TestComplexFloat) { + torch::headeronly::complex a(std::complex(1.0f, 2.0f)); + torch::headeronly::complex b(std::complex(3.0f, 4.0f)); + torch::headeronly::complex add(std::complex(4.0f, 6.0f)); + torch::headeronly::complex sub(std::complex(-2.0f, -2.0f)); + torch::headeronly::complex mul(std::complex(-5.0f, 10.0f)); + torch::headeronly::complex div(std::complex(0.44f, 0.08f)); + + EXPECT_EQ(a + b, add); + EXPECT_EQ(a - b, sub); + EXPECT_EQ(a * b, mul); + EXPECT_EQ(a / b, div); +} + +TEST(TestDtype, TestQuintsQintsAndBits) { + // There's not much you can do with these dtypes... + // so we'll just check that it compiles + auto a = torch::headeronly::quint8(0); + auto b = torch::headeronly::quint4x2(5); + auto c = torch::headeronly::quint2x4(1); + auto d = torch::headeronly::qint32(5); + auto e = torch::headeronly::qint8(1); + auto f = torch::headeronly::bits1x8(9); + auto g = torch::headeronly::bits2x4(9); + auto h = torch::headeronly::bits4x2(9); + auto i = torch::headeronly::bits8(2); + auto j = torch::headeronly::bits16(6); +} + +TEST(TestDtype, TestScalarType) { + using torch::headeronly::ScalarType; + constexpr ScalarType expected_scalar_types[] = { + ScalarType::Byte, + ScalarType::Char, + ScalarType::Short, + ScalarType::Int, + ScalarType::Long, + ScalarType::Half, + ScalarType::Float, + ScalarType::Double, + ScalarType::ComplexHalf, + ScalarType::ComplexFloat, + ScalarType::ComplexDouble, + ScalarType::Bool, + ScalarType::QInt8, + ScalarType::QUInt8, + ScalarType::QInt32, + ScalarType::BFloat16, + ScalarType::QUInt4x2, + ScalarType::QUInt2x4, + ScalarType::Bits1x8, + ScalarType::Bits2x4, + ScalarType::Bits4x2, + ScalarType::Bits8, + ScalarType::Bits16, + ScalarType::Float8_e5m2, + ScalarType::Float8_e4m3fn, + ScalarType::Float8_e5m2fnuz, + ScalarType::Float8_e4m3fnuz, + ScalarType::UInt16, + ScalarType::UInt32, + ScalarType::UInt64, + ScalarType::UInt1, + ScalarType::UInt2, + ScalarType::UInt3, + ScalarType::UInt4, + ScalarType::UInt5, + ScalarType::UInt6, + ScalarType::UInt7, + ScalarType::Int1, + ScalarType::Int2, + ScalarType::Int3, + ScalarType::Int4, + ScalarType::Int5, + ScalarType::Int6, + ScalarType::Int7, + ScalarType::Float8_e8m0fnu, + ScalarType::Float4_e2m1fn_x2, + ScalarType::BComplex32, + ScalarType::Undefined, + }; + for (int8_t i = 0; i < static_cast(torch::headeronly::NumScalarTypes); + i++) { + EXPECT_EQ(static_cast(i), expected_scalar_types[i]); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_exception.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_exception.cpp new file mode 100644 index 0000000000000000000000000000000000000000..41ac9753b1cc3550508f5cbde0d0072f0a287ecf --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_exception.cpp @@ -0,0 +1,25 @@ +#include + +#include +#include + +namespace torch { +namespace aot_inductor { + +TEST(TestExceptions, TestStdTorchCheck) { + EXPECT_NO_THROW(STD_TORCH_CHECK(true, "dummy true message")); + EXPECT_NO_THROW(STD_TORCH_CHECK(true, "dummy ", "true ", "message")); + EXPECT_THROW( + STD_TORCH_CHECK(false, "dummy false message"), std::runtime_error); + EXPECT_THROW( + STD_TORCH_CHECK(false, "dummy ", "false ", "message"), + std::runtime_error); +} + +TEST(TestExceptions, TestTorchErrorCodeCheck) { + EXPECT_NO_THROW(TORCH_ERROR_CODE_CHECK(0)); + EXPECT_THROW(TORCH_ERROR_CODE_CHECK(1), std::runtime_error); +} + +} // namespace aot_inductor +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_headeronlyarrayref.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_headeronlyarrayref.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f6dd0cdc7d0e659948925b0cc198f97a493863d4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_headeronlyarrayref.cpp @@ -0,0 +1,87 @@ +#include + +#include + +#include + +using torch::headeronly::HeaderOnlyArrayRef; + +TEST(TestHeaderOnlyArrayRef, TestEmpty) { + HeaderOnlyArrayRef arr; + ASSERT_TRUE(arr.empty()); +} + +TEST(TestHeaderOnlyArrayRef, TestSingleton) { + float val = 5.0f; + HeaderOnlyArrayRef arr(val); + ASSERT_FALSE(arr.empty()); + EXPECT_EQ(arr.size(), 1); + EXPECT_EQ(arr[0], val); +} + +TEST(TestHeaderOnlyArrayRef, TestAPIs) { + std::vector vec = {1, 2, 3, 4, 5, 6, 7}; + HeaderOnlyArrayRef arr(vec); + ASSERT_FALSE(arr.empty()); + EXPECT_EQ(arr.size(), 7); + for (size_t i = 0; i < arr.size(); i++) { + EXPECT_EQ(arr[i], i + 1); + EXPECT_EQ(arr.at(i), i + 1); + } + EXPECT_EQ(arr.front(), 1); + EXPECT_EQ(arr.back(), 7); + ASSERT_TRUE(arr.slice(3, 4).equals(arr.slice(3))); +} + +TEST(TestHeaderOnlyArrayRef, TestFromInitializerList) { + std::vector vec = {1, 2, 3, 4, 5, 6, 7}; + HeaderOnlyArrayRef arr({1, 2, 3, 4, 5, 6, 7}); + auto res_vec = arr.vec(); + for (size_t i = 0; i < vec.size(); i++) { + EXPECT_EQ(vec[i], res_vec[i]); + } +} + +TEST(TestHeaderOnlyArrayRef, TestFromRange) { + std::vector vec = {1, 2, 3, 4, 5, 6, 7}; + HeaderOnlyArrayRef arr(vec.data() + 3, vec.data() + 7); + auto res_vec = arr.vec(); + for (size_t i = 0; i < res_vec.size(); i++) { + EXPECT_EQ(vec[i + 3], res_vec[i]); + } +} + +TEST(TestHeaderOnlyArrayRef, TestEqualityOperators) { + std::vector vec = {1, 2, 3, 4}; + std::vector same_vec = {1, 2, 3, 4}; + std::vector diff_vec = {1, 2, 3, 5}; + std::vector short_vec = {1, 2, 3}; + + HeaderOnlyArrayRef arr(vec); + HeaderOnlyArrayRef same_arr(same_vec); + HeaderOnlyArrayRef diff_arr(diff_vec); + HeaderOnlyArrayRef short_arr(short_vec); + + // HeaderOnlyArrayRef vs HeaderOnlyArrayRef + EXPECT_TRUE(arr == same_arr); + EXPECT_FALSE(arr != same_arr); + EXPECT_FALSE(arr == diff_arr); + EXPECT_TRUE(arr != diff_arr); + EXPECT_FALSE(arr == short_arr); + EXPECT_TRUE(arr != short_arr); + + // HeaderOnlyArrayRef should agree with .equals() + EXPECT_EQ(arr == same_arr, arr.equals(same_arr)); + EXPECT_EQ(arr == diff_arr, arr.equals(diff_arr)); + + // HeaderOnlyArrayRef vs std::vector (both orderings) + EXPECT_TRUE(arr == same_vec); + EXPECT_TRUE(same_vec == arr); + EXPECT_FALSE(arr != same_vec); + EXPECT_FALSE(same_vec != arr); + + EXPECT_FALSE(arr == diff_vec); + EXPECT_FALSE(diff_vec == arr); + EXPECT_TRUE(arr != diff_vec); + EXPECT_TRUE(diff_vec != arr); +} diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_layout.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_layout.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9a753724d85ec56ab08b5283c739ca79167067eb --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_layout.cpp @@ -0,0 +1,20 @@ +#include + +#include + +TEST(TestLayout, TestLayout) { + using torch::headeronly::Layout; + constexpr Layout expected_layouts[] = { + torch::headeronly::kStrided, + torch::headeronly::kSparse, + torch::headeronly::kSparseCsr, + torch::headeronly::kMkldnn, + torch::headeronly::kSparseCsc, + torch::headeronly::kSparseBsr, + torch::headeronly::kSparseBsc, + torch::headeronly::kJagged, + }; + for (int8_t i = 0; i < static_cast(Layout::NumOptions); i++) { + EXPECT_EQ(static_cast(i), expected_layouts[i]); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_macros.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_macros.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c29b0b097846d22314aedf64ae7198e009bfdee6 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_macros.cpp @@ -0,0 +1,18 @@ +#include + +#include + +namespace torch { +namespace aot_inductor { + +C10_API bool equal(int a, int b) { + return a == b; +} + +TEST(TestMacros, TestC10API) { + EXPECT_TRUE(equal(1, 1)); + EXPECT_FALSE(equal(1, 2)); +} + +} // namespace aot_inductor +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_math.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_math.cpp new file mode 100644 index 0000000000000000000000000000000000000000..79b7662d6d634aa896598d688e107d1d8e7765a3 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_math.cpp @@ -0,0 +1,23 @@ +#include + +#include +#include +#include +namespace torch { +namespace aot_inductor { + +TEST(TestMath, TestDivFloor) { + EXPECT_EQ(c10::div_floor_floating(5., 0.), INFINITY); + EXPECT_DOUBLE_EQ(c10::div_floor_floating(5., 2.), 2.); + EXPECT_DOUBLE_EQ(c10::div_floor_floating(5., -2.), -3.); + EXPECT_EQ(c10::div_floor_integer(5, 2), 2); + EXPECT_EQ(c10::div_floor_integer(5, -2), -3); +} + +TEST(TestMath, TestNan) { + EXPECT_FALSE(at::_isnan(1.0)); + EXPECT_TRUE(at::_isnan(std::nan(""))); +} + +} // namespace aot_inductor +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_memoryformat.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_memoryformat.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1d28cbe95f9613545b7a863005d5daebbd9c599c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_memoryformat.cpp @@ -0,0 +1,23 @@ +#include + +#include + +TEST(TestMemoryFormat, TestMemoryFormat) { + using torch::headeronly::MemoryFormat; + constexpr MemoryFormat expected_memory_formats[] = { + MemoryFormat::Contiguous, + MemoryFormat::Preserve, + MemoryFormat::ChannelsLast, + MemoryFormat::ChannelsLast3d, + }; + for (int8_t i = 0; i < static_cast(MemoryFormat::NumOptions); i++) { + EXPECT_EQ(static_cast(i), expected_memory_formats[i]); + } +} + +TEST(TestMemoryFormat, get_contiguous_memory_format) { + using torch::headeronly::get_contiguous_memory_format; + using torch::headeronly::MemoryFormat; + + EXPECT_EQ(get_contiguous_memory_format(), MemoryFormat::Contiguous); +} diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_metaprogramming.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_metaprogramming.cpp new file mode 100644 index 0000000000000000000000000000000000000000..170d99cb6d90dd126d47a9662fb2e028844aca6e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_metaprogramming.cpp @@ -0,0 +1,312 @@ +#include +#include +#include + +using namespace torch::headeronly::guts; + +// NOLINTBEGIN(modernize*, cppcoreguidelines-special-member-functions) +namespace { + +namespace test_function_traits { +static_assert( + std::is_same< + void, + typename function_traits::return_type>::value, + ""); +static_assert( + std::is_same::return_type>:: + value, + ""); +static_assert( + std::is_same< + typelist::typelist, + typename function_traits::parameter_types>::value, + ""); +static_assert( + std::is_same< + typelist::typelist, + typename function_traits::parameter_types>::value, + ""); + +static_assert( + std::is_same< + bool, + typename make_function_traits_t>:: + return_type>::value, + ""); +static_assert( + std::is_same< + void, + typename make_function_traits_t>:: + return_type>::value, + ""); +static_assert( + std::is_same< + typelist::typelist, + typename make_function_traits_t>:: + parameter_types>::value, + ""); +static_assert( + std::is_same< + typelist::typelist, + typename make_function_traits_t>:: + parameter_types>::value, + ""); +static_assert( + std::is_same< + bool(int, float), + typename make_function_traits_t>:: + func_type>::value, + ""); +static_assert( + std::is_same< + void(int, float), + typename make_function_traits_t>:: + func_type>::value, + ""); + +struct Functor final { + std::string operator()(int64_t a, float b) const; +}; +static_assert( + std::is_same< + std::string(int64_t, float), + typename infer_function_traits_t::func_type>::value, + ""); +} // namespace test_function_traits + +struct MovableOnly { + constexpr MovableOnly(int val_) : val(val_) { /* no default constructor */ } + MovableOnly(const MovableOnly&) = delete; + MovableOnly(MovableOnly&&) = default; + MovableOnly& operator=(const MovableOnly&) = delete; + MovableOnly& operator=(MovableOnly&&) = default; + + friend bool operator==(const MovableOnly& lhs, const MovableOnly& rhs) { + return lhs.val == rhs.val; + } + + private: + int val; +}; + +template +using is_my_movable_only_class = + std::is_same>>; + +struct CopyCounting { + int move_count{0}; + int copy_count{0}; + + CopyCounting() {} + CopyCounting(const CopyCounting& rhs) + : move_count(rhs.move_count), copy_count(rhs.copy_count + 1) {} + CopyCounting(CopyCounting&& rhs) noexcept + : move_count(rhs.move_count + 1), copy_count(rhs.copy_count) {} + CopyCounting& operator=(const CopyCounting& rhs) { + move_count = rhs.move_count; + copy_count = rhs.copy_count + 1; + return *this; + } + CopyCounting& operator=(CopyCounting&& rhs) noexcept { + move_count = rhs.move_count + 1; + copy_count = rhs.copy_count; + return *this; + } +}; + +template +using is_my_copy_counting_class = + std::is_same>>; + +namespace test_tuple_elements { +// note: not testing empty selection, as some compilers will raise +// "parameter set but not used" in tuple_elements(). a good example +// of the friction that comes with using these tools + +TEST(MetaprogrammingTest, TupleElements_subsetSelection) { + auto x = std::make_tuple(0, "HEY", 2.0); + auto y = tuple_elements(x, std::index_sequence<0, 2>()); + auto z = std::make_tuple(0, 2.0); + EXPECT_EQ(y, z); +} + +TEST(MetaprogrammingTest, TupleElements_reorderSelection) { + auto x = std::make_tuple(0, "HEY", 2.0); + auto y = tuple_elements(x, std::index_sequence<0, 2, 1>()); + auto z = std::make_tuple(0, 2.0, "HEY"); + EXPECT_EQ(y, z); +} +} // namespace test_tuple_elements + +namespace test_tuple_take { +// note: not testing empty prefix, see note on empty selection above. + +TEST(MetaprogrammingTest, TupleTake_nonemptyPrefix) { + auto x = std::make_tuple(0, "HEY", 2.0); + auto y = tuple_take(x); + auto z = std::make_tuple(0, "HEY"); + EXPECT_EQ(y, z); +} + +TEST(MetaprogrammingTest, TupleTake_fullPrefix) { + auto x = std::make_tuple(0, "HEY", 2.0); + auto y = tuple_take(x); + EXPECT_EQ(x, y); +} + +TEST(MetaprogrammingTest, TupleTake_negative) { + auto x = std::make_tuple(0, "HEY", 2.0); + auto y = tuple_take(x); + auto z = std::make_tuple("HEY", 2.0); + EXPECT_EQ(y, z); +} +} // namespace test_tuple_take + +namespace test_tuple_slice { +TEST(MetaprogrammingTest, TupleSlice_middle) { + auto x = std::make_tuple(0, "HEY", 2.0, false); + auto y = tuple_slice(x); + auto z = std::make_tuple("HEY", 2.0); + EXPECT_EQ(y, z); +} + +TEST(MetaprogrammingTest, TupleSlice_full) { + auto x = std::make_tuple(0, "HEY", 2.0); + auto y = tuple_slice(x); + EXPECT_EQ(x, y); +} +} // namespace test_tuple_slice + +namespace test_tuple_map { +TEST(MetaprogrammingTest, TupleMap_simple) { + auto result = tuple_map( + std::tuple(3, 4, 5), + [](int32_t a) -> int16_t { return static_cast(a + 1); }); + static_assert( + std::is_same, decltype(result)>:: + value, + ""); + EXPECT_EQ(4, std::get<0>(result)); + EXPECT_EQ(5, std::get<1>(result)); + EXPECT_EQ(6, std::get<2>(result)); +} + +TEST(MetaprogrammingTest, TupleMap_mapperTakesDifferentButConvertibleType) { + auto result = tuple_map( + std::tuple(3, 4, 5), + [](int64_t a) -> int16_t { return static_cast(a + 1); }); + static_assert( + std::is_same, decltype(result)>:: + value, + ""); + EXPECT_EQ(4, std::get<0>(result)); + EXPECT_EQ(5, std::get<1>(result)); + EXPECT_EQ(6, std::get<2>(result)); +} + +TEST(MetaprogrammingTest, TupleMap_mapperTakesConstRef) { + auto result = tuple_map( + std::tuple(3, 4, 5), + [](const int32_t& a) -> int16_t { return static_cast(a + 1); }); + static_assert( + std::is_same, decltype(result)>:: + value, + ""); + EXPECT_EQ(4, std::get<0>(result)); + EXPECT_EQ(5, std::get<1>(result)); + EXPECT_EQ(6, std::get<2>(result)); +} + +TEST(MetaprogrammingTest, TupleMap_mapsToDifferentTypes) { + struct Mapper { + std::string operator()(int32_t a) const { + return std::to_string(a); + } + int32_t operator()(const std::string& a) const { + return atoi(a.c_str()); + } + }; + auto result = tuple_map(std::tuple(3, "4"), Mapper()); + static_assert( + std::is_same, decltype(result)>::value, + ""); + EXPECT_EQ("3", std::get<0>(result)); + EXPECT_EQ(4, std::get<1>(result)); +} + +TEST(MetaprogrammingTest, TupleMap_differentiatesLRValueReferences) { + struct Mapper { + // NOLINTNEXTLINE(*move*) + std::string operator()(std::string&& a) const { + return "moved"; + } + std::string operator()(const std::string& a) const { + return "copied"; + } + }; + std::string str1, str2; + auto result = tuple_map( + std::tuple(str1, std::move(str2)), + Mapper()); + static_assert( + std::is_same, decltype(result)>:: + value, + ""); + EXPECT_EQ("copied", std::get<0>(result)); + EXPECT_EQ("moved", std::get<1>(result)); +} + +TEST(MetaprogrammingTest, TupleMap_canWorkWithMovableOnlyType) { + auto result = tuple_map( + std::tuple(MovableOnly(7)), [](MovableOnly a) { return a; }); + static_assert( + std::is_same, decltype(result)>::value, ""); + EXPECT_EQ(MovableOnly(7), std::get<0>(result)); +} + +TEST(MetaprogrammingTest, TupleMap_doesntUnecessarilyCopyValues) { + auto result = tuple_map( + std::tuple(CopyCounting()), + [](CopyCounting a) { return a; }); + static_assert( + std::is_same, decltype(result)>::value, ""); + EXPECT_EQ(4, std::get<0>(result).move_count); + EXPECT_EQ(0, std::get<0>(result).copy_count); +} + +TEST(MetaprogrammingTest, TupleMap_doesntUnecessarilyMoveValues) { + CopyCounting a; + auto result = tuple_map( + std::tuple(std::move(a)), + [](CopyCounting&& a) -> CopyCounting&& { return std::move(a); }); + static_assert( + std::is_same, decltype(result)>::value, ""); + EXPECT_EQ(&a, &std::get<0>(result)); + EXPECT_EQ(0, std::get<0>(result).move_count); + EXPECT_EQ(0, std::get<0>(result).copy_count); +} + +TEST(MetaprogrammingTest, TupleMap_canBeUsedWithAutoLambdas) { + struct A final { + int32_t func() { + return 5; + } + }; + struct B final { + std::string func() { + return "5"; + } + }; + auto result = + tuple_map(std::make_tuple(A(), B()), [](auto a) { return a.func(); }); + static_assert( + std::is_same, decltype(result)>::value, + ""); + EXPECT_EQ(5, std::get<0>(result)); + EXPECT_EQ("5", std::get<1>(result)); +} +} // namespace test_tuple_map + +} // namespace +// NOLINTEND(modernize*, cppcoreguidelines-special-member-functions) diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_rand.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_rand.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6a646eda49b014d718722f836a7d7de972da4512 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_rand.cpp @@ -0,0 +1,39 @@ +#include + +#include + +#include +#include +namespace torch { +namespace aot_inductor { + +int64_t randint64_cpu( + uint32_t seed, + uint32_t offset, + int64_t low, + int64_t high) { + auto gen = at::Philox4_32(seed, 0, offset); + uint64_t r0 = gen(); + uint64_t r1 = gen(); + uint64_t result = r0 | (r1 << 32); + return static_cast(result % (high - low)) + low; +} + +TEST(TestRand, TestRandn) { + at::Philox4_32 engine_1(1, 0, 0); + float a = engine_1.randn(10); + at::Philox4_32 engine_2(1, 0, 0); + float b = engine_2.randn(10); + + EXPECT_EQ(a, b); +} + +TEST(TestRand, TestRandint64) { + int64_t a = randint64_cpu(0xffffffff, 100, 0, INT64_MAX); + int64_t b = randint64_cpu(0xffffffff, 100, 0, INT64_MAX); + + EXPECT_EQ(a, b); +} + +} // namespace aot_inductor +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_scalartype.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_scalartype.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8c2df2ceea8e007df0f25651698bb987202b7113 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_scalartype.cpp @@ -0,0 +1,114 @@ +#include + +#include + +TEST(TestScalarType, ScalarTypeToCPPTypeT) { + using torch::headeronly::ScalarType; + using torch::headeronly::impl::ScalarTypeToCPPTypeT; + +#define DEFINE_CHECK(TYPE, SCALARTYPE) \ + EXPECT_EQ(typeid(ScalarTypeToCPPTypeT), typeid(TYPE)); + + AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(DEFINE_CHECK); +#undef DEFINE_CHECK +} + +TEST(TestScalarType, CppTypeToScalarType) { + using torch::headeronly::CppTypeToScalarType; + using torch::headeronly::ScalarType; + +#define DEFINE_CHECK(TYPE, SCALARTYPE) \ + EXPECT_EQ(CppTypeToScalarType::value, ScalarType::SCALARTYPE); + + AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(DEFINE_CHECK); +#undef DEFINE_CHECK +} + +#define DEFINE_CHECK(TYPE, SCALARTYPE) \ + { \ + EXPECT_EQ( \ + typeid(ScalarTypeToCPPTypeT), typeid(TYPE)); \ + count++; \ + } + +#define TEST_FORALL(M, EXPECTEDCOUNT, ...) \ + TEST(TestScalarType, M) { \ + using torch::headeronly::ScalarType; \ + using torch::headeronly::impl::ScalarTypeToCPPTypeT; \ + int8_t count = 0; \ + M(__VA_ARGS__ DEFINE_CHECK); \ + EXPECT_EQ(count, EXPECTEDCOUNT); \ + } + +TEST_FORALL(AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_EXCEPT_COMPLEX_HALF_F8NZ, 14) +TEST_FORALL(AT_FORALL_SCALAR_TYPES_WITH_COMPLEX, 18) +TEST_FORALL(AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS, 47) +TEST_FORALL(AT_FORALL_INT_TYPES, 5) +TEST_FORALL(AT_FORALL_SCALAR_TYPES, 7) +TEST_FORALL(AT_FORALL_SCALAR_TYPES_AND, 8, Bool, ) +TEST_FORALL(AT_FORALL_SCALAR_TYPES_AND2, 9, Bool, Half, ) +TEST_FORALL(AT_FORALL_SCALAR_TYPES_AND3, 10, Bool, Half, ComplexFloat, ) +TEST_FORALL( + AT_FORALL_SCALAR_TYPES_AND7, + 14, + Bool, + Half, + ComplexHalf, + ComplexFloat, + ComplexDouble, + UInt16, + UInt32, ) +TEST_FORALL(AT_FORALL_QINT_TYPES, 5) +TEST_FORALL(AT_FORALL_FLOAT8_TYPES, 5) +TEST_FORALL(AT_FORALL_COMPLEX_TYPES, 2) + +#undef DEFINE_CHECK +#undef TEST_FORALL + +TEST(TestScalarType, toString) { + using torch::headeronly::ScalarType; + +#define DEFINE_CHECK(_, name) EXPECT_EQ(toString(ScalarType::name), #name); + AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(DEFINE_CHECK); +#undef DEFINE_CHECK +} + +TEST(TestScalarType, operator_left_shift) { + using torch::headeronly::ScalarType; + +#define DEFINE_CHECK(_, name) \ + { \ + std::stringstream ss; \ + ss << ScalarType::name; \ + EXPECT_EQ(std::move(ss).str(), #name); \ + } + AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(DEFINE_CHECK); +#undef DEFINE_CHECK +} + +TEST(TestScalarType, toUnderlying) { + using torch::headeronly::ScalarType; + using torch::headeronly::toUnderlying; + + EXPECT_EQ(toUnderlying(ScalarType::QUInt8), ScalarType::Byte); + EXPECT_EQ(toUnderlying(ScalarType::QUInt4x2), ScalarType::Byte); + EXPECT_EQ(toUnderlying(ScalarType::QUInt2x4), ScalarType::Byte); + EXPECT_EQ(toUnderlying(ScalarType::QInt8), ScalarType::Char); + EXPECT_EQ(toUnderlying(ScalarType::QInt32), ScalarType::Int); +#define DEFINE_CHECK(_, name) \ + EXPECT_EQ(toUnderlying(ScalarType::name), ScalarType::name); + AT_FORALL_SCALAR_TYPES_WITH_COMPLEX(DEFINE_CHECK); + AT_FORALL_FLOAT8_TYPES(DEFINE_CHECK); +#undef DEFINE_CHECK +} + +TEST(TestScalarType, isQIntType) { + using torch::headeronly::isQIntType; + using torch::headeronly::ScalarType; +#define DEFINE_CHECK(_, name) EXPECT_TRUE(isQIntType(ScalarType::name)); + AT_FORALL_QINT_TYPES(DEFINE_CHECK); +#undef DEFINE_CHECK +#define DEFINE_CHECK(_, name) EXPECT_FALSE(isQIntType(ScalarType::name)); + AT_FORALL_SCALAR_TYPES_WITH_COMPLEX(DEFINE_CHECK); +#undef DEFINE_CHECK +} diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_typelist.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_typelist.cpp new file mode 100644 index 0000000000000000000000000000000000000000..39ba9964ed5604ba501c82f0951b2ef702d7f495 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_typelist.cpp @@ -0,0 +1,363 @@ +#include +#include +#include + +using namespace torch::headeronly::guts::typelist; +// NOLINTBEGIN(modernize-unary-static-assert) +namespace test_size { +class MyClass {}; +static_assert(0 == size>::value, ""); +static_assert(1 == size>::value, ""); +static_assert(3 == size>::value, ""); +} // namespace test_size + +namespace test_from_tuple { +class MyClass {}; +static_assert( + std::is_same_v< + typelist, + from_tuple_t>>, + ""); +static_assert(std::is_same_v, from_tuple_t>>, ""); +} // namespace test_from_tuple + +namespace test_to_tuple { +class MyClass {}; +static_assert( + std::is_same_v< + std::tuple, + to_tuple_t>>, + ""); +static_assert(std::is_same_v, to_tuple_t>>, ""); +} // namespace test_to_tuple + +namespace test_concat { +class MyClass {}; +static_assert(std::is_same_v, concat_t<>>, ""); +static_assert(std::is_same_v, concat_t>>, ""); +static_assert(std::is_same_v, concat_t, typelist<>>>, ""); +static_assert(std::is_same_v, concat_t>>, ""); +static_assert( + std::is_same_v, concat_t, typelist<>>>, + ""); +static_assert( + std::is_same_v, concat_t, typelist>>, + ""); +static_assert( + std::is_same_v< + typelist, + concat_t, typelist, typelist<>>>, + ""); +static_assert( + std::is_same_v< + typelist, + concat_t, typelist>>, + ""); +static_assert( + std::is_same_v< + typelist, + concat_t, typelist, typelist<>>>, + ""); +static_assert( + std::is_same_v< + typelist, + concat_t, typelist, typelist>>, + ""); +} // namespace test_concat + +namespace test_filter { +class MyClass {}; +static_assert( + std::is_same_v, filter_t>>, + ""); +static_assert( + std::is_same_v< + typelist<>, + filter_t>>, + ""); +static_assert( + std::is_same_v< + typelist, + filter_t< + std::is_reference, + typelist>>, + ""); +} // namespace test_filter + +namespace test_count_if { +class MyClass final {}; +static_assert( + count_if< + std::is_reference, + typelist>::value == 2, + ""); +static_assert(count_if>::value == 0, ""); +static_assert(count_if>::value == 0, ""); +} // namespace test_count_if + +namespace test_true_for_each_type { +template +class Test; +class MyClass {}; +static_assert( + all>::value, + ""); +static_assert( + !all>::value, + ""); +static_assert(all>::value, ""); +} // namespace test_true_for_each_type + +namespace test_true_for_any_type { +template +class Test; +class MyClass {}; +static_assert( + true_for_any_type< + std::is_reference, + typelist>::value, + ""); +static_assert( + true_for_any_type< + std::is_reference, + typelist>::value, + ""); +static_assert( + !true_for_any_type< + std::is_reference, + typelist>::value, + ""); +static_assert(!true_for_any_type>::value, ""); +} // namespace test_true_for_any_type + +namespace test_map { +class MyClass {}; +static_assert( + std::is_same_v, map_t>>, + ""); +static_assert( + std::is_same_v< + typelist, + map_t>>, + ""); +static_assert( + std::is_same_v< + typelist, + map_t< + std::add_lvalue_reference_t, + typelist>>, + ""); +} // namespace test_map + +namespace test_head { +class MyClass {}; +static_assert(std::is_same_v>>, ""); +static_assert( + std::is_same_v>>, + ""); +static_assert( + std::is_same_v>>, + ""); +static_assert(std::is_same_v>>, ""); +} // namespace test_head + +namespace test_head_with_default { +class MyClass {}; +static_assert( + std::is_same_v>>, + ""); +static_assert( + std::is_same_v< + const MyClass&, + head_with_default_t>>, + ""); +static_assert( + std::is_same_v< + MyClass&&, + head_with_default_t>>, + ""); +static_assert( + std::is_same_v>>, + ""); +static_assert(std::is_same_v>>, ""); +} // namespace test_head_with_default + +namespace test_reverse { +class MyClass {}; +static_assert( + std::is_same_v< + typelist, + reverse_t>>, + ""); +static_assert(std::is_same_v, reverse_t>>, ""); +} // namespace test_reverse + +namespace test_map_types_to_values { +struct map_to_size { + template + constexpr size_t operator()(T) const { + return sizeof(typename T::type); + } +}; + +TEST(TypeListTest, MapTypesToValues_sametype) { + auto sizes = + map_types_to_values>(map_to_size()); + std::tuple expected(8, 1, 4); + static_assert(std::is_same_v, ""); + EXPECT_EQ(expected, sizes); +} + +struct map_make_shared { + template + std::shared_ptr operator()(T) { + return std::make_shared(); + } +}; + +TEST(TypeListTest, MapTypesToValues_differenttypes) { + auto shared_ptrs = + map_types_to_values>(map_make_shared()); + static_assert( + std::is_same_v< + std::tuple, std::shared_ptr>, + decltype(shared_ptrs)>, + ""); +} + +struct Class1 { + static int func() { + return 3; + } +}; +struct Class2 { + static double func() { + return 2.0; + } +}; + +struct mapper_call_func { + template + auto operator()(T) { + return T::type::func(); + } +}; + +TEST(TypeListTest, MapTypesToValues_members) { + auto result = + map_types_to_values>(mapper_call_func()); + std::tuple expected(3, 2.0); + static_assert(std::is_same_v, ""); + EXPECT_EQ(expected, result); +} + +struct mapper_call_nonexistent_function { + template + auto operator()(T) { + return T::type::this_doesnt_exist(); + } +}; + +TEST(TypeListTest, MapTypesToValues_empty) { + auto result = + map_types_to_values>(mapper_call_nonexistent_function()); + std::tuple<> expected; + static_assert(std::is_same_v, ""); + EXPECT_EQ(expected, result); +} +} // namespace test_map_types_to_values + +namespace test_find_if { +static_assert(0 == find_if, std::is_reference>::value, ""); +static_assert( + 0 == find_if, std::is_reference>::value, + ""); +static_assert( + 2 == find_if, std::is_reference>::value, + ""); +static_assert( + 3 == find_if, std::is_reference>::value, + ""); +} // namespace test_find_if + +namespace test_contains { +static_assert(contains, double>::value, ""); +static_assert(contains, double>::value, ""); +static_assert(!contains, float>::value, ""); +static_assert(!contains, double>::value, ""); +} // namespace test_contains + +namespace test_take { +static_assert(std::is_same_v, take_t, 0>>, ""); +static_assert(std::is_same_v, take_t, 0>>, ""); +static_assert( + std::is_same_v, take_t, 1>>, + ""); +static_assert( + std::is_same_v, take_t, 0>>, + ""); +static_assert( + std::is_same_v, take_t, 1>>, + ""); +static_assert( + std::is_same_v< + typelist, + take_t, 2>>, + ""); +} // namespace test_take + +namespace test_drop { +static_assert(std::is_same_v, drop_t, 0>>, ""); +static_assert( + std::is_same_v, drop_t, 0>>, + ""); +static_assert(std::is_same_v, drop_t, 1>>, ""); +static_assert( + std::is_same_v< + typelist, + drop_t, 0>>, + ""); +static_assert( + std::is_same_v, drop_t, 1>>, + ""); +static_assert( + std::is_same_v, drop_t, 2>>, + ""); +} // namespace test_drop + +namespace test_drop_if_nonempty { +static_assert( + std::is_same_v, drop_if_nonempty_t, 0>>, + ""); +static_assert( + std::is_same_v, drop_if_nonempty_t, 0>>, + ""); +static_assert( + std::is_same_v, drop_if_nonempty_t, 1>>, + ""); +static_assert( + std::is_same_v< + typelist, + drop_if_nonempty_t, 0>>, + ""); +static_assert( + std::is_same_v< + typelist, + drop_if_nonempty_t, 1>>, + ""); +static_assert( + std::is_same_v< + typelist<>, + drop_if_nonempty_t, 2>>, + ""); +static_assert( + std::is_same_v, drop_if_nonempty_t, 1>>, + ""); +static_assert( + std::is_same_v< + typelist<>, + drop_if_nonempty_t, 3>>, + ""); +} // namespace test_drop_if_nonempty +// NOLINTEND(modernize-unary-static-assert) diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_typetraits.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_typetraits.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cc808752993dd918f0ca487c1bb4d4d46d86aac6 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_typetraits.cpp @@ -0,0 +1,199 @@ +#include +#include + +using namespace torch::headeronly::guts; + +// NOLINTBEGIN(modernize-unary-static-assert) +namespace { + +namespace test_is_equality_comparable { +class NotEqualityComparable {}; +class EqualityComparable {}; + +inline bool operator==(const EqualityComparable&, const EqualityComparable&) { + return false; +} + +static_assert(!is_equality_comparable::value, ""); +static_assert(is_equality_comparable::value, ""); +static_assert(is_equality_comparable::value, ""); + +// v_ just exists to silence a compiler warning about +// operator==(EqualityComparable, EqualityComparable) not being needed +const bool v_ = EqualityComparable() == EqualityComparable(); +} // namespace test_is_equality_comparable + +namespace test_is_hashable { +class NotHashable {}; +class Hashable {}; +} // namespace test_is_hashable +} // namespace +namespace std { +template <> +struct hash final { + size_t operator()(const test_is_hashable::Hashable&) { + return 0; + } +}; +} // namespace std +namespace { +namespace test_is_hashable { +static_assert(is_hashable::value, ""); +static_assert(is_hashable::value, ""); +static_assert(!is_hashable::value, ""); +} // namespace test_is_hashable + +namespace test_is_function_type { +class MyClass {}; +struct Functor { + void operator()() {} +}; +auto lambda = []() {}; +// func() and func__ just exists to silence a compiler warning about lambda +// being unused +static bool func() { + lambda(); + return true; +} +bool func__ = func(); + +static_assert(is_function_type::value, ""); +static_assert(is_function_type::value, ""); +static_assert(is_function_type::value, ""); +static_assert(is_function_type::value, ""); +static_assert(is_function_type::value, ""); +static_assert(is_function_type::value, ""); +static_assert(is_function_type::value, ""); +static_assert(is_function_type::value, ""); +static_assert(is_function_type::value, ""); +static_assert(is_function_type::value, ""); +static_assert(is_function_type < MyClass && () > ::value, ""); +static_assert(is_function_type < MyClass && (MyClass&&) > ::value, ""); +static_assert(is_function_type::value, ""); + +static_assert(!is_function_type::value, ""); +static_assert(!is_function_type::value, ""); +static_assert(!is_function_type::value, ""); +static_assert(!is_function_type::value, ""); +static_assert(!is_function_type::value, ""); +static_assert(!is_function_type::value, ""); + +static_assert( + !is_function_type::value, + "function pointers aren't plain functions"); +static_assert( + !is_function_type::value, + "Functors aren't plain functions"); +static_assert( + !is_function_type::value, + "Lambdas aren't plain functions"); +} // namespace test_is_function_type + +namespace test_is_instantiation_of { +class MyClass {}; +template +class Single {}; +template +class Double {}; +template +class Multiple {}; + +static_assert(is_instantiation_of>::value, ""); +static_assert(is_instantiation_of>::value, ""); +static_assert(is_instantiation_of>::value, ""); +static_assert(is_instantiation_of>::value, ""); +static_assert(is_instantiation_of>::value, ""); +static_assert(is_instantiation_of>::value, ""); +static_assert(is_instantiation_of>::value, ""); +static_assert(is_instantiation_of>::value, ""); +static_assert( + is_instantiation_of>::value, + ""); +static_assert(is_instantiation_of>::value, ""); +static_assert(is_instantiation_of>::value, ""); +static_assert( + is_instantiation_of>::value, + ""); +static_assert( + is_instantiation_of>::value, + ""); +static_assert( + is_instantiation_of>:: + value, + ""); + +static_assert(!is_instantiation_of>::value, ""); +static_assert(!is_instantiation_of>::value, ""); +static_assert(!is_instantiation_of>::value, ""); +static_assert(!is_instantiation_of>::value, ""); +static_assert(!is_instantiation_of>::value, ""); +static_assert(!is_instantiation_of>::value, ""); +static_assert(!is_instantiation_of>::value, ""); +static_assert(!is_instantiation_of>::value, ""); +} // namespace test_is_instantiation_of + +namespace test_is_type_condition { +template +class NotATypeCondition {}; +static_assert(is_type_condition::value, ""); +static_assert(!is_type_condition::value, ""); +} // namespace test_is_type_condition +} // namespace + +namespace test_lambda_is_stateless { +template +struct MyStatelessFunctor final { + Result operator()(Args...) {} +}; + +template +struct MyStatelessConstFunctor final { + Result operator()(Args...) const {} +}; + +// NOLINTNEXTLINE(misc-use-internal-linkage) +void func() { + auto stateless_lambda = [](int a) { return a; }; + static_assert(is_stateless_lambda::value, ""); + + int b = 4; + // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores) + auto stateful_lambda_1 = [&](int a) { return a + b; }; + static_assert(!is_stateless_lambda::value, ""); + + // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores) + auto stateful_lambda_2 = [=](int a) { return a + b; }; + static_assert(!is_stateless_lambda::value, ""); + + // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores) + auto stateful_lambda_3 = [b](int a) { return a + b; }; + static_assert(!is_stateless_lambda::value, ""); + + static_assert( + !is_stateless_lambda>::value, + "even if stateless, a functor is not a lambda, so it's false"); + static_assert( + !is_stateless_lambda>::value, + "even if stateless, a functor is not a lambda, so it's false"); + static_assert( + !is_stateless_lambda>::value, + "even if stateless, a functor is not a lambda, so it's false"); + static_assert( + !is_stateless_lambda>::value, + "even if stateless, a functor is not a lambda, so it's false"); + + class Dummy final {}; + static_assert( + !is_stateless_lambda::value, + "A non-functor type is also not a lambda"); + + static_assert(!is_stateless_lambda::value, "An int is not a lambda"); + + using Func = int(int); + static_assert( + !is_stateless_lambda::value, "A function is not a lambda"); + static_assert( + !is_stateless_lambda::value, "A function pointer is not a lambda"); +} +} // namespace test_lambda_is_stateless +// NOLINTEND(modernize-unary-static-assert) diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_vec.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_vec.cpp new file mode 100644 index 0000000000000000000000000000000000000000..31fde99ee6dfc7a0e3e8f85342fa86a988adaefc --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_vec.cpp @@ -0,0 +1,88 @@ +#include + +#include + +namespace torch { +namespace aot_inductor { + +template +void ExpectVecEqual( + const at::vec::Vectorized& expected, + const at::vec::Vectorized& actual) { + using Vec = at::vec::Vectorized; + // Have to use std::vector for comparison because at::vec::Vectorized doesn't + // support operator[] on aarch64 + std::vector expected_data(Vec::size()); + std::vector actual_data(Vec::size()); + + expected.store(expected_data.data()); + actual.store(actual_data.data()); + + for (int i = 0; i < Vec::size(); i++) { + EXPECT_EQ(expected_data[i], actual_data[i]); + } +} + +TEST(TestVec, TestAdd) { + using Vec = at::vec::Vectorized; + std::vector a(1024, 1); + std::vector b(1024, 2); + Vec a_vec = Vec::loadu(a.data()); + Vec b_vec = Vec::loadu(b.data()); + Vec actual_vec = a_vec + b_vec; + std::vector expected(1024, 3); + Vec expected_vec = Vec::loadu(expected.data()); + + ExpectVecEqual(expected_vec, actual_vec); +} + +TEST(TestVec, TestMax) { + using Vec = at::vec::Vectorized; + std::vector a(1024, -1); + std::vector b(1024, 2); + Vec a_vec = Vec::loadu(a.data()); + Vec b_vec = Vec::loadu(b.data()); + Vec actual_vec = at::vec::maximum(a_vec, b_vec); + Vec expected_vec = b_vec; + + ExpectVecEqual(expected_vec, actual_vec); +} + +TEST(TestVec, TestMin) { + using Vec = at::vec::Vectorized; + std::vector a(1024, -1); + std::vector b(1024, 2); + Vec a_vec = Vec::loadu(a.data()); + Vec b_vec = Vec::loadu(b.data()); + Vec actual_vec = at::vec::minimum(a_vec, b_vec); + Vec expected_vec = a_vec; + + ExpectVecEqual(expected_vec, actual_vec); +} + +TEST(TestVec, TestConvert) { + std::vector a(1024, -1); + std::vector b(1024, -1.0); + at::vec::Vectorized a_vec = at::vec::Vectorized::loadu(a.data()); + at::vec::Vectorized b_vec = + at::vec::Vectorized::loadu(b.data()); + auto actual_vec = at::vec::convert(a_vec); + auto expected_vec = b_vec; + + ExpectVecEqual(expected_vec, actual_vec); +} + +TEST(TestVec, TestClampMin) { + using Vec = at::vec::Vectorized; + std::vector a(1024, -2.0); + std::vector min(1024, -1.0); + Vec a_vec = Vec::loadu(a.data()); + Vec min_vec = Vec::loadu(min.data()); + Vec actual_vec = at::vec::clamp_min(a_vec, min_vec); + Vec expected_vec = min_vec; + + ExpectVecEqual(expected_vec, actual_vec); +} + +} // namespace aot_inductor +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_vec_half.cpp b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_vec_half.cpp new file mode 100644 index 0000000000000000000000000000000000000000..28fd1eb7d4dede5877faf9c7aa60cbc0e01d9a36 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_abi_check/test_vec_half.cpp @@ -0,0 +1,22 @@ +#include +#include +#include + +TEST(TestVecHalf, TestConversion) { + float f32s[100]; + for (int i = 0; i < 100; i++) { + f32s[i] = static_cast(i + 0.3); + } + for (int i = 0; i < 100; i++) { +#if (defined(CPU_CAPABILITY_AVX2) || defined(CPU_CAPABILITY_AVX512)) && \ + !defined(__APPLE__) + uint16_t u16 = torch::headeronly::vec::float2half_scalar(f32s[i]); + float x = torch::headeronly::vec::half2float_scalar(u16); + EXPECT_EQ( + u16, torch::headeronly::detail::fp16_ieee_from_fp32_value(f32s[i])) + << "Test failed for float to uint16 " << f32s[i] << '\n'; + EXPECT_EQ(x, torch::headeronly::detail::fp16_ieee_to_fp32_value(u16)) + << "Test failed for uint16 to float " << u16 << '\n'; +#endif + } +} diff --git a/western_ai/usa/pytorch/test/cpp/aoti_eager/kernel_meta_info_test.cpp b/western_ai/usa/pytorch/test/cpp/aoti_eager/kernel_meta_info_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f6eb36a93af45daa36edf9fe06ee419f62a26067 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_eager/kernel_meta_info_test.cpp @@ -0,0 +1,138 @@ +// Tests for TensorMetadata::dynamic_check and ParameterMetadata::dynamic_check, +// which enable a single compiled AOTI kernel to serve multiple input shapes by +// matching dtype/device/rank instead of exact sizes and strides. + +#if !defined(C10_MOBILE) && !defined(ANDROID) + +#include + +#include + +namespace torch::inductor { + +namespace { + +TensorMetadata makeTensorMeta( + c10::ScalarType dtype, + std::vector sizes, + std::vector strides, + c10::Device device = c10::Device(c10::kCPU)) { + return TensorMetadata( + /*is_symbolic=*/false, + dtype, + device, + c10::DispatchKeySet(c10::DispatchKey::CPU), + std::move(sizes), + std::move(strides)); +} + +} // namespace + +// ---------- TensorMetadata::dynamic_check ---------- + +TEST(TensorMetadataDynamicCheckTest, MatchesSameDtypeDeviceRankDifferentSizes) { + auto a = makeTensorMeta(c10::kFloat, {4, 8}, {8, 1}); + auto b = makeTensorMeta(c10::kFloat, {16, 32}, {32, 1}); + EXPECT_TRUE(a.dynamic_check(b)); + // Exact comparison should fail because sizes differ + EXPECT_FALSE(a == b); +} + +TEST(TensorMetadataDynamicCheckTest, RejectsDifferentDtype) { + auto a = makeTensorMeta(c10::kFloat, {4, 8}, {8, 1}); + auto b = makeTensorMeta(c10::kHalf, {4, 8}, {8, 1}); + EXPECT_FALSE(a.dynamic_check(b)); +} + +TEST(TensorMetadataDynamicCheckTest, RejectsDifferentRank) { + auto a = makeTensorMeta(c10::kFloat, {4, 8}, {8, 1}); + auto b = makeTensorMeta(c10::kFloat, {4, 8, 2}, {16, 2, 1}); + EXPECT_FALSE(a.dynamic_check(b)); +} + +TEST(TensorMetadataDynamicCheckTest, RejectsDifferentDevice) { + auto cpu = makeTensorMeta(c10::kFloat, {4}, {1}, c10::Device(c10::kCPU)); + auto cuda = makeTensorMeta(c10::kFloat, {4}, {1}, c10::Device(c10::kCUDA, 0)); + EXPECT_FALSE(cpu.dynamic_check(cuda)); +} + +TEST(TensorMetadataDynamicCheckTest, MatchesIdenticalMetadata) { + auto a = makeTensorMeta(c10::kFloat, {4, 8}, {8, 1}); + auto b = makeTensorMeta(c10::kFloat, {4, 8}, {8, 1}); + EXPECT_TRUE(a.dynamic_check(b)); + EXPECT_TRUE(a == b); +} + +TEST(TensorMetadataDynamicCheckTest, MatchesScalarTensors) { + auto a = makeTensorMeta(c10::kFloat, {}, {}); + auto b = makeTensorMeta(c10::kFloat, {}, {}); + EXPECT_TRUE(a.dynamic_check(b)); +} + +// ---------- ParameterMetadata::dynamic_check ---------- + +TEST(ParameterMetadataDynamicCheckTest, TensorMatchesDifferentSizes) { + ParameterMetadata a(makeTensorMeta(c10::kFloat, {4, 8}, {8, 1}), 0); + ParameterMetadata b(makeTensorMeta(c10::kFloat, {16, 32}, {32, 1}), 0); + EXPECT_TRUE(a.dynamic_check(b)); + EXPECT_FALSE(a == b); +} + +TEST(ParameterMetadataDynamicCheckTest, TensorRejectsDifferentDtype) { + ParameterMetadata a(makeTensorMeta(c10::kFloat, {4}, {1}), 0); + ParameterMetadata b(makeTensorMeta(c10::kHalf, {4}, {1}), 0); + EXPECT_FALSE(a.dynamic_check(b)); +} + +TEST(ParameterMetadataDynamicCheckTest, RejectsDifferentOrder) { + ParameterMetadata a(makeTensorMeta(c10::kFloat, {4}, {1}), 0); + ParameterMetadata b(makeTensorMeta(c10::kFloat, {4}, {1}), 1); + EXPECT_FALSE(a.dynamic_check(b)); +} + +TEST(ParameterMetadataDynamicCheckTest, ScalarUsesExactMatch) { + ParameterMetadata a(c10::Scalar(1.5), 0); + ParameterMetadata b(c10::Scalar(1.5), 0); + EXPECT_TRUE(a.dynamic_check(b)); + + ParameterMetadata c(c10::Scalar(2.0), 0); + EXPECT_FALSE(a.dynamic_check(c)); +} + +TEST(ParameterMetadataDynamicCheckTest, TensorListMatchesDifferentSizes) { + std::vector list_a = { + makeTensorMeta(c10::kFloat, {4, 8}, {8, 1}), + makeTensorMeta(c10::kFloat, {2, 8}, {8, 1}), + }; + std::vector list_b = { + makeTensorMeta(c10::kFloat, {16, 32}, {32, 1}), + makeTensorMeta(c10::kFloat, {8, 32}, {32, 1}), + }; + ParameterMetadata a(list_a, 0); + ParameterMetadata b(list_b, 0); + EXPECT_TRUE(a.dynamic_check(b)); +} + +TEST(ParameterMetadataDynamicCheckTest, TensorListRejectsDifferentLengths) { + std::vector one = {makeTensorMeta(c10::kFloat, {4}, {1})}; + std::vector two = { + makeTensorMeta(c10::kFloat, {4}, {1}), + makeTensorMeta(c10::kFloat, {8}, {1}), + }; + ParameterMetadata a(one, 0); + ParameterMetadata b(two, 0); + EXPECT_FALSE(a.dynamic_check(b)); +} + +TEST(ParameterMetadataDynamicCheckTest, StringUsesExactMatch) { + ParameterMetadata a(std::string("sum"), 0); + ParameterMetadata b(std::string("sum"), 0); + EXPECT_TRUE(a.dynamic_check(b)); + + ParameterMetadata c(std::string("mean"), 0); + EXPECT_FALSE(a.dynamic_check(c)); +} + +} // namespace torch::inductor + +#endif // !defined(C10_MOBILE) && !defined(ANDROID) diff --git a/western_ai/usa/pytorch/test/cpp/aoti_inference/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/aoti_inference/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a2c7cb6bed8c69facae3a3c0bd70b442b4f3d8c0 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_inference/CMakeLists.txt @@ -0,0 +1,57 @@ +set(AOT_INDUCTOR_TEST_ROOT ${TORCH_ROOT}/test/cpp/aoti_inference) + +# Build custom TorchScript op for AOTInductor +add_library(aoti_custom_class SHARED aoti_custom_class.cpp) +set_target_properties(aoti_custom_class PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) +if(USE_CUDA) + target_compile_definitions(aoti_custom_class PRIVATE USE_CUDA) +elseif(USE_ROCM) + target_compile_definitions(aoti_custom_class PRIVATE USE_ROCM) +endif() + +# Link against LibTorch +target_link_libraries(aoti_custom_class torch) + +# Build the cpp gtest binary containing the cpp-only tests. +set(INDUCTOR_TEST_SRCS + ${AOT_INDUCTOR_TEST_ROOT}/test.cpp +) + +add_executable(test_aoti_inference + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${INDUCTOR_TEST_SRCS} +) +add_dependencies(test_aoti_inference aoti_custom_class) + +# TODO temporary until we can delete the old gtest polyfills. +target_compile_definitions(test_aoti_inference PRIVATE USE_GTEST) + +target_link_libraries(test_aoti_inference PRIVATE + torch + gtest_main + -Wl,--no-as-needed aoti_custom_class +) + +if(USE_CUDA) + target_include_directories(test_aoti_inference PRIVATE ${ATen_CUDA_INCLUDE}) + target_compile_definitions(test_aoti_inference PRIVATE USE_CUDA) +elseif(USE_ROCM) + target_include_directories(test_aoti_inference PRIVATE ${ATen_HIP_INCLUDE}) + target_compile_definitions(test_aoti_inference PRIVATE USE_ROCM) +endif() +target_compile_definitions(test_aoti_inference PRIVATE + CMAKE_CURRENT_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR} +) + +target_compile_options_if_supported(test_aoti_inference -Wno-unused-variable) +target_compile_options_if_supported(test_aoti_inference -Wno-unused-but-set-variable) +target_compile_options_if_supported(test_aoti_inference -Wno-unused-function) + +if(INSTALL_TEST) + install(TARGETS test_aoti_inference DESTINATION bin) + # Install PDB files for MSVC builds + if(MSVC AND BUILD_SHARED_LIBS) + install(FILES $ DESTINATION bin OPTIONAL) + endif() +endif() diff --git a/western_ai/usa/pytorch/test/cpp/aoti_inference/aoti_custom_class.cpp b/western_ai/usa/pytorch/test/cpp/aoti_inference/aoti_custom_class.cpp new file mode 100644 index 0000000000000000000000000000000000000000..74582e8bd439fdc9ba821b9e3d5c911f9fbfa61f --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_inference/aoti_custom_class.cpp @@ -0,0 +1,55 @@ +#include + +#include +#if defined(USE_CUDA) || defined(USE_ROCM) +#include +#endif + +#include "aoti_custom_class.h" + +namespace torch::aot_inductor { + +static auto registerMyAOTIClass = + torch::class_("aoti", "MyAOTIClass") + .def(torch::init()) + .def("forward", &MyAOTIClass::forward) + .def_pickle( + [](const c10::intrusive_ptr& self) + -> std::vector { + std::vector v; + v.push_back(self->lib_path()); + v.push_back(self->device()); + return v; + }, + [](std::vector params) { + return c10::make_intrusive(params[0], params[1]); + }); + +MyAOTIClass::MyAOTIClass( + const std::string& model_path, + const std::string& device) + : lib_path_(model_path), device_(device) { + if (device_ == "cpu") { + runner_ = std::make_unique( + model_path.c_str()); +#if defined(USE_CUDA) || defined(USE_ROCM) + } else if (device_ == "cuda") { + runner_ = std::make_unique( + model_path.c_str()); +#endif +#if defined(USE_XPU) + } else if (device_ == "xpu") { + runner_ = std::make_unique( + model_path.c_str()); +#endif + } else { + throw std::runtime_error("invalid device: " + device); + } +} + +std::vector MyAOTIClass::forward( + std::vector inputs) { + return runner_->run(inputs); +} + +} // namespace torch::aot_inductor diff --git a/western_ai/usa/pytorch/test/cpp/aoti_inference/aoti_custom_class.h b/western_ai/usa/pytorch/test/cpp/aoti_inference/aoti_custom_class.h new file mode 100644 index 0000000000000000000000000000000000000000..7e15390074e76086426edf23f4dc01f98137891a --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_inference/aoti_custom_class.h @@ -0,0 +1,45 @@ +#pragma once + +#include + +#include + +namespace torch::inductor { + +class AOTIModelContainerRunner; + +} // namespace torch::inductor + +namespace torch::aot_inductor { + +class MyAOTIClass : public torch::CustomClassHolder { + public: + explicit MyAOTIClass( + const std::string& model_path, + const std::string& device = "cuda"); + + ~MyAOTIClass() {} + + MyAOTIClass(const MyAOTIClass&) = delete; + MyAOTIClass& operator=(const MyAOTIClass&) = delete; + MyAOTIClass& operator=(MyAOTIClass&&) = delete; + + const std::string& lib_path() const { + return lib_path_; + } + + const std::string& device() const { + return device_; + } + + std::vector forward(std::vector inputs); + + private: + const std::string lib_path_; + + const std::string device_; + + std::unique_ptr runner_; +}; + +} // namespace torch::aot_inductor diff --git a/western_ai/usa/pytorch/test/cpp/aoti_inference/compile_model.py b/western_ai/usa/pytorch/test/cpp/aoti_inference/compile_model.py new file mode 100644 index 0000000000000000000000000000000000000000..b60528695dca2a1b6be2d88c3c09f28f385d6d02 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_inference/compile_model.py @@ -0,0 +1,95 @@ +import torch +from torch.export import Dim + + +# custom op that loads the aot-compiled model +AOTI_CUSTOM_OP_LIB = "libaoti_custom_class.so" +torch.classes.load_library(AOTI_CUSTOM_OP_LIB) + + +class TensorSerializer(torch.nn.Module): + def __init__(self, data): + super().__init__() + for key in data: + setattr(self, key, data[key]) + + +class SimpleModule(torch.nn.Module): + """ + a simple module to be compiled + """ + + def __init__(self) -> None: + super().__init__() + self.fc = torch.nn.Linear(4, 6) + self.relu = torch.nn.ReLU() + + def forward(self, x): + a = self.fc(x) + b = self.relu(a) + return b + + +class MyAOTIModule(torch.nn.Module): + """ + a wrapper nn.Module that instantiates its forward method + on MyAOTIClass + """ + + def __init__(self, lib_path, device): + super().__init__() + self.aoti_custom_op = torch.classes.aoti.MyAOTIClass( + lib_path, + device, + ) + + def forward(self, *x): + outputs = self.aoti_custom_op.forward(x) + return tuple(outputs) + + +def make_script_module(lib_path, device, *inputs): + m = MyAOTIModule(lib_path, device) + # sanity check + m(*inputs) + return torch.jit.trace(m, inputs) + + +def compile_model(device, data): + module = SimpleModule().to(device) + x = torch.randn((4, 4), device=device) + inputs = (x,) + # make batch dimension + batch_dim = Dim("batch", min=1, max=1024) + dynamic_shapes = { + "x": {0: batch_dim}, + } + with torch.no_grad(): + # aot-compile the module into a .so pointed by lib_path + lib_path = torch._export.aot_compile( + module, inputs, dynamic_shapes=dynamic_shapes + ) + script_module = make_script_module(lib_path, device, *inputs) + aoti_script_model = f"script_model_{device}.pt" + script_module.save(aoti_script_model) + + # save sample inputs and ref output + with torch.no_grad(): + ref_output = module(*inputs) + data.update( + { + f"inputs_{device}": list(inputs), + f"outputs_{device}": [ref_output], + } + ) + + +def main(): + data = {} + for device in ["cpu", "cuda"] if torch.cuda.is_available() else ["cpu"]: + compile_model(device, data) + torch.jit.script(TensorSerializer(data)).save("script_data.pt") + + +if __name__ == "__main__": + main() diff --git a/western_ai/usa/pytorch/test/cpp/aoti_inference/generate_lowered_cpu.py b/western_ai/usa/pytorch/test/cpp/aoti_inference/generate_lowered_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..c64a820e5d54585ef486011b6e16626f7dc2bcc3 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_inference/generate_lowered_cpu.py @@ -0,0 +1,62 @@ +import copy + +import click + +import torch + + +class Serializer(torch.nn.Module): + def __init__(self, data): + super().__init__() + for key in data: + setattr(self, key, data[key]) + + +@click.command() +@click.option( + "--input-path", + type=str, + default="", + required=True, + help="path to the ExportedProgram", +) +@click.option( + "--output-path", + type=str, + default="", + required=True, +) +def main( + input_path: str = "", + output_path: str = "", +) -> None: + data = {} + ep = torch.export.load(input_path) + with torch.no_grad(): + example_inputs = ep.example_inputs[0] + # Get scripted original module. + module = torch.jit.trace(copy.deepcopy(ep.module()), example_inputs) + + # Get aot compiled module. + so_path = torch._inductor.aot_compile(ep.module(), example_inputs) + runner = torch.fx.Interpreter(ep.module()) + output = runner.run(example_inputs) + if isinstance(output, (list, tuple)): + output = list(output) + else: + output = [output] + + data.update( + { + "script_module": module, + "model_so_path": so_path, + "inputs": list(example_inputs), + "outputs": output, + } + ) + + torch.jit.script(Serializer(data)).save(output_path) + + +if __name__ == "__main__": + main() diff --git a/western_ai/usa/pytorch/test/cpp/aoti_inference/standalone_compile.sh b/western_ai/usa/pytorch/test/cpp/aoti_inference/standalone_compile.sh new file mode 100644 index 0000000000000000000000000000000000000000..09247772719bcbb2f7a79e4c05f0d8be7a98e618 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_inference/standalone_compile.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +if [ -z "$CXX" ]; then + CXX="clang++" + echo "Using system default C++ compiler: $CXX" +else + echo "Using user-provided C++ compiler: $CXX" +fi + +if [ -z "$TORCH_ROOT_DIR" ]; then + echo "Error: The TORCH_ROOT_DIR environment variable must be set." >&2 + echo "Example: export TORCH_ROOT_DIR=/home/$USER/local/pytorch" >&2 + exit 1 +fi + +if [ $# -lt 2 ]; then + echo "Usage: $0 ." + echo "Example Usage: $0 standalone_test.cpp standalone_test.out." + exit 1 +fi + +# Building the wrapper +$CXX -I$TORCH_ROOT_DIR/build/aten/src -I$TORCH_ROOT_DIR/aten/src -I$TORCH_ROOT_DIR/build -I$TORCH_ROOT_DIR -I$TORCH_ROOT_DIR/build/caffe2/aten/src -I$TORCH_ROOT_DIR/torch/csrc/api -I$TORCH_ROOT_DIR/torch/csrc/api/include -std=gnu++20 -fPIE -o $1.o -c $1 + +# Linking +$CXX -rdynamic -Wl,--no-as-needed,$TORCH_ROOT_DIR/build/lib/libtorch.so $1.o -Wl,--no-as-needed,$TORCH_ROOT_DIR/build/lib/libtorch_cpu.so -Wl,--no-as-needed,$TORCH_ROOT_DIR/build/lib/libc10.so -o $2 diff --git a/western_ai/usa/pytorch/test/cpp/aoti_inference/standalone_test.cpp b/western_ai/usa/pytorch/test/cpp/aoti_inference/standalone_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ae6bdd93ef29d332f4ca279221de50df1762abce --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_inference/standalone_test.cpp @@ -0,0 +1,81 @@ +#include +#include + +#include +#include +#include + +int main(int argc, char* argv[]) { + if (argc < 2) { + std::cerr + << "Usage: ./standalone_test [benchmark iter] [warm-up iter]" + << std::endl; + return 1; + } + size_t benchmark_iter = 10; + size_t warmup_iter = 3; + + if (argc == 3) { + benchmark_iter = std::stoul(argv[2]); + } else if (argc == 4) { + benchmark_iter = std::stoul(argv[2]); + warmup_iter = std::stoul(argv[3]); + } + + std::string data_path = argv[1]; + torch::jit::script::Module data_loader = torch::jit::load(data_path); + torch::jit::script::Module model = + data_loader.attr("script_module").toModule(); + const auto& model_so_path = data_loader.attr("model_so_path").toStringRef(); + const auto& script_input_tensors = data_loader.attr("inputs").toList().vec(); + const auto& input_tensors = data_loader.attr("inputs").toTensorList().vec(); + const auto& output_tensors = data_loader.attr("outputs").toTensorList().vec(); + + std::unique_ptr runner; + runner = std::make_unique( + model_so_path); + + // Check results. + auto actual_output_tensors = runner->run(input_tensors); + assert(output_tensors.size() == actual_output_tensors.size()); + for (size_t i = 0; i < output_tensors.size(); i++) { + assert(torch::allclose(output_tensors[i], actual_output_tensors[i])); + } + + // Start benchmarking for scripted module. + // Warm up + for (size_t i = 0; i < warmup_iter; i++) { + model.forward(script_input_tensors); + } + + // Benchmark + auto start = std::chrono::high_resolution_clock::now(); + for (size_t i = 0; i < benchmark_iter; i++) { + model.forward(script_input_tensors); + } + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration non_lowered_duration = end - start; + + // Start benchmarking for lowered module. + // Warm up + for (size_t i = 0; i < warmup_iter; i++) { + runner->run(input_tensors); + } + + // Benchmark + start = std::chrono::high_resolution_clock::now(); + for (size_t i = 0; i < benchmark_iter; i++) { + runner->run(input_tensors); + } + end = std::chrono::high_resolution_clock::now(); + std::chrono::duration lowered_duration = end - start; + + std::cout << "[Non-lowered] Time for " << benchmark_iter + << "iter(s): " << non_lowered_duration.count() << " sec(s)" + << std::endl; + std::cout << "[Lowered] Time for " << benchmark_iter + << "iter(s): " << lowered_duration.count() << " sec(s)" + << std::endl; + + return 0; +} diff --git a/western_ai/usa/pytorch/test/cpp/aoti_inference/test.cpp b/western_ai/usa/pytorch/test/cpp/aoti_inference/test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1ec09a45d9e9e65779dcc3da16a997e996fe4b7f --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_inference/test.cpp @@ -0,0 +1,1330 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#if defined(USE_CUDA) +#include +#include +#include +#endif +#if defined(USE_CUDA) || defined(USE_ROCM) +#include +#include +#endif +#include +#include + +#define STR_VALUE(x) #x +#define STRINGIZE(x) STR_VALUE(x) + +namespace { + +// Function to check if test data files exist and are valid +bool testDataFilesExist() { + std::string bindir = STRINGIZE(CMAKE_CURRENT_BINARY_DIR); + std::array required_files = { + "data.pt", + "script_data.pt", + "script_model_cpu.pt", + "script_model_cuda.pt"}; + + for (const auto& filename : required_files) { + std::string filepath = bindir + "/" + filename; + std::ifstream file(filepath); + if (!file.good()) { + return false; + } + } + return true; +} + +// Function to ensure test data files are generated at runtime +void ensureTestDataGenerated() { + static std::once_flag generated_flag; + std::call_once(generated_flag, []() { + // Only generate if files don't exist or are placeholders + if (testDataFilesExist()) { + return; + } + + std::string bindir = STRINGIZE(CMAKE_CURRENT_BINARY_DIR); + + // Calculate path to source directory: build/test_aoti_inference -> build -> + // pytorch + std::string pytorch_root = bindir.substr(0, bindir.find_last_of("/")); + pytorch_root = pytorch_root.substr(0, pytorch_root.find_last_of("/")); + std::string source_dir = pytorch_root + "/test/cpp/aoti_inference"; + + // Generate test data files (data.pt, etc.) by running test.py directly + std::string test_script = source_dir + "/test.py"; + std::string test_data_cmd = "cd " + bindir + " && python " + test_script; + std::cout << "Generating test data: " << test_data_cmd << std::endl; + int result1 = std::system(test_data_cmd.c_str()); + if (result1 != 0) { + std::cerr << "Warning: Test data generation failed with code " << result1 + << std::endl; + } + + // Generate model files (script_*.pt) by running compile_model.py directly + std::string compile_script = source_dir + "/compile_model.py"; + std::string models_cmd = "cd " + bindir + " && python " + compile_script; + std::cout << "Generating model files: " << models_cmd << std::endl; + int result2 = std::system(models_cmd.c_str()); + if (result2 != 0) { + std::cerr << "Warning: Model generation failed with code " << result2 + << std::endl; + } + }); +} + +const std::unordered_map derefTensorConstantMap( + torch::inductor::TensorConstantMap tensor_constant_map) { + std::unordered_map ret; + for (const auto& pair : tensor_constant_map) { + ret.emplace(pair.first, *(pair.second)); + } + return ret; +} + +bool compareConstantMap( + const std::unordered_map& lhs, + const std::unordered_map& rhs) { + if (lhs.size() != rhs.size()) { + return false; + } + + for (const auto& pair : lhs) { + auto it = rhs.find(pair.first); + if (it == rhs.end() || !torch::allclose(pair.second, it->second)) { + return false; + } + } + return true; +} + +void test_aoti(const std::string& device, bool use_runtime_constant_folding) { + torch::NoGradGuard no_grad; + + std::string data_path = + (std::filesystem::path(STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "data.pt") + .string(); + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string suffix = use_runtime_constant_folding + ? device + "_use_runtime_constant_folding" + : device; + std::string path_attr = "model_so_path_" + suffix; + std::string inputs_attr = "inputs_" + suffix; + std::string outputs_attr = "outputs_" + suffix; + const auto& model_so_path = data_loader.attr(path_attr.c_str()).toStringRef(); + const auto& ref_output_tensors = + data_loader.attr(outputs_attr.c_str()).toTensorList().vec(); + + std::unique_ptr runner; + if (device == "cpu") { + runner = std::make_unique( + model_so_path); +#if defined(USE_CUDA) || defined(USE_ROCM) + } else if (device == "cuda") { + runner = std::make_unique( + model_so_path); +#endif + } else { + testing::AssertionFailure() << "unsupported device: " << device; + } + auto actual_output_tensors = + runner->run(data_loader.attr(inputs_attr.c_str()).toTensorList().vec()); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); +} + +void test_aoti_script(const std::string& device) { + torch::NoGradGuard no_grad; + + std::string script_model = "script_model_" + device + ".pt"; + std::string model_path = + (std::filesystem::path( + STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / script_model.c_str()) + .string(); + torch::jit::script::Module model = torch::jit::load(model_path); + + std::string sample_data_path = + (std::filesystem::path( + STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "script_data.pt") + .string(); + torch::jit::script::Module sample_data = torch::jit::load(sample_data_path); + std::string inputs_attr = "inputs_" + device; + std::string outputs_attr = "outputs_" + device; + const auto& inputs = sample_data.attr(inputs_attr.c_str()).toList().vec(); + const auto& ref_output_tensors = + sample_data.attr(outputs_attr.c_str()).toTensorVector(); + auto outputs = model.forward(inputs).toTuple()->elements(); + ASSERT_EQ(outputs.size(), ref_output_tensors.size()); + for (size_t i = 0; i < ref_output_tensors.size(); i++) { + ASSERT_TRUE(torch::allclose(outputs[i].toTensor(), ref_output_tensors[i])); + } +} + +void test_aoti_package_loader( + const std::string& device, + bool use_runtime_constant_folding) { + torch::NoGradGuard no_grad; + + std::string data_path = + (std::filesystem::path(STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "data.pt") + .string(); + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string suffix = use_runtime_constant_folding + ? device + "_use_runtime_constant_folding" + : device; + std::string path_attr = "pt2_package_path_" + suffix; + std::string inputs_attr = "inputs_" + suffix; + std::string outputs_attr = "outputs_" + suffix; + const auto& pt2_package_path = + data_loader.attr(path_attr.c_str()).toStringRef(); + const auto& ref_output_tensors = + data_loader.attr(outputs_attr.c_str()).toTensorList().vec(); + + torch::inductor::AOTIModelPackageLoader runner(pt2_package_path); + auto actual_output_tensors = + runner.run(data_loader.attr(inputs_attr.c_str()).toTensorList().vec()); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); +} + +void test_aoti_package_loader_multi_gpu( + const std::string& device, + bool use_runtime_constant_folding) { + torch::NoGradGuard no_grad; + // Ensure that this test will reset the default CUDA device on exit. + torch::DeviceGuard device_guard(c10::Device("cuda")); + + std::string data_path = + (std::filesystem::path(STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "data.pt") + .string(); + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string suffix = use_runtime_constant_folding + ? device + "_use_runtime_constant_folding" + : device; + std::string path_attr = "pt2_package_path_" + suffix; + std::string inputs_attr = "inputs_" + suffix; + std::string outputs_attr = "outputs_" + suffix; + const auto& pt2_package_path = + data_loader.attr(path_attr.c_str()).toStringRef(); + const auto& ref_output_tensors = + data_loader.attr(outputs_attr.c_str()).toTensorList().vec(); + + // For all available CUDA devices: Load PT2 package on this device, run + // inference, and validate results + auto input_tensors = + data_loader.attr(inputs_attr.c_str()).toTensorList().vec(); + for (int i = 0; i < torch::cuda::device_count(); i++) { + auto options = torch::TensorOptions().device(torch::kCUDA, i); + torch::inductor::AOTIModelPackageLoader runner( + pt2_package_path, "model", false, 1, i); + std::vector input_tensors_on_device; + for (auto input_tensor : input_tensors) { + input_tensors_on_device.push_back(input_tensor.clone().to(options)); + } + // Run loaded PT2 package on device + auto actual_output_tensors = runner.run(input_tensors_on_device); + ASSERT_TRUE(torch::allclose( + ref_output_tensors[0].cpu(), actual_output_tensors[0].cpu())); + } +} + +void test_aoti_constants_update( + const std::string& device, + bool use_runtime_constant_folding) { + torch::NoGradGuard no_grad; + + std::string data_path = + (std::filesystem::path(STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "data.pt") + .string(); + + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string suffix = use_runtime_constant_folding + ? device + "_use_runtime_constant_folding" + : device; + std::string path_attr = "model_so_path_" + suffix; + std::string inputs_attr = "inputs_" + suffix; + std::string outputs_attr = "outputs_" + suffix; + std::string weights_attr = "w_pre_" + suffix; + std::string add_attr = "w_add_" + suffix; + const auto& model_so_path = data_loader.attr(path_attr.c_str()).toStringRef(); + auto input_tensors = + data_loader.attr(inputs_attr.c_str()).toTensorList().vec(); + const auto& ref_output_tensors = + data_loader.attr(outputs_attr.c_str()).toTensorList().vec(); + + const auto& weight_tensors = + data_loader.attr(weights_attr.c_str()).toTensor(); + const auto& add_tensors = data_loader.attr(add_attr.c_str()).toTensor(); + + torch::inductor::TensorConstantMap missing_map, rand_map, real_map; + missing_map.emplace("L__self___w_pre", new at::Tensor(at::randn({4, 4}))); + rand_map.emplace("L__self___w_pre", new at::Tensor(at::randn({10}))); + rand_map.emplace("L__self___w_add", new at::Tensor(at::randn({10}))); + real_map.emplace("L__self___w_pre", new at::Tensor(weight_tensors)); + real_map.emplace("L__self___w_add", new at::Tensor(add_tensors)); + + std::unique_ptr runner; + if (device == "cpu") { + runner = std::make_unique( + model_so_path); +#if defined(USE_CUDA) || defined(USE_ROCM) + } else if (device == "cuda") { + runner = std::make_unique( + model_so_path); +#endif + } else { + testing::AssertionFailure() << "unsupported device: " << device; + } + // By default, buffer #1 get loaded with burned in weights. Correct results. + auto actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // Update with missing map which should throw. + // Somehow EXPECT_THROW doesn't work here when running tests in a row, but + // works when running AotInductorTest.RuntimeUpdateConstantsCuda individually. + try { + runner->update_constant_buffer(missing_map, false, true); + } catch (const std::runtime_error& e) { + EXPECT_THAT(e.what(), ::testing::HasSubstr("API call failed at")); + } + + // Update random weight to buffer #1. + runner->update_constant_buffer(missing_map, false, false); + actual_output_tensors = runner->run(input_tensors); + ASSERT_FALSE( + torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // Update with real map. + runner->update_constant_buffer(real_map, false, false); + actual_output_tensors = runner->run(input_tensors); + if (use_runtime_constant_folding) { + runner->run_const_fold(/* use_inactive = */ false); + } + actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // Update with full random map. + runner->update_constant_buffer(rand_map, false, false); + if (use_runtime_constant_folding) { + runner->run_const_fold(/* use_inactive = */ false); + } + actual_output_tensors = runner->run(input_tensors); + ASSERT_FALSE( + torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + for (auto& pair : missing_map) { + delete pair.second; + } + for (auto& pair : rand_map) { + delete pair.second; + } + for (auto& pair : real_map) { + delete pair.second; + } +} + +void test_aoti_extract_constants_map(const std::string& device) { + torch::NoGradGuard no_grad; + + std::string data_path = + (std::filesystem::path(STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "data.pt") + .string(); + + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string path_attr = "model_so_path_" + device; + std::string inputs_attr = "inputs_" + device; + std::string outputs_attr = "outputs_" + device; + std::string weights_attr = "w_pre_" + device; + std::string add_attr = "w_add_" + device; + const auto& model_so_path = data_loader.attr(path_attr.c_str()).toStringRef(); + auto input_tensors = + data_loader.attr(inputs_attr.c_str()).toTensorList().vec(); + const auto& ref_output_tensors = + data_loader.attr(outputs_attr.c_str()).toTensorList().vec(); + + const auto& weight_tensors = + data_loader.attr(weights_attr.c_str()).toTensor(); + const auto& add_tensors = data_loader.attr(add_attr.c_str()).toTensor(); + + torch::inductor::TensorConstantMap rand_map, real_map; + at::Tensor rand_pre, rand_add; + at::Tensor w_pre, w_add; + at::DeviceType device_type = device == "cuda" ? at::kCUDA : at::kCPU; + rand_pre = at::randn({4, 4}).to(device_type); + rand_add = at::randn({4, 4}).to(device_type); + w_pre = at::Tensor(weight_tensors).to(device_type); + w_add = at::Tensor(add_tensors).to(device_type); + + rand_map.emplace("L__self___w_pre", &rand_pre); + rand_map.emplace("L__self___w_add", &rand_add); + real_map.emplace("L__self___w_pre", &w_pre); + real_map.emplace("L__self___w_add", &w_add); + + std::unique_ptr runner; + if (device == "cpu") { + runner = std::make_unique( + model_so_path); +#if defined(USE_CUDA) || defined(USE_ROCM) + } else if (device == "cuda") { + runner = std::make_unique( + model_so_path); +#endif + } else { + testing::AssertionFailure() << "unsupported device: " << device; + } + + // By default, buffer #1 get loaded with burned in weights. Correct results. + auto actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // We update the weights to buffer #2 and activate it. This should still + // produce correct result, as it's the real constant map. + runner->update_inactive_constant_buffer(real_map); + auto extracted_inactive_weight = + runner->extract_constants_map(/* use_inactive = */ true); + auto extracted_active_weight = + runner->extract_constants_map(/* use_inactive = */ false); + auto cmp_real_map = derefTensorConstantMap(real_map); + auto cmp_rand_map = derefTensorConstantMap(rand_map); + ASSERT_TRUE(compareConstantMap(extracted_active_weight, cmp_real_map)); + ASSERT_TRUE(compareConstantMap(extracted_inactive_weight, cmp_real_map)); + + // We update random weights to buffer #1. But do not swap in the weight yet. + runner->update_inactive_constant_buffer(rand_map); + extracted_inactive_weight = + runner->extract_constants_map(/* use_inactive = */ true); + ASSERT_TRUE(compareConstantMap(extracted_inactive_weight, cmp_rand_map)); + + // We swap and activate the weight to buffer #1. + // Active weight now should be the new weight, while inactive should be the + // previous one. + runner->swap_constant_buffer(); + extracted_inactive_weight = + runner->extract_constants_map(/* use_inactive = */ true); + extracted_active_weight = + runner->extract_constants_map(/* use_inactive = */ false); + ASSERT_TRUE(compareConstantMap(extracted_active_weight, cmp_rand_map)); + ASSERT_TRUE(compareConstantMap(extracted_inactive_weight, cmp_real_map)); +} + +void test_aoti_double_buffering( + const std::string& device, + bool use_runtime_constant_folding) { + torch::NoGradGuard no_grad; + + std::string data_path = + (std::filesystem::path(STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "data.pt") + .string(); + + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string suffix = use_runtime_constant_folding + ? device + "_use_runtime_constant_folding" + : device; + std::string path_attr = "model_so_path_" + suffix; + std::string inputs_attr = "inputs_" + suffix; + std::string outputs_attr = "outputs_" + suffix; + std::string weights_attr = "w_pre_" + suffix; + std::string add_attr = "w_add_" + suffix; + const auto& model_so_path = data_loader.attr(path_attr.c_str()).toStringRef(); + auto input_tensors = + data_loader.attr(inputs_attr.c_str()).toTensorList().vec(); + const auto& ref_output_tensors = + data_loader.attr(outputs_attr.c_str()).toTensorList().vec(); + + const auto& weight_tensors = + data_loader.attr(weights_attr.c_str()).toTensor(); + const auto& add_tensors = data_loader.attr(add_attr.c_str()).toTensor(); + + torch::inductor::TensorConstantMap rand_map, real_map; + rand_map.emplace("L__self___w_pre", new at::Tensor(at::randn({4, 4}))); + rand_map.emplace("L__self___w_add", new at::Tensor(at::randn({4, 4}))); + real_map.emplace("L__self___w_pre", new at::Tensor(weight_tensors)); + real_map.emplace("L__self___w_add", new at::Tensor(add_tensors)); + + std::unique_ptr runner; + if (device == "cpu") { + runner = std::make_unique( + model_so_path); +#if defined(USE_CUDA) || defined(USE_ROCM) + } else if (device == "cuda") { + runner = std::make_unique( + model_so_path); +#endif + } else { + testing::AssertionFailure() << "unsupported device: " << device; + } + // By default, buffer #1 get loaded with burned in weights. Correct results. + auto actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // We update the weights to buffer #2 and activate it. This should still + // produce correct result, as it's the real constant map. + runner->update_inactive_constant_buffer(real_map); + if (use_runtime_constant_folding) { + runner->run_const_fold(/* use_inactive = */ true); + } + runner->swap_constant_buffer(); + actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // We update random weights to buffer #1. But do not swap in the weight yet. + runner->update_inactive_constant_buffer(rand_map); + if (use_runtime_constant_folding) { + runner->run_const_fold(/* use_inactive = */ true); + } + actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // We swap and activate the weight to buffer #1. This is random weight and + // should produce incorrect results. + runner->swap_constant_buffer(); + actual_output_tensors = runner->run(input_tensors); + ASSERT_FALSE( + torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // Swap back to buffer #2 which is the real constants. + runner->swap_constant_buffer(); + actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + for (auto& pair : rand_map) { + delete pair.second; + } + for (auto& pair : real_map) { + delete pair.second; + } +} + +#if defined(USE_CUDA) || defined(USE_ROCM) +void test_aoti_double_buffering_with_tensor_constants() { + torch::NoGradGuard no_grad; + + std::string data_path = (std::filesystem::path( + STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / + "data_with_tensor_constants.pt") + .string(); + + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string path_attr = "model_so_path"; + std::string inputs_attr = "inputs"; + std::string w_attr = "w"; + std::string outputs_attr = "outputs"; + const auto& model_so_path = data_loader.attr(path_attr.c_str()).toStringRef(); + auto input_tensors = + data_loader.attr(inputs_attr.c_str()).toTensorList().vec(); + const auto& w_tensors = data_loader.attr(w_attr.c_str()).toTensor(); + const auto& ref_output_tensors = + data_loader.attr(outputs_attr.c_str()).toTensorList().vec(); + + torch::inductor::TensorConstantMap real_map; + real_map.emplace("L__self___w", new at::Tensor(w_tensors)); + + std::unique_ptr runner; + runner = std::make_unique( + model_so_path.c_str()); + + // By default, buffer #1 get loaded with burned in weights. Correct results. + auto actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // We update the weights to buffer #2 and activate it. This should still + // produce correct result, since we would have copied the tensor_constants. + runner->update_inactive_constant_buffer(real_map); + runner->swap_constant_buffer(); + actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + for (auto& pair : real_map) { + delete pair.second; + } +} + +void test_aoti_user_managed_buffer() { + torch::NoGradGuard no_grad; + + std::string data_path = + (std::filesystem::path( + STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "large_data.pt") + .string(); + + // Memory information variable + size_t DATASIZE = 128 * 1024 * 1024; // We have 128MB of weight data. + + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string path_attr = "model_so_path"; + std::string inputs_attr = "inputs"; + std::string outputs_attr = "outputs"; + std::string weights_attr = "w_pre"; + std::string add_attr = "w_add"; + const auto& model_so_path = data_loader.attr(path_attr.c_str()).toStringRef(); + auto input_tensors = + data_loader.attr(inputs_attr.c_str()).toTensorList().vec(); + const auto& ref_output_tensors = + data_loader.attr(outputs_attr.c_str()).toTensorList().vec(); + + const auto& weight_tensors = + data_loader.attr(weights_attr.c_str()).toTensor(); + const auto& add_tensors = data_loader.attr(add_attr.c_str()).toTensor(); + + torch::inductor::TensorConstantMap rand_map, real_map; + at::Tensor rand_pre, rand_add; + at::Tensor w_pre, w_add; + rand_pre = at::randn({4096, 4096}).contiguous().to(at::kCUDA); + rand_add = at::randn({4096, 4096}).contiguous().to(at::kCUDA); + w_pre = at::Tensor(weight_tensors).contiguous().to(at::kCUDA); + w_add = at::Tensor(add_tensors).contiguous().to(at::kCUDA); + + rand_map.emplace("L__self___w_pre", &rand_pre); + rand_map.emplace("L__self___w_add", &rand_add); + real_map.emplace("L__self___w_pre", &w_pre); + real_map.emplace("L__self___w_add", &w_add); + + std::unique_ptr runner; + runner = std::make_unique( + model_so_path); + + // We extract the memory information starting from here. + int device_idx = -1; + cudaError_t cudaStatus; + cudaStatus = cudaGetDevice(&device_idx); + c10::cuda::CUDACachingAllocator::DeviceStats stats = + c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + size_t initTorchReserved = stats.reserved_bytes[0].current; + size_t torchReserved = stats.reserved_bytes[0].current; + if (cudaStatus != cudaSuccess || device_idx == -1) { + throw std::runtime_error("cudaGetDevice failed!"); + } + // This should contain one set of weight (128MB) loaded from .so + size_t initMemory = 0; + size_t totalMemory = 0; + size_t preFreeMemory = 0; + cudaStatus = cudaMemGetInfo(&preFreeMemory, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + // At this point, no memory should be consumed since we freed them all. + runner->swap_constant_buffer(); + runner->free_inactive_constant_buffer(); + runner->swap_constant_buffer(); + cudaStatus = cudaMemGetInfo(&initMemory, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + ASSERT_EQ(initMemory - DATASIZE, preFreeMemory); + + // We update the active buffer, but with user_managed = True. This shouldn't + // add any memory consumption. + runner->update_constant_buffer( + real_map, + /*use_inactive = */ false, + /*validate_full_updates = */ true, + /*user_managed = */ true); + size_t updateMemory = 0; + cudaStatus = cudaMemGetInfo(&updateMemory, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + ASSERT_EQ(initMemory, updateMemory); + + // Make sure the output is correct with user managed buffer. + auto actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // Update with rand_map and extract the output of rand_map. + // We let user_managed = false for rand_map, this should increase memory + // consumption. + cudaStatus = cudaMemGetInfo(&initMemory, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + runner->update_constant_buffer( + rand_map, + /*use_inactive = */ true, + /*validate_full_updates = */ true, + /*user_managed = */ false); + cudaStatus = cudaMemGetInfo(&updateMemory, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + ASSERT_EQ(initMemory - DATASIZE, updateMemory); + + runner->swap_constant_buffer(); + auto ref_rand_output_tensors = runner->run(input_tensors); + ASSERT_FALSE( + torch::allclose(ref_output_tensors[0], ref_rand_output_tensors[0])); + + // Free everything. + runner->free_inactive_constant_buffer(); + runner->swap_constant_buffer(); + runner->free_inactive_constant_buffer(); + + // Set buffer #1 user_managed, and #2 not user managed, and compare the + // underlying data + runner->update_constant_buffer( + real_map, + /*use_inactive = */ false, + /*validate_full_updates = */ true, + /*user_managed = */ false); + runner->update_constant_buffer( + real_map, + /*use_inactive = */ true, + /*validate_full_updates = */ true, + /*user_managed = */ true); + + auto extracted_active_weight = + runner->extract_constants_map(/* use_inactive = */ false); + auto extracted_inactive_weight = + runner->extract_constants_map(/* use_inactive = */ true); + auto cmp_real_map = derefTensorConstantMap(real_map); + // Value-wise all weights are equal + ASSERT_TRUE(compareConstantMap(extracted_active_weight, cmp_real_map)); + ASSERT_TRUE(compareConstantMap(extracted_inactive_weight, cmp_real_map)); + // Only when user_managed has the same underlying if set to true. + ASSERT_FALSE( + extracted_active_weight["L__self___w_pre"].data_ptr() == + cmp_real_map["L__self___w_pre"].data_ptr()); + ASSERT_TRUE( + extracted_inactive_weight["L__self___w_pre"].data_ptr() == + cmp_real_map["L__self___w_pre"].data_ptr()); + + // From non user_managed + actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // From user_managed + runner->swap_constant_buffer(); + actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); + + // We modify the buffer by the data's pointer outside of container. + cudaMemcpy( + real_map["L__self___w_add"]->data_ptr(), + rand_map["L__self___w_add"]->data_ptr(), + 4096 * 4096 * sizeof(float), + cudaMemcpyDeviceToDevice); + cudaMemcpy( + real_map["L__self___w_pre"]->data_ptr(), + rand_map["L__self___w_pre"]->data_ptr(), + 4096 * 4096 * sizeof(float), + cudaMemcpyDeviceToDevice); + + // We should get the result of the rand output. + actual_output_tensors = runner->run(input_tensors); + ASSERT_TRUE( + torch::allclose(ref_rand_output_tensors[0], actual_output_tensors[0])); +} + +void test_aoti_free_buffer(bool use_runtime_constant_folding) { + torch::NoGradGuard no_grad; + + std::string data_path = + (std::filesystem::path( + STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "large_data.pt") + .string(); + + // Memory information variable + size_t DATASIZE = 128 * 1024 * 1024; // We have 128MB of weight data. + size_t FOLDEDDATASIZE = use_runtime_constant_folding + ? 64 * 1024 * 1024 + : 0; // We have 64MB of folded data. + + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string path_attr = "model_so_path"; + if (use_runtime_constant_folding) { + path_attr += std::string("_use_runtime_constant_folding"); + } + std::string inputs_attr = "inputs"; + std::string outputs_attr = "outputs"; + std::string weights_attr = "w_pre"; + std::string add_attr = "w_add"; + const auto& model_so_path = data_loader.attr(path_attr.c_str()).toStringRef(); + auto input_tensors = + data_loader.attr(inputs_attr.c_str()).toTensorList().vec(); + const auto& ref_output_tensors = + data_loader.attr(outputs_attr.c_str()).toTensorList().vec(); + + const auto& weight_tensors = + data_loader.attr(weights_attr.c_str()).toTensor(); + const auto& add_tensors = data_loader.attr(add_attr.c_str()).toTensor(); + + torch::inductor::TensorConstantMap rand_map, real_map; + rand_map.emplace("L__self___w_pre", new at::Tensor(at::randn({4096, 4096}))); + rand_map.emplace("L__self___w_add", new at::Tensor(at::randn({4096, 4096}))); + real_map.emplace("L__self___w_pre", new at::Tensor(weight_tensors)); + real_map.emplace("L__self___w_add", new at::Tensor(add_tensors)); + + std::unique_ptr runner; + runner = std::make_unique( + model_so_path); + + // We extract the memory information starting from here. + int device_idx = -1; + cudaError_t cudaStatus; + cudaStatus = cudaGetDevice(&device_idx); + if (cudaStatus != cudaSuccess || device_idx == -1) { + throw std::runtime_error("cudaGetDevice failed!"); + } + c10::cuda::CUDACachingAllocator::DeviceStats stats = + c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + size_t initTorchActive = stats.active_bytes[0].current; + size_t initTorchReserved = stats.reserved_bytes[0].current; + // This should contain one set of weight (128MB) loaded from .so + size_t torchActive1, torchActive2; + size_t torchReserved1, torchReserved2; + size_t initMemory = 0; + size_t totalMemory = 0; + cudaStatus = cudaMemGetInfo(&initMemory, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + + // We update inactive buffer, this should create one copy (128MB) at buffer #2 + runner->update_inactive_constant_buffer(real_map); + size_t updateMemory2 = 0; + cudaStatus = cudaMemGetInfo(&updateMemory2, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + ASSERT_EQ(initMemory - DATASIZE, updateMemory2); + + // Call run, this should run const_fold and create the folded constant in #2 + // (64MB). + if (use_runtime_constant_folding) { + runner->run_const_fold(/* use_inactive = */ true); + stats = c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + torchActive1 = stats.active_bytes[0].current; + torchReserved1 = stats.reserved_bytes[0].current; + size_t constFoldMemory = 0; + cudaStatus = cudaMemGetInfo(&constFoldMemory, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + ASSERT_EQ( + initMemory - DATASIZE - (torchReserved1 - initTorchReserved), + constFoldMemory); + ASSERT_EQ(torchActive1 - initTorchActive, FOLDEDDATASIZE); + } + + // We swap and free the inactive buffer. (Use #2 and free #1) + // Note that buffer #1 does not include folded-const + stats = c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + torchActive1 = stats.active_bytes[0].current; + torchReserved1 = stats.reserved_bytes[0].current; + runner->swap_constant_buffer(); + runner->free_inactive_constant_buffer(); + stats = c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + torchActive2 = stats.active_bytes[0].current; + torchReserved2 = stats.reserved_bytes[0].current; + size_t postFreeMemory = 0; + cudaStatus = cudaMemGetInfo(&postFreeMemory, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + // We should only have one set of buffer (#2), available memory should equal + // initial memory minus the folded constants. + ASSERT_EQ(initMemory - (torchReserved2 - initTorchReserved), postFreeMemory); + // Buffer #1 does not include folded-consts + ASSERT_EQ(torchActive2 - torchActive1, 0); + + // We update random weights to buffer #1 and run const fold. + // We will have 2 full set of data plus 2 set of const-folded data. + runner->update_inactive_constant_buffer(rand_map); + runner->run_const_fold(/* use_inactive = */ true); + stats = c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + torchActive1 = stats.active_bytes[0].current; + torchReserved1 = stats.reserved_bytes[0].current; + size_t updateMemory1 = 0; + cudaStatus = cudaMemGetInfo(&updateMemory1, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + ASSERT_EQ( + initMemory - DATASIZE - (torchReserved1 - initTorchReserved), + updateMemory1); + ASSERT_EQ(torchActive1 - initTorchActive, 2 * FOLDEDDATASIZE); + + // We directly free the buffer #1. This would free the DATASIZE weight. + // If folded constant exists, it will not directly free the cudaMalloc, but + // decrease the active buffer in CachingAllocator instead. + stats = c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + torchActive1 = stats.active_bytes[0].current; + runner->free_inactive_constant_buffer(); + cudaStatus = cudaMemGetInfo(&updateMemory1, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + stats = c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + torchActive2 = stats.active_bytes[0].current; + torchReserved2 = stats.reserved_bytes[0].current; + ASSERT_EQ(initMemory - (torchReserved2 - initTorchReserved), updateMemory1); + ASSERT_EQ(FOLDEDDATASIZE, torchActive1 - torchActive2); + + // Free buffer #1 again, since #1 is freed, nothing should change. + stats = c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + torchActive1 = stats.active_bytes[0].current; + runner->free_inactive_constant_buffer(); + stats = c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + torchActive2 = stats.active_bytes[0].current; + cudaStatus = cudaMemGetInfo(&updateMemory1, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + ASSERT_EQ(initMemory - (torchReserved2 - initTorchReserved), updateMemory1); + ASSERT_EQ(torchActive1 - torchActive2, 0); + + // Swap and free #2, no data should exist in memory now. + // However, the folded constants might still occupies the CUDA memory in + // CachedAllocator. + stats = c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + torchActive1 = stats.active_bytes[0].current; + torchReserved1 = stats.reserved_bytes[0].current; + runner->swap_constant_buffer(); + runner->free_inactive_constant_buffer(); + stats = c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + torchActive2 = stats.active_bytes[0].current; + torchReserved2 = stats.reserved_bytes[0].current; + cudaStatus = cudaMemGetInfo(&updateMemory1, &totalMemory); + if (cudaStatus != cudaSuccess) { + throw std::runtime_error("cudaMemGetInfo failed!"); + } + + ASSERT_EQ( + initMemory + DATASIZE - (torchReserved2 - initTorchReserved), + updateMemory1); + ASSERT_EQ(FOLDEDDATASIZE, torchActive1 - torchActive2); + ASSERT_EQ(0, torchActive2 - initTorchActive); + + for (auto& pair : rand_map) { + delete pair.second; + } + for (auto& pair : real_map) { + delete pair.second; + } +} + +void test_cuda_alloc_test() { + torch::NoGradGuard no_grad; + + std::string data_path = + (std::filesystem::path( + STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "cuda_alloc_data.pt") + .string(); + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string path_attr = "model_so_path"; + std::string inputs_attr = "inputs"; + std::string outputs_attr = "outputs"; + const auto& model_so_path = data_loader.attr(path_attr.c_str()).toStringRef(); + const auto& ref_output_tensors = + data_loader.attr(outputs_attr.c_str()).toTensorList().vec(); + + size_t DATASIZE = 128 * 1024 * 1024; // We have 128MB of weight data. + + int device_idx = -1; + cudaError_t cudaStatus; + cudaStatus = cudaGetDevice(&device_idx); + if (cudaStatus != cudaSuccess || device_idx == -1) { + throw std::runtime_error("cudaGetDevice failed!"); + } + + c10::cuda::CUDACachingAllocator::emptyCache(); + c10::cuda::CUDACachingAllocator::DeviceStats stats = + c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + size_t initTorchActive = stats.allocated_bytes[0].current; + auto runner = std::make_unique( + model_so_path); + stats = c10::cuda::CUDACachingAllocator::getDeviceStats(device_idx); + size_t torchActive = stats.allocated_bytes[0].current; + + ASSERT_EQ(initTorchActive + DATASIZE, torchActive); + + auto actual_output_tensors = + runner->run(data_loader.attr(inputs_attr.c_str()).toTensorList().vec()); + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], actual_output_tensors[0])); +} + +#ifdef USE_CUDA +class ThreadPool { + private: + struct Task { + int id; + std::vector inputs; + }; + + std::vector workers; + std::vector cuda_streams; + std::queue tasks; + std::mutex queue_mutex; + std::condition_variable condition; + std::condition_variable completion_condition; + std::atomic active_tasks{0}; + std::atomic stop; + + public: + ThreadPool(size_t num_threads) : stop(false) { + // Create CUDA streams + cuda_streams.reserve(num_threads); + for (size_t i = 0; i < num_threads; ++i) { + cuda_streams.push_back(c10::cuda::getStreamFromPool()); + } + + // Create worker threads + for (size_t i = 0; i < num_threads; ++i) { + workers.emplace_back([this, i] { + while (true) { + Task task; + { + std::unique_lock lock(this->queue_mutex); + this->condition.wait( + lock, [this] { return this->stop || !this->tasks.empty(); }); + + if (this->stop && this->tasks.empty()) { + return; + } + + task = std::move(this->tasks.front()); + this->tasks.pop(); + } + + // Process the task with this thread's CUDA stream + process_function(task.id, task.inputs, this->cuda_streams[i]); + + // Mark task as completed + { + std::unique_lock lock(this->queue_mutex); + active_tasks--; + if (active_tasks == 0 && this->tasks.empty()) { + completion_condition.notify_all(); + } + } + } + }); + } + } + + // Updated processing function for vector of tensors and CUDA stream + std::function< + void(int, const std::vector&, c10::cuda::CUDAStream&)> + process_function; + + // Enqueue task with vector of tensors as input + void enqueue(int i, std::vector inputs) { + { + std::unique_lock lock(queue_mutex); + tasks.push({i, std::move(inputs)}); + active_tasks++; + } + condition.notify_one(); + } + + // Wait for all tasks to complete + void wait() { + std::unique_lock lock(queue_mutex); + completion_condition.wait( + lock, [this] { return active_tasks == 0 && tasks.empty(); }); + } + + ~ThreadPool() { + { + std::unique_lock lock(queue_mutex); + stop = true; + } + + condition.notify_all(); + for (std::thread& worker : workers) { + worker.join(); + } + } +}; + +void test_multi_cuda_streams(const std::string& device) { + c10::InferenceMode mode; + std::string data_path = + (std::filesystem::path(STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "data.pt") + .string(); + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string path_attr = "pt2_package_path_" + device; + std::string inputs_attr = "inputs_" + device; + std::string outputs_attr = "outputs_" + device; + const auto& pt2_package_path = + data_loader.attr(path_attr.c_str()).toStringRef(); + const auto& ref_output_tensors = + data_loader.attr(outputs_attr.c_str()).toTensorList().vec(); + auto inputs = data_loader.attr(inputs_attr.c_str()).toTensorList().vec(); + + constexpr int N = 16; + constexpr int num_threads = 4; + std::vector> all_outputs(N); + // Create thread pool with desired number of threads + torch::inductor::AOTIModelPackageLoader loader( + pt2_package_path, "model", false, num_threads); + ThreadPool pool(num_threads); + std::mutex results_mutex; + + // Set the processing function + pool.process_function = [&](int i, + const std::vector& inputs, + c10::cuda::CUDAStream& stream) { + // Run inference with the task-specific input + std::vector outputs = loader.run(inputs, stream.stream()); + // Store results safely + { + std::lock_guard lock(results_mutex); + all_outputs[i] = outputs; + } + }; + // Enqueue all tasks + for (int i = 0; i < N; i++) { + pool.enqueue(i, inputs); + } + // Wait for all tasks to complete + pool.wait(); + + for (int i = 0; i < N; i++) { + ASSERT_TRUE(torch::allclose(ref_output_tensors[0], all_outputs[i][0])); + } +} +#endif // USE_CUDA + +void test_concurrent_run_with_const_fold(const std::string& device) { + torch::NoGradGuard no_grad; + + std::string data_path = + (std::filesystem::path(STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "data.pt") + .string(); + + torch::jit::script::Module data_loader = torch::jit::load(data_path); + std::string suffix = device + "_use_runtime_constant_folding"; + const auto& model_so_path = + data_loader.attr(("model_so_path_" + suffix).c_str()).toStringRef(); + auto input_tensors = + data_loader.attr(("inputs_" + suffix).c_str()).toTensorList().vec(); + const auto& ref_output_tensors = + data_loader.attr(("outputs_" + suffix).c_str()).toTensorList().vec(); + + // num_models=1 forces all threads to contend for the single model instance. + std::unique_ptr runner; + if (device == "cuda") { + runner = std::make_unique( + model_so_path, /* num_models = */ 1); + } else { + FAIL() << "unsupported device: " << device; + } + + constexpr int num_threads = 4; + constexpr int num_iters = 4; + std::atomic ready{0}; + std::atomic failed{false}; + + std::vector threads; + for (int t = 0; t < num_threads; t++) { + threads.emplace_back([&]() { + ready.fetch_add(1); + while (ready.load() < num_threads) { + } + for (int i = 0; i < num_iters; i++) { + auto outputs = runner->run(input_tensors); + if (!torch::allclose(ref_output_tensors[0], outputs[0])) { + failed.store(true); + } + } + }); + } + + for (auto& t : threads) { + t.join(); + } + + ASSERT_FALSE(failed.load()) + << "One or more threads produced incorrect output"; +} + +// S638065 regression: with runtime constant folding, update_constant_buffer +// copies the new weights into the (inactive) constant buffer on the default +// stream (async device-to-device cudaMemcpy), while run_const_fold reads those +// constants on a separate stream. On ROCm the default stream is not implicitly +// ordered with other streams, so without a barrier the fold can read +// not-yet-copied weights and bake stale values into the folded constants. +// This exercises that exact edge: fold on a dedicated pool stream, then check +// the folded result against an independently computed reference. Each iteration +// uses fresh random weights so a stale read produces a detectably wrong output. +// The race is timing-dependent (the copy usually wins), hence the loop; a clean +// run depends on the cudaStreamSynchronize(0) at the end of +// update_constant_buffer (model_container.h). On NVIDIA the legacy default +// stream implicitly serializes, so this passes regardless. +void test_aoti_const_fold_separate_stream() { + torch::NoGradGuard no_grad; + + // Use the large (size=4096) model so the per-weight D2D copy is big enough to + // still be in flight when the fold starts reading. + std::string data_path = + (std::filesystem::path( + STRINGIZE(CMAKE_CURRENT_BINARY_DIR)) / "large_data.pt") + .string(); + torch::jit::script::Module data_loader = torch::jit::load(data_path); + const auto& model_so_path = + data_loader.attr("model_so_path_use_runtime_constant_folding") + .toStringRef(); + auto input_tensors = data_loader.attr("inputs").toTensorList().vec(); + const auto& x = input_tensors[0]; + const int64_t size = x.size(1); + + auto runner = std::make_unique( + model_so_path); + + // Dedicated non-blocking pool stream for folding, distinct from the default + // stream that update_constant_buffer copies the weights on. + c10::cuda::CUDAStream fold_stream = c10::cuda::getStreamFromPool(); + + constexpr int kIters = 100; + for (int i = 0; i < kIters; ++i) { + at::Tensor w_pre = at::randn({size, size}, x.options()); + at::Tensor w_add = at::randn({size, size}, x.options()); + // Independent reference, mirroring Net.forward in test.py. + at::Tensor expected = + at::matmul(x, at::relu(at::transpose(w_pre, 0, 1)) + w_add); + + torch::inductor::TensorConstantMap weight_map; + weight_map.emplace("L__self___w_pre", &w_pre); + weight_map.emplace("L__self___w_add", &w_add); + + // Copy weights into the inactive buffer (default stream), then fold on a + // separate stream. We deliberately do NOT synchronize between these two: + // ordering the copy before the fold is exactly what the fix must guarantee. + runner->update_inactive_constant_buffer(weight_map); + runner->run_const_fold( + /* use_inactive = */ true, + reinterpret_cast(fold_stream.stream())); + // Isolate the copy->fold edge (under test) from the fold->inference edge by + // finishing the fold before swapping the freshly folded buffer in. + fold_stream.synchronize(); + runner->swap_constant_buffer(); + + auto actual = runner->run(input_tensors); + ASSERT_TRUE(torch::allclose( + expected, actual[0], /* rtol = */ 1e-2, /* atol = */ 1e-2)) + << "iter " << i + << ": folded constants reflect stale weights (S638065); the const-fold " + "read the constant buffer before the default-stream weight copy " + "completed"; + } +} +#endif // USE_CUDA || USE_ROCM +} // namespace + +namespace torch::aot_inductor { + +// Test fixture that ensures test data is generated once for all tests +class AotInductorTest : public ::testing::Test { + public: + // This runs once before all tests in this test suite + static void SetUpTestSuite() { + ensureTestDataGenerated(); + } +}; + +TEST_F(AotInductorTest, BasicTestCpu) { + test_aoti("cpu", false); +} + +TEST_F(AotInductorTest, BasicScriptTestCpu) { + test_aoti_script("cpu"); +} + +TEST_F(AotInductorTest, BasicPackageLoaderTestCpu) { + test_aoti_package_loader("cpu", false); +} + +TEST_F(AotInductorTest, ExtractConstantsMapCpu) { + test_aoti_extract_constants_map("cpu"); +} + +#ifdef USE_CUDA +TEST_F(AotInductorTest, BasicTestCuda) { + test_aoti("cuda", true); + test_aoti("cuda", false); +} + +TEST_F(AotInductorTest, BasicScriptTestCuda) { + test_aoti_script("cuda"); +} + +TEST_F(AotInductorTest, BasicPackageLoaderTestCuda) { + test_aoti_package_loader("cuda", false); +} + +TEST_F(AotInductorTest, BasicPackageLoaderTestMultiGpuCuda) { + test_aoti_package_loader_multi_gpu("cuda", false); +} + +TEST_F(AotInductorTest, UpdateUserManagedConstantsCuda) { + test_aoti_user_managed_buffer(); +} + +TEST_F(AotInductorTest, RuntimeUpdateConstantsCuda) { + test_aoti_constants_update("cuda", true); +} + +TEST_F(AotInductorTest, UpdateConstantsCuda) { + test_aoti_constants_update("cuda", false); +} + +TEST_F(AotInductorTest, ExtractConstantsMapCuda) { + test_aoti_extract_constants_map("cuda"); +} + +TEST_F(AotInductorTest, RuntimeUpdateInactiveConstantsCuda) { + test_aoti_double_buffering("cuda", true); +} + +TEST_F(AotInductorTest, UpdateInactiveConstantsCuda) { + test_aoti_double_buffering("cuda", false); +} + +TEST_F(AotInductorTest, UpdateInactiveConstantsWithTensorConstantsCuda) { + test_aoti_double_buffering_with_tensor_constants(); +} + +TEST_F(AotInductorTest, FreeInactiveConstantBufferCuda) { + test_aoti_free_buffer(false); +} + +TEST_F(AotInductorTest, FreeInactiveConstantBufferRuntimeConstantFoldingCuda) { + test_aoti_free_buffer(true); +} + +TEST_F(AotInductorTest, MultiStreamTestCuda) { + test_multi_cuda_streams("cuda"); +} + +TEST_F(AotInductorTest, CudaAllocTestCuda) { + test_cuda_alloc_test(); +} + +TEST_F(AotInductorTest, ConcurrentRunConstFoldCuda) { + test_concurrent_run_with_const_fold("cuda"); +} +#endif + +// Registered for ROCm as well as CUDA: the S638065 race only manifests on AMD +// (on NVIDIA the legacy default stream implicitly orders with the fold stream). +#if defined(USE_CUDA) || defined(USE_ROCM) +TEST_F(AotInductorTest, ConstFoldSeparateStreamCuda) { + test_aoti_const_fold_separate_stream(); +} +#endif // USE_CUDA || USE_ROCM + +} // namespace torch::aot_inductor diff --git a/western_ai/usa/pytorch/test/cpp/aoti_inference/test.py b/western_ai/usa/pytorch/test/cpp/aoti_inference/test.py new file mode 100644 index 0000000000000000000000000000000000000000..38a53a6416346a591a1edfaf83563672dbb528a7 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/aoti_inference/test.py @@ -0,0 +1,219 @@ +import torch +import torch._inductor.config +from torch._export import aot_compile +from torch.export import Dim + + +torch.manual_seed(1337) + + +class Net(torch.nn.Module): + def __init__(self, device, size=4): + super().__init__() + self.w_pre = torch.randn(size, size, device=device) + self.w_add = torch.randn(size, size, device=device) + + def forward(self, x): + w_transpose = torch.transpose(self.w_pre, 0, 1) + w_relu = torch.nn.functional.relu(w_transpose) + w = w_relu + self.w_add + return torch.matmul(x, w) + + +class NetWithTensorConstants(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.w = torch.randn(30, 1, device="cuda") + + def forward(self, x, y): + z = self.w * x * y + return z[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17]] + + +data = {} +large_data = {} +cuda_alloc_data = {} +data_with_tensor_constants = {} + + +# Basice AOTI model test generation. +def generate_basic_tests(): + for device in ["cpu", "cuda"] if torch.cuda.is_available() else ["cpu"]: + for use_runtime_constant_folding in [True, False]: + if device == "cpu" and use_runtime_constant_folding: + # We do not test runtime const folding for cpu mode. + continue + model = Net(device).to(device=device) + x = torch.randn((4, 4), device=device) + with torch.no_grad(): + ref_output = model(x) + + torch._dynamo.reset() + with torch.no_grad(): + dim0_x = Dim("dim0_x", min=1, max=1024) + dynamic_shapes = {"x": {0: dim0_x}} + model_so_path = aot_compile( + model, + (x,), + dynamic_shapes=dynamic_shapes, + options={ + "aot_inductor.use_runtime_constant_folding": use_runtime_constant_folding + }, + ) + # Also store a .pt2 file using the aoti_compile_and_package API + pt2_package_path = torch._inductor.aoti_compile_and_package( + torch.export.export( + model, + (x,), + dynamic_shapes=dynamic_shapes, + ), + inductor_configs={ + "aot_inductor.use_runtime_constant_folding": use_runtime_constant_folding + }, + ) + + suffix = f"{device}" + if use_runtime_constant_folding: + suffix += "_use_runtime_constant_folding" + data.update( + { + f"model_so_path_{suffix}": model_so_path, + f"pt2_package_path_{suffix}": pt2_package_path, + f"inputs_{suffix}": [x], + f"outputs_{suffix}": [ref_output], + f"w_pre_{suffix}": model.w_pre, + f"w_add_{suffix}": model.w_add, + } + ) + + +def generate_basic_tests_consts_cpp(): + backup_consts_asm_cfg: bool = ( + torch._inductor.config.aot_inductor.use_consts_asm_build + ) + torch._inductor.config.aot_inductor.use_consts_asm_build = False + + # Test consts cpp build again. + generate_basic_tests() + + torch._inductor.config.aot_inductor.use_consts_asm_build = backup_consts_asm_cfg + + +def generate_large_tests(): + device = "cuda" + model = Net(device, size=4096).to(device=device) + x = torch.randn((4096, 4096), device=device) + with torch.no_grad(): + ref_output = model(x) + + torch._dynamo.reset() + for use_runtime_constant_folding in [True, False]: + with torch.no_grad(): + model_so_path = aot_compile( + model, + (x,), + options={ + "aot_inductor.use_runtime_constant_folding": use_runtime_constant_folding + }, + ) + # Also store a .pt2 file using the aoti_compile_and_package API + pt2_package_path = torch._inductor.aoti_compile_and_package( + torch.export.export( + model, + (x,), + ), + inductor_configs={ + "aot_inductor.use_runtime_constant_folding": use_runtime_constant_folding + }, + ) + + suffix = "_use_runtime_constant_folding" if use_runtime_constant_folding else "" + large_data.update( + { + f"model_so_path{suffix}": model_so_path, + f"pt2_package_path{suffix}": pt2_package_path, + "inputs": [x], + "outputs": [ref_output], + "w_pre": model.w_pre, + "w_add": model.w_add, + } + ) + + +def generate_cuda_alloc_test(): + device = "cuda" + model = Net(device, size=4096).to(device=device) + x = torch.randn((4096, 4096), device=device) + with torch.no_grad(): + ref_output = model(x) + + torch._dynamo.reset() + with torch.no_grad(): + model_so_path = aot_compile( + model, + (x,), + options={"aot_inductor.weight_use_caching_allocator": True}, + ) + + cuda_alloc_data.update( + { + "model_so_path": model_so_path, + "inputs": [x], + "outputs": [ref_output], + "w_pre": model.w_pre, + "w_add": model.w_add, + } + ) + + +# AOTI model which will create additional tensors during autograd. +def generate_test_with_additional_tensors(): + if not torch.cuda.is_available(): + return + + model = NetWithTensorConstants() + x = torch.randn((30, 1), device="cuda") + y = torch.randn((30, 1), device="cuda") + with torch.no_grad(): + ref_output = model(x, y) + + torch._dynamo.reset() + with torch.no_grad(): + model_so_path = aot_compile(model, (x, y)) + # Also store a .pt2 file using the aoti_compile_and_package API + pt2_package_path = torch._inductor.aoti_compile_and_package( + torch.export.export(model, (x, y)) + ) + + data_with_tensor_constants.update( + { + "model_so_path": model_so_path, + "pt2_package_path": pt2_package_path, + "inputs": [x, y], + "outputs": [ref_output], + "w": model.w, + } + ) + + +generate_basic_tests() +generate_basic_tests_consts_cpp() +generate_large_tests() +generate_test_with_additional_tensors() +generate_cuda_alloc_test() + + +# Use this to communicate tensors to the cpp code +class Serializer(torch.nn.Module): + def __init__(self, data): + super().__init__() + for key in data: + setattr(self, key, data[key]) + + +torch.jit.script(Serializer(data)).save("data.pt") +torch.jit.script(Serializer(large_data)).save("large_data.pt") +torch.jit.script(Serializer(data_with_tensor_constants)).save( + "data_with_tensor_constants.pt" +) +torch.jit.script(Serializer(cuda_alloc_data)).save("cuda_alloc_data.pt") diff --git a/western_ai/usa/pytorch/test/cpp/api/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/api/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..afb7d4ce74c2152d6ee1ee6808c0e81f4eb21aad --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/CMakeLists.txt @@ -0,0 +1,93 @@ +set(TORCH_API_TEST_DIR "${TORCH_ROOT}/test/cpp/api") +set(TORCH_API_TEST_SOURCES + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${TORCH_API_TEST_DIR}/autograd.cpp + ${TORCH_API_TEST_DIR}/any.cpp + ${TORCH_API_TEST_DIR}/dataloader.cpp + ${TORCH_API_TEST_DIR}/enum.cpp + ${TORCH_API_TEST_DIR}/expanding-array.cpp + ${TORCH_API_TEST_DIR}/fft.cpp + ${TORCH_API_TEST_DIR}/functional.cpp + ${TORCH_API_TEST_DIR}/init.cpp + ${TORCH_API_TEST_DIR}/integration.cpp + ${TORCH_API_TEST_DIR}/ivalue.cpp + ${TORCH_API_TEST_DIR}/jit.cpp + ${TORCH_API_TEST_DIR}/memory.cpp + ${TORCH_API_TEST_DIR}/meta_tensor.cpp + ${TORCH_API_TEST_DIR}/misc.cpp + ${TORCH_API_TEST_DIR}/module.cpp + ${TORCH_API_TEST_DIR}/moduledict.cpp + ${TORCH_API_TEST_DIR}/modulelist.cpp + ${TORCH_API_TEST_DIR}/modules.cpp + ${TORCH_API_TEST_DIR}/nested.cpp + ${TORCH_API_TEST_DIR}/parameterdict.cpp + ${TORCH_API_TEST_DIR}/parameterlist.cpp + ${TORCH_API_TEST_DIR}/namespace.cpp + ${TORCH_API_TEST_DIR}/nn_utils.cpp + ${TORCH_API_TEST_DIR}/optim.cpp + ${TORCH_API_TEST_DIR}/ordered_dict.cpp + ${TORCH_API_TEST_DIR}/rnn.cpp + ${TORCH_API_TEST_DIR}/sequential.cpp + ${TORCH_API_TEST_DIR}/transformer.cpp + ${TORCH_API_TEST_DIR}/serialize.cpp + ${TORCH_API_TEST_DIR}/special.cpp + ${TORCH_API_TEST_DIR}/static.cpp + ${TORCH_API_TEST_DIR}/support.cpp + ${TORCH_API_TEST_DIR}/tensor_cuda.cpp + ${TORCH_API_TEST_DIR}/tensor_indexing.cpp + ${TORCH_API_TEST_DIR}/tensor_options_cuda.cpp + ${TORCH_API_TEST_DIR}/tensor_options.cpp + ${TORCH_API_TEST_DIR}/tensor.cpp + ${TORCH_API_TEST_DIR}/torch_include.cpp + ${TORCH_API_TEST_DIR}/inference_mode.cpp + ${TORCH_API_TEST_DIR}/grad_mode.cpp + ${TORCH_API_TEST_DIR}/operations.cpp + ${TORCH_API_TEST_DIR}/nested_int.cpp +) +if(USE_CUDA OR USE_ROCM) + list(APPEND TORCH_API_TEST_SOURCES ${TORCH_API_TEST_DIR}/parallel.cpp) +endif() + +add_executable(test_api ${TORCH_API_TEST_SOURCES}) +target_include_directories(test_api PRIVATE ${ATen_CPU_INCLUDE}) +target_link_libraries(test_api PRIVATE torch gtest_main gmock) + +if(USE_CUDA) + target_compile_definitions(test_api PRIVATE "USE_CUDA") +endif() + +if(NOT MSVC) + # Clang has an unfixed bug leading to spurious missing braces + # warnings, see https://bugs.llvm.org/show_bug.cgi?id=21629 + target_compile_options_if_supported(test_api "-Wno-missing-braces") + # Considered to be flaky. See the discussion at + # https://github.com/pytorch/pytorch/pull/9608 + target_compile_options_if_supported(test_api "-Wno-maybe-uninitialized") + # gcc gives nonsensical warnings about variadic.h + target_compile_options_if_supported(test_api "-Wno-unused-but-set-parameter") + + # Add -Wno-error=nonnull for GCC 12+ + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12) + target_compile_options_if_supported(test_api "-Wno-error=nonnull") + endif() + + # Add -Wno-error=array-bounds for GCC 13+ + # See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=113239 + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13) + target_compile_options_if_supported(test_api "-Wno-error=array-bounds") + endif() + +endif() + +if(INSTALL_TEST) + set_target_properties(test_api PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS test_api DESTINATION bin) + # Install PDB files for MSVC builds + if(MSVC AND BUILD_SHARED_LIBS) + install(FILES $ DESTINATION bin OPTIONAL) + endif() +endif() + +add_executable(parallel_benchmark ${TORCH_API_TEST_DIR}/parallel_benchmark.cpp) +target_include_directories(parallel_benchmark PRIVATE ${ATen_CPU_INCLUDE}) +target_link_libraries(parallel_benchmark PRIVATE torch) diff --git a/western_ai/usa/pytorch/test/cpp/api/README.md b/western_ai/usa/pytorch/test/cpp/api/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9921168274846d71a15b7c49eed1f6c86f699544 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/README.md @@ -0,0 +1,35 @@ +# C++ Frontend Tests + +In this folder live the tests for PyTorch's C++ Frontend. They use the +[GoogleTest](https://github.com/google/googletest) test framework. + +## CUDA Tests + +To make a test runnable only on platforms with CUDA, you should suffix your +test with `_CUDA`, e.g. + +```cpp +TEST(MyTestSuite, MyTestCase_CUDA) { } +``` + +To make it runnable only on platforms with at least two CUDA machines, suffix +it with `_MultiCUDA` instead of `_CUDA`, e.g. + +```cpp +TEST(MyTestSuite, MyTestCase_MultiCUDA) { } +``` + +There is logic in `main.cpp` that detects the availability and number of CUDA +devices and supplies the appropriate negative filters to GoogleTest. + +## Integration Tests + +Integration tests use the MNIST dataset. You must download it by running the +following command from the PyTorch root folder: + +```sh +$ python tools/download_mnist.py -d test/cpp/api/mnist +``` + +The required paths will be referenced as `test/cpp/api/mnist/...` in the test +code, so you *must* run the integration tests from the PyTorch root folder. diff --git a/western_ai/usa/pytorch/test/cpp/api/any.cpp b/western_ai/usa/pytorch/test/cpp/api/any.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a0d9b603348a8cb1294475bae77bba06049b6cb0 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/any.cpp @@ -0,0 +1,457 @@ +#include + +#include + +#include + +#include +#include + +using namespace torch::nn; + +struct AnyModuleTest : torch::test::SeedingFixture {}; + +TEST_F(AnyModuleTest, SimpleReturnType) { + struct M : torch::nn::Module { + int forward() { + return 123; + } + }; + AnyModule any(M{}); + ASSERT_EQ(any.forward(), 123); +} + +TEST_F(AnyModuleTest, SimpleReturnTypeAndSingleArgument) { + struct M : torch::nn::Module { + int forward(int x) { + return x; + } + }; + AnyModule any(M{}); + ASSERT_EQ(any.forward(5), 5); +} + +TEST_F(AnyModuleTest, StringLiteralReturnTypeAndArgument) { + struct M : torch::nn::Module { + const char* forward(const char* x) { + return x; + } + }; + AnyModule any(M{}); + ASSERT_EQ(any.forward("hello"), std::string("hello")); +} + +TEST_F(AnyModuleTest, StringReturnTypeWithConstArgument) { + struct M : torch::nn::Module { + std::string forward(int x, const double f) { + return std::to_string(static_cast(x + f)); + } + }; + AnyModule any(M{}); + int x = 4; + ASSERT_EQ(any.forward(x, 3.14), std::string("7")); +} + +TEST_F( + AnyModuleTest, + TensorReturnTypeAndStringArgumentsWithFunkyQualifications) { + struct M : torch::nn::Module { + torch::Tensor forward( + std::string a, + const std::string& b, + std::string&& c) { + const auto s = a + b + c; + return torch::ones({static_cast(s.size())}); + } + }; + AnyModule any(M{}); + ASSERT_TRUE( + any.forward(std::string("a"), std::string("ab"), std::string("abc")) + .sum() + .item() == 6); +} + +TEST_F(AnyModuleTest, WrongArgumentType) { + struct M : torch::nn::Module { + int forward(float x) { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + return x; + } + }; + AnyModule any(M{}); + ASSERT_THROWS_WITH( + any.forward(5.0), + "Expected argument #0 to be of type float, " + "but received value of type double"); +} + +struct M_test_wrong_number_of_arguments : torch::nn::Module { + int forward(int a, int b) { + return a + b; + } +}; + +TEST_F(AnyModuleTest, WrongNumberOfArguments) { + AnyModule any(M_test_wrong_number_of_arguments{}); +#if defined(_MSC_VER) + std::string module_name = "struct M_test_wrong_number_of_arguments"; +#else + std::string module_name = "M_test_wrong_number_of_arguments"; +#endif + ASSERT_THROWS_WITH( + any.forward(), + module_name + + "'s forward() method expects 2 argument(s), but received 0. " + "If " + + module_name + + "'s forward() method has default arguments, " + "please make sure the forward() method is declared with a corresponding `FORWARD_HAS_DEFAULT_ARGS` macro."); + ASSERT_THROWS_WITH( + any.forward(5), + module_name + + "'s forward() method expects 2 argument(s), but received 1. " + "If " + + module_name + + "'s forward() method has default arguments, " + "please make sure the forward() method is declared with a corresponding `FORWARD_HAS_DEFAULT_ARGS` macro."); + ASSERT_THROWS_WITH( + any.forward(1, 2, 3), + module_name + + "'s forward() method expects 2 argument(s), but received 3."); +} + +struct M_default_arg_with_macro : torch::nn::Module { + double forward(int a, int b = 2, double c = 3.0) { + return a + b + c; + } + + protected: + FORWARD_HAS_DEFAULT_ARGS( + {1, torch::nn::AnyValue(2)}, + {2, torch::nn::AnyValue(3.0)}) +}; + +struct M_default_arg_without_macro : torch::nn::Module { + double forward(int a, int b = 2, double c = 3.0) { + return a + b + c; + } +}; + +TEST_F( + AnyModuleTest, + PassingArgumentsToModuleWithDefaultArgumentsInForwardMethod) { + { + AnyModule any(M_default_arg_with_macro{}); + + ASSERT_EQ(any.forward(1), 6.0); + ASSERT_EQ(any.forward(1, 3), 7.0); + ASSERT_EQ(any.forward(1, 3, 5.0), 9.0); + + ASSERT_THROWS_WITH( + any.forward(), + "M_default_arg_with_macro's forward() method expects at least 1 argument(s) and at most 3 argument(s), but received 0."); + ASSERT_THROWS_WITH( + any.forward(1, 2, 3.0, 4), + "M_default_arg_with_macro's forward() method expects at least 1 argument(s) and at most 3 argument(s), but received 4."); + } + { + AnyModule any(M_default_arg_without_macro{}); + + ASSERT_EQ(any.forward(1, 3, 5.0), 9.0); + +#if defined(_MSC_VER) + std::string module_name = "struct M_default_arg_without_macro"; +#else + std::string module_name = "M_default_arg_without_macro"; +#endif + + ASSERT_THROWS_WITH( + any.forward(), + module_name + + "'s forward() method expects 3 argument(s), but received 0. " + "If " + + module_name + + "'s forward() method has default arguments, " + "please make sure the forward() method is declared with a corresponding `FORWARD_HAS_DEFAULT_ARGS` macro."); + ASSERT_THROWS_WITH( + any.forward(1), + module_name + + "'s forward() method expects 3 argument(s), but received 1. " + "If " + + module_name + + "'s forward() method has default arguments, " + "please make sure the forward() method is declared with a corresponding `FORWARD_HAS_DEFAULT_ARGS` macro."); + ASSERT_THROWS_WITH( + any.forward(1, 3), + module_name + + "'s forward() method expects 3 argument(s), but received 2. " + "If " + + module_name + + "'s forward() method has default arguments, " + "please make sure the forward() method is declared with a corresponding `FORWARD_HAS_DEFAULT_ARGS` macro."); + ASSERT_THROWS_WITH( + any.forward(1, 2, 3.0, 4), + module_name + + "'s forward() method expects 3 argument(s), but received 4."); + } +} + +struct M : torch::nn::Module { + explicit M(int value_) : torch::nn::Module("M"), value(value_) {} + int value; + int forward(float x) { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + return x; + } +}; + +TEST_F(AnyModuleTest, GetWithCorrectTypeSucceeds) { + AnyModule any(M{5}); + ASSERT_EQ(any.get().value, 5); +} + +TEST_F(AnyModuleTest, GetWithIncorrectTypeThrows) { + struct N : torch::nn::Module { + torch::Tensor forward(torch::Tensor input) { + return input; + } + }; + AnyModule any(M{5}); + ASSERT_THROWS_WITH(any.get(), "Attempted to cast module"); +} + +TEST_F(AnyModuleTest, PtrWithBaseClassSucceeds) { + AnyModule any(M{5}); + auto ptr = any.ptr(); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(ptr->name(), "M"); +} + +TEST_F(AnyModuleTest, PtrWithGoodDowncastSuccceeds) { + AnyModule any(M{5}); + auto ptr = any.ptr(); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(ptr->value, 5); +} + +TEST_F(AnyModuleTest, PtrWithBadDowncastThrows) { + struct N : torch::nn::Module { + torch::Tensor forward(torch::Tensor input) { + return input; + } + }; + AnyModule any(M{5}); + ASSERT_THROWS_WITH(any.ptr(), "Attempted to cast module"); +} + +TEST_F(AnyModuleTest, DefaultStateIsEmpty) { + struct M : torch::nn::Module { + explicit M(int value_) : value(value_) {} + int value; + int forward(float x) { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + return x; + } + }; + AnyModule any; + ASSERT_TRUE(any.is_empty()); + any = std::make_shared(5); + ASSERT_FALSE(any.is_empty()); + ASSERT_EQ(any.get().value, 5); +} + +TEST_F(AnyModuleTest, AllMethodsThrowForEmptyAnyModule) { + struct M : torch::nn::Module { + int forward(int x) { + return x; + } + }; + AnyModule any; + ASSERT_TRUE(any.is_empty()); + ASSERT_THROWS_WITH(any.get(), "Cannot call get() on an empty AnyModule"); + ASSERT_THROWS_WITH(any.ptr(), "Cannot call ptr() on an empty AnyModule"); + ASSERT_THROWS_WITH(any.ptr(), "Cannot call ptr() on an empty AnyModule"); + ASSERT_THROWS_WITH( + any.type_info(), "Cannot call type_info() on an empty AnyModule"); + ASSERT_THROWS_WITH( + any.forward(5), "Cannot call forward() on an empty AnyModule"); +} + +TEST_F(AnyModuleTest, CanMoveAssignDifferentModules) { + struct M : torch::nn::Module { + std::string forward(int x) { + return std::to_string(x); + } + }; + struct N : torch::nn::Module { + int forward(float x) { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + return 3 + x; + } + }; + AnyModule any; + ASSERT_TRUE(any.is_empty()); + any = std::make_shared(); + ASSERT_FALSE(any.is_empty()); + ASSERT_EQ(any.forward(5), "5"); + any = std::make_shared(); + ASSERT_FALSE(any.is_empty()); + ASSERT_EQ(any.forward(5.0f), 8); +} + +TEST_F(AnyModuleTest, ConstructsFromModuleHolder) { + struct MImpl : torch::nn::Module { + explicit MImpl(int value_) : torch::nn::Module("M"), value(value_) {} + int value; + int forward(float x) { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + return x; + } + }; + + struct M : torch::nn::ModuleHolder { + using torch::nn::ModuleHolder::ModuleHolder; + using torch::nn::ModuleHolder::get; + }; + + AnyModule any(M{5}); + ASSERT_EQ(any.get().value, 5); + ASSERT_EQ(any.get()->value, 5); + + AnyModule module(Linear(3, 4)); + std::shared_ptr ptr = module.ptr(); + Linear linear(module.get()); +} + +TEST_F(AnyModuleTest, ConvertsVariableToTensorCorrectly) { + struct M : torch::nn::Module { + torch::Tensor forward(torch::Tensor input) { + return input; + } + }; + + // When you have an autograd::Variable, it should be converted to a + // torch::Tensor before being passed to the function (to avoid a type + // mismatch). + AnyModule any(M{}); + ASSERT_TRUE( + any.forward(torch::autograd::Variable(torch::ones(5))) + .sum() + .item() == 5); + // at::Tensors that are not variables work too. + ASSERT_EQ(any.forward(at::ones(5)).sum().item(), 5); +} + +namespace torch { +namespace nn { +struct TestAnyValue { + template + // NOLINTNEXTLINE(bugprone-forwarding-reference-overload) + explicit TestAnyValue(T&& value) : value_(std::forward(value)) {} + AnyValue operator()() { + return std::move(value_); + } + AnyValue value_; +}; +template +AnyValue make_value(T&& value) { + return TestAnyValue(std::forward(value))(); +} +} // namespace nn +} // namespace torch + +struct AnyValueTest : torch::test::SeedingFixture {}; + +TEST_F(AnyValueTest, CorrectlyAccessesIntWhenCorrectType) { + auto value = make_value(5); + ASSERT_NE(value.try_get(), nullptr); + // const and non-const types have the same typeid(), + // but casting Holder to Holder is undefined + // behavior according to UBSAN: + // https://github.com/pytorch/pytorch/issues/26964 + // ASSERT_NE(value.try_get(), nullptr); + ASSERT_EQ(value.get(), 5); +} +// This test does not work at all, because it looks like make_value +// decays const int into int. +// TEST_F(AnyValueTest, CorrectlyAccessesConstIntWhenCorrectType) { +// auto value = make_value(5); +// ASSERT_NE(value.try_get(), nullptr); +// // ASSERT_NE(value.try_get(), nullptr); +// ASSERT_EQ(value.get(), 5); +//} +TEST_F(AnyValueTest, CorrectlyAccessesStringLiteralWhenCorrectType) { + auto value = make_value("hello"); + ASSERT_NE(value.try_get(), nullptr); + ASSERT_EQ(value.get(), std::string("hello")); +} +TEST_F(AnyValueTest, CorrectlyAccessesStringWhenCorrectType) { + auto value = make_value(std::string("hello")); + ASSERT_NE(value.try_get(), nullptr); + ASSERT_EQ(value.get(), "hello"); +} +TEST_F(AnyValueTest, CorrectlyAccessesPointersWhenCorrectType) { + std::string s("hello"); + std::string* p = &s; + auto value = make_value(p); + ASSERT_NE(value.try_get(), nullptr); + ASSERT_EQ(*value.get(), "hello"); +} +TEST_F(AnyValueTest, CorrectlyAccessesReferencesWhenCorrectType) { + std::string s("hello"); + const std::string& t = s; + auto value = make_value(t); + ASSERT_NE(value.try_get(), nullptr); + ASSERT_EQ(value.get(), "hello"); +} + +TEST_F(AnyValueTest, TryGetReturnsNullptrForTheWrongType) { + auto value = make_value(5); + ASSERT_NE(value.try_get(), nullptr); + ASSERT_EQ(value.try_get(), nullptr); + ASSERT_EQ(value.try_get(), nullptr); + ASSERT_EQ(value.try_get(), nullptr); +} + +TEST_F(AnyValueTest, GetThrowsForTheWrongType) { + auto value = make_value(5); + ASSERT_NE(value.try_get(), nullptr); + ASSERT_THROWS_WITH( + value.get(), + "Attempted to cast AnyValue to float, " + "but its actual type is int"); + ASSERT_THROWS_WITH( + value.get(), + "Attempted to cast AnyValue to long, " + "but its actual type is int"); +} + +TEST_F(AnyValueTest, MoveConstructionIsAllowed) { + auto value = make_value(5); + auto copy = make_value(std::move(value)); + ASSERT_NE(copy.try_get(), nullptr); + ASSERT_EQ(copy.get(), 5); +} + +TEST_F(AnyValueTest, MoveAssignmentIsAllowed) { + auto value = make_value(5); + auto copy = make_value(10); + copy = std::move(value); + ASSERT_NE(copy.try_get(), nullptr); + ASSERT_EQ(copy.get(), 5); +} + +TEST_F(AnyValueTest, TypeInfoIsCorrectForInt) { + auto value = make_value(5); + ASSERT_EQ(value.type_info().hash_code(), typeid(int).hash_code()); +} + +TEST_F(AnyValueTest, TypeInfoIsCorrectForStringLiteral) { + auto value = make_value("hello"); + ASSERT_EQ(value.type_info().hash_code(), typeid(const char*).hash_code()); +} + +TEST_F(AnyValueTest, TypeInfoIsCorrectForString) { + auto value = make_value(std::string("hello")); + ASSERT_EQ(value.type_info().hash_code(), typeid(std::string).hash_code()); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/autograd.cpp b/western_ai/usa/pytorch/test/cpp/api/autograd.cpp new file mode 100644 index 0000000000000000000000000000000000000000..16e972d9539e062402fc24cac991e8da26e5512e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/autograd.cpp @@ -0,0 +1,1722 @@ +#include +#include + +#include +#include + +#include +#include +#include + +#include + +using namespace torch::autograd; +using namespace torch::test; + +#define ASSERT_VARIABLE_EQ(a, b) ASSERT_TRUE(torch::allclose((a), (b))) +#define EXPECT_VARIABLE_EQ(a, b) EXPECT_TRUE(torch::allclose((a), (b))) + +std::string graph_desc(c10::intrusive_ptr node) { + if (!node) { + return "None"; + } + auto result = node->name() + "("; + auto next_edges = node->next_edges(); + for (auto& edge : next_edges) { + result += graph_desc(edge.function); + } + return result + ")"; +} + +Variable simple_fn(const Variable& x, const Variable& y) { + return x + 2 * y + x * y; +} + +TEST(AutogradAPITests, RegisterHookVoidReturnAcceptsUndefinedTensor) { + auto x = at::zeros({}, at::kCPU); + x.requires_grad_(); + x.register_hook([](at::TensorBase x) { return; }); + auto y = torch::autograd::UndefinedGrad().apply({x}); + y[0].backward(); +} + +TEST(AutogradAPITests, RegisterHookTensorReturnAcceptsUndefinedTensor) { + auto x = at::zeros({}, at::kCPU); + x.requires_grad_(); + x.register_hook([](at::Tensor x) -> at::Tensor { return x; }); + auto y = torch::autograd::UndefinedGrad().apply({x}); + y[0].backward(); +} + +TEST(AutogradAPITests, BackwardSimpleTest) { + Variable x = torch::randn({2, 2}, torch::requires_grad()); + Variable y = torch::randn({2, 2}, torch::requires_grad()); + auto res = simple_fn(x, y); + backward({res.sum()}, {}); + + ASSERT_VARIABLE_EQ(x.grad(), y + torch::ones({2, 2})); + ASSERT_VARIABLE_EQ(y.grad(), x + torch::ones({2, 2}) * 2); +} + +TEST(AutogradAPITests, BackwardTest) { + Variable x = torch::randn({2, 2}, torch::requires_grad()); + Variable y = torch::randn({2, 2}, torch::requires_grad()); + auto res = simple_fn(x, y); + backward({res}, {torch::ones({2, 2})}, {}, true); + + backward({res}, {torch::ones({2, 2})}); + + ASSERT_VARIABLE_EQ(x.grad(), 2 * (y + torch::ones({2, 2}))); + ASSERT_VARIABLE_EQ(y.grad(), 2 * (x + torch::ones({2, 2}) * 2)); +} + +TEST(AutogradAPITests, GradSimpleTest) { + // basic grad + Variable x = torch::randn({2, 2}, torch::requires_grad()); + Variable y = torch::randn({2, 2}, torch::requires_grad()); + auto res = simple_fn(x, y); + auto grad_res = grad({res}, {x, y}, {torch::ones({2, 2})}); + + ASSERT_VARIABLE_EQ(grad_res[0], y + torch::ones({2, 2})); + ASSERT_VARIABLE_EQ(grad_res[1], x + torch::ones({2, 2}) * 2); +} + +TEST(AutogradAPITests, GradTest) { + Variable x = torch::randn({2, 2}, torch::requires_grad()); + Variable y = torch::randn({2, 2}, torch::requires_grad()); + auto res = simple_fn(x, y); + res.backward(torch::ones({2, 2}), false, true); + + Variable x_grad = y + torch::ones({2, 2}); + Variable y_grad = x + torch::ones({2, 2}) * 2; + ASSERT_VARIABLE_EQ(x.grad(), x_grad); + ASSERT_VARIABLE_EQ(y.grad(), y_grad); + + Variable grad_sum = 2 * x.grad() + y.grad(); + auto x_hv = grad({grad_sum}, {x}, {torch::ones({2, 2})}, {}, true); + + ASSERT_VARIABLE_EQ(x_hv[0], torch::ones({2, 2})); + ASSERT_VARIABLE_EQ(x.grad(), x_grad); + ASSERT_VARIABLE_EQ(y.grad(), y_grad); +} + +TEST(AutogradAPITests, GradNonLeafTest) { + Variable x_init = torch::randn({2, 2}, torch::requires_grad()); + Variable x = x_init; + Variable y = torch::randn({2, 2}, torch::requires_grad()); + Variable grad_output = torch::ones({2, 2}); + + for (int i = 0; i < 5; ++i) { + auto res = simple_fn(x, y); + auto input_grads = grad({res}, {x}, {grad_output}, {}, true); + + Variable grad_x_expected = y + torch::ones({2, 2}); + ASSERT_VARIABLE_EQ(input_grads[0], grad_x_expected); + ASSERT_FALSE(x.grad().defined()); + ASSERT_FALSE(y.grad().defined()); + x = x + 0.05 * input_grads[0]; + } + + float val_init = simple_fn(x_init, y).sum().item().toFloat(); + float val_final = simple_fn(x, y).sum().item().toFloat(); + ASSERT_TRUE(val_final > val_init); + + x.backward(grad_output, false, true); + ASSERT_TRUE(x_init.grad().defined()); + ASSERT_TRUE(y.grad().defined()); +} + +TEST(AutogradAPITests, GradUnreachableTest) { + Variable x = torch::ones({1}, torch::requires_grad()); + Variable y = torch::ones({1}, torch::requires_grad()); + + Variable z = x * 2; + Variable w = y * 2; + + auto grad_res = grad({x * 2}, {x, y}, {}, {}, false, true); + ASSERT_VARIABLE_EQ(grad_res[0], x * 2); + ASSERT_FALSE(grad_res[1].defined()); + + // This is slightly different than the case above, because z doesn't even + // have a grad accumulator allocated. + z = torch::ones({1}, torch::requires_grad()); + grad_res = grad({x * 2}, {x, z}, {}, {}, false, true); + + ASSERT_VARIABLE_EQ(grad_res[0], x * 2); + ASSERT_FALSE(grad_res[1].defined()); + + // allow_unused=False, but grads contains None inside, should throw + ASSERT_THROWS_WITH( + grad({x * 2}, {x, y}, {}, {}, false, false), "Set allow_unused=True"); +} + +TEST(CustomAutogradTest, GradUnreachableDiscoveryTest) { + // Test that certain nodes are not erroneously executed when an input + // is unreachable. See #39784 + struct MyFunction : public Function { + static Variable forward(AutogradContext* ctx, Variable var) { + return var; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + ADD_FAILURE() << "This node should not be executed!"; + return grad_output; + } + }; + + auto x = torch::randn(1, torch::requires_grad()); + auto x1 = torch::randn(1); + auto x2 = MyFunction::apply(x + x1); + + auto y = torch::randn(1, torch::requires_grad()); + auto grad_res = torch::autograd::grad({x2}, {y}, {}, {}, false, true); + ASSERT_FALSE(grad_res[0].defined()); +} + +TEST(AutogradAPITests, EmptyInput) { + Variable x = torch::ones({1}, torch::requires_grad()); + ASSERT_THROWS_WITH( + grad({x * 2}, /*inputs=*/{}, {x}), "grad requires non-empty inputs."); +} + +TEST(AutogradAPITests, RetainGrad) { + auto input = torch::rand({1, 3}, torch::requires_grad()); + auto h1 = input * 3; + auto out = (h1 * h1).sum(); + + { + // Warning when grad is accessed for non-leaf tensor + WarningCapture warnings; + ASSERT_FALSE(h1.grad().defined()); + ASSERT_TRUE(warnings.str().find("is not a leaf") != std::string::npos); + } + // It should be possible to call retain_grad() multiple times + h1.retain_grad(); + h1.retain_grad(); + { + // If retain_grad is true for a non-leaf tensor, + // there should not be any warning when grad is accessed + WarningCapture warnings; + ASSERT_FALSE(h1.grad().defined()); + ASSERT_FALSE(warnings.str().find("is not a leaf") != std::string::npos); + } + + // Gradient should be accumulated + // NOLINTNEXTLINE(bugprone-argument-comment) + out.backward({}, /*keep_graph=*/true); + ASSERT_VARIABLE_EQ(h1 * 2, h1.grad()); + // NOLINTNEXTLINE(bugprone-argument-comment) + out.backward({}, /*keep_graph=*/true); + ASSERT_VARIABLE_EQ(h1 * 4, h1.grad()); + + { + torch::NoGradGuard no_grad; + input.grad().zero_(); + } + // It should be a no-op for leaves + input.retain_grad(); + input.retain_grad(); + out.backward(); + ASSERT_VARIABLE_EQ(input * 18, input.grad()); +} + +TEST(AutogradAPITests, AnomalyMode) { + // Needs to have backtrace as warning and then throw an error + torch::autograd::DetectAnomalyGuard detect_anomaly; + { + WarningCapture warnings; + auto x = torch::tensor({5.0}, torch::requires_grad()); + auto y = x * x; + auto z = y * y; + y += 1; + ASSERT_THROWS_WITH(z.backward(), "inplace"); + ASSERT_TRUE( + warnings.str().find("Traceback of forward") != std::string::npos); + } + auto double_backward_produce_nan = [](bool should_throw) { + auto x = torch::tensor({0.0}, torch::requires_grad()); + auto y = x.pow(1.5); + auto gr = + // NOLINTNEXTLINE(bugprone-argument-comment) + grad({y}, {x}, {}, /*retain_graph=*/true, /*create_backward=*/true); + if (should_throw) { + WarningCapture warnings; + ASSERT_THROWS_WITH(grad({gr[0]}, {x}, {torch::tensor({0.0})}); + , "returned nan"); + auto msgs = warnings.messages(); + ASSERT_EQ(msgs.size(), 2); + ASSERT_TRUE( + msgs[0].find("Traceback of forward call that caused the error") != + std::string::npos); + ASSERT_TRUE( + msgs[1].find( + "Traceback of forward call that induced the previous calculation") != + std::string::npos); + } else { + grad({gr[0]}, {x}, {torch::tensor({0.0})}); + } + }; + + double_backward_produce_nan(true); + { + torch::autograd::DetectAnomalyGuard detect_anomaly(/*check_nan=*/false); + double_backward_produce_nan(false); + { + torch::autograd::DetectAnomalyGuard detect_anomaly(/*check_nan=*/true); + double_backward_produce_nan(true); + } + } + double_backward_produce_nan(true); +} + +TEST(CustomAutogradTest, CustomFunctionReturnInputAsIsAndSavesIt) { + struct MyFunction : public Function { + static Variable forward( + AutogradContext* ctx, + Variable var1, + Variable var2) { + ctx->save_for_backward({var1, var2}); + return var1 * var2, var1; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + return {}; + } + }; + + Variable x = torch::randn({5, 5}, torch::requires_grad()); + Variable y = torch::randn({5, 5}, torch::requires_grad()); + MyFunction::apply(x, y); +} + +// Regression test: CppNode must override release_resources() to reset the +// AutogradContext. Otherwise the weak self-reference in ctx_.grad_fn_ keeps +// weakcount > 0 and leaks the CppNode. +TEST(CustomAutogradTest, CppNodeReleaseResourcesBreaksRefCycle) { + struct MyFunction : public Function { + static Variable forward( + AutogradContext* ctx, + Variable x, + Variable stashed) { + ctx->saved_data["stashed"] = stashed; + return x * 2; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + return {grad_output[0] * 2, Variable()}; + } + }; + + c10::weak_intrusive_ptr weak_node(c10::intrusive_ptr{}); + c10::weak_intrusive_ptr + weak_stashed( + c10::intrusive_ptr{}); + { + Variable x = torch::randn({5, 5}, torch::requires_grad()); + Variable stashed = torch::randn({5, 5}); + weak_stashed = + c10::weak_intrusive_ptr( + stashed.getIntrusivePtr()); + Variable y = MyFunction::apply(x, stashed); + weak_node = c10::weak_intrusive_ptr(y.grad_fn()); + } + // After the scope exits, the CppNode should be destroyed. If + // release_resources() is not overridden, ctx_.grad_fn_ (a weak reference to + // the node itself) keeps the weakcount > 1 so the node is leaked, which + // also leaks anything stashed in ctx_->saved_data. + EXPECT_TRUE(weak_node.expired()); + EXPECT_EQ(weak_node.weak_use_count(), 1); + EXPECT_TRUE(weak_stashed.expired()); +} + +TEST(CustomAutogradTest, CustomFunction) { + struct MyFunction : public Function { + static Variable forward( + AutogradContext* ctx, + Variable var1, + int mul, + Variable var2) { + ctx->saved_data["mul"] = mul; + ctx->save_for_backward({var1, var2}); + return var1 + mul * var2 + var1 * var2; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + int mul = ctx->saved_data["mul"].toInt(); + auto saved = ctx->get_saved_variables(); + auto var1 = saved[0]; + auto var2 = saved[1]; + variable_list output = { + grad_output[0] + grad_output[0] * var2, + Variable(), + grad_output[0] * mul + grad_output[0] * var1}; + return output; + } + }; + + Variable x = torch::randn({5, 5}, torch::requires_grad()); + Variable y = torch::randn({5, 5}, torch::requires_grad()); + auto res = MyFunction::apply(x, 2, y); + auto go = torch::ones({}, torch::requires_grad()); + res.sum().backward(go, false, true); + + ASSERT_VARIABLE_EQ(x.grad(), y + torch::ones({5, 5})); + ASSERT_VARIABLE_EQ(y.grad(), x + torch::ones({5, 5}) * 2); +} + +TEST(CustomAutogradTest, CustomFunctionWithTensorList) { + struct MyFunction : public Function { + static Variable forward(AutogradContext* ctx, at::TensorList tensors) { + torch::autograd::variable_list vars; + for (const at::Tensor& tensor : tensors) { + vars.push_back(tensor); + } + ctx->save_for_backward(vars); + return tensors[0] + tensors[1] + tensors[0] * tensors[1]; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + auto saved = ctx->get_saved_variables(); + auto var1 = saved[0]; + auto var2 = saved[1]; + variable_list output = { + grad_output[0] + grad_output[0] * var2, + grad_output[0] + grad_output[0] * var1}; + return output; + } + }; + + at::Tensor x = torch::randn({5, 5}, torch::requires_grad()); + at::Tensor y = torch::randn({5, 5}, torch::requires_grad()); + torch::autograd::variable_list variables = {x, y}; + at::TensorList tensors = variables; + auto res = MyFunction::apply(tensors); + auto go = torch::ones({}, torch::requires_grad()); + res.sum().backward(go, false, true); + + ASSERT_VARIABLE_EQ(x.grad(), y + torch::ones({5, 5})); + ASSERT_VARIABLE_EQ(y.grad(), x + torch::ones({5, 5})); +} + +TEST(CustomAutogradTest, GraphTaskTrimEdges) { + struct MyFunction : public Function { + static Variable forward( + AutogradContext* ctx, + Variable var1, + Variable var2, + int mul, + bool needs_input1_grad, + bool needs_input2_grad) { + // setup the expected should and should not compute idx + ctx->saved_data["needs_input1_grad"] = needs_input1_grad; + ctx->saved_data["needs_input2_grad"] = needs_input2_grad; + + ctx->saved_data["mul"] = mul; + ctx->save_for_backward({var1, var2}); + return var1 + mul * var2 + var1 * var2; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + // Test `needs_input_grad` method is working correctly. + // We have to test this within the backward function. + auto needs_input1_grad = ctx->saved_data["needs_input1_grad"].toBool(); + auto needs_input2_grad = ctx->saved_data["needs_input2_grad"].toBool(); + IndexRange var1_idx = {0, 1}; + IndexRange var2_idx = {1, 2}; + EXPECT_EQ(ctx->needs_input_grad(0), needs_input1_grad); + EXPECT_EQ(ctx->needs_input_grad(1), needs_input2_grad); + EXPECT_EQ(ctx->needs_input_grad({var1_idx}), needs_input1_grad); + EXPECT_EQ(ctx->needs_input_grad({var2_idx}), needs_input2_grad); + EXPECT_EQ( + ctx->needs_input_grad({var1_idx, var2_idx}), + needs_input1_grad || needs_input2_grad); + + // calculate gradients + int mul = ctx->saved_data["mul"].toInt(); + auto saved = ctx->get_saved_variables(); + auto var1 = saved[0]; + auto var2 = saved[1]; + + Variable grad_var1, grad_var2; + if (ctx->needs_input_grad(0)) { + grad_var1 = grad_output[0] + grad_output[0] * var2; + } + if (ctx->needs_input_grad(1)) { + grad_var2 = grad_output[0] * mul + grad_output[0] * var1; + } + variable_list output = { + grad_var1, + grad_var2, + Variable(), + Variable(), + Variable(), + }; + return output; + } + }; + + Variable x = torch::randn({5, 5}, torch::requires_grad()); + Variable y = torch::randn({5, 5}, torch::requires_grad()); + auto go = torch::ones_like(x); + Variable out; + + // grad_x + out = MyFunction::apply( + x, + y, + 2, + /* needs_input1_grad= */ true, + /* needs_input2_grad= */ false); + auto grad_x = torch::autograd::grad({out}, {x}, {go})[0]; + ASSERT_VARIABLE_EQ(grad_x, y + torch::ones({5, 5})); + + // grad_y + out = MyFunction::apply( + x, + y, + 2, + /* needs_input1_grad= */ false, + /* needs_input2_grad= */ true); + auto grad_y = torch::autograd::grad({out}, {y}, {go})[0]; + ASSERT_VARIABLE_EQ(grad_y, x + torch::ones({5, 5}) * 2); + + // grad_x and grad_y + out = MyFunction::apply( + x, + y, + 2, + /* needs_input1_grad= */ true, + /* needs_input2_grad= */ true); + auto grads = torch::autograd::grad({out}, {x, y}, {go}); + ASSERT_VARIABLE_EQ(grads[0], y + torch::ones({5, 5})); + ASSERT_VARIABLE_EQ(grads[1], x + torch::ones({5, 5}) * 2); +} + +TEST(CustomAutogradTest, FunctionReturnsInput) { + struct MyFunction : public Function { + static Variable forward(AutogradContext* ctx, Variable var1) { + return var1; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + return {grad_output[0] * 2}; + } + }; + + Variable x(torch::ones(1, torch::requires_grad())); + MyFunction::apply(x).backward(torch::ones(1), true, true); + ASSERT_VARIABLE_EQ(x.grad(), torch::full(1, 2.)); +} + +TEST(CustomAutogradTest, FunctionReturnsUndefined) { + struct MyFunction : public Function { + static Variable forward(AutogradContext* ctx, Variable var) { + return var * 2; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + at::Tensor undefined_tensor; + return {undefined_tensor}; + } + }; + + auto x = torch::ones(1, torch::requires_grad()); + + MyFunction::apply(x).backward(); + ASSERT_FALSE(x.grad().defined()); + + MyFunction::apply(x.pow(2)).backward(); + ASSERT_FALSE(x.grad().defined()); + + MyFunction::apply(x).sum().backward(); + ASSERT_FALSE(x.grad().defined()); + + ASSERT_FALSE(torch::autograd::grad( + {MyFunction::apply(x)}, {x}, {}, false, false, true)[0] + .defined()); +} + +TEST(CustomAutogradTest, MaterializeGrads) { + struct MyFunction : public Function { + static Variable forward(AutogradContext* ctx, Variable var) { + return var; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + EXPECT_VARIABLE_EQ(grad_output[0], torch::zeros(1)); + return grad_output; + } + }; + + auto x = torch::ones(1, torch::requires_grad()); + UndefinedGrad().apply({MyFunction::apply(x)})[0].backward(); +} + +TEST(CustomAutogradTest, DontMaterializeGrads) { + struct MyFunction : public Function { + static Variable forward(AutogradContext* ctx, Variable var) { + ctx->set_materialize_grads(false); + return var; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + EXPECT_FALSE(grad_output[0].defined()); + return grad_output; + } + }; + + auto x = torch::ones(1, torch::requires_grad()); + UndefinedGrad().apply({MyFunction::apply(x)})[0].backward(); +} + +TEST(CustomAutogradTest, NoGradCustomFunction) { + // Custom Function should respect grad mode + struct MyOp : public Function { + static Variable forward(AutogradContext* ctx, Variable x) { + return x + 1; + } + + static variable_list backward(AutogradContext* ctx, variable_list dy) { + return dy; + } + }; + + auto x = torch::ones({5, 5}, torch::requires_grad()); + { + at::NoGradGuard no_grad; + auto y = MyOp::apply(x); + ASSERT_FALSE(y.requires_grad()); + } +} + +TEST(CustomAutogradTest, MarkDirty) { + struct MyFunction : public Function { + static Variable forward(AutogradContext* ctx, Variable v) { + // Change the value inplace + auto v_data = v.data_ptr(); + v_data[0] = 2; + ctx->mark_dirty({v}); + return v; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + return {(grad_output[0] * 2.0)}; + } + }; + + // Clone here because modifying leaves inplace is not allowed + auto x = torch::randn({5, 5}, torch::requires_grad()).clone(); + auto version_before = x._version(); + auto out = MyFunction::apply(x); + auto version_after = x._version(); + ASSERT_TRUE(version_after >= (version_before + 1)); + out.sum().backward(); +} + +TEST(CustomAutogradTest, MarkNonDifferentiable) { + struct MyFunction : public Function { + static Variable forward(AutogradContext* ctx, Variable v) { + Variable output = v > 0; + ctx->mark_non_differentiable({output}); + return output; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + return {(grad_output[0] * 0.0)}; + } + }; + + auto x = torch::randn({5, 5}, torch::requires_grad()); + auto mask = MyFunction::apply(x); + ASSERT_FALSE(mask.requires_grad()); + auto y = x.masked_fill(mask, 0); + y.sum().backward(); +} + +TEST(CustomAutogradTest, MarkNonDifferentiableMixed) { + struct MyFunction : public Function { + static variable_list forward(AutogradContext* ctx, Variable input) { + Variable a = input + 1; + Variable b = input + 2; + ctx->mark_non_differentiable({a}); + return {a, b}; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + const Variable &grad_a = grad_output[0], &grad_b = grad_output[1]; + EXPECT_VARIABLE_EQ(grad_a, torch::zeros({5, 5})); + EXPECT_VARIABLE_EQ(grad_b, torch::ones({5, 5})); + return {grad_b}; + } + }; + + auto x = torch::randn({5, 5}, torch::requires_grad()); + auto out = MyFunction::apply(x); + + ASSERT_FALSE(out[0].requires_grad()); + ASSERT_TRUE(out[1].requires_grad()); + out[1].sum().backward(); + ASSERT_VARIABLE_EQ(x.grad(), torch::ones({5, 5})); +} + +TEST(CustomAutogradTest, MarkNonDifferentiableNone) { + struct MyFunction : public Function { + static Variable forward(AutogradContext* ctx, Variable input) { + auto output = input.clone(); + ctx->mark_non_differentiable({output}); + return output; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_outputs) { + return {}; + } + }; + + auto x = torch::randn({5, 5}, torch::requires_grad()); + auto r = MyFunction::apply(x * x); + (r * x).sum().backward(); +} + +TEST(CustomAutogradTest, ReturnLeafInplace) { + struct Inplace : public Function { + static variable_list forward(AutogradContext* ctx, Variable a, Variable b) { + ctx->mark_dirty({a}); + return {a.add_(b), b + 2}; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + return {grad_output[0], grad_output[0] + grad_output[1]}; + } + }; + + Variable x = torch::randn({5, 5}); + Variable y = torch::randn({5, 5}, torch::requires_grad()); + + auto out = Inplace::apply(x, y); + auto& q = out[0]; + ASSERT_TRUE(torch::equal(q, x)); + ASSERT_TRUE(q.requires_grad()); + q.sum().backward(); + ASSERT_VARIABLE_EQ(y.grad(), torch::ones({5, 5})); +} + +TEST(CustomAutogradTest, ReturnDuplicateInplace) { + struct DoubleInplace : public Function { + static variable_list forward(AutogradContext* ctx, Variable x) { + x.mul_(2); + ctx->mark_dirty({x}); + return {x, x}; + } + + static variable_list backward( + AutogradContext* ctsx, + variable_list grad_outputs) { + return {grad_outputs[0] * 2 + grad_outputs[1] * 2}; + } + }; + + auto x = torch::randn({5, 5}, torch::requires_grad()); + + ASSERT_THROWS_WITH( + DoubleInplace::apply(x), "leaf Variable that requires grad"); + // TODO ASSERT_THROWS_WITH(DoubleInplace::apply(x.clone()[0]), "only one + // output"); + + auto out = DoubleInplace::apply(x.clone()); + ASSERT_TRUE(torch::equal(out[0], out[1])); +} + +TEST(CustomAutogradTest, ReturnDuplicate) { + struct DoubleDuplicate : public Function { + static variable_list forward(AutogradContext* ctx, Variable x) { + auto output = x * 2; + return {output, output}; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_outputs) { + return {grad_outputs[0] * 2 + grad_outputs[1] * 2}; + } + }; + + auto x = torch::randn({5, 5}, torch::requires_grad()); + auto out = DoubleDuplicate::apply(x); + ASSERT_TRUE(torch::equal(out[0], out[1])); +} + +TEST(CustomAutogradTest, SaveEmptyForBackward) { + struct MyFunction : public Function { + static Variable forward(AutogradContext* ctx, Variable input) { + ctx->save_for_backward({Variable(), input, Variable()}); + return input * input; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + auto saved = ctx->get_saved_variables(); + EXPECT_FALSE(saved[0].defined()); + EXPECT_FALSE(saved[2].defined()); + return {saved[1] * 2 * grad_output[0]}; + } + }; + + Variable x = torch::randn({5, 5}, torch::requires_grad()); + auto y = MyFunction::apply(x); + y.sum().backward(); + ASSERT_VARIABLE_EQ(x.grad(), 2 * x); +} + +TEST(CustomAutogradTest, InvalidGradients) { + struct MyFunction : public Function { + static Variable forward(AutogradContext* ctx, Variable x) { + return x * 2; + } + + static variable_list backward( + AutogradContext* ctsx, + variable_list grad_outputs) { + return { + torch::randn(10, torch::dtype(torch::kFloat).requires_grad(true))}; + } + }; + + auto input1 = + torch::randn({5, 5}, torch::dtype(torch::kFloat).requires_grad(true)); + ASSERT_THROWS_WITH( + MyFunction::apply(input1).sum().backward(), "expected shape"); + auto input2 = + torch::randn(10, torch::dtype(torch::kDouble).requires_grad(true)); +} + +TEST(CustomAutogradTest, NoGradInput) { + struct MyFunction : public Function { + static Variable forward(AutogradContext*, Variable x) { + return x; + } + + static variable_list backward( + AutogradContext*, + variable_list grad_outputs) { + return grad_outputs; + } + }; + + Variable x = torch::randn({5, 5}, torch::requires_grad()); + Variable y; + { + at::NoGradGuard no_grad; + y = MyFunction::apply(x); + } + + ASSERT_TRUE(x.requires_grad()); + ASSERT_FALSE(y.grad_fn()); +} + +TEST(CustomAutogradTest, TooManyGrads) { + struct MyFunction : public Function { + static Variable forward(AutogradContext*, Variable input) { + return input; + } + + static variable_list backward(AutogradContext*, variable_list grad_output) { + grad_output.insert(grad_output.end(), {Variable(), Variable()}); + return grad_output; + } + }; +} + +TEST(CustomAutogradTest, DepNoGrad) { + struct F1 : public Function { + static variable_list forward(AutogradContext* ctx, Variable input) { + auto out = torch::randn(input.sizes()); + ctx->mark_non_differentiable({out}); + return {input, out}; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + return {grad_output[0]}; + } + }; + + struct F2 : public Function { + static Variable forward(AutogradContext*, Variable input, Variable ignore) { + return input; + } + + static variable_list backward(AutogradContext*, variable_list grad_output) { + return {grad_output[0], Variable()}; + } + }; + + auto x = torch::randn(5, torch::requires_grad()); + auto out = F1::apply(x); + Variable &a = out[0], &b = out[1]; + b = b + 1; // Separate F1 and F2 by another operation + ASSERT_TRUE(a.requires_grad()); + ASSERT_FALSE(b.requires_grad()); + + auto c = F2::apply(a, b); + c.backward(torch::ones(c.sizes()), false, false); + ASSERT_VARIABLE_EQ(x.grad(), torch::ones(x.sizes())); +} + +TEST(CustomAutogradTest, Reentrant) { + static Variable y_data = torch::randn({2, 2}); + struct Reenter : public Function { + static Variable forward(AutogradContext* ctx, Variable input) { + Variable output; + { + at::AutoGradMode enable_grad(true); + auto x = make_variable(input.tensor_data(), true); + auto y = make_variable(y_data.tensor_data(), true); + output = x * y; + + ctx->saved_data["x"] = x; + ctx->saved_data["y"] = y; + ctx->saved_data["output_var"] = output; + } + return output.detach(); + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + { + at::AutoGradMode enable_grad(true); + auto out = ctx->saved_data["output_var"].toTensor(); + out.sum().backward(); + } + return {ctx->saved_data["x"].toTensor().grad() * grad_output[0]}; + } + }; + + auto x = torch::randn({2, 2}, torch::requires_grad()); + auto out = Reenter::apply(x); + out.sum().backward(); + ASSERT_VARIABLE_EQ(x.grad(), y_data); +} + +// NOTE: If this fails for apparently unrelated reasons in TSAN be aware of +// the TSAN limit on mutex: https://github.com/google/sanitizers/issues/950 +TEST(CustomAutogradTest, DeepReentrant) { + struct DeepReenter : public Function { + static Variable forward(AutogradContext* ctx, Variable x) { + { + at::AutoGradMode enable_grad(true); + ctx->saved_data["x"] = make_variable(x.tensor_data(), true) - 1; + } + return ctx->saved_data["x"].toTensor().detach(); + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + if (!at::native::is_nonzero(ctx->saved_data["x"].toTensor())) { + return grad_output; + } + { + at::AutoGradMode enable_grad(true); + apply(ctx->saved_data["x"].toTensor())[0].sum().backward(); + return grad_output; + } + } + }; + + // This should not stack overflow + auto v = + torch::tensor({8193}, torch::dtype(torch::kFloat).requires_grad(true)); + DeepReenter::apply(v).sum().backward(); +} + +TEST(CustomAutogradTest, ReentrantPriority) { + static std::vector order; + + struct MyFunction : public Function { + static Variable forward(AutogradContext*, Variable x) { + return x; + } + + static variable_list backward(AutogradContext*, variable_list grad) { + order.push_back(0); + return grad; + } + }; + + struct Reenter : public Function { + static Variable forward(AutogradContext* ctx, Variable x) { + { + at::AutoGradMode enable_grad(true); + ctx->saved_data["x"] = make_variable(x.tensor_data(), true) - 1; + } + return ctx->saved_data["x"].toTensor().detach(); + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + order.push_back(1); + if (!at::native::is_nonzero(ctx->saved_data["x"].toTensor())) { + return grad_output; + } + { + at::AutoGradMode enable_grad(true); + apply(ctx->saved_data["x"].toTensor())[0].sum().backward(); + return grad_output; + } + } + }; + + auto a = MyFunction::apply( + torch::tensor({6}, torch::dtype(torch::kFloat).requires_grad(true))); + auto b = Reenter::apply( + torch::tensor({9}, torch::dtype(torch::kFloat).requires_grad(true))); + auto v = a * b; + v.backward(); + + // All the reentrant tasks should be prioritized over the MyFunction backward + // task. + ASSERT_EQ(order.size(), 10); + ASSERT_EQ(std::count(order.begin(), order.end(), 1), 9); + ASSERT_EQ(order.back(), 0); + // Clear static variable in case test get executed in a loop + order.clear(); +} + +TEST(CustomAutogradTest, Hooks) { + Variable x = torch::ones({5, 5}, torch::requires_grad()); + Variable y = torch::ones({5, 5}) * 4; + y.set_requires_grad(true); + + int counter = 0; + + std::function bw_hook( + [&counter](int inc, Variable grad) { counter += inc; }); + + Variable z = x * x + x * 2 + x * y + y; + x.register_hook([&bw_hook](Variable grad) { bw_hook(0, grad); }); + auto hook_1 = + z.register_hook([&bw_hook](Variable grad) { bw_hook(1, grad); }); + z.backward(torch::ones({5, 5}), true, true); + ASSERT_EQ(counter, 1); + + auto hook_2 = + z.register_hook([&bw_hook](Variable grad) { bw_hook(2, grad); }); + z.backward(torch::ones({5, 5}), true, true); + ASSERT_EQ(counter, 4); + + z.remove_hook(hook_2); + z.backward(torch::ones({5, 5}), true, true); + ASSERT_EQ(counter, 5); + + std::function bw_hook_modify( + [](Variable grad) { return grad.mul(2); }); + + z.remove_hook(hook_1); + z.register_hook(bw_hook_modify); + y.grad().zero_(); + z.backward(torch::ones({5, 5}), true, false); + ASSERT_VARIABLE_EQ(y.grad(), (x + 1) * 2); + + y.register_hook(bw_hook_modify); + y.grad().zero_(); + z.backward(torch::ones({5, 5}), false, false); + ASSERT_VARIABLE_EQ(y.grad(), (x + 1) * 4); + + ASSERT_THROWS_WITH(y.remove_hook(3), "Invalid index"); +} + +TEST(CustomAutogradTest, HooksInplace) { + auto a = torch::ones({5, 5}, torch::requires_grad()).clone(); + + int hook1_count = 0; + auto hook1 = ([&hook1_count](Variable grad) { + hook1_count++; + ASSERT_VARIABLE_EQ(grad, torch::ones({5, 5}) * 2); + }); + + int hook2_count = 0; + auto hook2 = ([&hook2_count](Variable grad) { + hook2_count++; + ASSERT_VARIABLE_EQ(grad, torch::ones({5, 5})); + }); + + a.register_hook(hook1); + a.mul_(2); + a.register_hook(hook2); + + auto out = (a + 1).sum(); + out.backward(); + + ASSERT_EQ(hook1_count, 1); + ASSERT_EQ(hook2_count, 1); +} + +TEST(CustomAutogradTest, HooksInplaceWithRetainsGrad) { + auto a = torch::ones({5, 5}, torch::requires_grad()).clone(); + + int hook1_count = 0; + auto hook1 = ([&hook1_count](Variable grad) { + hook1_count++; + ASSERT_VARIABLE_EQ(grad, torch::ones({5, 5}) * 2); + }); + + int hook2_count = 0; + auto hook2 = ([&hook2_count](Variable grad) { + hook2_count++; + ASSERT_VARIABLE_EQ(grad, torch::ones({5, 5}) * 2); + }); + + int hook3_count = 0; + auto hook3 = ([&hook3_count](Variable grad) { + hook3_count++; + ASSERT_VARIABLE_EQ(grad, torch::ones({5, 5})); + }); + + a.register_hook(hook1); + a.retain_grad(); + a.register_hook(hook2); + + a.mul_(2); + a.register_hook(hook3); + + auto out = (a + 1).sum(); + out.backward(); + + ASSERT_EQ(hook1_count, 1); + ASSERT_EQ(hook2_count, 1); + ASSERT_EQ(hook3_count, 1); + + ASSERT_TRUE(a.retains_grad()); + ASSERT_VARIABLE_EQ(a.grad(), torch::ones({5, 5})); +} + +TEST(CustomAutogradTest, HooksInplaceTwiceWithRetainsGrad) { + auto a = torch::ones({5, 5}, torch::requires_grad()).clone(); + + int hook1_count = 0; + auto hook1 = ([&hook1_count](Variable grad) { + hook1_count++; + ASSERT_VARIABLE_EQ(grad, torch::ones({5, 5}) * 4); + }); + + int hook2_count = 0; + auto hook2 = ([&hook2_count](Variable grad) { + hook2_count++; + ASSERT_VARIABLE_EQ(grad, torch::ones({5, 5}) * 4); + }); + + int hook3_count = 0; + auto hook3 = ([&hook3_count](Variable grad) { + hook3_count++; + ASSERT_VARIABLE_EQ(grad, torch::ones({5, 5})); + }); + + a.register_hook(hook1); + a.retain_grad(); + a.register_hook(hook2); + + a.mul_(2); + a.mul_(2); + a.register_hook(hook3); + + auto out = (a + 1).sum(); + out.backward(); + + ASSERT_EQ(hook1_count, 1); + ASSERT_EQ(hook2_count, 1); + ASSERT_EQ(hook3_count, 1); + + ASSERT_TRUE(a.retains_grad()); + ASSERT_VARIABLE_EQ(a.grad(), torch::ones({5, 5})); +} + +TEST(CustomAutogradTest, HookNone) { + struct NoneGradientFunction : public Function { + static variable_list forward(AutogradContext* ctx, Variable x, Variable y) { + return {x, y}; + } + + static variable_list backward(AutogradContext* ctx, variable_list grad) { + return {grad[0], Variable()}; + } + }; + + bool was_called = false; + + auto hook = ([&was_called](Variable grad) { + ASSERT_TRUE(grad.defined()); + was_called = true; + }); + + auto x = torch::randn({5, 5}, torch::requires_grad()); + auto y = torch::randn({5, 5}); + + auto out = NoneGradientFunction::apply(x, y); + Variable rx = x[0], ry = x[1]; + + rx.register_hook(hook); + ry.register_hook(hook); + (rx + ry).sum().backward(); + ASSERT_TRUE(was_called); +} + +TEST(CustomAutogradTest, BackwardWithInputs) { + Variable x = torch::randn({5, 5}, torch::requires_grad()); + Variable y = torch::randn({5, 5}, torch::requires_grad()); + Variable z = x * x + x * y + y * y; + Variable x_grad_expected = 2 * x + y; + Variable y_grad_expected = x + 2 * y; + + z.backward(torch::ones({5, 5}), false, false, {x}); + + ASSERT_VARIABLE_EQ(x.grad(), x_grad_expected); + ASSERT_FALSE(y.grad().defined()); +} + +TEST(CustomAutogradTest, BackwardWithEmptyInputs) { + Variable x = torch::randn({5, 5}, torch::requires_grad()); + Variable y = torch::randn({5, 5}, torch::requires_grad()); + Variable z = x * x + x * y + y * y; + Variable x_grad_expected = 2 * x + y; + Variable y_grad_expected = x + 2 * y; + ASSERT_THROWS_WITH( + z.backward(torch::ones({5, 5}), false, false, std::vector{}), + "cannot be empty"); +} + +TEST(CustomAutogradTest, BackwardWithNonLeafInputs) { + Variable x = torch::randn({5, 5}, torch::requires_grad()); + Variable y = torch::randn({5, 5}, torch::requires_grad()); + Variable z = x * x; + Variable w = y * z + x * y + y * y; + + Variable x_grad_expected = 2 * x * y + y; + Variable z_grad_expected = y; + + w.backward(torch::ones({5, 5}), false, false, std::vector{x, z}); + + ASSERT_VARIABLE_EQ(x.grad(), x_grad_expected); + ASSERT_VARIABLE_EQ(z.grad(), z_grad_expected); + ASSERT_FALSE(y.grad().defined()); +} + +TEST(CustomAutogradTest, BackwardWithCreateGraphWarns) { + c10::WarningUtils::WarnAlways guard(true); + + torch::Tensor x = torch::randn({5, 5}).set_requires_grad(true); + auto z = x * x; + { + WarningCapture warnings; + z.backward(torch::ones({5, 5}), std::nullopt, true); + ASSERT_TRUE( + warnings.str().find("Using backward() with create_graph=True") != + std::string::npos); + } + + { + WarningCapture warnings; + torch::autograd::backward({z}, {torch::ones({5, 5})}, std::nullopt, true); + ASSERT_TRUE( + warnings.str().find("Using backward() with create_graph=True") != + std::string::npos); + } +} + +/** + * Tests for AutogradNotImplementedFallback + * - Check that we created the NotImplemented kernel when inputs require grad + * but when no inputs require grad, we should not create this node + * - check_inplace logic + * - view ops + * - TODO: Tests for debug-only checks? Don't need for now because CI doesn't + * test non-NDEBUG builds. + * - tensorlist input and output + * - multiple outputs / non-tensor output + * - rebase_history vs set_history + */ +namespace { + +torch::Tensor inplace_op( + const torch::Tensor& self, + const torch::Tensor& other) { + return self.add_(other); +} + +std::tuple two_arg_inplace_op( + const torch::Tensor& self, + const torch::Tensor& other) { + other.add_(self); + self.add_(other); + return std::tuple(self, other); +} + +std::tuple two_pairs_of_view_op( + const torch::Tensor& self, + const torch::Tensor& other) { + // This is not allowed. We test below that this calling into the boxed kernel + // will raise an error + return std::tuple(self, other); +} + +std::tuple non_first_view_op( + const torch::Tensor& self, + const torch::Tensor& other) { + // This is not allowed. We test below that this calling into the boxed kernel + // will raise an error + return std::tuple(self.clone(), other); +} + +int64_t ret_single_non_tensor( + const torch::Tensor& self, + const torch::Tensor& other) { + return 12; +} + +torch::Tensor opt_op( + const torch::Tensor& self, + const std::optional& other) { + if (other.has_value()) { + return self + other.value(); + } else { + return self.clone(); + } +} + +torch::Tensor my_custom_op( + const torch::Tensor& self, + const torch::Tensor& other) { + return self + other; +} + +std::tuple ret_tuple_non_tensor( + const torch::Tensor& self, + const torch::Tensor& other) { + auto a = self - other; + auto b = self + other; + return std::tuple(a, b, 12); +} + +torch::Tensor view_op(const torch::Tensor& self) { + return self.alias(); +} + +std::vector ret_tensor_vector_view( + const torch::Tensor& self, + const torch::Tensor& other) { + return {self.alias(), self.alias()}; +} + +std::vector ret_tensor_vector( + const torch::Tensor& self, + const torch::Tensor& other) { + std::vector out; + out.push_back(self + other); + out.push_back(self - other); + return out; +} + +torch::Tensor tensorlist_op(const torch::Tensor& self, at::TensorList other) { + const auto& res = self.clone(); + for (const auto& t : other) { + res.add_(t); + } + return res; +} + +#define REGISTER_TEST_OP(name, schema, fn) \ + auto m = MAKE_TORCH_LIBRARY(_test); \ + m.def(schema); \ + auto m_autograd = MAKE_TORCH_LIBRARY_IMPL(_test, Autograd); \ + auto m_cpu = MAKE_TORCH_LIBRARY_IMPL(_test, CPU); \ + auto m_inplaceorview = MAKE_TORCH_LIBRARY_IMPL(_test, ADInplaceOrView); \ + m_cpu.impl(name, c10::DispatchKey::CPU, TORCH_FN(fn)); \ + m_autograd.impl( \ + name, c10::DispatchKey::Autograd, autogradNotImplementedFallback()); \ + m_inplaceorview.impl( \ + name, \ + c10::DispatchKey::ADInplaceOrView, \ + autogradNotImplementedInplaceOrViewFallback()); + +template +void assertBasicChecks(F op) { + auto a = torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true); + auto b = torch::tensor({1.}, {torch::kFloat32}); + auto c = torch::tensor({1.}, {torch::kFloat32}); + + // If any inputs require grad, + auto out1 = op(a, b); + ASSERT_THROWS_WITH(out1.backward(), "is not implemented"); + + // # Should not have grad_fn if none require grad + auto out2 = op(b, c); + ASSERT_THROWS_WITH( + out2.backward(), + "element 0 of tensors does not require grad and does not have a grad_fn"); + + // TODO: Forward AD Tests? +} + +} // namespace + +TEST(TestAutogradNotImplementedFallback, RetSingleNonTensor) { + REGISTER_TEST_OP( + "ret_single_non_tensor", + "_test::ret_single_non_tensor(Tensor self, Tensor other) -> int", + ret_single_non_tensor); + auto opHandle = c10::Dispatcher::singleton().findSchemaOrThrow( + "_test::ret_single_non_tensor", ""); + auto op = [&](const torch::Tensor& _1, const torch::Tensor& _2) { + return callOpUnboxed( + opHandle, _1, _2); + }; + + auto a = torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true); + auto b = torch::tensor({1.}, {torch::kFloat32}); + + ASSERT_EQ(op(a, b), ret_single_non_tensor(a, b)); +} + +TEST(TestAutogradNotImplementedFallback, InplaceOp) { + REGISTER_TEST_OP( + "inplace_op", + "_test::inplace_op(Tensor(a!) self, Tensor other) -> Tensor(a!)", + inplace_op); + auto opHandle = + c10::Dispatcher::singleton().findSchemaOrThrow("_test::inplace_op", ""); + auto op = [&](const torch::Tensor& _1, const torch::Tensor& _2) { + return callOpUnboxed< + torch::Tensor, + const torch::Tensor&, + const torch::Tensor&>(opHandle, _1, _2); + }; + + auto a = torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true); + auto b = torch::tensor({1.}, {torch::kFloat32}); + + // Check in-place + ASSERT_THROWS_WITH( + op(a, b), + "a leaf Variable that requires grad is being used in an in-place operation"); + op(b, a); + a = a.clone(); + b = b.clone(); + auto c = op(a, b); + ASSERT_TRUE(torch::allclose(c, inplace_op(a, b))); + + // Test in-place on view + auto base = + torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true).clone(); + auto view = base.view(-1); + auto t = torch::tensor({1.}, {torch::kFloat32}); + + torch::Tensor v_nograd; + { + c10::NoGradGuard guard; + v_nograd = base.view(-1); + op(v_nograd, t); + } + + ASSERT_THROWS_WITH(op(v_nograd, t), "A view was created in no_grad mode"); + ASSERT_EQ(op(view, t).unsafeGetTensorImpl(), view.unsafeGetTensorImpl()); + ASSERT_THAT( + op(view, t).grad_fn()->name(), ::testing::HasSubstr("AsStridedBackward")); +} + +TEST(TestAutogradNotImplementedFallback, DoubleInplaceOp) { + REGISTER_TEST_OP( + "two_arg_inplace_op", + "_test::two_arg_inplace_op(Tensor(a!) self, Tensor(b!) other) -> (Tensor(a!), Tensor(b!))", + two_arg_inplace_op); + auto opHandle = c10::Dispatcher::singleton().findSchemaOrThrow( + "_test::two_arg_inplace_op", ""); + auto op = [&](const torch::Tensor& _1, const torch::Tensor& _2) { + return callOpUnboxed< + std::tuple, + const torch::Tensor&, + const torch::Tensor&>(opHandle, _1, _2); + }; + auto a = torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true); + auto b = torch::tensor({1.}, {torch::kFloat32}); + + // Both are modified in-place! + ASSERT_THROWS_WITH( + op(a, b), + "a leaf Variable that requires grad is being used in an in-place operation"); + ASSERT_THROWS_WITH( + op(b, a), + "a leaf Variable that requires grad is being used in an in-place operation"); + + auto c = + torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true).clone(); + auto d = + torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true).clone(); + + auto saved_version_c = c._version(); + auto saved_version_d = d._version(); + op(c, d); + ASSERT_NE(c._version(), saved_version_c); + ASSERT_NE(d._version(), saved_version_d); +} + +TEST(TestAutogradNotImplementedFallback, OptOp) { + REGISTER_TEST_OP( + "opt_op", "_test::opt_op(Tensor self, Tensor? other) -> Tensor", opt_op); + auto opHandle = + c10::Dispatcher::singleton().findSchemaOrThrow("_test::opt_op", ""); + auto op = [&](const torch::Tensor& _1, + const std::optional& _2) { + return callOpUnboxed< + torch::Tensor, + const torch::Tensor&, + const std::optional&>(opHandle, _1, _2); + }; + + auto a = torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true); + auto b = torch::tensor({1.}, {torch::kFloat32}); + + ASSERT_TRUE(torch::allclose(op(a, b), opt_op(a, b))); + ASSERT_TRUE(torch::allclose(op(a, {}), opt_op(a, {}))); +} + +TEST(TestAutogradNotImplementedFallback, OutOfPlaceAddition) { + REGISTER_TEST_OP( + "my_custom_op", + "_test::my_custom_op(Tensor self, Tensor other) -> Tensor", + my_custom_op); + auto opHandle = + c10::Dispatcher::singleton().findSchemaOrThrow("_test::my_custom_op", ""); + auto op = [&](const torch::Tensor& _1, const torch::Tensor& _2) { + return callOpUnboxed< + torch::Tensor, + const torch::Tensor&, + const torch::Tensor&>(opHandle, _1, _2); + }; + + assertBasicChecks(op); +} + +TEST(TestAutogradNotImplementedFallback, RetTupleNonTensor) { + REGISTER_TEST_OP( + "ret_tuple_non_tensor", + "_test::ret_tuple_non_tensor(Tensor self, Tensor other) -> (Tensor, Tensor, int)", + ret_tuple_non_tensor); + auto opHandle = c10::Dispatcher::singleton().findSchemaOrThrow( + "_test::ret_tuple_non_tensor", ""); + auto op = [&](const torch::Tensor& _1, const torch::Tensor& _2) { + auto out = callOpUnboxed< + std::tuple, + const torch::Tensor&, + const torch::Tensor&>(opHandle, _1, _2); + auto [out0, out1, out2] = std::move(out); + return out0; + }; + + assertBasicChecks(op); +} + +TEST(TestAutogradNotImplementedFallback, ViewOp) { + REGISTER_TEST_OP( + "view_op", "_test::view_op(Tensor(a) self) -> Tensor(a)", view_op); + auto opHandle = + c10::Dispatcher::singleton().findSchemaOrThrow("_test::view_op", ""); + auto op = [&](const torch::Tensor& _1) { + return callOpUnboxed(opHandle, _1); + }; + auto b = torch::tensor({1.}, {torch::kFloat32}); + auto v = op(b); + ASSERT_TRUE(v.is_view()); + ASSERT_EQ(v._base().unsafeGetTensorImpl(), b.unsafeGetTensorImpl()); + + auto b1 = + torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true).clone(); + auto v1 = op(b1); + ASSERT_TRUE(v1.is_view()); + ASSERT_EQ(v1._base().unsafeGetTensorImpl(), b1.unsafeGetTensorImpl()); + + // Test inplace on view + auto t = torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true); + + // this works as we can properly replay the view given by the user + v1.add_(t); + b1.add_(t); +} + +TEST(TestAutogradNotImplementedFallback, RetTensorVectorView) { + REGISTER_TEST_OP( + "ret_tensor_vector_view", + "_test::ret_tensor_vector_view(Tensor(a) self, Tensor other) -> Tensor[](a)", + ret_tensor_vector_view); + auto opHandle = c10::Dispatcher::singleton().findSchemaOrThrow( + "_test::ret_tensor_vector_view", ""); + auto op = [&](const torch::Tensor& _1, const torch::Tensor& _2) { + return callOpUnboxed< + std::vector, + const torch::Tensor&, + const torch::Tensor&>(opHandle, _1, _2); + }; + auto a = torch::tensor({1.}, {torch::kFloat32}); + auto b = torch::tensor({1.}, {torch::kFloat32}); + auto out = op(a, b); + ASSERT_TRUE(out[0].is_view()); + ASSERT_EQ(out[0]._base().unsafeGetTensorImpl(), a.unsafeGetTensorImpl()); + ASSERT_TRUE(out[1].is_view()); + ASSERT_EQ(out[1]._base().unsafeGetTensorImpl(), a.unsafeGetTensorImpl()); +} + +TEST(TestAutogradNotImplementedFallback, DoubleViewOP) { + REGISTER_TEST_OP( + "two_pairs_of_view_op", + "_test::two_pairs_of_view_op(Tensor(a) self, Tensor(b) other) -> (Tensor(a), Tensor(b))", + two_pairs_of_view_op); + auto opHandle = c10::Dispatcher::singleton().findSchemaOrThrow( + "_test::two_pairs_of_view_op", ""); + auto op = [&](const torch::Tensor& _1, const torch::Tensor& _2) { + return callOpUnboxed< + std::tuple, + const torch::Tensor&, + const torch::Tensor&>(opHandle, _1, _2); + }; + auto a = torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true); + auto b = torch::tensor({1.}, {torch::kFloat32}); + ASSERT_THROWS_WITH( + op(a, b), + "Expected only a single output in the operator schema to have a non-write alias annotation"); +} + +TEST(TestAutogradNotImplementedFallback, NonFirstViewOP) { + REGISTER_TEST_OP( + "non_first_view_op", + "_test::non_first_view_op(Tensor self, Tensor(b) other) -> (Tensor, Tensor(b))", + non_first_view_op); + auto opHandle = c10::Dispatcher::singleton().findSchemaOrThrow( + "_test::non_first_view_op", ""); + auto op = [&](const torch::Tensor& _1, const torch::Tensor& _2) { + return callOpUnboxed< + std::tuple, + const torch::Tensor&, + const torch::Tensor&>(opHandle, _1, _2); + }; + auto a = torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true); + auto b = torch::tensor({1.}, {torch::kFloat32}); + ASSERT_THROWS_WITH( + op(a, b), "can only create view relationships between the first"); +} + +TEST(TestAutogradNotImplementedFallback, RetTensorVector) { + REGISTER_TEST_OP( + "ret_tensor_vector", + "_test::ret_tensor_vector(Tensor self, Tensor other) -> Tensor[]", + ret_tensor_vector); + auto opHandle = c10::Dispatcher::singleton().findSchemaOrThrow( + "_test::ret_tensor_vector", ""); + auto op = [&](const torch::Tensor& _1, const torch::Tensor& _2) { + return callOpUnboxed< + std::vector, + const torch::Tensor&, + const torch::Tensor&>(opHandle, _1, _2)[0]; + }; + assertBasicChecks(op); +} + +TEST(TestAutogradNotImplementedFallback, TensorlistOp) { + REGISTER_TEST_OP( + "tensorlist_op", + "_test::tensorlist_op(Tensor self, Tensor[] other) -> Tensor", + tensorlist_op); + auto opHandle = c10::Dispatcher::singleton().findSchemaOrThrow( + "_test::tensorlist_op", ""); + auto op = [&](torch::Tensor _1, at::TensorList _2) { + return callOpUnboxed( + opHandle, _1, _2); + }; + + auto a = torch::tensor({1.}, {torch::kFloat32}); + auto b = torch::tensor({1.}, {torch::kFloat32}); + auto c = torch::tensor({1.}, {torch::kFloat32}).set_requires_grad(true); + std::vector vec = {b, c}; + auto out = op(a, vec); + + ASSERT_THROWS_WITH( + torch::autograd::grad({out}, {vec[0]}), + "element 0 of the input tensors does not require grad"); + ASSERT_THROWS_WITH( + torch::autograd::grad({out}, {vec[1]}), "is not implemented"); + + ASSERT_TRUE(at::allclose(op(a, vec), tensorlist_op(a, vec))); +} + +static std::string test_format_error(const std::string& s) { + return s; +} + +TEST(TestAutogradUtils, ValidateOutputsReduce) { + auto input = torch::ones({}, {torch::kFloat32}); + auto grad = torch::ones({2, 3}, {torch::kFloat32}); + + std::vector> input_metadata; + input_metadata.emplace_back(InputMetadata(input)); + std::vector grads; + grads.emplace_back(grad); + + torch::autograd::validate_outputs(input_metadata, grads, test_format_error); + ASSERT_TRUE(at::allclose(grads[0], grad.sum())); +} + +TEST(TestAutogradUtils, ValidateOutputsBasic) { + auto input = torch::zeros({2, 3}, {torch::kFloat32}); + auto grad = torch::ones({2, 3}, {torch::kFloat32}); + + std::vector> input_metadata; + input_metadata.emplace_back(InputMetadata(input)); + std::vector grads; + grads.emplace_back(grad); + + torch::autograd::validate_outputs(input_metadata, grads, test_format_error); + ASSERT_TRUE(at::allclose(grad, torch::ones({2, 3}))); +} + +// TODO add these tests if needed +// test_once_differentiable +// test_sparse_backward +// test_save_output_nr +// test_free_deep_graph_pyfunction +// test_naughty_anomaly_access +// test_naughty_autograd-function_stashing_ctx +// test_custom_autograd_repeated_grad_grad +// test_return_leaf +// test_anomaly_detect_nan +// test_no_grad_copy diff --git a/western_ai/usa/pytorch/test/cpp/api/dataloader.cpp b/western_ai/usa/pytorch/test/cpp/api/dataloader.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c979cbacc7c9c78cecd52bf3b2cc0d75e376fa91 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/dataloader.cpp @@ -0,0 +1,2321 @@ +#include + +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace torch::data; // NOLINT + +const std::chrono::milliseconds kMillisecond(1); + +struct DummyDataset : datasets::Dataset { + explicit DummyDataset(size_t size = 100) : size_(size) {} + + int get(size_t index) override { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + return 1 + index; + } + std::optional size() const override { + return size_; + } + + size_t size_; +}; + +TEST(DataTest, DatasetCallsGetCorrectly) { + DummyDataset d; + std::vector batch = d.get_batch({0, 1, 2, 3, 4}); + std::vector expected = {1, 2, 3, 4, 5}; + ASSERT_EQ(batch, expected); +} + +TEST(DataTest, TransformCallsGetApplyCorrectly) { + struct T : transforms::Transform { + std::string apply(int input) override { + return std::to_string(input); + } + }; + + auto d = DummyDataset{}.map(T{}); + std::vector batch = d.get_batch({0, 1, 2, 3, 4}); + std::vector expected = {"1", "2", "3", "4", "5"}; + ASSERT_EQ(batch, expected); +} + +// dummy chunk data reader with 3 chunks and 35 examples in total. Each chunk +// contains 10, 5, 20 examples respectively. + +struct DummyChunkDataReader : public datasets::ChunkDataReader { + public: + using BatchType = datasets::ChunkDataReader::ChunkType; + using DataType = datasets::ChunkDataReader::ExampleType; + + /// Read an entire chunk. + BatchType read_chunk(size_t chunk_index) override { + BatchType batch_data; + int start_index = chunk_index == 0 + ? 0 + // NOLINTNEXTLINE(bugprone-fold-init-type) + : std::accumulate(chunk_sizes, chunk_sizes + chunk_index, 0); + + batch_data.resize(chunk_sizes[chunk_index]); + + std::iota(batch_data.begin(), batch_data.end(), start_index); + + return batch_data; + } + + size_t chunk_count() override { + return chunk_count_; + }; + + void reset() override {}; + + const static size_t chunk_count_ = 3; + // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-magic-numbers,cppcoreguidelines-avoid-c-arrays) + size_t chunk_sizes[chunk_count_] = {10, 5, 20}; +}; + +TEST(DataTest, ChunkDataSetWithInvalidInitParameter) { + DummyChunkDataReader data_reader; + samplers::SequentialSampler sampler(0); + + auto initialization_function = [&](size_t preloader_count, + size_t batch_size, + size_t cache_size, + size_t cross_chunk_shuffle_count = 1) { + datasets::SharedBatchDataset> + dataset = datasets::make_shared_dataset>( + data_reader, + sampler, + sampler, + datasets::ChunkDatasetOptions( + preloader_count, + batch_size, + cache_size, + cross_chunk_shuffle_count)); + }; + + ASSERT_THROWS_WITH( + initialization_function(0, 1, 1), + "Preloader count is 0. At least one preloader needs to be specified."); + + ASSERT_THROWS_WITH( + initialization_function(1, 0, 1), + "Batch size is 0. A positive batch size needs to be specified."); + + ASSERT_THROWS_WITH( + initialization_function(1, 1, 0), + "Cache size is 0. A positive cache size needs to be specified."); + + ASSERT_THROWS_WITH( + initialization_function(1, 10, 5), + "Cache size is less than batch size. Cache needs to be large enough to " + "hold at least one batch."); + ASSERT_THROWS_WITH( + initialization_function(1, 10, 20, 0), + "cross_chunk_shuffle_count needs to be greater than 0."); +} + +struct InfiniteStreamDataset + : datasets::StreamDataset> { + std::vector get_batch(size_t batch_size) override { + std::vector batch(batch_size); + for (auto& i : batch) { + i = counter++; + } + return batch; + } + + std::optional size() const override { + return std::nullopt; + } + + size_t counter = 0; +}; + +TEST(DataTest, InfiniteStreamDataset) { + const size_t kBatchSize = 13; + + auto dataset = InfiniteStreamDataset().map( + transforms::Lambda([](int x) { return x + 1; })); + + auto data_loader = torch::data::make_data_loader( + std::move(dataset), + samplers::StreamSampler(/*epoch_size=*/39), + kBatchSize); + + size_t batch_index = 0; + for (auto& batch : *data_loader) { + ASSERT_LT(batch_index, 3); + ASSERT_EQ(batch.size(), kBatchSize); + for (const auto j : c10::irange(kBatchSize)) { + ASSERT_EQ(batch.at(j), 1 + (batch_index * kBatchSize) + j); + } + batch_index += 1; + } + ASSERT_EQ(batch_index, 3); +} + +TEST(DataTest, NoSequencerIsIdentity) { + using namespace torch::data::detail::sequencers; // NOLINT + NoSequencer no_sequencer; + const auto value = no_sequencer.next([] { return 5; }).value(); + ASSERT_EQ(value, 5); +} + +TEST(DataTest, OrderedSequencerIsSetUpWell) { + using namespace torch::data::detail::sequencers; // NOLINT + struct S { + size_t sequence_number; + }; + const size_t kMaxJobs = 5; + OrderedSequencer sequencer(kMaxJobs); + ASSERT_EQ(sequencer.next_sequence_number_, 0); + ASSERT_EQ(sequencer.buffer_.size(), kMaxJobs); +} + +TEST(DataTest, OrderedSequencerReOrdersValues) { + using namespace torch::data::detail::sequencers; // NOLINT + struct S { + size_t sequence_number; + }; + const size_t kMaxJobs = 5; + OrderedSequencer sequencer(kMaxJobs); + + std::vector v = {0, 2, 4, 3, 1}; + size_t index = 0; + auto getter = [&v, &index]() { return S{v.at(index++)}; }; + + // Let's say the sequence number matches for the batch one, then it should + // return immediately. + const auto batch = sequencer.next(getter); + ASSERT_EQ(batch.value().sequence_number, 0); + ASSERT_EQ(index, 1); + + // Now it should call the getter until it gets the next value. + ASSERT_EQ(1, sequencer.next(getter).value().sequence_number); + ASSERT_EQ(index, 5); + + // The next three should come in order. + for (size_t i = 2; i <= 4; ++i) { + // New value doesn't matter. In fact, it shouldn't be accessed. + ASSERT_EQ(i, sequencer.next(getter).value().sequence_number); + // The index doesn't change. + ASSERT_EQ(index, 5); + } +} + +TEST(DataTest, BatchLambdaAppliesFunctionToBatch) { + using InputBatch = std::vector; + using OutputBatch = std::string; + DummyDataset d; + auto e = d.map(transforms::BatchLambda( + [](std::vector input) { + return std::to_string(std::accumulate(input.begin(), input.end(), 0)); + })); + ASSERT_EQ(e.get_batch({1, 2, 3, 4, 5}), std::string("20")); +} + +TEST(DataTest, LambdaAppliesFunctionToExample) { + auto d = DummyDataset().map(transforms::Lambda( + static_cast(std::to_string))); + std::vector expected = {"1", "2", "3", "4", "5"}; + ASSERT_EQ(d.get_batch({0, 1, 2, 3, 4}), expected); +} + +TEST(DataTest, CollateReducesBatch) { + auto d = + DummyDataset().map(transforms::Collate([](std::vector input) { + return std::accumulate(input.begin(), input.end(), 0); + })); + ASSERT_EQ(d.get_batch({1, 2, 3, 4, 5}), 20); +} + +TEST(DataTest, CollationReducesBatch) { + struct Summer : transforms::Collation { + int apply_batch(std::vector input) override { + return std::accumulate(input.begin(), input.end(), 0); + } + }; + auto d = DummyDataset().map(Summer{}); + ASSERT_EQ(d.get_batch({1, 2, 3, 4, 5}), 20); +} + +TEST(DataTest, SequentialSamplerReturnsIndicesInOrder) { + samplers::SequentialSampler sampler(10); + ASSERT_EQ(sampler.next(3).value(), std::vector({0, 1, 2})); + ASSERT_EQ(sampler.next(5).value(), std::vector({3, 4, 5, 6, 7})); + ASSERT_EQ(sampler.next(2).value(), std::vector({8, 9})); + ASSERT_FALSE(sampler.next(2).has_value()); +} + +TEST(DataTest, SequentialSamplerReturnsLessValuesForLastBatch) { + samplers::SequentialSampler sampler(5); + ASSERT_EQ(sampler.next(3).value(), std::vector({0, 1, 2})); + ASSERT_EQ(sampler.next(100).value(), std::vector({3, 4})); + ASSERT_FALSE(sampler.next(2).has_value()); +} + +TEST(DataTest, SequentialSamplerResetsWell) { + samplers::SequentialSampler sampler(5); + ASSERT_EQ(sampler.next(5).value(), std::vector({0, 1, 2, 3, 4})); + ASSERT_FALSE(sampler.next(2).has_value()); + sampler.reset(); + ASSERT_EQ(sampler.next(5).value(), std::vector({0, 1, 2, 3, 4})); + ASSERT_FALSE(sampler.next(2).has_value()); +} + +TEST(DataTest, SequentialSamplerResetsWithNewSizeWell) { + samplers::SequentialSampler sampler(5); + ASSERT_EQ(sampler.next(5).value(), std::vector({0, 1, 2, 3, 4})); + ASSERT_FALSE(sampler.next(2).has_value()); + sampler.reset(7); + ASSERT_EQ( + sampler.next(7).value(), std::vector({0, 1, 2, 3, 4, 5, 6})); + ASSERT_FALSE(sampler.next(2).has_value()); + sampler.reset(3); + ASSERT_EQ(sampler.next(3).value(), std::vector({0, 1, 2})); + ASSERT_FALSE(sampler.next(2).has_value()); +} + +TEST(DataTest, CanSaveAndLoadSequentialSampler) { + { + samplers::SequentialSampler a(10); + ASSERT_EQ(a.index(), 0); + std::stringstream stream; + torch::save(a, stream); + + samplers::SequentialSampler b(10); + torch::load(b, stream); + ASSERT_EQ(b.index(), 0); + } + { + samplers::SequentialSampler a(10); + a.next(3); + a.next(4); + ASSERT_EQ(a.index(), 7); + std::stringstream stream; + torch::save(a, stream); + + samplers::SequentialSampler b(10); + torch::load(b, stream); + ASSERT_EQ(b.index(), 7); + } +} + +TEST(DataTest, RandomSamplerReturnsIndicesInCorrectRange) { + samplers::RandomSampler sampler(10); + + std::vector indices = sampler.next(3).value(); + for (auto i : indices) { + ASSERT_GE(i, 0); + ASSERT_LT(i, 10); + } + + indices = sampler.next(5).value(); + for (auto i : indices) { + ASSERT_GE(i, 0); + ASSERT_LT(i, 10); + } + + indices = sampler.next(2).value(); + for (auto i : indices) { + ASSERT_GE(i, 0); + ASSERT_LT(i, 10); + } + + ASSERT_FALSE(sampler.next(10).has_value()); +} + +TEST(DataTest, RandomSamplerReturnsLessValuesForLastBatch) { + samplers::RandomSampler sampler(5); + ASSERT_EQ(sampler.next(3).value().size(), 3); + ASSERT_EQ(sampler.next(100).value().size(), 2); + ASSERT_FALSE(sampler.next(2).has_value()); +} + +TEST(DataTest, RandomSamplerResetsWell) { + samplers::RandomSampler sampler(5); + ASSERT_EQ(sampler.next(5).value().size(), 5); + ASSERT_FALSE(sampler.next(2).has_value()); + sampler.reset(); + ASSERT_EQ(sampler.next(5).value().size(), 5); + ASSERT_FALSE(sampler.next(2).has_value()); +} + +TEST(DataTest, RandomSamplerResetsWithNewSizeWell) { + samplers::RandomSampler sampler(5); + ASSERT_EQ(sampler.next(5).value().size(), 5); + ASSERT_FALSE(sampler.next(2).has_value()); + sampler.reset(7); + ASSERT_EQ(sampler.next(7).value().size(), 7); + ASSERT_FALSE(sampler.next(2).has_value()); + sampler.reset(3); + ASSERT_EQ(sampler.next(3).value().size(), 3); + ASSERT_FALSE(sampler.next(2).has_value()); +} + +TEST(DataTest, SavingAndLoadingRandomSamplerYieldsSameSequence) { + { + samplers::RandomSampler a(10); + + std::stringstream stream; + torch::save(a, stream); + + samplers::RandomSampler b(10); + torch::load(b, stream); + + ASSERT_EQ(a.next(10).value(), b.next(10).value()); + } + { + samplers::RandomSampler a(10); + a.next(3); + ASSERT_EQ(a.index(), 3); + + std::stringstream stream; + torch::save(a, stream); + + samplers::RandomSampler b(10); + torch::load(b, stream); + ASSERT_EQ(b.index(), 3); + + auto b_sequence = b.next(10).value(); + ASSERT_EQ(b_sequence.size(), 7); + ASSERT_EQ(a.next(10).value(), b_sequence); + } +} + +TEST(DataTest, StreamSamplerReturnsTheBatchSizeAndThenRemainder) { + samplers::StreamSampler sampler(/*epoch_size=*/100); + ASSERT_EQ(sampler.next(10).value(), 10); + ASSERT_EQ(sampler.next(2).value(), 2); + ASSERT_EQ(sampler.next(85).value(), 85); + ASSERT_EQ(sampler.next(123).value(), 3); + ASSERT_FALSE(sampler.next(1).has_value()); +} + +TEST(DataTest, StreamSamplerResetsWell) { + samplers::StreamSampler sampler(/*epoch_size=*/5); + ASSERT_EQ(sampler.next(5).value().size(), 5); + ASSERT_FALSE(sampler.next(2).has_value()); + sampler.reset(); + ASSERT_EQ(sampler.next(5).value().size(), 5); + ASSERT_FALSE(sampler.next(2).has_value()); +} + +TEST(DataTest, StreamSamplerResetsWithNewSizeWell) { + samplers::StreamSampler sampler(/*epoch_size=*/5); + ASSERT_EQ(sampler.next(5).value().size(), 5); + ASSERT_FALSE(sampler.next(2).has_value()); + sampler.reset(7); + ASSERT_EQ(sampler.next(7).value().size(), 7); + ASSERT_FALSE(sampler.next(2).has_value()); + sampler.reset(3); + ASSERT_EQ(sampler.next(3).value().size(), 3); + ASSERT_FALSE(sampler.next(2).has_value()); +} + +TEST(DataTest, TensorDatasetConstructsFromSingleTensor) { + datasets::TensorDataset dataset(torch::eye(5)); + ASSERT_TRUE( + torch::tensor({0, 0, 1, 0, 0}, torch::kFloat32).allclose(dataset.get(2))); +} + +TEST(DataTest, TensorDatasetConstructsFromInitializerListOfTensors) { + std::vector vector = torch::eye(5).chunk(5); + datasets::TensorDataset dataset(vector); + ASSERT_TRUE( + torch::tensor({0, 0, 1, 0, 0}, torch::kFloat32).allclose(dataset.get(2))); +} + +TEST(DataTest, StackTransformWorksForExample) { + struct D : public datasets::Dataset { + Example<> get(size_t index) override { + return {tensor[index], 1 + tensor[index]}; + } + + std::optional size() const override { + return tensor.size(0); + } + + torch::Tensor tensor{torch::eye(4)}; + }; + + auto d = D().map(transforms::Stack>()); + + Example<> batch = d.get_batch({0, 1}); + ASSERT_TRUE(batch.data.allclose(torch::eye(4).slice(/*dim=*/0, 0, 2))); + ASSERT_TRUE(batch.target.allclose(1 + torch::eye(4).slice(/*dim=*/0, 0, 2))); + + Example<> second = d.get_batch({2, 3}); + ASSERT_TRUE(second.data.allclose(torch::eye(4).slice(/*dim=*/0, 2, 4))); + ASSERT_TRUE(second.target.allclose(1 + torch::eye(4).slice(/*dim=*/0, 2, 4))); +} + +TEST(DataTest, StackTransformWorksForTensorExample) { + auto d = datasets::TensorDataset(torch::eye(4)) + .map(transforms::Stack()); + + TensorExample batch = d.get_batch({0, 1}); + ASSERT_TRUE(batch.data.allclose(torch::eye(4).slice(/*dim=*/0, 0, 2))); + + TensorExample second = d.get_batch({2, 3}); + ASSERT_TRUE(second.data.allclose(torch::eye(4).slice(/*dim=*/0, 2, 4))); +} + +// Template classes cannot be nested in functions. +template +struct T : transforms::TensorTransform { + torch::Tensor operator()(torch::Tensor input) override { + return input * 2; + } +}; + +struct TensorStringDataset + : datasets:: + Dataset> { + Example get(size_t index) override { + return {torch::tensor(static_cast(index)), std::to_string(index)}; + } + + std::optional size() const override { + return 100; + } +}; + +TEST(DataTest, TensorTransformWorksForAnyTargetType) { + auto d = TensorStringDataset().map(T{}); + std::vector> batch = d.get_batch({1, 2}); + + ASSERT_EQ(batch.size(), 2); + ASSERT_TRUE(batch[0].data.allclose(torch::tensor(2.0))); + ASSERT_EQ(batch[0].target, "1"); + + ASSERT_TRUE(batch[1].data.allclose(torch::tensor(4.0))); + ASSERT_EQ(batch[1].target, "2"); +} + +TEST(DataTest, TensorLambdaWorksforAnyTargetType) { + auto d = TensorStringDataset().map(transforms::TensorLambda( + [](torch::Tensor input) { return input * 2; })); + std::vector> batch = d.get_batch({1, 2}); + + ASSERT_EQ(batch.size(), 2); + ASSERT_TRUE(batch[0].data.allclose(torch::tensor(2.0))); + ASSERT_EQ(batch[0].target, "1"); + + ASSERT_TRUE(batch[1].data.allclose(torch::tensor(4.0))); + ASSERT_EQ(batch[1].target, "2"); +} + +struct DummyTensorDataset + : datasets::Dataset> { + Example get(size_t index) override { + const auto channels = static_cast(index); + torch::Tensor tensor = + (channels > 0) ? torch::ones({channels, 4, 4}) : torch::ones({4, 4}); + return {tensor, static_cast(channels)}; + } + + std::optional size() const override { + return 100; + } +}; + +TEST(DataTest, NormalizeTransform) { + auto dataset = DummyTensorDataset().map(transforms::Normalize(0.5, 0.1)); + + // Works for zero (one implicit) channels + std::vector> output = dataset.get_batch(0); + ASSERT_EQ(output.size(), 1); + // (1 - 0.5) / 0.1 = 5 + ASSERT_TRUE(output[0].data.allclose(torch::ones({4, 4}) * 5)) + << output[0].data; + + // Works for one explicit channel + output = dataset.get_batch(1); + ASSERT_EQ(output.size(), 1); + ASSERT_EQ(output[0].data.size(0), 1); + ASSERT_TRUE(output[0].data.allclose(torch::ones({1, 4, 4}) * 5)) + << output[0].data; + + // Works for two channels with different moments + dataset = DummyTensorDataset().map( + transforms::Normalize({0.5, 1.5}, {0.1, 0.2})); + output = dataset.get_batch(2); + ASSERT_EQ(output.size(), 1); + ASSERT_EQ(output[0].data.size(0), 2); + ASSERT_TRUE(output[0] + .data.slice(/*dim=*/0, /*start=*/0, /*end=*/1) + .allclose(torch::ones({1, 4, 4}) * 5)) + << output[0].data; + ASSERT_TRUE(output[0] + .data.slice(/*dim=*/0, /*start=*/1) + .allclose(torch::ones({1, 4, 4}) * -2.5)) + << output[0].data; + + // Works for three channels with one moment value + dataset = DummyTensorDataset().map(transforms::Normalize(1.5, 0.2)); + output = dataset.get_batch(3); + ASSERT_EQ(output.size(), 1); + ASSERT_EQ(output[0].data.size(0), 3); + ASSERT_TRUE(output[0].data.allclose(torch::ones({3, 4, 4}) * -2.5)) + << output[0].data; + + // Works for three channels with different moments + dataset = DummyTensorDataset().map( + transforms::Normalize({0.5, 1.5, -1.5}, {0.1, 0.2, 0.2})); + output = dataset.get_batch(3); + ASSERT_EQ(output.size(), 1); + ASSERT_EQ(output[0].data.size(0), 3); + ASSERT_TRUE(output[0] + .data.slice(/*dim=*/0, /*start=*/0, /*end=*/1) + .allclose(torch::ones({1, 4, 4}) * 5)) + << output[0].data; + ASSERT_TRUE(output[0] + .data.slice(/*dim=*/0, /*start=*/1, /*end=*/2) + .allclose(torch::ones({1, 4, 4}) * -2.5)) + << output[0].data; + ASSERT_TRUE(output[0] + .data.slice(/*dim=*/0, /*start=*/2) + .allclose(torch::ones({1, 4, 4}) * 12.5)) + << output[0].data; +} + +struct UnCopyableDataset : public datasets::Dataset { + UnCopyableDataset() = default; + + UnCopyableDataset(const UnCopyableDataset&) = delete; + UnCopyableDataset& operator=(const UnCopyableDataset&) = delete; + + UnCopyableDataset(UnCopyableDataset&&) = default; + UnCopyableDataset& operator=(UnCopyableDataset&&) = default; + + ~UnCopyableDataset() override = default; + + Example<> get(size_t index) override { + return { + torch::tensor({static_cast(index)}), + torch::tensor({static_cast(index)})}; + } + + std::optional size() const override { + return 100; + } +}; + +TEST(DataTest, MapDoesNotCopy) { + auto dataset = UnCopyableDataset() + .map(transforms::TensorLambda<>( + [](torch::Tensor tensor) { return tensor + 1; })) + .map(transforms::TensorLambda<>( + [](torch::Tensor tensor) { return tensor + 2; })) + .map(transforms::TensorLambda<>( + [](torch::Tensor tensor) { return tensor + 3; })); + + auto data = dataset.get_batch(1).at(0).data; + ASSERT_EQ(data.numel(), 1); + ASSERT_EQ(data[0].item(), 7); +} + +TEST(DataTest, QueuePushAndPopFromSameThread) { + torch::data::detail::Queue queue; + queue.push(1); + queue.push(2); + ASSERT_EQ(queue.pop(), 1); + ASSERT_EQ(queue.pop(), 2); +} + +TEST(DataTest, QueuePopWithTimeoutThrowsUponTimeout) { + torch::data::detail::Queue queue; + ASSERT_THROWS_WITH( + queue.pop(10 * kMillisecond), + "Timeout in DataLoader queue while waiting for next batch " + "(timeout was 10 ms)"); +} + +TEST(DataTest, QueuePushAndPopFromDifferentThreads) { + using torch::data::detail::Queue; + + // First test: push batch and the pop in thread. + { + Queue queue; + queue.push(1); + auto future = + std::async(std::launch::async, [&queue] { return queue.pop(); }); + ASSERT_EQ(future.get(), 1); + } + + // Second test: attempt to pop batch (and block), then push. + { + Queue queue; + std::thread thread([&queue] { + std::this_thread::sleep_for(20 * kMillisecond); + queue.push(123); + }); + ASSERT_EQ(queue.pop(), 123); + thread.join(); + } +} + +TEST(DataTest, QueueClearEmptiesTheQueue) { + torch::data::detail::Queue queue; + queue.push(1); + queue.push(2); + queue.push(3); + ASSERT_EQ(queue.clear(), 3); + ASSERT_THROWS_WITH(queue.pop(1 * kMillisecond), "Timeout"); +} + +TEST(DataTest, DataShuttleCanPushAndPopJob) { + torch::data::detail::DataShuttle shuttle; + shuttle.push_job(1); + shuttle.push_job(2); + ASSERT_EQ(shuttle.pop_job(), 1); + ASSERT_EQ(shuttle.pop_job(), 2); +} + +TEST(DataTest, DataShuttleCanPushAndPopResult) { + torch::data::detail::DataShuttle shuttle; + // pop_result() will only attempt to pop if there was a push_job() batch. + shuttle.push_job(1); + shuttle.push_job(2); + + shuttle.pop_job(); + shuttle.push_result(1); + ASSERT_EQ(shuttle.pop_result().value(), 1); + + shuttle.pop_job(); + shuttle.push_result(2); + ASSERT_EQ(shuttle.pop_result().value(), 2); +} + +TEST(DataTest, DataShuttlePopResultReturnsNulloptWhenNoJobsInFlight) { + torch::data::detail::DataShuttle shuttle; + ASSERT_FALSE(shuttle.pop_result().has_value()); + shuttle.push_job(1); + shuttle.pop_job(); + shuttle.push_result(1); + ASSERT_EQ(shuttle.pop_result().value(), 1); + ASSERT_FALSE(shuttle.pop_result().has_value()); + ASSERT_FALSE(shuttle.pop_result().has_value()); +} + +TEST(DataTest, DataShuttleDrainMeansPopResultReturnsNullopt) { + torch::data::detail::DataShuttle shuttle; + shuttle.push_job(1); + shuttle.push_result(1); + shuttle.drain(); + ASSERT_FALSE(shuttle.pop_result().has_value()); +} + +TEST(DataTest, DataShuttlePopResultTimesOut) { + torch::data::detail::DataShuttle shuttle; + shuttle.push_job(1); + ASSERT_THROWS_WITH(shuttle.pop_result(10 * kMillisecond), "Timeout"); +} + +struct UncopyableDataset : datasets::Dataset { + UncopyableDataset(const std::string& /* unused */) {} + + UncopyableDataset(UncopyableDataset&&) = default; + UncopyableDataset& operator=(UncopyableDataset&&) = default; + + UncopyableDataset(const UncopyableDataset&) = delete; + UncopyableDataset& operator=(const UncopyableDataset&) = delete; + + int get(size_t index) override { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + return 1 + index; + } + std::optional size() const override { + return 100; + } +}; + +TEST(DataTest, SharedBatchDatasetReallyIsShared) { + // This test will only compile if we really are not making any copies. + // There is otherwise no logic to test and because it is not deterministic + // how many and when worker threads access the shareddataset, we don't have + // any additional assertions here. + + auto shared_dataset = + torch::data::datasets::make_shared_dataset( + "uncopyable"); + + auto data_loader = torch::data::make_data_loader( + shared_dataset, torch::data::DataLoaderOptions().workers(3)); + + for (auto batch : *data_loader) { + /* exhaust */ + } +} + +TEST(DataTest, SharedBatchDatasetDoesNotIncurCopyWhenPassedDatasetObject) { + // This will not compile if a copy is made. + auto shared_dataset = + torch::data::datasets::make_shared_dataset( + UncopyableDataset("uncopyable")); + ASSERT_EQ(shared_dataset.size().value(), 100); +} + +struct TestIndex : public torch::data::samplers::CustomBatchRequest { + explicit TestIndex(size_t offset, std::vector index) + : offset(offset), index(std::move(index)) {} + size_t size() const override { + return index.size(); + } + size_t offset; + std::vector index; +}; + +struct TestIndexDataset + : datasets::BatchDataset, TestIndex> { + explicit TestIndexDataset(size_t size) : data(size) { + std::iota(data.begin(), data.end(), size_t(0)); + } + std::vector get_batch(TestIndex index) override { + std::vector batch; + for (auto i : index.index) { + batch.push_back(index.offset + data.at(i)); + } + return batch; + } + std::optional size() const override { + return data.size(); + } + std::vector data; +}; + +struct TestIndexSampler : public samplers::Sampler { + explicit TestIndexSampler(size_t size) : size_(size) {} + void reset(std::optional new_size = std::nullopt) override {} + std::optional next(size_t batch_size) override { + if (index_ >= size_) { + return std::nullopt; + } + std::vector indices(batch_size); + std::iota(indices.begin(), indices.end(), size_t(0)); + index_ += batch_size; + return TestIndex(batch_size, std::move(indices)); + } + void save(torch::serialize::OutputArchive& archive) const override {} + void load(torch::serialize::InputArchive& archive) override {} + size_t index_ = 0; + size_t size_; +}; + +TEST(DataTest, CanUseCustomTypeAsIndexType) { + const int kBatchSize = 10; + auto data_loader = torch::data::make_data_loader( + TestIndexDataset(23), TestIndexSampler(23), kBatchSize); + + for (auto batch : *data_loader) { + for (const auto j : c10::irange(kBatchSize)) { + ASSERT_EQ(batch.at(j), 10 + j); + } + } +} + +TEST(DataTest, DistributedRandomSamplerSingleReplicaProduceCorrectSamples) { + size_t sample_count = 10; + samplers::DistributedRandomSampler drs(sample_count); + + std::vector res; + std::optional> idx; + while ((idx = drs.next(3)).has_value()) { + res.insert(std::end(res), std::begin(*idx), std::end(*idx)); + } + + ASSERT_EQ(res.size(), sample_count); + + std::sort(res.begin(), res.end()); + for (const auto i : c10::irange(res.size())) { + ASSERT_EQ(res[i], i); + } +} + +TEST(DataTest, DistributedRandomSamplerMultiReplicaProduceCorrectSamples) { + size_t sample_count = 10; + size_t num_replicas = 3; + + auto test_function = [&](bool allow_duplicates, + size_t local_sample_count, + std::vector& output, + size_t batch_size) { + std::vector> samplers; + + for (const auto i : c10::irange(num_replicas)) { + samplers.emplace_back( + std::make_unique( + sample_count, num_replicas, i, allow_duplicates)); + } + + std::vector res; + for (const auto i : c10::irange(num_replicas)) { + (*samplers[i]).reset(); + std::optional> idx; + while ((idx = (*samplers[i]).next(batch_size)).has_value()) { + res.insert(std::end(res), std::begin(*idx), std::end(*idx)); + } + ASSERT_EQ(res.size(), local_sample_count * (i + 1)); + } + std::sort(res.begin(), res.end()); + ASSERT_EQ(res, output); + }; + + for (size_t batch_size = 1; batch_size <= 3; ++batch_size) { + size_t local_sample_count = + static_cast(std::ceil(sample_count * 1.0 / num_replicas)); + std::vector output1{0, 0, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + test_function(true, local_sample_count, output1, batch_size); + + local_sample_count = + static_cast(std::floor(sample_count * 1.0 / num_replicas)); + std::vector output2{0, 1, 2, 3, 4, 5, 6, 7, 8}; + test_function(false, local_sample_count, output2, batch_size); + } +} + +TEST(DataTest, CanSaveAndLoadDistributedRandomSampler) { + { + samplers::DistributedRandomSampler a(10); + ASSERT_EQ(a.index(), 0); + std::stringstream stream; + torch::save(a, stream); + + samplers::DistributedRandomSampler b(10); + torch::load(b, stream); + ASSERT_EQ(b.index(), 0); + } + { + samplers::DistributedRandomSampler a(10); + a.next(3); + a.next(4); + ASSERT_EQ(a.index(), 7); + std::stringstream stream; + torch::save(a, stream); + + samplers::DistributedRandomSampler b(10); + torch::load(b, stream); + ASSERT_EQ(b.index(), 7); + } + { + samplers::DistributedRandomSampler a(10); + a.set_epoch(3); + std::stringstream stream; + torch::save(a, stream); + + samplers::DistributedRandomSampler b(10); + torch::load(b, stream); + ASSERT_EQ(b.epoch(), 3); + } +} + +TEST(DataTest, DistributedSequentialSamplerSingleReplicaProduceCorrectSamples) { + size_t sample_count = 10; + size_t batch_size = 3; + samplers::DistributedSequentialSampler dss(sample_count); + + std::vector res; + std::optional> idx; + while ((idx = dss.next(batch_size)).has_value()) { + res.insert(std::end(res), std::begin(*idx), std::end(*idx)); + } + + ASSERT_EQ(res.size(), sample_count); + + std::sort(res.begin(), res.end()); + for (const auto i : c10::irange(res.size())) { + ASSERT_EQ(res[i], i); + } +} + +TEST(DataTest, DistributedSequentialSamplerMultiReplicaProduceCorrectSamples) { + size_t sample_count = 10; + size_t num_replicas = 3; + + auto test_function = [&](bool allow_duplicates, + size_t local_sample_count, + std::vector& output, + size_t batch_size) { + std::vector> + samplers; + + for (const auto i : c10::irange(num_replicas)) { + samplers.emplace_back( + std::make_unique( + sample_count, num_replicas, i, allow_duplicates)); + } + + std::vector res; + for (const auto i : c10::irange(num_replicas)) { + (*samplers[i]).reset(); + std::optional> idx; + while ((idx = (*samplers[i]).next(batch_size)).has_value()) { + res.insert(std::end(res), std::begin(*idx), std::end(*idx)); + } + ASSERT_EQ(res.size(), local_sample_count * (i + 1)); + } + std::sort(res.begin(), res.end()); + ASSERT_EQ(res, output); + }; + + for (size_t batch_size = 1; batch_size <= 3; ++batch_size) { + size_t local_sample_count = + static_cast(std::ceil(sample_count * 1.0 / num_replicas)); + std::vector output1{0, 0, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + test_function(true, local_sample_count, output1, batch_size); + + local_sample_count = + static_cast(std::floor(sample_count * 1.0 / num_replicas)); + std::vector output2{0, 1, 2, 3, 4, 5, 6, 7, 8}; + test_function(false, local_sample_count, output2, batch_size); + } +} + +TEST(DataTest, CanSaveAndLoadDistributedSequentialSampler) { + { + samplers::DistributedSequentialSampler a(10); + ASSERT_EQ(a.index(), 0); + std::stringstream stream; + torch::save(a, stream); + + samplers::DistributedSequentialSampler b(10); + torch::load(b, stream); + ASSERT_EQ(b.index(), 0); + } + { + samplers::DistributedSequentialSampler a(10); + a.next(3); + a.next(4); + ASSERT_EQ(a.index(), 7); + std::stringstream stream; + torch::save(a, stream); + + samplers::DistributedSequentialSampler b(10); + torch::load(b, stream); + ASSERT_EQ(b.index(), 7); + } +} + +TEST(DataLoaderTest, DataLoaderOptionsDefaultAsExpected) { + DataLoaderOptions partial_options; + FullDataLoaderOptions full_options(partial_options); + ASSERT_EQ(full_options.batch_size, 1); + ASSERT_FALSE(full_options.drop_last); + ASSERT_EQ(full_options.workers, 0); + ASSERT_EQ(full_options.max_jobs, 0); + ASSERT_FALSE(full_options.timeout.has_value()); + ASSERT_TRUE(full_options.enforce_ordering); +} + +TEST(DataLoaderTest, DataLoaderOptionsCoalesceOptionalValues) { + auto partial_options = DataLoaderOptions(32).workers(10); + FullDataLoaderOptions full_options(partial_options); + ASSERT_EQ(full_options.batch_size, 32); + ASSERT_EQ(full_options.max_jobs, 2 * 10); +} + +TEST(DataLoaderTest, MakeDataLoaderDefaultsAsExpected) { + auto data_loader = torch::data::make_data_loader( + DummyDataset().map(transforms::Lambda([](int x) { return x + 1; }))); + ASSERT_EQ(data_loader->options().batch_size, 1); +} + +struct UnsizedDataset : public datasets::Dataset { + torch::data::Example<> get(size_t i) override { + return {torch::ones(i), torch::ones(i)}; + } + std::optional size() const noexcept override { + return std::nullopt; + } +}; + +TEST( + DataLoaderTest, + MakeDataLoaderThrowsWhenConstructingSamplerWithUnsizedDataset) { + ASSERT_THROWS_WITH( + torch::data::make_data_loader(UnsizedDataset{}), + "Expected the dataset to be sized in order to construct the Sampler"); +} + +TEST(DataLoaderTest, IteratorsCompareEqualToThemselves) { + auto data_loader = torch::data::make_data_loader(DummyDataset(), 32); + auto begin = data_loader->begin(); + ASSERT_EQ(begin, begin); + auto end = data_loader->end(); + ASSERT_EQ(end, end); +} + +TEST(DataLoaderTest, ValidIteratorsCompareUnequalToEachOther) { + auto data_loader = torch::data::make_data_loader(DummyDataset(), 32); + auto i = data_loader->begin(); + auto j = data_loader->begin(); + ASSERT_NE(i, j); + ++j; + ASSERT_NE(i, j); +} + +TEST(DataLoaderTest, SentinelIteratorsCompareEqualToEachOther) { + auto data_loader = torch::data::make_data_loader(DummyDataset(), 32); + auto i = data_loader->end(); + auto j = data_loader->end(); + ASSERT_EQ(i, j); +} + +TEST(DataLoaderTest, IteratorsCompareEqualToSentinelWhenExhausted) { + DummyDataset dataset; + auto data_loader = + torch::data::make_data_loader(dataset, dataset.size().value() / 4); + auto i = data_loader->begin(); + auto end = data_loader->end(); + ASSERT_NE(i, end); + ++i; + ASSERT_NE(i, end); + ++i; + ASSERT_NE(i, end); + ++i; + ASSERT_NE(i, end); + ++i; + ASSERT_EQ(i, end); +} + +TEST(DataLoaderTest, IteratorsShareState) { + DummyDataset dataset; + auto data_loader = + torch::data::make_data_loader(dataset, dataset.size().value() / 2); + auto i = data_loader->begin(); + auto j = i; + auto end = data_loader->end(); + ASSERT_NE(i, end); + ASSERT_NE(j, end); + ++i; + ASSERT_NE(i, end); + ASSERT_NE(j, end); + ++j; + ASSERT_EQ(i, end); + ASSERT_EQ(j, end); +} + +TEST(DataLoaderTest, CanDereferenceIteratorMultipleTimes) { + DummyDataset dataset; + auto data_loader = + torch::data::make_data_loader( + dataset, + // NOLINTNEXTLINE(bugprone-argument-comment) + /*batch_size=*/1); + auto iterator = data_loader->begin(); + std::vector expected = {1}; + ASSERT_EQ(*iterator, expected); + ASSERT_EQ(*iterator, expected); + ++iterator; + expected[0] = 2; + ASSERT_EQ(*iterator, expected); + ASSERT_EQ(*iterator, expected); + ++iterator; + expected[0] = 3; + ASSERT_EQ(*iterator, expected); + ASSERT_EQ(*iterator, expected); +} + +TEST(DataLoaderTest, CanUseIteratorAlgorithms) { + struct D : datasets::BatchDataset { + int get_batch(torch::ArrayRef indices) override { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + return 1 + indices.front(); + } + std::optional size() const override { + return 10; + } + }; + + D dataset; + auto data_loader = + torch::data::make_data_loader( + dataset, 1); + std::vector values; + std::copy( + data_loader->begin(), data_loader->end(), std::back_inserter(values)); + std::vector expected(dataset.size().value()); + std::iota(expected.begin(), expected.end(), size_t(1)); + ASSERT_EQ(values, expected); +} + +TEST(DataLoaderTest, CallingBeginWhileOtherIteratorIsInFlightThrows) { + DummyDataset dataset; + auto data_loader = + torch::data::make_data_loader(dataset, DataLoaderOptions(1).workers(2)); + auto i = data_loader->begin(); + ASSERT_THROWS_WITH( + data_loader->begin(), + "Attempted to get a new DataLoader iterator " + "while another iterator is not yet exhausted"); +} + +TEST(DataLoaderTest, IncrementingExhaustedValidIteratorThrows) { + DummyDataset dataset; + auto data_loader = + torch::data::make_data_loader(dataset, dataset.size().value()); + auto i = data_loader->begin(); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_NO_THROW(++i); + ASSERT_THROWS_WITH(++i, "Attempted to increment iterator past the end"); +} + +TEST(DataLoaderTest, DereferencingExhaustedValidIteratorThrows) { + DummyDataset dataset; + auto data_loader = + torch::data::make_data_loader(dataset, dataset.size().value()); + auto i = data_loader->begin(); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_NO_THROW(++i); + ASSERT_THROWS_WITH( + *i, "Attempted to dereference iterator that was past the end"); +} + +TEST(DataLoaderTest, IncrementingSentinelIteratorThrows) { + DummyDataset dataset; + auto data_loader = + torch::data::make_data_loader(dataset, dataset.size().value()); + auto i = data_loader->end(); + ASSERT_THROWS_WITH( + ++i, + "Incrementing the DataLoader's past-the-end iterator is not allowed"); +} + +TEST(DataLoaderTest, DereferencingSentinelIteratorThrows) { + DummyDataset dataset; + auto data_loader = + torch::data::make_data_loader(dataset, dataset.size().value()); + auto i = data_loader->end(); + ASSERT_THROWS_WITH( + *i, + "Dereferencing the DataLoader's past-the-end iterator is not allowed"); +} + +TEST(DataLoaderTest, YieldsCorrectBatchSize) { + DummyDataset dataset; + auto data_loader = torch::data::make_data_loader(dataset, 25); + auto iterator = data_loader->begin(); + ASSERT_EQ(iterator->size(), 25); + ASSERT_EQ((++iterator)->size(), 25); + ASSERT_EQ((++iterator)->size(), 25); + ASSERT_EQ((++iterator)->size(), 25); + ASSERT_EQ(++iterator, data_loader->end()); +} + +TEST( + DataLoaderTest, + ReturnsLastBatchWhenSmallerThanBatchSizeWhenDropLastIsFalse) { + DummyDataset dataset; + auto data_loader = torch::data::make_data_loader( + dataset, DataLoaderOptions(33).drop_last(false)); + auto iterator = data_loader->begin(); + ASSERT_EQ(iterator->size(), 33); + ASSERT_EQ((++iterator)->size(), 33); + ASSERT_EQ((++iterator)->size(), 33); + ASSERT_EQ((++iterator)->size(), 1); + ASSERT_EQ(++iterator, data_loader->end()); +} + +TEST( + DataLoaderTest, + DoesNotReturnLastBatchWhenSmallerThanBatchSizeWhenDropLastIsTrue) { + DummyDataset dataset; + auto data_loader = torch::data::make_data_loader( + dataset, DataLoaderOptions(33).drop_last(true)); + auto iterator = data_loader->begin(); + ASSERT_EQ(iterator->size(), 33); + ASSERT_EQ((++iterator)->size(), 33); + ASSERT_EQ((++iterator)->size(), 33); + ASSERT_EQ(++iterator, data_loader->end()); +} + +TEST(DataLoaderTest, RespectsTimeout) { + struct Baton { + std::condition_variable cv; + std::mutex mutex; + }; + + struct D : datasets::Dataset { + D(std::shared_ptr b) : baton(std::move(b)) {} + int get(size_t index) override { + std::unique_lock lock(baton->mutex); + baton->cv.wait_for(lock, 1000 * kMillisecond); + return 0; + } + std::optional size() const override { + return 100; + } + std::shared_ptr baton; + }; + + auto baton = std::make_shared(); + + auto data_loader = torch::data::make_data_loader( + D{baton}, DataLoaderOptions().workers(1).timeout(10 * kMillisecond)); + + auto start = std::chrono::system_clock::now(); + + ASSERT_THROWS_WITH(*data_loader->begin(), "Timeout"); + baton->cv.notify_one(); + + auto end = std::chrono::system_clock::now(); + auto duration = std::chrono::duration_cast(end - start); + ASSERT_LT(duration.count(), 1); +} + +// stackoverflow.com/questions/24465533/implementing-boostbarrier-in-c11 +struct Barrier { + explicit Barrier(size_t target) : counter_(target) {} + void wait() { + std::unique_lock lock(mutex_); + if (--counter_ == 0) { + cv_.notify_all(); + } else { + cv_.wait(lock, [this] { return this->counter_ == 0; }); + } + } + + size_t counter_; + std::condition_variable cv_; + std::mutex mutex_; +}; + +// On the OrderingTest: This test is intended to verify that the +// `enforce_ordering` option of the dataloader works correctly. The reason this +// flag exists is because when the dataloader has multiple workers (threads) +// enabled and this flag is not set, the order in which worker threads finish +// loading their respective batch and push it back to the dataloader's main +// thread (for outside consumption) is not deterministic. Imagine the sampler is +// a SequentialSampler with indices 0, 1, 2, 3. With batch size 1, each index +// will be a single "job". Inside the dataloader, worker threads block until a +// job is available. It is not deterministic which worker thread wakes up batch +// to dequeue a particular batch. Further, some worker threads may take longer +// than others to read the data for their index. As such, it could be that +// worker thread 2 finishes before all other threads and returns its batch to +// the main thread. In that case, the dataloader iterator would return the datum +// at index 2 batch, and afterwards the datum from whatever thread finishes +// next. As such, the user may see data from indices 2, 0, 3, 1. On another run +// of the same dataloader on the same data, threads may be scheduled differently +// and return in order 0, 2, 3, 1. To force this ordering to deterministically +// be 0, 1, 2, 3, the `enforce_ordering` flag can be set to true. In that case, +// the dataloader will use a *sequencer* internally which keeps track of which +// datum is expected next, and buffers any other results until that next +// expected value arrives. For example, workers 1, 2, 3 may finish before worker +// 0. If `enforce_ordering` is true, the sequencer will internally buffer the +// results from 1, 2, 3 until worker 0 finishes. Only then does the dataloader +// return the datum from worker 0 to the user (and then datum 1 the next time, +// then 2 and so on). +// +// The way the test works is that we start +// `kNumberOfWorkers` workers in the dataloader, which each get an index from a +// `SequentialSampler` in the range `0...kNumberOfWorkers-1`. Each worker thread +// has a copy of the dataset, and thus `get_batch()` is called on the +// thread-local copy in each worker. We want to simulate out-of-order completion +// of these threads. For this, we batch set a barrier in the `get_batch()` +// method to make sure every worker has some index to fetch assigned. Further, +// each worker thread has a unique ID in `0...kNumberOfWorkers-1`. +// There is a hard-coded ordering, `kOrderInWhichWorkersReturnTheirBatch`, in +// which we want the worker threads to return. For this, an iterator into this +// order is maintained. When the dereferenced iterator (the current order index) +// matches the thread ID of a worker, it knows it can now return its index as +// well as progress the iterator. Inside the dataloader, the sequencer should +// buffer these indices such that they are ultimately returned in order. + +namespace ordering_test { +namespace { +const size_t kNumberOfWorkers = 10; +const std::vector kOrderInWhichWorkersReturnTheirBatch = + {3, 7, 0, 5, 4, 8, 2, 1, 9, 6}; +} // namespace + +struct Dataset : datasets::BatchDataset { + Dataset() = default; + + // This copy constructor will be called when we copy the dataset into a + // particular thread. + Dataset(const Dataset& other) { + static std::atomic counter{0}; + thread_id_ = counter.fetch_add(1); + } + + Dataset(Dataset&& other) noexcept = default; + Dataset& operator=(const Dataset& other) = delete; + Dataset& operator=(Dataset&& other) noexcept = delete; + + size_t get_batch(torch::ArrayRef indices) override { + static Barrier barrier(kNumberOfWorkers); + static auto order_iterator = kOrderInWhichWorkersReturnTheirBatch.begin(); + static std::condition_variable cv; + static std::mutex mutex; + + // Wait for all threads to get an index batch and arrive here. + barrier.wait(); + + std::unique_lock lock(mutex); + cv.wait(lock, [this] { return *order_iterator == this->thread_id_; }); + ++order_iterator; + lock.unlock(); + cv.notify_all(); + + return indices.front(); + } + + std::optional size() const override { + return kNumberOfWorkers; + } + + size_t thread_id_ = 0; +}; + +} // namespace ordering_test + +TEST(DataLoaderTest, EnforcesOrderingAmongThreadsWhenConfigured) { + auto data_loader = torch::data::make_data_loader( + ordering_test::Dataset{}, + torch::data::samplers::SequentialSampler(ordering_test::kNumberOfWorkers), + DataLoaderOptions() + .batch_size(1) + .workers(ordering_test::kNumberOfWorkers) + .enforce_ordering(true)); + std::vector output; + for (size_t value : *data_loader) { + output.push_back(value); + } + std::vector expected(ordering_test::kNumberOfWorkers); + std::iota(expected.begin(), expected.end(), size_t(0)); + ASSERT_EQ(expected, output); +} + +TEST(DataLoaderTest, Reset) { + DummyDataset dataset; + auto data_loader = + torch::data::make_data_loader(dataset, dataset.size().value() / 2); + auto end = data_loader->end(); + + auto iterator = data_loader->begin(); + ASSERT_NE(iterator, end); + ASSERT_NE(++iterator, end); + ASSERT_EQ(++iterator, end); + + iterator = data_loader->begin(); + ASSERT_NE(iterator, end); + ASSERT_NE(++iterator, end); + ASSERT_EQ(++iterator, end); + + iterator = data_loader->begin(); + ASSERT_NE(iterator, end); + ASSERT_NE(++iterator, end); + ASSERT_EQ(++iterator, end); +} + +TEST(DataLoaderTest, TestExceptionsArePropagatedFromWorkers) { + struct D : datasets::Dataset { + int get(size_t index) override { + throw std::invalid_argument("badness"); + } + std::optional size() const override { + return 100; + } + }; + + auto data_loader = torch::data::make_data_loader( + D{}, samplers::RandomSampler(100), DataLoaderOptions().workers(2)); + auto iterator = data_loader->begin(); + + try { + (void)*iterator; + } catch (torch::data::WorkerException& e) { + ASSERT_EQ( + e.what(), + std::string("Caught exception in DataLoader worker thread. " + "Original message: badness")); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_THROW( + std::rethrow_exception(e.original_exception), std::invalid_argument); + } +} + +TEST(DataLoaderTest, StatefulDatasetWithNoWorkers) { + const int kNumberOfExamplesAfterWhichTheDatasetExhausts = 10; + + struct D : datasets::StatefulDataset { + std::optional get_batch(size_t) override { + if (counter < kNumberOfExamplesAfterWhichTheDatasetExhausts) { + return counter++; + } + return std::nullopt; + } + std::optional size() const override { + return 100; + } + void reset() override { + counter = 0; + } + void save(torch::serialize::OutputArchive& archive) const override {}; + void load(torch::serialize::InputArchive& archive) override {} + int counter = 0; + }; + + auto data_loader = torch::data::make_data_loader(D{}); + + for (const auto i : c10::irange(10)) { + const auto number_of_iterations = + std::distance(data_loader->begin(), data_loader->end()); + ASSERT_EQ( + number_of_iterations, kNumberOfExamplesAfterWhichTheDatasetExhausts) + << "epoch " << i; + } + + for (const int i : *data_loader) { + ASSERT_LT(i, kNumberOfExamplesAfterWhichTheDatasetExhausts); + } +} + +TEST(DataLoaderTest, StatefulDatasetWithManyWorkers) { + const int kNumberOfExamplesAfterWhichTheDatasetExhausts = 10; + const int kNumberOfWorkers = 4; + + struct D : datasets::StatefulDataset { + std::optional get_batch(size_t) override { + std::lock_guard lock(mutex); + if (counter < kNumberOfExamplesAfterWhichTheDatasetExhausts) { + return counter++; + } + return std::nullopt; + } + std::optional size() const override { + return 100; + } + void reset() override { + counter = 0; + } + void save(torch::serialize::OutputArchive& archive) const override {}; + void load(torch::serialize::InputArchive& archive) override {} + int counter = 0; + std::mutex mutex; + }; + + auto data_loader = torch::data::make_data_loader( + torch::data::datasets::make_shared_dataset(), + DataLoaderOptions().workers(kNumberOfWorkers)); + + for (const auto i : c10::irange(10)) { + const auto number_of_iterations = + std::distance(data_loader->begin(), data_loader->end()); + ASSERT_EQ( + number_of_iterations, kNumberOfExamplesAfterWhichTheDatasetExhausts) + << "epoch " << i; + } + + for (const int i : *data_loader) { + ASSERT_LT(i, kNumberOfExamplesAfterWhichTheDatasetExhausts); + } +} + +TEST(DataLoaderTest, StatefulDatasetWithMap) { + const int kNumberOfExamplesAfterWhichTheDatasetExhausts = 10; + + struct D : datasets::StatefulDataset { + std::optional get_batch(size_t) override { + if (counter < kNumberOfExamplesAfterWhichTheDatasetExhausts) { + return counter++; + } + return std::nullopt; + } + std::optional size() const override { + return 100; + } + void reset() override { + counter = 0; + } + void save(torch::serialize::OutputArchive& archive) const override {}; + void load(torch::serialize::InputArchive& archive) override {} + int counter = 0; + }; + + auto data_loader = torch::data::make_data_loader( + D().map(transforms::BatchLambda( + [](int x) { return std::to_string(x); })) + .map(transforms::BatchLambda( + [](const std::string& x) { + return torch::tensor(static_cast(std::stoi(x))); + })), + DataLoaderOptions{}); + + for (const auto i : c10::irange(10)) { + const auto number_of_iterations = + std::distance(data_loader->begin(), data_loader->end()); + ASSERT_EQ( + number_of_iterations, kNumberOfExamplesAfterWhichTheDatasetExhausts) + << "epoch " << i; + } + + for (const torch::Tensor& t : *data_loader) { + ASSERT_LT(t.item(), kNumberOfExamplesAfterWhichTheDatasetExhausts); + } +} + +TEST(DataLoaderTest, StatefulDatasetWithCollate) { + const int kNumberOfExamplesAfterWhichTheDatasetExhausts = 10; + + struct D : datasets::StatefulDataset { + std::optional>> get_batch( + size_t batch_size) override { + if (counter < kNumberOfExamplesAfterWhichTheDatasetExhausts) { + counter += batch_size; + std::vector> batch( + /*count=*/batch_size, + Example<>{ + torch::ones(batch_size + 1), torch::zeros(batch_size - 1)}); + return batch; + } + return std::nullopt; + } + std::optional size() const override { + return 100; + } + void reset() override { + counter = 0; + } + void save(torch::serialize::OutputArchive& archive) const override {}; + void load(torch::serialize::InputArchive& archive) override {} + int counter = 0; + }; + + auto d = D().map(transforms::Stack>()); + + const size_t kBatchSize = 5; + + // Notice that the `get_batch()` of the dataset returns a vector, but + // the `Stack` collation stacks the tensors into one. + std::optional> batch = d.get_batch(kBatchSize); + ASSERT_TRUE(batch.has_value()); + ASSERT_EQ(batch->data.size(0), kBatchSize); + ASSERT_EQ(batch->data.size(1), kBatchSize + 1); + ASSERT_EQ(batch->target.size(0), kBatchSize); + ASSERT_EQ(batch->target.size(1), kBatchSize - 1); + + ASSERT_TRUE(batch->data[0].allclose(torch::ones(kBatchSize + 1))); + ASSERT_TRUE(batch->target[0].allclose(torch::zeros(kBatchSize - 1))); +} + +// This test tests the core function for iterate through a chunk dataset. It +// contains test cases with different parameter combination. (For example, +// different prefetch count, batch size and data loader worker count). It +// verifies the return batches size and content when the order is deterministic. +TEST(DataLoaderTest, ChunkDataSetGetBatch) { + // different prefetch count for testing. + // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + const size_t prefetch_counts[] = {1, 2, 3, 4}; + + // different batch size for testing. + // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + const size_t batch_sizes[] = {5, 7}; + + // test with/without worker threads + // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + const size_t dataloader_worker_counts[] = {0, 2}; + + const size_t total_example_count = 35; + DummyChunkDataReader data_reader; + samplers::SequentialSampler sampler(0); + + // test functionality across epoch boundary + const int epoch_count = 2; + + for (auto prefetch_count : prefetch_counts) { + for (auto batch_size : batch_sizes) { + for (auto dataloader_worker_count : dataloader_worker_counts) { + datasets::SharedBatchDataset> + dataset = datasets::make_shared_dataset>( + data_reader, + sampler, + sampler, + datasets::ChunkDatasetOptions(prefetch_count, batch_size)); + + auto data_loader = torch::data::make_data_loader( + dataset, + DataLoaderOptions(batch_size).workers(dataloader_worker_count)); + + for (const auto epoch_index : c10::irange(epoch_count)) { + (void)epoch_index; // Suppress unused variable warning + std::vector result(total_example_count, false); + int iteration_count = 0; + for (auto iterator = data_loader->begin(); + iterator != data_loader->end(); + ++iterator, ++iteration_count) { + DummyChunkDataReader::BatchType& batch = *iterator; + ASSERT_EQ(batch.size(), batch_size); + + // When prefetch_count is equal to 1 and no worker thread, the batch + // order is deterministic. So we can verify elements in each batch. + if (prefetch_count == 1 && dataloader_worker_count == 0) { + for (const auto j : c10::irange(batch_size)) { + ASSERT_EQ(batch[j], iteration_count * batch_size + j); + } + } + for (const auto j : c10::irange(batch_size)) { + result[batch[j]] = true; + } + } + + for (auto data : result) { + ASSERT_EQ(data, true); + } + } + } + } + } +} + +TEST(DataLoaderTest, ChunkDataSetWithBatchSizeMismatch) { + const size_t prefetch_count = 1; + const size_t batch_size = 5; + const size_t requested_batch_size = 6; + + DummyChunkDataReader data_reader; + samplers::SequentialSampler sampler(0); + + datasets::SharedBatchDataset> + dataset = datasets::make_shared_dataset>( + data_reader, + sampler, + sampler, + datasets::ChunkDatasetOptions(prefetch_count, batch_size)); + + auto data_loader = torch::data::make_data_loader( + dataset, DataLoaderOptions(requested_batch_size).workers(0)); + + std::string exception_msg = + "The requested batch size does not match with the initialized batch " + "size.\n The requested batch size is 6, while the dataset is created" + " with batch size equal to 5"; + + ASSERT_THROWS_WITH(*(data_loader->begin()), exception_msg); +} + +TEST(DataLoaderTest, ChunkDataSetWithEmptyBatch) { + struct DummyEmptyChunkDataReader : datasets::ChunkDataReader { + public: + using BatchType = datasets::ChunkDataReader::ChunkType; + + BatchType read_chunk(size_t chunk_index) override { + return {}; + } + + size_t chunk_count() override { + return 1; + }; + + void reset() override {}; + }; + + const size_t prefetch_count = 1; + const size_t batch_size = 5; + DummyEmptyChunkDataReader data_reader; + samplers::SequentialSampler sampler(0); + + datasets::SharedBatchDataset> + dataset = datasets::make_shared_dataset>( + data_reader, + sampler, + sampler, + datasets::ChunkDatasetOptions(prefetch_count, batch_size)); + + auto data_loader = torch::data::make_data_loader( + dataset, DataLoaderOptions(batch_size).workers(0)); + + for (auto iterator = data_loader->begin(); iterator != data_loader->end(); + ++iterator) { + ASSERT_EQ(iterator->size(), 0); + } +} + +TEST(DataLoaderTest, ChunkDataSetGetBatchWithUnevenBatchSize) { + struct D : public datasets::ChunkDataReader { + public: + using BatchType = datasets::ChunkDataReader::ChunkType; + + BatchType read_chunk(size_t chunk_index) override { + BatchType batch_data(10, 0); + return batch_data; + } + + size_t chunk_count() override { + return 2; + }; + + void reset() override {}; + }; + + // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + const size_t batch_sizes[] = {17, 30}; + D data_reader; + samplers::SequentialSampler sampler(0); + + for (auto batch_size : batch_sizes) { + datasets::SharedBatchDataset> + dataset = datasets::make_shared_dataset>( + data_reader, + sampler, + sampler, + datasets::ChunkDatasetOptions(1, batch_size)); + + auto data_loader = torch::data::make_data_loader( + dataset, DataLoaderOptions(batch_size).workers(0)); + + for (auto iterator = data_loader->begin(); iterator != data_loader->end(); + ++iterator) { + DummyChunkDataReader::BatchType batch = *iterator; + auto batch_size = batch.size(); + if (batch_size == 17) { + ASSERT_TRUE(batch.size() == 17 || batch.size() == 3); + } + if (batch_size == 30) { + ASSERT_TRUE(batch.size() == 20); + } + } + } +} + +TEST(DataLoaderTest, CanAccessChunkSamplerWithChunkDataSet) { + const size_t prefetch_count = 2; + const size_t batch_size = 5; + + DummyChunkDataReader data_reader; + samplers::SequentialSampler sampler(0); + datasets::SharedBatchDataset> + dataset = datasets::make_shared_dataset>( + data_reader, + sampler, + sampler, + datasets::ChunkDatasetOptions(prefetch_count, batch_size)); + + samplers::SequentialSampler& chunk_sampler = dataset->chunk_sampler(); + + auto data_loader = torch::data::make_data_loader( + dataset.map(transforms::BatchLambda< + DummyChunkDataReader::BatchType, + DummyChunkDataReader::DataType>( + [](DummyChunkDataReader::BatchType batch) { + return std::accumulate(batch.begin(), batch.end(), 0); + })), + DataLoaderOptions(batch_size).workers(0)); + + // before we start, the index should be 0. + ASSERT_EQ(chunk_sampler.index(), 0); + + size_t sum = 0; + for (auto iterator = data_loader->begin(); iterator != data_loader->end(); + ++iterator) { + sum += *iterator; + } + ASSERT_EQ(sum, 595); // sum([0, 35)) + // 3 chunks, and when exhausted the value is already incremented. + ASSERT_EQ(chunk_sampler.index(), 3); +} + +TEST(DataLoaderTest, ChunkDatasetDoesNotHang) { + const size_t prefetch_count = 2; + const size_t batch_size = 5; + // this will make the preloaders to wait till the `get_batch()` calls. + const size_t cache_size = 10; + + DummyChunkDataReader data_reader; + samplers::SequentialSampler sampler(0); + datasets::SharedBatchDataset> + dataset = datasets::make_shared_dataset>( + data_reader, + sampler, + sampler, + datasets::ChunkDatasetOptions( + prefetch_count, batch_size, cache_size)); + + auto data_loader = torch::data::make_data_loader( + dataset.map(transforms::BatchLambda< + DummyChunkDataReader::BatchType, + DummyChunkDataReader::DataType>( + [](DummyChunkDataReader::BatchType batch) { + return std::accumulate(batch.begin(), batch.end(), 0); + })), + DataLoaderOptions(batch_size).workers(0)); + // simply creates the iterator but no iteration. chunk preloaders are waiting + // to fill the batch buffer but it is not draining. Still we need to exit + // cleanly. + auto iterator = data_loader->begin(); +} + +// Test ChunkDataset save function. +// Note [save/load ChunkDataset as ChunkSampler]: +// The chunk sampler inside ChunkDataset is used in a separate thread pool other +// than the main thread. Thus it is very hard to accurately estimate its status +// when ChunkDataset::save/ChunkDataset::load is called. For the pure purpose of +// testing, we utilize the implementation fact that the file format for sampler +// serialization is the same as ChunkDataset serialization, and manually control +// the chunk sampler by calling the sampler's save/load method for value +// validation. This is only for testing the specific save/load functionality. In +// real user case, the user should still use matching ChunkDataset::save and +// ChunkDataset::load method. +TEST(DataLoaderTest, ChunkDatasetSave) { + const size_t chunk_count_ = 6; + const size_t chunk_size = 10; + + struct DummyTestChunkDataReader : datasets::ChunkDataReader { + public: + using BatchType = datasets::ChunkDataReader::ChunkType; + + BatchType read_chunk(size_t chunk_index) override { + return batch_data_; + } + + size_t chunk_count() override { + return chunk_count_; + }; + + void reset() override {}; + BatchType batch_data_ = BatchType(chunk_size, 0); + }; + + const size_t prefetch_count = 1; + const size_t batch_size = chunk_size; + const size_t dataloader_worker_count = 0; + samplers::SequentialSampler sampler(0); + const int epoch_count = 2; + + DummyTestChunkDataReader data_reader; + + // tested save_intervals + // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + const size_t save_intervals[] = {1, 2}; + + using datasets::ChunkDatasetOptions; + + for (auto save_interval : save_intervals) { + auto tempfile = c10::make_tempfile(); + + datasets::SharedBatchDataset> + dataset = datasets::make_shared_dataset>( + data_reader, + sampler, + sampler, + ChunkDatasetOptions( + prefetch_count, batch_size, chunk_size /*cache size*/)); + + auto data_loader = torch::data::make_data_loader( + dataset, + DataLoaderOptions(batch_size).workers(dataloader_worker_count)); + + for (const auto epoch_index : c10::irange(epoch_count)) { + (void)epoch_index; // Suppress unused variable warning + unsigned iteration_count = 0; + for (auto iterator = data_loader->begin(); iterator != data_loader->end(); + ++iterator, ++iteration_count) { + if ((iteration_count + 1) % save_interval == 0) { + torch::save(*dataset, tempfile.name); + + samplers::SequentialSampler new_sampler(0); + + // See Note [save/load ChunkDataset as ChunkSampler] + torch::load(new_sampler, tempfile.name); + + // Verify save logic. For ChunkDataset, the chunk data is stored in a + // cache inside the dataset. One pool of threads are constantly + // writing to the cache, and a different pool of thread are constantly + // reading from the cache. Due to the nature of asynchronization, at + // the time of get_batch(), which chunk is written to the cache is not + // fully deterministic. + // But we can still calculate a restricted window on the expected + // output, hence verify the logic. In this test, the cache size is + // configured to be the same as chunk size and batch size. So the + // chunk data is written to the cache one by one. Only the current + // batch is retrieved, the next chunk is written. Now in iteration 0, + // after the first batch is retrieved, when we save the dataset + // statues, there are three possible scenarios for the writer thread: + // 1. it hasn't started loading the next chunk data yet, so the + // sequential sampler index is still 0; + // 2. it started to load the second chunk, so the sequential sampler + // index is at 1; + // 3. it finished loading the second chunk, and start to load the + // third chunk, because the cache is still fully occupied by the data + // from the second chunk, it is waiting to write to the cache. At this + // point, the sampler index is at 2. + // So now we have a window of [0, 2], which is what we expected the + // sampler to save the index from. Now noted for sequential sampler, + // it advances to the next index automatically in the call next(). So + // when save the index, it saves the next index in stead of the + // current one. In other word, after getting the first index from + // sequential sampler, it already moves to the second index. So when + // we save it, it is the second index we save. As a result, + // we need to advance the window by one. Now we have the expected + // window of [1, 3]. + // This analysis applies to all scenarios. So extend it to a more + // general case: the expected saved index should falling into the + // range of [iteration, iteration + 3], which is the validation + // below. + ASSERT_TRUE( + new_sampler.index() >= iteration_count + 1 && + new_sampler.index() <= iteration_count + 3); + } + } + } + } +} + +// Test ChunkDataset load function. +TEST(DataLoaderTest, ChunkDatasetLoad) { + auto tempfile = c10::make_tempfile(); + + const size_t prefetch_count = 1; + const size_t batch_size = 10; + const size_t dataloader_worker_count = 0; + + DummyChunkDataReader data_reader; + samplers::SequentialSampler sampler(0); + + const size_t skipped_chunk = 2; + + // Configure sampler to skip 2 chunks + { + sampler.reset(data_reader.chunk_count()); + sampler.next(skipped_chunk); + + // See Note [save/load ChunkDataset as ChunkSampler] + torch::save(sampler, tempfile.name); + } + + // test functionality across epoch boundary. The first epoch should be + // affected by the checkpoint, but the second should start normally. + const int epoch_count = 2; + + datasets::SharedBatchDataset> + dataset = datasets::make_shared_dataset>( + data_reader, + sampler, + sampler, + datasets::ChunkDatasetOptions( + prefetch_count, batch_size, 20 /*cache size*/)); + + torch::load(*dataset, tempfile.name); + + auto data_loader = torch::data::make_data_loader( + dataset, DataLoaderOptions(batch_size).workers(dataloader_worker_count)); + + for (const auto epoch_index : c10::irange(epoch_count)) { + int iteration_count = 0; + + // For the first epoch, the returned batch should be returned from the + // third chunk, because the check point skipped the first two chunks. But + // for the next epoch, it should start from the first batch. + int initial_value = epoch_index == 0 ? 15 : 0; + + for (auto iterator = data_loader->begin(); iterator != data_loader->end(); + ++iterator, ++iteration_count) { + DummyChunkDataReader::BatchType batch = *iterator; + + std::vector expected_result; + size_t expected_size = (epoch_index > 0 && iteration_count == 3) ? 5 : 10; + expected_result.resize(expected_size); + std::iota(expected_result.begin(), expected_result.end(), initial_value); + + ASSERT_EQ(batch.size(), expected_result.size()); + ASSERT_TRUE( + std::equal(batch.begin(), batch.end(), expected_result.begin())); + + initial_value += batch_size; + } + } + + samplers::SequentialSampler new_sampler(0); + + // See Note [save/load ChunkDataset as ChunkSampler] + torch::load(new_sampler, tempfile.name); + + ASSERT_EQ(new_sampler.index(), skipped_chunk); +} + +TEST(DataLoaderTest, ChunkDatasetCrossChunkShuffle) { + const size_t chunk_size = 5; + const size_t batch_size = 5; + + class S : public samplers::Sampler<> { + public: + explicit S(size_t size) : size_(size), index_(0) {}; + + void reset(std::optional new_size = std::nullopt) override { + if (new_size.has_value()) { + size_ = *new_size; + } + indices_.resize(size_); + size_t index = 0; + + // Repeatedly sample every 5 indices. + for (const auto i : c10::irange(batch_size)) { + for (size_t j = 0; j < size_ / batch_size; ++j) { + indices_[index++] = i + batch_size * j; + } + } + index_ = 0; + } + + // Returns the next batch of indices. + std::optional> next(size_t batch_size) override { + const auto remaining_indices = size_ - index_; + if (remaining_indices == 0) { + return std::nullopt; + } + auto return_size = std::min(batch_size, remaining_indices); + std::vector index_batch( + indices_.begin() + index_, indices_.begin() + index_ + return_size); + index_ += return_size; + + return index_batch; + } + + void save(torch::serialize::OutputArchive& archive) const override {} + void load(torch::serialize::InputArchive& archive) override {} + + private: + size_t size_; + std::vector indices_; + size_t index_{0}; + }; + + struct D : public datasets::ChunkDataReader { + public: + using BatchType = datasets::ChunkDataReader::ChunkType; + D(size_t chunk_count) : chunk_count_(chunk_count) {} + + BatchType read_chunk(size_t chunk_index) override { + BatchType batch_data(chunk_size, chunk_index); + return batch_data; + } + + size_t chunk_count() override { + return chunk_count_; + }; + + void reset() override {}; + size_t chunk_count_; + }; + + const size_t prefetch_count = 1; + const size_t cache_size = 10; + // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + const size_t cross_chunk_shuffle_counts[] = {2, 3}; + // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + const size_t chunk_counts[] = {3, 4, 5}; + + samplers::SequentialSampler chunk_sampler(0); + S example_sampler(0); + + for (auto chunk_count : chunk_counts) { + for (auto cross_chunk_shuffle_count : cross_chunk_shuffle_counts) { + D data_reader(chunk_count); + + datasets::SharedBatchDataset< + datasets::ChunkDataset> + dataset = datasets::make_shared_dataset< + datasets::ChunkDataset>( + data_reader, + chunk_sampler, + example_sampler, + datasets::ChunkDatasetOptions( + prefetch_count, + batch_size, + cache_size, + cross_chunk_shuffle_count)); + + auto data_loader = torch::data::make_data_loader( + dataset, DataLoaderOptions(batch_size).workers(0)); + + std::vector result; + for (auto iterator = data_loader->begin(); iterator != data_loader->end(); + ++iterator) { + auto batch_result = *iterator; + std::copy( + batch_result.begin(), + batch_result.end(), + std::back_inserter(result)); + } + + std::vector expected_result; + { + // construct expected result + for (const auto i : c10::irange( + (chunk_count + cross_chunk_shuffle_count - 1) / + cross_chunk_shuffle_count)) { + for ([[maybe_unused]] const auto j : c10::irange(chunk_size)) { + for (const auto k : c10::irange(cross_chunk_shuffle_count)) { + if (i * cross_chunk_shuffle_count + k < chunk_count) { + expected_result.push_back(i * cross_chunk_shuffle_count + k); + } + } + } + } + } + + ASSERT_EQ(result.size(), expected_result.size()); + ASSERT_TRUE( + std::equal(result.begin(), result.end(), expected_result.begin())); + } + } +} + +TEST(DataLoaderTest, CustomPreprocessPolicy) { + const size_t chunk_size = 5; + const size_t batch_size = 10; + + struct D : public datasets::ChunkDataReader { + public: + using BatchType = datasets::ChunkDataReader::ChunkType; + D(size_t chunk_count) : chunk_count_(chunk_count) {} + + BatchType read_chunk(size_t chunk_index) override { + BatchType batch_data(chunk_size); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,clang-analyzer-security.insecureAPI.rand) + auto rand_gen = []() { return std::rand() % 100; }; + std::generate(batch_data.begin(), batch_data.end(), rand_gen); + return batch_data; + } + + size_t chunk_count() override { + return chunk_count_; + }; + + void reset() override {}; + size_t chunk_count_; + }; + + // custom preprocessing policy - sort the data ascendingly + auto sorting_policy = [](std::vector& raw_batch_data) { + std::sort(raw_batch_data.begin(), raw_batch_data.end()); + }; + std::function&)> policy_function = sorting_policy; + + const size_t prefetch_count = 1; + const size_t cache_size = 10; + // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + const size_t cross_chunk_shuffle_counts[] = {1, 2}; + // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + const size_t chunk_counts[] = {3, 4}; + + samplers::SequentialSampler chunk_sampler(0); + + for (auto chunk_count : chunk_counts) { + for (auto cross_chunk_shuffle_count : cross_chunk_shuffle_counts) { + D data_reader(chunk_count); + + datasets::SharedBatchDataset> + dataset = datasets::make_shared_dataset>( + data_reader, + chunk_sampler, + chunk_sampler, + datasets::ChunkDatasetOptions( + prefetch_count, + batch_size, + cache_size, + cross_chunk_shuffle_count), + policy_function); + + auto data_loader = torch::data::make_data_loader( + dataset, DataLoaderOptions(batch_size).workers(0)); + + std::vector result; + for (auto iterator = data_loader->begin(); iterator != data_loader->end(); + ++iterator) { + auto batch_result = *iterator; + if (batch_result.size() > chunk_size * cross_chunk_shuffle_count) { + for (unsigned i = 0; i < batch_result.size(); i += chunk_size) { + ASSERT_TRUE(std::is_sorted( + batch_result.begin() + i, + batch_result.begin() + i + chunk_size)); + } + } else { + ASSERT_TRUE(std::is_sorted(batch_result.begin(), batch_result.end())); + } + } + } + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/dispatch.cpp b/western_ai/usa/pytorch/test/cpp/api/dispatch.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9f17f392d5645f38b3da731c895087afd8facb10 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/dispatch.cpp @@ -0,0 +1,49 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct DispatchTest : torch::test::SeedingFixture {}; + +TEST_F(DispatchTest, TestAVX2) { + const std::vector ints{1, 2, 3, 4}; + const std::vector result{1, 4, 27, 256}; + const auto vals_tensor = torch::tensor(ints); + const auto pows_tensor = torch::tensor(ints); + c10::utils::set_env("ATEN_CPU_CAPABILITY", "avx2"); + const auto actual_pow_avx2 = vals_tensor.pow(pows_tensor); + for (const auto i : c10::irange(4)) { + ASSERT_EQ(result[i], actual_pow_avx2[i].item()); + } +} + +TEST_F(DispatchTest, TestAVX512) { + const std::vector ints{1, 2, 3, 4}; + const std::vector result{1, 4, 27, 256}; + const auto vals_tensor = torch::tensor(ints); + const auto pows_tensor = torch::tensor(ints); + c10::utils::set_env("ATEN_CPU_CAPABILITY", "avx512"); + const auto actual_pow_avx512 = vals_tensor.pow(pows_tensor); + for (const auto i : c10::irange(4)) { + ASSERT_EQ(result[i], actual_pow_avx512[i].item()); + } +} + +TEST_F(DispatchTest, TestDefault) { + const std::vector ints{1, 2, 3, 4}; + const std::vector result{1, 4, 27, 256}; + const auto vals_tensor = torch::tensor(ints); + const auto pows_tensor = torch::tensor(ints); + c10::utils::set_env("ATEN_CPU_CAPABILITY", "default"); + const auto actual_pow_default = vals_tensor.pow(pows_tensor); + for (const auto i : c10::irange(4)) { + ASSERT_EQ(result[i], actual_pow_default[i].item()); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/enum.cpp b/western_ai/usa/pytorch/test/cpp/api/enum.cpp new file mode 100644 index 0000000000000000000000000000000000000000..45c0d07cc21fb654c55dbfcae26b006830f647d1 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/enum.cpp @@ -0,0 +1,88 @@ +#include + +#include +#include + +#include + +#define TORCH_ENUM_PRETTY_PRINT_TEST(name) \ + { \ + v = torch::k##name; \ + std::string pretty_print_name("k"); \ + pretty_print_name.append(#name); \ + ASSERT_EQ(torch::enumtype::get_enum_name(v), pretty_print_name); \ + } + +TEST(EnumTest, AllEnums) { + std::variant< + torch::enumtype::kLinear, + torch::enumtype::kConv1D, + torch::enumtype::kConv2D, + torch::enumtype::kConv3D, + torch::enumtype::kConvTranspose1D, + torch::enumtype::kConvTranspose2D, + torch::enumtype::kConvTranspose3D, + torch::enumtype::kSigmoid, + torch::enumtype::kTanh, + torch::enumtype::kReLU, + torch::enumtype::kLeakyReLU, + torch::enumtype::kFanIn, + torch::enumtype::kFanOut, + torch::enumtype::kConstant, + torch::enumtype::kReflect, + torch::enumtype::kReplicate, + torch::enumtype::kCircular, + torch::enumtype::kNearest, + torch::enumtype::kBilinear, + torch::enumtype::kBicubic, + torch::enumtype::kTrilinear, + torch::enumtype::kArea, + torch::enumtype::kSum, + torch::enumtype::kMean, + torch::enumtype::kMax, + torch::enumtype::kNone, + torch::enumtype::kBatchMean, + torch::enumtype::kZeros, + torch::enumtype::kBorder, + torch::enumtype::kReflection, + torch::enumtype::kRNN_TANH, + torch::enumtype::kRNN_RELU, + torch::enumtype::kLSTM, + torch::enumtype::kGRU> + v; + + TORCH_ENUM_PRETTY_PRINT_TEST(Linear) + TORCH_ENUM_PRETTY_PRINT_TEST(Conv1D) + TORCH_ENUM_PRETTY_PRINT_TEST(Conv2D) + TORCH_ENUM_PRETTY_PRINT_TEST(Conv3D) + TORCH_ENUM_PRETTY_PRINT_TEST(ConvTranspose1D) + TORCH_ENUM_PRETTY_PRINT_TEST(ConvTranspose2D) + TORCH_ENUM_PRETTY_PRINT_TEST(ConvTranspose3D) + TORCH_ENUM_PRETTY_PRINT_TEST(Sigmoid) + TORCH_ENUM_PRETTY_PRINT_TEST(Tanh) + TORCH_ENUM_PRETTY_PRINT_TEST(ReLU) + TORCH_ENUM_PRETTY_PRINT_TEST(LeakyReLU) + TORCH_ENUM_PRETTY_PRINT_TEST(FanIn) + TORCH_ENUM_PRETTY_PRINT_TEST(FanOut) + TORCH_ENUM_PRETTY_PRINT_TEST(Constant) + TORCH_ENUM_PRETTY_PRINT_TEST(Reflect) + TORCH_ENUM_PRETTY_PRINT_TEST(Replicate) + TORCH_ENUM_PRETTY_PRINT_TEST(Circular) + TORCH_ENUM_PRETTY_PRINT_TEST(Nearest) + TORCH_ENUM_PRETTY_PRINT_TEST(Bilinear) + TORCH_ENUM_PRETTY_PRINT_TEST(Bicubic) + TORCH_ENUM_PRETTY_PRINT_TEST(Trilinear) + TORCH_ENUM_PRETTY_PRINT_TEST(Area) + TORCH_ENUM_PRETTY_PRINT_TEST(Sum) + TORCH_ENUM_PRETTY_PRINT_TEST(Mean) + TORCH_ENUM_PRETTY_PRINT_TEST(Max) + TORCH_ENUM_PRETTY_PRINT_TEST(None) + TORCH_ENUM_PRETTY_PRINT_TEST(BatchMean) + TORCH_ENUM_PRETTY_PRINT_TEST(Zeros) + TORCH_ENUM_PRETTY_PRINT_TEST(Border) + TORCH_ENUM_PRETTY_PRINT_TEST(Reflection) + TORCH_ENUM_PRETTY_PRINT_TEST(RNN_TANH) + TORCH_ENUM_PRETTY_PRINT_TEST(RNN_RELU) + TORCH_ENUM_PRETTY_PRINT_TEST(LSTM) + TORCH_ENUM_PRETTY_PRINT_TEST(GRU) +} diff --git a/western_ai/usa/pytorch/test/cpp/api/expanding-array.cpp b/western_ai/usa/pytorch/test/cpp/api/expanding-array.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7ec187dec7aa691b441345bceb15ca3034ebb0b7 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/expanding-array.cpp @@ -0,0 +1,60 @@ +#include + +#include +#include + +#include + +#include +#include +#include + +struct ExpandingArrayTest : torch::test::SeedingFixture {}; + +TEST_F(ExpandingArrayTest, CanConstructFromInitializerList) { + torch::ExpandingArray<5> e({1, 2, 3, 4, 5}); + ASSERT_EQ(e.size(), 5); + for (const auto i : c10::irange(e.size())) { + ASSERT_EQ((*e)[i], i + 1); + } +} + +TEST_F(ExpandingArrayTest, CanConstructFromVector) { + torch::ExpandingArray<5> e(std::vector{1, 2, 3, 4, 5}); + ASSERT_EQ(e.size(), 5); + for (const auto i : c10::irange(e.size())) { + ASSERT_EQ((*e)[i], i + 1); + } +} + +TEST_F(ExpandingArrayTest, CanConstructFromArray) { + torch::ExpandingArray<5> e(std::array({1, 2, 3, 4, 5})); + ASSERT_EQ(e.size(), 5); + for (const auto i : c10::irange(e.size())) { + ASSERT_EQ((*e)[i], i + 1); + } +} + +TEST_F(ExpandingArrayTest, CanConstructFromSingleValue) { + torch::ExpandingArray<5> e(5); + ASSERT_EQ(e.size(), 5); + for (const auto i : c10::irange(e.size())) { + ASSERT_EQ((*e)[i], 5); + } +} + +TEST_F( + ExpandingArrayTest, + ThrowsWhenConstructedWithIncorrectNumberOfArgumentsInInitializerList) { + ASSERT_THROWS_WITH( + torch::ExpandingArray<5>({1, 2, 3, 4, 5, 6, 7}), + "Expected 5 values, but instead got 7"); +} + +TEST_F( + ExpandingArrayTest, + ThrowsWhenConstructedWithIncorrectNumberOfArgumentsInVector) { + ASSERT_THROWS_WITH( + torch::ExpandingArray<5>(std::vector({1, 2, 3, 4, 5, 6, 7})), + "Expected 5 values, but instead got 7"); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/fft.cpp b/western_ai/usa/pytorch/test/cpp/api/fft.cpp new file mode 100644 index 0000000000000000000000000000000000000000..37dbe68503ffa375027e23e21d766a78a86947dc --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/fft.cpp @@ -0,0 +1,130 @@ +#include + +#include +#include +#include + +// Naive DFT of a 1 dimensional tensor +torch::Tensor naive_dft(torch::Tensor x, bool forward = true) { + TORCH_INTERNAL_ASSERT(x.dim() == 1); + x = x.contiguous(); + auto out_tensor = torch::zeros_like(x); + const int64_t len = x.size(0); + + // Roots of unity, exp(-2*pi*j*n/N) for n in [0, N), reversed for inverse + // transform + std::vector> roots(len); + const auto angle_base = (forward ? -2.0 : 2.0) * M_PI / len; + for (const auto i : c10::irange(len)) { + auto angle = i * angle_base; + roots[i] = c10::complex(std::cos(angle), std::sin(angle)); + } + + const auto in = x.data_ptr>(); + const auto out = out_tensor.data_ptr>(); + for (const auto i : c10::irange(len)) { + for (const auto j : c10::irange(len)) { + out[i] += roots[(j * i) % len] * in[j]; + } + } + return out_tensor; +} + +// NOTE: Visual Studio and ROCm builds don't understand complex literals +// as of August 2020 + +TEST(FFTTest, fft) { + auto t = torch::randn(128, torch::kComplexDouble); + auto actual = torch::fft::fft(t); + auto expect = naive_dft(t); + ASSERT_TRUE(torch::allclose(actual, expect)); +} + +TEST(FFTTest, fft_real) { + auto t = torch::randn(128, torch::kDouble); + auto actual = torch::fft::fft(t); + auto expect = torch::fft::fft(t.to(torch::kComplexDouble)); + ASSERT_TRUE(torch::allclose(actual, expect)); +} + +TEST(FFTTest, fft_pad) { + auto t = torch::randn(128, torch::kComplexDouble); + auto actual = torch::fft::fft(t, 200); + auto expect = torch::fft::fft(torch::constant_pad_nd(t, {0, 72})); + ASSERT_TRUE(torch::allclose(actual, expect)); + + actual = torch::fft::fft(t, 64); + expect = torch::fft::fft(torch::constant_pad_nd(t, {0, -64})); + ASSERT_TRUE(torch::allclose(actual, expect)); +} + +TEST(FFTTest, fft_norm) { + auto t = torch::randn(128, torch::kComplexDouble); + // NOLINTNEXTLINE(bugprone-argument-comment) + auto unnorm = torch::fft::fft(t, /*n=*/{}, /*axis=*/-1, /*norm=*/{}); + // NOLINTNEXTLINE(bugprone-argument-comment) + auto norm = torch::fft::fft(t, /*n=*/{}, /*axis=*/-1, /*norm=*/"forward"); + ASSERT_TRUE(torch::allclose(unnorm / 128, norm)); + + // NOLINTNEXTLINE(bugprone-argument-comment) + auto ortho_norm = torch::fft::fft(t, /*n=*/{}, /*axis=*/-1, /*norm=*/"ortho"); + ASSERT_TRUE(torch::allclose(unnorm / std::sqrt(128), ortho_norm)); +} + +TEST(FFTTest, ifft) { + auto T = torch::randn(128, torch::kComplexDouble); + auto actual = torch::fft::ifft(T); + auto expect = naive_dft(T, /*forward=*/false) / 128; + ASSERT_TRUE(torch::allclose(actual, expect)); +} + +TEST(FFTTest, fft_ifft) { + auto t = torch::randn(77, torch::kComplexDouble); + auto T = torch::fft::fft(t); + ASSERT_EQ(T.size(0), 77); + ASSERT_EQ(T.scalar_type(), torch::kComplexDouble); + + auto t_round_trip = torch::fft::ifft(T); + ASSERT_EQ(t_round_trip.size(0), 77); + ASSERT_EQ(t_round_trip.scalar_type(), torch::kComplexDouble); + ASSERT_TRUE(torch::allclose(t, t_round_trip)); +} + +TEST(FFTTest, rfft) { + auto t = torch::randn(129, torch::kDouble); + auto actual = torch::fft::rfft(t); + auto expect = torch::fft::fft(t.to(torch::kComplexDouble)).slice(0, 0, 65); + ASSERT_TRUE(torch::allclose(actual, expect)); +} + +TEST(FFTTest, rfft_irfft) { + auto t = torch::randn(128, torch::kDouble); + auto T = torch::fft::rfft(t); + ASSERT_EQ(T.size(0), 65); + ASSERT_EQ(T.scalar_type(), torch::kComplexDouble); + + auto t_round_trip = torch::fft::irfft(T); + ASSERT_EQ(t_round_trip.size(0), 128); + ASSERT_EQ(t_round_trip.scalar_type(), torch::kDouble); + ASSERT_TRUE(torch::allclose(t, t_round_trip)); +} + +TEST(FFTTest, ihfft) { + auto T = torch::randn(129, torch::kDouble); + auto actual = torch::fft::ihfft(T); + auto expect = torch::fft::ifft(T.to(torch::kComplexDouble)).slice(0, 0, 65); + ASSERT_TRUE(torch::allclose(actual, expect)); +} + +TEST(FFTTest, hfft_ihfft) { + auto t = torch::randn(64, torch::kComplexDouble); + t[0] = .5; // Must be purely real to satisfy hermitian symmetry + auto T = torch::fft::hfft(t, 127); + ASSERT_EQ(T.size(0), 127); + ASSERT_EQ(T.scalar_type(), torch::kDouble); + + auto t_round_trip = torch::fft::ihfft(T); + ASSERT_EQ(t_round_trip.size(0), 64); + ASSERT_EQ(t_round_trip.scalar_type(), torch::kComplexDouble); + ASSERT_TRUE(torch::allclose(t, t_round_trip)); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/functional.cpp b/western_ai/usa/pytorch/test/cpp/api/functional.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a9e6bee7dce9fa7d48c7deabceb7cd8ffbb01138 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/functional.cpp @@ -0,0 +1,3301 @@ +#include + +#include +#include + +#include + +namespace F = torch::nn::functional; + +using namespace torch::nn; + +struct FunctionalTest : torch::test::SeedingFixture {}; + +TEST_F(FunctionalTest, Conv1d) { + auto x = torch::arange(30, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({2, 3, 5}); + auto weight = + torch::arange(18, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({2, 3, 3}); + auto y = F::conv1d(x, weight, F::Conv1dFuncOptions().stride(1)); + auto expected = torch::tensor( + {{{312., 348., 384.}, {798., 915., 1032.}}, + + {{852., 888., 924.}, {2553., 2670., 2787.}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, expected)); + + auto y_no_options = F::conv1d(x, weight); + ASSERT_TRUE(torch::allclose(y_no_options, expected)); +} + +TEST_F(FunctionalTest, Conv2dEven) { + auto x = torch::arange(75, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({1, 3, 5, 5}); + auto weight = + torch::arange(54, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({2, 3, 3, 3}); + auto y = F::conv2d(x, weight, F::Conv2dFuncOptions().stride(1)); + auto expected = torch::tensor( + {{{{15219., 15570., 15921.}, + {16974., 17325., 17676.}, + {18729., 19080., 19431.}}, + + {{37818., 38898., 39978.}, + {43218., 44298., 45378.}, + {48618., 49698., 50778.}}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, expected)); + + auto y_no_options = F::conv2d(x, weight); + ASSERT_TRUE(torch::allclose(y_no_options, expected)); +} + +TEST_F(FunctionalTest, Conv2dUneven) { + auto x = torch::arange(60, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({1, 3, 5, 4}); + auto weight = + torch::arange(36, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({2, 3, 3, 2}); + auto y = F::conv2d(x, weight, F::Conv2dFuncOptions().stride(1)); + auto expected = torch::tensor( + {{{{5289., 5442., 5595.}, {5901., 6054., 6207.}, {6513., 6666., 6819.}}, + + {{13227., 13704., 14181.}, + {15135., 15612., 16089.}, + {17043., 17520., 17997.}}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, expected)); + + auto y_no_options = F::conv2d(x, weight); + ASSERT_TRUE(torch::allclose(y_no_options, expected)); +} + +TEST_F(FunctionalTest, Conv3d) { + auto x = torch::arange(375, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({1, 3, 5, 5, 5}); + auto weight = + torch::arange(162, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({2, 3, 3, 3, 3}); + auto y = F::conv3d(x, weight, F::Conv3dFuncOptions().stride(1)); + auto expected = torch::tensor( + {{{{{700704., 703944., 707184.}, + {716904., 720144., 723384.}, + {733104., 736344., 739584.}}, + + {{781704., 784944., 788184.}, + {797904., 801144., 804384.}, + {814104., 817344., 820584.}}, + + {{862704., 865944., 869184.}, + {878904., 882144., 885384.}, + {895104., 898344., 901584.}}}, + + {{{1724220., 1734021., 1743822.}, + {1773225., 1783026., 1792827.}, + {1822230., 1832031., 1841832.}}, + + {{1969245., 1979046., 1988847.}, + {2018250., 2028051., 2037852.}, + {2067255., 2077056., 2086857.}}, + + {{2214270., 2224071., 2233872.}, + {2263275., 2273076., 2282877.}, + {2312280., 2322081., 2331882.}}}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, expected)); + + auto y_no_options = F::conv3d(x, weight); + ASSERT_TRUE(torch::allclose(y_no_options, expected)); +} + +TEST_F(FunctionalTest, MaxPool1d) { + auto x = torch::ones({1, 1, 5}); + auto y = F::max_pool1d(x, F::MaxPool1dFuncOptions(3).stride(2)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({1, 1, 2}))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 2})); +} + +TEST_F(FunctionalTest, MaxPool2d) { + auto x = torch::ones({2, 5, 5}); + auto y = F::max_pool2d(x, F::MaxPool2dFuncOptions(3).stride(2)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2}))); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2})); +} + +TEST_F(FunctionalTest, MaxPool2dBackward) { + auto input = torch::rand( + {1, 2, 4, 4}, torch::dtype(torch::kFloat).requires_grad(true)); + auto output = F::max_pool2d(input, F::MaxPool2dFuncOptions(2)); + auto s = output.sum(); + s.backward(); + ASSERT_TRUE(input.sizes() == input.grad().sizes()); +} + +TEST_F(FunctionalTest, MaxPool3d) { + auto x = torch::ones({2, 5, 5, 5}); + auto y = F::max_pool3d(x, F::MaxPool3dFuncOptions(3).stride(2)); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2, 2}))); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2, 2})); +} + +TEST_F(FunctionalTest, AvgPool1d) { + auto x = torch::ones({1, 1, 5}); + auto y = F::avg_pool1d(x, F::AvgPool1dFuncOptions(3).stride(2)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({1, 1, 2}))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 2})); +} + +TEST_F(FunctionalTest, AvgPool2d) { + auto x = torch::ones({2, 5, 5}); + auto y = F::avg_pool2d(x, F::AvgPool2dFuncOptions(3).stride(2)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2}))); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2})); +} + +TEST_F(FunctionalTest, AvgPool3d) { + auto x = torch::ones({2, 5, 5, 5}); + auto y = F::avg_pool3d(x, F::AvgPool3dFuncOptions(3).stride(2)); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2, 2}))); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2, 2})); +} + +TEST_F(FunctionalTest, FractionalMaxPool2d) { + auto x = torch::ones({2, 5, 5}); + auto y = F::fractional_max_pool2d( + x, F::FractionalMaxPool2dFuncOptions(3).output_size(2)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2}))); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2})); + + auto y_with_indices = F::fractional_max_pool2d_with_indices( + x, F::FractionalMaxPool2dFuncOptions(3).output_size(2)); + ASSERT_TRUE(torch::equal(y, std::get<0>(y_with_indices))); + ASSERT_TRUE(torch::allclose( + std::get<1>(y_with_indices), + torch::tensor({{{0, 2}, {10, 12}}, {{0, 2}, {10, 12}}}))); + ASSERT_EQ( + std::get<1>(y_with_indices).sizes(), std::vector({2, 2, 2})); + + auto x1 = torch::ones({2, 2, 5, 5}); + auto y1 = F::fractional_max_pool2d( + x1, F::FractionalMaxPool2dFuncOptions(3).output_size(2)); + + ASSERT_EQ(y1.ndimension(), 4); + ASSERT_TRUE(torch::allclose(y1, torch::ones({2, 2, 2, 2}))); + ASSERT_EQ(y1.sizes(), std::vector({2, 2, 2, 2})); + + auto y1_with_indices = F::fractional_max_pool2d_with_indices( + x1, F::FractionalMaxPool2dFuncOptions(3).output_size(2)); + ASSERT_TRUE(torch::equal(y1, std::get<0>(y1_with_indices))); + ASSERT_TRUE(torch::allclose( + std::get<1>(y1_with_indices), + torch::tensor( + {{{{0, 2}, {10, 12}}, {{0, 2}, {10, 12}}}, + {{{0, 2}, {10, 12}}, {{0, 2}, {10, 12}}}}))); + ASSERT_EQ( + std::get<1>(y1_with_indices).sizes(), std::vector({2, 2, 2, 2})); +} + +TEST_F(FunctionalTest, FractionalMaxPool3d) { + auto x = torch::ones({2, 5, 5, 5}); + auto y = F::fractional_max_pool3d( + x, F::FractionalMaxPool3dFuncOptions(3).output_size(2)); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2, 2}))); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2, 2})); + + auto y_with_indices = F::fractional_max_pool3d_with_indices( + x, F::FractionalMaxPool3dFuncOptions(3).output_size(2)); + ASSERT_TRUE(torch::equal(y, std::get<0>(y_with_indices))); + ASSERT_TRUE(torch::allclose( + std::get<1>(y_with_indices), + torch::tensor( + {{{{0, 2}, {10, 12}}, {{50, 52}, {60, 62}}}, + {{{0, 2}, {10, 12}}, {{50, 52}, {60, 62}}}}))); + ASSERT_EQ( + std::get<1>(y_with_indices).sizes(), std::vector({2, 2, 2, 2})); +} + +TEST_F(FunctionalTest, LPPool1d) { + int norm_type = 2; + int stride = 2; + int kernel_size = 3; + + auto x = torch::ones({1, 1, 5}); + auto y = F::lp_pool1d( + x, F::LPPool1dFuncOptions(norm_type, kernel_size).stride(stride)); + auto expected = + (torch::pow(torch::tensor({{{1, 1}}}, torch::kFloat), norm_type) * + kernel_size) + .pow(1. / norm_type); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, expected)); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({1, 1, 2})); +} + +TEST_F(FunctionalTest, LPPool2d) { + int norm_type = 2; + int stride = 2; + std::vector kernel_size({2, 3}); + + auto x = torch::ones({1, 1, 2, 5}); + auto y = F::lp_pool2d( + x, F::LPPool2dFuncOptions(norm_type, kernel_size).stride(stride)); + auto expected = + (torch::pow(torch::tensor({{{{1, 1}}}}, torch::kFloat), norm_type) * + (kernel_size[0] * kernel_size[1])) + .pow(1. / norm_type); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose(y, expected)); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({1, 1, 1, 2})); +} + +TEST_F(FunctionalTest, LPPool3d) { + int norm_type = 2; + int stride = 2; + std::vector kernel_size({1, 2, 3}); + + auto x = torch::ones({1, 1, 1, 2, 5}); + auto y = F::lp_pool3d( + x, F::LPPool3dFuncOptions(norm_type, kernel_size).stride(stride)); + auto expected = + (torch::pow(torch::tensor({{{{{1, 1}}}}}, torch::kFloat), norm_type) * + (kernel_size[0] * kernel_size[1] * kernel_size[2])) + .pow(1. / norm_type); + + ASSERT_EQ(y.ndimension(), 5); + ASSERT_TRUE(torch::allclose(y, expected)); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({1, 1, 1, 1, 2})); +} + +TEST_F(FunctionalTest, CosineSimilarity) { + auto input1 = torch::tensor({{1, 2, 3}, {4, 5, 6}}, torch::kFloat); + auto input2 = torch::tensor({{1, 8, 3}, {2, 1, 6}}, torch::kFloat); + auto output = F::cosine_similarity( + input1, input2, F::CosineSimilarityFuncOptions().dim(1)); + auto expected = torch::tensor({0.8078, 0.8721}, torch::kFloat); + ASSERT_TRUE(output.allclose(expected, 1e-04)); +} + +TEST_F(FunctionalTest, SmoothL1LossDefaultOptions) { + auto input = torch::tensor( + {0.1, 1.2, 4.7}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto output = F::smooth_l1_loss(input, target); + auto expected = torch::tensor(0.0233335, torch::kFloat); + auto s = output.sum(); + s.backward(); + ASSERT_TRUE(output.allclose(expected)); + ASSERT_TRUE(input.sizes() == input.grad().sizes()); +} + +TEST_F(FunctionalTest, SmoothL1LossBeta) { + auto input = torch::tensor( + {0.1, 1.5, 10.0}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto output = + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,bugprone-argument-comment) + F::smooth_l1_loss( + input, target, /*reduction=*/torch::kMean, /*beta=*/0.5); + auto expected = torch::tensor(1.67, torch::kFloat); + auto s = output.sum(); + s.backward(); + ASSERT_TRUE(output.allclose(expected)); + ASSERT_TRUE(input.sizes() == input.grad().sizes()); +} + +TEST_F(FunctionalTest, SmoothL1LossBetaOptions) { + auto input = torch::tensor( + {0.1, 1.5, 10.0}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto output = + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) + F::smooth_l1_loss( + input, + target, + F::SmoothL1LossFuncOptions().reduction(torch::kMean).beta(0.5)); + auto expected = torch::tensor(1.67, torch::kFloat); + auto s = output.sum(); + s.backward(); + ASSERT_TRUE(output.allclose(expected)); + ASSERT_TRUE(input.sizes() == input.grad().sizes()); +} + +TEST_F(FunctionalTest, SmoothL1LossNoReduction) { + auto input = torch::tensor( + {0.1, 1.2, 4.7}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto output = + // NOLINTNEXTLINE(bugprone-argument-comment) + F::smooth_l1_loss(input, target, /*reduction=*/torch::kNone); + auto expected = torch::tensor({0.005, 0.02, 0.045}, torch::kFloat); + auto s = output.sum(); + s.backward(); + ASSERT_TRUE(output.allclose(expected)); + ASSERT_TRUE(input.sizes() == input.grad().sizes()); +} + +TEST_F(FunctionalTest, HuberLossDefaultOptions) { + auto input = torch::tensor( + {0.1, 1.2, 4.7}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto output = F::huber_loss(input, target); + auto expected = torch::tensor(0.0233335, torch::kFloat); + auto s = output.sum(); + s.backward(); + ASSERT_TRUE(output.allclose(expected)); + ASSERT_TRUE(input.sizes() == input.grad().sizes()); +} + +TEST_F(FunctionalTest, HuberLossDelta) { + auto input = torch::tensor( + {0.1, 1.5, 10.0}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto options = F::HuberLossFuncOptions().reduction(torch::kMean).delta(0.5); + auto output = F::huber_loss(input, target, options); + auto expected = torch::tensor(1.67 * 0.5, torch::kFloat); + auto s = output.sum(); + s.backward(); + ASSERT_TRUE(output.allclose(expected)); + ASSERT_TRUE(input.sizes() == input.grad().sizes()); +} + +TEST_F(FunctionalTest, HuberLossNoReduction) { + auto input = torch::tensor( + {0.1, 1.2, 4.7}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto options = F::HuberLossFuncOptions().reduction(torch::kNone); + auto output = F::huber_loss(input, target, options); + auto expected = torch::tensor({0.005, 0.02, 0.045}, torch::kFloat); + auto s = output.sum(); + s.backward(); + ASSERT_TRUE(output.allclose(expected)); + ASSERT_TRUE(input.sizes() == input.grad().sizes()); +} + +TEST_F(FunctionalTest, SoftMarginLossDefaultOptions) { + auto input = torch::tensor( + {2., 4., 1., 3.}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({-1., 1., 1., -1.}, torch::kFloat); + auto output = F::soft_margin_loss(input, target); + auto expected = torch::tensor({1.3767317}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(FunctionalTest, MultiLabelSoftMarginLossDefaultOptions) { + auto input = torch::tensor( + {{0., 2., 2., 0.}, {2., 1., 0., 1.}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto target = + torch::tensor({{0., 0., 1., 0.}, {1., 0., 1., 1.}}, torch::kFloat); + auto output = F::multilabel_soft_margin_loss(input, target); + auto expected = torch::tensor({0.7608436}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(FunctionalTest, SoftMarginLossNoReduction) { + auto input = torch::tensor( + {2., 4., 1., 3.}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({-1., 1., 1., -1.}, torch::kFloat); + auto output = F::soft_margin_loss(input, target, torch::kNone); + auto expected = torch::tensor( + {2.1269281, 0.01814993, 0.3132617, 3.0485873}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(FunctionalTest, MultiLabelSoftMarginLossWeightedNoReduction) { + auto input = torch::tensor( + {{0., 2., 2., 0.}, {2., 1., 0., 1.}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto target = + torch::tensor({{0., 0., 1., 0.}, {1., 0., 1., 1.}}, torch::kFloat); + auto weight = torch::tensor({0.1, 0.6, 0.4, 0.8}, torch::kFloat); + auto options = F::MultilabelSoftMarginLossFuncOptions() + .reduction(torch::kNone) + .weight(weight); + auto output = F::multilabel_soft_margin_loss(input, target, options); + auto expected = torch::tensor({0.4876902, 0.3321295}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(FunctionalTest, PairwiseDistance) { + auto input1 = torch::tensor({{1, 2, 3}, {4, 5, 6}}, torch::kFloat); + auto input2 = torch::tensor({{1, 8, 3}, {2, 1, 6}}, torch::kFloat); + auto output = F::pairwise_distance( + input1, input2, F::PairwiseDistanceFuncOptions().p(1)); + auto expected = torch::tensor({6, 6}, torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(FunctionalTest, PDist) { + { + auto input = torch::tensor({{-1.0, -5.0, -1.0}, {2.0, 4.0, 6.0}}); + auto output = F::pdist(input); + auto expected = torch::tensor({11.7898}); + ASSERT_TRUE(output.allclose(expected)); + } + { + auto input = torch::tensor({{1.0, -1.0}, {1.0, 3.0}, {3.0, 3.0}}); + auto output = F::pdist(input, 1.5); + auto expected = torch::tensor({4.0, 4.8945, 2.0}); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(FunctionalTest, AdaptiveMaxPool1d) { + auto x = torch::ones({1, 1, 5}); + auto y = F::adaptive_max_pool1d(x, F::AdaptiveMaxPool1dFuncOptions(3)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({1, 1, 3}))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 3})); +} + +TEST_F(FunctionalTest, AdaptiveMaxPool2d) { + auto x = torch::ones({2, 5, 5}); + auto y = F::adaptive_max_pool2d(x, F::AdaptiveMaxPool2dFuncOptions(3)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 3, 3}))); + ASSERT_EQ(y.sizes(), std::vector({2, 3, 3})); +} + +TEST_F(FunctionalTest, AdaptiveMaxPool3d) { + auto x = torch::ones({2, 5, 5, 5}); + auto y = F::adaptive_max_pool3d(x, F::AdaptiveMaxPool3dFuncOptions(3)); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 3, 3, 3}))); + ASSERT_EQ(y.sizes(), std::vector({2, 3, 3, 3})); +} + +TEST_F(FunctionalTest, AdaptiveAvgPool1d) { + auto x = torch::ones({1, 1, 5}); + auto y = F::adaptive_avg_pool1d(x, F::AdaptiveAvgPool1dFuncOptions(3)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({1, 1, 3}))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 3})); +} + +TEST_F(FunctionalTest, AdaptiveAvgPool2d) { + auto x = torch::ones({2, 5, 5}); + auto y = F::adaptive_avg_pool2d(x, F::AdaptiveAvgPool2dFuncOptions(3)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 3, 3}))); + ASSERT_EQ(y.sizes(), std::vector({2, 3, 3})); +} + +TEST_F(FunctionalTest, AdaptiveAvgPool3d) { + auto x = torch::ones({2, 5, 5, 5}); + auto y = F::adaptive_avg_pool3d(x, F::AdaptiveAvgPool3dFuncOptions(3)); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 3, 3, 3}))); + ASSERT_EQ(y.sizes(), std::vector({2, 3, 3, 3})); +} + +TEST_F(FunctionalTest, L1Loss) { + auto input = torch::randn({5, 6}, torch::requires_grad()); + auto target = torch::empty({5, 6}).random_(2); + auto output = F::l1_loss(torch::sigmoid(input), target); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(output.sizes(), torch::IntArrayRef()); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(FunctionalTest, MSELoss) { + auto input = torch::randn({5, 6}, torch::requires_grad()); + auto target = torch::empty({5, 6}).random_(2); + auto output = F::mse_loss(torch::sigmoid(input), target); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(output.sizes(), torch::IntArrayRef()); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(FunctionalTest, BCELoss) { + auto input = torch::randn({5, 6}, torch::requires_grad()); + auto target = torch::empty({5, 6}).random_(2); + auto output = F::binary_cross_entropy(torch::sigmoid(input), target); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(output.sizes(), torch::IntArrayRef()); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(FunctionalTest, KLDivLoss) { + KLDivLoss loss; + auto input = torch::randn({5, 6}, torch::requires_grad()); + auto target = torch::empty({5, 6}).random_(2); + auto output = F::kl_div(torch::sigmoid(input), target); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(output.sizes(), torch::IntArrayRef()); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(FunctionalTest, HingeEmbeddingLoss) { + auto input = torch::tensor({{2, 22, 4}, {20, 10, 0}}, torch::kFloat); + auto target = torch::tensor({{2, 6, 4}, {1, 10, 0}}, torch::kFloat); + auto output = F::hinge_embedding_loss( + input, target, F::HingeEmbeddingLossFuncOptions().margin(2)); + auto expected = torch::tensor({10}, torch::kFloat); + + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(FunctionalTest, GridSample) { + auto input = + torch::arange(9, torch::kFloat).view(std::vector({1, 1, 3, 3})); + auto grid = torch::tensor( + {{{{-2., -1.}, {-1., -1.}, {0., -1.}}, + {{-1., 0.}, {0., 0.}, {1., 0.}}, + {{0., 1.}, {1., 1.}, {2., 1.}}}}, + torch::kFloat); + + // bilinear, zeros, true + auto options = F::GridSampleFuncOptions() + .mode(torch::kBilinear) + .padding_mode(torch::kZeros) + .align_corners(true); + auto output = F::grid_sample(input, grid, options); + auto expected = torch::tensor( + {{{{0., 0., 1.}, {3., 4., 5.}, {7., 8., 0.}}}}, torch::kFloat); + + ASSERT_TRUE(output.allclose(expected)); + + // bilinear, zeros, false + options = F::GridSampleFuncOptions() + .mode(torch::kBilinear) + .padding_mode(torch::kZeros) + .align_corners(false); + output = F::grid_sample(input, grid, options); + expected = torch::tensor( + {{{{0., 0., 0.5}, {1.5, 4., 2.5}, {3.5, 2., 0.}}}}, torch::kFloat); + + ASSERT_TRUE(output.allclose(expected)); + + // default options (bilinear, zeros, false) same result as above + output = F::grid_sample(input, grid); + + ASSERT_TRUE(output.allclose(expected)); + + // nearest, zeros, true + options = F::GridSampleFuncOptions() + .mode(torch::kNearest) + .padding_mode(torch::kZeros) + .align_corners(true); + output = F::grid_sample(input, grid, options); + expected = torch::tensor( + {{{{0., 0., 1.}, {3., 4., 5.}, {7., 8., 0.}}}}, torch::kFloat); + + ASSERT_TRUE(output.allclose(expected)); + + // bicubic, zeros, true + options = F::GridSampleFuncOptions() + .mode(torch::kBicubic) + .padding_mode(torch::kZeros) + .align_corners(true); + output = F::grid_sample(input, grid, options); + expected = torch::tensor( + {{{{0., 0., 1.}, {3., 4., 5.}, {7., 8., 0.}}}}, torch::kFloat); + + ASSERT_TRUE(output.allclose(expected)); + + // bilinear, border, true + options = F::GridSampleFuncOptions() + .mode(torch::kBilinear) + .padding_mode(torch::kBorder) + .align_corners(true); + output = F::grid_sample(input, grid, options); + expected = torch::tensor( + {{{{0., 0., 1.}, {3., 4., 5.}, {7., 8., 8.}}}}, torch::kFloat); + + ASSERT_TRUE(output.allclose(expected)); + + // bilinear, reflection, true + options = F::GridSampleFuncOptions() + .mode(torch::kBilinear) + .padding_mode(torch::kReflection) + .align_corners(true); + output = F::grid_sample(input, grid, options); + expected = torch::tensor( + {{{{1., 0., 1.}, {3., 4., 5.}, {7., 8., 7.}}}}, torch::kFloat); + + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(FunctionalTest, AffineGrid) { + { + // 2D affine. + auto theta = torch::arange(1., 13).view(std::vector({2, 2, 3})); + auto size = std::vector({2, 3, 2, 2}); + auto align_corners = true; + auto output = F::affine_grid(theta, size, !align_corners); + auto expected = torch::tensor( + {{{{1.50, 1.50}, {2.50, 5.50}}, {{3.50, 6.50}, {4.50, 10.50}}}, + {{{1.50, 1.50}, {8.50, 11.50}}, {{9.50, 12.50}, {16.50, 22.50}}}}); + auto output_aligned = F::affine_grid(theta, size, align_corners); + auto expected_aligned = torch::tensor( + {{{{0.0, -3.0}, {2.0, 5.0}}, {{4.0, 7.0}, {6.0, 15.0}}}, + {{{-6.0, -9.0}, {8.0, 11.0}}, {{10.0, 13.0}, {24.0, 33.0}}}}); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_TRUE(output_aligned.allclose(expected_aligned)); + } + { + // 3D affine. + auto theta = torch::arange(1., 13).view(std::vector({1, 3, 4})); + auto size = std::vector({1, 1, 3, 2, 2}); + auto align_corners = true; + auto output = F::affine_grid(theta, size, !align_corners); + auto expected = torch::tensor( + {{{{{0.5000, -2.1667, -4.8333}, {1.5000, 2.8333, 4.1667}}, + {{2.5000, 3.8333, 5.1667}, {3.5000, 8.8333, 14.1667}}}, + {{{2.5000, 2.5000, 2.5000}, {3.5000, 7.5000, 11.5000}}, + {{4.5000, 8.5000, 12.5000}, {5.5000, 13.5000, 21.5000}}}, + {{{4.5000, 7.1667, 9.8333}, {5.5000, 12.1667, 18.8333}}, + {{6.5000, 13.1667, 19.8333}, {7.5000, 18.1667, 28.8333}}}}}); + auto output_aligned = F::affine_grid(theta, size, align_corners); + auto expected_aligned = torch::tensor( + {{{{{-2.0, -10.0, -18.0}, {0.0, 0.0, 0.0}}, + {{2.0, 2.0, 2.0}, {4.0, 12.0, 20.0}}}, + {{{1.0, -3.0, -7.0}, {3.0, 7.0, 11.0}}, + {{5.0, 9.0, 13.0}, {7.0, 19.0, 31.0}}}, + {{{4.0, 4.0, 4.0}, {6.0, 14.0, 22.0}}, + {{8.0, 16.0, 24.0}, {10.0, 26.0, 42.0}}}}}); + + ASSERT_TRUE(output.allclose(expected, 1e-2)); + ASSERT_TRUE(output_aligned.allclose(expected_aligned)); + } + { + auto theta = torch::empty({1, 2, 3}, torch::kDouble); + auto size = std::vector({1, 1, 2, 2}); + ASSERT_THROWS_WITH( + F::affine_grid(torch::empty({2, 2, 3}), {-1, 1, 2, 2}), + "Expected non-zero, positive output size. Got [-1, 1, 2, 2]"); + ASSERT_THROWS_WITH( + F::affine_grid(torch::empty({2, 2, 3}, torch::kInt), size), + "Expected theta to have floating point type, but got int"); + ASSERT_THROWS_WITH( + F::affine_grid(theta[0], size), + "Expected a batch of 2D affine matrices of shape Nx2x3 for size " + "[1, 1, 2, 2]. Got [2, 3]."); + ASSERT_THROWS_WITH( + F::affine_grid(theta.unsqueeze(0), size), + "Expected a batch of 2D affine matrices of shape Nx2x3 for size " + "[1, 1, 2, 2]. Got [1, 1, 2, 3]."); + ASSERT_THROWS_WITH( + F::affine_grid(theta.repeat({1, 2, 1}), size), + "Expected a batch of 2D affine matrices of shape Nx2x3 for size " + "[1, 1, 2, 2]. Got [1, 4, 3]."); + ASSERT_THROWS_WITH( + F::affine_grid(theta.repeat({1, 1, 2}), size), + "Expected a batch of 2D affine matrices of shape Nx2x3 for size " + "[1, 1, 2, 2]. Got [1, 2, 6]."); + } + { + auto theta = torch::empty({1, 3, 4}, torch::kDouble); + auto size = std::vector({1, 1, 2, 2, 3}); + ASSERT_THROWS_WITH( + F::affine_grid(theta[0], size), + "Expected a batch of 3D affine matrices of shape Nx3x4 for size " + "[1, 1, 2, 2, 3]. Got [3, 4]."); + ASSERT_THROWS_WITH( + F::affine_grid(theta.unsqueeze(0), size), + "Expected a batch of 3D affine matrices of shape Nx3x4 for size " + "[1, 1, 2, 2, 3]. Got [1, 1, 3, 4]."); + ASSERT_THROWS_WITH( + F::affine_grid(theta.repeat({1, 2, 1}), size), + "Expected a batch of 3D affine matrices of shape Nx3x4 for size " + "[1, 1, 2, 2, 3]. Got [1, 6, 4]."); + ASSERT_THROWS_WITH( + F::affine_grid(theta.repeat({1, 1, 2}), size), + "Expected a batch of 3D affine matrices of shape Nx3x4 for size " + "[1, 1, 2, 2, 3]. Got [1, 3, 8]."); + ASSERT_THROWS_WITH( + F::affine_grid(theta, {1, 1, 1, 2, 2, 3}), + "affine_grid only supports 4D and 5D sizes, for 2D and 3D affine " + "transforms, respectively. Got size [1, 1, 1, 2, 2, 3]"); + ASSERT_THROWS_WITH( + F::affine_grid(theta, {1, 1}), + "affine_grid only supports 4D and 5D sizes, for 2D and 3D affine " + "transforms, respectively. Got size [1, 1]"); + } +} + +TEST_F(FunctionalTest, MultiMarginLoss) { + auto weight = torch::tensor({0.3, 0.3, 0.4}, torch::kFloat); + auto input = torch::tensor( + {{0.2, 0.2, 0.6}, {0.1, 0.8, 0.1}, {0.9, 0.09, 0.01}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({2, 1, 0}, torch::kLong); + auto output = F::multi_margin_loss( + input, target, F::MultiMarginLossFuncOptions().margin(2).weight(weight)); + auto expected = torch::tensor({0.305556}, torch::kFloat); + + ASSERT_TRUE(output.allclose(expected, 1e-04)); +} + +TEST_F(FunctionalTest, CosineEmbeddingLoss) { + auto input1 = torch::tensor({{2, 3, 4}, {6, 2, 4}}); + auto input2 = torch::tensor({{2, 3, 5}, {9, 12, 0}}); + auto target = torch::tensor({1, -1}); + auto output = F::cosine_embedding_loss( + input1, input2, target, F::CosineEmbeddingLossFuncOptions().margin(0.5)); + auto expected = torch::tensor({0.1004}, torch::kFloat); + + ASSERT_TRUE(output.allclose(expected, 1e-4)); +} + +TEST_F(FunctionalTest, MultiLabelMarginLossDefaultOptions) { + auto input = torch::tensor( + {{0.1, 0.2, 0.4, 0.8}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({{3, 0, -1, 1}}, torch::kLong); + auto output = F::multilabel_margin_loss(input, target); + auto expected = torch::tensor({0.8500}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(FunctionalTest, MultiLabelMarginLossNoReduction) { + auto input = torch::tensor( + {{0.1, 0.2, 0.4, 0.8}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({{3, 0, -1, 1}}, torch::kLong); + auto output = F::multilabel_margin_loss(input, target, torch::kNone); + auto expected = torch::tensor({0.8500}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(FunctionalTest, TripletMarginLoss) { + auto anchor = torch::tensor({{3., 3.}}, torch::kFloat); + auto positive = torch::tensor({{2., 2.}}, torch::kFloat); + auto negative = torch::tensor({{0., 0.}}, torch::kFloat); + auto output = F::triplet_margin_loss( + anchor, + positive, + negative, + F::TripletMarginLossFuncOptions().margin(1.0)); + auto expected = torch::tensor({0.}, torch::kFloat); + + ASSERT_TRUE(output.allclose(expected, 1e-04)); +} + +TEST_F(FunctionalTest, TripletMarginWithDistanceLossDefaultParity) { + // Check that if we use torch::pairwise_distance with the default + // TripletMarginLoss options as our distance function, the outputs + // are equal (i.e., equal under defaults). + + std::vector reductions = { + torch::kSum, torch::kMean, torch::kNone}; + std::vector margins = {0.5, 1.0, 1.5}; + std::vector swaps = {true, false}; + + for (auto& reduction : reductions) { + for (auto& margin : margins) { + for (const auto& swap : swaps) { + auto anchor = torch::randn( + {100, 128}, torch::dtype(torch::kFloat).requires_grad(true)); + auto positive = torch::randn( + {100, 128}, torch::dtype(torch::kFloat).requires_grad(true)); + auto negative = torch::randn( + {100, 128}, torch::dtype(torch::kFloat).requires_grad(true)); + + auto basicOptions = F::TripletMarginLossFuncOptions() + .reduction(reduction) + .margin(margin) + .swap(swap); + auto distanceOptions = F::TripletMarginWithDistanceLossFuncOptions() + .reduction(reduction) + .margin(margin) + .swap(swap); + TripletMarginLoss basicLoss(basicOptions); + TripletMarginWithDistanceLoss distanceLoss(distanceOptions); + + auto basicOutput = + F::triplet_margin_loss(anchor, positive, negative, basicOptions); + auto distanceOutput = F::triplet_margin_with_distance_loss( + anchor, positive, negative, distanceOptions); + + ASSERT_TRUE(distanceOutput.allclose(basicOutput, 1e-6, 1e-6)); + + // handle for torch::kNone reduction + auto sum = distanceOutput.sum(); + sum.backward(); + ASSERT_EQ(anchor.sizes(), anchor.grad().sizes()); + ASSERT_EQ(positive.sizes(), positive.grad().sizes()); + ASSERT_EQ(negative.sizes(), negative.grad().sizes()); + } + } + } +} + +TEST_F(FunctionalTest, NLLLoss) { + auto input = torch::tensor( + {{-0.1315, -3.1315, -2.5315}, + {-3.7038, -0.1038, -2.6038}, + {-2.3422, -1.3422, -0.4422}}, + torch::kFloat); + auto target = torch::tensor({1, 0, 2}, torch::kLong); + auto output = F::nll_loss( + input, + target, + F::NLLLossFuncOptions().ignore_index(-100).reduction(torch::kMean)); + auto expected = torch::tensor(2.4258, torch::kFloat); + ASSERT_TRUE(output.allclose(expected, 1e-04)); + ASSERT_TRUE(F::nll_loss(input, target).allclose(expected, 1e-04)); +} + +TEST_F(FunctionalTest, CrossEntropy) { + auto input = torch::tensor({{3., 3.}, {2., 2.}}, torch::kFloat); + auto target = torch::tensor({0, 1}, torch::kLong); + auto output = F::cross_entropy( + input, + target, + F::CrossEntropyFuncOptions().ignore_index(-100).reduction(torch::kMean)); + auto expected = torch::tensor(0.6931, torch::kFloat); + + ASSERT_TRUE(output.allclose(expected, 1e-04)); + ASSERT_TRUE(F::cross_entropy(input, target).allclose(expected, 1e-04)); + + // label smoothing with class indices + input = torch::tensor({{3., 1.}, {1., 2.}}, torch::kFloat); + output = F::cross_entropy( + input, + target, + F::CrossEntropyFuncOptions().label_smoothing(0.15).reduction( + torch::kMean)); + expected = torch::tensor(0.3326, torch::kFloat); + ASSERT_TRUE(output.allclose(expected, 1e-04)); + + // label smoothing with target probabilities + target = torch::tensor({{0.8, 0.2}, {0.1, 0.9}}, torch::kFloat); + output = F::cross_entropy( + input, + target, + F::CrossEntropyFuncOptions().label_smoothing(0.2).reduction( + torch::kMean)); + expected = torch::tensor(0.5701, torch::kFloat); + ASSERT_TRUE(output.allclose(expected, 1e-04)); +} + +TEST_F(FunctionalTest, MaxUnpool1d) { + auto x = torch::tensor( + {{{2, 4, 5}}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto indices = torch::tensor({{{1, 3, 4}}}, torch::kLong); + auto y = F::max_unpool1d(x, indices, F::MaxUnpool1dFuncOptions(3)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose( + y, torch::tensor({{{0, 2, 0, 4, 5, 0, 0, 0, 0}}}, torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 9})); + + x = torch::tensor( + {{2, 4, 5}}, torch::dtype(torch::kFloat).requires_grad(true)); + indices = torch::tensor({{1, 3, 4}}, torch::kLong); + y = F::max_unpool1d(x, indices, F::MaxUnpool1dFuncOptions(3)); + + ASSERT_EQ(y.ndimension(), 2); + ASSERT_TRUE(torch::allclose( + y, torch::tensor({{0, 2, 0, 4, 5, 0, 0, 0, 0}}, torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 9})); + + x = torch::tensor( + {{{2, 4, 5}}}, torch::dtype(torch::kFloat).requires_grad(true)); + indices = torch::tensor({{{1, 3, 4}}}, torch::kLong); + y = F::max_unpool1d( + x, + indices, + F::MaxUnpool1dFuncOptions(3).output_size( + std::vector({1, 1, 9}))); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose( + y, torch::tensor({{{0, 2, 0, 4, 5, 0, 0, 0, 0}}}, torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 9})); + + x = torch::tensor( + {{{2, 4, 5}}}, torch::dtype(torch::kFloat).requires_grad(true)); + indices = torch::tensor({{{1, 3, 4}}}, torch::kLong); + y = F::max_unpool1d( + x, indices, F::MaxUnpool1dFuncOptions(3).stride(2).padding(1)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE( + torch::allclose(y, torch::tensor({{{0, 2, 0, 4, 5}}}, torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 5})); +} + +TEST_F(FunctionalTest, MaxUnpool2d) { + auto indices = torch::tensor( + {{{{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}}, + {{{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}}}, + torch::kLong); + auto x = torch::tensor( + {{{{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}}, + {{{31, 33, 34}, {41, 43, 44}, {46, 48, 49}}}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto y = F::max_unpool2d( + x, indices, F::MaxUnpool2dFuncOptions(3).stride(2).padding(1)); + + ASSERT_EQ(y.dim(), 4); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + {{{{0, 0, 0, 0, 0}, + {0, 6, 0, 8, 9}, + {0, 0, 0, 0, 0}, + {0, 16, 0, 18, 19}, + {0, 21, 0, 23, 24}}}, + {{{0, 0, 0, 0, 0}, + {0, 31, 0, 33, 34}, + {0, 0, 0, 0, 0}, + {0, 41, 0, 43, 44}, + {0, 46, 0, 48, 49}}}}, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({2, 1, 5, 5})); + + indices = torch::tensor( + {{{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}, + {{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}}, + torch::kLong); + x = torch::tensor( + {{{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}, + {{31, 33, 34}, {41, 43, 44}, {46, 48, 49}}}, + torch::dtype(torch::kFloat).requires_grad(true)); + y = F::max_unpool2d( + x, indices, F::MaxUnpool2dFuncOptions(3).stride(2).padding(1)); + + ASSERT_EQ(y.dim(), 3); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + {{{0, 0, 0, 0, 0}, + {0, 6, 0, 8, 9}, + {0, 0, 0, 0, 0}, + {0, 16, 0, 18, 19}, + {0, 21, 0, 23, 24}}, + {{0, 0, 0, 0, 0}, + {0, 31, 0, 33, 34}, + {0, 0, 0, 0, 0}, + {0, 41, 0, 43, 44}, + {0, 46, 0, 48, 49}}}, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({2, 5, 5})); +} + +TEST_F(FunctionalTest, MaxUnpool3d) { + auto indices = torch::tensor({{{{{26}}}}}, torch::kLong); + auto x = torch::tensor( + {{{{{26}}}}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto y = F::max_unpool3d(x, indices, F::MaxUnpool3dFuncOptions(3)); + + ASSERT_EQ(y.dim(), 5); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + {{{{{0, 0, 0}, {0, 0, 0}, {0, 0, 0}}, + {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}}, + {{0, 0, 0}, {0, 0, 0}, {0, 0, 26}}}}}, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 3, 3, 3})); + + indices = torch::tensor({{{{26}}}}, torch::kLong); + x = torch::tensor( + {{{{26}}}}, torch::dtype(torch::kFloat).requires_grad(true)); + y = F::max_unpool3d(x, indices, F::MaxUnpool3dFuncOptions(3)); + + ASSERT_EQ(y.dim(), 4); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + {{{{0, 0, 0}, {0, 0, 0}, {0, 0, 0}}, + {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}}, + {{0, 0, 0}, {0, 0, 0}, {0, 0, 26}}}}, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 3, 3, 3})); +} + +TEST_F(FunctionalTest, ELU) { + const auto size = 3; + for (const auto inplace : {false, true}) { + for (const auto alpha : {0.0, 0.42, 1.0, 4.2, 42.42}) { + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + auto x_bf16 = + torch::linspace(-10.0, 10.0, size * size * size).to(torch::kBFloat16); + x_bf16.resize_({size, size, size}); + + auto y_exp = torch::max(torch::zeros_like(x), x) + + torch::min(torch::zeros_like(x), alpha * (torch::expm1(x))); + auto y = F::elu(x, F::ELUFuncOptions().alpha(alpha).inplace(inplace)); + auto y_bf16 = + F::elu(x_bf16, F::ELUFuncOptions().alpha(alpha).inplace(inplace)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + ASSERT_TRUE(torch::allclose(y_bf16.to(torch::kFloat), y, 1e-2, 1e-2)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + ASSERT_TRUE(torch::allclose(x_bf16.to(torch::kFloat), y, 1e-2, 1e-2)); + } + } + } + ASSERT_TRUE(F::elu(torch::tensor(1.)).defined()); +} + +TEST_F(FunctionalTest, SELU) { + { + const double scale = 1.0507009873554804934193349852946; + const double alpha = 1.6732632423543772848170429916717; + for (const auto inplace : {false, true}) { + auto input = torch::randn({5, 5}); + auto input_bf16 = input.clone().to(torch::kBFloat16); + auto expected = scale * + (torch::max(torch::zeros_like(input), input) + + torch::min(torch::zeros_like(input), alpha * (torch::expm1(input)))); + auto output = F::selu(input, inplace); + auto output_bf16 = F::selu(input_bf16, inplace); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_TRUE(output_bf16.to(torch::kFloat).allclose(output, 1e-2, 1e-2)); + if (inplace) { + ASSERT_TRUE(input.allclose(expected)); + ASSERT_TRUE(input_bf16.to(torch::kFloat).allclose(output, 1e-2, 1e-2)); + } + } + } + { + auto input = torch::arange(0, 9, torch::kDouble).view({3, 3}); + auto output = F::selu(input); + auto expected = F::selu(input, false); + ASSERT_TRUE(output.allclose(expected)); + } + ASSERT_TRUE(F::selu(torch::tensor(1.)).defined()); +} + +TEST_F(FunctionalTest, GLU) { + int64_t dim = 1; + auto input = torch::randn({4, 2}, torch::requires_grad()); + auto output = F::glu(input, dim); + auto input_size = input.sizes()[dim] / 2; + auto first_half = input.narrow(dim, 0, input_size); + auto second_half = input.narrow(dim, input_size, input_size); + auto expected = first_half * torch::sigmoid(second_half); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_TRUE(F::glu(input).allclose(expected)); +} + +TEST_F(FunctionalTest, GELU) { + const auto x = torch::linspace(-3.0, 3.0, 100); + const auto y_exp = x * 0.5 * (1.0 + torch::erf(x / std::sqrt(2.0))); + const auto y = F::gelu(x, F::GELUFuncOptions().approximate("none")); + ASSERT_TRUE(torch::allclose(y, y_exp, 1.4e-06, 1e-05)); +} + +TEST_F(FunctionalTest, TanhGELU) { + const auto x = torch::linspace(-3.0, 3.0, 100); + const auto inner = std::sqrt(2 / M_PI) * (x + 0.044715 * x.pow(3.0)); + const auto y_exp = 0.5 * x * (1.0 + inner.tanh()); + const auto y = F::gelu(x, F::GELUFuncOptions().approximate("tanh")); + ASSERT_TRUE(torch::allclose(y, y_exp, 1.4e-06, 1e-05)); +} + +TEST_F(FunctionalTest, Hardshrink) { + const auto size = 3; + for (const auto lambda : {-4.2, -1.0, -0.42, 0.0, 0.42, 1.0, 4.2, 42.42}) { + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}).set_requires_grad(true); + auto y = F::hardshrink(x, F::HardshrinkFuncOptions().lambda(lambda)); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = (x.abs() > lambda) * x; + ASSERT_TRUE(torch::allclose(y, y_exp)); + } +} + +TEST_F(FunctionalTest, OneHot) { + { // Test #1 + auto x = torch::arange(0, 5, torch::kLong); + auto y = F::one_hot(x % 3); + auto expected = torch::tensor( + {{1, 0, 0}, {0, 1, 0}, {0, 0, 1}, {1, 0, 0}, {0, 1, 0}}, torch::kLong); + + ASSERT_EQ(y.ndimension(), 2); + ASSERT_TRUE(torch::allclose(y, expected)); + ASSERT_EQ(y.sizes(), std::vector({5, 3})); + } + + { // Test #2 + auto x = torch::arange(0, 5, torch::kLong); + auto y = F::one_hot(x % 3, 5); + auto expected = torch::tensor( + {{1, 0, 0, 0, 0}, + {0, 1, 0, 0, 0}, + {0, 0, 1, 0, 0}, + {1, 0, 0, 0, 0}, + {0, 1, 0, 0, 0}}, + torch::kLong); + + ASSERT_EQ(y.ndimension(), 2); + ASSERT_TRUE(torch::allclose(y, expected)); + ASSERT_EQ(y.sizes(), std::vector({5, 5})); + } + + { // Test #3 + auto x = torch::arange(0, 6, torch::kLong); + auto y = F::one_hot(x.view(std::vector({3, 2})) % 3); + auto expected = torch::tensor( + {{{1, 0, 0}, {0, 1, 0}}, + {{0, 0, 1}, {1, 0, 0}}, + {{0, 1, 0}, {0, 0, 1}}}, + torch::kLong); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, expected)); + ASSERT_EQ(y.sizes(), std::vector({3, 2, 3})); + } +} + +TEST_F(FunctionalTest, Hardtanh) { + const auto size = 3; + for (const auto min_val : {-4.2, -1.0, -0.42, 0.0}) { + for (const auto max_val : {0.0, 0.42, 1.0, 4.2}) { + for (const auto inplace : {false, true}) { + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + auto y_exp = (x < min_val) * min_val + + ((x >= min_val) * (x <= max_val)) * x + (x > max_val) * max_val; + auto y = F::hardtanh( + x, + F::HardtanhFuncOptions().min_val(min_val).max_val(max_val).inplace( + inplace)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } + } + } + } + ASSERT_TRUE(F::hardtanh(torch::tensor(1.)).defined()); +} + +TEST_F(FunctionalTest, LeakyReLU) { + const auto size = 3; + for (const auto negative_slope : {0.0, 0.42, 1.0}) { + for (const auto inplace : {false, true}) { + for (const auto type : {torch::kFloat, torch::kBFloat16}) { + auto x = torch::linspace(-10.0, 10.0, size * size * size).to(type); + x.resize_({size, size, size}); + auto y_exp = (x < 0) * x * negative_slope + (x >= 0) * x; + auto y = F::leaky_relu( + x, + F::LeakyReLUFuncOptions() + .negative_slope(negative_slope) + .inplace(inplace)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } + } + } + } + ASSERT_TRUE(F::leaky_relu(torch::tensor(1.)).defined()); +} + +TEST_F(FunctionalTest, LogSigmoid) { + const auto size = 3; + LogSigmoid model; + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + auto y = F::logsigmoid(x); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = torch::log( + torch::ones_like(x) / (torch::ones_like(x) + torch::exp(torch::neg(x)))); + ASSERT_TRUE(torch::allclose(y, y_exp, 1e-4, 1e-7)); +} + +TEST_F(FunctionalTest, GumbelSoftmax) { + // Test 1: No-options + { + auto logits = torch::randn({5}); + int expected_count = 1; + auto y_draw = F::gumbel_softmax(logits); + + // All values positive + ASSERT_GE(y_draw.min().item(), 0); + // Shape unchanged + ASSERT_TRUE(y_draw.sizes() == logits.sizes()); + // One choice per draw + ASSERT_TRUE(torch::allclose( + y_draw.sum(), torch::tensor(expected_count, torch::kFloat))); + } + + // Test 2: 1D shape, 0 and -1 dim + for (const auto dim : {0, -1}) { + auto logits = torch::randn({5}); + int expected_count = 1; + auto y_draw = F::gumbel_softmax( + logits, F::GumbelSoftmaxFuncOptions().hard(true).dim(dim)); + + // All values positive + ASSERT_GE(y_draw.min().item(), 0); + // Shape unchanged + ASSERT_TRUE(y_draw.sizes() == logits.sizes()); + // One choice per draw + ASSERT_TRUE(torch::allclose( + y_draw.sum(), torch::tensor(expected_count, torch::kFloat))); + } + + { // Test 3: 2D shape, 1 dim + auto logits = torch::randn({5, 4}); + int expected_count = 5; + auto y_draw = F::gumbel_softmax( + logits, F::GumbelSoftmaxFuncOptions().hard(true).dim(1)); + + // All values positive + ASSERT_GE(y_draw.min().item(), 0); + // Shape unchanged + ASSERT_TRUE(y_draw.sizes() == logits.sizes()); + // One choice per draw + ASSERT_TRUE(torch::allclose( + y_draw.sum(), torch::tensor(expected_count, torch::kFloat))); + } + + // Test 4: 3D shape, 1 and -1 dim + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) + int dims[] = {1, -1}; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-avoid-magic-numbers) + int expected[] = {5 * 3, 5 * 4}; + for (const auto i : c10::irange(2)) { + auto logits = torch::randn({5, 4, 3}); + int expected_count = expected[i]; + auto y_draw = F::gumbel_softmax( + logits, F::GumbelSoftmaxFuncOptions().hard(true).dim(dims[i])); + + // All values positive + ASSERT_GE(y_draw.min().item(), 0); + // Shape unchanged + ASSERT_TRUE(y_draw.sizes() == logits.sizes()); + // One choice per draw + ASSERT_TRUE(torch::allclose( + y_draw.sum(), torch::tensor(expected_count, torch::kFloat))); + } + + { // Test 5: Straight through + int num_draws = 100; + auto logits = torch::tensor({{0.2, 0.8, 0.1}}); + logits = logits.reshape({1, 3}); + logits.requires_grad(); + auto probs = logits.softmax(-1); + + auto counts = torch::zeros_like(logits); + torch::Tensor y_draw; + for ([[maybe_unused]] const auto i : c10::irange(num_draws)) { + y_draw = + F::gumbel_softmax(logits, F::GumbelSoftmaxFuncOptions().hard(true)); + counts += y_draw; + } + + // All values positive + ASSERT_GE(y_draw.min().item(), 0); + // Each experiment should result in 1 draw + ASSERT_EQ(counts.sum().item(), num_draws); + + // Check results are asymptotically as expected + auto expected = probs * num_draws; + // ~z is approximately N(0,1) for unbiased count + auto z = (counts - expected) / (expected * (1 - probs)).sqrt(); + // A (lazy) approximate 99% two-sided test: + // occurs with prob alpha~>=0.01 if unbiased + ASSERT_LT(z.abs().max().item(), 2.58); + } +} + +TEST_F(FunctionalTest, Softmax) { + auto input = torch::arange(10, torch::kFloat).reshape({2, 5}); + // NOLINTNEXTLINE(bugprone-argument-comment) + auto output = F::softmax(input, /*dim=*/1); + auto sum = torch::sum(torch::exp(input), 1); + + for (const auto i : c10::irange(2)) { + auto expected = torch::exp(input[i]) / sum[i]; + ASSERT_TRUE(torch::allclose(output[i], expected)); + } +} + +TEST_F(FunctionalTest, Softmin) { + auto input = torch::arange(10, torch::kFloat).reshape({2, 5}); + // NOLINTNEXTLINE(bugprone-argument-comment) + auto output = F::softmin(input, /*dim=*/1); + auto sum = torch::sum(torch::exp(-input), 1); + + for (const auto i : c10::irange(2)) { + auto expected = torch::exp(-input[i]) / sum[i]; + ASSERT_TRUE(torch::allclose(output[i], expected)); + } +} + +TEST_F(FunctionalTest, LogSoftmax) { + auto input = torch::arange(10, torch::kFloat).reshape({2, 5}); + // NOLINTNEXTLINE(bugprone-argument-comment) + auto output = F::log_softmax(input, /*dim=*/1); + auto sum = torch::sum(torch::exp(input), 1); + + for (const auto i : c10::irange(2)) { + auto expected = torch::log(torch::exp(input[i]) / sum[i]); + ASSERT_TRUE(torch::allclose(output[i], expected)); + } +} + +TEST_F(FunctionalTest, PReLU) { + const auto x = torch::rand({42, 24}) * 200 - 100; + const auto w = torch::rand(24) * 200 - 100; + const auto y = F::prelu(x, w); + ASSERT_EQ(y.sizes(), std::vector({42, 24})); + const auto y_exp = (x < 0) * w * x + (x >= 0) * x; + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(FunctionalTest, LayerNorm) { + const auto input = torch::randn({2, 2}); + auto y = F::layer_norm(input, F::LayerNormFuncOptions({2, 2}).eps(2e-5)); + auto y_exp = + torch::layer_norm(input, {2, 2}, torch::Tensor(), torch::Tensor(), 2e-5); + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(FunctionalTest, GroupNorm) { + const auto input = torch::randn({2, 2}); + auto y = F::group_norm(input, F::GroupNormFuncOptions(2).eps(2e-5)); + auto y_exp = + torch::group_norm(input, 2, torch::Tensor(), torch::Tensor(), 2e-5); + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(FunctionalTest, LocalResponseNorm) { + const auto x = torch::arange(100, 118).resize_({3, 3, 2}); + const auto y = F::local_response_norm(x, F::LocalResponseNormFuncOptions(2)); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({3, 3, 2})); + const auto y_exp = torch::tensor( + {{{73.7788, 74.1462}, {60.1942, 60.3302}, {60.4609, 60.5865}}, + {{75.8729, 76.2011}, {60.9331, 61.0390}, {61.1403, 61.2370}}, + {{77.7387, 78.0303}, {61.5011, 61.5807}, {61.6563, 61.7279}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, y_exp, 1e-4, 1e-7)); +} + +TEST_F(FunctionalTest, Linear) { + { + const auto x = torch::arange(100., 118).resize_({3, 3, 2}); + const auto w = torch::arange(200., 206).resize_({3, 2}); + const auto b = torch::arange(300., 303); + const auto y = F::linear(x, w, b); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({3, 3, 3})); + const auto y_exp = torch::tensor( + {{{40601, 41004, 41407}, {41403, 41814, 42225}, {42205, 42624, 43043}}, + {{43007, 43434, 43861}, {43809, 44244, 44679}, {44611, 45054, 45497}}, + {{45413, 45864, 46315}, {46215, 46674, 47133}, {47017, 47484, 47951}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, y_exp)); + } + { + const auto x = torch::arange(100., 118).resize_({3, 3, 2}); + const auto w = torch::arange(200., 206).resize_({3, 2}); + const auto y = F::linear(x, w); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({3, 3, 3})); + const auto y_exp = torch::tensor( + {{{40301, 40703, 41105}, {41103, 41513, 41923}, {41905, 42323, 42741}}, + {{42707, 43133, 43559}, {43509, 43943, 44377}, {44311, 44753, 45195}}, + {{45113, 45563, 46013}, {45915, 46373, 46831}, {46717, 47183, 47649}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, y_exp)); + } +} + +TEST_F(FunctionalTest, Embedding) { + const auto input = torch::tensor({{1, 2, 4, 5}, {4, 3, 2, 9}}, torch::kLong); + auto weight = torch::empty({10, 3}); + torch::nn::init::normal_(weight); + auto y = F::embedding(input, weight); + auto y_exp = torch::embedding(weight, input.contiguous(), -1, false, false); + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(FunctionalTest, EmbeddingBag) { + const auto input = torch::tensor({1, 2, 4, 5, 4, 3, 2, 9}, torch::kLong); + auto offsets = torch::tensor({0, 4}, torch::kLong); + auto weight = torch::empty({10, 3}); + torch::nn::init::normal_(weight); + auto y = F::embedding_bag( + input, + weight, + F::EmbeddingBagFuncOptions() + .mode(torch::kSum) + .offsets(offsets) + .padding_idx(4)); + auto y_exp = std::get<0>(torch::embedding_bag( + weight, input, offsets, false, 0, false, torch::Tensor(), false, 4)); + ASSERT_TRUE(torch::allclose(y, y_exp)); + + // no options test + const auto input_ = torch::tensor({{1, 2, 4, 5}, {4, 3, 2, 9}}, torch::kLong); + auto offsets_ = torch::arange( + 0, + input_.numel(), + input_.size(1), + torch::TensorOptions().dtype(torch::kLong).device(input.device())); + y = F::embedding_bag(input_, weight); + y_exp = std::get<0>(torch::embedding_bag( + weight, input_.reshape(-1), offsets_, false, 1, false, torch::Tensor())); + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(FunctionalTest, Bilinear) { + auto input1 = torch::tensor({{1, 2, 3}, {7, 6, 5}}); + auto input2 = torch::tensor({{7, 4}, {8, 9}}); + auto weight = torch::tensor({{{2, 3}, {9, 7}, {8, 6}}}); + auto bias = torch::tensor({1}); + + auto y_with_bias = F::bilinear(input1, input2, weight, bias); + ASSERT_EQ(y_with_bias.ndimension(), 2); + ASSERT_EQ(y_with_bias.sizes(), torch::IntArrayRef({2, 1})); + auto y_with_bias_exp = torch::tensor({{449}, {1702}}).reshape({2, 1}); + ASSERT_TRUE(torch::allclose(y_with_bias, y_with_bias_exp, 1e-4, 1e-7)); + + auto y_no_bias = F::bilinear(input1, input2, weight); + ASSERT_EQ(y_no_bias.ndimension(), 2); + ASSERT_EQ(y_no_bias.sizes(), torch::IntArrayRef({2, 1})); + auto y_no_bias_exp = torch::tensor({{448, 1701}}).reshape({2, 1}); + ASSERT_TRUE(torch::allclose(y_no_bias, y_no_bias_exp, 1e-4, 1e-7)); + + input1 = input1.to(torch::kFloat64); + input2 = input2.to(torch::kInt32); + weight = weight.to(torch::kInt32); + ASSERT_THROWS_WITH( + F::bilinear(input1, input2, weight), + "All tensors must have the same dtype, got input1: double, input2: int, weight: int"); +} + +TEST_F(FunctionalTest, Normalize) { + const auto expected = torch::tensor( + {{{0.00000000, 0.10000000, 0.2000, 0.30000000, 0.40000000}, + {0.14285715, 0.17142858, 0.2000, 0.22857143, 0.25714287}}}, + torch::requires_grad().dtype(torch::kFloat)); + { // Test #1 + auto input = torch::tensor( + {{{0, 1, 2, 3, 4}, {5, 6, 7, 8, 9}}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto norm = F::normalize(input, F::NormalizeFuncOptions().p(1).dim(-1)); + + // reduce to scalar to call .backward() + torch::Tensor s = norm.sum(); + s.backward(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(input.grad().numel(), 10); + ASSERT_TRUE(torch::allclose(norm, expected)); + } + + { // Test #2 Check variations of optional arguments + auto input = torch::tensor( + {{{0, 1, 2, 3, 4}, {5, 6, 7, 8, 9}}}, torch::dtype(torch::kFloat)); + auto output = torch::randn({1, 2, 5}, torch::dtype(torch::kFloat)); + // non-null output argument + F::normalize(input, F::NormalizeFuncOptions().p(1).dim(-1).out(output)); + // default options + F::normalize(input); + + ASSERT_TRUE(torch::allclose(output, expected)); + } + + { // Test #3 Base case of scalar tensor + auto input = torch::randn({}, torch::requires_grad()); + torch::Tensor norm = + F::normalize(input, F::NormalizeFuncOptions().p(1).dim(-1)); + norm.backward(); + + ASSERT_EQ(input.grad().numel(), 1); + } +} + +TEST_F(FunctionalTest, ReLU) { + const auto size = 3; + for (const auto inplace : {false, true}) { + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + auto y_exp = (x < 0) * 0 + (x >= 0) * x; + auto y = F::relu(x, F::ReLUFuncOptions().inplace(inplace)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } + + // NOLINTNEXTLINE(bugprone-argument-comment) + y = F::relu(x, /*inplace=*/inplace); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } + } + ASSERT_TRUE(F::relu(torch::tensor(1.)).defined()); +} + +TEST_F(FunctionalTest, ReLUDefaultOptions) { + const auto size = 3; + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + auto y_exp = (x < 0) * 0 + (x >= 0) * x; + auto y = F::relu(x); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(FunctionalTest, ReLU6) { + const auto size = 3; + for (const auto inplace : {false, true}) { + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + auto y_exp = (x < 0) * 0 + ((x >= 0) * (x <= 6)) * x + (x > 6) * 6; + auto y = F::relu6(x, F::ReLU6FuncOptions().inplace(inplace)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } + + // NOLINTNEXTLINE(bugprone-argument-comment) + y = F::relu6(x, /*inplace=*/inplace); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } + } + ASSERT_TRUE(F::relu6(torch::tensor(1.)).defined()); +} + +TEST_F(FunctionalTest, ReLU6DefaultOptions) { + const auto size = 3; + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + auto y_exp = (x < 0) * 0 + ((x >= 0) * (x <= 6)) * x + (x > 6) * 6; + auto y = F::relu6(x); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(FunctionalTest, RReLU) { + const auto size = 3; + for (const auto lower : {0.01, 0.1, 0.2}) { + for (const auto upper : {0.3, 0.4, 0.5}) { + for (const auto inplace : {false, true}) { + for (const auto type : {torch::kFloat, torch::kBFloat16}) { + auto x = torch::linspace(-10.0, 10.0, size * size * size).to(type); + x.resize_({size, size, size}); + auto x_copy = x.clone(); + auto y = F::rrelu( + x, + F::RReLUFuncOptions().lower(lower).upper(upper).inplace(inplace)); + auto z = + ((x_copy >= 0) * (x_copy == y) + + (x_copy < 0) * (y >= x_copy * upper) * (y <= lower * x_copy)) * + 1.0; + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(z, torch::ones_like(z))); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y)); + } + } + } + } + } + ASSERT_TRUE(F::rrelu(torch::tensor(1.)).defined()); +} + +TEST_F(FunctionalTest, RReLUDefaultOptions) { + const auto size = 3; + const auto lower = 1.0 / 8.0; + const auto upper = 1.0 / 3.0; + for (const auto type : {torch::kFloat, torch::kBFloat16}) { + auto x = torch::linspace(-10.0, 10.0, size * size * size).to(type); + x.resize_({size, size, size}); + auto x_copy = x.clone(); + auto y = F::rrelu(x); + auto z = ((x_copy >= 0) * (x_copy == y) + + (x_copy < 0) * (y >= x_copy * upper) * (y <= lower * x_copy)) * + 1.0; + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(z, torch::ones_like(z))); + } +} + +TEST_F(FunctionalTest, CELU) { + const auto size = 3; + for (const auto inplace : {false, true}) { + for (const auto alpha : {0.42, 1.0, 4.2, 42.42}) { + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + auto x_bf16 = x.clone().to(torch::kBFloat16); + auto y_exp = torch::max(torch::zeros_like(x), x) + + torch::min(torch::zeros_like(x), alpha * (torch::expm1(x / alpha))); + auto y = F::celu(x, F::CELUFuncOptions().alpha(alpha).inplace(inplace)); + auto y_bf16 = + F::celu(x_bf16, F::CELUFuncOptions().alpha(alpha).inplace(inplace)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + ASSERT_TRUE(torch::allclose(y_bf16.to(torch::kFloat), y, 1e-2, 1e-2)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + ASSERT_TRUE(torch::allclose(x_bf16.to(torch::kFloat), y, 1e-2, 1e-2)); + } + } + } + ASSERT_TRUE(F::celu(torch::tensor(1.)).defined()); +} + +TEST_F(FunctionalTest, CELUDefaultOptions) { + const auto size = 3; + const auto alpha = 1.0; + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + auto x_bf16 = x.clone().to(torch::kBFloat16); + auto y_exp = torch::max(torch::zeros_like(x), x) + + torch::min(torch::zeros_like(x), alpha * (torch::expm1(x / alpha))); + auto y = F::celu(x); + auto y_bf16 = F::celu(x_bf16); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + ASSERT_TRUE(torch::allclose(y_bf16.to(torch::kFloat), y, 1e-2, 1e-2)); +} + +TEST_F(FunctionalTest, PixelShuffle) { + auto x = torch::tensor( + {{{{-17, 19}, {-1, 2}}, + {{7, 14}, {-3, 1}}, + {{0, -2}, {-12, 14}}, + {{-15, 0}, {-3, 9}}}}, + torch::kFloat); + auto y_exp = torch::tensor( + {{{{-17, 7, 19, 14}, {0, -15, -2, 0}, {-1, -3, 2, 1}, {-12, -3, 14, 9}}}}, + torch::kFloat); + auto y = F::pixel_shuffle(x, 2); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({1, 1, 4, 4})); + ASSERT_TRUE(y.allclose(y_exp)); +} + +TEST_F(FunctionalTest, PixelUnshuffle) { + auto x = torch::tensor( + {{{{-17, 7, 19, 14}, {0, -15, -2, 0}, {-1, -3, 2, 1}, {-12, -3, 14, 9}}}}, + torch::kFloat); + auto y_exp = torch::tensor( + {{{{-17, 19}, {-1, 2}}, + {{7, 14}, {-3, 1}}, + {{0, -2}, {-12, 14}}, + {{-15, 0}, {-3, 9}}}}, + torch::kFloat); + auto y = F::pixel_unshuffle(x, 2); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({1, 4, 2, 2})); + ASSERT_TRUE(y.allclose(y_exp)); +} + +TEST_F(FunctionalTest, Softplus) { + const auto size = 3; + for (const auto beta : {0.5, 1.0, 2.0}) { + for (const auto threshold : {1.0, 3.0, 5.0}) { + auto x = torch::linspace(-3.0, 3.0, 61); + x.resize_({size, size, size}); + auto y_exp = + (x <= threshold) * torch::log(1 + torch::exp(x * beta)) / beta + + (x > threshold) * x; + auto y = F::softplus( + x, F::SoftplusFuncOptions().beta(beta).threshold(threshold)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + } + } +} + +TEST_F(FunctionalTest, SoftplusDefaultOptions) { + const auto size = 3; + const auto beta = 1.0; + const auto threshold = 20.0; + auto x = torch::linspace(-3.0, 3.0, 61); + x.resize_({size, size, size}); + auto y_exp = (x <= threshold) * torch::log(1 + torch::exp(x * beta)) / beta + + (x > threshold) * x; + auto y = F::softplus(x); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(FunctionalTest, Fold) { + auto input = torch::ones({1, 3 * 2 * 2, 2}, torch::kDouble); + auto output = F::fold(input, F::FoldFuncOptions({3, 2}, {2, 2})); + auto expected = torch::tensor( + {{{{1.0, 1.0}, {2.0, 2.0}, {1.0, 1.0}}, + {{1.0, 1.0}, {2.0, 2.0}, {1.0, 1.0}}, + {{1.0, 1.0}, {2.0, 2.0}, {1.0, 1.0}}}}, + torch::kDouble); + + ASSERT_EQ(output.sizes(), std::vector({1, 3, 3, 2})); + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(FunctionalTest, Unfold) { + auto input = torch::arange(0, 12, torch::kDouble).view({1, 2, 2, 3}); + auto output = + F::unfold(input, F::UnfoldFuncOptions({2, 2}).padding(1).stride(2)); + auto expected = torch::tensor( + {{{0.0, 0.0, 0.0, 4.0}, + {0.0, 0.0, 3.0, 5.0}, + {0.0, 1.0, 0.0, 0.0}, + {0.0, 2.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 10.0}, + {0.0, 0.0, 9.0, 11.0}, + {0.0, 7.0, 0.0, 0.0}, + {6.0, 8.0, 0.0, 0.0}}}, + torch::kDouble); + + ASSERT_EQ(output.sizes(), std::vector({1, 8, 4})); + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(FunctionalTest, Softshrink) { + const auto size = 3; + for (const auto lambda : {0.0, 0.42, 1.0, 4.2, 42.42}) { + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}).set_requires_grad(true); + // NOLINTNEXTLINE(bugprone-argument-comment) + auto y = F::softshrink(x, /*lambda=*/lambda); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = (x < -lambda) * (x + lambda) + (x > lambda) * (x - lambda); + ASSERT_TRUE(torch::allclose(y, y_exp)); + } +} + +TEST_F(FunctionalTest, SoftshrinkDefaultOptions) { + const auto size = 3; + const auto lambda = 0.5; + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}).set_requires_grad(true); + auto y = F::softshrink(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = (x < -lambda) * (x + lambda) + (x > lambda) * (x - lambda); +} + +TEST_F(FunctionalTest, Softsign) { + auto x = torch::randn(100) * 10; + auto y_exp = x / (1 + x.abs()); + auto y = F::softsign(x); + + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(FunctionalTest, Mish) { + auto x = torch::randn(100) * 10; + auto y_exp = x * x.exp().log1p().tanh(); + auto y = F::mish(x); + + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(FunctionalTest, Tanhshrink) { + auto x = torch::randn(100) * 10; + auto y_exp = x - x.tanh(); + auto y = F::tanhshrink(x); + + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(FunctionalTest, Threshold) { + const auto size = 3; + for (const auto threshold : {0.5, 1.0, 2.0}) { + for (const auto value : {0.5, 1.0, 2.0}) { + for (const auto inplace : {false, true}) { + auto x = torch::linspace(-3.0, 3.0, 61); + x.resize_({size, size, size}); + auto y_exp = (x <= threshold) * value + (x > threshold) * x; + auto y = F::threshold( + x, F::ThresholdFuncOptions(threshold, value).inplace(inplace)); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } + } + } + } + ASSERT_TRUE(F::threshold(torch::tensor(1.), F::ThresholdFuncOptions(0.5, 0.5)) + .defined()); +} + +TEST_F(FunctionalTest, BatchNorm1d) { + int num_features = 5; + double eps = 1e-05; + double momentum = 0.1; + + auto input = torch::randn({2, 5}); + auto mean = torch::randn(5); + auto variance = torch::rand(5); + auto weight = torch::ones({num_features}); + auto bias = torch::zeros({num_features}); + auto output = F::batch_norm( + input, + mean, + variance, + F::BatchNormFuncOptions() + .weight(weight) + .bias(bias) + .momentum(momentum) + .eps(eps) + .training(false)); + auto expected = (input - mean) / torch::sqrt(variance + eps); + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(FunctionalTest, BatchNorm1dDefaultOptions) { + auto input = torch::randn({2, 5}); + auto mean = torch::randn(5); + auto variance = torch::rand(5); + auto output = F::batch_norm(input, mean, variance); + auto expected = (input - mean) / torch::sqrt(variance + 1e-5); + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(FunctionalTest, BatchNorm2d) { + int num_features = 5; + double eps = 1e-05; + double momentum = 0.1; + + auto input = torch::randn({2, num_features, 4, 4}); + auto mean = torch::randn(num_features); + auto variance = torch::rand(num_features); + auto weight = torch::ones({num_features}); + auto bias = torch::zeros({num_features}); + auto output = F::batch_norm( + input, + mean, + variance, + F::BatchNormFuncOptions() + .weight(weight) + .bias(bias) + .momentum(momentum) + .eps(eps) + .training(false)); + auto expected = torch::transpose( + (torch::transpose(input, 1, 3) - mean) / torch::sqrt(variance + eps), + 1, + 3); + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(FunctionalTest, BatchNorm2dDefaultOptions) { + int num_features = 5; + double eps = 1e-05; + + auto input = torch::randn({2, num_features, 4, 4}); + auto mean = torch::randn(num_features); + auto variance = torch::rand(num_features); + auto output = F::batch_norm(input, mean, variance); + auto expected = torch::transpose( + (torch::transpose(input, 1, 3) - mean) / torch::sqrt(variance + eps), + 1, + 3); + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(FunctionalTest, BatchNorm3d) { + int num_features = 5; + double eps = 1e-05; + double momentum = 0.1; + + auto input = torch::randn({2, num_features, 2, 2, 2}); + auto mean = torch::randn(num_features); + auto variance = torch::rand(num_features); + auto weight = torch::ones({num_features}); + auto bias = torch::zeros({num_features}); + auto output = F::batch_norm( + input, + mean, + variance, + F::BatchNormFuncOptions() + .weight(weight) + .bias(bias) + .momentum(momentum) + .eps(eps) + .training(false)); + auto expected = torch::transpose( + (torch::transpose(input, 1, 4) - mean) / torch::sqrt(variance + eps), + 1, + 4); + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(FunctionalTest, BatchNorm3dDefaultOptions) { + int num_features = 5; + double eps = 1e-05; + + auto input = torch::randn({2, num_features, 2, 2, 2}); + auto mean = torch::randn(num_features); + auto variance = torch::rand(num_features); + auto output = F::batch_norm(input, mean, variance); + auto expected = torch::transpose( + (torch::transpose(input, 1, 4) - mean) / torch::sqrt(variance + eps), + 1, + 4); + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(FunctionalTest, InstanceNorm1d) { + int num_features = 5; + double eps = 1e-05; + double momentum = 0.1; + + auto input = torch::arange(40.).view({2, 5, 4}); + auto mean = torch::arange(5.); + auto variance = torch::arange(5.); + auto weight = torch::arange((double)num_features); + auto bias = torch::arange((double)num_features); + auto output = F::instance_norm( + input, + F::InstanceNormFuncOptions() + .running_mean(mean) + .running_var(variance) + .weight(weight) + .bias(bias) + .momentum(momentum) + .eps(eps)); + auto expected = torch::tensor( + {{{0.0000, 0.0000, 0.0000, 0.0000}, + {-0.3416, 0.5528, 1.4472, 2.3416}, + {-0.6833, 1.1056, 2.8944, 4.6833}, + {-1.0249, 1.6584, 4.3416, 7.0249}, + {-1.3665, 2.2112, 5.7888, 9.3665}}, + {{0.0000, 0.0000, 0.0000, 0.0000}, + {-0.3416, 0.5528, 1.4472, 2.3416}, + {-0.6833, 1.1056, 2.8944, 4.6833}, + {-1.0249, 1.6584, 4.3416, 7.0249}, + {-1.3665, 2.2112, 5.7888, 9.3665}}}); + ASSERT_TRUE(output.allclose(expected, 2e-04)); +} + +TEST_F(FunctionalTest, InstanceNorm1dDefaultOptions) { + auto input = torch::arange(40.).view({2, 5, 4}); + auto output = F::instance_norm(input); + auto expected = torch::tensor( + {{{-1.3416, -0.4472, 0.4472, 1.3416}, + {-1.3416, -0.4472, 0.4472, 1.3416}, + {-1.3416, -0.4472, 0.4472, 1.3416}, + {-1.3416, -0.4472, 0.4472, 1.3416}, + {-1.3416, -0.4472, 0.4472, 1.3416}}, + {{-1.3416, -0.4472, 0.4472, 1.3416}, + {-1.3416, -0.4472, 0.4472, 1.3416}, + {-1.3416, -0.4472, 0.4472, 1.3416}, + {-1.3416, -0.4472, 0.4472, 1.3416}, + {-1.3416, -0.4472, 0.4472, 1.3416}}}); + ASSERT_TRUE(output.allclose(expected, 2e-04)); +} + +TEST_F(FunctionalTest, InstanceNorm2d) { + int num_features = 5; + double eps = 1e-05; + double momentum = 0.1; + + auto input = + torch::arange(2. * num_features * 2 * 2).view({2, num_features, 2, 2}); + auto mean = torch::arange((double)num_features); + auto variance = torch::arange((double)num_features); + auto weight = torch::arange((double)num_features); + auto bias = torch::arange((double)num_features); + auto output = F::instance_norm( + input, + F::InstanceNormFuncOptions() + .running_mean(mean) + .running_var(variance) + .weight(weight) + .bias(bias) + .momentum(momentum) + .eps(eps)); + auto expected = torch::tensor( + {{{{0.0000, 0.0000}, {0.0000, 0.0000}}, + {{-0.3416, 0.5528}, {1.4472, 2.3416}}, + {{-0.6833, 1.1056}, {2.8944, 4.6833}}, + {{-1.0249, 1.6584}, {4.3416, 7.0249}}, + {{-1.3665, 2.2112}, {5.7888, 9.3665}}}, + {{{0.0000, 0.0000}, {0.0000, 0.0000}}, + {{-0.3416, 0.5528}, {1.4472, 2.3416}}, + {{-0.6833, 1.1056}, {2.8944, 4.6833}}, + {{-1.0249, 1.6584}, {4.3416, 7.0249}}, + {{-1.3665, 2.2112}, {5.7888, 9.3665}}}}); + ASSERT_TRUE(output.allclose(expected, 2e-04)); +} + +TEST_F(FunctionalTest, InstanceNorm2dDefaultOptions) { + int num_features = 5; + + auto input = + torch::arange(2. * num_features * 2 * 2).view({2, num_features, 2, 2}); + auto output = F::instance_norm(input); + auto expected = torch::tensor( + {{{{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}}, + {{{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}}}); + ASSERT_TRUE(output.allclose(expected, 2e-04)); +} + +TEST_F(FunctionalTest, InstanceNorm3d) { + int num_features = 5; + double eps = 1e-05; + double momentum = 0.1; + + auto input = torch::arange(2. * num_features * 2 * 2 * 2) + .view({2, num_features, 2, 2, 2}); + auto mean = torch::arange((double)num_features); + auto variance = torch::arange((double)num_features); + auto weight = torch::arange((double)num_features); + auto bias = torch::arange((double)num_features); + auto output = F::instance_norm( + input, + F::InstanceNormFuncOptions() + .running_mean(mean) + .running_var(variance) + .weight(weight) + .bias(bias) + .momentum(momentum) + .eps(eps)); + auto expected = torch::tensor( + {{{{{0.0000, 0.0000}, {0.0000, 0.0000}}, + {{0.0000, 0.0000}, {0.0000, 0.0000}}}, + {{{-0.5275, -0.0911}, {0.3453, 0.7818}}, + {{1.2182, 1.6547}, {2.0911, 2.5275}}}, + {{{-1.0550, -0.1822}, {0.6907, 1.5636}}, + {{2.4364, 3.3093}, {4.1822, 5.0550}}}, + {{{-1.5826, -0.2733}, {1.0360, 2.3453}}, + {{3.6547, 4.9640}, {6.2733, 7.5826}}}, + {{{-2.1101, -0.3644}, {1.3814, 3.1271}}, + {{4.8729, 6.6186}, {8.3644, 10.1101}}}}, + {{{{0.0000, 0.0000}, {0.0000, 0.0000}}, + {{0.0000, 0.0000}, {0.0000, 0.0000}}}, + {{{-0.5275, -0.0911}, {0.3453, 0.7818}}, + {{1.2182, 1.6547}, {2.0911, 2.5275}}}, + {{{-1.0550, -0.1822}, {0.6907, 1.5636}}, + {{2.4364, 3.3093}, {4.1822, 5.0550}}}, + {{{-1.5826, -0.2733}, {1.0360, 2.3453}}, + {{3.6547, 4.9640}, {6.2733, 7.5826}}}, + {{{-2.1101, -0.3644}, {1.3814, 3.1271}}, + {{4.8729, 6.6186}, {8.3644, 10.1101}}}}}); + ASSERT_TRUE(output.allclose(expected, 2e-04)); +} + +TEST_F(FunctionalTest, InstanceNorm3dDefaultOptions) { + int num_features = 5; + + auto input = torch::arange(2. * num_features * 2 * 2 * 2) + .view({2, num_features, 2, 2, 2}); + auto output = F::instance_norm(input); + auto expected = torch::tensor( + {{{{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}}, + {{{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}}}); + ASSERT_TRUE(output.allclose(expected, 2e-04)); +} + +TEST_F(FunctionalTest, Interpolate) { + { + // 1D interpolation + auto input = torch::ones({1, 1, 2}); + auto options = F::InterpolateFuncOptions() + .size(std::vector({4})) + .mode(torch::kNearest); + auto output = F::interpolate(input, options); + auto expected = torch::ones({1, 1, 4}); + + ASSERT_TRUE(output.allclose(expected)); + } + { + // 2D interpolation + for (const auto align_corners : {true, false}) { + // test float scale factor up & down sampling + for (const auto scale_factor : {0.5, 1.5, 2.0}) { + auto input = torch::ones({1, 1, 2, 2}); + auto options = + F::InterpolateFuncOptions() + .scale_factor(std::vector({scale_factor, scale_factor})) + .mode(torch::kBilinear) + .align_corners(align_corners); + auto output = F::interpolate(input, options); + auto expected_size = + static_cast(std::floor(input.size(-1) * scale_factor)); + auto expected = torch::ones({1, 1, expected_size, expected_size}); + + ASSERT_TRUE(output.allclose(expected)); + } + } + } + { + // 3D interpolation + for (const auto align_corners : {true, false}) { + for (const auto scale_factor : {0.5, 1.5, 2.0}) { + auto input = torch::ones({1, 1, 2, 2, 2}); + auto options = F::InterpolateFuncOptions() + .scale_factor(std::vector( + {scale_factor, scale_factor, scale_factor})) + .mode(torch::kTrilinear) + .align_corners(align_corners); + auto output = F::interpolate(input, options); + auto expected_size = + static_cast(std::floor(input.size(-1) * scale_factor)); + auto expected = + torch::ones({1, 1, expected_size, expected_size, expected_size}); + + ASSERT_TRUE(output.allclose(expected)); + } + } + } + { + ASSERT_THROWS_WITH( + F::interpolate( + torch::randn({1}), + F::InterpolateFuncOptions().size(std::vector({1}))), + "Input Error: Only 3D, 4D and 5D input Tensors supported (got 1D) "); + } + { + auto input = torch::randn({3, 2, 2}); + ASSERT_THROWS_WITH( + F::interpolate( + input[0], + F::InterpolateFuncOptions().size(std::vector({4, 4}))), + "Input Error: Only 3D, 4D and 5D input Tensors supported (got 2D) " + "for the modes: nearest | linear | bilinear | bicubic | trilinear (got kNearest)"); + ASSERT_THROWS_WITH( + F::interpolate( + torch::reshape(input, {1, 1, 1, 3, 2, 2}), + F::InterpolateFuncOptions().size( + std::vector({1, 1, 1, 3, 4, 4}))), + "Input Error: Only 3D, 4D and 5D input Tensors supported (got 6D) " + "for the modes: nearest | linear | bilinear | bicubic | trilinear (got kNearest)"); + ASSERT_THROWS_WITH( + F::interpolate(input, F::InterpolateFuncOptions()), + "either size or scale_factor should be defined"); + ASSERT_THROWS_WITH( + F::interpolate( + input, + F::InterpolateFuncOptions() + .size(std::vector({3, 4, 4})) + .scale_factor(std::vector({0.5}))), + "only one of size or scale_factor should be defined"); + ASSERT_THROWS_WITH( + F::interpolate( + input, + F::InterpolateFuncOptions().scale_factor( + std::vector({3, 2}))), + "scale_factor shape must match input shape. " + "Input is 1D, scale_factor size is [3, 2]"); + ASSERT_THROWS_WITH( + F::interpolate( + input, + F::InterpolateFuncOptions() + .mode(torch::kNearest) + .align_corners(true)), + "align_corners option can only be set with the " + "interpolating modes: linear | bilinear | bicubic | trilinear"); + } + { + auto tensor = torch::rand({2, 3, 32, 32}); + std::vector osize = {8, 10}; + auto expected = + at::native::_upsample_nearest_exact2d(tensor, osize, std::nullopt); + + auto options = F::InterpolateFuncOptions() + .size(osize) + .mode(torch::kNearestExact) + .align_corners(false); + auto output = F::interpolate(tensor, options); + + ASSERT_TRUE(output.allclose(expected)); + } + { + auto tensor = torch::rand({2, 3, 32, 32}); + std::vector osize = {8, 10}; + auto expected = + at::native::_upsample_bilinear2d_aa(tensor, osize, false, std::nullopt); + + auto options = F::InterpolateFuncOptions() + .size(osize) + .mode(torch::kBilinear) + .align_corners(false) + .antialias(true); + auto output = F::interpolate(tensor, options); + ASSERT_TRUE(output.allclose(expected)); + } + { + auto tensor = torch::rand({2, 3, 32, 32}); + std::vector osize = {8, 10}; + auto expected = + at::native::_upsample_bicubic2d_aa(tensor, osize, false, std::nullopt); + + auto options = F::InterpolateFuncOptions() + .size(osize) + .mode(torch::kBicubic) + .align_corners(false) + .antialias(true); + auto output = F::interpolate(tensor, options); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(FunctionalTest, Pad1) { + { + auto input = torch::arange(6, torch::kDouble).reshape({1, 2, 3}); + auto output = + F::pad(input, F::PadFuncOptions({1, 2}).mode(torch::kCircular)); + auto expected = torch::tensor( + {{{2., 0., 1., 2., 0., 1.}, {5., 3., 4., 5., 3., 4.}}}, torch::kDouble); + ASSERT_EQ(output.sizes(), std::vector({1, 2, 6})); + ASSERT_TRUE(output.allclose(expected, 1e-04)); + } +} +TEST_F(FunctionalTest, Pad2) { + { + auto input = torch::arange(9, torch::kDouble).reshape({1, 1, 3, 3}); + auto output = + F::pad(input, F::PadFuncOptions({3, 3, 3, 1}).mode(torch::kCircular)); + auto expected = torch::tensor( + {{{{0., 1., 2., 0., 1., 2., 0., 1., 2.}, + {3., 4., 5., 3., 4., 5., 3., 4., 5.}, + {6., 7., 8., 6., 7., 8., 6., 7., 8.}, + {0., 1., 2., 0., 1., 2., 0., 1., 2.}, + {3., 4., 5., 3., 4., 5., 3., 4., 5.}, + {6., 7., 8., 6., 7., 8., 6., 7., 8.}, + {0., 1., 2., 0., 1., 2., 0., 1., 2.}}}}, + torch::kDouble); + ASSERT_EQ(output.sizes(), std::vector({1, 1, 7, 9})); + ASSERT_TRUE(output.allclose(expected, 1e-04)); + } +} +TEST_F(FunctionalTest, Pad3) { + { + auto input = torch::arange(12, torch::kDouble).reshape({1, 1, 2, 2, 3}); + auto output = F::pad( + input, F::PadFuncOptions({3, 3, 2, 1, 2, 2}).mode(torch::kCircular)); + auto expected = torch::tensor( + {{{{{0., 1., 2., 0., 1., 2., 0., 1., 2.}, + {3., 4., 5., 3., 4., 5., 3., 4., 5.}, + {0., 1., 2., 0., 1., 2., 0., 1., 2.}, + {3., 4., 5., 3., 4., 5., 3., 4., 5.}, + {0., 1., 2., 0., 1., 2., 0., 1., 2.}}, + + {{6., 7., 8., 6., 7., 8., 6., 7., 8.}, + {9., 10., 11., 9., 10., 11., 9., 10., 11.}, + {6., 7., 8., 6., 7., 8., 6., 7., 8.}, + {9., 10., 11., 9., 10., 11., 9., 10., 11.}, + {6., 7., 8., 6., 7., 8., 6., 7., 8.}}, + + {{0., 1., 2., 0., 1., 2., 0., 1., 2.}, + {3., 4., 5., 3., 4., 5., 3., 4., 5.}, + {0., 1., 2., 0., 1., 2., 0., 1., 2.}, + {3., 4., 5., 3., 4., 5., 3., 4., 5.}, + {0., 1., 2., 0., 1., 2., 0., 1., 2.}}, + + {{6., 7., 8., 6., 7., 8., 6., 7., 8.}, + {9., 10., 11., 9., 10., 11., 9., 10., 11.}, + {6., 7., 8., 6., 7., 8., 6., 7., 8.}, + {9., 10., 11., 9., 10., 11., 9., 10., 11.}, + {6., 7., 8., 6., 7., 8., 6., 7., 8.}}, + + {{0., 1., 2., 0., 1., 2., 0., 1., 2.}, + {3., 4., 5., 3., 4., 5., 3., 4., 5.}, + {0., 1., 2., 0., 1., 2., 0., 1., 2.}, + {3., 4., 5., 3., 4., 5., 3., 4., 5.}, + {0., 1., 2., 0., 1., 2., 0., 1., 2.}}, + + {{6., 7., 8., 6., 7., 8., 6., 7., 8.}, + {9., 10., 11., 9., 10., 11., 9., 10., 11.}, + {6., 7., 8., 6., 7., 8., 6., 7., 8.}, + {9., 10., 11., 9., 10., 11., 9., 10., 11.}, + {6., 7., 8., 6., 7., 8., 6., 7., 8.}}}}}, + torch::kDouble); + ASSERT_EQ(output.sizes(), std::vector({1, 1, 6, 5, 9})); + ASSERT_TRUE(output.allclose(expected, 1e-04)); + } +} +TEST_F(FunctionalTest, Pad4) { + { + auto input = torch::arange(16, torch::kDouble).reshape({2, 2, 2, 2}); + auto output = + F::pad(input, F::PadFuncOptions({1, 1, 1, 1}).mode(torch::kReflect)); + auto expected = torch::tensor( + {{{{3., 2., 3., 2.}, + {1., 0., 1., 0.}, + {3., 2., 3., 2.}, + {1., 0., 1., 0.}}, + + {{7., 6., 7., 6.}, + {5., 4., 5., 4.}, + {7., 6., 7., 6.}, + {5., 4., 5., 4.}}}, + + {{{11., 10., 11., 10.}, + {9., 8., 9., 8.}, + {11., 10., 11., 10.}, + {9., 8., 9., 8.}}, + + {{15., 14., 15., 14.}, + {13., 12., 13., 12.}, + {15., 14., 15., 14.}, + {13., 12., 13., 12.}}}}, + torch::kDouble); + ASSERT_EQ(output.sizes(), std::vector({2, 2, 4, 4})); + ASSERT_TRUE(output.allclose(expected, 1e-04)); + } +} +TEST_F(FunctionalTest, Pad5) { + { + auto input = torch::arange(12, torch::kDouble).reshape({1, 1, 2, 2, 3}); + auto output = F::pad( + input, F::PadFuncOptions({1, 2, 2, 1, 1, 2}).mode(torch::kReplicate)); + auto expected = torch::tensor( + {{{{{0., 0., 1., 2., 2., 2.}, + {0., 0., 1., 2., 2., 2.}, + {0., 0., 1., 2., 2., 2.}, + {3., 3., 4., 5., 5., 5.}, + {3., 3., 4., 5., 5., 5.}}, + + {{0., 0., 1., 2., 2., 2.}, + {0., 0., 1., 2., 2., 2.}, + {0., 0., 1., 2., 2., 2.}, + {3., 3., 4., 5., 5., 5.}, + {3., 3., 4., 5., 5., 5.}}, + + {{6., 6., 7., 8., 8., 8.}, + {6., 6., 7., 8., 8., 8.}, + {6., 6., 7., 8., 8., 8.}, + {9., 9., 10., 11., 11., 11.}, + {9., 9., 10., 11., 11., 11.}}, + + {{6., 6., 7., 8., 8., 8.}, + {6., 6., 7., 8., 8., 8.}, + {6., 6., 7., 8., 8., 8.}, + {9., 9., 10., 11., 11., 11.}, + {9., 9., 10., 11., 11., 11.}}, + + {{6., 6., 7., 8., 8., 8.}, + {6., 6., 7., 8., 8., 8.}, + {6., 6., 7., 8., 8., 8.}, + {9., 9., 10., 11., 11., 11.}, + {9., 9., 10., 11., 11., 11.}}}}}, + torch::kDouble); + ASSERT_EQ(output.sizes(), std::vector({1, 1, 5, 5, 6})); + ASSERT_TRUE(output.allclose(expected, 1e-04)); + } +} +TEST_F(FunctionalTest, Pad6) { + { + auto input = torch::arange(18, torch::kDouble).reshape({1, 1, 3, 2, 3}); + auto output = F::pad( + input, F::PadFuncOptions({0, 2, 1, 0, 1, 2}).mode(torch::kReflect)); + auto expected = torch::tensor( + {{{{{9., 10., 11., 10., 9.}, + {6., 7., 8., 7., 6.}, + {9., 10., 11., 10., 9.}}, + + {{3., 4., 5., 4., 3.}, {0., 1., 2., 1., 0.}, {3., 4., 5., 4., 3.}}, + + {{9., 10., 11., 10., 9.}, + {6., 7., 8., 7., 6.}, + {9., 10., 11., 10., 9.}}, + + {{15., 16., 17., 16., 15.}, + {12., 13., 14., 13., 12.}, + {15., 16., 17., 16., 15.}}, + + {{9., 10., 11., 10., 9.}, + {6., 7., 8., 7., 6.}, + {9., 10., 11., 10., 9.}}, + + {{3., 4., 5., 4., 3.}, + {0., 1., 2., 1., 0.}, + {3., 4., 5., 4., 3.}}}}}, + torch::kDouble); + ASSERT_EQ(output.sizes(), std::vector({1, 1, 6, 3, 5})); + ASSERT_TRUE(output.allclose(expected, 1e-04)); + } +} +TEST_F(FunctionalTest, Pad7) { + { + auto input = torch::ones({1, 1, 1, 1}, torch::kDouble); + auto output = F::pad( + input, F::PadFuncOptions({1, 1}).mode(torch::kConstant).value(0)); + ASSERT_EQ(output.sizes(), std::vector({1, 1, 1, 3})); + auto expected = torch::tensor({{{{0., 1., 0.}}}}, torch::kDouble); + } +} +TEST_F(FunctionalTest, Pad8) { + { + auto input = torch::ones({1, 1, 1, 1}, torch::kDouble); + auto output = F::pad(input, F::PadFuncOptions({1, 1})); + ASSERT_EQ(output.sizes(), std::vector({1, 1, 1, 3})); + auto expected = torch::tensor({{{{0., 1., 0.}}}}, torch::kDouble); + } +} + +TEST_F(FunctionalTest, CTCLoss) { + { // test CTCLoss typechecks + const auto target_lengths = torch::tensor({30, 25, 20}); + const auto input_lengths = torch::tensor({50, 50, 50}); + const auto targets = + torch::randint(1, 15, {target_lengths.sum().item()}, torch::kInt); + const auto log_probs = + torch::randn({50, 3, 15}, torch::kFloat).log_softmax(2); + + const auto _input_lengths = input_lengths.to(torch::kFloat); + ASSERT_THROWS_WITH( + F::ctc_loss(log_probs, targets, _input_lengths, target_lengths), + "input_lengths must be integral"); + + const auto target_lengths_ = target_lengths.to(torch::kFloat); + ASSERT_THROWS_WITH( + F::ctc_loss(log_probs, targets, input_lengths, target_lengths_), + "target_lengths must be integral"); + } + { // test CTCLoss length checks + const auto target_lengths = torch::tensor({30, 25, 20}); + const auto input_lengths = torch::tensor({50, 50, 50}); + const auto targets = torch::randint(1, 15, {3, 29}, torch::kInt); + const auto log_probs = + torch::randn({50, 3, 15}, torch::kFloat).log_softmax(2); + ASSERT_THROWS_WITH( + F::ctc_loss(log_probs, targets, input_lengths, target_lengths), + "Expected tensor to have size at least 30 at dimension 1"); + } + { // test CTCLoss empty target + { + const auto target_lengths = torch::tensor({0, 0, 0}); + const auto input_lengths = torch::tensor({50, 50, 50}); + const auto targets = + torch::randint(1, 15, at::IntArrayRef({0}), torch::kLong); + const auto log_probs = + torch::randn({50, 3, 15}, torch::kDouble).log_softmax(2); + const auto loss = F::ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + F::CTCLossFuncOptions().reduction(torch::kNone)); + ASSERT_TRUE(loss.ge(0).all().item()); + ASSERT_TRUE(torch::allclose( + -log_probs.sum(0).slice(1, 0, 1).view_as(loss), loss)); + } + { + const auto target_lengths = torch::tensor({0, 9, 0}); + const auto input_lengths = torch::tensor({50, 50, 50}); + const auto targets = torch::randint(1, 15, {9}, torch::kLong); + const auto log_probs = + torch::randn({50, 3, 15}, torch::kDouble).log_softmax(2); + const auto loss = F::ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + F::CTCLossFuncOptions().reduction(torch::kNone)); + ASSERT_TRUE(loss.ge(0).all().item()); + ASSERT_TRUE(torch::allclose( + -log_probs.sum(0) + .index_select(0, torch::tensor({0, 2}, torch::kLong)) + .slice(1, 0, 1) + .view({2}), + loss.index_select(0, torch::tensor({0, 2}, torch::kLong)))); + } + } +} + +TEST_F(FunctionalTest, PoissonNLLLoss) { + const auto input = torch::tensor({0.5, 1.5, 2.5}); + const auto target = torch::tensor({1., 2., 3.}); + const auto component_wise_loss = torch::exp(input) - target * input; + ASSERT_TRUE(torch::allclose( + torch::mean(component_wise_loss), F::poisson_nll_loss(input, target))); + ASSERT_TRUE(torch::allclose( + component_wise_loss, + F::poisson_nll_loss( + input, + target, + F::PoissonNLLLossFuncOptions().reduction(torch::kNone)))); + ASSERT_TRUE(torch::allclose( + torch::sum(component_wise_loss), + F::poisson_nll_loss( + input, + target, + F::PoissonNLLLossFuncOptions().reduction(torch::kSum)))); + ASSERT_TRUE(torch::allclose( + torch::mean(component_wise_loss), + F::poisson_nll_loss( + input, + target, + F::PoissonNLLLossFuncOptions().reduction(torch::kMean)))); +} + +TEST_F(FunctionalTest, MarginRankingLoss) { + { + const auto input1 = torch::randn(15) * 10; + const auto input2 = torch::randn(15) * 10; + const auto target = torch::randn(15).sign(); + ASSERT_TRUE(torch::allclose( + F::margin_ranking_loss(input1, input2, target), + (-target * (input1 - input2)).clamp(0).mean())); + } + { + const auto input1 = torch::randn(15) * 10; + const auto input2 = torch::randn(15) * 10; + const auto target = torch::randn(15).sign(); + const auto margin = 0.5; + ASSERT_TRUE(torch::allclose( + F::margin_ranking_loss( + input1, + input2, + target, + F::MarginRankingLossFuncOptions().margin(0.5).reduction( + torch::kSum)), + (-target * (input1 - input2) + margin).clamp(0).sum())); + } + { + const auto input1 = torch::randn(15) * 10; + const auto input2 = torch::randn(15) * 10; + const auto target = torch::randn(15).sign(); + const auto margin = 0.5; + ASSERT_TRUE(torch::allclose( + F::margin_ranking_loss( + input1, + input2, + target, + F::MarginRankingLossFuncOptions().margin(0.5).reduction( + torch::kMean)), + (-target * (input1 - input2) + margin).clamp(0).mean())); + } +} + +TEST_F(FunctionalTest, ConvTranspose1d) { + auto x = torch::arange(20.).view({2, 2, 5}); + auto weight = torch::arange(18.).view({2, 3, 3}); + auto y = + F::conv_transpose1d(x, weight, F::ConvTranspose1dFuncOptions().stride(1)); + auto expected = torch::tensor( + {{{45., 104., 179., 212., 245., 188., 107.}, + {60., 140., 242., 293., 344., 260., 146.}, + {75., 176., 305., 374., 443., 332., 185.}}, + {{135., 304., 509., 542., 575., 428., 237.}, + {210., 460., 752., 803., 854., 620., 336.}, + {285., 616., 995., 1064., 1133., 812., 435.}}}); + ASSERT_TRUE(torch::allclose(y, expected)); + + auto y_no_options = F::conv_transpose1d(x, weight); + ASSERT_TRUE(torch::allclose(y_no_options, expected)); +} + +TEST_F(FunctionalTest, ConvTranspose2dEven) { + auto x = torch::arange(50.).view({1, 2, 5, 5}); + auto weight = torch::arange(54.).view({2, 3, 3, 3}); + auto y = + F::conv_transpose2d(x, weight, F::ConvTranspose2dFuncOptions().stride(1)); + auto expected = torch::tensor( + {{{{675., 1402., 2183., 2270., 2357., 1634., 849.}, + {1560., 3240., 5044., 5236., 5428., 3760., 1952.}, + {2685., 5574., 8673., 8988., 9303., 6438., 3339.}, + {3180., 6594., 10248., 10563., 10878., 7518., 3894.}, + {3675., 7614., 11823., 12138., 12453., 8598., 4449.}, + {2820., 5832., 9040., 9268., 9496., 6544., 3380.}, + {1605., 3314., 5129., 5252., 5375., 3698., 1907.}}, + {{900., 1870., 2912., 3053., 3194., 2210., 1146.}, + {2100., 4356., 6772., 7072., 7372., 5092., 2636.}, + {3630., 7518., 11670., 12147., 12624., 8706., 4500.}, + {4395., 9078., 14055., 14532., 15009., 10326., 5325.}, + {5160., 10638., 16440., 16917., 17394., 11946., 6150.}, + {3900., 8028., 12388., 12724., 13060., 8956., 4604.}, + {2190., 4502., 6938., 7115., 7292., 4994., 2564.}}, + {{1125., 2338., 3641., 3836., 4031., 2786., 1443.}, + {2640., 5472., 8500., 8908., 9316., 6424., 3320.}, + {4575., 9462., 14667., 15306., 15945., 10974., 5661.}, + {5610., 11562., 17862., 18501., 19140., 13134., 6756.}, + {6645., 13662., 21057., 21696., 22335., 15294., 7851.}, + {4980., 10224., 15736., 16180., 16624., 11368., 5828.}, + {2775., 5690., 8747., 8978., 9209., 6290., 3221.}}}}); + ASSERT_TRUE(torch::allclose(y, expected)); + + auto y_no_options = F::conv_transpose2d(x, weight); + ASSERT_TRUE(torch::allclose(y_no_options, expected)); +} + +TEST_F(FunctionalTest, ConvTranspose2dUneven) { + auto x = torch::arange(40.).view({1, 2, 5, 4}); + auto weight = torch::arange(36.).view({2, 3, 3, 2}); + auto y = + F::conv_transpose2d(x, weight, F::ConvTranspose2dFuncOptions().stride(1)); + auto expected = torch::tensor( + {{{{360., 758., 796., 834., 440.}, + {832., 1752., 1836., 1920., 1012.}, + {1432., 3014., 3152., 3290., 1732.}, + {1696., 3566., 3704., 3842., 2020.}, + {1960., 4118., 4256., 4394., 2308.}, + {1504., 3152., 3252., 3352., 1756.}, + {856., 1790., 1844., 1898., 992.}}, + {{480., 1010., 1072., 1134., 596.}, + {1120., 2352., 2484., 2616., 1372.}, + {1936., 4058., 4268., 4478., 2344.}, + {2344., 4898., 5108., 5318., 2776.}, + {2752., 5738., 5948., 6158., 3208.}, + {2080., 4328., 4476., 4624., 2404.}, + {1168., 2426., 2504., 2582., 1340.}}, + {{600., 1262., 1348., 1434., 752.}, + {1408., 2952., 3132., 3312., 1732.}, + {2440., 5102., 5384., 5666., 2956.}, + {2992., 6230., 6512., 6794., 3532.}, + {3544., 7358., 7640., 7922., 4108.}, + {2656., 5504., 5700., 5896., 3052.}, + {1480., 3062., 3164., 3266., 1688.}}}}); + ASSERT_TRUE(torch::allclose(y, expected)); + + auto y_no_options = F::conv_transpose2d(x, weight); + ASSERT_TRUE(torch::allclose(y_no_options, expected)); +} + +TEST_F(FunctionalTest, ConvTranspose3d) { + auto x = torch::arange(16.).view({1, 2, 2, 2, 2}); + auto weight = torch::arange(32.).view({2, 2, 2, 2, 2}); + auto y = + F::conv_transpose3d(x, weight, F::ConvTranspose3dFuncOptions().stride(1)); + auto expected = torch::tensor( + {{{{{128., 280., 154.}, {304., 664., 364.}, {184., 400., 218.}}, + {{352., 768., 420.}, {832., 1808., 984.}, {496., 1072., 580.}}, + {{256., 552., 298.}, {592., 1272., 684.}, {344., 736., 394.}}}, + {{{192., 424., 234.}, {464., 1016., 556.}, {280., 608., 330.}}, + {{544., 1184., 644.}, {1280., 2768., 1496.}, {752., 1616., 868.}}, + {{384., 824., 442.}, {880., 1880., 1004.}, {504., 1072., 570.}}}}}); + ASSERT_TRUE(torch::allclose(y, expected)); + + auto y_no_options = F::conv_transpose3d(x, weight); + ASSERT_TRUE(torch::allclose(y_no_options, expected)); +} + +TEST_F(FunctionalTest, AlphaDropout) { + auto input = torch::randn(5000); + auto input_mean = input.mean(); + auto input_std = input.std(); + + for (const auto rate : {0.2, 0.5, 0.8}) { + for (const auto inplace : {false, true}) { + auto input_ = input.clone(); + auto output = F::alpha_dropout( + input_, + F::AlphaDropoutFuncOptions().p(rate).training(false).inplace( + inplace)); + ASSERT_TRUE(torch::allclose(input_mean, output.mean(), 0.1)); + ASSERT_TRUE(torch::allclose(input_std, output.std(), 0.1)); + if (inplace) { + ASSERT_TRUE(torch::allclose(input_, output)); + } + } + } + auto output = F::detail::alpha_dropout(input, 0.5, false, false); + ASSERT_TRUE(torch::allclose(input_mean, output.mean(), 0.1)); + ASSERT_TRUE(torch::allclose(input_std, output.std(), 0.1)); +} + +TEST_F(FunctionalTest, FeatureAlphaDropout) { + auto input = torch::randn(5000); + auto input_mean = input.mean(); + auto input_std = input.std(); + + for (const auto rate : {0.2, 0.5, 0.8}) { + for (const auto inplace : {false, true}) { + auto input_ = input.clone(); + auto output = F::feature_alpha_dropout( + input_, + F::FeatureAlphaDropoutFuncOptions().p(rate).training(false).inplace( + inplace)); + ASSERT_TRUE(torch::allclose(input_mean, output.mean(), 0.1)); + ASSERT_TRUE(torch::allclose(input_std, output.std(), 0.1)); + if (inplace) { + ASSERT_TRUE(torch::allclose(input_, output)); + } + } + } + auto output = F::feature_alpha_dropout(input); + ASSERT_TRUE(torch::allclose(input_mean, output.mean(), 0.1)); + ASSERT_TRUE(torch::allclose(input_std, output.std(), 0.1)); +} + +TEST_F(FunctionalTest, Dropout) { + auto input = torch::randn(5000); + auto input_mean = input.mean(); + auto input_std = input.std(); + + for (const auto rate : {0.2, 0.5, 0.8}) { + auto output = F::dropout(input, F::DropoutFuncOptions().p(rate)); + ASSERT_TRUE(torch::allclose(input_mean, output.mean(), 0.01, 0.05)); + ASSERT_TRUE((input_std <= output.std()).all().item()); + } + auto output = F::dropout(input); + ASSERT_TRUE(torch::allclose(input_mean, output.mean(), 0.01, 0.05)); + ASSERT_TRUE((input_std <= output.std()).all().item()); + ASSERT_TRUE(F::dropout(torch::tensor(1.)).defined()); +} + +TEST_F(FunctionalTest, Dropout2d) { + auto input = torch::randn({2, 2, 50, 100}); + auto input_mean = input.mean(); + auto input_std = input.std(); + + for (const auto rate : {0.2, 0.5, 0.8}) { + auto output = F::dropout2d(input, F::Dropout2dFuncOptions().p(rate)); + ASSERT_TRUE(torch::allclose(input_mean, output.mean(), 0.01, 0.05)); + } + auto output = F::dropout2d(input); + ASSERT_TRUE(torch::allclose(input_mean, output.mean(), 0.01, 0.05)); + ASSERT_TRUE(F::dropout2d(torch::randn({2, 50, 100})).defined()); +} + +TEST_F(FunctionalTest, Dropout3d) { + auto input = torch::randn({2, 2, 50, 10, 10}); + auto input_mean = input.mean(); + auto input_std = input.std(); + + for (const auto rate : {0.2, 0.5, 0.8}) { + auto output = F::dropout3d(input, F::Dropout3dFuncOptions().p(rate)); + ASSERT_TRUE(torch::allclose(input_mean, output.mean(), 0.01, 0.05)); + } + auto output = F::dropout3d(input); + ASSERT_TRUE(torch::allclose(input_mean, output.mean(), 0.01, 0.05)); + ASSERT_TRUE(F::dropout3d(torch::randn({2, 50, 10, 10})).defined()); +} + +template +void test_isfinite(const at::Device& device) { + const std::vector values = { + std::numeric_limits::lowest(), + 0, + 1, + 42, + std::numeric_limits::min(), + std::numeric_limits::max()}; + for (const auto value : values) { + const auto x = torch::full( + {3, 3}, value, torch::TensorOptions().dtype(S).device(device)); + ASSERT_TRUE(torch::isfinite(x).all().template item()); + } + if (std::numeric_limits::has_infinity) { + const auto inf = std::numeric_limits::infinity(); + const auto x = torch::tensor( + {-inf, + std::numeric_limits::lowest(), + static_cast(0), + static_cast(1), + static_cast(42), + std::numeric_limits::min(), + std::numeric_limits::max(), + inf}, + torch::TensorOptions().dtype(S).device(device)); + ASSERT_TRUE(torch::allclose( + // torch::allclose does not support comparing torch::kBool + torch::isfinite(x).toType(torch::kInt), + torch::tensor( + {false, true, true, true, true, true, true, false}, + torch::TensorOptions().device(device)) + .toType(torch::kInt))); + } + if (std::numeric_limits::has_quiet_NaN) { + const auto x = torch::tensor( + {std::numeric_limits::quiet_NaN()}, + torch::TensorOptions().dtype(S).device(device)); + ASSERT_FALSE(torch::isfinite(x).all().template item()); + } + if (std::numeric_limits::has_signaling_NaN) { + const auto x = torch::tensor( + {std::numeric_limits::signaling_NaN()}, + torch::TensorOptions().dtype(S).device(device)); + ASSERT_FALSE(torch::isfinite(x).all().template item()); + } +} + +TEST_F(FunctionalTest, isfinite) { + const at::Device device("cpu"); + test_isfinite(device); + test_isfinite(device); + test_isfinite(device); + test_isfinite(device); + test_isfinite(device); + test_isfinite(device); + test_isfinite(device); +} + +TEST_F(FunctionalTest, isfinite_CUDA) { + const at::Device device("cuda"); + test_isfinite(device); + test_isfinite(device); + test_isfinite(device); + test_isfinite(device); + test_isfinite(device); + test_isfinite(device); + test_isfinite(device); + test_isfinite(device); +} + +template +void test_isinf(const at::Device& device) { + const std::vector values = { + std::numeric_limits::lowest(), + 0, + 1, + 42, + std::numeric_limits::min(), + std::numeric_limits::max()}; + for (const auto value : values) { + const auto x = torch::full( + {3, 3}, value, torch::TensorOptions().dtype(S).device(device)); + ASSERT_FALSE(torch::isinf(x).all().template item()); + } + if (std::numeric_limits::has_infinity) { + const auto inf = std::numeric_limits::infinity(); + const auto x = torch::tensor( + {-inf, + std::numeric_limits::lowest(), + static_cast(0), + static_cast(1), + static_cast(42), + std::numeric_limits::min(), + std::numeric_limits::max(), + inf}, + torch::TensorOptions().dtype(S).device(device)); + ASSERT_TRUE(torch::allclose( + // torch::allclose does not support comparing torch::kBool + torch::isinf(x).toType(torch::kInt), + torch::tensor( + {true, false, false, false, false, false, false, true}, + torch::TensorOptions().device(device)) + .toType(torch::kInt))); + } + if (std::numeric_limits::has_quiet_NaN) { + const auto x = torch::tensor( + {std::numeric_limits::quiet_NaN()}, + torch::TensorOptions().dtype(S).device(device)); + ASSERT_FALSE(torch::isinf(x).all().template item()); + } + if (std::numeric_limits::has_signaling_NaN) { + const auto x = torch::tensor( + {std::numeric_limits::signaling_NaN()}, + torch::TensorOptions().dtype(S).device(device)); + ASSERT_FALSE(torch::isinf(x).all().template item()); + } +} + +TEST_F(FunctionalTest, isinf) { + const at::Device device("cpu"); + test_isinf(device); + test_isinf(device); + test_isinf(device); + test_isinf(device); + test_isinf(device); + test_isinf(device); + test_isinf(device); +} + +TEST_F(FunctionalTest, isinf_CUDA) { + const at::Device device("cuda"); + test_isinf(device); + test_isinf(device); + test_isinf(device); + test_isinf(device); + test_isinf(device); + test_isinf(device); + test_isinf(device); + test_isinf(device); +} + +template +void test_allclose(const at::Device& device) { + const std::vector values = { + std::numeric_limits::lowest(), + 0, + 1, + 42, + std::numeric_limits::min(), + std::numeric_limits::max()}; + for (const auto value : values) { + const auto x = + torch::full({1}, value, torch::TensorOptions().dtype(S).device(device)); + const auto y = + torch::full({1}, value, torch::TensorOptions().dtype(S).device(device)); + ASSERT_TRUE(torch::allclose(x, x)); + ASSERT_TRUE(torch::allclose(x, y)); + ASSERT_TRUE(torch::allclose(y, x)); + ASSERT_FALSE(torch::allclose(1.1 * x + 0.1, 1.0 * x)); + ASSERT_TRUE(torch::allclose(0.99 * x + 0.1, 1.0 * x, 1.1, 0.1)); + } + if (std::numeric_limits::has_infinity) { + const auto inf = std::numeric_limits::infinity(); + const auto x = torch::tensor( + {-inf, inf}, torch::TensorOptions().dtype(S).device(device)); + const auto y = torch::tensor( + {-inf, inf}, torch::TensorOptions().dtype(S).device(device)); + ASSERT_TRUE(torch::allclose(x, x)); + ASSERT_TRUE(torch::allclose(x, y)); + ASSERT_TRUE(torch::allclose(y, x)); + } + if (std::numeric_limits::has_quiet_NaN) { + const auto x = torch::tensor( + {std::numeric_limits::quiet_NaN()}, + torch::TensorOptions().dtype(S).device(device)); + const auto y = torch::tensor( + {std::numeric_limits::quiet_NaN()}, + torch::TensorOptions().dtype(S).device(device)); + ASSERT_TRUE(torch::allclose(x, x, 1.0, 0.0, /*equal_nan=*/true)); + ASSERT_TRUE(torch::allclose(x, y, 1.0, 0.0, /*equal_nan=*/true)); + ASSERT_TRUE(torch::allclose(y, x, 1.0, 0.0, /*equal_nan=*/true)); + } + if (std::numeric_limits::has_signaling_NaN) { + const auto x = torch::tensor( + {std::numeric_limits::signaling_NaN()}, + torch::TensorOptions().dtype(S).device(device)); + const auto y = torch::tensor( + {std::numeric_limits::signaling_NaN()}, + torch::TensorOptions().dtype(S).device(device)); + ASSERT_TRUE(torch::allclose(x, x, 1.0, 0.0, /*equal_nan=*/true)); + ASSERT_TRUE(torch::allclose(x, y, 1.0, 0.0, /*equal_nan=*/true)); + ASSERT_TRUE(torch::allclose(y, x, 1.0, 0.0, /*equal_nan=*/true)); + } +} + +TEST_F(FunctionalTest, AllClose) { + const at::Device device("cpu"); + test_allclose(device); + test_allclose(device); + test_allclose(device); + test_allclose(device); + test_allclose(device); + test_allclose(device); + test_allclose(device); +} + +TEST_F(FunctionalTest, AllClose_CUDA) { + const at::Device device("cuda"); + test_allclose(device); + test_allclose(device); + test_allclose(device); + test_allclose(device); + test_allclose(device); + test_allclose(device); + test_allclose(device); + test_allclose(device); +} + +TEST_F(FunctionalTest, BCEWithLogitsLoss) { + { // test BCE with logits raises if target and input are different size + { + const auto target = torch::rand(5); + const auto input = torch::rand({5, 1}); + ASSERT_THROWS_WITH( + F::binary_cross_entropy_with_logits(input, target), + "must be the same as input size"); + } + { + const auto target = torch::rand({5, 1}); + const auto input = torch::rand(5); + ASSERT_THROWS_WITH( + F::binary_cross_entropy_with_logits(input, target), + "must be the same as input size"); + } + } + { // test BCE with logits gives same result as sigmoid and bce loss + auto sigmoid = Sigmoid(); + + auto target = torch::rand({64, 4}); + auto output = torch::rand({64, 4}) - 0.5; + + ASSERT_TRUE(torch::allclose( + F::binary_cross_entropy_with_logits(output, target), + F::binary_cross_entropy(sigmoid(output), target))); + + auto weight = torch::rand(4); + ASSERT_TRUE(torch::allclose( + F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().weight(weight)), + F::binary_cross_entropy( + sigmoid(output), + target, + F::BinaryCrossEntropyFuncOptions().weight(weight)))); + + target = torch::zeros({4, 1}, torch::kFloat); + output = torch::empty({4, 1}, torch::kFloat).fill_(-100); + + ASSERT_TRUE(torch::allclose( + F::binary_cross_entropy_with_logits(output, target), + F::binary_cross_entropy(sigmoid(output), target))); + + ASSERT_TRUE(torch::allclose( + F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().reduction( + torch::kNone)), + F::binary_cross_entropy( + sigmoid(output), + target, + F::BinaryCrossEntropyFuncOptions().reduction(torch::kNone)))); + + weight = torch::rand({1}, torch::kFloat); + ASSERT_TRUE(torch::allclose( + F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().weight(weight)), + F::binary_cross_entropy( + sigmoid(output), + target, + F::BinaryCrossEntropyFuncOptions().weight(weight)))); + } + { // test BCE with logits has correct grad at zero + const auto output = torch::zeros({3, 1}, torch::requires_grad()); + const auto target = torch::zeros({3, 1}); + F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().reduction(torch::kSum)) + .backward(); + const auto expected_grad = torch::empty({3, 1}).fill_(0.5); + ASSERT_TRUE(torch::allclose(output.grad(), expected_grad)); + } + { // test BCE with logits broadcasts weights + const auto target = torch::rand({16, 4}); + const auto output = torch::rand({16, 4}) - 0.5; + + auto weight = torch::rand(4); + auto out1 = F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().weight(weight)); + + weight = weight.expand({16, 4}).contiguous(); + auto out2 = F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().weight(weight)); + + ASSERT_TRUE(torch::allclose(out1, out2)); + + weight = torch::rand({16, 1}); + out1 = F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().weight(weight)); + + weight = weight.expand({16, 4}).contiguous(); + out2 = F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().weight(weight)); + + ASSERT_TRUE(torch::allclose(out1, out2)); + } + { // test BCE with logits ones in pos weights are the same as none + const auto target = torch::rand({64, 4}); + const auto output = torch::rand({64, 4}) - 0.5; + const auto pos_weight = torch::ones({64, 4}); + + ASSERT_TRUE(torch::allclose( + F::binary_cross_entropy_with_logits(output, target), + F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().pos_weight( + pos_weight)))); + } + { // test BCE with logits broadcasts pos weights + const auto target = torch::rand({64, 4}); + const auto output = torch::rand({64, 4}) - 0.5; + const auto pos_weight = torch::rand(4); + const auto out1 = F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().pos_weight(pos_weight)); + + const auto pos_weight1 = pos_weight.expand({1, 4}); + const auto out2 = F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().pos_weight(pos_weight)); + + const auto pos_weight2 = pos_weight.expand({64, 4}); + const auto out3 = F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().pos_weight(pos_weight)); + + ASSERT_TRUE(torch::allclose(out1, out2)); + ASSERT_TRUE(torch::allclose(out1, out3)); + } + { // test BCE with logits with pos weight has correct grad at zero + const auto output = torch::zeros({3, 1}, torch::requires_grad()); + const auto target = torch::zeros({3, 1}); + const auto pos_weight = torch::ones({3, 1}); + F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions() + .pos_weight(pos_weight) + .reduction(torch::kSum)) + .backward(); + const auto expected_grad = torch::empty({3, 1}).fill_(0.5); + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + const auto grad = output.grad(); + ASSERT_TRUE(torch::allclose(grad, expected_grad)); + } + { // test BCE with logits stability + const auto output = torch::tensor({0., -120.}); + const auto target = torch::tensor({0., 1.}); + const auto pos_weight = torch::tensor({1., 1.}); + + const auto out1 = F::binary_cross_entropy_with_logits(output, target); + ASSERT_TRUE(torch::isfinite(out1).all().item()); + + const auto out2 = F::binary_cross_entropy_with_logits( + output, + target, + F::BinaryCrossEntropyWithLogitsFuncOptions().pos_weight(pos_weight)); + ASSERT_TRUE(torch::isfinite(out2).all().item()); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/grad_mode.cpp b/western_ai/usa/pytorch/test/cpp/api/grad_mode.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3612870cd7400716ca311be7d773372206421162 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/grad_mode.cpp @@ -0,0 +1,78 @@ +#include +#include +#include + +using namespace torch::autograd; +using namespace torch::test; + +TEST(GradModeTest, TestRequiresGradFunctionalOp) { + torch::AutoGradMode mode(false); + for (bool requires_grad : {true, false}) { + torch::Tensor c = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + + torch::Tensor func_out = c * c; + ASSERT_FALSE(func_out.requires_grad()); + ASSERT_TRUE(func_out.is_leaf()); + } +} + +TEST(GradModeTest, TestRequiresGradInplaceOp) { + torch::AutoGradMode mode(false); + for (bool requires_grad : {true, false}) { + torch::Tensor c = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + + c.mul_(2); + ASSERT_EQ(c.requires_grad(), requires_grad); + } +} + +TEST(GradModeTest, TestRequiresGradViewOp) { + torch::AutoGradMode mode(false); + for (bool requires_grad : {true, false}) { + torch::Tensor c = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + + torch::Tensor view_out = c.view({2, 3}); + ASSERT_EQ(view_out.requires_grad(), requires_grad); + ASSERT_TRUE(view_out.is_leaf()); + } +} + +TEST(GradModeTest, TestRequiresGradViewOpExiting) { + for (bool requires_grad : {true, false}) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + torch::Tensor a = s.clone(); + torch::Tensor view_out, tmp; + + { + torch::AutoGradMode mode(false); + view_out = a.view( + {2, 3}); // go through kernels: VariableType, ADInplaceOrView, CPU + assert_tensor_creation_meta( + view_out, torch::autograd::CreationMeta::NO_GRAD_MODE); + ASSERT_EQ(view_out.requires_grad(), requires_grad); + ASSERT_TRUE(view_out.is_leaf()); + } + + tmp = view_out * view_out; + ASSERT_EQ(tmp.requires_grad(), requires_grad); + if (requires_grad) { + tmp.backward(torch::ones_like(tmp)); + // TODO: this behavior is a side effect of issue #11390. + ASSERT_FALSE(view_out.grad().defined()); + } + + if (requires_grad) { + ASSERT_THROWS_WITH( + view_out.mul_( + 2), // go through kernels: VariableType, ADInplaceOrView, CPU + "A view was created in no_grad mode and is being modified inplace"); + } else { + view_out.mul_(2); + } + + tmp = view_out.view({2, 3}); + ASSERT_EQ(tmp.requires_grad(), requires_grad); + assert_tensor_creation_meta( + tmp, torch::autograd::CreationMeta::NO_GRAD_MODE); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/inference_mode.cpp b/western_ai/usa/pytorch/test/cpp/api/inference_mode.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0ef950cc4b7da485d88a0d3dbcb1710392319259 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/inference_mode.cpp @@ -0,0 +1,657 @@ +#include +#include +#include + +using namespace torch::autograd; +using namespace torch::test; + +namespace { +torch::Tensor functional_op(torch::Tensor& x) { + return x * x; +} + +void inplace_op(torch::Tensor& x) { + x.mul_(1); +} + +torch::Tensor view_op(torch::Tensor& x) { + return x.view({2, 3}); +} + +/* + Only the following combos of Autograd & ADInplaceOrView keys on tensors are + valid: + - Autograd=true, ADInplaceOrView=true (normal tensor) + - Autograd=false, ADInplaceOrView=false (inference tensor) + Tensors created in InferenceMode are mostly inference tensors. The only + exception is that view of normal tensors created in InferenceMode still + produce normal tensor. +*/ +void assert_TLS_states(bool inference_mode) { + ASSERT_EQ(InferenceMode::is_enabled(), inference_mode); + ASSERT_FALSE(c10::impl::tls_is_dispatch_key_excluded( + c10::DispatchKey::ADInplaceOrView)); + ASSERT_FALSE(c10::impl::tls_is_dispatch_keyset_included( + c10::autograd_dispatch_keyset)); + ASSERT_EQ( + c10::impl::tls_is_dispatch_keyset_excluded(c10::autograd_dispatch_keyset), + inference_mode); + ASSERT_EQ( + c10::impl::tls_is_dispatch_key_included( + c10::DispatchKey::ADInplaceOrView), + !inference_mode); + ASSERT_EQ(GradMode::is_enabled(), !inference_mode); +} +} // namespace + +TEST(InferenceModeTest, TestTLSState) { + assert_TLS_states(false); + { + InferenceMode guard; + assert_TLS_states(true); + { + InferenceMode guard(false); + assert_TLS_states(false); + } + assert_TLS_states(true); + } + assert_TLS_states(false); +} + +TEST(InferenceModeTest, TestInferenceTensorCreation) { + { + InferenceMode guard; + // New tensor created through constructors are inference tensors. + torch::Tensor c = torch::ones({1, 2, 3}); + ASSERT_FALSE(c.requires_grad()); + ASSERT_TRUE(c.is_inference()); + + // requires_grad doesn't change inference tensor behavior inside + // InferenceMode. + torch::Tensor tmp = torch::ones({1, 2, 3}).set_requires_grad(true); + ASSERT_TRUE(tmp.requires_grad()); + ASSERT_TRUE(tmp.is_inference()); + + tmp = torch::ones({1, 2, 3}).set_requires_grad(false); + ASSERT_FALSE(tmp.requires_grad()); + ASSERT_TRUE(tmp.is_inference()); + } +} + +TEST(InferenceModeTest, TestExistingAutogradSession) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(true); + torch::Tensor a = s.clone(); + + // Save `a` in an existing autograd session + torch::Tensor out = a * a; + { + InferenceMode guard; + inplace_op(a); + } + // Performing backward should trigger error since `a`'s version has been + // bumped. + ASSERT_THROWS_WITH( + out.backward(torch::ones_like(out)), + "one of the variables needed for gradient computation has been modified by an inplace operation") +} + +TEST(InferenceModeTest, TestInferenceTensorInInferenceModeFunctionalOp) { + c10::InferenceMode guard; + for (bool requires_grad : {true, false}) { + torch::Tensor c = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + + torch::Tensor func_out = functional_op(c); // go through kernels: CPU + ASSERT_TRUE(func_out.is_inference()); + ASSERT_FALSE(func_out.requires_grad()); + } +} + +TEST(InferenceModeTest, TestInferenceTensorInInferenceModeInplaceOp) { + c10::InferenceMode guard; + for (bool requires_grad : {true, false}) { + torch::Tensor c = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + + inplace_op(c); // go through kernels: CPU + ASSERT_TRUE(c.is_inference()); + ASSERT_EQ(c.requires_grad(), requires_grad); + } +} + +TEST(InferenceModeTest, TestInferenceTensorInInferenceModeViewOp) { + c10::InferenceMode guard; + for (bool requires_grad : {true, false}) { + torch::Tensor c = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + + torch::Tensor view_out = view_op(c); // go through kernels: CPU + ASSERT_TRUE(view_out.is_inference()); + // Note this is different from NoGradMode but makes sense. + ASSERT_FALSE(view_out.requires_grad()); + ASSERT_FALSE(view_out.is_view()); + } +} + +TEST(InferenceModeTest, TestInferenceTensorInNormalModeFunctionalOp) { + torch::Tensor inference_tensor; + for (bool requires_grad : {true, false}) { + { + InferenceMode guard; + inference_tensor = + torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + } + + // Due to issue #54614, this might run slower compared to InferenceMode + // since intermediate tensors are normal tensors, and they might dispatch to + // VariableType kernels. This is fine since users can easily fix it by + // moving it inside InferenceMode block. + torch::Tensor tmp = + functional_op(inference_tensor); // go through kernels: + // ADInplaceOrView(fallthrough), CPU + ASSERT_FALSE(tmp.is_inference()); + ASSERT_FALSE(tmp.requires_grad()); + } +} + +TEST(InferenceModeTest, TestInferenceTensorInNormalModeInplaceOp) { + torch::Tensor inference_tensor; + for (bool requires_grad : {true, false}) { + { + InferenceMode guard; + inference_tensor = + torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + } + ASSERT_THROWS_WITH( + inplace_op( + inference_tensor), // go through kernels: ADInplaceOrView, CPU + "Inplace update to inference tensor outside InferenceMode is not allowed"); + } +} + +TEST(InferenceModeTest, TestInferenceTensorInNormalModeViewOp) { + torch::Tensor inference_tensor; + for (bool requires_grad : {true, false}) { + { + InferenceMode guard; + inference_tensor = + torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + } + torch::Tensor out = + view_op(inference_tensor); // go through kernels: ADInplaceOrView, CPU + ASSERT_TRUE(out.is_inference()); + ASSERT_FALSE(out.requires_grad()); + ASSERT_FALSE(out.is_view()); + ASSERT_TRUE(out.is_leaf()); + } +} + +TEST(InferenceModeTest, TestNormalTensorInplaceOutputInInferenceMode) { + for (bool requires_grad : {true, false}) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + torch::Tensor a = s.clone(); + + { + c10::InferenceMode guard; + + inplace_op(a); // go through kernels: ADInplaceOrView, CPU + ASSERT_FALSE(a.is_inference()); + ASSERT_EQ(a.requires_grad(), requires_grad); + + // inplace -> inplace + inplace_op(a); // go through kernels: ADInplaceOrView, CPU + ASSERT_FALSE(a.is_inference()); + ASSERT_EQ(a.requires_grad(), requires_grad); + + // inplace -> inplace -> view + torch::Tensor view_out = + view_op(a); // go through kernels: ADInplaceOrView, CPU + ASSERT_FALSE(view_out.is_inference()); + ASSERT_EQ(view_out.requires_grad(), requires_grad); + } + } +} + +TEST(InferenceModeTest, TestNormalTensorInplaceOutputInNormalMode) { + for (bool requires_grad : {true, false}) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + torch::Tensor a = s.clone(); + + { + c10::InferenceMode guard; + + inplace_op(a); // go through kernels: ADInplaceOrView, CPU + ASSERT_FALSE(a.is_inference()); + ASSERT_EQ(a.requires_grad(), requires_grad); + } + + torch::Tensor tmp = functional_op(a); // go through kernels: VariableType, + // ADInplaceOrView(fallthrough), CPU + ASSERT_FALSE(tmp.is_inference()); + ASSERT_EQ(tmp.requires_grad(), requires_grad); + + inplace_op(a); // go through kernels: VariableType, ADInplaceOrView, CPU + ASSERT_FALSE(a.is_inference()); + ASSERT_EQ(a.requires_grad(), requires_grad); + + tmp = view_op(a); // go through kernels: VariableType, ADInplaceOrView, CPU + ASSERT_FALSE(tmp.is_inference()); + ASSERT_EQ(tmp.requires_grad(), requires_grad); + } +} + +TEST(InferenceModeTest, TestNormalTensorViewOutputInInferenceMode) { + for (bool requires_grad : {true, false}) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + torch::Tensor a = s.clone(); + torch::Tensor view_out, tmp; + + { + c10::InferenceMode guard; + // View ops on normal tensor produce normal tensors as output. + // - For view ops it has both dispatch keys since due to the way we create + // view Tensors in alias_with_sizes_and_strides: + // ``` + // auto impl = c10::make_intrusive( + // Storage(self.storage()), self.key_set(), self.dtype()); + // ``` + // In addition, these view output tensors are normal in the sense they + // have both Autograd and ADInplaceOrView keys. But they're still + // special since they'll have CreationMeta::INFERENCE_MODE. In other + // words they behave exactly the same as a view tensor created in + // no_grad mode. + + view_out = view_op(a); // go through kernels: ADInplaceOrView, CPU + ASSERT_FALSE(view_out.is_inference()); + assert_tensor_creation_meta(view_out, CreationMeta::INFERENCE_MODE); + ASSERT_EQ(view_out.requires_grad(), requires_grad); + ASSERT_TRUE(view_out.is_leaf()); + + // view -> view + tmp = view_op(view_out); // go through kernels: ADInplaceOrView, CPU + ASSERT_FALSE(tmp.is_inference()); + assert_tensor_creation_meta(tmp, CreationMeta::INFERENCE_MODE); + ASSERT_EQ(tmp.requires_grad(), requires_grad); + ASSERT_TRUE(tmp.is_leaf()); + + // view -> view -> inplace + inplace_op(tmp); // kernels: ADInplaceOrView, CPU + assert_tensor_creation_meta(tmp, CreationMeta::INFERENCE_MODE); + ASSERT_FALSE(tmp.is_inference()); + ASSERT_EQ(tmp.requires_grad(), requires_grad); + ASSERT_TRUE(tmp.is_leaf()); + ASSERT_EQ(a._version(), tmp._version()); + } + } +} + +TEST(InferenceModeTest, TestNormalTensorViewOutputInNormalMode) { + for (bool requires_grad : {true, false}) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + torch::Tensor a = s.clone(); + torch::Tensor view_out, tmp; + + { + c10::InferenceMode guard; + view_out = view_op(a); // go through kernels: ADInplaceOrView, CPU + ASSERT_FALSE(view_out.is_inference()); + assert_tensor_creation_meta(view_out, CreationMeta::INFERENCE_MODE); + ASSERT_EQ(view_out.requires_grad(), requires_grad); + ASSERT_TRUE(view_out.is_leaf()); + } + + tmp = functional_op(view_out); + ASSERT_FALSE(view_out.is_inference()); + ASSERT_EQ(tmp.requires_grad(), requires_grad); + + if (requires_grad) { + ASSERT_THROWS_WITH( + inplace_op(view_out), // go through kernels: VariableType, + // ADInplaceOrView, CPU + "A view was created in inference mode and is being modified inplace") + } else { + inplace_op(view_out); + } + + tmp = view_op(view_out); + ASSERT_FALSE(view_out.is_inference()); + ASSERT_EQ(tmp.requires_grad(), requires_grad); + } +} + +TEST(InferenceModeTest, TestMixInferenceAndNormalTensorFunctionalOp) { + for (bool requires_grad : {true, false}) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + torch::Tensor c; + { + InferenceMode guard; + c = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + } + + // add(Tensor, Tensor) is safe with inference tensor since it doesn't save + // any variable for backward. + torch::Tensor out = c.add(s); // go through kernels: VariableType, + // ADInplaceOrView(fallthrough), CPU + ASSERT_FALSE(out.is_inference()); + ASSERT_EQ(out.requires_grad(), requires_grad); + if (requires_grad) { + // leaf inference tensor with requires_grad=true can still have gradient. + // Note this behavior is different from NoGradMode which has empty grad. + out.backward(torch::ones_like(out)); + assert_tensor_equal(c.grad(), torch::ones_like(c)); + } + + if (requires_grad) { + // mul(self, other) saves variable when requires_grad=true + ASSERT_THROWS_WITH( + c.mul(s), "Inference tensors cannot be saved for backward."); + + // Inference tensor in TensorList input + // stack does not capture anymore, so disabled + // TODO: find alternative Function that captures a list (maybe custom fn) + /* + std::vector inputs = {s, c}; + ASSERT_THROWS_WITH( + torch::stack(inputs), // go through kernels: VariableType(ERROR)!, + // ADInplaceOrView(fallthrough), CPU + "Inference tensors cannot be saved for backward.") + */ + } + } +} + +TEST(InferenceModeTest, TestMixInferenceAndNormalTensorInplaceOp) { + for (bool requires_grad : {true, false}) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + torch::Tensor a = s.clone(); + torch::Tensor c; + { + InferenceMode guard; + c = torch::ones({1, 2, 3}); + } + + if (requires_grad) { + ASSERT_THROWS_WITH( + a.mul_(c), // go through kernels: VariableType(ERROR!), InferenceMode, + // CPU + "Inference tensors cannot be saved for backward."); + + ASSERT_THROWS_WITH( + torch::mul_out( + /*out=*/c, s, s), // go through kernels: VariableType(ERROR!), + // ADInplaceOrView, CPU + "out=... arguments don't support automatic differentiation, but one of the arguments requires grad") + } else { + a.mul_(c); + + ASSERT_THROWS_WITH( + torch::mul_out(/*out=*/c, s, s), // go through kernels: VariableType, + // ADInplaceOrView(ERROR!), CPU + "Inplace update to inference tensor outside InferenceMode is not allowed"); + } + } +} + +TEST(InferenceModeTest, TestMixInferenceAndNormalTensorViewOp) { + for (bool requires_grad : {true, false}) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + torch::Tensor c; + { + InferenceMode guard; + c = torch::ones({1, 2, 3}); + } + + // view_as is a composite op which calls view() with only one tensor + // argument. So there isn't a mixed inference tensor and normal tensor + // inputs for view ops. + torch::Tensor tmp1 = + c.view_as(s); // go through kernels: ADInplaceOrView, CPU + ASSERT_TRUE(tmp1.is_inference()); + ASSERT_FALSE(tmp1.requires_grad()); + + // This is fine since it's equivalent as s.view(c.sizes()) which + // isn't a mixed input scenario. + torch::Tensor tmp2 = + s.view_as(c); // go through kernels: VariableType, ADInplaceOrView, CPU + ASSERT_FALSE(tmp2.is_inference()); + ASSERT_EQ(tmp2.requires_grad(), requires_grad); + } +} + +TEST(InferenceModeTest, TestHandleDirectViewOnRebase) { + for (bool requires_grad : {true, false}) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + torch::Tensor a = s.clone(); + torch::Tensor view_out; + { + InferenceMode guard; + view_out = view_op(a); // go through kernels: ADInplaceOrView, CPU + } + if (requires_grad) { + ASSERT_THROWS_WITH( + inplace_op(view_out), + "A view was created in inference mode and is being modified inplace") + } else { + inplace_op(view_out); + } + } +} + +TEST(InferenceModeTest, TestHandleInDirectViewOnRebase) { + for (bool requires_grad : {true, false}) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + torch::Tensor a = s.clone(); + torch::Tensor view_out; + { + InferenceMode guard; + view_out = view_op(a); // go through kernels: ADInplaceOrView, CPU + } + inplace_op(a); + if (requires_grad) { + ASSERT_THROWS_WITH( + view_out.grad_fn(), + "A view was created in inference mode and its base or another view of its base has been modified inplace"); + } else { + view_out.grad_fn(); + } + } +} + +TEST(InferenceModeTest, TestCreationMetaPropagation) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(true); + torch::Tensor b, c; + { + InferenceMode guard; + b = s.view_as(s); + } + ASSERT_THROWS_WITH( + b.add_(1), + "A view was created in inference mode and is being modified inplace"); + { + AutoGradMode mode(false); + c = b.view_as(b); + } + ASSERT_THROWS_WITH( + c.add_(1), + "A view was created in inference mode and is being modified inplace"); +} + +TEST(InferenceModeTest, TestCreationMetaPropagationInput) { + torch::Tensor s = torch::ones({2, 2, 3}).set_requires_grad(true); + auto s_view = s.view_as(s); + std::vector b, c; + { + InferenceMode guard; + b = s_view.split_with_sizes({1, 1}); + + s = s.view_as(s); + c = s.split_with_sizes({1, 1}); + } + for (auto& b_el : b) { + assert_tensor_creation_meta(b_el, CreationMeta::INFERENCE_MODE); + ASSERT_THROWS_WITH( + b_el.add_(1), + "A view was created in inference mode and is being modified inplace"); + } + for (auto& c_el : c) { + assert_tensor_creation_meta(c_el, CreationMeta::INFERENCE_MODE); + ASSERT_THROWS_WITH( + c_el.add_(1), + "A view was created in inference mode and is being modified inplace"); + } +} + +TEST(InferenceModeTest, TestInplaceCopyOnInferenceTensor) { + for (bool requires_grad : {true, false}) { + torch::Tensor s = torch::ones({1, 2, 3}).set_requires_grad(requires_grad); + torch::Tensor t; + { + InferenceMode guard; + t = torch::ones({1, 2, 3}); + t.copy_(s); + ASSERT_TRUE(t.is_inference()); + ASSERT_FALSE(t.requires_grad()); + } + + ASSERT_THROWS_WITH( + t.copy_(s), + "Inplace update to inference tensor outside InferenceMode is not allowed"); + } +} + +TEST(InferenceModeTest, TestSetRequiresGradInNormalMode) { + torch::Tensor t; + { + InferenceMode guard; + t = torch::ones({1, 2, 3}); + } + t.set_requires_grad(false); + ASSERT_THROWS_WITH( + t.set_requires_grad(true), + "Setting requires_grad=True on inference tensor outside InferenceMode is not allowed."); +} + +TEST(InferenceModeTest, TestAccessVersionCounter) { + torch::Tensor t; + { + InferenceMode guard; + t = torch::ones({1, 2, 3}); + ASSERT_THROWS_WITH( + t.unsafeGetTensorImpl()->version_counter().current_version(), + "Inference tensors do not track version counter."); + t.unsafeGetTensorImpl()->bump_version(); + } + ASSERT_THROWS_WITH( + t.unsafeGetTensorImpl()->version_counter().current_version(), + "Inference tensors do not track version counter."); + ASSERT_THROWS_WITH( + t.unsafeGetTensorImpl()->bump_version(), + "Inplace update to inference tensor outside InferenceMode is not allowed."); + // Suggested workaround + torch::Tensor c = t.clone(); + uint32_t v = c.unsafeGetTensorImpl()->version_counter().current_version(); + c.unsafeGetTensorImpl()->bump_version(); + ASSERT_EQ( + c.unsafeGetTensorImpl()->version_counter().current_version(), v + 1); +} + +TEST(InferenceModeTest, TestInplaceUpdateInferenceTensorWithNormalTensor) { + torch::Tensor s = torch::ones({1, 2, 3}); + torch::Tensor t; + { + InferenceMode guard; + t = torch::ones({1, 2, 3}); + // Testing both copy_ from VariableTypeManual and add_ from generated code. + s.copy_(t); + s.add_(t); + t.add_(s); + t.copy_(s); + } + s.copy_(t); + s.add_(t); + ASSERT_THROWS_WITH( + t.copy_(s), + "Inplace update to inference tensor outside InferenceMode is not allowed"); + + ASSERT_THROWS_WITH( + t.add_(s), + "Inplace update to inference tensor outside InferenceMode is not allowed"); +} + +TEST(InferenceModeTest, TestComplexViewInInferenceMode) { + torch::Tensor s = torch::ones({3, 3, 2}); + torch::Tensor t = torch::view_as_complex(s); + { + InferenceMode guard; + torch::Tensor tmp; + + tmp = torch::view_as_real(t); + ASSERT_FALSE(tmp.is_inference()); + tmp = torch::view_as_complex(s); + ASSERT_FALSE(tmp.is_inference()); + + torch::Tensor e = torch::ones({3, 3, 2}); + tmp = torch::view_as_complex(e); + ASSERT_TRUE(tmp.is_inference()); + tmp = torch::view_as_real(tmp); + ASSERT_TRUE(tmp.is_inference()); + } +} + +TEST(InferenceModeTest, TestComplexViewInNormalMode) { + torch::Tensor s; + { + InferenceMode guard; + s = torch::ones({3, 3, 2}); + } + torch::Tensor tmp = torch::view_as_complex(s); + ASSERT_TRUE(tmp.is_inference()); + tmp = torch::view_as_real(tmp); + ASSERT_TRUE(tmp.is_inference()); +} + +TEST(InferenceModeTest, TestCustomFunction) { + struct MyFunction : public Function { + static Variable forward( + AutogradContext* ctx, + Variable var1, + int mul, + Variable var2) { + ctx->saved_data["mul"] = mul; + ctx->save_for_backward({var1, var2}); + return var1 + mul * var2 + var1 * var2; + } + + static variable_list backward( + AutogradContext* ctx, + variable_list grad_output) { + int mul = ctx->saved_data["mul"].toInt(); + auto saved = ctx->get_saved_variables(); + auto var1 = saved[0]; + auto var2 = saved[1]; + variable_list output = { + grad_output[0] + grad_output[0] * var2, + Variable(), + grad_output[0] * mul + grad_output[0] * var1}; + return output; + } + }; + + { + InferenceMode guard; + torch::Tensor var1 = torch::ones({3, 3}).set_requires_grad(true); + auto var2 = var1.clone(); + int mul = 2; + // If InferenceMode didn't set NoGradGuard automatically, this line + // would error out when trying to save `var1` and `var2` for backward. + auto y = MyFunction::apply(var1, mul, var2); + torch::Tensor expected = var1 + mul * var2 + var1 * var2; + assert_tensor_equal(y, expected); + } +} + +TEST(InferenceModeTest, TestLegacyAutoNonVariableTypeModeWarning) { + c10::WarningUtils::WarnAlways warn_always(true); + WarningCapture warnings; + at::AutoNonVariableTypeMode guard; + ASSERT_TRUE( + warnings.str().find("AutoNonVariableTypeMode is deprecated") != + std::string::npos); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/init.cpp b/western_ai/usa/pytorch/test/cpp/api/init.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b3af895e0c80cc45e998b4cdd961da4140b0501a --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/init.cpp @@ -0,0 +1,131 @@ +#include + +#include +#include + +#include +#include + +#include +#include + +void check_exact_values( + const std::vector& parameters, + const std::vector>& expected_parameters) { + ASSERT_EQ(parameters.size(), expected_parameters.size()); + + for (const auto i : c10::irange(parameters.size())) { + auto layerParameters = parameters[i]; + auto expectedLayerParameters = expected_parameters[i]; + + if (static_cast(layerParameters.size(0)) != + expectedLayerParameters.size()) { + std::cout << "layer #" << i + << " layerParameters size: " << layerParameters.size(0) + << " != " + << " expectedLayerParameters size: " + << expectedLayerParameters.size() << std::endl; + ASSERT_TRUE(false); + } + + for (const auto p : c10::irange(layerParameters.size(0))) { + // Always compare using double dtype, regardless of the original dtype of + // the tensors + auto tensor = layerParameters[p].to(torch::kFloat64); + auto expectedTensor = expectedLayerParameters[p].to(torch::kFloat64); + + if (!tensor.allclose(expectedTensor, /*rtol=*/1e-3, /*atol=*/5e-4)) { + std::cout << "layer " << i << ": " << tensor << " != " << expectedTensor + << " (parameter " << p << ')' << std::endl; + ASSERT_TRUE(false); + } + } + } +} + +void check_initializer_against_baseline( + std::function initializer, + std::vector> expected) { + torch::manual_seed(0); + + auto layer1 = torch::nn::Linear(7, 15); + initializer(layer1->weight); + layer1->to(torch::kFloat64); + + auto layer2 = torch::nn::Linear(15, 15); + initializer(layer2->weight); + layer2->to(torch::kFloat64); + + auto layer3 = torch::nn::Linear(15, 2); + initializer(layer3->weight); + layer3->to(torch::kFloat64); + + auto parameters = std::vector{ + layer1->weight, + layer2->weight, + layer3->weight, + }; + + check_exact_values(parameters, expected); +} + +TEST(InitTest, ProducesPyTorchValues_XavierUniform) { + auto expected = expected_parameters::Xavier_Uniform(); + auto initializer = [](torch::Tensor tensor) { + torch::nn::init::xavier_uniform_(tensor); + }; + check_initializer_against_baseline(initializer, expected); +} + +TEST(InitTest, ProducesPyTorchValues_XavierNormal) { + auto expected = expected_parameters::Xavier_Normal(); + auto initializer = [](torch::Tensor tensor) { + torch::nn::init::xavier_normal_(tensor); + }; + check_initializer_against_baseline(initializer, expected); +} + +TEST(InitTest, ProducesPyTorchValues_KaimingNormal) { + auto expected = expected_parameters::Kaiming_Normal(); + auto initializer = [](torch::Tensor tensor) { + torch::nn::init::kaiming_normal_(tensor); + }; + check_initializer_against_baseline(initializer, expected); +} + +TEST(InitTest, ProducesPyTorchValues_KaimingUniform) { + auto expected = expected_parameters::Kaiming_Uniform(); + auto initializer = [](torch::Tensor tensor) { + torch::nn::init::kaiming_uniform_(tensor); + }; + check_initializer_against_baseline(initializer, expected); +} + +TEST(InitTest, CanInitializeTensorThatRequiresGrad) { + auto tensor = torch::empty({3, 4}, torch::requires_grad()); + ASSERT_THROWS_WITH( + tensor.fill_(1), + "a leaf Variable that requires grad " + "is being used in an in-place operation"); + ASSERT_EQ(torch::nn::init::ones_(tensor).sum().item(), 12); +} + +TEST(InitTest, CalculateGainWithTanh) { + double gain = torch::nn::init::calculate_gain(torch::kTanh); + ASSERT_DOUBLE_EQ(gain, 5.0 / 3.0); +} + +TEST(InitTest, CalculateGainWithRelu) { + double gain = torch::nn::init::calculate_gain(torch::kReLU); + ASSERT_DOUBLE_EQ(gain, std::sqrt(2.0)); +} + +TEST(InitTest, CalculateGainWithLeakyRelu) { + double gain = torch::nn::init::calculate_gain(torch::kLeakyReLU); + ASSERT_DOUBLE_EQ(gain, std::sqrt(2.0 / (1 + pow(0.01, 2)))); +} + +TEST(InitTest, CanInitializeCnnWithOrthogonal) { + torch::nn::Conv2d conv_layer(torch::nn::Conv2dOptions(3, 2, 3).stride(2)); + torch::nn::init::orthogonal_(conv_layer->named_parameters()["weight"]); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/init_baseline.h b/western_ai/usa/pytorch/test/cpp/api/init_baseline.h new file mode 100644 index 0000000000000000000000000000000000000000..e538e2a07628ae9711a2591d65a88c451dc13ed8 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/init_baseline.h @@ -0,0 +1,1621 @@ +// @generated from /home/josh/git/pytorch/test/cpp/api/init_baseline.py + +#include + +#include + +namespace expected_parameters { + +inline std::vector> Xavier_Uniform() { + return { + { + torch::tensor( + {0.20493895, + 0.03128183, + -0.25481248, + 0.2471149, + -0.5009514, + -0.30953097, + -0.13073051}), + torch::tensor( + {-0.25438663, + -0.18269452, + -0.42803344, + -0.111086845, + 0.11163068, + -0.34021688, + -0.026800662}), + torch::tensor( + {0.37384087, + -0.053685665, + 0.014514029, + -0.04505247, + 0.10569024, + 0.33205634, + 0.4946832}), + torch::tensor( + {0.3316471, + 0.49581498, + -0.03776875, + -0.46913308, + -0.24757874, + 0.35559112, + -0.003385365}), + torch::tensor( + {-0.259574, + -0.40019324, + -0.48873278, + -0.4407689, + -0.10592803, + 0.28639567, + 0.2823406}), + torch::tensor( + {-0.5036581, + 0.32575953, + -0.40865222, + -0.110405415, + -0.21175116, + -0.10059005, + -0.10253662}), + torch::tensor( + {-0.46862572, + -0.45091572, + -0.08171874, + 0.0067536235, + -0.2372373, + 0.19672471, + -0.47004014}), + torch::tensor( + {-0.035244048, + 0.45926183, + -0.21301463, + 0.47157794, + 0.18912864, + -0.47129482, + 0.33041543}), + torch::tensor( + {-0.0602628, + -0.23312837, + 0.41760528, + -0.42201394, + 0.05603814, + -0.10933924, + 0.37293315}), + torch::tensor( + {0.14577848, + 0.25093573, + 0.18443125, + -0.1255838, + -0.10982844, + -0.43036246, + 0.28296888}), + torch::tensor( + {0.4146415, + 0.35732478, + -0.36837178, + 0.023291528, + -0.3681398, + -0.28748098, + -0.304308}), + torch::tensor( + {0.17847049, + -0.3112055, + -0.011393666, + 0.021969378, + 0.3366434, + -0.39476636, + -0.35851932}), + torch::tensor( + {-0.3032406, + 0.3655283, + -0.18772447, + 0.44049758, + 0.18884337, + 0.066128254, + -0.0038875341}), + torch::tensor( + {-0.10323581, + 0.06552267, + -0.119249105, + -0.0036694407, + 0.066633284, + -0.40849328, + -0.2737187}), + torch::tensor( + {0.4216993, + -0.4238164, + -0.037499547, + 0.51661307, + 0.1886499, + 0.014786005, + -0.45257226}), + }, + { + torch::tensor( + {0.2400673, + 0.06230098, + -0.29922318, + -0.3467335, + -0.13797283, + 0.19630808, + 0.44112986, + 0.25716078, + -0.05036038, + 0.15680045, + -0.43874466, + -0.3819657, + 0.20867342, + -0.25330856, + 0.2151525}), + torch::tensor( + {-0.31570244, + -0.22150977, + -0.3683649, + 0.23337424, + -0.045568883, + 0.34417605, + 0.27676803, + 0.24746233, + 0.014380664, + -0.13826859, + -0.09723839, + 0.05943495, + 0.22168803, + -0.3133133, + 0.37533647}), + torch::tensor( + {-0.04862556, + -0.37474066, + -0.24196841, + 0.39570248, + 0.408989, + -0.41424486, + 0.31541896, + 0.2241252, + 0.264714, + 0.37857938, + -0.24102591, + 0.1412192, + 0.18301463, + -0.13214865, + 0.14966142}), + torch::tensor( + {-0.12866935, + 0.27649486, + -0.12408146, + -0.16671932, + 0.112585604, + 0.15862381, + -0.21849588, + 0.03953293, + 0.25917625, + -0.044496298, + 0.13610226, + -0.107862085, + 0.15674818, + -0.32395893, + -0.26297447}), + torch::tensor( + {-0.22700138, + 0.41099417, + -0.12033805, + -0.0012210608, + -0.21667299, + 0.44644886, + 0.43678015, + -0.3372825, + -0.3625426, + -0.33898476, + -0.002156794, + -0.113996506, + -0.29272172, + -0.16040304, + 0.084492445}), + torch::tensor( + {-0.23366496, + 0.09909475, + -0.10255319, + -0.21670331, + 0.061440647, + 0.36772507, + -0.30235183, + 0.02076611, + -0.16491795, + 0.4388535, + -0.4242998, + -0.42872632, + 0.44067752, + -0.28294754, + 0.08574128}), + torch::tensor( + {-0.038597256, + -0.09420869, + -0.09988415, + 0.28417766, + 0.021375448, + -0.43541366, + -0.2640171, + -0.15245524, + 0.2250452, + -0.28940699, + 0.42168647, + -0.09960759, + -0.08030257, + 0.3504178, + 0.22477299}), + torch::tensor( + {0.37929094, + 0.25868815, + -0.13566399, + -0.2967139, + -0.033274055, + 0.37013078, + -0.15009373, + -0.41473246, + 0.18332559, + 0.43534964, + -0.12731421, + -0.3703034, + -0.40564942, + 0.112071455, + -0.03386289}), + torch::tensor( + {-0.22583716, + 0.090396106, + 0.1698333, + 0.35567743, + 0.34720868, + -0.066940606, + -0.39433825, + -0.40411252, + 0.41755867, + 0.19769311, + 0.19494373, + -0.3869386, + 0.4141268, + 0.4236647, + 0.4037714}), + torch::tensor( + {-0.37726268, + -0.16874415, + -0.3075773, + 0.4234959, + -0.19215873, + -0.2041774, + 0.23430097, + -0.20687759, + -0.22026259, + -0.03911844, + -0.042985946, + -0.34836975, + 0.3728277, + -0.19727562, + 0.15863329}), + torch::tensor( + {0.38897908, + 0.22553718, + 0.063316464, + 0.3805148, + 0.060117245, + -0.20690633, + 0.4230638, + 0.10584676, + -0.43633774, + -0.12731794, + -0.30462736, + 0.39209586, + -0.07385549, + -0.40764633, + -0.028113335}), + torch::tensor( + {0.2808559, + 0.11618626, + 0.14141095, + 0.041534156, + 0.16672957, + -0.10896447, + -0.17790166, + -0.41801435, + -0.3369025, + 0.19382352, + -0.26480114, + 0.06416017, + 0.14274675, + 0.03166446, + -0.28995082}), + torch::tensor( + {0.42768306, + -0.26005447, + 0.36783344, + -0.35576212, + -0.10757655, + 0.24327022, + -0.18272284, + 0.3756786, + -0.30775294, + -0.37555724, + -0.20165718, + 0.07229227, + 0.41177452, + -0.21350017, + 0.15993619}), + torch::tensor( + {-0.112119585, + -0.09698379, + 0.3288377, + -0.34658423, + 0.047500044, + 0.42056376, + -0.061452597, + 0.34723365, + -0.13772246, + 0.35999, + -0.43260327, + -0.06445408, + -0.07855147, + 0.14493519, + 0.17545414}), + torch::tensor( + {0.34337813, + -0.066628546, + -0.01773429, + 0.3062569, + -0.121003985, + 0.39204246, + -0.29776025, + -0.04839292, + -0.024020255, + 0.1995511, + 0.30574924, + -0.07088503, + -0.37050778, + 0.22159088, + 0.13377577}), + }, + { + torch::tensor( + {-0.095182985, + -0.35154456, + -0.33943355, + 0.14092201, + 0.5576401, + -0.4759267, + 0.35954505, + -0.30801514, + -0.11571318, + 0.47157025, + -0.1343652, + 0.05409521, + -0.41528726, + 0.5057125, + -0.076797724}), + torch::tensor( + {-0.4345107, + 0.17395526, + -0.42240727, + -0.4714136, + 0.036191404, + 0.47101927, + -0.16811755, + 0.2796178, + 0.51051295, + 0.39403576, + -0.3116357, + -0.065123916, + -0.18695068, + -0.47772023, + 0.00024545193}), + }, + }; +} + +inline std::vector> Xavier_Normal() { + return { + { + torch::tensor( + {-0.21151732, + 0.31257284, + -0.18201339, + -0.3855622, + 0.028025549, + -0.20083663, + 0.18333313}), + torch::tensor( + {-0.22010927, + 0.41458952, + 0.19888626, + 0.14368738, + -0.30642822, + 0.05438269, + 0.032663286}), + torch::tensor( + {-0.22758777, + 0.07366481, + 0.34382868, + -0.027100464, + 0.22004184, + -0.5563846, + -0.0075437957}), + torch::tensor( + {0.4128839, + 0.80112267, + 0.29702467, + 0.11372463, + 0.3320348, + -0.34456047, + 0.011332423}), + torch::tensor( + {0.81295794, + 0.37259677, + 0.16366935, + 0.15845336, + -0.2500635, + -0.42430383, + 0.49051273}), + torch::tensor( + {0.051942326, + -0.48588628, + -0.14455895, + -0.04322197, + -0.09566793, + 0.17296337, + 0.3008876}), + torch::tensor( + {0.16390441, + 0.023760417, + 0.26016212, + -0.005876346, + 0.29881194, + -0.23449583, + -0.090267815}), + torch::tensor( + {-0.05661722, + 0.57766485, + 0.20810172, + -0.70001936, + -0.36073124, + 0.059406225, + -0.35497883}), + torch::tensor( + {0.034237247, + 0.33308733, + -0.4206602, + 0.14325368, + -0.24534757, + 0.27866775, + -0.07457621}), + torch::tensor( + {-0.42675552, + 0.29772332, + -0.44859084, + 0.17689292, + 0.04772785, + 0.03324038, + -0.24688111}), + torch::tensor( + {0.19078615, + -0.57796097, + 0.39555034, + -0.063268825, + 0.23570086, + 0.29840353, + 0.12504078}), + torch::tensor( + {-0.45496583, + 0.61388826, + 0.039676014, + -0.15409455, + -0.5167084, + -0.15379032, + -0.14318566}), + torch::tensor( + {-0.19097842, + -0.44253433, + -0.26487318, + -0.6266602, + -0.33180708, + 0.4737542, + 0.05777406}), + torch::tensor( + {0.11455678, + -0.04364589, + 0.19224901, + -0.084812194, + -0.40097243, + -0.197128, + -0.161051}), + torch::tensor( + {-0.15780471, + 0.25975332, + -0.26742947, + 0.2529001, + 0.34761095, + -0.5309911, + -0.3337848}), + }, + { + torch::tensor( + {-0.11110223, + -0.08603837, + -0.39927804, + -0.0037998888, + 0.3163225, + 0.41147578, + -0.4212508, + 0.27163807, + 0.16257726, + 0.07001552, + -0.17712006, + -0.28190005, + 0.43368816, + -0.22742769, + 0.14976384}), + torch::tensor( + {0.15162551, + 0.022756116, + -0.337443, + -0.1823836, + -0.04240044, + -0.25083202, + -0.26616275, + 0.16712677, + -0.3765372, + 0.18503848, + -0.51644665, + -0.622171, + 0.05665474, + -0.4386439, + 0.33809182}), + torch::tensor( + {-0.42894447, + 0.44781545, + -0.16271128, + -0.16386595, + 0.25165728, + 0.054181106, + 0.0077175284, + 0.44132262, + -0.18739006, + -0.37522528, + 0.15394184, + -0.32285675, + 0.2957909, + 0.19089724, + 0.32357433}), + torch::tensor( + {-0.11477537, + 0.21132338, + 0.0032186622, + 0.09303321, + -0.41689086, + -0.6386192, + 0.009335384, + -0.088360295, + -0.09855254, + -0.014685968, + -0.22642778, + 0.17631726, + 0.87642914, + -0.4308766, + 0.13190313}), + torch::tensor( + {-0.07384103, + 0.086509995, + 0.30259287, + -0.10428588, + 0.23001948, + -0.12647822, + -0.30277884, + -0.17739438, + -0.60285777, + 0.024281243, + -0.052177392, + 0.807063, + 0.5194553, + -0.08695924, + 0.084167756}), + torch::tensor( + {0.13818856, + 0.50950116, + -0.053579096, + -0.007894313, + 0.067052916, + 0.0014321166, + 0.20509668, + 0.105126604, + -0.093184546, + 0.33830634, + -0.24917552, + 0.22737288, + -0.2688545, + -0.17480324, + -0.106032155}), + torch::tensor( + {-0.4179092, + 0.1311413, + 0.5997842, + 0.059327736, + -0.13675568, + 0.47349188, + 0.0011002938, + -0.32478496, + -0.28000802, + 0.19441846, + 0.08356549, + -0.07100728, + 0.33710754, + 0.27447432, + 0.070220366}), + torch::tensor( + {-0.23930988, + -0.7056576, + -0.14566903, + -0.0707464, + 0.03609119, + 0.13131015, + 0.085740715, + -0.25335756, + 0.22949918, + 0.40511283, + -0.021134255, + -0.090214714, + 0.052266303, + -0.07446028, + -0.0024477358}), + torch::tensor( + {0.6245137, + 0.34285438, + -0.068128414, + 0.09410469, + 0.6568622, + -0.6944174, + 0.63067895, + 0.10417306, + -0.25729227, + 0.2526477, + -0.1139792, + -0.067401096, + 0.20603673, + -0.28586346, + 0.601752}), + torch::tensor( + {-0.26997554, + -0.12263605, + -0.12787469, + -0.05121646, + 0.57187945, + -0.03529285, + -0.26288798, + 0.046064075, + 0.33631605, + -0.1457349, + -0.23712163, + -0.19353649, + -0.024511583, + 0.28424242, + 0.3383595}), + torch::tensor( + {-0.07561234, + 0.10794748, + -0.0437702, + -0.56154686, + 0.18596707, + 0.07370188, + 0.059135318, + 0.33133864, + -0.35610923, + 0.36255547, + -0.24472283, + 0.052193344, + -0.09056281, + 0.14071976, + 0.3979389}), + torch::tensor( + {0.15498109, + -0.08727514, + -0.1047872, + 0.23060486, + -0.37544468, + 0.30660948, + -0.07733662, + 0.59290683, + 0.08534651, + 0.56152266, + -0.058316797, + 0.64462274, + -0.1825164, + 0.29704455, + -0.14194927}), + torch::tensor( + {0.21056584, + 0.5165385, + -0.11343903, + -0.2790672, + -0.12210965, + 0.08408014, + -0.25138792, + 0.20601495, + -0.14802553, + 0.053201754, + 0.055186458, + 0.17179312, + 0.18927598, + -0.37573355, + -0.5282227}), + torch::tensor( + {-0.045720562, + 0.16112846, + -0.43607467, + 0.21109799, + -0.008630521, + 0.32987764, + 0.13289018, + 0.0899868, + -0.2679775, + -0.24753189, + 0.23707163, + 0.12620693, + -0.6093915, + 0.25720116, + 0.33193505}), + torch::tensor( + {-0.4173836, + -0.19854523, + -0.3151219, + 0.14798103, + 0.18053184, + 0.07821397, + 0.051448464, + -0.024434408, + 0.41581196, + -0.031413753, + -0.35996363, + -0.23361506, + -0.08951727, + -0.052789558, + 0.054483347}), + }, + { + torch::tensor( + {0.41098386, + 0.45162097, + -0.11947367, + 0.4141703, + -0.2222035, + 0.1805965, + 0.3349034, + -0.3735036, + 0.13126858, + -0.14054178, + -0.25333884, + 0.56775486, + 0.17848605, + -0.079786636, + 0.24059108}), + torch::tensor( + {-0.19057135, + -0.13091972, + -0.015175606, + 0.17821532, + 0.26070553, + -0.35303566, + -0.42888534, + -0.038996607, + -0.21060736, + 0.2925792, + 0.053198263, + 0.3793282, + 0.3753942, + 0.19560203, + 0.2662913}), + }, + }; +} + +inline std::vector> Kaiming_Normal() { + return { + { + torch::tensor( + {-0.37498012, + 0.5541324, + -0.32267526, + -0.6835287, + 0.049683988, + -0.35604528, + 0.3250149}), + torch::tensor( + {-0.39021203, + 0.7349887, + 0.35258764, + 0.2547305, + -0.5432392, + 0.0964102, + 0.057905816}), + torch::tensor( + {-0.40347, + 0.13059375, + 0.6095431, + -0.048043985, + 0.3900925, + -0.9863645, + -0.01337372}), + torch::tensor( + {0.7319649, + 1.4202386, + 0.5265685, + 0.2016122, + 0.5886347, + -0.61084044, + 0.02009024}), + torch::tensor( + {1.4412204, + 0.66054344, + 0.29015473, + 0.28090778, + -0.44331524, + -0.75221026, + 0.8695861}), + torch::tensor( + {0.0920839, + -0.8613843, + -0.25627562, + -0.076624356, + -0.16960111, + 0.30663127, + 0.5334167}), + torch::tensor( + {0.29057145, + 0.042122718, + 0.46121812, + -0.010417648, + 0.52973694, + -0.41571668, + -0.16002773}), + torch::tensor( + {-0.1003716, + 1.0240903, + 0.36892492, + -1.2410016, + -0.6395081, + 0.105315976, + -0.6293102}), + torch::tensor( + {0.06069615, + 0.5905007, + -0.7457508, + 0.25396162, + -0.43495473, + 0.4940251, + -0.13220948}), + torch::tensor( + {-0.75655663, + 0.52780706, + -0.79526657, + 0.31359762, + 0.08461243, + 0.058928896, + -0.43767342}), + torch::tensor( + {0.3382277, + -1.0246153, + 0.7012358, + -0.11216363, + 0.41785294, + 0.5290129, + 0.22167361}), + torch::tensor( + {-0.8065682, + 1.0883075, + 0.070338055, + -0.27318043, + -0.91602606, + -0.2726411, + -0.25384104}), + torch::tensor( + {-0.33856857, + -0.78452945, + -0.46956995, + -1.1109498, + -0.5882311, + 0.83987635, + 0.10242246}), + torch::tensor( + {0.20308746, + -0.07737589, + 0.34082106, + -0.15035595, + -0.71084815, + -0.3494706, + -0.28551292}), + torch::tensor( + {-0.27975786, + 0.4604934, + -0.47410175, + 0.4483439, + 0.6162483, + -0.9413465, + -0.59173715}), + }, + { + torch::tensor( + {-0.15712228, + -0.12167663, + -0.5646644, + -0.005373854, + 0.44734758, + 0.58191466, + -0.59573853, + 0.38415426, + 0.22991896, + 0.0990169, + -0.2504856, + -0.3986669, + 0.6133277, + -0.3216313, + 0.21179804}), + torch::tensor( + {0.21443085, + 0.032182008, + -0.47721645, + -0.25792935, + -0.059963275, + -0.35473004, + -0.376411, + 0.23635295, + -0.532504, + 0.26168394, + -0.7303659, + -0.8798827, + 0.0801219, + -0.6203361, + 0.47813404}), + torch::tensor( + {-0.60661906, + 0.6333067, + -0.2301085, + -0.23174146, + 0.35589716, + 0.076623656, + 0.0109142335, + 0.6241244, + -0.26500958, + -0.53064865, + 0.21770664, + -0.45658842, + 0.4183115, + 0.26996946, + 0.45760322}), + torch::tensor( + {-0.16231689, + 0.29885638, + 0.004551876, + 0.13156882, + -0.5895727, + -0.9031439, + 0.013202227, + -0.124960326, + -0.13937433, + -0.020769095, + -0.32021725, + 0.24935026, + 1.239458, + -0.6093515, + 0.18653919}), + torch::tensor( + {-0.10442699, + 0.12234361, + 0.42793095, + -0.14748251, + 0.32529667, + -0.17886722, + -0.42819393, + -0.25087354, + -0.85256964, + 0.03433886, + -0.07378998, + 1.1413594, + 0.73462075, + -0.12297893, + 0.11903118}), + torch::tensor( + {0.19542812, + 0.72054344, + -0.075772285, + -0.011164244, + 0.09482714, + 0.0020253188, + 0.2900505, + 0.14867148, + -0.13178284, + 0.4784374, + -0.3523874, + 0.32155383, + -0.3802177, + -0.24720912, + -0.14995211}), + torch::tensor( + {-0.5910129, + 0.18546182, + 0.8482229, + 0.08390209, + -0.19340174, + 0.6696186, + 0.0015560504, + -0.4593153, + -0.39599115, + 0.27494922, + 0.11817944, + -0.10041946, + 0.47674203, + 0.38816532, + 0.0993066}), + torch::tensor( + {-0.33843526, + -0.9979506, + -0.20600711, + -0.10005052, + 0.05104065, + 0.1857006, + 0.12125569, + -0.3583017, + 0.32456085, + 0.57291603, + -0.02988835, + -0.12758288, + 0.07391571, + -0.105302736, + -0.003461621}), + torch::tensor( + {0.88319576, + 0.4848693, + -0.09634813, + 0.13308413, + 0.9289434, + -0.98205453, + 0.8919147, + 0.14732295, + -0.3638662, + 0.35729778, + -0.16119093, + -0.09531955, + 0.29137993, + -0.404272, + 0.8510058}), + torch::tensor( + {-0.38180307, + -0.17343357, + -0.1808421, + -0.07243101, + 0.8087596, + -0.049911626, + -0.37177977, + 0.06514444, + 0.4756227, + -0.20610029, + -0.33534062, + -0.27370194, + -0.034664612, + 0.4019795, + 0.4785126}), + torch::tensor( + {-0.106931984, + 0.15266079, + -0.06190041, + -0.7941472, + 0.26299715, + 0.1042302, + 0.083629966, + 0.46858358, + -0.5036145, + 0.5127309, + -0.34609035, + 0.07381254, + -0.12807515, + 0.19900778, + 0.5627706}), + torch::tensor( + {0.21917637, + -0.123425685, + -0.14819148, + 0.32612452, + -0.53095895, + 0.4336113, + -0.10937049, + 0.83849686, + 0.1206982, + 0.794113, + -0.08247241, + 0.91163427, + -0.25811717, + 0.42008442, + -0.20074657}), + torch::tensor( + {0.29778504, + 0.73049575, + -0.16042702, + -0.3946606, + -0.17268912, + 0.11890727, + -0.35551623, + 0.2913491, + -0.20933971, + 0.07523864, + 0.078045435, + 0.24295215, + 0.26767665, + -0.5313675, + -0.74701965}), + torch::tensor( + {-0.06465864, + 0.22787006, + -0.61670274, + 0.2985376, + -0.0122054, + 0.46651745, + 0.1879351, + 0.12726055, + -0.37897742, + -0.35006294, + 0.33526993, + 0.17848356, + -0.86180973, + 0.3637374, + 0.46942705}), + torch::tensor( + {-0.59026957, + -0.28078535, + -0.44564965, + 0.2092768, + 0.25531057, + 0.110611245, + 0.072759114, + -0.034555472, + 0.5880469, + -0.044425756, + -0.50906545, + -0.33038157, + -0.12659654, + -0.07465571, + 0.07705109}), + }, + { + torch::tensor( + {0.43752572, + 0.48078725, + -0.12718944, + 0.44091794, + -0.23655368, + 0.19225967, + 0.35653186, + -0.39762494, + 0.13974607, + -0.14961815, + -0.26969978, + 0.6044212, + 0.19001292, + -0.08493936, + 0.25612876}), + torch::tensor( + {-0.20287868, + -0.13937469, + -0.016155666, + 0.1897247, + 0.27754223, + -0.37583515, + -0.45658332, + -0.041515056, + -0.22420865, + 0.31147435, + 0.056633875, + 0.4038257, + 0.39963764, + 0.20823427, + 0.28348872}), + }, + }; +} + +inline std::vector> Kaiming_Uniform() { + return { + { + torch::tensor( + {0.36331797, + 0.055456758, + -0.45173424, + 0.43808794, + -0.8880919, + -0.5487398, + -0.23176038}), + torch::tensor( + {-0.45097935, + -0.32388276, + -0.7588222, + -0.19693595, + 0.19790006, + -0.6031401, + -0.04751247}), + torch::tensor( + {0.66274905, + -0.09517455, + 0.02573061, + -0.07986951, + 0.18736875, + 0.588673, + 0.8769796}), + torch::tensor( + {0.5879475, + 0.8789861, + -0.06695682, + -0.8316841, + -0.43891022, + 0.63039577, + -0.0060015917}), + torch::tensor( + {-0.4601755, + -0.7094668, + -0.86643064, + -0.7813998, + -0.18779033, + 0.50772536, + 0.5005363}), + torch::tensor( + {-0.89289045, + 0.57751, + -0.724463, + -0.19572788, + -0.3753947, + -0.17832708, + -0.18177801}), + torch::tensor( + {-0.8307846, + -0.7993882, + -0.14487183, + 0.011972845, + -0.4205768, + 0.34875572, + -0.8332921}), + torch::tensor( + {-0.062481046, + 0.8141842, + -0.37763458, + 0.83601844, + 0.33528924, + -0.83551645, + 0.58576393}), + torch::tensor( + {-0.10683453, + -0.4132924, + 0.7403351, + -0.7481508, + 0.09934509, + -0.19383776, + 0.66113985}), + torch::tensor( + {0.25843763, + 0.44486153, + 0.32696164, + -0.22263634, + -0.19470501, + -0.76295114, + 0.5016502}), + torch::tensor( + {0.73508084, + 0.6334691, + -0.6530534, + 0.041291475, + -0.6526422, + -0.50964934, + -0.53948045}), + torch::tensor( + {0.31639445, + -0.5517084, + -0.020198822, + 0.038947523, + 0.596805, + -0.69984597, + -0.63558686}), + torch::tensor( + {-0.5375881, + 0.6480124, + -0.3327999, + 0.78091884, + 0.33478355, + 0.11723292, + -0.0068918467}), + torch::tensor( + {-0.18301755, + 0.11615932, + -0.21140611, + -0.0065051913, + 0.11812818, + -0.72418123, + -0.4852514}), + torch::tensor( + {0.74759305, + -0.75134623, + -0.06647962, + 0.9158571, + 0.33444047, + 0.026212811, + -0.8023249}), + }, + { + torch::tensor( + {0.33950645, + 0.08810693, + -0.42316547, + -0.49035522, + -0.19512305, + 0.2776215, + 0.62385184, + 0.3636803, + -0.07122034, + 0.2217493, + -0.62047863, + -0.5401811, + 0.29510874, + -0.3582324, + 0.30427164}), + torch::tensor( + {-0.44647068, + -0.3132621, + -0.5209466, + 0.33004105, + -0.064444125, + 0.4867385, + 0.3914091, + 0.34996456, + 0.020337284, + -0.19554132, + -0.13751587, + 0.084053695, + 0.31351423, + -0.44309196, + 0.5308059}), + torch::tensor( + {-0.06876695, + -0.5299633, + -0.342195, + 0.5596078, + 0.5783978, + -0.5858307, + 0.44606978, + 0.31696087, + 0.37436205, + 0.5353921, + -0.34086213, + 0.19971412, + 0.2588218, + -0.1868864, + 0.21165323}), + torch::tensor( + {-0.18196595, + 0.39102274, + -0.17547771, + -0.2357767, + 0.1592201, + 0.22432798, + -0.30899984, + 0.055908024, + 0.3665306, + -0.062927246, + 0.1924777, + -0.15254003, + 0.22167546, + -0.4581471, + -0.37190205}), + torch::tensor( + {-0.32102844, + 0.58123356, + -0.17018372, + -0.0017268658, + -0.30642188, + 0.63137406, + 0.6177004, + -0.4769895, + -0.51271266, + -0.47939685, + -0.0030501485, + -0.1612154, + -0.413971, + -0.22684419, + 0.119490385}), + torch::tensor( + {-0.33045214, + 0.14014107, + -0.14503211, + -0.30646476, + 0.08689022, + 0.52004176, + -0.42759007, + 0.029367685, + -0.23322919, + 0.6206326, + -0.60005057, + -0.60631055, + 0.62321216, + -0.40014827, + 0.12125647}), + torch::tensor( + {-0.05458474, + -0.1332312, + -0.14125755, + 0.40188795, + 0.03022945, + -0.6157679, + -0.37337655, + -0.21560428, + 0.31826198, + -0.40928328, + 0.59635466, + -0.1408664, + -0.11356497, + 0.4955656, + 0.317877}), + torch::tensor( + {0.53639835, + 0.36584032, + -0.19185784, + -0.4196168, + -0.047056615, + 0.523444, + -0.2122646, + -0.58652025, + 0.2592615, + 0.6156774, + -0.18004948, + -0.5236881, + -0.5736749, + 0.15849298, + -0.04788935}), + torch::tensor( + {-0.31938198, + 0.12783945, + 0.24018055, + 0.5030039, + 0.49102718, + -0.09466827, + -0.5576785, + -0.57150143, + 0.5905171, + 0.2795803, + 0.27569205, + -0.5472138, + 0.58566374, + 0.5991524, + 0.571019}), + torch::tensor( + {-0.53353, + -0.23864025, + -0.43498003, + 0.5989136, + -0.2717535, + -0.28875044, + 0.33135164, + -0.2925691, + -0.31149834, + -0.055321813, + -0.060791314, + -0.49266922, + 0.527258, + -0.27898985, + 0.22434139}), + torch::tensor( + {0.55009943, + 0.31895775, + 0.089542985, + 0.53812927, + 0.085018635, + -0.29260972, + 0.59830266, + 0.14968991, + -0.6170747, + -0.18005475, + -0.43080813, + 0.5545073, + -0.104447424, + -0.576499, + -0.039758265}), + torch::tensor( + {0.39719027, + 0.16431224, + 0.19998527, + 0.058738172, + 0.23579127, + -0.15409905, + -0.25159094, + -0.59116155, + -0.4764521, + 0.2741078, + -0.37448537, + 0.09073615, + 0.20187438, + 0.044780314, + -0.4100524}), + torch::tensor( + {0.6048352, + -0.36777255, + 0.52019507, + -0.5031236, + -0.15213624, + 0.34403604, + -0.25840908, + 0.53128976, + -0.43522838, + -0.53111815, + -0.28518632, + 0.10223669, + 0.5823371, + -0.30193484, + 0.22618395}), + torch::tensor( + {-0.15856105, + -0.13715577, + 0.4650467, + -0.49014413, + 0.06717521, + 0.59476703, + -0.08690709, + 0.49106258, + -0.194769, + 0.50910276, + -0.6117934, + -0.09115183, + -0.111088574, + 0.20496935, + 0.24812967}), + torch::tensor( + {0.48561007, + -0.094227016, + -0.025080085, + 0.43311268, + -0.17112547, + 0.55443174, + -0.42109656, + -0.068437934, + -0.03396976, + 0.2822079, + 0.4323948, + -0.10024661, + -0.52397716, + 0.31337678, + 0.18918753}), + }, + { + torch::tensor( + {-0.10132998, + -0.37424773, + -0.3613546, + 0.15002292, + 0.59365314, + -0.5066626, + 0.38276488, + -0.32790715, + -0.12318605, + 0.5020248, + -0.14304265, + 0.057588696, + -0.442107, + 0.538372, + -0.081757426}), + torch::tensor( + {-0.46257195, + 0.18518949, + -0.44968688, + -0.5018581, + 0.03852868, + 0.5014383, + -0.1789748, + 0.29767585, + 0.5434825, + 0.419483, + -0.33176154, + -0.06932968, + -0.19902417, + -0.508572, + 0.00026130676}), + }, + }; +} + +} // namespace expected_parameters diff --git a/western_ai/usa/pytorch/test/cpp/api/init_baseline.py b/western_ai/usa/pytorch/test/cpp/api/init_baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..619c677d6eb0b8b7eae5ed65646deb6e534725c2 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/init_baseline.py @@ -0,0 +1,75 @@ +"""Script to generate baseline values from PyTorch initialization algorithms""" + +import sys + +import torch + + +HEADER = """ +#include + +#include + +namespace expected_parameters { +""" + +FOOTER = "} // namespace expected_parameters" + +PARAMETERS = "inline std::vector> {}() {{" + +INITIALIZERS = { + "Xavier_Uniform": lambda w: torch.nn.init.xavier_uniform(w), + "Xavier_Normal": lambda w: torch.nn.init.xavier_normal(w), + "Kaiming_Normal": lambda w: torch.nn.init.kaiming_normal(w), + "Kaiming_Uniform": lambda w: torch.nn.init.kaiming_uniform(w), +} + + +def emit(initializer_parameter_map): + # Don't write generated with an @ in front, else this file is recognized as generated. + print("// @{} from {}".format("generated", __file__)) + print(HEADER) + for initializer_name, weights in initializer_parameter_map.items(): + print(PARAMETERS.format(initializer_name)) + print(" return {") + for sample in weights: + print(" {") + for parameter in sample: + parameter_values = "{{{}}}".format(", ".join(map(str, parameter))) + print(f" torch::tensor({parameter_values}),") + print(" },") + print(" };") + print("}\n") + print(FOOTER) + + +def run(initializer): + torch.manual_seed(0) + + layer1 = torch.nn.Linear(7, 15) + INITIALIZERS[initializer](layer1.weight) + + layer2 = torch.nn.Linear(15, 15) + INITIALIZERS[initializer](layer2.weight) + + layer3 = torch.nn.Linear(15, 2) + INITIALIZERS[initializer](layer3.weight) + + weight1 = layer1.weight.data.numpy() + weight2 = layer2.weight.data.numpy() + weight3 = layer3.weight.data.numpy() + + return [weight1, weight2, weight3] + + +def main(): + initializer_parameter_map = {} + for initializer in INITIALIZERS: + sys.stderr.write(f"Evaluating {initializer} ...\n") + initializer_parameter_map[initializer] = run(initializer) + + emit(initializer_parameter_map) + + +if __name__ == "__main__": + main() diff --git a/western_ai/usa/pytorch/test/cpp/api/integration.cpp b/western_ai/usa/pytorch/test/cpp/api/integration.cpp new file mode 100644 index 0000000000000000000000000000000000000000..99582124568b2cd81c713e73a9e25be25c8d232a --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/integration.cpp @@ -0,0 +1,323 @@ +#include + +#include +#include + +#include + +#include +#include +#include + +using namespace torch::nn; +using namespace torch::test; + +const double kPi = 3.1415926535898; + +class CartPole { + // Translated from openai/gym's cartpole.py + public: + double gravity = 9.8; + double masscart = 1.0; + double masspole = 0.1; + double total_mass = (masspole + masscart); + double length = 0.5; // actually half the pole's length; + double polemass_length = (masspole * length); + double force_mag = 10.0; + double tau = 0.02; // seconds between state updates; + + // Angle at which to fail the episode + double theta_threshold_radians = 12 * 2 * kPi / 360; + double x_threshold = 2.4; + int steps_beyond_done = -1; + + torch::Tensor state; + double reward; + bool done; + int step_ = 0; + + torch::Tensor getState() { + return state; + } + + double getReward() { + return reward; + } + + double isDone() { + return done; + } + + void reset() { + state = torch::empty({4}).uniform_(-0.05, 0.05); + steps_beyond_done = -1; + step_ = 0; + } + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) + CartPole() { + reset(); + } + + void step(int action) { + auto x = state[0].item(); + auto x_dot = state[1].item(); + auto theta = state[2].item(); + auto theta_dot = state[3].item(); + + auto force = (action == 1) ? force_mag : -force_mag; + auto costheta = std::cos(theta); + auto sintheta = std::sin(theta); + auto temp = (force + polemass_length * theta_dot * theta_dot * sintheta) / + total_mass; + auto thetaacc = (gravity * sintheta - costheta * temp) / + (length * (4.0 / 3.0 - masspole * costheta * costheta / total_mass)); + auto xacc = temp - polemass_length * thetaacc * costheta / total_mass; + + x = x + tau * x_dot; + x_dot = x_dot + tau * xacc; + theta = theta + tau * theta_dot; + theta_dot = theta_dot + tau * thetaacc; + state = torch::tensor({x, x_dot, theta, theta_dot}); + + done = x < -x_threshold || x > x_threshold || + theta < -theta_threshold_radians || theta > theta_threshold_radians || + step_ > 200; + + if (!done) { + reward = 1.0; + } else if (steps_beyond_done == -1) { + // Pole just fell! + steps_beyond_done = 0; + reward = 0; + } else { + if (steps_beyond_done == 0) { + AT_ASSERT(false); // Can't do this + } + } + step_++; + } +}; + +template +bool test_mnist( + size_t batch_size, + size_t number_of_epochs, + bool with_cuda, + M&& model, + F&& forward_op, + O&& optimizer) { + std::string mnist_path = "mnist"; + if (const char* user_mnist_path = getenv("TORCH_CPP_TEST_MNIST_PATH")) { + mnist_path = user_mnist_path; + } + + auto train_dataset = + torch::data::datasets::MNIST( + mnist_path, torch::data::datasets::MNIST::Mode::kTrain) + .map(torch::data::transforms::Stack<>()); + + auto data_loader = + torch::data::make_data_loader(std::move(train_dataset), batch_size); + + torch::Device device(with_cuda ? torch::kCUDA : torch::kCPU); + model->to(device); + + for ([[maybe_unused]] const auto epoch : c10::irange(number_of_epochs)) { + // NOLINTNEXTLINE(performance-for-range-copy) + for (torch::data::Example<> batch : *data_loader) { + auto data = batch.data.to(device); + auto targets = batch.target.to(device); + torch::Tensor prediction = forward_op(std::move(data)); + // NOLINTNEXTLINE(performance-move-const-arg) + torch::Tensor loss = torch::nll_loss(prediction, std::move(targets)); + AT_ASSERT(!torch::isnan(loss).any().item()); + optimizer.zero_grad(); + loss.backward(); + optimizer.step(); + } + } + + torch::NoGradGuard guard; + torch::data::datasets::MNIST test_dataset( + mnist_path, torch::data::datasets::MNIST::Mode::kTest); + auto images = test_dataset.images().to(device), + targets = test_dataset.targets().to(device); + + auto result = std::get<1>(forward_op(images).max(/*dim=*/1)); + torch::Tensor correct = (result == targets).to(torch::kFloat32); + return correct.sum().item() > (test_dataset.size().value() * 0.8); +} + +struct IntegrationTest : torch::test::SeedingFixture {}; + +TEST_F(IntegrationTest, CartPole) { + torch::manual_seed(0); + auto model = std::make_shared(); + auto linear = model->add(Linear(4, 128), "linear"); + auto policyHead = model->add(Linear(128, 2), "policy"); + auto valueHead = model->add(Linear(128, 1), "action"); + auto optimizer = torch::optim::Adam(model->parameters(), 1e-3); + + std::vector saved_log_probs; + std::vector saved_values; + std::vector rewards; + + auto forward = [&](torch::Tensor inp) { + auto x = linear->forward(inp).clamp_min(0); + torch::Tensor actions = policyHead->forward(x); + torch::Tensor value = valueHead->forward(x); + return std::make_tuple(torch::softmax(actions, -1), value); + }; + + auto selectAction = [&](torch::Tensor state) { + // Only work on single state right now, change index to gather for batch + auto out = forward(state); + auto probs = torch::Tensor(std::get<0>(out)); + auto value = torch::Tensor(std::get<1>(out)); + auto action = probs.multinomial(1)[0].item(); + // Compute the log prob of a multinomial distribution. + // This should probably be actually implemented in autogradpp... + auto p = probs / probs.sum(-1, true); + auto log_prob = p[action].log(); + saved_log_probs.emplace_back(log_prob); + saved_values.push_back(value); + return action; + }; + + auto finishEpisode = [&] { + auto R = 0.; + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + for (int i = rewards.size() - 1; i >= 0; i--) { + R = rewards[i] + 0.99 * R; + rewards[i] = R; + } + auto r_t = torch::from_blob( + rewards.data(), {static_cast(rewards.size())}); + r_t = (r_t - r_t.mean()) / (r_t.std() + 1e-5); + + std::vector policy_loss; + std::vector value_loss; + for (const auto i : c10::irange(0U, saved_log_probs.size())) { + auto advantage = r_t[i] - saved_values[i].item(); + policy_loss.push_back(-advantage * saved_log_probs[i]); + value_loss.push_back( + torch::smooth_l1_loss(saved_values[i], torch::ones(1) * r_t[i])); + } + + auto loss = + torch::stack(policy_loss).sum() + torch::stack(value_loss).sum(); + + optimizer.zero_grad(); + loss.backward(); + optimizer.step(); + + rewards.clear(); + saved_log_probs.clear(); + saved_values.clear(); + }; + + auto env = CartPole(); + double running_reward = 10.0; + for (size_t episode = 0;; episode++) { + env.reset(); + auto state = env.getState(); + int t = 0; + for (; t < 10000; t++) { + auto action = selectAction(state); + env.step(action); + state = env.getState(); + auto reward = env.getReward(); + auto done = env.isDone(); + + rewards.push_back(reward); + if (done) + break; + } + + running_reward = running_reward * 0.99 + t * 0.01; + finishEpisode(); + /* + if (episode % 10 == 0) { + printf("Episode %i\tLast length: %5d\tAverage length: %.2f\n", + episode, t, running_reward); + } + */ + if (running_reward > 150) { + break; + } + ASSERT_LT(episode, 3000); + } +} + +TEST_F(IntegrationTest, MNIST_CUDA) { + torch::manual_seed(0); + auto model = std::make_shared(); + auto conv1 = model->add(Conv2d(1, 10, 5), "conv1"); + auto conv2 = model->add(Conv2d(10, 20, 5), "conv2"); + auto drop = Dropout(0.3); + auto drop2d = Dropout2d(0.3); + auto linear1 = model->add(Linear(320, 50), "linear1"); + auto linear2 = model->add(Linear(50, 10), "linear2"); + + auto forward = [&](torch::Tensor x) { + x = torch::max_pool2d(conv1->forward(x), {2, 2}).relu(); + x = conv2->forward(x); + x = drop2d->forward(x); + x = torch::max_pool2d(x, {2, 2}).relu(); + + x = x.view({-1, 320}); + x = linear1->forward(x).clamp_min(0); + x = drop->forward(x); + x = linear2->forward(x); + x = torch::log_softmax(x, 1); + return x; + }; + + auto optimizer = torch::optim::SGD( + model->parameters(), torch::optim::SGDOptions(1e-2).momentum(0.5)); + + ASSERT_TRUE(test_mnist( + 32, // batch_size + 3, // number_of_epochs + true, // with_cuda + model, + forward, + optimizer)); +} + +TEST_F(IntegrationTest, MNISTBatchNorm_CUDA) { + torch::manual_seed(0); + auto model = std::make_shared(); + auto conv1 = model->add(Conv2d(1, 10, 5), "conv1"); + auto batchnorm2d = model->add(BatchNorm2d(10), "batchnorm2d"); + auto conv2 = model->add(Conv2d(10, 20, 5), "conv2"); + auto linear1 = model->add(Linear(320, 50), "linear1"); + auto batchnorm1 = model->add(BatchNorm1d(50), "batchnorm1"); + auto linear2 = model->add(Linear(50, 10), "linear2"); + + auto forward = [&](torch::Tensor x) { + x = torch::max_pool2d(conv1->forward(x), {2, 2}).relu(); + x = batchnorm2d->forward(x); + x = conv2->forward(x); + x = torch::max_pool2d(x, {2, 2}).relu(); + + x = x.view({-1, 320}); + x = linear1->forward(x).clamp_min(0); + x = batchnorm1->forward(x); + x = linear2->forward(x); + x = torch::log_softmax(x, 1); + return x; + }; + + auto optimizer = torch::optim::SGD( + model->parameters(), torch::optim::SGDOptions(1e-2).momentum(0.5)); + + ASSERT_TRUE(test_mnist( + 32, // batch_size + 3, // number_of_epochs + true, // with_cuda + model, + forward, + optimizer)); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/ivalue.cpp b/western_ai/usa/pytorch/test/cpp/api/ivalue.cpp new file mode 100644 index 0000000000000000000000000000000000000000..85cf58f9e7c514fd0dc78f82cc3fa8e261f66f0e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/ivalue.cpp @@ -0,0 +1,63 @@ +#include + +#include + +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include + +using namespace torch::test; +using namespace torch::nn; +using namespace torch::optim; + +TEST(IValueTest, DeepcopyTensors) { + torch::Tensor t0 = torch::randn({2, 3}); + torch::Tensor t1 = torch::randn({3, 4}); + torch::Tensor t2 = t0.detach(); + torch::Tensor t3 = t0; + torch::Tensor t4 = t1.as_strided({2, 3}, {3, 1}, 2); + std::vector tensor_vector = {t0, t1, t2, t3, t4}; + c10::List tensor_list(tensor_vector); + torch::IValue tensor_list_ivalue(tensor_list); + + c10::IValue::CompIdentityIValues ivalue_compare; + + // Make sure our setup configuration is correct + ASSERT_TRUE(ivalue_compare(tensor_list[0].get(), tensor_list[3].get())); + ASSERT_FALSE(ivalue_compare(tensor_list[0].get(), tensor_list[1].get())); + ASSERT_FALSE(ivalue_compare(tensor_list[0].get(), tensor_list[2].get())); + ASSERT_FALSE(ivalue_compare(tensor_list[1].get(), tensor_list[4].get())); + ASSERT_TRUE(tensor_list[0].get().isAliasOf(tensor_list[2].get())); + + c10::IValue copied_ivalue = tensor_list_ivalue.deepcopy(); + c10::List copied_list = copied_ivalue.toList(); + + // Make sure our setup configuration is correct + ASSERT_TRUE(ivalue_compare(copied_list[0].get(), copied_list[3].get())); + ASSERT_FALSE(ivalue_compare(copied_list[0].get(), copied_list[1].get())); + ASSERT_FALSE(ivalue_compare(copied_list[0].get(), copied_list[2].get())); + ASSERT_FALSE(ivalue_compare(copied_list[1].get(), copied_list[4].get())); + // NOTE: this is actually incorrect. Ideally, these _should_ be aliases. + ASSERT_FALSE(copied_list[0].get().isAliasOf(copied_list[2].get())); + + ASSERT_TRUE(copied_list[0].get().toTensor().allclose( + tensor_list[0].get().toTensor())); + ASSERT_TRUE(copied_list[1].get().toTensor().allclose( + tensor_list[1].get().toTensor())); + ASSERT_TRUE(copied_list[2].get().toTensor().allclose( + tensor_list[2].get().toTensor())); + ASSERT_TRUE(copied_list[3].get().toTensor().allclose( + tensor_list[3].get().toTensor())); + ASSERT_TRUE(copied_list[4].get().toTensor().allclose( + tensor_list[4].get().toTensor())); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/jit.cpp b/western_ai/usa/pytorch/test/cpp/api/jit.cpp new file mode 100644 index 0000000000000000000000000000000000000000..494cdc0d90e532d8ef14deaeffd1435e83b5ba06 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/jit.cpp @@ -0,0 +1,126 @@ +#include + +#include +#include +#include + +#include + +TEST(TorchScriptTest, CanCompileMultipleFunctions) { + auto module = torch::jit::compile(R"JIT( + def test_mul(a, b): + return a * b + def test_relu(a, b): + return torch.relu(a + b) + def test_while(a, i): + while bool(i < 10): + a += a + i += 1 + return a + def test_len(a : List[int]): + return len(a) + )JIT"); + auto a = torch::ones(1); + auto b = torch::ones(1); + + ASSERT_EQ(1, module->run_method("test_mul", a, b).toTensor().item()); + + ASSERT_EQ( + 2, module->run_method("test_relu", a, b).toTensor().item()); + + ASSERT_TRUE( + 0x200 == + module->run_method("test_while", a, b).toTensor().item()); + + at::IValue list = c10::List({3, 4}); + ASSERT_EQ(2, module->run_method("test_len", list).toInt()); +} + +TEST(TorchScriptTest, TestNestedIValueModuleArgMatching) { + auto module = torch::jit::compile(R"JIT( + def nested_loop(a: List[List[Tensor]], b: int): + return torch.tensor(1.0) + b + )JIT"); + + auto b = 3; + + torch::List list({torch::rand({4, 4})}); + + torch::List> list_of_lists; + list_of_lists.push_back(list); + module->run_method("nested_loop", list_of_lists, b); + + auto generic_list = c10::impl::GenericList(at::TensorType::get()); + auto empty_generic_list = + c10::impl::GenericList(at::ListType::create(at::TensorType::get())); + empty_generic_list.push_back(generic_list); + module->run_method("nested_loop", empty_generic_list, b); + + auto too_many_lists = c10::impl::GenericList( + at::ListType::create(at::ListType::create(at::TensorType::get()))); + too_many_lists.push_back(empty_generic_list); + try { + module->run_method("nested_loop", too_many_lists, b); + AT_ASSERT(false); + } catch (const c10::Error& error) { + AT_ASSERT( + std::string(error.what_without_backtrace()) + .find("nested_loop() Expected a value of type 'List[List[Tensor]]'" + " for argument 'a' but instead found type " + "'List[List[List[Tensor]]]'") == 0); + }; +} + +TEST(TorchScriptTest, TestDictArgMatching) { + auto module = torch::jit::compile(R"JIT( + def dict_op(a: Dict[str, Tensor], b: str): + return a[b] + )JIT"); + c10::Dict dict; + dict.insert("hello", torch::ones({2})); + auto output = module->run_method("dict_op", dict, std::string("hello")); + ASSERT_EQ(1, output.toTensor()[0].item()); +} + +TEST(TorchScriptTest, TestTupleArgMatching) { + auto module = torch::jit::compile(R"JIT( + def tuple_op(a: Tuple[List[int]]): + return a + )JIT"); + + c10::List int_list({1}); + auto tuple_generic_list = c10::ivalue::Tuple::create({int_list}); + + // doesn't fail on arg matching + module->run_method("tuple_op", tuple_generic_list); +} + +TEST(TorchScriptTest, TestOptionalArgMatching) { + auto module = torch::jit::compile(R"JIT( + def optional_tuple_op(a: Optional[Tuple[int, str]]): + if a is None: + return 0 + else: + return a[0] + )JIT"); + + auto optional_tuple = c10::ivalue::Tuple::create({2, std::string("hi")}); + + ASSERT_EQ(2, module->run_method("optional_tuple_op", optional_tuple).toInt()); + ASSERT_EQ( + 0, module->run_method("optional_tuple_op", torch::jit::IValue()).toInt()); +} + +TEST(TorchScriptTest, TestPickle) { + torch::IValue float_value(2.3); + + // TODO: when tensors are stored in the pickle, delete this + std::vector tensor_table; + auto data = torch::jit::pickle(float_value, &tensor_table); + + torch::IValue ivalue = torch::jit::unpickle(data.data(), data.size()); + + double diff = ivalue.toDouble() - float_value.toDouble(); + double eps = 0.0001; + ASSERT_TRUE(diff < eps && diff > -eps); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/memory.cpp b/western_ai/usa/pytorch/test/cpp/api/memory.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1eccf66fc1ed81d051ca2e4ba2f1a7dbeb0953c7 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/memory.cpp @@ -0,0 +1,35 @@ +#include + +#include + +struct TestValue { + explicit TestValue(const int& x) : lvalue_(x) {} + explicit TestValue(int&& x) : rvalue_(x) {} + + std::optional lvalue_; + std::optional rvalue_; +}; + +TEST(MakeUniqueTest, ForwardRvaluesCorrectly) { + auto ptr = std::make_unique(123); + ASSERT_FALSE(ptr->lvalue_.has_value()); + ASSERT_TRUE(ptr->rvalue_.has_value()); + ASSERT_EQ(*ptr->rvalue_, 123); +} + +TEST(MakeUniqueTest, ForwardLvaluesCorrectly) { + int x = 5; + auto ptr = std::make_unique(x); + ASSERT_TRUE(ptr->lvalue_.has_value()); + ASSERT_EQ(*ptr->lvalue_, 5); + ASSERT_FALSE(ptr->rvalue_.has_value()); +} + +TEST(MakeUniqueTest, CanConstructUniquePtrOfArray) { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) + auto ptr = std::make_unique(3); + // Value initialization is required by the standard. + ASSERT_EQ(ptr[0], 0); + ASSERT_EQ(ptr[1], 0); + ASSERT_EQ(ptr[2], 0); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/meta_tensor.cpp b/western_ai/usa/pytorch/test/cpp/api/meta_tensor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..88c2616baafbefa482b9c96b703e147b21439744 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/meta_tensor.cpp @@ -0,0 +1,35 @@ +#include + +#include +#include + +#include + +TEST(MetaTensorTest, MetaDeviceApi) { + auto a = at::ones({4}, at::kFloat); + auto b = at::ones({3, 4}, at::kFloat); + // at::add() will return a meta tensor if its inputs are also meta tensors. + auto out_meta = at::add(a.to(c10::kMeta), b.to(c10::kMeta)); + + ASSERT_EQ(a.device(), c10::kCPU); + ASSERT_EQ(b.device(), c10::kCPU); + ASSERT_EQ(out_meta.device(), c10::kMeta); + c10::IntArrayRef sizes_actual = out_meta.sizes(); + std::vector sizes_expected = std::vector{3, 4}; + ASSERT_EQ(sizes_actual, sizes_expected); +} + +TEST(MetaTensorTest, MetaNamespaceApi) { + auto a = at::ones({4}, at::kFloat); + auto b = at::ones({3, 4}, at::kFloat); + // The at::meta:: namespace take in tensors from any backend + // and return a meta tensor. + auto out_meta = at::meta::add(a, b); + + ASSERT_EQ(a.device(), c10::kCPU); + ASSERT_EQ(b.device(), c10::kCPU); + ASSERT_EQ(out_meta.device(), c10::kMeta); + c10::IntArrayRef sizes_actual = out_meta.sizes(); + std::vector sizes_expected = std::vector{3, 4}; + ASSERT_EQ(sizes_actual, sizes_expected); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/misc.cpp b/western_ai/usa/pytorch/test/cpp/api/misc.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c1971186f3b9669146279d72514762d77c424e59 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/misc.cpp @@ -0,0 +1,104 @@ +#include + +#include + +#include + +#include + +using namespace torch::test; + +void torch_warn_once_A() { + TORCH_WARN_ONCE("warn once"); +} + +void torch_warn_once_B() { + TORCH_WARN_ONCE("warn something else once"); +} + +void torch_warn() { + TORCH_WARN("warn multiple times"); +} + +TEST(UtilsTest, WarnOnce) { + { + WarningCapture warnings; + + torch_warn_once_A(); + torch_warn_once_A(); + torch_warn_once_B(); + torch_warn_once_B(); + + ASSERT_EQ(count_substr_occurrences(warnings.str(), "warn once"), 1); + ASSERT_EQ( + count_substr_occurrences(warnings.str(), "warn something else once"), + 1); + } + { + WarningCapture warnings; + + torch_warn(); + torch_warn(); + torch_warn(); + + ASSERT_EQ( + count_substr_occurrences(warnings.str(), "warn multiple times"), 3); + } +} + +TEST(NoGradTest, SetsGradModeCorrectly) { + torch::manual_seed(0); + torch::NoGradGuard guard; + torch::nn::Linear model(5, 2); + auto x = torch::randn({10, 5}, torch::requires_grad()); + auto y = model->forward(x); + torch::Tensor s = y.sum(); + + // Mimicking python API behavior: + ASSERT_THROWS_WITH( + s.backward(), + "element 0 of tensors does not require grad and does not have a grad_fn") +} + +struct AutogradTest : torch::test::SeedingFixture { + AutogradTest() { + x = torch::randn({3, 3}, torch::requires_grad()); + y = torch::randn({3, 3}); + z = x * y; + } + torch::Tensor x, y, z; +}; + +TEST_F(AutogradTest, CanTakeDerivatives) { + z.backward(torch::ones_like(z)); + ASSERT_TRUE(x.grad().allclose(y)); +} + +TEST_F(AutogradTest, CanTakeDerivativesOfZeroDimTensors) { + z.sum().backward(); + ASSERT_TRUE(x.grad().allclose(y)); +} + +TEST_F(AutogradTest, CanPassCustomGradientInputs) { + z.sum().backward(torch::ones({}) * 2); + ASSERT_TRUE(x.grad().allclose(y * 2)); +} + +TEST(UtilsTest, AmbiguousOperatorDefaults) { + auto tmp = at::empty({}, at::kCPU); + at::_test_ambiguous_defaults(tmp); + at::_test_ambiguous_defaults(tmp, 1); + at::_test_ambiguous_defaults(tmp, 1, 1); + at::_test_ambiguous_defaults(tmp, 2, "2"); +} + +int64_t get_first_element(c10::OptionalIntArrayRef arr) { + return arr.value()[0]; +} + +TEST(OptionalArrayRefTest, DanglingPointerFix) { + // Ensure that the converting constructor of `OptionalArrayRef` does not + // create a dangling pointer when given a single value + ASSERT_TRUE(get_first_element(300) == 300); + ASSERT_TRUE(get_first_element({400}) == 400); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/module.cpp b/western_ai/usa/pytorch/test/cpp/api/module.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b9704166154ff0764a485c94cc485d1505b99ad8 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/module.cpp @@ -0,0 +1,1057 @@ +#include + +#include +#include + +#include + +using namespace torch::nn; +using namespace torch::test; + +struct AGIUnit : torch::nn::Module {}; + +namespace test { +struct AGIUnit : torch::nn::Module {}; +struct AGIUnit2 : torch::nn::Module { + AGIUnit2() : torch::nn::Module("Foo") {} +}; +} // namespace test + +struct ModuleTest : torch::test::SeedingFixture {}; + +TEST_F(ModuleTest, CanEnableAndDisableTrainingMode) { + Linear module(3, 4); + ASSERT_TRUE(module->is_training()); + + module->eval(); + ASSERT_FALSE(module->is_training()); + + module->train(); + ASSERT_TRUE(module->is_training()); +} + +TEST_F(ModuleTest, ZeroGrad) { + Linear module(3, 4); + auto weight = torch::ones({8, 3}, torch::requires_grad()); + auto loss = module(weight).sum(); + loss.backward(); + for (auto& parameter : module->parameters()) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto grad = parameter.grad(); + ASSERT_TRUE(grad.defined()); + ASSERT_NE(grad.sum().item(), 0); + } + module->zero_grad(); + for (auto& parameter : module->parameters()) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto grad = parameter.grad(); + ASSERT_FALSE(grad.defined()); + } +} + +TEST_F(ModuleTest, ZeroGradWithUndefined) { + struct TestModule : torch::nn::Module { + TestModule() { + x = register_parameter("x", torch::ones(5, torch::requires_grad())); + y = register_parameter("y", torch::ones(5, torch::requires_grad())); + } + torch::Tensor x, y; + }; + + TestModule module; + auto z = module.x * 2; + z.sum().backward(); + + ASSERT_TRUE(module.x.grad().defined()); + ASSERT_FALSE(module.y.grad().defined()); + + module.zero_grad(false); // set_to_none = false + + ASSERT_TRUE(module.x.grad().defined()); + ASSERT_FALSE(module.y.grad().defined()); + + ASSERT_EQ(module.x.grad().sum().item(), 0); + + module.zero_grad(); + + ASSERT_FALSE(module.x.grad().defined()); + ASSERT_FALSE(module.y.grad().defined()); +} + +TEST_F(ModuleTest, RegisterModuleThrowsForEmptyOrDottedName) { + struct TestModel : public torch::nn::Module {}; + ASSERT_THROWS_WITH( + TestModel{}.register_module("name.with.dot", torch::nn::Linear(3, 4)), + "Submodule name must not contain a dot (got 'name.with.dot')"); + ASSERT_THROWS_WITH( + TestModel{}.register_module("", torch::nn::Linear(3, 4)), + "Submodule name must not be empty"); +} + +TEST_F(ModuleTest, RegisterModuleThrowsForDuplicateModuleName) { + struct TestModel : public torch::nn::Module {}; + TestModel model; + model.register_module("linear", torch::nn::Linear(3, 4)); + ASSERT_THROWS_WITH( + model.register_module("linear", torch::nn::Linear(3, 4)), + "Submodule 'linear' already defined"); +} + +TEST_F(ModuleTest, ReplaceModuleThrowsForUnknownModuleName) { + torch::nn::Module model; + ASSERT_THROWS_WITH( + model.replace_module("linear", torch::nn::Linear(3, 4)), + "Submodule 'linear' is not defined"); +} + +TEST_F(ModuleTest, ReplaceModule) { + struct TestModel : public torch::nn::Module { + torch::nn::Linear l1{nullptr}; + TestModel() { + l1 = register_module("l1", torch::nn::Linear(3, 4)); + } + }; + auto model = std::make_shared(); + model->l1 = model->replace_module("l1", torch::nn::Linear(5, 6)); + ASSERT_EQ(model->named_parameters()["l1.weight"].size(0), 6); + ASSERT_EQ(model->l1.get(), model->named_modules()["l1"]->as()); +} + +TEST_F(ModuleTest, UnregisterModule) { + struct TestModel : public torch::nn::Module {}; + TestModel model; + ASSERT_THROWS_WITH( + model.unregister_module("linear"), + "No Module with name `linear` is registered"); + model.register_module("linear", torch::nn::Linear(3, 4)); + model.unregister_module("linear"); + ASSERT_TRUE(model.children().empty()); +} + +TEST_F(ModuleTest, RegisterParameterThrowsForEmptyOrDottedName) { + struct TestModel : public torch::nn::Module {}; + ASSERT_THROWS_WITH( + TestModel{}.register_parameter("name.with.dot", torch::ones(5)), + "Parameter name must not contain a dot (got 'name.with.dot')"); + ASSERT_THROWS_WITH( + TestModel{}.register_parameter("", torch::ones(5)), + "Parameter name must not be empty"); +} + +TEST_F(ModuleTest, RegisterParameterThrowsForDuplicateModuleName) { + struct TestModel : public torch::nn::Module {}; + TestModel model; + model.register_parameter("p", torch::ones(5)); + ASSERT_THROWS_WITH( + model.register_parameter("p", torch::ones(5)), + "Parameter 'p' already defined"); +} + +TEST_F(ModuleTest, RegisterParameterUndefinedTensor) { + struct TestModel : public torch::nn::Module {}; + { + TestModel model; + model.register_parameter( + "undefined_tensor", torch::Tensor(), /*requires_grad=*/false); + ASSERT_EQ(model.parameters().size(), 0); + } + { + WarningCapture warnings; + + TestModel model; + model.register_parameter("undefined_tensor", torch::Tensor()); + ASSERT_EQ(model.parameters().size(), 0); + + ASSERT_EQ( + count_substr_occurrences( + warnings.str(), + "Ignoring the `requires_grad=true` function parameter"), + 1); + } +} + +TEST_F(ModuleTest, RegisterBufferThrowsForEmptyOrDottedName) { + struct TestModel : public torch::nn::Module {}; + ASSERT_THROWS_WITH( + TestModel{}.register_buffer("name.with.dot", torch::ones(5)), + "Buffer name must not contain a dot (got 'name.with.dot')"); + ASSERT_THROWS_WITH( + TestModel{}.register_buffer("", torch::ones(5)), + "Buffer name must not be empty"); +} + +TEST_F(ModuleTest, RegisterBufferThrowsForDuplicateModuleName) { + struct TestModel : public torch::nn::Module {}; + TestModel model; + model.register_buffer("p", torch::ones(5)); + ASSERT_THROWS_WITH( + model.register_buffer("p", torch::ones(5)), "Buffer 'p' already defined"); +} + +TEST_F(ModuleTest, CanGetName) { + // CHECK instead of REQUIRE because demangling may fail. + AGIUnit agi; + // Call it twice just to make sure there are no bugs in the lazy + // initialization semantics. + EXPECT_EQ(agi.name(), "AGIUnit"); + EXPECT_EQ(agi.name(), "AGIUnit"); + EXPECT_EQ(test::AGIUnit().name(), "test::AGIUnit"); + EXPECT_EQ(test::AGIUnit2().name(), "Foo"); +} + +TEST_F(ModuleTest, AsCastsModulesCorrectly) { + Linear module(3, 4); + ASSERT_EQ(module->as(), module.get()); + ASSERT_EQ(module->as(), module.get()); + ASSERT_EQ(module->as(), module.get()); + ASSERT_EQ(module->as(), nullptr); + + std::shared_ptr raw = module.ptr(); + ASSERT_EQ(raw->as(), module.get()); + ASSERT_EQ(raw->as(), module.get()); + ASSERT_EQ(raw->as(), module.get()); + ASSERT_EQ(raw->as(), nullptr); + + Module& raw_ref = *raw.get(); + ASSERT_EQ(raw_ref.as(), module.get()); + ASSERT_EQ(raw_ref.as(), module.get()); + ASSERT_EQ(raw_ref.as(), module.get()); + ASSERT_EQ(raw_ref.as(), nullptr); + if (auto* linear = raw_ref.as()) { + ASSERT_EQ(linear->weight.ndimension(), 2); + } + + AGIUnit unit; + ASSERT_EQ(unit.as(), nullptr); + ASSERT_EQ(unit.as(), nullptr); + ASSERT_EQ(unit.as(), &unit); +} + +void test_DeviceOrDtypeConversionSkipsUndefinedTensor( + torch::Device to_device, + torch::Dtype to_dtype) { + { + // Case 1: Undefined tensors as parameters + Linear module(LinearOptions(10, 20).bias(false)); + ASSERT_TRUE(module->weight.defined()); + ASSERT_FALSE(module->bias.defined()); + + module->to(to_device); + ASSERT_TRUE(module->weight.defined()); + ASSERT_EQ(module->weight.device().type(), to_device.type()); + ASSERT_FALSE(module->bias.defined()); + + module->to(to_dtype); + ASSERT_TRUE(module->weight.defined()); + ASSERT_EQ(module->weight.dtype(), to_dtype); + ASSERT_FALSE(module->bias.defined()); + } + { + // Case 2: Undefined tensors as buffers + BatchNorm1d module( + BatchNorm1dOptions(5).track_running_stats(false).affine(true)); + ASSERT_TRUE(module->weight.defined()); + ASSERT_FALSE(module->running_mean.defined()); + + module->to(to_device); + ASSERT_TRUE(module->weight.defined()); + ASSERT_EQ(module->weight.device().type(), to_device.type()); + ASSERT_FALSE(module->running_mean.defined()); + + module->to(to_dtype); + ASSERT_TRUE(module->weight.defined()); + ASSERT_EQ(module->weight.dtype(), to_dtype); + ASSERT_FALSE(module->running_mean.defined()); + } +} + +TEST_F(ModuleTest, DeviceOrDtypeConversionSkipsUndefinedTensor) { + test_DeviceOrDtypeConversionSkipsUndefinedTensor(torch::kCPU, torch::kDouble); +} + +TEST_F(ModuleTest, DeviceOrDtypeConversionSkipsUndefinedTensor_CUDA) { + test_DeviceOrDtypeConversionSkipsUndefinedTensor( + torch::kCUDA, torch::kDouble); +} + +TEST_F(ModuleTest, ParametersAndBuffersAccessorSkipsUndefinedTensor) { + { + Linear module(LinearOptions(10, 20).bias(false)); + + auto params = module->parameters(); + ASSERT_EQ(params.size(), 1); + auto named_params = module->named_parameters(); + ASSERT_EQ(named_params.size(), 1); + + ASSERT_TRUE(pointer_equal(params[0], named_params["weight"])); + ASSERT_TRUE(pointer_equal(named_params["weight"], module->weight)); + } + { + BatchNorm1d module( + BatchNorm1dOptions(5).track_running_stats(false).affine(false)); + + auto buffers = module->buffers(); + ASSERT_EQ(buffers.size(), 0); + auto named_buffers = module->named_buffers(); + ASSERT_EQ(named_buffers.size(), 0); + } + { + BatchNorm1d module( + BatchNorm1dOptions(5).track_running_stats(true).affine(false)); + + auto buffers = module->buffers(); + ASSERT_EQ(buffers.size(), 3); + auto named_buffers = module->named_buffers(); + ASSERT_EQ(named_buffers.size(), 3); + + ASSERT_TRUE(pointer_equal(buffers[0], named_buffers["running_mean"])); + ASSERT_TRUE( + pointer_equal(named_buffers["running_mean"], module->running_mean)); + ASSERT_TRUE(pointer_equal(buffers[1], named_buffers["running_var"])); + ASSERT_TRUE( + pointer_equal(named_buffers["running_var"], module->running_var)); + ASSERT_TRUE( + pointer_equal(buffers[2], named_buffers["num_batches_tracked"])); + ASSERT_TRUE(pointer_equal( + named_buffers["num_batches_tracked"], module->num_batches_tracked)); + } +} + +TEST_F(ModuleTest, Conversion_MultiCUDA) { + Linear module(128, 64); + for (auto& parameter : module->parameters()) { + ASSERT_EQ(parameter.device(), torch::Device(torch::kCPU)); + ASSERT_EQ(parameter.dtype(), torch::kFloat32); + } + { + module->to({torch::kCUDA, 0}); + for (auto& parameter : module->parameters()) { + ASSERT_EQ(parameter.device().type(), torch::Device::Type::CUDA); + ASSERT_EQ(parameter.device().index(), 0); + } + module->to({torch::kCUDA, 1}); + for (auto& parameter : module->parameters()) { + ASSERT_EQ(parameter.device().type(), torch::Device::Type::CUDA); + ASSERT_EQ(parameter.device().index(), 1); + } + } + { + module->to(torch::Device(torch::kCPU)); + for (auto& parameter : module->parameters()) { + ASSERT_EQ(parameter.device().type(), torch::Device::Type::CPU); + } + } + { + module->to(torch::kFloat64); + for (auto& parameter : module->parameters()) { + ASSERT_EQ(parameter.dtype(), torch::kFloat64); + } + } +} + +TEST_F(ModuleTest, Conversion_NoGrad_MultiCUDA) { + Linear module(128, 64); + for (auto& parameter : module->parameters()) { + parameter.requires_grad_(false); + } + { + module->to(torch::kInt32); + for (auto& parameter : module->parameters()) { + ASSERT_EQ(parameter.dtype(), torch::kInt32); + } + } + { + module->to(torch::Device(torch::kCUDA, 1), torch::kUInt8); + for (auto& parameter : module->parameters()) { + ASSERT_EQ(parameter.device().type(), torch::Device::Type::CUDA); + ASSERT_EQ(parameter.device().index(), 1); + } + for (auto& parameter : module->parameters()) { + ASSERT_EQ(parameter.dtype(), torch::kUInt8); + } + } +} + +TEST_F(ModuleTest, CallingCloneOnModuleThatDoesNotOverrideCloneThrows) { + struct UnCloneable : Module {}; + UnCloneable module; + ASSERT_THROWS_WITH(module.clone(), "clone() has not been implemented"); +} + +TEST_F(ModuleTest, CallingCloneOnModuleThatDoesOverrideCloneDoesNotThrow) { + struct Cloneable : Module { + std::shared_ptr clone( + const std::optional& device = + std::nullopt) const override { + return nullptr; + } + }; + Cloneable module; + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_NO_THROW({ module.clone(); }); +} + +// NOLINTNEXTLINE(bugprone-exception-escape) +struct TestDistinctParametersModule + : public Cloneable { + TestDistinctParametersModule() { + // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) + reset(); + } + void reset() override { + l1 = register_module("l1", Linear(10, 3)); + l2 = register_module("l2", Linear(3, 5)); + l3 = register_module("l3", Linear(5, 100)); + buffer = register_buffer("buf", torch::ones({2, 2})); + } + + Linear l1{nullptr}, l2{nullptr}, l3{nullptr}; + torch::Tensor buffer; +}; + +void testDistinctParameters( + std::shared_ptr m1, + std::shared_ptr m2) { + auto params1 = m1->named_parameters(); + auto params2 = m2->named_parameters(); + ASSERT_EQ(params1.size(), 6); + ASSERT_EQ(params2.size(), 6); + for (auto& param : params1) { + ASSERT_FALSE(pointer_equal(param.value(), params2[param.key()])); + ASSERT_TRUE(param->allclose(params2[param.key()])); + param->add_(2); + } + for (auto& param : params1) { + ASSERT_FALSE(param->allclose(params2[param.key()])); + } + + auto buffers1 = m1->named_buffers(); + auto buffers2 = m2->named_buffers(); + ASSERT_EQ(buffers1.size(), 1); + ASSERT_EQ(buffers2.size(), 1); + for (auto& buffer : buffers1) { + ASSERT_FALSE(pointer_equal(buffer.value(), buffers2[buffer.key()])); + ASSERT_TRUE(buffer->allclose(buffers2[buffer.key()])); + buffer->add_(2); + } + for (auto& buffer : buffers1) { + ASSERT_FALSE(buffer->allclose(buffers2[buffer.key()])); + } +} + +TEST_F(ModuleTest, CloneCreatesDistinctParameters) { + auto module = std::make_shared(); + torch::NoGradGuard no_grad; + auto module2 = module->clone(); + testDistinctParameters(module, module2); +} + +TEST_F(ModuleTest, CloneCreatesDistinctParametersExplicitDevice_CUDA) { + auto module = std::make_shared(); + torch::NoGradGuard no_grad; + torch::Device device(torch::kCUDA, 0); + module->to(device); + auto module2 = module->clone(device); + testDistinctParameters(module, module2); +} + +TEST_F(ModuleTest, CloneCreatesDistinctParametersExplicitDevice_MultiCUDA) { + auto module = std::make_shared(); + torch::NoGradGuard no_grad; + torch::Device d0(torch::kCUDA, 0); + torch::Device d1(torch::kCUDA, 1); + module->to(d0); + auto module2 = module->clone(d1); + + for (auto& param : module->parameters()) { + ASSERT_EQ(param.device(), d0); + } + + for (auto& param : module2->parameters()) { + ASSERT_EQ(param.device(), d1); + } + + // need to move the module back to d0 as allclose expects two tensors on + // the same device. + module2->to(d0); + testDistinctParameters(module, module2); +} + +TEST_F(ModuleTest, ClonePreservesExternalReferences) { + // NOLINTNEXTLINE(bugprone-exception-escape) + struct TestModule : public Cloneable { + TestModule() { + // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) + reset(); + } + void reset() override { + weight = register_parameter("weight", torch::ones({4, 4})); + } + torch::Tensor weight; + }; + auto module = std::make_shared(); + { + torch::NoGradGuard no_grad; + module->weight += 1; + } + ASSERT_TRUE( + pointer_equal(module->weight, module->named_parameters()["weight"])); + ASSERT_TRUE(module->weight.allclose(module->named_parameters()["weight"])); + + auto module2 = std::dynamic_pointer_cast( + std::shared_ptr(module->clone())); + ASSERT_FALSE(pointer_equal(module2->weight, module->weight)); + ASSERT_TRUE( + pointer_equal(module2->weight, module2->named_parameters()["weight"])); + ASSERT_TRUE(module2->weight.allclose(module2->named_parameters()["weight"])); + ASSERT_TRUE(module2->weight.allclose(module->weight)); + ASSERT_FALSE( + pointer_equal(module2->weight, module->named_parameters()["weight"])); +} + +TEST_F(ModuleTest, CloneCopiesTheValuesOfVariablesOfSubmodules) { + // NOLINTNEXTLINE(bugprone-exception-escape) + struct TestModule : public Cloneable { + TestModule() { + // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) + reset(); + } + void reset() override { + weight = register_parameter("weight", torch::ones({4, 4})); + } + + torch::Tensor weight; + int value = 0; + }; + // NOLINTNEXTLINE(bugprone-exception-escape) + struct NestedModule : public Cloneable { + NestedModule() { + // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) + reset(); + } + void reset() override { + module = register_module("module", std::make_shared()); + } + std::shared_ptr module; + }; + + auto a = std::make_shared(); + { + torch::NoGradGuard no_grad; + a->module->weight += 1; + a->module->value = 123; + } + + auto b = std::dynamic_pointer_cast(a->clone()); + + ASSERT_FALSE(pointer_equal(b->module->weight, a->module->weight)); + ASSERT_TRUE(pointer_equal( + b->module->weight, b->module->named_parameters()["weight"])); + ASSERT_TRUE( + b->module->named_parameters()["weight"].allclose(a->module->weight)); + ASSERT_TRUE(b->module->weight.allclose(a->module->weight)); + ASSERT_EQ(b->module->value, a->module->value); +} + +TEST_F(ModuleTest, CloneToDevicePreservesTheDeviceOfParameters_CUDA) { + // NOLINTNEXTLINE(bugprone-exception-escape) + struct TestModule : public Cloneable { + TestModule() { + // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) + reset(); + } + void reset() override { + l1 = register_module("l1", Linear(10, 3)); + l2 = register_module("l2", Linear(3, 5)); + l3 = register_module("l3", Linear(5, 100)); + buffer = register_buffer("buf", torch::ones({2, 2})); + } + + Linear l1{nullptr}, l2{nullptr}, l3{nullptr}; + torch::Tensor buffer; + }; + + TestModule m; + torch::Device device(torch::kCUDA, 0); + + m.to(device); + + auto clone = m.clone(); + for (const auto& parameter : clone->parameters()) { + ASSERT_EQ(parameter.device().type(), device.type()); + ASSERT_EQ(parameter.device().index(), device.index()); + } + for (const auto& buffer : clone->buffers()) { + ASSERT_EQ(buffer.device().type(), device.type()); + ASSERT_EQ(buffer.device().index(), device.index()); + } +} + +TEST_F( + ModuleTest, + CloningToAParticularDevicePlacesAllParametersThere_MultiCUDA) { + // NOLINTNEXTLINE(bugprone-exception-escape) + struct TestModule : public Cloneable { + TestModule() { + // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) + reset(); + } + void reset() override { + l1 = register_module("l1", Linear(10, 3)); + l2 = register_module("l2", Linear(3, 5)); + l3 = register_module("l3", Linear(5, 100)); + buffer = register_buffer("buf", torch::ones({2, 2})); + } + + Linear l1{nullptr}, l2{nullptr}, l3{nullptr}; + torch::Tensor buffer; + }; + + TestModule m; + torch::Device device(torch::kCUDA, 1); + // everything is on CPU here + auto clone = m.clone(device); + for (const auto& parameter : clone->parameters()) { + ASSERT_EQ(parameter.device().type(), device.type()); + ASSERT_EQ(parameter.device().index(), device.index()); + } + for (const auto& buffer : clone->buffers()) { + ASSERT_EQ(buffer.device().type(), device.type()); + ASSERT_EQ(buffer.device().index(), device.index()); + } +} + +struct ParameterTestModule : Module { + ParameterTestModule() { + a = register_parameter("a", torch::zeros({2, 2})); + b = register_parameter("b", torch::ones({2, 2})); + c = register_parameter("c", torch::ones({2, 2}) * 2); + } + + torch::Tensor a, b, c; +}; + +TEST_F(ModuleTest, HasCorrectNumberOfParameters) { + ParameterTestModule module; + ASSERT_EQ(module.parameters().size(), 3); + ASSERT_EQ(module.named_parameters().size(), 3); +} + +TEST_F(ModuleTest, ContainsParametersWithTheCorrectName) { + ParameterTestModule module; + auto parameters = module.named_parameters(); + ASSERT_TRUE(parameters.contains("a")); + ASSERT_TRUE(parameters.contains("b")); + ASSERT_TRUE(parameters.contains("c")); +} + +struct BufferTestModule : Module { + BufferTestModule() { + a = register_buffer("a", torch::zeros({2, 2})); + b = register_buffer("b", torch::ones({2, 2})); + c = register_buffer("c", torch::ones({2, 2}) * 2); + } + + torch::Tensor a, b, c; +}; + +TEST_F(ModuleTest, HasCorrectNumberOfBuffers) { + BufferTestModule module; + ASSERT_EQ(module.buffers().size(), 3); + ASSERT_EQ(module.named_buffers().size(), 3); +} + +TEST_F(ModuleTest, ContainsBuffersWithTheCorrectName) { + BufferTestModule module; + auto buffers = module.named_buffers(); + ASSERT_TRUE(buffers.contains("a")); + ASSERT_TRUE(buffers.contains("b")); + ASSERT_TRUE(buffers.contains("c")); +} + +struct AImpl : torch::nn::Module { + AImpl() : x_(123) {} + AImpl(int x) : x_(x) {} + int x_; +}; +TORCH_MODULE(A); + +TEST_F( + ModuleTest, + DefaultConstructorOfModuleHolderCallsDefaultConstructorOfImpl) { + A a; + ASSERT_TRUE(a); + ASSERT_FALSE(a.is_empty()); + ASSERT_EQ(a->x_, 123); +} + +TEST_F( + ModuleTest, + ValueConstructorOfModuleHolderCallsCorrectConstructorInImpl) { + A a(5); + ASSERT_TRUE(a); + ASSERT_FALSE(a.is_empty()); + ASSERT_EQ(a->x_, 5); +} + +TEST_F(ModuleTest, NullptrConstructorLeavesTheModuleHolderInEmptyState) { + A a = nullptr; + ASSERT_FALSE(a); + ASSERT_TRUE(a.is_empty()); + ASSERT_THROWS_WITH(a->x_, "Accessing empty ModuleHolder"); +} + +struct TestModule : public torch::nn::Module { + TestModule(int64_t size) { + p1 = register_parameter("p1", torch::randn({size})); + p2 = register_parameter("p2", torch::randn({size})); + b1 = register_buffer("b1", torch::randn({size})); + b2 = register_buffer("b2", torch::randn({size})); + } + + torch::Tensor forward(torch::Tensor input) { + return input; + } + + torch::Tensor p1, p2, b1, b2; +}; + +TEST_F(ModuleTest, ModulesReturnsExpectedSubmodulesForFlatModel) { + torch::nn::Sequential model(TestModule(1), TestModule(2), TestModule(3)); + std::vector> modules = model->modules(); + std::vector> expected = { + model.ptr(), model[0], model[1], model[2]}; + ASSERT_EQ(modules.size(), expected.size()); + for (const auto i : c10::irange(expected.size())) { + // Assert pointer equality. + ASSERT_EQ(modules[i].get(), expected[i].get()); + } +} + +TEST_F(ModuleTest, ModulesExcludesSelfWhenIncludeSelfSetToFalse) { + torch::nn::Sequential model(TestModule(1), TestModule(2), TestModule(3)); + std::vector> modules = + model->modules(/*include_self=*/false); + std::vector> expected = { + model[0], model[1], model[2]}; + ASSERT_EQ(modules.size(), expected.size()); + for (const auto i : c10::irange(expected.size())) { + // Assert pointer equality. + ASSERT_EQ(modules[i].get(), expected[i].get()); + } +} + +TEST_F(ModuleTest, NamedModulesReturnsExpectedNamedSubmodulesForFlatModel) { + torch::nn::Sequential model(TestModule(1), TestModule(2), TestModule(3)); + torch::OrderedDict> modules = + model->named_modules(); + std::vector> expected = { + model.ptr(), model[0], model[1], model[2]}; + ASSERT_EQ(modules.size(), expected.size()); + for (const auto i : c10::irange(expected.size())) { + // Assert pointer equality. + ASSERT_EQ(modules[i].key(), i ? std::to_string(i - 1) : std::string()); + ASSERT_EQ(modules[i].value().get(), expected[i].get()); + } +} + +TEST_F(ModuleTest, NamedModulesExcludesSelfWhenIncludeSelfSetToFalse) { + torch::nn::Sequential model(TestModule(1), TestModule(2), TestModule(3)); + torch::OrderedDict> modules = + model->named_modules( + /*name_prefix=*/std::string(), /*include_self=*/false); + std::vector> expected = { + model[0], model[1], model[2]}; + ASSERT_EQ(modules.size(), expected.size()); + for (const auto i : c10::irange(expected.size())) { + // Assert pointer equality. + ASSERT_EQ(modules[i].key(), std::to_string(i)); + ASSERT_EQ(modules[i].value().get(), expected[i].get()); + } +} + +TEST_F(ModuleTest, ChildrenReturnsExpectedSubmodulesForFlatModel) { + torch::nn::Sequential model(TestModule(1), TestModule(2), TestModule(3)); + std::vector> modules = model->children(); + std::vector> expected = { + model[0], model[1], model[2]}; + ASSERT_EQ(modules.size(), expected.size()); + for (const auto i : c10::irange(expected.size())) { + // Assert pointer equality. + ASSERT_EQ(modules[i].get(), expected[i].get()); + } + + // For this flat model, this should be true. + ASSERT_EQ(modules, model->modules(/*include_self=*/false)); +} + +TEST_F(ModuleTest, NamedChildrenReturnsExpectedNamedSubmodulesForFlatModel) { + torch::nn::Sequential model(TestModule(1), TestModule(2), TestModule(3)); + torch::OrderedDict> modules = + model->named_children(); + std::vector> expected = { + model[0], model[1], model[2]}; + ASSERT_EQ(modules.size(), expected.size()); + for (const auto i : c10::irange(expected.size())) { + // Assert pointer equality. + ASSERT_EQ(modules[i].key(), std::to_string(i)); + ASSERT_EQ(modules[i].value().get(), expected[i].get()); + } +} + +TEST_F(ModuleTest, ParametersReturnsExpectedTensorsForFlatModel) { + TestModule module(1); + std::vector parameters = module.parameters(); + ASSERT_EQ(parameters.size(), 2); + ASSERT_EQ(parameters[0].data_ptr(), module.p1.data_ptr()); + ASSERT_EQ(parameters[1].data_ptr(), module.p2.data_ptr()); +} + +TEST_F(ModuleTest, NamedParametersReturnsExpectedTensorsForFlatModel) { + TestModule module(1); + torch::OrderedDict parameters = + module.named_parameters(); + ASSERT_EQ(parameters.size(), 2); + ASSERT_EQ(parameters[0].key(), "p1"); + ASSERT_EQ(parameters[0]->data_ptr(), module.p1.data_ptr()); + ASSERT_EQ(parameters[1].key(), "p2"); + ASSERT_EQ(parameters[1]->data_ptr(), module.p2.data_ptr()); +} + +TEST_F(ModuleTest, BuffersReturnsExpectedTensorsForFlatModel) { + TestModule module(1); + std::vector buffers = module.buffers(); + ASSERT_EQ(buffers.size(), 2); + ASSERT_EQ(buffers[0].data_ptr(), module.b1.data_ptr()); + ASSERT_EQ(buffers[1].data_ptr(), module.b2.data_ptr()); +} + +TEST_F(ModuleTest, NamedBuffersReturnsExpectedTensorsForFlatModel) { + TestModule module(1); + torch::OrderedDict buffers = + module.named_buffers(); + ASSERT_EQ(buffers.size(), 2); + ASSERT_EQ(buffers[0].key(), "b1"); + ASSERT_EQ(buffers[0]->data_ptr(), module.b1.data_ptr()); + ASSERT_EQ(buffers[1].key(), "b2"); + ASSERT_EQ(buffers[1]->data_ptr(), module.b2.data_ptr()); +} + +struct TestContainer : torch::nn::Module { + TestContainer(int64_t number, std::vector modules = {}) + : tensor(torch::tensor(number)) { + for (const auto i : c10::irange(modules.size())) { + register_module( + std::to_string(i), + std::make_shared(std::move(modules[i]))); + } + } + torch::Tensor tensor; +}; + +int64_t get_test_container_item(std::shared_ptr module) { + return std::dynamic_pointer_cast(module) + ->tensor.item(); +} + +std::shared_ptr make_deeply_nested_test_container() { + return std::make_shared(TestContainer( + 0, + {TestContainer(1, {TestContainer(2), TestContainer(3)}), + TestContainer(4), + TestContainer( + 5, + {TestContainer(6), + TestContainer(7, {TestContainer(8), TestContainer(9)})})})); +} + +std::vector> +make_key_value_pairs_for_deeply_nested_container() { + return { + {"test_prefix", 0}, + {"test_prefix.0", 1}, + {"test_prefix.0.0", 2}, + {"test_prefix.0.1", 3}, + {"test_prefix.1", 4}, + {"test_prefix.2", 5}, + {"test_prefix.2.0", 6}, + {"test_prefix.2.1", 7}, + {"test_prefix.2.1.0", 8}, + {"test_prefix.2.1.1", 9}}; +} + +TEST_F(ModuleTest, ModulesReturnsExpectedSubmodulesForDeepModel) { + auto model = make_deeply_nested_test_container(); + std::vector> modules = model->modules(); + + ASSERT_EQ(modules.size(), 10); + for (const auto i : c10::irange(modules.size())) { + ASSERT_EQ(get_test_container_item(modules[i]), i); + } +} + +TEST_F(ModuleTest, NamedModulesReturnsExpectedNamedSubmodulesForDeepModel) { + auto model = make_deeply_nested_test_container(); + torch::OrderedDict> modules = + model->named_modules(/*name_prefix=*/"test_prefix"); + auto expected = make_key_value_pairs_for_deeply_nested_container(); + + ASSERT_EQ(modules.size(), expected.size()); + + for (const auto i : c10::irange(expected.size())) { + ASSERT_EQ(modules[i].key(), expected[i].first); + ASSERT_EQ(get_test_container_item(modules[i].value()), expected[i].second); + } +} + +TEST_F(ModuleTest, ChildrensReturnsExpectedSubmodulesForDeepModel) { + auto model = make_deeply_nested_test_container(); + std::vector> modules = model->children(); + + ASSERT_EQ(modules.size(), 3); + ASSERT_EQ(get_test_container_item(modules[0]), 1); + ASSERT_EQ(get_test_container_item(modules[1]), 4); + ASSERT_EQ(get_test_container_item(modules[2]), 5); +} + +TEST_F(ModuleTest, NamedChildrensReturnsExpectedNamedSubmodulesForDeepModel) { + auto model = make_deeply_nested_test_container(); + torch::OrderedDict> modules = + model->named_children(); + + ASSERT_EQ(modules.size(), 3); + + ASSERT_EQ(get_test_container_item(modules[0].value()), 1); + ASSERT_EQ(modules[0].key(), "0"); + + ASSERT_EQ(get_test_container_item(modules[1].value()), 4); + ASSERT_EQ(modules[1].key(), "1"); + + ASSERT_EQ(get_test_container_item(modules[2].value()), 5); + ASSERT_EQ(modules[2].key(), "2"); +} + +TEST_F(ModuleTest, ModuleApplyIteratesCorreclty) { + auto model = make_deeply_nested_test_container(); + int64_t index = 0; + model->apply([&index](torch::nn::Module& module) { + ASSERT_EQ(module.as()->tensor.item(), index++); + }); + ASSERT_EQ(index, 10); +} + +TEST_F(ModuleTest, ConstModuleApplyIteratesCorreclty) { + std::shared_ptr model = + make_deeply_nested_test_container(); + int64_t index = 0; + model->apply([&index](const torch::nn::Module& module) { + ASSERT_EQ(module.as()->tensor.item(), index++); + }); + ASSERT_EQ(index, 10); +} + +TEST_F(ModuleTest, NamedModuleApplyIteratesCorreclty) { + auto model = make_deeply_nested_test_container(); + auto expected = make_key_value_pairs_for_deeply_nested_container(); + int64_t index = 0; + model->apply( + [&index, expected](const std::string& name, torch::nn::Module& module) { + ASSERT_EQ(name, expected[index].first); + ASSERT_EQ( + module.as()->tensor.item(), + expected[index++].second); + }, + /*name_prefix=*/"test_prefix"); + ASSERT_EQ(index, 10); +} + +TEST_F(ModuleTest, ConstNamedModuleApplyIteratesCorreclty) { + std::shared_ptr model = + make_deeply_nested_test_container(); + auto expected = make_key_value_pairs_for_deeply_nested_container(); + int64_t index = 0; + model->apply( + [&index, &expected]( + const std::string& name, const torch::nn::Module& module) { + ASSERT_EQ(name, expected[index].first); + ASSERT_EQ( + module.as()->tensor.item(), + expected[index++].second); + }, + /*name_prefix=*/"test_prefix"); + ASSERT_EQ(index, 10); +} + +TEST_F(ModuleTest, ModulePointerApplyIteratesCorreclty) { + auto model = make_deeply_nested_test_container(); + int64_t index = 0; + model->apply([&index](const std::shared_ptr& module) { + ASSERT_EQ(get_test_container_item(module), index++); + }); + ASSERT_EQ(index, 10); +} + +TEST_F(ModuleTest, NamedModulePointerApplyIteratesCorreclty) { + auto model = make_deeply_nested_test_container(); + auto expected = make_key_value_pairs_for_deeply_nested_container(); + int64_t index = 0; + model->apply( + [&index, &expected]( + const std::string& name, + const std::shared_ptr& module) { + ASSERT_EQ(name, expected[index].first); + ASSERT_EQ(get_test_container_item(module), expected[index++].second); + }, + /*name_prefix=*/"test_prefix"); + ASSERT_EQ(index, 10); +} + +TEST_F(ModuleTest, ThrowsWhenAttemptingtoGetTopLevelModuleAsSharedPtr) { + { + TestModule module(1); + ASSERT_THROWS_WITH( + module.modules(), + "It looks like you attempted to retrieve " + "your top-level module as a shared_ptr") + } + { + TestModule module(1); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_NO_THROW(module.modules(/*include_self=*/false)); + } + { + auto module = std::make_shared(1); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_NO_THROW(module->modules()); + } +} + +struct EmptyModule : torch::nn::Module {}; + +TEST_F(ModuleTest, PrettyPrint) { + struct TestModule : torch::nn::Module { + TestModule(int x, float y) : x_(x), y_(y) {} + + void pretty_print(std::ostream& stream) const override { + stream << "TestModule(x=" << x_ << ", y=" << y_ << ')'; + } + + int x_; + float y_; + }; + + ASSERT_EQ(c10::str(EmptyModule{}), "EmptyModule"); + ASSERT_EQ(c10::str(TestModule(1, 3.14)), "TestModule(x=1, y=3.14)"); +} + +struct ModuleWithNonTensorForwardImpl : torch::nn::Module { + int64_t forward(torch::Tensor x) { + return x.numel(); + } +}; +TORCH_MODULE(ModuleWithNonTensorForward); + +TEST_F(ModuleTest, CanCallForwardOnNonTensorForwardThroughPimpl) { + ModuleWithNonTensorForward m; + ASSERT_EQ(m(torch::ones(123)), 123); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/moduledict.cpp b/western_ai/usa/pytorch/test/cpp/api/moduledict.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1fc011b81f7ab41987cea37c4b79f05bef284b70 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/moduledict.cpp @@ -0,0 +1,309 @@ +#include +#include +#include +#include +#include + +#include + +using namespace torch::nn; +using namespace torch::test; + +struct ModuleDictTest : torch::test::SeedingFixture {}; + +TEST_F(ModuleDictTest, ConstructsFromList) { + struct M : Module { + explicit M(int value_) : value(value_) {} + int value; + }; + + std::vector>> list = { + {"module_1", std::make_shared(1)}, + {"module_2", std::make_shared(2)}, + {"module_3", std::make_shared(3)}}; + ModuleDict dict(list); + ASSERT_EQ(dict->size(), 3); +} + +TEST_F(ModuleDictTest, ConstructsFromordereddict) { + struct M : Module { + explicit M(int value_) : value(value_) {} + int value; + }; + + torch::OrderedDict> ordereddict = { + {"module_1", std::make_shared(1)}, + {"module_2", std::make_shared(2)}, + {"module_3", std::make_shared(3)}, + }; + ModuleDict dict(ordereddict); + ASSERT_EQ(dict->size(), 3); +} + +TEST_F(ModuleDictTest, UpdatePopClearContains) { + struct M : Module { + explicit M(int value_) : value(value_) {} + int value; + }; + + ModuleDict dict; + ASSERT_TRUE(dict->empty()); + // Update by List + std::vector>> list1 = { + {"module_1", std::make_shared(1)}}; + dict->update(list1); + ASSERT_EQ(dict->size(), 1); + ASSERT_TRUE(dict->contains("module_1")); + // Update by OrderedDict + torch::OrderedDict> ordereddict = { + {"module_2", std::make_shared(2)}}; + dict->update(ordereddict); + ASSERT_EQ(dict->size(), 2); + ASSERT_TRUE(dict->contains("module_2")); + // Update by another ModuleDict + std::vector>> list2 = { + {"module_3", std::make_shared(3)}}; + ModuleDict updatedict(list2); + dict->update(*updatedict); + ASSERT_EQ(dict->size(), 3); + ASSERT_TRUE(dict->contains("module_3")); + // Pop + dict->pop("module_1"); + ASSERT_EQ(dict->size(), 2); + // Pop unexist + ASSERT_THROWS_WITH(dict->pop("module_4"), " 'module_4' is not defined"); + // Clear + dict->clear(); + ASSERT_EQ(dict->size(), 0); +} + +TEST_F(ModuleDictTest, UpdateExist) { + struct M : Module { + explicit M(int value_) : value(value_) {} + int value; + }; + std::vector>> list1 = { + {"module_1", std::make_shared(1)}, + {"module_2", std::make_shared(2)}}; + ModuleDict dict(list1); + ASSERT_EQ(dict->at("module_2").value, 2); + // Update by list + std::vector>> list2 = { + {"module_2", std::make_shared(0)}, + {"module_3", std::make_shared(3)}}; + dict->update(list2); + ASSERT_EQ(dict->size(), 3); + ASSERT_EQ(dict->at("module_2").value, 0); + // Update by ordereddict + torch::OrderedDict> ordereddict = { + {"module_3", std::make_shared(0)}, + {"module_4", std::make_shared(4)}}; + dict->update(ordereddict); + ASSERT_EQ(dict->size(), 4); + ASSERT_EQ(dict->at("module_3").value, 0); + // Update by ModuleDict + std::vector>> list3 = { + {"module_4", std::make_shared(0)}, + {"module_1", std::make_shared(0)}}; + ModuleDict dict2(list3); + dict->update(*dict2); + ASSERT_EQ(dict->size(), 4); + ASSERT_EQ(dict->at("module_1").value, 0); + ASSERT_EQ(dict->at("module_4").value, 0); +} + +TEST_F(ModuleDictTest, Keys) { + struct M : Module { + explicit M(int value_) : value(value_) {} + int value; + }; + + torch::OrderedDict> ordereddict = { + {"linear", Linear(10, 3).ptr()}, + {"conv", Conv2d(1, 2, 3).ptr()}, + {"dropout", Dropout(0.5).ptr()}, + }; + ModuleDict dict(ordereddict); + const auto& keys = dict->keys(); + std::vector expected{"linear", "conv", "dropout"}; + ASSERT_EQ(keys, expected); + ASSERT_THROWS_WITH(dict["batch"], " 'batch' is not defined"); + + ASSERT_TRUE(dict["linear"]->as()); + ASSERT_TRUE(dict["conv"]->as()); + ASSERT_TRUE(dict["dropout"]->as()); +} + +TEST_F(ModuleDictTest, Values) { + struct M : Module { + explicit M(int value_) : value(value_) {} + int value; + }; + + torch::OrderedDict> ordereddict = { + {"module_1", std::make_shared(1)}, + {"module_2", std::make_shared(2)}, + }; + ModuleDict dict(ordereddict); + const auto& values = dict->values(); + const auto& expected = ordereddict.values(); + ASSERT_EQ(values, expected); + ASSERT_TRUE(std::equal( + dict->begin(), + dict->end(), + ordereddict.begin(), + [](const auto& lhs, const auto& rhs) { + return lhs.value().get() == rhs.value().get(); + })); +} + +TEST_F(ModuleDictTest, SanityCheckForHoldingStandardModules) { + torch::OrderedDict> ordereddict = { + {"linear", Linear(10, 3).ptr()}, + {"conv", Conv2d(1, 2, 3).ptr()}, + {"dropout", Dropout(0.5).ptr()}, + {"batch", BatchNorm2d(5).ptr()}, + {"embedding", Embedding(4, 10).ptr()}, + {"lstm", LSTM(4, 5).ptr()}}; + ModuleDict dict(ordereddict); +} + +TEST_F(ModuleDictTest, HasReferenceSemantics) { + torch::OrderedDict> ordereddict = { + {"linear1", Linear(2, 3).ptr()}, + {"linear2", Linear(3, 4).ptr()}, + {"linear3", Linear(4, 5).ptr()}, + }; + ModuleDict first(ordereddict); + ModuleDict second(ordereddict); + + ASSERT_EQ(first->size(), second->size()); + ASSERT_TRUE(std::equal( + first->begin(), + first->end(), + second->begin(), + [](const auto& lhs, const auto& rhs) { + return lhs.value().get() == rhs.value().get(); + })); +} + +void iscloneable_helper(torch::Device device) { + torch::OrderedDict> ordereddict = { + {"linear", Linear(2, 3).ptr()}, + {"relu", Functional(torch::relu).ptr()}, + {"batch", BatchNorm1d(3).ptr()}, + }; + ModuleDict dict(ordereddict); + dict->to(device); + ModuleDict clone = + std::dynamic_pointer_cast(dict->clone(device)); + ASSERT_EQ(dict->size(), clone->size()); + + for (auto it = dict->begin(), it_c = clone->begin(); it != dict->end(); + ++it, ++it_c) { + // The key should be same + ASSERT_EQ(it->key(), it_c->key()); + // The modules should be the same kind (type). + ASSERT_EQ(it->value()->name(), it_c->value()->name()); + // But not pointer-equal (distinct objects). + ASSERT_NE(it->value(), it_c->value()); + } + + // Verify that the clone is deep, i.e. parameters of modules are cloned too. + torch::NoGradGuard no_grad; + + auto params1 = dict->named_parameters(); + auto params2 = clone->named_parameters(); + ASSERT_EQ(params1.size(), params2.size()); + for (auto& param : params1) { + ASSERT_FALSE(pointer_equal(param.value(), params2[param.key()])); + ASSERT_EQ(param->device(), params2[param.key()].device()); + ASSERT_TRUE(param->allclose(params2[param.key()])); + param->add_(2); + } + for (auto& param : params1) { + ASSERT_FALSE(param->allclose(params2[param.key()])); + } +} + +TEST_F(ModuleDictTest, IsCloneable) { + iscloneable_helper(torch::kCPU); +} + +TEST_F(ModuleDictTest, IsCloneable_CUDA) { + iscloneable_helper({torch::kCUDA, 0}); +} + +TEST_F(ModuleDictTest, RegistersElementsAsSubmodules) { + torch::OrderedDict> ordereddict1 = { + {"linear", Linear(10, 3).ptr()}, + {"conv", Conv2d(1, 2, 3).ptr()}, + {"test", Dropout(0.5).ptr()}, + }; + ModuleDict dict(ordereddict1); + + auto modules = dict->children(); + ASSERT_TRUE(modules[0]->as()); + ASSERT_TRUE(modules[1]->as()); + ASSERT_TRUE(modules[2]->as()); + + // Update Existing + torch::OrderedDict> ordereddict2 = { + {"lstm", LSTM(4, 5).ptr()}, {"test", BatchNorm2d(5).ptr()}}; + dict->update(ordereddict2); + + modules = dict->children(); + ASSERT_TRUE(modules[0]->as()); + ASSERT_TRUE(modules[1]->as()); + // Keep Order + ASSERT_TRUE(modules[2]->as()); + ASSERT_TRUE(modules[3]->as()); +} + +TEST_F(ModuleDictTest, CloneToDevice_CUDA) { + torch::OrderedDict> ordereddict = { + {"linear", Linear(2, 3).ptr()}, + {"relu", Functional(torch::relu).ptr()}, + {"batch", BatchNorm1d(3).ptr()}, + }; + ModuleDict dict(ordereddict); + torch::Device device(torch::kCUDA, 0); + ModuleDict clone = + std::dynamic_pointer_cast(dict->clone(device)); + for (const auto& p : clone->parameters()) { + ASSERT_EQ(p.device(), device); + } + for (const auto& b : clone->buffers()) { + ASSERT_EQ(b.device(), device); + } +} + +TEST_F(ModuleDictTest, PrettyPrintModuleDict) { + torch::OrderedDict> ordereddict = { + {"linear", Linear(10, 3).ptr()}, + {"conv", Conv2d(1, 2, 3).ptr()}, + {"dropout", Dropout(0.5).ptr()}, + {"batch", BatchNorm2d(5).ptr()}, + {"embedding", Embedding(4, 10).ptr()}, + {"lstm", LSTM(4, 5).ptr()}}; + ModuleDict dict(ordereddict); + + ASSERT_EQ( + c10::str(dict), + "torch::nn::ModuleDict(\n" + " (linear): torch::nn::Linear(in_features=10, out_features=3, bias=true)\n" + " (conv): torch::nn::Conv2d(1, 2, kernel_size=[3, 3], stride=[1, 1])\n" + " (dropout): torch::nn::Dropout(p=0.5, inplace=false)\n" + " (batch): torch::nn::BatchNorm2d(5, eps=1e-05, momentum=0.1, affine=true, track_running_stats=true)\n" + " (embedding): torch::nn::Embedding(num_embeddings=4, embedding_dim=10)\n" + " (lstm): torch::nn::LSTM(input_size=4, hidden_size=5, num_layers=1, bias=true, batch_first=false, dropout=0, bidirectional=false)\n" + ")"); +} + +TEST_F(ModuleDictTest, InvalidAt) { + torch::OrderedDict> ordereddict = { + {"linear", Linear(10, 3).ptr()}}; + ModuleDict dict(ordereddict); + ASSERT_THROWS_WITH( + dict->at("linear"), "Unable to cast module"); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/modulelist.cpp b/western_ai/usa/pytorch/test/cpp/api/modulelist.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a09d898d2d89fbe92f8e550fe445f072b217f63b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/modulelist.cpp @@ -0,0 +1,308 @@ +#include + +#include +#include + +#include +#include +#include + +#include + +using namespace torch::nn; +using namespace torch::test; + +struct ModuleListTest : torch::test::SeedingFixture {}; + +TEST_F(ModuleListTest, ConstructsFromSharedPointer) { + struct M : torch::nn::Module { + explicit M(int value_) : value(value_) {} + int value; + }; + ModuleList list( + std::make_shared(1), std::make_shared(2), std::make_shared(3)); + ASSERT_EQ(list->size(), 3); +} + +TEST_F(ModuleListTest, ConstructsFromConcreteType) { + static int copy_count; + + struct M : torch::nn::Module { + explicit M(int value_) : value(value_) {} + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) + M(const M& other) : torch::nn::Module(other) { + copy_count++; + } + int value; + }; + + copy_count = 0; + ModuleList list(M(1), M(2), M(3)); + ASSERT_EQ(list->size(), 3); + // NOTE: The current implementation expects each module to be copied exactly + // once, which happens when the module is passed into `std::make_shared()`. + // TODO: Find a way to avoid copying, and then delete the copy constructor of + // `M`. + ASSERT_EQ(copy_count, 3); +} + +TEST_F(ModuleListTest, ConstructsFromModuleHolder) { + struct MImpl : torch::nn::Module { + explicit MImpl(int value_) : value(value_) {} + int value; + }; + + struct M : torch::nn::ModuleHolder { + using torch::nn::ModuleHolder::ModuleHolder; + using torch::nn::ModuleHolder::get; + }; + + ModuleList list(M(1), M(2), M(3)); + ASSERT_EQ(list->size(), 3); +} + +TEST_F(ModuleListTest, PushBackAddsAnElement) { + struct M : torch::nn::Module { + explicit M(int value_) : value(value_) {} + int value; + }; + + ModuleList list; + ASSERT_EQ(list->size(), 0); + ASSERT_TRUE(list->is_empty()); + list->push_back(Linear(3, 4)); + ASSERT_EQ(list->size(), 1); + list->push_back(std::make_shared(1)); + ASSERT_EQ(list->size(), 2); + list->push_back(M(2)); + ASSERT_EQ(list->size(), 3); +} + +TEST_F(ModuleListTest, Insertion) { + struct MImpl : torch::nn::Module { + explicit MImpl(int value_) : value(value_) {} + int value; + }; + TORCH_MODULE(M); + + ModuleList list; + list->push_back(MImpl(1)); + ASSERT_EQ(list->size(), 1); + list->insert(0, std::make_shared(2)); + ASSERT_EQ(list->size(), 2); + list->insert(1, M(3)); + ASSERT_EQ(list->size(), 3); + list->insert(3, M(4)); + ASSERT_EQ(list->size(), 4); + ASSERT_EQ(list->at(0).value, 2); + ASSERT_EQ(list->at(1).value, 3); + ASSERT_EQ(list->at(2).value, 1); + ASSERT_EQ(list->at(3).value, 4); + + std::unordered_map U = {{0, 2}, {1, 3}, {2, 1}, {3, 4}}; + for (const auto& P : list->named_modules("", false)) + ASSERT_EQ(U[std::stoul(P.key())], P.value()->as()->value); +} + +TEST_F(ModuleListTest, AccessWithAt) { + struct M : torch::nn::Module { + explicit M(int value_) : value(value_) {} + int value; + }; + std::vector> modules = { + std::make_shared(1), std::make_shared(2), std::make_shared(3)}; + + ModuleList list; + for (auto& module : modules) { + list->push_back(module); + } + ASSERT_EQ(list->size(), 3); + + // returns the correct module for a given index + for (const auto i : c10::irange(modules.size())) { + ASSERT_EQ(&list->at(i), modules[i].get()); + } + + // throws for a bad index + ASSERT_THROWS_WITH(list->at(modules.size() + 1), "Index out of range"); + ASSERT_THROWS_WITH( + list->at(modules.size() + 1000000), "Index out of range"); +} + +TEST_F(ModuleListTest, AccessWithPtr) { + struct M : torch::nn::Module { + explicit M(int value_) : value(value_) {} + int value; + }; + std::vector> modules = { + std::make_shared(1), std::make_shared(2), std::make_shared(3)}; + + ModuleList list; + for (auto& module : modules) { + list->push_back(module); + } + ASSERT_EQ(list->size(), 3); + + // returns the correct module for a given index + for (const auto i : c10::irange(modules.size())) { + ASSERT_EQ(list->ptr(i).get(), modules[i].get()); + ASSERT_EQ(list[i].get(), modules[i].get()); + ASSERT_EQ(list->ptr(i).get(), modules[i].get()); + } + + // throws for a bad index + ASSERT_THROWS_WITH(list->ptr(modules.size() + 1), "Index out of range"); + ASSERT_THROWS_WITH(list->ptr(modules.size() + 1000000), "Index out of range"); +} + +TEST_F(ModuleListTest, SanityCheckForHoldingStandardModules) { + ModuleList list( + Linear(10, 3), + Conv2d(1, 2, 3), + Dropout(0.5), + BatchNorm2d(5), + Embedding(4, 10), + LSTM(4, 5)); +} + +TEST_F(ModuleListTest, ExtendPushesModulesFromOtherModuleList) { + struct A : torch::nn::Module {}; + struct B : torch::nn::Module {}; + struct C : torch::nn::Module {}; + struct D : torch::nn::Module {}; + ModuleList a(A{}, B{}); + ModuleList b(C{}, D{}); + a->extend(*b); + + ASSERT_EQ(a->size(), 4); + ASSERT_TRUE(a[0]->as()); + ASSERT_TRUE(a[1]->as()); + ASSERT_TRUE(a[2]->as()); + ASSERT_TRUE(a[3]->as()); + + ASSERT_EQ(b->size(), 2); + ASSERT_TRUE(b[0]->as()); + ASSERT_TRUE(b[1]->as()); + + std::vector> c = { + std::make_shared(), std::make_shared()}; + b->extend(c); + + ASSERT_EQ(b->size(), 4); + ASSERT_TRUE(b[0]->as()); + ASSERT_TRUE(b[1]->as()); + ASSERT_TRUE(b[2]->as()); + ASSERT_TRUE(b[3]->as()); +} + +TEST_F(ModuleListTest, HasReferenceSemantics) { + ModuleList first(Linear(2, 3), Linear(4, 4), Linear(4, 5)); + ModuleList second(first); + + ASSERT_EQ(first.get(), second.get()); + ASSERT_EQ(first->size(), second->size()); + ASSERT_TRUE(std::equal( + first->begin(), + first->end(), + second->begin(), + [](const std::shared_ptr& first, + const std::shared_ptr& second) { + return first.get() == second.get(); + })); +} + +TEST_F(ModuleListTest, IsCloneable) { + ModuleList list(Linear(3, 4), Functional(torch::relu), BatchNorm1d(3)); + ModuleList clone = std::dynamic_pointer_cast(list->clone()); + ASSERT_EQ(list->size(), clone->size()); + + for (size_t i = 0; i < list->size(); ++i) { + // The modules should be the same kind (type). + ASSERT_EQ(list[i]->name(), clone[i]->name()); + // But not pointer-equal (distinct objects). + ASSERT_NE(list[i], clone[i]); + } + + // Verify that the clone is deep, i.e. parameters of modules are cloned too. + + torch::NoGradGuard no_grad; + + auto params1 = list->named_parameters(); + auto params2 = clone->named_parameters(); + ASSERT_EQ(params1.size(), params2.size()); + for (auto& param : params1) { + ASSERT_FALSE(pointer_equal(param.value(), params2[param.key()])); + ASSERT_EQ(param->device(), params2[param.key()].device()); + ASSERT_TRUE(param->allclose(params2[param.key()])); + param->add_(2); + } + for (auto& param : params1) { + ASSERT_FALSE(param->allclose(params2[param.key()])); + } +} + +TEST_F(ModuleListTest, RegistersElementsAsSubmodules) { + ModuleList list(Linear(10, 3), Conv2d(1, 2, 3), Dropout2d(0.5)); + + auto modules = list->children(); + ASSERT_TRUE(modules[0]->as()); + ASSERT_TRUE(modules[1]->as()); + ASSERT_TRUE(modules[2]->as()); +} + +TEST_F(ModuleListTest, NestingIsPossible) { + ModuleList list( + (ModuleList(Dropout(), Dropout())), + (ModuleList(Dropout(), Dropout()), Dropout())); +} + +TEST_F(ModuleListTest, CloneToDevice_CUDA) { + ModuleList list(Linear(3, 4), Functional(torch::relu), BatchNorm1d(3)); + torch::Device device(torch::kCUDA, 0); + ModuleList clone = + std::dynamic_pointer_cast(list->clone(device)); + for (const auto& p : clone->parameters()) { + ASSERT_EQ(p.device(), device); + } + for (const auto& b : clone->buffers()) { + ASSERT_EQ(b.device(), device); + } +} + +TEST_F(ModuleListTest, PrettyPrintModuleList) { + ModuleList list( + Linear(10, 3), + Conv2d(1, 2, 3), + Dropout(0.5), + BatchNorm2d(5), + Embedding(4, 10), + LSTM(4, 5)); + ASSERT_EQ( + c10::str(list), + "torch::nn::ModuleList(\n" + " (0): torch::nn::Linear(in_features=10, out_features=3, bias=true)\n" + " (1): torch::nn::Conv2d(1, 2, kernel_size=[3, 3], stride=[1, 1])\n" + " (2): torch::nn::Dropout(p=0.5, inplace=false)\n" + " (3): torch::nn::BatchNorm2d(5, eps=1e-05, momentum=0.1, affine=true, track_running_stats=true)\n" + " (4): torch::nn::Embedding(num_embeddings=4, embedding_dim=10)\n" + " (5): torch::nn::LSTM(input_size=4, hidden_size=5, num_layers=1, bias=true, batch_first=false, dropout=0, bidirectional=false)\n" + ")"); +} + +TEST_F(ModuleListTest, RangeBasedForLoop) { + torch::nn::ModuleList mlist( + torch::nn::Linear(3, 4), + torch::nn::BatchNorm1d(4), + torch::nn::Dropout(0.5)); + + std::stringstream buffer; + for (const auto& module : *mlist) { + module->pretty_print(buffer); + } +} + +TEST_F(ModuleListTest, InvalidAt) { + torch::nn::ModuleList m(torch::nn::Linear(1, 2)); + ASSERT_THROWS_WITH( + m->at(0), "Unable to cast module"); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/modules.cpp b/western_ai/usa/pytorch/test/cpp/api/modules.cpp new file mode 100644 index 0000000000000000000000000000000000000000..92949ed2cb4588bcdd7b1333a256809d25aafa24 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/modules.cpp @@ -0,0 +1,5564 @@ +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include + +using namespace torch::nn; +using namespace torch::test; + +class TestModel : public torch::nn::Module { + public: + TestModel() + : l1(register_module("l1", Linear(10, 3))), + l2(register_module("l2", Linear(3, 5))), + l3(register_module("l3", Linear(5, 100))) {} + + Linear l1, l2, l3; +}; + +class NestedModel : public torch::nn::Module { + public: + NestedModel() + : param_(register_parameter("param", torch::empty({3, 2, 21}))), + l1(register_module("l1", Linear(5, 20))), + t(register_module("test", std::make_shared())) {} + + torch::Tensor param_; + Linear l1; + std::shared_ptr t; +}; + +struct ModulesTest : torch::test::SeedingFixture {}; + +TEST_F(ModulesTest, Conv1d) { + Conv1d model(Conv1dOptions(3, 2, 3).stride(1).bias(false)); + model->weight.set_data( + torch::arange(18, torch::dtype(torch::kFloat)).reshape({2, 3, 3})); + auto x = torch::arange(30, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({2, 3, 5}); + auto y = model(x); + auto expected = torch::tensor( + {{{312., 348., 384.}, {798., 915., 1032.}}, + + {{852., 888., 924.}, {2553., 2670., 2787.}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, expected)); + + torch::Tensor s = y.sum(); + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(model->weight.grad().numel(), 3 * 2 * 3); +} + +TEST_F(ModulesTest, Conv1dSameStrided) { + auto options = Conv1dOptions(3, 2, 3); + options.stride(1).padding(torch::kSame); + Conv1d model_valid(options); + ASSERT_THROWS_WITH( + [&] { Conv1d model_invalid(options.stride(2)); }(), + "padding='same' is not supported for strided convolutions"); +} + +TEST_F(ModulesTest, Conv1dIvalidArg) { + auto options = Conv1dOptions(3, 2, 3).groups(-1); + ASSERT_THROWS_WITH( + Conv1d(options), "in_channels, groups and out_channels must"); +} + +TEST_F(ModulesTest, Conv2dEven) { + Conv2d model(Conv2dOptions(3, 2, 3).stride(1).bias(false)); + model->weight.set_data( + torch::arange(54, torch::dtype(torch::kFloat)).reshape({2, 3, 3, 3})); + auto x = torch::arange(75, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({1, 3, 5, 5}); + auto y = model(x); + auto expected = torch::tensor( + {{{{15219., 15570., 15921.}, + {16974., 17325., 17676.}, + {18729., 19080., 19431.}}, + + {{37818., 38898., 39978.}, + {43218., 44298., 45378.}, + {48618., 49698., 50778.}}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, expected)); + + torch::Tensor s = y.sum(); + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(model->weight.grad().numel(), 3 * 2 * 3 * 3); +} + +TEST_F(ModulesTest, Conv2dUneven) { + Conv2d model(Conv2dOptions(3, 2, {3, 2}).stride({1, 1}).bias(false)); + model->weight.set_data( + torch::arange(36, torch::dtype(torch::kFloat)).reshape({2, 3, 3, 2})); + auto x = torch::arange(60, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({1, 3, 5, 4}); + auto y = model(x); + auto expected = torch::tensor( + {{{{5289., 5442., 5595.}, {5901., 6054., 6207.}, {6513., 6666., 6819.}}, + + {{13227., 13704., 14181.}, + {15135., 15612., 16089.}, + {17043., 17520., 17997.}}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, expected)); + + torch::Tensor s = y.sum(); + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(model->weight.grad().numel(), 3 * 2 * 3 * 2); +} + +TEST_F(ModulesTest, Conv2dSameStrided) { + auto options = Conv2dOptions(3, 2, {3, 4}); + options.stride(1).padding(torch::kSame); + Conv2d model_valid(options); + ASSERT_THROWS_WITH( + [&] { Conv2d model_invalid(options.stride(2)); }(), + "padding='same' is not supported for strided convolutions"); + ASSERT_THROWS_WITH( + [&] { Conv2d model_invalid(options.stride({1, 2})); }(), + "padding='same' is not supported for strided convolutions"); +} + +TEST_F(ModulesTest, Conv3d) { + Conv3d model(Conv3dOptions(3, 2, 3).stride(1).bias(false)); + model->weight.set_data( + torch::arange(162, torch::dtype(torch::kFloat)).reshape({2, 3, 3, 3, 3})); + auto x = torch::arange(375, torch::dtype(torch::kFloat).requires_grad(true)) + .reshape({1, 3, 5, 5, 5}); + auto y = model(x); + auto expected = torch::tensor( + {{{{{700704., 703944., 707184.}, + {716904., 720144., 723384.}, + {733104., 736344., 739584.}}, + + {{781704., 784944., 788184.}, + {797904., 801144., 804384.}, + {814104., 817344., 820584.}}, + + {{862704., 865944., 869184.}, + {878904., 882144., 885384.}, + {895104., 898344., 901584.}}}, + + {{{1724220., 1734021., 1743822.}, + {1773225., 1783026., 1792827.}, + {1822230., 1832031., 1841832.}}, + + {{1969245., 1979046., 1988847.}, + {2018250., 2028051., 2037852.}, + {2067255., 2077056., 2086857.}}, + + {{2214270., 2224071., 2233872.}, + {2263275., 2273076., 2282877.}, + {2312280., 2322081., 2331882.}}}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, expected)); + + torch::Tensor s = y.sum(); + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_TRUE(model->weight.grad().numel() == 3 * 2 * 3 * 3 * 3); +} + +TEST_F(ModulesTest, Conv3dSameStrided) { + auto options = Conv3dOptions(3, 2, {3, 4, 5}); + options.stride(1).padding(torch::kSame); + Conv3d model_valid(options); + ASSERT_THROWS_WITH( + [&] { Conv3d model_invalid(options.stride(2)); }(), + "padding='same' is not supported for strided convolutions"); + ASSERT_THROWS_WITH( + [&] { Conv3d model_invalid(options.stride({1, 2, 1})); }(), + "padding='same' is not supported for strided convolutions"); +} + +TEST_F(ModulesTest, ConvTranspose1d) { + ConvTranspose1d model(ConvTranspose1dOptions(3, 2, 3).stride(1).bias(false)); + model->weight.set_data(torch::arange(18.).view({2, 3, 3})); + auto x = torch::arange(20.).reshape({2, 2, 5}); + auto y = model(x); + auto expected = torch::tensor( + {{{45., 104., 179., 212., 245., 188., 107.}, + {60., 140., 242., 293., 344., 260., 146.}, + {75., 176., 305., 374., 443., 332., 185.}}, + {{135., 304., 509., 542., 575., 428., 237.}, + {210., 460., 752., 803., 854., 620., 336.}, + {285., 616., 995., 1064., 1133., 812., 435.}}}); + ASSERT_TRUE(torch::allclose(y, expected)); + + torch::Tensor s = y.sum(); + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(model->weight.grad().numel(), 3 * 2 * 3); +} + +TEST_F(ModulesTest, ConvTranspose2dEven) { + ConvTranspose2d model(ConvTranspose2dOptions(3, 2, 3).stride(1).bias(false)); + model->weight.set_data(torch::arange(54.).view({2, 3, 3, 3})); + auto x = torch::arange(50.).view({1, 2, 5, 5}); + auto y = model(x); + auto expected = torch::tensor( + {{{{675., 1402., 2183., 2270., 2357., 1634., 849.}, + {1560., 3240., 5044., 5236., 5428., 3760., 1952.}, + {2685., 5574., 8673., 8988., 9303., 6438., 3339.}, + {3180., 6594., 10248., 10563., 10878., 7518., 3894.}, + {3675., 7614., 11823., 12138., 12453., 8598., 4449.}, + {2820., 5832., 9040., 9268., 9496., 6544., 3380.}, + {1605., 3314., 5129., 5252., 5375., 3698., 1907.}}, + {{900., 1870., 2912., 3053., 3194., 2210., 1146.}, + {2100., 4356., 6772., 7072., 7372., 5092., 2636.}, + {3630., 7518., 11670., 12147., 12624., 8706., 4500.}, + {4395., 9078., 14055., 14532., 15009., 10326., 5325.}, + {5160., 10638., 16440., 16917., 17394., 11946., 6150.}, + {3900., 8028., 12388., 12724., 13060., 8956., 4604.}, + {2190., 4502., 6938., 7115., 7292., 4994., 2564.}}, + {{1125., 2338., 3641., 3836., 4031., 2786., 1443.}, + {2640., 5472., 8500., 8908., 9316., 6424., 3320.}, + {4575., 9462., 14667., 15306., 15945., 10974., 5661.}, + {5610., 11562., 17862., 18501., 19140., 13134., 6756.}, + {6645., 13662., 21057., 21696., 22335., 15294., 7851.}, + {4980., 10224., 15736., 16180., 16624., 11368., 5828.}, + {2775., 5690., 8747., 8978., 9209., 6290., 3221.}}}}); + ASSERT_TRUE(torch::allclose(y, expected)); + + torch::Tensor s = y.sum(); + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(model->weight.grad().numel(), 3 * 2 * 3 * 3); +} + +TEST_F(ModulesTest, ConvTranspose2dUneven) { + ConvTranspose2d model( + ConvTranspose2dOptions(3, 2, {3, 2}).stride({1, 1}).bias(false)); + model->weight.set_data(torch::arange(36.).view({2, 3, 3, 2})); + auto x = torch::arange(40.).view({1, 2, 5, 4}); + auto y = model(x); + auto expected = torch::tensor( + {{{{360., 758., 796., 834., 440.}, + {832., 1752., 1836., 1920., 1012.}, + {1432., 3014., 3152., 3290., 1732.}, + {1696., 3566., 3704., 3842., 2020.}, + {1960., 4118., 4256., 4394., 2308.}, + {1504., 3152., 3252., 3352., 1756.}, + {856., 1790., 1844., 1898., 992.}}, + {{480., 1010., 1072., 1134., 596.}, + {1120., 2352., 2484., 2616., 1372.}, + {1936., 4058., 4268., 4478., 2344.}, + {2344., 4898., 5108., 5318., 2776.}, + {2752., 5738., 5948., 6158., 3208.}, + {2080., 4328., 4476., 4624., 2404.}, + {1168., 2426., 2504., 2582., 1340.}}, + {{600., 1262., 1348., 1434., 752.}, + {1408., 2952., 3132., 3312., 1732.}, + {2440., 5102., 5384., 5666., 2956.}, + {2992., 6230., 6512., 6794., 3532.}, + {3544., 7358., 7640., 7922., 4108.}, + {2656., 5504., 5700., 5896., 3052.}, + {1480., 3062., 3164., 3266., 1688.}}}}); + ASSERT_TRUE(torch::allclose(y, expected)); + + torch::Tensor s = y.sum(); + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(model->weight.grad().numel(), 3 * 2 * 3 * 2); +} + +TEST_F(ModulesTest, ConvTranspose3d) { + ConvTranspose3d model(ConvTranspose3dOptions(2, 2, 2).stride(1).bias(false)); + model->weight.set_data(torch::arange(32.).reshape({2, 2, 2, 2, 2})); + auto x = torch::arange(16.).reshape({1, 2, 2, 2, 2}); + auto y = model(x); + auto expected = torch::tensor( + {{{{{128., 280., 154.}, {304., 664., 364.}, {184., 400., 218.}}, + {{352., 768., 420.}, {832., 1808., 984.}, {496., 1072., 580.}}, + {{256., 552., 298.}, {592., 1272., 684.}, {344., 736., 394.}}}, + {{{192., 424., 234.}, {464., 1016., 556.}, {280., 608., 330.}}, + {{544., 1184., 644.}, {1280., 2768., 1496.}, {752., 1616., 868.}}, + {{384., 824., 442.}, {880., 1880., 1004.}, {504., 1072., 570.}}}}}); + ASSERT_TRUE(torch::allclose(y, expected)); + + torch::Tensor s = y.sum(); + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_TRUE(model->weight.grad().numel() == 2 * 2 * 2 * 2 * 2); +} + +TEST_F(ModulesTest, MaxPool1d) { + MaxPool1d model(MaxPool1dOptions(3).stride(2)); + auto x = torch::ones({1, 1, 5}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({1, 1, 2}))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 2})); +} + +TEST_F(ModulesTest, MaxPool1dReturnIndices) { + MaxPool1d model(MaxPool1dOptions(3).stride(2)); + auto x = torch::ones({1, 1, 5}, torch::requires_grad()); + auto [y, indices] = model->forward_with_indices(x); + + ASSERT_EQ(y.dim(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({1, 1, 2}))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 2})); + + ASSERT_TRUE( + torch::allclose(indices, torch::tensor({{{0, 2}}}, torch::kLong))); + ASSERT_EQ(indices.sizes(), std::vector({1, 1, 2})); +} + +TEST_F(ModulesTest, MaxPool2dEven) { + MaxPool2d model(MaxPool2dOptions(3).stride(2)); + auto x = torch::ones({2, 5, 5}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2}))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2})); +} + +TEST_F(ModulesTest, MaxPool2dUneven) { + MaxPool2d model(MaxPool2dOptions({3, 2}).stride({2, 2})); + auto x = torch::ones({2, 5, 4}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2}))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2})); +} + +TEST_F(ModulesTest, MaxPool2dReturnIndices) { + MaxPool2d model(MaxPool2dOptions(3).stride(2)); + auto x = torch::ones({2, 5, 5}, torch::requires_grad()); + auto [y, indices] = model->forward_with_indices(x); + + ASSERT_EQ(y.dim(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2}))); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2})); + ASSERT_TRUE(torch::allclose( + indices, + torch::tensor({{{0, 2}, {10, 12}}, {{0, 2}, {10, 12}}}, torch::kLong))); + ASSERT_EQ(indices.sizes(), std::vector({2, 2, 2})); +} + +TEST_F(ModulesTest, MaxPool3d) { + MaxPool3d model(MaxPool3dOptions(3).stride(2)); + auto x = torch::ones({2, 5, 5, 5}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2, 2}))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2, 2})); +} + +TEST_F(ModulesTest, MaxPool3dReturnIndices) { + MaxPool3d model(MaxPool3dOptions(3).stride(2)); + auto x = torch::ones({2, 5, 5, 5}, torch::requires_grad()); + auto [y, indices] = model->forward_with_indices(x); + + ASSERT_EQ(y.dim(), 4); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2, 2}))); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2, 2})); + + ASSERT_TRUE(torch::allclose( + indices, + torch::tensor( + {{{{0, 2}, {10, 12}}, {{50, 52}, {60, 62}}}, + {{{0, 2}, {10, 12}}, {{50, 52}, {60, 62}}}}, + torch::kLong))); + ASSERT_EQ(indices.sizes(), std::vector({2, 2, 2, 2})); +} + +TEST_F(ModulesTest, AvgPool1d) { + AvgPool1d model(AvgPool1dOptions(3).stride(2)); + auto x = torch::ones({1, 1, 5}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({1, 1, 2}))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 2})); +} + +TEST_F(ModulesTest, AvgPool2dEven) { + AvgPool2d model(AvgPool2dOptions(3).stride(2)); + auto x = torch::ones({2, 5, 5}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2}))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2})); +} + +TEST_F(ModulesTest, AvgPool2dUneven) { + AvgPool2d model(AvgPool2dOptions({3, 2}).stride({2, 2})); + auto x = torch::ones({2, 5, 4}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2}))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2})); +} + +TEST_F(ModulesTest, AvgPool3d) { + AvgPool3d model(AvgPool3dOptions(3).stride(2)); + auto x = torch::ones({2, 5, 5, 5}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2, 2}))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2, 2})); +} + +TEST_F(ModulesTest, FractionalMaxPool2d) { + FractionalMaxPool2d model(FractionalMaxPool2dOptions(3).output_size(2)); + auto x = torch::ones({2, 5, 5}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2}))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2})); +} + +TEST_F(ModulesTest, FractionalMaxPool2dReturnIndices) { + FractionalMaxPool2d model(FractionalMaxPool2dOptions(3).output_size(2)); + auto x = torch::ones({2, 5, 5}, torch::requires_grad()); + auto [y, indices] = model->forward_with_indices(x); + + ASSERT_EQ(y.dim(), 3); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2}))); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2})); + ASSERT_TRUE(torch::allclose( + indices, torch::tensor({{{0, 2}, {10, 12}}, {{0, 2}, {10, 12}}}))); + ASSERT_EQ(indices.sizes(), std::vector({2, 2, 2})); +} + +TEST_F(ModulesTest, FractionalMaxPool3d) { + FractionalMaxPool3d model(FractionalMaxPool3dOptions(3).output_size(2)); + auto x = torch::ones({2, 5, 5, 5}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2, 2}))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2, 2})); +} + +TEST_F(ModulesTest, FractionalMaxPool3dReturnIndices) { + FractionalMaxPool3d model(FractionalMaxPool3dOptions(3).output_size(2)); + auto x = torch::ones({2, 5, 5, 5}, torch::requires_grad()); + auto [y, indices] = model->forward_with_indices(x); + + ASSERT_EQ(y.dim(), 4); + ASSERT_TRUE(torch::allclose(y, torch::ones({2, 2, 2, 2}))); + ASSERT_EQ(y.sizes(), std::vector({2, 2, 2, 2})); + + ASSERT_TRUE(torch::allclose( + indices, + torch::tensor( + {{{{0, 2}, {10, 12}}, {{50, 52}, {60, 62}}}, + {{{0, 2}, {10, 12}}, {{50, 52}, {60, 62}}}}))); + ASSERT_EQ(indices.sizes(), std::vector({2, 2, 2, 2})); +} + +TEST_F(ModulesTest, LPPool1d) { + int norm_type = 2; + int stride = 2; + int kernel_size = 3; + + LPPool1d model(LPPool1dOptions(norm_type, kernel_size).stride(stride)); + auto x = torch::ones({1, 1, 5}); + auto y = model(x); + auto expected = + (torch::pow(torch::tensor({{{1, 1}}}, torch::kFloat), norm_type) * + kernel_size) + .pow(1. / norm_type); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, expected)); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({1, 1, 2})); +} + +TEST_F(ModulesTest, LPPool2d) { + int norm_type = 2; + int stride = 2; + std::vector kernel_size({2, 3}); + + LPPool2d model(LPPool2dOptions(norm_type, kernel_size).stride(stride)); + auto x = torch::ones({1, 1, 2, 5}); + auto y = model(x); + auto expected = + (torch::pow(torch::tensor({{{{1, 1}}}}, torch::kFloat), norm_type) * + (kernel_size[0] * kernel_size[1])) + .pow(1. / norm_type); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose(y, expected)); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({1, 1, 1, 2})); +} + +TEST_F(ModulesTest, LPPool3d) { + int norm_type = 2; + int stride = 2; + std::vector kernel_size({1, 2, 3}); + + LPPool3d model(LPPool3dOptions(norm_type, kernel_size).stride(stride)); + auto x = torch::ones({1, 1, 1, 2, 5}); + auto y = model(x); + auto expected = + (torch::pow(torch::tensor({{{{{1, 1}}}}}, torch::kFloat), norm_type) * + (kernel_size[0] * kernel_size[1] * kernel_size[2])) + .pow(1. / norm_type); + + ASSERT_EQ(y.ndimension(), 5); + ASSERT_TRUE(torch::allclose(y, expected)); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({1, 1, 1, 1, 2})); +} + +TEST_F(ModulesTest, Identity) { + Identity identity; + auto input = torch::tensor( + {{1, 3, 4}, {2, 3, 4}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto output = identity->forward(input); + auto expected = torch::tensor({{1, 3, 4}, {2, 3, 4}}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(torch::equal(output, expected)); + ASSERT_TRUE(torch::equal(input.grad(), torch::ones_like(input))); +} + +TEST_F(ModulesTest, Flatten) { + Flatten flatten; + auto input = torch::tensor( + {{1, 3, 4}, {2, 5, 6}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto output = flatten->forward(input); + auto expected = torch::tensor({{1, 3, 4}, {2, 5, 6}}, torch::kFloat); + auto s = output.sum(); + + s.backward(); + ASSERT_TRUE(torch::equal(output, expected)); + ASSERT_TRUE(torch::equal(input.grad(), torch::ones_like(input))); + + // Testing with optional arguments start_dim and end_dim + Flatten flatten_optional_dims(FlattenOptions().start_dim(2).end_dim(3)); + input = torch::tensor( + {{{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}}, + {{{9, 10}, {11, 12}}, {{13, 14}, {15, 16}}}}, + torch::dtype(torch::kFloat) + .requires_grad(true)); // Tensor with sizes (2, 2, 2, 2) + + output = flatten_optional_dims->forward(input); + expected = torch::tensor( + {{{1, 2, 3, 4}, {5, 6, 7, 8}}, {{9, 10, 11, 12}, {13, 14, 15, 16}}}, + torch::kFloat); // Tensor with sizes (2, 2, 4) + + s = output.sum(); + s.backward(); + ASSERT_TRUE(torch::equal(output, expected)); + ASSERT_TRUE(torch::equal(input.grad(), torch::ones_like(input))); +} + +TEST_F(ModulesTest, Unflatten) { + Unflatten unflatten(UnflattenOptions(0, {2, 2})); + auto output = unflatten->forward(torch::tensor({1, 2, 3, 4})); + auto expected = torch::tensor({{1, 2}, {3, 4}}); + ASSERT_TRUE(torch::equal(output, expected)); +} + +TEST_F(ModulesTest, AdaptiveMaxPool1d) { + AdaptiveMaxPool1d model(3); + auto x = torch::tensor( + {{{1, 2, 3, 4, 5}}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose(y, torch::tensor({{{2, 4, 5}}}, torch::kFloat))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 3})); +} + +TEST_F(ModulesTest, AdaptiveMaxPool1dReturnIndices) { + AdaptiveMaxPool1d model(3); + auto x = torch::tensor( + {{{1, 2, 3, 4, 5}}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto [y, indices] = model->forward_with_indices(x); + + ASSERT_EQ(y.dim(), 3); + ASSERT_TRUE(torch::allclose(y, torch::tensor({{{2, 4, 5}}}, torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 3})); + ASSERT_TRUE( + torch::allclose(indices, torch::tensor({{{1, 3, 4}}}, torch::kLong))); + ASSERT_EQ(indices.sizes(), std::vector({1, 1, 3})); +} + +TEST_F(ModulesTest, AdaptiveMaxPool2dEven) { + AdaptiveMaxPool2d model(3); + auto x = torch::arange(0., 50); + x.resize_({2, 5, 5}).set_requires_grad(true); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + { + {{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}, + {{31, 33, 34}, {41, 43, 44}, {46, 48, 49}}, + }, + torch::kFloat))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({2, 3, 3})); +} + +TEST_F(ModulesTest, AdaptiveMaxPool2dUneven) { + AdaptiveMaxPool2d model(AdaptiveMaxPool2dOptions({3, 2})); + auto x = torch::arange(0., 40); + x.resize_({2, 5, 4}).set_requires_grad(true); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + { + {{5, 7}, {13, 15}, {17, 19}}, + {{25, 27}, {33, 35}, {37, 39}}, + }, + torch::kFloat))); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), std::vector({2, 3, 2})); +} + +TEST_F(ModulesTest, AdaptiveMaxPool2dReturnIndicesEven) { + AdaptiveMaxPool2d model(3); + auto x = torch::arange(0., 50); + x.resize_({2, 5, 5}).set_requires_grad(true); + auto [y, indices] = model->forward_with_indices(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + { + {{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}, + {{31, 33, 34}, {41, 43, 44}, {46, 48, 49}}, + }, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({2, 3, 3})); + + ASSERT_EQ(indices.ndimension(), 3); + ASSERT_TRUE(torch::allclose( + indices, + torch::tensor( + { + {{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}, + {{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}, + }, + torch::kLong))); + ASSERT_EQ(indices.sizes(), std::vector({2, 3, 3})); +} + +TEST_F(ModulesTest, AdaptiveMaxPool2dReturnIndicesUneven) { + AdaptiveMaxPool2d model(AdaptiveMaxPool2dOptions({3, 2})); + auto x = torch::arange(0., 40); + x.resize_({2, 5, 4}).set_requires_grad(true); + auto [y, indices] = model->forward_with_indices(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + { + {{5, 7}, {13, 15}, {17, 19}}, + {{25, 27}, {33, 35}, {37, 39}}, + }, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({2, 3, 2})); + + ASSERT_EQ(indices.ndimension(), 3); + ASSERT_TRUE(torch::allclose( + indices, + torch::tensor( + { + {{5, 7}, {13, 15}, {17, 19}}, + {{5, 7}, {13, 15}, {17, 19}}, + }, + torch::kLong))); + ASSERT_EQ(indices.sizes(), std::vector({2, 3, 2})); +} + +TEST_F(ModulesTest, AdaptiveMaxPool3d) { + AdaptiveMaxPool3d model(3); + auto x = torch::arange(0., 64); + x.resize_({1, 4, 4, 4}).set_requires_grad(true); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + { + {{21, 22, 23}, {25, 26, 27}, {29, 30, 31}}, + {{37, 38, 39}, {41, 42, 43}, {45, 46, 47}}, + {{53, 54, 55}, {57, 58, 59}, {61, 62, 63}}, + }, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 3, 3, 3})); +} + +TEST_F(ModulesTest, AdaptiveMaxPool3dReturnIndices) { + AdaptiveMaxPool3d model(3); + auto x = torch::arange(0., 64); + x.resize_({1, 4, 4, 4}).set_requires_grad(true); + auto [y, indices] = model->forward_with_indices(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + { + {{21, 22, 23}, {25, 26, 27}, {29, 30, 31}}, + {{37, 38, 39}, {41, 42, 43}, {45, 46, 47}}, + {{53, 54, 55}, {57, 58, 59}, {61, 62, 63}}, + }, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 3, 3, 3})); + + ASSERT_EQ(indices.ndimension(), 4); + ASSERT_TRUE(torch::allclose( + indices, + torch::tensor( + { + {{21, 22, 23}, {25, 26, 27}, {29, 30, 31}}, + {{37, 38, 39}, {41, 42, 43}, {45, 46, 47}}, + {{53, 54, 55}, {57, 58, 59}, {61, 62, 63}}, + }, + torch::kLong))); + ASSERT_EQ(indices.sizes(), std::vector({1, 3, 3, 3})); +} + +TEST_F(ModulesTest, AdaptiveAvgPool1d) { + AdaptiveAvgPool1d model(3); + auto x = torch::tensor( + {{{1, 2, 3, 4, 5}}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE( + torch::allclose(y, torch::tensor({{{1.5, 3.0, 4.5}}}, torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 3})); +} + +TEST_F(ModulesTest, AdaptiveAvgPool2dEven) { + AdaptiveAvgPool2d model(3); + auto x = torch::arange(0., 50); + x.resize_({2, 5, 5}).set_requires_grad(true); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + { + {{3.0, 4.5, 6.0}, {10.5, 12.0, 13.5}, {18.0, 19.5, 21.0}}, + {{28.0, 29.5, 31.0}, {35.5, 37.0, 38.5}, {43.0, 44.5, 46.0}}, + }, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({2, 3, 3})); +} + +TEST_F(ModulesTest, AdaptiveAvgPool2dUneven) { + AdaptiveAvgPool2d model(AdaptiveAvgPool2dOptions({3, 2})); + auto x = torch::arange(0., 40); + x.resize_({2, 5, 4}).set_requires_grad(true); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + { + {{2.5, 4.5}, {8.5, 10.5}, {14.5, 16.5}}, + {{22.5, 24.5}, {28.5, 30.5}, {34.5, 36.5}}, + }, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({2, 3, 2})); +} + +TEST_F(ModulesTest, AdaptiveAvgPool3d) { + AdaptiveAvgPool3d model(3); + auto x = torch::arange(0., 64); + x.resize_({1, 4, 4, 4}).set_requires_grad(true); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + { + {{10.5, 11.5, 12.5}, {14.5, 15.5, 16.5}, {18.5, 19.5, 20.5}}, + {{26.5, 27.5, 28.5}, {30.5, 31.5, 32.5}, {34.5, 35.5, 36.5}}, + {{42.5, 43.5, 44.5}, {46.5, 47.5, 48.5}, {50.5, 51.5, 52.5}}, + }, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 3, 3, 3})); +} + +TEST_F(ModulesTest, MaxUnpool1d) { + auto indices = torch::tensor({{{1, 3, 4}}}, torch::kLong); + auto x = torch::tensor( + {{{2, 4, 5}}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto model = MaxUnpool1d{3}; + auto y = model->forward(x, indices); + + ASSERT_EQ(y.dim(), 3); + ASSERT_TRUE(torch::allclose( + y, torch::tensor({{{0, 2, 0, 4, 5, 0, 0, 0, 0}}}, torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 9})); + + indices = torch::tensor({{{1, 3, 4}}}, torch::kLong); + x = torch::tensor( + {{{2, 4, 5}}}, torch::dtype(torch::kFloat).requires_grad(true)); + model = MaxUnpool1d{MaxUnpool1dOptions(3).stride(2).padding(1)}; + y = model->forward(x, indices, std::vector({1, 1, 5})); + + ASSERT_EQ(y.dim(), 3); + ASSERT_TRUE( + torch::allclose(y, torch::tensor({{{0, 2, 0, 4, 5}}}, torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 5})); +} + +TEST_F(ModulesTest, MaxPool1d_MaxUnpool1d) { + MaxPool1d pool{MaxPool1dOptions(2).stride(2)}; + MaxUnpool1d unpool{MaxUnpool1dOptions(2).stride(2)}; + auto input = torch::tensor({{{1, 2, 3, 4, 5, 6, 7, 8}}}, torch::kFloat); + auto [output, indices] = pool->forward_with_indices(input); + ASSERT_TRUE(torch::allclose( + unpool(output, indices), + torch::tensor({{{0, 2, 0, 4, 0, 6, 0, 8}}}, torch::kFloat))); + + // Example showcasing the use of output_size + input = torch::tensor({{{1, 2, 3, 4, 5, 6, 7, 8, 9}}}, torch::kFloat); + std::tie(output, indices) = pool->forward_with_indices(input); + ASSERT_TRUE(torch::allclose( + unpool(output, indices, input.sizes().vec()), + torch::tensor({{{0, 2, 0, 4, 0, 6, 0, 8, 0}}}, torch::kFloat))); + ASSERT_TRUE(torch::allclose( + unpool(output, indices), + torch::tensor({{{0, 2, 0, 4, 0, 6, 0, 8}}}, torch::kFloat))); +} + +TEST_F(ModulesTest, MaxUnpool2d) { + auto indices = torch::tensor( + {{{{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}}, + {{{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}}}, + torch::kLong); + auto x = torch::tensor( + {{{{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}}, + {{{31, 33, 34}, {41, 43, 44}, {46, 48, 49}}}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto model = MaxUnpool2d{MaxUnpool2dOptions(3).stride(2).padding(1)}; + auto y = model->forward(x, indices); + + ASSERT_EQ(y.dim(), 4); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + {{{{0, 0, 0, 0, 0}, + {0, 6, 0, 8, 9}, + {0, 0, 0, 0, 0}, + {0, 16, 0, 18, 19}, + {0, 21, 0, 23, 24}}}, + {{{0, 0, 0, 0, 0}, + {0, 31, 0, 33, 34}, + {0, 0, 0, 0, 0}, + {0, 41, 0, 43, 44}, + {0, 46, 0, 48, 49}}}}, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({2, 1, 5, 5})); +} + +TEST_F(ModulesTest, MaxPool2d_MaxUnpool2d) { + MaxPool2d pool{MaxPool2dOptions(2).stride(2)}; + MaxUnpool2d unpool{MaxUnpool2dOptions(2).stride(2)}; + auto input = torch::tensor( + {{{{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}, {13, 14, 15, 16}}}}, + torch::kFloat); + auto [output, indices] = pool->forward_with_indices(input); + ASSERT_TRUE(torch::allclose( + unpool(output, indices), + torch::tensor( + {{{{0, 0, 0, 0}, {0, 6, 0, 8}, {0, 0, 0, 0}, {0, 14, 0, 16}}}}, + torch::kFloat))); + + ASSERT_TRUE(torch::allclose( + unpool(output, indices, std::vector{1, 1, 5, 5}), + torch::tensor( + {{{{0, 0, 0, 0, 0}, + {6, 0, 8, 0, 0}, + {0, 0, 0, 14, 0}, + {16, 0, 0, 0, 0}, + {0, 0, 0, 0, 0}}}}, + torch::kFloat))); +} + +TEST_F(ModulesTest, MaxUnpool3d) { + auto indices = torch::tensor({{{{{26}}}}}, torch::kLong); + auto x = torch::tensor( + {{{{{26}}}}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto model = MaxUnpool3d{3}; + auto y = model->forward(x, indices); + + ASSERT_EQ(y.dim(), 5); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + {{{{{0, 0, 0}, {0, 0, 0}, {0, 0, 0}}, + {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}}, + {{0, 0, 0}, {0, 0, 0}, {0, 0, 26}}}}}, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 3, 3, 3})); +} + +TEST_F(ModulesTest, MaxUnpool3dOutputSize) { + auto indices = torch::tensor( + {{{{{21, 23}, {29, 31}}, {{53, 55}, {61, 63}}}}}, torch::kLong); + auto x = torch::tensor( + {{{{{21, 23}, {29, 31}}, {{53, 55}, {61, 63}}}}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto model = MaxUnpool3d{MaxUnpool3dOptions(3).stride(2).padding(1)}; + auto y = model->forward(x, indices, std::vector({1, 1, 4, 4, 4})); + + ASSERT_EQ(y.dim(), 5); + ASSERT_TRUE(torch::allclose( + y, + torch::tensor( + {{{{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, + {{0, 0, 0, 0}, {0, 21, 0, 23}, {0, 0, 0, 0}, {0, 29, 0, 31}}, + {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, + {{0, 0, 0, 0}, {0, 53, 0, 55}, {0, 0, 0, 0}, {0, 61, 0, 63}}}}}, + torch::kFloat))); + ASSERT_EQ(y.sizes(), std::vector({1, 1, 4, 4, 4})); +} + +TEST_F(ModulesTest, MaxPool3d_MaxUnpool3d) { + MaxPool3d pool{MaxPool3dOptions(3).stride(2)}; + MaxUnpool3d unpool{MaxUnpool3dOptions(3).stride(2)}; + auto input = torch::randn({20, 16, 51, 33, 15}); + auto [output, indices] = pool->forward_with_indices(input); + auto unpooled_output = unpool(output, indices); + ASSERT_EQ( + unpooled_output.sizes(), std::vector({20, 16, 51, 33, 15})); +} + +TEST_F(ModulesTest, Linear) { + { + Linear model(5, 2); + auto x = torch::randn({10, 5}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 2); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.size(0), 10); + ASSERT_EQ(y.size(1), 2); + + ASSERT_EQ(model->weight.grad().numel(), 2 * 5); + + auto y_exp = torch::addmm(model->bias, x, model->weight.t()); + ASSERT_TRUE(torch::allclose(y, y_exp)); + } + { + Linear model(LinearOptions(5, 2).bias(false)); + auto x = torch::randn({10, 5}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 2); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.size(0), 10); + ASSERT_EQ(y.size(1), 2); + + ASSERT_EQ(model->weight.grad().numel(), 2 * 5); + + auto y_exp = torch::mm(x, model->weight.t()); + ASSERT_TRUE(torch::allclose(y, y_exp)); + } +} + +TEST_F(ModulesTest, LocalResponseNorm) { + { + LocalResponseNorm model(LocalResponseNormOptions(2)); + const auto x = + torch::arange(100., 136, torch::requires_grad()).reshape({2, 3, 3, 2}); + auto y = model(x); + const auto y_exp = torch::tensor( + {{{{73.7788, 74.1462}, {74.5031, 74.8572}, {75.2010, 75.5420}}, + + {{61.6057, 61.7227}, {61.8347, 61.9418}, {62.0441, 62.1418}}, + + {{62.2349, 62.3235}, {62.4077, 62.4877}, {62.5635, 62.6353}}}, + + {{{79.3915, 79.6491}, {79.8978, 80.1446}, {80.3827, 80.6190}}, + + {{63.0317, 63.0742}, {63.1135, 63.1496}, {63.1826, 63.2126}}, + + {{63.2396, 63.2637}, {63.2850, 63.3036}, {63.3195, 63.3328}}}}, + torch::kFloat); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 4); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.sizes(), x.sizes()); + ASSERT_TRUE(torch::allclose(y, y_exp, 1e-4, 1e-7)); + } +} + +TEST_F(ModulesTest, LayerNorm) { + LayerNorm model(LayerNormOptions({2, 2}).eps(2e-5)); + auto x = torch::randn({2, 2}, torch::requires_grad()); + auto y = model(x); + auto y_exp = torch::layer_norm(x, {2, 2}, model->weight, model->bias, 2e-5); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 2); + ASSERT_EQ(s.ndimension(), 0); + for (const auto i : c10::irange(2)) { + ASSERT_EQ(y.size(i), 2); + } + + ASSERT_EQ(model->weight.grad().numel(), 2 * 2); + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(ModulesTest, GroupNorm) { + GroupNorm model(GroupNormOptions(2, 2).eps(2e-5)); + auto x = torch::randn({2, 2}, torch::requires_grad()); + auto y = model(x); + auto y_exp = torch::group_norm(x, 2, model->weight, model->bias, 2e-5); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 2); + ASSERT_EQ(s.ndimension(), 0); + for (const auto i : c10::irange(2)) { + ASSERT_EQ(y.size(i), 2); + } + + ASSERT_EQ(model->weight.grad().numel(), 2); + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(ModulesTest, Bilinear) { + Bilinear model(5, 3, 2); + auto x1 = torch::randn({10, 5}, torch::requires_grad()); + auto x2 = torch::randn({10, 3}, torch::requires_grad()); + auto y = model(x1, x2); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 2); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.size(0), 10); + ASSERT_EQ(y.size(1), 2); + + ASSERT_EQ(model->weight.grad().numel(), 2 * 5 * 3); +} + +TEST_F(ModulesTest, Fold) { + { + Fold model(FoldOptions({3, 2}, {2, 2})); + auto input = torch::ones({1, 3 * 2 * 2, 2}, torch::requires_grad()); + auto output = model(input); + auto expected = torch::tensor( + {{{{1.0, 1.0}, {2.0, 2.0}, {1.0, 1.0}}, + {{1.0, 1.0}, {2.0, 2.0}, {1.0, 1.0}}, + {{1.0, 1.0}, {2.0, 2.0}, {1.0, 1.0}}}}, + torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(output.sizes(), std::vector({1, 3, 3, 2})); + ASSERT_TRUE(output.allclose(expected)); + } + { + // input wrong dimension + Fold model(FoldOptions({8, 8}, {3, 3})); + ASSERT_THROWS_WITH( + model(torch::randn({1, 3, 16, 16})), + "Input Error: Only unbatched (2D) or batched (3D) input Tensors are supported (got 4D)"); + } +} + +TEST_F(ModulesTest, Unfold) { + { + Unfold model(UnfoldOptions({2, 2}).padding(1).stride(2)); + auto input = + torch::arange(2., 14, torch::requires_grad()).view({1, 2, 2, 3}); + auto output = model(input); + auto expected = torch::tensor( + {{{0.0, 0.0, 0.0, 6.0}, + {0.0, 0.0, 5.0, 7.0}, + {0.0, 3.0, 0.0, 0.0}, + {2.0, 4.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 12.0}, + {0.0, 0.0, 11.0, 13.0}, + {0.0, 9.0, 0.0, 0.0}, + {8.0, 10.0, 0.0, 0.0}}}, + torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(output.sizes(), std::vector({1, 8, 4})); + ASSERT_TRUE(output.allclose(expected)); + } + { + // input wrong dimension + Unfold model(UnfoldOptions({2, 4})); + ASSERT_THROWS_WITH( + model(torch::randn({1, 5, 2})), + "Input Error: Only 4D input Tensors are supported (got 3D)"); + } + { + // calculated output shape is too small + Unfold model(UnfoldOptions({2, 3})); + ASSERT_THROWS_WITH( + model(torch::randn({1, 2, 2, 2})), + "Given input with spatial size (2, 2), kernel_size=(2, 3), " + "dilation=(1, 1), padding=(0, 0), calculated shape of the array of " + "sliding blocks as (1, 0), but its components must be at least one."); + } +} + +TEST_F(ModulesTest, SimpleContainer) { + auto model = std::make_shared(); + auto l1 = model->add(Linear(10, 3), "l1"); + auto l2 = model->add(Linear(3, 5), "l2"); + auto l3 = model->add(Linear(5, 100), "l3"); + + auto x = torch::randn({1000, 10}, torch::requires_grad()); + x = l1(x).clamp_min(0); + x = l2(x).clamp_min(0); + x = l3(x).clamp_min(0); + + x.backward(torch::ones_like(x)); + ASSERT_EQ(x.ndimension(), 2); + ASSERT_EQ(x.size(0), 1000); + ASSERT_EQ(x.size(1), 100); + ASSERT_EQ(x.min().item(), 0); +} + +TEST_F(ModulesTest, EmbeddingBasic) { + const int64_t dict_size = 10; + Embedding model(dict_size, 2); + ASSERT_TRUE(model->named_parameters().contains("weight")); + ASSERT_EQ(model->weight.ndimension(), 2); + ASSERT_EQ(model->weight.size(0), dict_size); + ASSERT_EQ(model->weight.size(1), 2); + + // Cannot get gradients to change indices (input) - only for embedding + // params + auto x = torch::full({10}, dict_size - 1, torch::kInt64); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 2); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.size(0), 10); + ASSERT_EQ(y.size(1), 2); + + ASSERT_EQ(model->weight.grad().numel(), 2 * dict_size); +} + +TEST_F(ModulesTest, EmbeddingList) { + Embedding model(6, 4); + auto x = torch::full({2, 3}, 5, torch::kInt64); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.size(0), 2); + ASSERT_EQ(y.size(1), 3); + ASSERT_EQ(y.size(2), 4); +} + +TEST_F(ModulesTest, EmbeddingFromPretrained) { + auto weight = torch::tensor({{1., 2.3, 3.}, {4., 5.1, 6.3}}); + Embedding embedding = torch::nn::Embedding::from_pretrained(weight); + auto input = torch::tensor({1}, torch::kLong); + ASSERT_TRUE(torch::allclose( + embedding(input), torch::tensor({4.0000, 5.1000, 6.3000}))); +} + +TEST_F(ModulesTest, EmbeddingBagFromPretrained) { + auto weight = torch::tensor({{1., 2.3, 3.}, {4., 5.1, 6.3}}); + EmbeddingBag embeddingbag = torch::nn::EmbeddingBag::from_pretrained(weight); + auto input = torch::zeros({{1, 2}}, torch::kLong); + input[0] = torch::tensor({1, 0}); + ASSERT_TRUE(torch::allclose( + embeddingbag(input), torch::tensor({2.5000, 3.7000, 4.6500}))); +} + +TEST_F(ModulesTest, AlphaDropout) { + AlphaDropout alpha_dropout(0.5); + torch::Tensor x = torch::ones(100, torch::requires_grad()); + torch::Tensor y = alpha_dropout(x); + + y.backward(torch::ones_like(y)); + + ASSERT_EQ(y.ndimension(), 1); + ASSERT_EQ(y.size(0), 100); + ASSERT_LT(y.sum().item(), 130); // Probably + ASSERT_GT(y.sum().item(), 40); // Probably + + alpha_dropout->eval(); + y = alpha_dropout(x); + + ASSERT_EQ(y.sum().item(), 100); +} + +TEST_F(ModulesTest, FeatureAlphaDropout) { + FeatureAlphaDropout feature_alpha_dropout(0.5); + torch::Tensor x = torch::ones({10, 10}, torch::requires_grad()); + torch::Tensor y = feature_alpha_dropout(x); + + y.backward(torch::ones_like(y)); + + ASSERT_EQ(y.ndimension(), 2); + ASSERT_EQ(y.size(0), 10); + ASSERT_EQ(y.size(1), 10); + ASSERT_LT(y.sum().item(), 130); // Probably + ASSERT_GT(y.sum().item(), 40); // Probably + + feature_alpha_dropout->eval(); + y = feature_alpha_dropout(x); + + ASSERT_EQ(y.sum().item(), 100); +} + +TEST_F(ModulesTest, Dropout) { + for (const auto inplace : {false, true}) { + Dropout dropout(DropoutOptions(0.5).inplace(inplace)); + torch::Tensor x = torch::ones(100); + if (!inplace) { + x.requires_grad_(true); + } + torch::Tensor y = dropout(x); + + ASSERT_EQ(y.ndimension(), 1); + ASSERT_EQ(y.size(0), 100); + ASSERT_LT(y.sum().item(), 130); // Probably + ASSERT_GT(y.sum().item(), 70); // Probably + if (inplace) { + ASSERT_TRUE(y.allclose(x)); + } else { + y.backward(torch::ones_like(y)); + } + + dropout->eval(); + y = dropout(torch::ones(100)); + ASSERT_EQ(y.sum().item(), 100); + } +} + +TEST_F(ModulesTest, Dropout2d) { + auto p = 0.5; + for (const auto inplace : {false, true}) { + Dropout2d dropout(Dropout2dOptions(p).inplace(inplace)); + torch::Tensor x = torch::empty({50, 50, 2, 2}).fill_(1 - p); + if (!inplace) { + x.requires_grad_(true); + } + torch::Tensor y = dropout(x); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_EQ(y.size(0), 50); + ASSERT_EQ(y.size(1), 50); + ASSERT_EQ(y.size(2), 2); + ASSERT_EQ(y.size(3), 2); + ASSERT_LT((y.mean() - (1 - p)).abs().item(), 0.05); + + if (inplace) { + ASSERT_TRUE(y.allclose(x)); + } else { + y.backward(torch::ones_like(y)); + } + + dropout->eval(); + y = dropout(torch::ones({2, 2, 10, 10})); + ASSERT_EQ(y.sum().item(), 400); + } +} + +TEST_F(ModulesTest, Dropout3d) { + for (const auto inplace : {false, true}) { + auto p = 0.5; + Dropout3d dropout(Dropout3dOptions(p).inplace(inplace)); + torch::Tensor x = torch::empty({50, 50, 2, 2, 2}).fill_(1 - p); + if (!inplace) { + x.requires_grad_(true); + } + torch::Tensor y = dropout(x); + + ASSERT_EQ(y.ndimension(), 5); + ASSERT_EQ(y.size(0), 50); + ASSERT_EQ(y.size(1), 50); + ASSERT_EQ(y.size(2), 2); + ASSERT_EQ(y.size(3), 2); + ASSERT_EQ(y.size(4), 2); + ASSERT_LT((y.mean() - (1 - p)).abs().item(), 0.05); + + if (inplace) { + ASSERT_TRUE(y.allclose(x)); + } else { + y.backward(torch::ones_like(y)); + } + + dropout->eval(); + y = dropout(torch::ones({4, 4, 5, 5})); + ASSERT_EQ(y.sum().item(), 400); + } +} + +TEST_F(ModulesTest, Parameters) { + auto model = std::make_shared(); + auto parameters = model->named_parameters(); + ASSERT_EQ(parameters["param"].size(0), 3); + ASSERT_EQ(parameters["param"].size(1), 2); + ASSERT_EQ(parameters["param"].size(2), 21); + ASSERT_EQ(parameters["l1.bias"].size(0), 20); + ASSERT_EQ(parameters["l1.weight"].size(0), 20); + ASSERT_EQ(parameters["l1.weight"].size(1), 5); + ASSERT_EQ(parameters["test.l1.bias"].size(0), 3); + ASSERT_EQ(parameters["test.l1.weight"].size(0), 3); + ASSERT_EQ(parameters["test.l1.weight"].size(1), 10); + ASSERT_EQ(parameters["test.l2.bias"].size(0), 5); + ASSERT_EQ(parameters["test.l2.weight"].size(0), 5); + ASSERT_EQ(parameters["test.l2.weight"].size(1), 3); + ASSERT_EQ(parameters["test.l3.bias"].size(0), 100); + ASSERT_EQ(parameters["test.l3.weight"].size(0), 100); + ASSERT_EQ(parameters["test.l3.weight"].size(1), 5); +} + +TEST_F(ModulesTest, FunctionalCallsSuppliedFunction) { + bool was_called = false; + auto functional = Functional([&was_called](torch::Tensor input) { + was_called = true; + return input; + }); + auto output = functional(torch::ones(5, torch::requires_grad())); + ASSERT_TRUE(was_called); + ASSERT_TRUE(output.equal(torch::ones(5, torch::requires_grad()))); + + was_called = false; + // Use the call operator overload here. + output = functional(torch::ones(5, torch::requires_grad())); + ASSERT_TRUE(was_called); + ASSERT_TRUE(output.equal(torch::ones(5, torch::requires_grad()))); +} + +TEST_F(ModulesTest, FunctionalWithTorchFunction) { + auto functional = Functional(torch::relu); + ASSERT_EQ(functional(torch::ones({})).item(), 1); + ASSERT_EQ(functional(torch::ones({})).item(), 1); + ASSERT_EQ(functional(torch::ones({}) * -1).item(), 0); +} + +TEST_F(ModulesTest, FunctionalArgumentBinding) { + auto functional = + Functional(torch::elu, /*alpha=*/1, /*scale=*/0, /*input_scale=*/1); + ASSERT_EQ(functional(torch::ones({})).item(), 0); +} + +TEST_F(ModulesTest, BatchNorm1dStateful) { + BatchNorm1d bn(5); + + ASSERT_TRUE(bn->options.track_running_stats()); + + ASSERT_TRUE(bn->running_mean.defined()); + ASSERT_EQ(bn->running_mean.dim(), 1); + ASSERT_EQ(bn->running_mean.size(0), 5); + + ASSERT_TRUE(bn->running_var.defined()); + ASSERT_EQ(bn->running_var.dim(), 1); + ASSERT_EQ(bn->running_var.size(0), 5); + + ASSERT_TRUE(bn->num_batches_tracked.defined()); + ASSERT_EQ(bn->num_batches_tracked.dim(), 0); + + ASSERT_TRUE(bn->options.affine()); + + ASSERT_TRUE(bn->weight.defined()); + ASSERT_EQ(bn->weight.dim(), 1); + ASSERT_EQ(bn->weight.size(0), 5); + + ASSERT_TRUE(bn->bias.defined()); + ASSERT_EQ(bn->bias.dim(), 1); + ASSERT_EQ(bn->bias.size(0), 5); +} + +TEST_F(ModulesTest, BatchNorm1dStateless) { + BatchNorm1d bn( + BatchNorm1dOptions(5).track_running_stats(false).affine(false)); + + ASSERT_FALSE(bn->running_mean.defined()); + ASSERT_FALSE(bn->running_var.defined()); + ASSERT_FALSE(bn->num_batches_tracked.defined()); + ASSERT_FALSE(bn->weight.defined()); + ASSERT_FALSE(bn->bias.defined()); +} + +TEST_F(ModulesTest, BatchNorm1d) { + BatchNorm1d bn(5); + bn->eval(); + + auto input = torch::arange(2. * 5 * 2).view({2, 5, 2}).requires_grad_(); + auto output = bn->forward(input); + auto expected = torch::tensor( + {{{0.0000, 1.0000}, + {2.0000, 3.0000}, + {4.0000, 5.0000}, + {6.0000, 7.0000}, + {8.0000, 9.0000}}, + {{10.0000, 10.9999}, + {11.9999, 12.9999}, + {13.9999, 14.9999}, + {15.9999, 16.9999}, + {17.9999, 18.9999}}}); + ASSERT_TRUE(output.allclose(expected)); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, BatchNorm2dStateful) { + BatchNorm2d bn(5); + + ASSERT_TRUE(bn->options.track_running_stats()); + + ASSERT_TRUE(bn->running_mean.defined()); + ASSERT_EQ(bn->running_mean.dim(), 1); + ASSERT_EQ(bn->running_mean.size(0), 5); + + ASSERT_TRUE(bn->running_var.defined()); + ASSERT_EQ(bn->running_var.dim(), 1); + ASSERT_EQ(bn->running_var.size(0), 5); + + ASSERT_TRUE(bn->num_batches_tracked.defined()); + ASSERT_EQ(bn->num_batches_tracked.dim(), 0); + + ASSERT_TRUE(bn->options.affine()); + + ASSERT_TRUE(bn->weight.defined()); + ASSERT_EQ(bn->weight.dim(), 1); + ASSERT_EQ(bn->weight.size(0), 5); + + ASSERT_TRUE(bn->bias.defined()); + ASSERT_EQ(bn->bias.dim(), 1); + ASSERT_EQ(bn->bias.size(0), 5); +} + +TEST_F(ModulesTest, BatchNorm2dStateless) { + BatchNorm2d bn( + BatchNorm2dOptions(5).track_running_stats(false).affine(false)); + + ASSERT_FALSE(bn->running_mean.defined()); + ASSERT_FALSE(bn->running_var.defined()); + ASSERT_FALSE(bn->num_batches_tracked.defined()); + ASSERT_FALSE(bn->weight.defined()); + ASSERT_FALSE(bn->bias.defined()); +} + +TEST_F(ModulesTest, BatchNorm2d) { + BatchNorm2d bn(5); + bn->eval(); + + auto input = + torch::arange(2. * 5 * 2 * 2).view({2, 5, 2, 2}).requires_grad_(); + auto output = bn->forward(input); + auto expected = torch::tensor( + {{{{0.0000, 1.0000}, {2.0000, 3.0000}}, + {{4.0000, 5.0000}, {6.0000, 7.0000}}, + {{8.0000, 9.0000}, {10.0000, 10.9999}}, + {{11.9999, 12.9999}, {13.9999, 14.9999}}, + {{15.9999, 16.9999}, {17.9999, 18.9999}}}, + {{{19.9999, 20.9999}, {21.9999, 22.9999}}, + {{23.9999, 24.9999}, {25.9999, 26.9999}}, + {{27.9999, 28.9999}, {29.9998, 30.9998}}, + {{31.9998, 32.9998}, {33.9998, 34.9998}}, + {{35.9998, 36.9998}, {37.9998, 38.9998}}}}); + ASSERT_TRUE(output.allclose(expected)); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, BatchNorm3dStateful) { + BatchNorm3d bn(5); + + ASSERT_TRUE(bn->options.track_running_stats()); + + ASSERT_TRUE(bn->running_mean.defined()); + ASSERT_EQ(bn->running_mean.dim(), 1); + ASSERT_EQ(bn->running_mean.size(0), 5); + + ASSERT_TRUE(bn->running_var.defined()); + ASSERT_EQ(bn->running_var.dim(), 1); + ASSERT_EQ(bn->running_var.size(0), 5); + + ASSERT_TRUE(bn->num_batches_tracked.defined()); + ASSERT_EQ(bn->num_batches_tracked.dim(), 0); + + ASSERT_TRUE(bn->options.affine()); + + ASSERT_TRUE(bn->weight.defined()); + ASSERT_EQ(bn->weight.dim(), 1); + ASSERT_EQ(bn->weight.size(0), 5); + + ASSERT_TRUE(bn->bias.defined()); + ASSERT_EQ(bn->bias.dim(), 1); + ASSERT_EQ(bn->bias.size(0), 5); +} + +TEST_F(ModulesTest, BatchNorm3dStateless) { + BatchNorm3d bn( + BatchNorm3dOptions(5).track_running_stats(false).affine(false)); + + ASSERT_FALSE(bn->running_mean.defined()); + ASSERT_FALSE(bn->running_var.defined()); + ASSERT_FALSE(bn->num_batches_tracked.defined()); + ASSERT_FALSE(bn->weight.defined()); + ASSERT_FALSE(bn->bias.defined()); +} + +TEST_F(ModulesTest, BatchNorm3d) { + BatchNorm3d bn(5); + bn->eval(); + + auto input = + torch::arange(2. * 5 * 2 * 2 * 2).view({2, 5, 2, 2, 2}).requires_grad_(); + auto output = bn->forward(input); + auto expected = torch::tensor( + {{{{{0.0000, 1.0000}, {2.0000, 3.0000}}, + {{4.0000, 5.0000}, {6.0000, 7.0000}}}, + {{{8.0000, 9.0000}, {10.0000, 10.9999}}, + {{11.9999, 12.9999}, {13.9999, 14.9999}}}, + {{{15.9999, 16.9999}, {17.9999, 18.9999}}, + {{19.9999, 20.9999}, {21.9999, 22.9999}}}, + {{{23.9999, 24.9999}, {25.9999, 26.9999}}, + {{27.9999, 28.9999}, {29.9998, 30.9998}}}, + {{{31.9998, 32.9998}, {33.9998, 34.9998}}, + {{35.9998, 36.9998}, {37.9998, 38.9998}}}}, + {{{{39.9998, 40.9998}, {41.9998, 42.9998}}, + {{43.9998, 44.9998}, {45.9998, 46.9998}}}, + {{{47.9998, 48.9998}, {49.9997, 50.9997}}, + {{51.9997, 52.9997}, {53.9997, 54.9997}}}, + {{{55.9997, 56.9997}, {57.9997, 58.9997}}, + {{59.9997, 60.9997}, {61.9997, 62.9997}}}, + {{{63.9997, 64.9997}, {65.9997, 66.9997}}, + {{67.9997, 68.9997}, {69.9996, 70.9996}}}, + {{{71.9996, 72.9996}, {73.9996, 74.9996}}, + {{75.9996, 76.9996}, {77.9996, 78.9996}}}}}); + ASSERT_TRUE(output.allclose(expected)); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, InstanceNorm1dStateful) { + InstanceNorm1d instance_norm( + InstanceNorm1dOptions(5).track_running_stats(true).affine(true)); + + ASSERT_TRUE(instance_norm->options.track_running_stats()); + + ASSERT_TRUE(instance_norm->running_mean.defined()); + ASSERT_EQ(instance_norm->running_mean.dim(), 1); + ASSERT_EQ(instance_norm->running_mean.size(0), 5); + + ASSERT_TRUE(instance_norm->running_var.defined()); + ASSERT_EQ(instance_norm->running_var.dim(), 1); + ASSERT_EQ(instance_norm->running_var.size(0), 5); + + ASSERT_TRUE(instance_norm->num_batches_tracked.defined()); + ASSERT_EQ(instance_norm->num_batches_tracked.dim(), 0); + + ASSERT_TRUE(instance_norm->options.affine()); + + ASSERT_TRUE(instance_norm->weight.defined()); + ASSERT_EQ(instance_norm->weight.dim(), 1); + ASSERT_EQ(instance_norm->weight.size(0), 5); + + ASSERT_TRUE(instance_norm->bias.defined()); + ASSERT_EQ(instance_norm->bias.dim(), 1); + ASSERT_EQ(instance_norm->bias.size(0), 5); +} + +TEST_F(ModulesTest, InstanceNorm1dStateless) { + InstanceNorm1d instance_norm( + InstanceNorm1dOptions(5).track_running_stats(false).affine(false)); + + ASSERT_FALSE(instance_norm->running_mean.defined()); + ASSERT_FALSE(instance_norm->running_var.defined()); + ASSERT_FALSE(instance_norm->num_batches_tracked.defined()); + ASSERT_FALSE(instance_norm->weight.defined()); + ASSERT_FALSE(instance_norm->bias.defined()); +} + +TEST_F(ModulesTest, InstanceNorm1d) { + InstanceNorm1d instance_norm(5); + instance_norm->eval(); + + auto input = torch::arange(2. * 5 * 2).view({2, 5, 2}).requires_grad_(); + auto output = instance_norm->forward(input); + auto expected = torch::tensor( + {{{-1.0000, 1.0000}, + {-1.0000, 1.0000}, + {-1.0000, 1.0000}, + {-1.0000, 1.0000}, + {-1.0000, 1.0000}}, + {{-1.0000, 1.0000}, + {-1.0000, 1.0000}, + {-1.0000, 1.0000}, + {-1.0000, 1.0000}, + {-1.0000, 1.0000}}}); + ASSERT_TRUE(output.allclose(expected, 1e-3)); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, InstanceNorm2dStateful) { + InstanceNorm2d instance_norm( + InstanceNorm2dOptions(5).track_running_stats(true).affine(true)); + + ASSERT_TRUE(instance_norm->options.track_running_stats()); + + ASSERT_TRUE(instance_norm->running_mean.defined()); + ASSERT_EQ(instance_norm->running_mean.dim(), 1); + ASSERT_EQ(instance_norm->running_mean.size(0), 5); + + ASSERT_TRUE(instance_norm->running_var.defined()); + ASSERT_EQ(instance_norm->running_var.dim(), 1); + ASSERT_EQ(instance_norm->running_var.size(0), 5); + + ASSERT_TRUE(instance_norm->num_batches_tracked.defined()); + ASSERT_EQ(instance_norm->num_batches_tracked.dim(), 0); + + ASSERT_TRUE(instance_norm->options.affine()); + + ASSERT_TRUE(instance_norm->weight.defined()); + ASSERT_EQ(instance_norm->weight.dim(), 1); + ASSERT_EQ(instance_norm->weight.size(0), 5); + + ASSERT_TRUE(instance_norm->bias.defined()); + ASSERT_EQ(instance_norm->bias.dim(), 1); + ASSERT_EQ(instance_norm->bias.size(0), 5); +} + +TEST_F(ModulesTest, InstanceNorm2dStateless) { + InstanceNorm2d instance_norm( + InstanceNorm2dOptions(5).track_running_stats(false).affine(false)); + + ASSERT_FALSE(instance_norm->running_mean.defined()); + ASSERT_FALSE(instance_norm->running_var.defined()); + ASSERT_FALSE(instance_norm->num_batches_tracked.defined()); + ASSERT_FALSE(instance_norm->weight.defined()); + ASSERT_FALSE(instance_norm->bias.defined()); +} + +TEST_F(ModulesTest, InstanceNorm2d) { + InstanceNorm2d instance_norm(5); + instance_norm->eval(); + + auto input = + torch::arange(2. * 5 * 2 * 2).view({2, 5, 2, 2}).requires_grad_(); + auto output = instance_norm->forward(input); + auto expected = torch::tensor( + {{{{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}}, + {{{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}, + {{-1.3416, -0.4472}, {0.4472, 1.3416}}}}); + ASSERT_TRUE(output.allclose(expected, 1e-3)); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, InstanceNorm3dStateful) { + InstanceNorm3d instance_norm( + InstanceNorm3dOptions(5).track_running_stats(true).affine(true)); + + ASSERT_TRUE(instance_norm->options.track_running_stats()); + + ASSERT_TRUE(instance_norm->running_mean.defined()); + ASSERT_EQ(instance_norm->running_mean.dim(), 1); + ASSERT_EQ(instance_norm->running_mean.size(0), 5); + + ASSERT_TRUE(instance_norm->running_var.defined()); + ASSERT_EQ(instance_norm->running_var.dim(), 1); + ASSERT_EQ(instance_norm->running_var.size(0), 5); + + ASSERT_TRUE(instance_norm->num_batches_tracked.defined()); + ASSERT_EQ(instance_norm->num_batches_tracked.dim(), 0); + + ASSERT_TRUE(instance_norm->options.affine()); + + ASSERT_TRUE(instance_norm->weight.defined()); + ASSERT_EQ(instance_norm->weight.dim(), 1); + ASSERT_EQ(instance_norm->weight.size(0), 5); + + ASSERT_TRUE(instance_norm->bias.defined()); + ASSERT_EQ(instance_norm->bias.dim(), 1); + ASSERT_EQ(instance_norm->bias.size(0), 5); +} + +TEST_F(ModulesTest, InstanceNorm3dStateless) { + InstanceNorm3d instance_norm( + InstanceNorm3dOptions(5).track_running_stats(false).affine(false)); + + ASSERT_FALSE(instance_norm->running_mean.defined()); + ASSERT_FALSE(instance_norm->running_var.defined()); + ASSERT_FALSE(instance_norm->num_batches_tracked.defined()); + ASSERT_FALSE(instance_norm->weight.defined()); + ASSERT_FALSE(instance_norm->bias.defined()); +} + +TEST_F(ModulesTest, InstanceNorm3d) { + InstanceNorm3d instance_norm(5); + instance_norm->eval(); + + auto input = + torch::arange(2. * 5 * 2 * 2 * 2).view({2, 5, 2, 2, 2}).requires_grad_(); + auto output = instance_norm->forward(input); + auto expected = torch::tensor( + {{{{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}}, + {{{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}, + {{{-1.5275, -1.0911}, {-0.6547, -0.2182}}, + {{0.2182, 0.6547}, {1.0911, 1.5275}}}}}); + ASSERT_TRUE(output.allclose(expected, 1e-3)); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, Linear_CUDA) { + Linear model(5, 2); + model->to(torch::kCUDA); + auto x = + torch::randn({10, 5}, torch::device(torch::kCUDA).requires_grad(true)); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 2); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.size(0), 10); + ASSERT_EQ(y.size(1), 2); + + ASSERT_EQ(model->weight.grad().numel(), 2 * 5); +} + +TEST_F(ModulesTest, Linear2_CUDA) { + Linear model(5, 2); + model->to(torch::kCUDA); + model->to(torch::kCPU); + auto x = torch::randn({10, 5}, torch::requires_grad()); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(y.ndimension(), 2); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.size(0), 10); + ASSERT_EQ(y.size(1), 2); + + ASSERT_EQ(model->weight.grad().numel(), 2 * 5); +} + +TEST_F(ModulesTest, L1Loss) { + L1Loss loss; + auto input = torch::randn({5, 6}, torch::requires_grad()); + auto target = torch::empty({5, 6}).random_(2); + auto output = loss->forward(torch::sigmoid(input), target); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(output.sizes(), std::vector()); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, MSELoss) { + MSELoss loss; + auto input = torch::randn({5, 6}, torch::requires_grad()); + auto target = torch::empty({5, 6}).random_(2); + auto output = loss->forward(torch::sigmoid(input), target); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(output.sizes(), torch::IntArrayRef()); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, BCELoss) { + BCELoss loss; + auto input = torch::randn({5, 6}, torch::requires_grad()); + auto target = torch::empty({5, 6}).random_(2); + auto output = loss->forward(torch::sigmoid(input), target); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(output.sizes(), torch::IntArrayRef()); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, KLDivLoss) { + KLDivLoss loss; + auto input = torch::randn({5, 6}, torch::requires_grad()); + auto target = torch::empty({5, 6}).random_(2); + auto output = loss->forward(torch::sigmoid(input), target); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(output.sizes(), torch::IntArrayRef()); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, HingeEmbeddingLoss) { + HingeEmbeddingLoss loss(HingeEmbeddingLossOptions().margin(2)); + auto input = torch::tensor( + {{2, 22, 4}, {20, 10, 0}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({{2, 6, 4}, {1, 10, 0}}, torch::kFloat); + auto output = loss->forward(input, target); + auto expected = torch::tensor({10}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, MultiMarginLoss) { + auto weight = torch::tensor({0.3, 0.3, 0.4}, torch::kFloat); + MultiMarginLoss loss(MultiMarginLossOptions().margin(2).weight(weight)); + auto input = torch::tensor( + {{0.2, 0.2, 0.6}, {0.1, 0.8, 0.1}, {0.9, 0.09, 0.01}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({2, 1, 0}, torch::kLong); + auto output = loss->forward(input, target); + auto expected = torch::tensor({0.305556}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected, 1e-04)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, CosineEmbeddingLoss) { + CosineEmbeddingLoss cos(CosineEmbeddingLossOptions().margin(0.5)); + auto input1 = torch::tensor( + {{2, 3, 4}, {6, 2, 4}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto input2 = torch::tensor( + {{2, 3, 5}, {9, 12, 0}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({1, -1}); + auto output = cos(input1, input2, target); + auto expected = torch::tensor({0.1004}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected, 1e-4)); + ASSERT_EQ(input1.sizes(), input1.grad().sizes()); + ASSERT_EQ(input2.sizes(), input2.grad().sizes()); +} + +TEST_F(ModulesTest, SmoothL1LossDefaultOptions) { + SmoothL1Loss loss; + auto input = torch::tensor( + {0.1, 1.2, 4.7}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto output = loss(input, target); + auto expected = torch::tensor(0.0233335, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, HuberLossDefaultOptions) { + HuberLoss loss; + auto input = torch::tensor( + {0.1, 1.2, 4.7}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto output = loss(input, target); + auto expected = torch::tensor(0.0233335, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, MultiLabelMarginLossDefaultOptions) { + MultiLabelMarginLoss loss; + auto input = torch::tensor( + {{0.1, 0.2, 0.4, 0.8}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({{3, 0, -1, 1}}, torch::kLong); + auto output = loss->forward(input, target); + auto expected = torch::tensor({0.8500}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, SmoothL1LossNoReduction) { + SmoothL1Loss loss(/*reduction=*/torch::kNone); + auto input = torch::tensor( + {0.1, 1.2, 4.7}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto output = loss(input, target); + auto expected = torch::tensor({0.005, 0.02, 0.045}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, HuberLossNoReduction) { + HuberLoss loss(/*reduction=*/torch::kNone); + auto input = torch::tensor( + {0.1, 1.2, 4.7}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto output = loss(input, target); + auto expected = torch::tensor({0.005, 0.02, 0.045}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, MultiLabelMarginLossNoReduction) { + MultiLabelMarginLoss loss(torch::kNone); + auto input = torch::tensor( + {{0.1, 0.2, 0.4, 0.8}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({{3, 0, -1, 1}}, torch::kLong); + auto output = loss->forward(input, target); + auto expected = torch::tensor({0.8500}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, SmoothL1LossBeta) { + auto options = SmoothL1LossOptions().beta(0.2); + SmoothL1Loss loss(options); + auto input = torch::tensor( + {0.1, 1.2, 4.7}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto output = loss(input, target); + auto expected = torch::tensor(0.108333, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, HuberLossDelta) { + auto options = HuberLossOptions().delta(0.2); + HuberLoss loss(options); + auto input = torch::tensor( + {0.1, 1.2, 4.7}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0., 1., 5.}, torch::kFloat); + auto output = loss(input, target); + auto expected = torch::tensor(0.0216666, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, TripletMarginLoss) { + TripletMarginLoss loss(TripletMarginLossOptions().margin(1.0)); + auto anchor = torch::tensor( + {{3., 3.}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto positive = torch::tensor( + {{2., 2.}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto negative = torch::tensor( + {{0., 0.}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto output = loss->forward(anchor, positive, negative); + auto expected = torch::tensor({0.}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected, 1e-04)); + ASSERT_EQ(anchor.sizes(), anchor.grad().sizes()); +} + +TEST_F(ModulesTest, TripletMarginWithDistanceLossDefaultParity) { + // Check that if we use torch::pairwise_distance with the default + // TripletMarginLoss options as our distance function, the outputs + // are equal (i.e., equal under defaults). + + std::vector reductions = { + torch::kSum, torch::kMean, torch::kNone}; + std::vector margins = {0.5, 1.0, 1.5}; + std::vector swaps = {true, false}; + + for (auto& reduction : reductions) { + for (auto& margin : margins) { + for (const auto swap : swaps) { + auto anchor = torch::randn( + {100, 128}, torch::dtype(torch::kFloat).requires_grad(true)); + auto positive = torch::randn( + {100, 128}, torch::dtype(torch::kFloat).requires_grad(true)); + auto negative = torch::randn( + {100, 128}, torch::dtype(torch::kFloat).requires_grad(true)); + + auto basicOptions = + TripletMarginLossOptions().reduction(reduction).margin(margin).swap( + swap); + auto distanceOptions = TripletMarginWithDistanceLossOptions() + .reduction(reduction) + .margin(margin) + .swap(swap); + TripletMarginLoss basicLoss(basicOptions); + TripletMarginWithDistanceLoss distanceLoss(distanceOptions); + + auto basicOutput = basicLoss->forward(anchor, positive, negative); + auto distanceOutput = distanceLoss->forward(anchor, positive, negative); + auto basicOperatorOutput = basicLoss(anchor, positive, negative); + auto distanceOperatorOutput = distanceLoss(anchor, positive, negative); + + ASSERT_TRUE(distanceOutput.allclose(basicOutput, 1e-6, 1e-6)); + ASSERT_TRUE( + distanceOperatorOutput.allclose(distanceOutput, 1e-6, 1e-6)); + ASSERT_TRUE( + distanceOperatorOutput.allclose(basicOperatorOutput, 1e-6, 1e-6)); + + // handle for torch::kNone reduction + auto sum = distanceOutput.sum(); + sum.backward(); + ASSERT_EQ(anchor.sizes(), anchor.grad().sizes()); + ASSERT_EQ(positive.sizes(), positive.grad().sizes()); + ASSERT_EQ(negative.sizes(), negative.grad().sizes()); + } + } + } +} + +TEST_F(ModulesTest, TripletMarginWithDistanceLossFunctionalParity) { + // Check for parity between F::triplet_margin_with_distance_loss and + // TripletMarginWithDistanceLoss. + auto pairwise_distance = [&](const torch::Tensor& x, const torch::Tensor& y) { + return torch::pairwise_distance(x, y); + }; + auto cosine_distance = [&](const torch::Tensor& x, const torch::Tensor& y) { + return 1.0 - torch::cosine_similarity(x, y); + }; + std::vector + distance_functions = {pairwise_distance, cosine_distance}; + + std::vector reductions = { + torch::kSum, torch::kMean, torch::kNone}; + std::vector margins = {0.5, 1.0, 1.5}; + std::vector swaps = {true, false}; + + for (auto& function : distance_functions) { + for (auto& reduction : reductions) { + for (auto& margin : margins) { + for (const auto swap : swaps) { + auto moduleOptions = TripletMarginWithDistanceLossOptions() + .distance_function(function) + .reduction(reduction) + .margin(margin) + .swap(swap); + auto functionOptions = + torch::nn::functional::TripletMarginWithDistanceLossFuncOptions() + .distance_function(function) + .reduction(reduction) + .margin(margin) + .swap(swap); + + auto anchor = torch::randn( + {100, 128}, torch::dtype(torch::kFloat).requires_grad(true)); + auto positive = torch::randn( + {100, 128}, torch::dtype(torch::kFloat).requires_grad(true)); + auto negative = torch::randn( + {100, 128}, torch::dtype(torch::kFloat).requires_grad(true)); + + TripletMarginWithDistanceLoss distanceLoss(moduleOptions); + + auto moduleOutput = distanceLoss->forward(anchor, positive, negative); + auto moduleOperatorOutput = distanceLoss(anchor, positive, negative); + auto functionOutput = + torch::nn::functional::triplet_margin_with_distance_loss( + anchor, positive, negative, functionOptions); + + ASSERT_TRUE(moduleOutput.allclose(functionOutput, 1e-6, 1e-6)); + ASSERT_TRUE( + moduleOperatorOutput.allclose(functionOutput, 1e-6, 1e-6)); + } + } + } + } +} + +TEST_F(ModulesTest, NLLLoss) { + NLLLoss loss; + auto input = torch::tensor( + {{-0.1315, -3.1315, -2.5315}, + {-3.7038, -0.1038, -2.6038}, + {-2.3422, -1.3422, -0.4422}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({1, 0, 2}, torch::kLong); + auto output = loss->forward(input, target); + auto expected = torch::tensor(2.4258, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected, 1e-04)); + ASSERT_TRUE( + NLLLoss(NLLLossOptions().ignore_index(-100).reduction(torch::kMean)) + ->forward(input, target) + .allclose(expected, 1e-04)); +} + +TEST_F(ModulesTest, CrossEntropyLoss) { + CrossEntropyLoss loss; + auto input = torch::tensor( + {{3., 3.}, {2., 2.}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({0, 1}, torch::kLong); + auto output = loss->forward(input, target); + auto expected = torch::tensor(0.6931, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected, 1e-04)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); + ASSERT_TRUE( + CrossEntropyLoss( + CrossEntropyLossOptions().ignore_index(-100).reduction(torch::kMean)) + ->forward(input, target) + .allclose(expected, 1e-04)); + + // label smoothing with class indices + loss = CrossEntropyLoss( + CrossEntropyLossOptions().label_smoothing(0.15).reduction(torch::kMean)); + input = torch::tensor( + {{3., 1.}, {1., 2.}}, torch::dtype(torch::kFloat).requires_grad(true)); + target = torch::tensor({0, 1}, torch::kLong); + output = loss->forward(input, target); + expected = torch::tensor(0.3326, torch::kFloat); + s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected, 1e-04)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); + + // label smoothing with with target probabilities + loss = CrossEntropyLoss( + CrossEntropyLossOptions().label_smoothing(0.2).reduction(torch::kMean)); + input = torch::tensor( + {{3., 1.}, {1., 2.}}, torch::dtype(torch::kFloat).requires_grad(true)); + target = torch::tensor({{0.8, 0.2}, {0.1, 0.9}}, torch::kFloat); + output = loss->forward(input, target); + expected = torch::tensor(0.5701, torch::kFloat); + s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected, 1e-04)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, CosineSimilarity) { + CosineSimilarity cos(CosineSimilarityOptions().dim(1)); + auto input1 = torch::tensor( + {{1, 2, 3}, {4, 5, 6}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto input2 = torch::tensor( + {{1, 8, 3}, {2, 1, 6}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto output = cos->forward(input1, input2); + auto expected = torch::tensor({0.8078, 0.8721}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected, 1e-04)); + ASSERT_EQ(input1.sizes(), input1.grad().sizes()); +} + +TEST_F(ModulesTest, SoftMarginLossDefaultOptions) { + SoftMarginLoss loss; + auto input = torch::tensor( + {2., 4., 1., 3.}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({-1., 1., 1., -1.}, torch::kFloat); + auto output = loss->forward(input, target); + auto expected = torch::tensor({1.3767317}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, MultiLabelSoftMarginLossDefaultOptions) { + MultiLabelSoftMarginLoss loss; + auto input = torch::tensor( + {{0., 2., 2., 0.}, {2., 1., 0., 1.}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto target = + torch::tensor({{0., 0., 1., 0.}, {1., 0., 1., 1.}}, torch::kFloat); + auto output = loss->forward(input, target); + auto expected = torch::tensor({0.7608436}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, SoftMarginLossNoReduction) { + SoftMarginLoss loss(torch::kNone); + auto input = torch::tensor( + {2., 4., 1., 3.}, torch::dtype(torch::kFloat).requires_grad(true)); + auto target = torch::tensor({-1., 1., 1., -1.}, torch::kFloat); + auto output = loss->forward(input, target); + auto expected = torch::tensor( + {2.1269281, 0.01814993, 0.3132617, 3.0485873}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, MultiLabelSoftMarginLossWeightedNoReduction) { + auto input = torch::tensor( + {{0., 2., 2., 0.}, {2., 1., 0., 1.}}, + torch::dtype(torch::kFloat).requires_grad(true)); + auto target = + torch::tensor({{0., 0., 1., 0.}, {1., 0., 1., 1.}}, torch::kFloat); + auto weight = torch::tensor({0.1, 0.6, 0.4, 0.8}, torch::kFloat); + auto options = + MultiLabelSoftMarginLossOptions().reduction(torch::kNone).weight(weight); + MultiLabelSoftMarginLoss loss = MultiLabelSoftMarginLoss(options); + auto output = loss->forward(input, target); + auto expected = torch::tensor({0.4876902, 0.3321295}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input.sizes(), input.grad().sizes()); +} + +TEST_F(ModulesTest, PairwiseDistance) { + PairwiseDistance dist(PairwiseDistanceOptions().p(1)); + auto input1 = torch::tensor( + {{1, 2, 3}, {4, 5, 6}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto input2 = torch::tensor( + {{1, 8, 3}, {2, 1, 6}}, torch::dtype(torch::kFloat).requires_grad(true)); + auto output = dist->forward(input1, input2); + auto expected = torch::tensor({6, 6}, torch::kFloat); + auto s = output.sum(); + s.backward(); + + ASSERT_TRUE(output.allclose(expected)); + ASSERT_EQ(input1.sizes(), input1.grad().sizes()); +} + +TEST_F(ModulesTest, ELU) { + const auto size = 3; + for (const auto alpha : {0.0, 0.42, 1.0, 4.2, 42.42}) { + for (const auto inplace : {false, true}) { + ELU model{ELUOptions().alpha(alpha).inplace(inplace)}; + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + if (!inplace) { + x.requires_grad_(true); + } + auto x_orig = x.clone(); + auto y = model(x); + torch::Tensor s = y.sum(); + + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = torch::max(torch::zeros_like(x_orig), x_orig) + + torch::min(torch::zeros_like(x_orig), alpha * (torch::expm1(x_orig))); + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } else { + s.backward(); + } + } + } +} + +TEST_F(ModulesTest, SELU) { + for (const auto inplace : {false, true}) { + SELU model(inplace); + auto input = torch::randn({5, 5}); + if (!inplace) { + input.requires_grad_(true); + } + auto input_orig = input.clone(); + auto output = model->forward(input); + const double scale = 1.0507009873554804934193349852946; + const double alpha = 1.6732632423543772848170429916717; + auto zero = torch::zeros_like(input); + auto expected = scale * + (torch::max(zero, input_orig) + + torch::min(zero, alpha * (torch::expm1(input_orig)))); + auto s = output.sum(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_TRUE(output.allclose(expected)); + if (inplace) { + ASSERT_TRUE(input.allclose(expected)); + } else { + s.backward(); + } + } +} + +TEST_F(ModulesTest, Hardshrink) { + const auto size = 3; + for (const auto lambda : {-4.2, -1.0, -0.42, 0.0, 0.42, 1.0, 4.2, 42.42}) { + Hardshrink model{HardshrinkOptions().lambda(lambda)}; + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}).set_requires_grad(true); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = (x.abs() > lambda) * x; + ASSERT_TRUE(torch::allclose(y, y_exp)); + } +} + +TEST_F(ModulesTest, Hardtanh) { + const auto size = 3; + for (const auto min_val : {-4.2, -1.0, -0.42, 0.0}) { + for (const auto max_val : {0.42, 1.0, 4.2}) { + for (const auto inplace : {false, true}) { + Hardtanh model{ + HardtanhOptions().min_val(min_val).max_val(max_val).inplace( + inplace)}; + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + if (!inplace) { + x.requires_grad_(true); + } + auto x_orig = x.clone(); + auto y = model(x); + torch::Tensor s = y.sum(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = (x_orig < min_val) * min_val + + ((x_orig >= min_val) * (x_orig <= max_val)) * x_orig + + (x_orig > max_val) * max_val; + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } else { + s.backward(); + } + } + } + } +} + +TEST_F(ModulesTest, HardtanhMinValGEMaxVal) { + ASSERT_THROWS_WITH( + Hardtanh{HardtanhOptions().min_val(0.42).max_val(0.42)}, + "max_val must be greater than min_val"); + ASSERT_THROWS_WITH( + Hardtanh{HardtanhOptions().min_val(0.42).max_val(-0.42)}, + "max_val must be greater than min_val"); + + Hardtanh ht{HardtanhOptions().min_val(-0.42).max_val(0.42)}; + ht->options.min_val(0.42); + ASSERT_THROWS_WITH(ht->reset(), "max_val must be greater than min_val"); + ht->options.max_val(-0.42); + ASSERT_THROWS_WITH(ht->reset(), "max_val must be greater than min_val"); +} + +TEST_F(ModulesTest, LeakyReLU) { + const auto size = 3; + for (const auto inplace : {false, true}) { + for (const auto negative_slope : {0.0, 0.42, 1.0}) { + for (const auto type : {torch::kFloat, torch::kBFloat16}) { + LeakyReLU model{ + LeakyReLUOptions().negative_slope(negative_slope).inplace(inplace)}; + auto x = torch::linspace(-10.0, 10.0, size * size * size).to(type); + x.resize_({size, size, size}); + if (!inplace) { + x.requires_grad_(true); + } + auto x_orig = x.clone(); + auto y = model(x); + torch::Tensor s = y.sum(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = + (x_orig < 0) * x_orig * negative_slope + (x_orig >= 0) * x_orig; + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } else { + s.backward(); + } + } + } + } +} + +TEST_F(ModulesTest, LogSigmoid) { + const auto size = 3; + LogSigmoid model; + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}).set_requires_grad(true); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = torch::log( + torch::ones_like(x) / (torch::ones_like(x) + torch::exp(torch::neg(x)))); + ASSERT_TRUE(torch::allclose(y, y_exp, 1e-4, 1e-7)); +} + +TEST_F(ModulesTest, Softmax) { + Softmax m(/*dim=*/1); + auto input = torch::arange(10, torch::kFloat).reshape({2, 5}); + auto output = m(input); + auto sum = torch::sum(torch::exp(input), 1); + + for (const auto i : c10::irange(2)) { + auto expected = torch::exp(input[i]) / sum[i]; + ASSERT_TRUE(torch::allclose(output[i], expected)); + } +} + +TEST_F(ModulesTest, Softmin) { + Softmin m(/*dim=*/1); + auto input = torch::arange(10, torch::kFloat).reshape({2, 5}); + auto output = m(input); + auto sum = torch::sum(torch::exp(-input), 1); + + for (const auto i : c10::irange(2)) { + auto expected = torch::exp(-input[i]) / sum[i]; + ASSERT_TRUE(torch::allclose(output[i], expected)); + } +} + +TEST_F(ModulesTest, LogSoftmax) { + LogSoftmax m(/*dim=*/1); + auto input = torch::arange(10, torch::kFloat).reshape({2, 5}); + auto output = m(input); + auto sum = torch::sum(torch::exp(input), 1); + + for (const auto i : c10::irange(2)) { + auto expected = torch::log(torch::exp(input[i]) / sum[i]); + ASSERT_TRUE(torch::allclose(output[i], expected)); + } +} + +TEST_F(ModulesTest, AdaptiveLogSoftmaxWithLoss) { + { + // log_probs actually returns log_proba + AdaptiveLogSoftmaxWithLoss asfm( + AdaptiveLogSoftmaxWithLossOptions(8, 4, {2}).div_value(2.)); + auto x = torch::randn({4, 8}); + auto logprob_out = asfm->log_prob(x); + ASSERT_TRUE( + torch::allclose(torch::exp(logprob_out).data().sum(1), torch::ones(4))); + } + { + // test predict + AdaptiveLogSoftmaxWithLoss asfm( + AdaptiveLogSoftmaxWithLossOptions(8, 10, {4, 8}) + .div_value(2.) + .head_bias(true)); + auto x = torch::randn({64, 8}); + auto logprob_out = asfm->log_prob(x); + auto predict_out = asfm->predict(x); + ASSERT_TRUE(torch::allclose(predict_out, logprob_out.argmax(1))); + } + { + // cluster sizes + AdaptiveLogSoftmaxWithLoss asfm( + AdaptiveLogSoftmaxWithLossOptions(16, 20, {4, 10, 15}).div_value(2.)); + auto x = torch::arange(100, 132, torch::kFloat).reshape({2, 16}); + auto y = torch::tensor({0, 17}, torch::kLong); + auto asm_out = asfm(x, y); + ASSERT_EQ(asm_out.output.sizes(), std::vector({2})); + } + { + // forward returns the same thing as log_probs + AdaptiveLogSoftmaxWithLoss asfm( + AdaptiveLogSoftmaxWithLossOptions(8, 4, {2}).div_value(2.)); + auto x = torch::randn({4, 8}); + auto logprob_out = asfm->log_prob(x); + NLLLoss nll_loss; + + for (const auto v : c10::irange(4)) { + auto y = torch::full({4}, v, torch::kLong); + auto asm_out = asfm(x, y); + auto out = asm_out.output; + auto loss = torch::tensor(asm_out.loss, torch::kFloat); + auto expected = nll_loss->forward(logprob_out, y); + + ASSERT_TRUE(torch::allclose(loss, expected)); + ASSERT_TRUE(torch::allclose( + out, logprob_out.gather(1, y.unsqueeze(1)).squeeze())); + } + } + { + // test no batch dim + AdaptiveLogSoftmaxWithLoss asfm( + AdaptiveLogSoftmaxWithLossOptions(16, 20, {4, 10, 15}).div_value(2.)); + auto x = torch::randn({1, 16}); + auto y = torch::tensor({17}); + auto x2 = x.squeeze(0); + auto y2 = y.squeeze(0); + ASSERT_TRUE( + torch::allclose(asfm(x, y).output.squeeze(0), asfm(x2, y2).output)); + } + { + // test div_value + auto options = + AdaptiveLogSoftmaxWithLossOptions(16, 20, {4, 10, 15}).div_value(0.); + ASSERT_THROWS_WITH( + AdaptiveLogSoftmaxWithLoss(options), + "div_value should not be equal to 0"); + + options = + AdaptiveLogSoftmaxWithLossOptions(16, 20, {4, 10, 15}).div_value(0.25); + ASSERT_TRUE(AdaptiveLogSoftmaxWithLoss(options)); + } +} + +TEST_F(ModulesTest, Softmax2d) { + Softmax2d m; + auto input = torch::arange(24, torch::kFloat).reshape({1, 2, 3, 4}); + auto output = m(input); + auto sum = torch::sum(torch::exp(input), 1); + + for (const auto i : c10::irange(1)) { + for (const auto j : c10::irange(2)) { + for (const auto k : c10::irange(3)) { + for (const auto l : c10::irange(4)) { + auto expected = torch::exp(input[i][j][k][l]) / sum[i][k][l]; + ASSERT_TRUE(torch::allclose(output[i][j][k][l], expected)); + } + } + } + } +} + +TEST_F(ModulesTest, PReLU) { + const auto num_parameters = 42; + const auto init = 0.42; + + PReLU model{PReLUOptions().num_parameters(num_parameters).init(init)}; + + ASSERT_EQ(model->weight.sizes(), std::vector({num_parameters})); + ASSERT_TRUE( + torch::allclose(model->weight, torch::full(num_parameters, init))); + + const auto x = torch::rand({100, num_parameters}) * 200 - 100; + const auto y = model(x); + const auto s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), x.ndimension()); + ASSERT_EQ(y.sizes(), x.sizes()); + const auto y_exp = (x < 0) * model->weight * x + (x >= 0) * x; + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(ModulesTest, ReLU) { + for (const auto inplace : {false, true}) { + const auto size = 3; + ReLU model(inplace); + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + if (!inplace) { + x.requires_grad_(true); + } + auto x_orig = x.clone(); + auto y = model(x); + torch::Tensor s = y.sum(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = (x_orig < 0) * 0 + (x_orig >= 0) * x_orig; + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } else { + s.backward(); + } + } +} + +TEST_F(ModulesTest, ReLU6) { + for (const auto inplace : {false, true}) { + const auto size = 3; + ReLU6 model(inplace); + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + if (!inplace) { + x.requires_grad_(true); + } + auto x_orig = x.clone(); + auto y = model(x); + torch::Tensor s = y.sum(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = (x_orig < 0) * 0 + ((x_orig >= 0) * (x_orig <= 6)) * x_orig + + (x_orig > 6) * 6; + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } else { + s.backward(); + } + } +} + +TEST_F(ModulesTest, RReLU) { + const auto size = 3; + for (const auto lower : {0.01, 0.1, 0.2}) { + for (const auto upper : {0.3, 0.4, 0.5}) { + for (const auto inplace : {false, true}) { + for (const auto type : {torch::kFloat, torch::kBFloat16}) { + RReLU model{ + RReLUOptions().lower(lower).upper(upper).inplace(inplace)}; + auto x = torch::linspace(-10.0, 10.0, size * size * size).to(type); + x.resize_({size, size, size}); + if (!inplace) { + x.requires_grad_(true); + } + auto x_orig = x.clone(); + auto y = model(x); + torch::Tensor s = y.sum(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto z = + ((x_orig >= 0) * (x_orig == y) + + (x_orig < 0) * (y >= x_orig * upper) * (y <= lower * x_orig)) * + 1.0; + ASSERT_TRUE(torch::allclose(z, torch::ones_like(z))); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y)); + } else { + s.backward(); + } + } + } + } + } +} + +TEST_F(ModulesTest, CELU) { + const auto size = 3; + for (const auto inplace : {false, true}) { + for (const auto alpha : {0.42, 1.0, 4.2, 42.42}) { + CELU model{CELUOptions().alpha(alpha).inplace(inplace)}; + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}); + if (!inplace) { + x.requires_grad_(true); + } + auto x_orig = x.clone(); + auto y = model(x); + torch::Tensor s = y.sum(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = torch::max(torch::zeros_like(x_orig), x_orig) + + torch::min(torch::zeros_like(x_orig), + alpha * (torch::expm1(x_orig / alpha))); + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } else { + s.backward(); + } + } + } +} + +TEST_F(ModulesTest, GLU) { + int64_t dim = 1; + GLU model(dim); + auto input = torch::randn({4, 2}, torch::requires_grad()); + auto output = model->forward(input); + auto input_size = input.sizes()[dim] / 2; + auto first_half = input.narrow(dim, 0, input_size); + auto second_half = input.narrow(dim, input_size, input_size); + auto expected = first_half * torch::sigmoid(second_half); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_TRUE(output.allclose(expected)); + + GLU model_default_options; + ASSERT_TRUE(model_default_options->forward(input).allclose(expected)); +} + +TEST_F(ModulesTest, GELU) { + GELU model(GELUOptions().approximate("none")); + const auto x = torch::linspace(-3.0, 3.0, 100); + const auto y_exp = x * 0.5 * (1.0 + torch::erf(x / std::sqrt(2.0))); + const auto y = model(x); + ASSERT_TRUE(torch::allclose(y, y_exp, 1.4e-06, 1e-05)); +} + +TEST_F(ModulesTest, TanhGELU) { + GELU model(GELUOptions().approximate("tanh")); + const auto x = torch::linspace(-3.0, 3.0, 100); + const auto inner = std::sqrt(2 / M_PI) * (x + 0.044715 * x.pow(3.0)); + const auto y_exp = 0.5 * x * (1.0 + inner.tanh()); + const auto y = model(x); + ASSERT_TRUE(torch::allclose(y, y_exp, 1.4e-06, 1e-05)); +} + +TEST_F(ModulesTest, Mish) { + Mish model; + auto x = torch::randn(100) * 10; + auto y_exp = x * x.exp().log1p().tanh(); + auto y = model(x); + + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(ModulesTest, Sigmoid) { + Sigmoid model; + auto x = torch::randn(100) * 10; + auto y_exp = 1 / (1 + torch::exp(-x)); + auto y = model(x); + + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(ModulesTest, PixelShuffle) { + PixelShuffle module(/*upscale_factor=*/2); + auto x = torch::tensor( + {{{{-17, 19}, {-1, 2}}, + {{7, 14}, {-3, 1}}, + {{0, -2}, {-12, 14}}, + {{-15, 0}, {-3, 9}}}}, + torch::kFloat); + auto y_exp = torch::tensor( + {{{{-17, 7, 19, 14}, {0, -15, -2, 0}, {-1, -3, 2, 1}, {-12, -3, 14, 9}}}}, + torch::kFloat); + auto y = module(x); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({1, 1, 4, 4})); + ASSERT_TRUE(y.allclose(y_exp)); +} + +TEST_F(ModulesTest, PixelUnshuffle) { + PixelUnshuffle module(/*downscale_factor=*/2); + auto x = torch::tensor( + {{{{-17, 7, 19, 14}, {0, -15, -2, 0}, {-1, -3, 2, 1}, {-12, -3, 14, 9}}}}, + torch::kFloat); + auto y_exp = torch::tensor( + {{{{-17, 19}, {-1, 2}}, + {{7, 14}, {-3, 1}}, + {{0, -2}, {-12, 14}}, + {{-15, 0}, {-3, 9}}}}, + torch::kFloat); + auto y = module(x); + + ASSERT_EQ(y.ndimension(), 4); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({1, 4, 2, 2})); + ASSERT_TRUE(y.allclose(y_exp)); +} + +TEST_F(ModulesTest, Softplus) { + const auto size = 3; + for (const auto beta : {0.5, 1.0, 2.0}) { + for (const auto threshold : {1.0, 3.0, 5.0}) { + Softplus model{SoftplusOptions().beta(beta).threshold(threshold)}; + auto x = torch::linspace(-3.0, 3.0, 61); + x.resize_({size, size, size}); + auto y_exp = + (x <= threshold) * torch::log(1 + torch::exp(x * beta)) / beta + + (x > threshold) * x; + auto y = model(x); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + } + } +} + +TEST_F(ModulesTest, Softshrink) { + const auto size = 3; + for (const auto lambda : {0.0, 0.42, 1.0, 4.2, 42.42}) { + Softshrink model{/*lambda=*/lambda}; + auto x = torch::linspace(-10.0, 10.0, size * size * size); + x.resize_({size, size, size}).set_requires_grad(true); + auto y = model(x); + torch::Tensor s = y.sum(); + + s.backward(); + ASSERT_EQ(s.ndimension(), 0); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + auto y_exp = (x < -lambda) * (x + lambda) + (x > lambda) * (x - lambda); + ASSERT_TRUE(torch::allclose(y, y_exp)); + } +} + +TEST_F(ModulesTest, Softsign) { + Softsign model; + auto x = torch::randn(100) * 10; + auto y_exp = x / (1 + x.abs()); + auto y = model(x); + + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(ModulesTest, Tanh) { + Tanh model; + auto x = torch::randn(100) * 10; + auto y_exp = (x.exp() - (-x).exp()) / (x.exp() + (-x).exp()); + auto y = model(x); + + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(ModulesTest, Tanhshrink) { + Tanhshrink model; + auto x = torch::randn(100) * 10; + auto y_exp = x - x.tanh(); + auto y = model(x); + + ASSERT_TRUE(torch::allclose(y, y_exp)); +} + +TEST_F(ModulesTest, Threshold) { + const auto size = 3; + for (const auto threshold : {0.5, 1.0, 2.0}) { + for (const auto value : {0.5, 1.0, 2.0}) { + for (const auto inplace : {false, true}) { + Threshold model{ThresholdOptions(threshold, value).inplace(inplace)}; + auto x = torch::linspace(-3.0, 3.0, 61); + x.resize_({size, size, size}); + auto x_orig = x.clone(); + auto y_exp = + (x_orig <= threshold) * value + (x_orig > threshold) * x_orig; + auto y = model(x); + + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), std::vector({size, size, size})); + ASSERT_TRUE(torch::allclose(y, y_exp)); + if (inplace) { + ASSERT_TRUE(torch::allclose(x, y_exp)); + } + } + } + } +} + +TEST_F(ModulesTest, Upsampling1D) { + { + Upsample model(UpsampleOptions() + .size(std::vector({4})) + .mode(torch::kNearest)); + auto input = torch::ones({1, 1, 2}, torch::requires_grad()); + auto output = model->forward(input); + auto expected = torch::ones({1, 1, 4}); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_TRUE(output.allclose(expected)); + } + { + for (const auto align_corners : {true, false}) { + // test float scale factor up & down sampling + for (const auto scale_factor : {0.5, 1.5, 2.0}) { + Upsample model(UpsampleOptions() + .scale_factor(std::vector({scale_factor})) + .mode(torch::kLinear) + .align_corners(align_corners)); + auto input = torch::ones({1, 1, 2}, torch::requires_grad()); + auto output = model->forward(input); + auto expected_size = + static_cast(std::floor(input.size(-1) * scale_factor)); + auto expected = torch::ones({1, 1, expected_size}); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_TRUE(output.allclose(expected)); + } + } + } + { + // linear (1D) upsampling spatial invariance + Upsample model(UpsampleOptions() + .scale_factor(std::vector({3})) + .mode(torch::kLinear) + .align_corners(false)); + auto input = torch::zeros({1, 1, 9}); + input.narrow(2, 0, 4).normal_(); + auto output = model->forward(input); + auto expected = model->forward(input.narrow(2, 0, 5)); + + ASSERT_TRUE(torch::allclose(output.narrow(2, 0, 15), expected)); + } +} + +TEST_F(ModulesTest, Upsampling2D) { + { + Upsample model(UpsampleOptions() + .size(std::vector({4, 4})) + .mode(torch::kNearest)); + auto input = torch::ones({1, 1, 2, 2}, torch::requires_grad()); + auto output = model->forward(input); + auto expected = torch::ones({1, 1, 4, 4}); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_TRUE(output.allclose(expected)); + } + { + for (const auto align_corners : {true, false}) { + // test float scale factor up & down sampling + for (const auto scale_factor : {0.5, 1.5, 2.0}) { + Upsample model( + UpsampleOptions() + .scale_factor(std::vector({scale_factor, scale_factor})) + .mode(torch::kBilinear) + .align_corners(align_corners)); + auto input = torch::ones({1, 1, 2, 2}, torch::requires_grad()); + auto output = model->forward(input); + auto expected_size = + static_cast(std::floor(input.size(-1) * scale_factor)); + auto expected = torch::ones({1, 1, expected_size, expected_size}); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_TRUE(output.allclose(expected)); + } + } + } + { + for (const auto align_corners : {true, false}) { + // test float scale factor up & down sampling + for (const auto scale_factor : {0.5, 1.5, 2.0}) { + Upsample model( + UpsampleOptions() + .scale_factor(std::vector({scale_factor, scale_factor})) + .mode(torch::kBicubic) + .align_corners(align_corners)); + auto input = torch::ones({1, 1, 2, 2}, torch::requires_grad()); + auto output = model->forward(input); + auto expected_size = + static_cast(std::floor(input.size(-1) * scale_factor)); + auto expected = torch::ones({1, 1, expected_size, expected_size}); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_TRUE(output.allclose(expected)); + } + } + } +} + +TEST_F(ModulesTest, Upsampling3D) { + { + Upsample model(UpsampleOptions() + .size(std::vector({4, 4, 4})) + .mode(torch::kNearest)); + auto input = torch::ones({1, 1, 2, 2, 2}, torch::requires_grad()); + auto output = model->forward(input); + auto expected = torch::ones({1, 1, 4, 4, 4}); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_TRUE(output.allclose(expected)); + } + { + for (const auto align_corners : {true, false}) { + // test float scale factor up & down sampling + for (const auto scale_factor : {0.5, 1.5, 2.0}) { + Upsample model(UpsampleOptions() + .scale_factor(std::vector( + {scale_factor, scale_factor, scale_factor})) + .mode(torch::kTrilinear) + .align_corners(align_corners)); + auto input = torch::ones({1, 1, 2, 2, 2}, torch::requires_grad()); + auto output = model->forward(input); + auto expected_size = + static_cast(std::floor(input.size(-1) * scale_factor)); + auto expected = + torch::ones({1, 1, expected_size, expected_size, expected_size}); + auto s = output.sum(); + s.backward(); + + ASSERT_EQ(s.ndimension(), 0); + ASSERT_TRUE(output.allclose(expected)); + } + } + } +} + +TEST_F(ModulesTest, CTCLoss) { + CTCLoss loss{CTCLossOptions().reduction(torch::kNone)}; + const auto target_lengths = torch::tensor({0, 0, 0}); + const auto input_lengths = torch::tensor({50, 50, 50}); + const auto targets = + torch::randint(1, 15, at::IntArrayRef({0}), torch::kLong); + const auto log_probs = + torch::randn({50, 3, 15}, torch::kDouble).log_softmax(2); + const auto output = + loss->forward(log_probs, targets, input_lengths, target_lengths); + ASSERT_TRUE(output.ge(0).all().item()); + ASSERT_TRUE(torch::allclose( + -log_probs.sum(0).slice(1, 0, 1).view_as(output), output)); +} + +TEST_F(ModulesTest, PoissonNLLLoss) { + const auto input = torch::tensor({0.5, 1.5, 2.5}); + const auto target = torch::tensor({1., 2., 3.}); + const auto component_wise_loss = torch::exp(input) - target * input; + { + PoissonNLLLoss loss{PoissonNLLLossOptions().reduction(torch::kNone)}; + ASSERT_TRUE( + torch::allclose(component_wise_loss, loss->forward(input, target))); + } + { + PoissonNLLLoss loss{PoissonNLLLossOptions().reduction(torch::kSum)}; + ASSERT_TRUE(torch::allclose( + torch::sum(component_wise_loss), loss->forward(input, target))); + } + { + PoissonNLLLoss loss{PoissonNLLLossOptions().reduction(torch::kMean)}; + ASSERT_TRUE(torch::allclose( + torch::mean(component_wise_loss), loss->forward(input, target))); + } +} + +TEST_F(ModulesTest, MarginRankingLoss) { + { + MarginRankingLoss loss; + const auto input1 = torch::randn(15) * 10; + const auto input2 = torch::randn(15) * 10; + const auto target = torch::randn(15).sign(); + ASSERT_TRUE(torch::allclose( + loss->forward(input1, input2, target), + (-target * (input1 - input2)).clamp(0).mean())); + } + { + MarginRankingLoss loss{ + MarginRankingLossOptions().margin(0.5).reduction(torch::kSum)}; + const auto input1 = torch::randn(15) * 10; + const auto input2 = torch::randn(15) * 10; + const auto target = torch::randn(15).sign(); + const auto margin = 0.5; + ASSERT_TRUE(torch::allclose( + loss->forward(input1, input2, target), + (-target * (input1 - input2) + margin).clamp(0).sum())); + } + { + MarginRankingLoss loss{ + MarginRankingLossOptions().margin(0.5).reduction(torch::kMean)}; + const auto input1 = torch::randn(15) * 10; + const auto input2 = torch::randn(15) * 10; + const auto target = torch::randn(15).sign(); + const auto margin = 0.5; + ASSERT_TRUE(torch::allclose( + loss->forward(input1, input2, target), + (-target * (input1 - input2) + margin).clamp(0).mean())); + } +} + +TEST_F(ModulesTest, BCEWithLogitsLoss) { + { // test BCE with logits raises if target and input are different size + { + const auto target = torch::rand(5); + const auto input = torch::rand({5, 1}); + ASSERT_THROWS_WITH( + BCEWithLogitsLoss()(input, target), "must be the same as input size"); + } + { + const auto target = torch::rand({5, 1}); + const auto input = torch::rand(5); + ASSERT_THROWS_WITH( + BCEWithLogitsLoss()(input, target), "must be the same as input size"); + } + } + { // test BCE with logits gives same result as sigmoid and bce loss + auto sigmoid = Sigmoid(); + + auto target = torch::rand({64, 4}); + auto output = torch::rand({64, 4}) - 0.5; + + ASSERT_TRUE(torch::allclose( + BCEWithLogitsLoss()(output, target), + BCELoss()(sigmoid(output), target))); + + auto weight = torch::rand(4); + ASSERT_TRUE(torch::allclose( + BCEWithLogitsLoss(BCEWithLogitsLossOptions().weight(weight))( + output, target), + BCELoss(BCELossOptions().weight(weight))(sigmoid(output), target))); + + target = torch::zeros({4, 1}, torch::kFloat); + output = torch::empty({4, 1}, torch::kFloat).fill_(-100); + + ASSERT_TRUE(torch::allclose( + BCEWithLogitsLoss()(output, target), + BCELoss()(sigmoid(output), target))); + + ASSERT_TRUE(torch::allclose( + BCEWithLogitsLoss(BCEWithLogitsLossOptions().reduction(torch::kNone))( + output, target), + BCELoss(BCELossOptions().reduction(torch::kNone))( + sigmoid(output), target))); + + weight = torch::rand({1}, torch::kFloat); + ASSERT_TRUE(torch::allclose( + BCEWithLogitsLoss(BCEWithLogitsLossOptions().weight(weight))( + output, target), + BCELoss(BCELossOptions().weight(weight))(sigmoid(output), target))); + } + { // test BCE with logits has correct grad at zero + const auto output = torch::zeros({3, 1}, torch::requires_grad()); + const auto target = torch::zeros({3, 1}); + BCEWithLogitsLoss(BCEWithLogitsLossOptions().reduction(torch::kSum))( + output, target) + .backward(); + const auto expected_grad = torch::empty({3, 1}).fill_(0.5); + ASSERT_TRUE(torch::allclose(output.grad(), expected_grad)); + } + { // test BCE with logits broadcasts weights + const auto target = torch::rand({16, 4}); + const auto output = torch::rand({16, 4}) - 0.5; + + auto weight = torch::rand(4); + auto out1 = BCEWithLogitsLoss(BCEWithLogitsLossOptions().weight(weight))( + output, target); + + weight = weight.expand({16, 4}).contiguous(); + auto out2 = BCEWithLogitsLoss(BCEWithLogitsLossOptions().weight(weight))( + output, target); + + ASSERT_TRUE(torch::allclose(out1, out2)); + + weight = torch::rand({16, 1}); + out1 = BCEWithLogitsLoss(BCEWithLogitsLossOptions().weight(weight))( + output, target); + + weight = weight.expand({16, 4}).contiguous(); + out2 = BCEWithLogitsLoss(BCEWithLogitsLossOptions().weight(weight))( + output, target); + + ASSERT_TRUE(torch::allclose(out1, out2)); + } + { // test BCE with logits ones in pos weights are the same as none + const auto target = torch::rand({64, 4}); + const auto output = torch::rand({64, 4}) - 0.5; + const auto pos_weight = torch::ones({64, 4}); + + ASSERT_TRUE(torch::allclose( + BCEWithLogitsLoss()(output, target), + BCEWithLogitsLoss(BCEWithLogitsLossOptions().pos_weight(pos_weight))( + output, target))); + } + { // test BCE with logits broadcasts pos weights + const auto target = torch::rand({64, 4}); + const auto output = torch::rand({64, 4}) - 0.5; + const auto pos_weight = torch::rand(4); + const auto out1 = BCEWithLogitsLoss( + BCEWithLogitsLossOptions().pos_weight(pos_weight))(output, target); + + const auto pos_weight1 = pos_weight.expand({1, 4}); + const auto out2 = BCEWithLogitsLoss( + BCEWithLogitsLossOptions().pos_weight(pos_weight))(output, target); + + const auto pos_weight2 = pos_weight.expand({64, 4}); + const auto out3 = BCEWithLogitsLoss( + BCEWithLogitsLossOptions().pos_weight(pos_weight))(output, target); + + ASSERT_TRUE(torch::allclose(out1, out2)); + ASSERT_TRUE(torch::allclose(out1, out3)); + } + { // test BCE with logits with pos weight has correct grad at zero + const auto output = torch::zeros({3, 1}, torch::requires_grad()); + const auto target = torch::zeros({3, 1}); + const auto pos_weight = torch::ones({3, 1}); + BCEWithLogitsLoss(BCEWithLogitsLossOptions() + .pos_weight(pos_weight) + .reduction(torch::kSum))(output, target) + .backward(); + const auto expected_grad = torch::empty({3, 1}).fill_(0.5); + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + const auto grad = output.grad(); + ASSERT_TRUE(torch::allclose(grad, expected_grad)); + } + { // test BCE with logits stability + const auto output = torch::tensor({0., -120.}); + const auto target = torch::tensor({0., 1.}); + const auto pos_weight = torch::tensor({1., 1.}); + + const auto out1 = BCEWithLogitsLoss()(output, target); + ASSERT_TRUE(torch::isfinite(out1).all().item()); + + const auto out2 = BCEWithLogitsLoss( + BCEWithLogitsLossOptions().pos_weight(pos_weight))(output, target); + ASSERT_TRUE(torch::isfinite(out2).all().item()); + } +} + +namespace detail { + +namespace F = torch::nn::functional; + +torch::Tensor _batchmatmul(const torch::Tensor& a, const torch::Tensor& b) { + TORCH_INTERNAL_ASSERT(a.size(0) == b.size(0)); + TORCH_INTERNAL_ASSERT(a.size(1) == b.size(1)); + auto retval = torch::zeros( + {a.size(0), a.size(1), a.size(2), b.size(3)}, torch::kFloat32); + for (const auto i : c10::irange(a.size(0))) { + for (const auto j : c10::irange(a.size(1))) { + retval[i][j] = torch::matmul(a[i][j], b[i][j]); + } + } + return retval; +} + +torch::Tensor _softmax(const torch::Tensor& x) { + auto output = torch::zeros(x.sizes()); + for (const auto i : c10::irange(x.size(0))) { + for (const auto j : c10::irange(x.size(1))) { + for (const auto k : c10::irange(x.size(2))) { + const auto& x_curr = x[i][j][k]; + const auto e_x = torch::exp(x_curr - torch::max(x_curr)); + output[i][j][k] = e_x / torch::sum(e_x); + } + } + } + return output; +} + +std::tuple _scaled_dot_attn_ref( + const torch::Tensor& Q, + const torch::Tensor& K, + const torch::Tensor& V, + at::IntArrayRef dims, + const torch::Tensor& unseen_mask = {}, + const torch::Tensor& key_padding_mask = {}, + bool average_attn_weights = true) { + auto QKT = _batchmatmul(Q, K.permute({0, 1, 3, 2}) / std::sqrt(dims[3])); + const auto b1 = QKT.size(0); + const auto b2 = QKT.size(1); + const auto s1 = QKT.size(2); + const auto s2 = QKT.size(3); + if (unseen_mask.defined() || key_padding_mask.defined()) { + for (const auto i : c10::irange(b1)) { + for (const auto j : c10::irange(b2)) { + for (const auto m : c10::irange(s1)) { + for (const auto n : c10::irange(s2)) { + if (unseen_mask.defined() && + unseen_mask[m][n].item() == 0) { + QKT[i][j][m][n] = -std::numeric_limits::infinity(); + } + if (key_padding_mask.defined() && + key_padding_mask[i][n].item() != 0) { + QKT[i][j][m][n] = -std::numeric_limits::infinity(); + } + } + } + } + } + } + auto reference = _softmax(QKT); + auto ref_attn_weight = reference; + if (average_attn_weights) { + // NOLINTNEXTLINE(bugprone-argument-comment) + ref_attn_weight = torch::sum(ref_attn_weight, /*axis=*/1) / b2; + } + reference = _batchmatmul(reference, V); + return std::tie(reference, ref_attn_weight); +} + +torch::Tensor _split_heads_ref( + const torch::Tensor& X, + at::IntArrayRef dims, + int nheads, + int d_head) { + auto X_split = X.reshape({dims[0], dims[1], nheads, d_head}); + auto X_split_transposed = X_split.permute({0, 2, 1, 3}); + return X_split_transposed.reshape({dims[0], nheads, dims[1], d_head}); +} + +torch::Tensor _combine_heads_ref( + const torch::Tensor& X, + at::IntArrayRef dims, + int nheads, + int d_head) { + auto X_transposed = X.permute({0, 2, 1, 3}); + auto reference = X_transposed.reshape({dims[0], dims[1], nheads * d_head}); + return reference; +} + +torch::Tensor _fc( + torch::Tensor X, + torch::Tensor X_weight, + torch::Tensor X_bias) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto X_fc_b = X_bias; + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto X_fc_w = X_weight; + return torch::matmul(X, torch::t(X_fc_w)) + X_fc_b; +} + +void _multihead_attn_test_helper( + bool add_key_padding_mask = false, + bool add_bias_kv = false, + bool add_zero_attn = false, + bool saved_kv = false, + bool same_embed_dim = false, + bool average_attn_weights = true) { + std::random_device device; + std::mt19937 generator(device()); + std::uniform_int_distribution d_2_10(2, 10); + std::uniform_int_distribution d_3_10(3, 10); + bool registration_checked = false; + for ([[maybe_unused]] const auto i : c10::irange(100)) { + const auto batch_sz = d_2_10(generator); + const auto seq_len = d_2_10(generator); + const auto d_head = d_3_10(generator); + const auto nheads = d_3_10(generator); + const auto d_model = d_head * nheads; + // NOLINTNEXTLINE(cppcoreguidelines-init-variables) + int kv_dim; + if (same_embed_dim) { + kv_dim = d_model; + } else { + std::uniform_int_distribution d(5, 20); + kv_dim = d(generator); + while (kv_dim == d_model) { + kv_dim = d(generator); + } + } + std::vector dims{batch_sz, seq_len, kv_dim}; + torch::Tensor saved_k; + torch::Tensor saved_k_tensor; + torch::Tensor saved_v; + torch::Tensor saved_v_tensor; + if (saved_kv) { + saved_k = torch::rand({batch_sz * nheads, seq_len, d_head}); + saved_k_tensor = saved_k; + saved_v = torch::rand({batch_sz * nheads, seq_len, d_head}); + saved_v_tensor = saved_v; + } + torch::Tensor key_padding_mask; + torch::Tensor key_padding_mask_tensor; + if (add_key_padding_mask) { + const auto seq_mask = torch::randint(0, 2, {1, seq_len}); + key_padding_mask = seq_mask.repeat({batch_sz, 1}) == 1; + key_padding_mask_tensor = key_padding_mask; + } + const auto decoder_state = torch::rand({batch_sz, d_model}); + const torch::Tensor K = torch::rand(dims); + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + const torch::Tensor V = K; + const torch::Tensor Q = + decoder_state.clone().resize_({batch_sz, 1, d_model}); + auto attn_mask = torch::randint(0, 2, {1, seq_len}, torch::kFloat); + const torch::Tensor attn_mask_tensor = attn_mask.clone(); + attn_mask_tensor.masked_fill_( + attn_mask_tensor == 0, -std::numeric_limits::infinity()); + attn_mask_tensor.masked_fill_(attn_mask_tensor > 0, double(0.0)); + + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + const torch::Tensor decoder_state_tensor = decoder_state; + const torch::Tensor source_hid_tensor = K.transpose(0, 1); + + const auto options = MultiheadAttentionOptions(d_model, nheads) + .add_bias_kv(add_bias_kv) + .add_zero_attn(add_zero_attn) + .kdim(kv_dim) + .vdim(kv_dim); + const auto multihead_attn_module = MultiheadAttention(options); + + if (!registration_checked) { + // make sure parameters are all registered correctly + auto named_parameters = multihead_attn_module->named_parameters(); + if (same_embed_dim) { + ASSERT_TRUE(named_parameters.contains("in_proj_weight")); + } else { + ASSERT_TRUE(named_parameters.contains("q_proj_weight")); + ASSERT_TRUE(named_parameters.contains("k_proj_weight")); + ASSERT_TRUE(named_parameters.contains("v_proj_weight")); + } + if (add_bias_kv) { + ASSERT_TRUE(named_parameters.contains("bias_k")); + ASSERT_TRUE(named_parameters.contains("bias_v")); + } + // make sure sub modules are all registered correctly + auto submodules = multihead_attn_module->named_children(); + ASSERT_TRUE(submodules.contains("out_proj")); + registration_checked = true; + } + + torch::Tensor bias_k; + torch::Tensor bias_v; + if (add_bias_kv) { + bias_k = multihead_attn_module->bias_k.detach(); + bias_v = multihead_attn_module->bias_v.detach(); + } else { + bias_k.reset(); + bias_v.reset(); + } + + torch::Tensor _Q = decoder_state_tensor.unsqueeze(1).transpose(0, 1); + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + torch::Tensor _V = source_hid_tensor; + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + torch::Tensor _K = source_hid_tensor; + + torch::Tensor result; + torch::Tensor result_weight; + if (multihead_attn_module->_qkv_same_embed_dim) { + std::tie(result, result_weight) = F::multi_head_attention_forward( + _Q, + _K, + _V, + F::MultiheadAttentionForwardFuncOptions( + /*embed_dim_to_check=*/d_model, + /*num_heads=*/nheads, + /*in_proj_weight=*/multihead_attn_module->in_proj_weight, + /*in_proj_bias=*/multihead_attn_module->in_proj_bias, + /*bias_k=*/multihead_attn_module->bias_k, + /*bias_v=*/multihead_attn_module->bias_v, + /*add_zero_attn=*/multihead_attn_module->options.add_zero_attn(), + /*dropout_p=*/multihead_attn_module->options.dropout(), + /*out_proj_weight=*/multihead_attn_module->out_proj->weight, + /*out_proj_bias=*/multihead_attn_module->out_proj->bias) + .training(multihead_attn_module->is_training()) + .key_padding_mask(key_padding_mask_tensor) + .need_weights(true) + .attn_mask(attn_mask_tensor) + .static_k(saved_k_tensor) + .static_v(saved_v_tensor) + .average_attn_weights(average_attn_weights)); + } else { + std::tie(result, result_weight) = F::multi_head_attention_forward( + _Q, + _K, + _V, + F::MultiheadAttentionForwardFuncOptions( + /*embed_dim_to_check=*/d_model, + /*num_heads=*/nheads, + /*in_proj_weight=*/{}, + /*in_proj_bias=*/multihead_attn_module->in_proj_bias, + /*bias_k=*/multihead_attn_module->bias_k, + /*bias_v=*/multihead_attn_module->bias_v, + /*add_zero_attn=*/multihead_attn_module->options.add_zero_attn(), + /*dropout_p=*/multihead_attn_module->options.dropout(), + /*out_proj_weight=*/multihead_attn_module->out_proj->weight, + /*out_proj_bias=*/multihead_attn_module->out_proj->bias) + .training(multihead_attn_module->is_training()) + .key_padding_mask(key_padding_mask_tensor) + .need_weights(true) + .attn_mask(attn_mask_tensor) + .use_separate_proj_weight(true) + .q_proj_weight(multihead_attn_module->q_proj_weight) + .k_proj_weight(multihead_attn_module->k_proj_weight) + .v_proj_weight(multihead_attn_module->v_proj_weight) + .static_k(saved_k_tensor) + .static_v(saved_v_tensor) + .average_attn_weights(average_attn_weights)); + } + result = result.squeeze(0).detach(); + torch::Tensor q_proj_weight; + torch::Tensor k_proj_weight; + torch::Tensor v_proj_weight; + if (multihead_attn_module->_qkv_same_embed_dim) { + q_proj_weight = + multihead_attn_module->in_proj_weight.slice(/*dim=*/0, 0, d_model); + k_proj_weight = multihead_attn_module->in_proj_weight.slice( + /*dim=*/0, d_model, (d_model * 2)); + v_proj_weight = + multihead_attn_module->in_proj_weight.slice(/*dim=*/0, (d_model * 2)); + } else { + q_proj_weight = multihead_attn_module->q_proj_weight; + k_proj_weight = multihead_attn_module->k_proj_weight; + v_proj_weight = multihead_attn_module->v_proj_weight; + } + auto Q_fc = + _fc(Q, + q_proj_weight, + multihead_attn_module->in_proj_bias.slice(/*dim=*/0, 0, d_model)); + auto K_fc = + _fc(K, + k_proj_weight, + multihead_attn_module->in_proj_bias.slice( + /*dim=*/0, d_model, (d_model * 2))); + auto V_fc = _fc( + V, + v_proj_weight, + multihead_attn_module->in_proj_bias.slice(/*dim=*/0, (d_model * 2))); + + if (add_bias_kv) { + K_fc = torch::cat( + {K_fc, + bias_k.repeat({K_fc.size(0) / bias_k.size(0), 1, 1} /*, axis=0*/)}, + /*dim=*/1); + V_fc = torch::cat( + {V_fc, + bias_v.repeat({V_fc.size(0) / bias_v.size(0), 1, 1} /*, axis=0*/)}, + /*dim=*/1); + if (attn_mask.defined()) { + attn_mask = torch::cat({attn_mask, torch::ones({1, 1})}, /*dim=*/1); + } + if (key_padding_mask.defined()) { + key_padding_mask = torch::cat( + {key_padding_mask, torch::full({batch_sz, 1}, false, torch::kBool)}, + /*dim=*/1); + } + dims[1] += 1; + } + const auto Q_split = + _split_heads_ref(Q_fc, {batch_sz, 1, d_model}, nheads, d_head); + torch::Tensor K_split; + if (saved_k.defined()) { + K_split = saved_k.reshape({dims[0], nheads, dims[1], d_head}); + } else { + K_split = _split_heads_ref(K_fc, dims, nheads, d_head); + } + torch::Tensor V_split; + if (saved_v.defined()) { + V_split = saved_v.reshape({dims[0], nheads, dims[1], d_head}); + } else { + V_split = _split_heads_ref(V_fc, dims, nheads, d_head); + } + if (add_zero_attn) { + dims[1] += 1; + K_split = torch::cat( + {K_split, + torch::zeros( + {K_split.size(0), K_split.size(1), 1, K_split.size(3)})}, + /*dim=*/2); + V_split = torch::cat( + {V_split, + torch::zeros( + {V_split.size(0), V_split.size(1), 1, V_split.size(3)})}, + /*dim=*/2); + if (attn_mask.defined()) { + attn_mask = torch::cat({attn_mask, torch::ones({1, 1})}, /*dim=*/1); + } + if (key_padding_mask.defined()) { + key_padding_mask = torch::cat( + {key_padding_mask, torch::full({batch_sz, 1}, false, torch::kBool)}, + /*dim=*/1); + } + } + auto [attn_heads, ref_attn_weight] = _scaled_dot_attn_ref( + Q_split, + K_split, + V_split, + Q_split.sizes(), + attn_mask, + key_padding_mask, + average_attn_weights); + const auto combined_attn_heads = + _combine_heads_ref(attn_heads, {batch_sz, 1}, nheads, d_head); + auto reference = + _fc(combined_attn_heads, + multihead_attn_module->out_proj->weight, + multihead_attn_module->out_proj->bias); + // NOLINTNEXTLINE(bugprone-argument-comment) + reference = torch::squeeze(reference, /*axis=*/1); + + // result = reference + ASSERT_EQ(result.sizes(), std::vector({batch_sz, d_model})); + ASSERT_TRUE( + torch::allclose(result, reference, 1e-5, 1e-5, /*equal_nan=*/true)); + + // result_weight = ref_attn_weight + result_weight = result_weight.detach(); + ASSERT_EQ(result_weight.sizes(), ref_attn_weight.sizes()); + ASSERT_TRUE(torch::allclose( + result_weight, ref_attn_weight, 1e-5, 1e-5, /*equal_nan=*/true)); + } +} +} // namespace detail + +TEST_F(ModulesTest, MultiheadAttention) { + using namespace ::detail; + + for (auto average_attn_weights : {false, true}) { + // test_multihead_attn_add_zero_attn + _multihead_attn_test_helper( + /*add_key_padding_mask=*/false, + /*add_bias_kv=*/false, + /*add_zero_attn=*/true, + /*saved_kv=*/false, + /*same_embed_dim=*/false, + /*average_attn_weights=*/average_attn_weights); + + // test_multihead_attn_add_bias_kv + _multihead_attn_test_helper( + /*add_key_padding_mask=*/false, + /*add_bias_kv=*/true, + /*add_zero_attn=*/false, + /*saved_kv=*/false, + /*same_embed_dim=*/false, + /*average_attn_weights=*/average_attn_weights); + + // test_multihead_attn_no_masking(): + _multihead_attn_test_helper(); + + // test_multihead_attn_key_padding_mask + _multihead_attn_test_helper( + /*add_key_padding_mask=*/true, + /*add_bias_kv=*/false, + /*add_zero_attn=*/false, + /*saved_kv=*/false, + /*same_embed_dim=*/false, + /*average_attn_weights=*/average_attn_weights); + + // test_multihead_attn_saved_kv + _multihead_attn_test_helper( + /*add_key_padding_mask=*/false, + /*add_bias_kv=*/false, + /*add_zero_attn=*/false, + /*saved_kv=*/true, + /*same_embed_dim=*/false, + /*average_attn_weights=*/average_attn_weights); + + // test_multihead_attn_add_bias_kv_zero_attn + _multihead_attn_test_helper( + /*add_key_padding_mask=*/true, + /*add_bias_kv=*/true, + /*add_zero_attn=*/true, + /*saved_kv=*/false, + /*same_embed_dim=*/false, + /*average_attn_weights=*/average_attn_weights); + + // test_multihead_attn_all_arguments1 + _multihead_attn_test_helper( + /*add_key_padding_mask=*/true, + /*add_bias_kv=*/false, + /*add_zero_attn=*/true, + /*saved_kv=*/true, + /*same_embed_dim=*/false, + /*average_attn_weights=*/average_attn_weights); + + ASSERT_THROWS_WITH( + // test_multihead_attn_all_arguments2 + _multihead_attn_test_helper( + /*add_key_padding_mask=*/true, + /*add_bias_kv=*/true, + /*add_zero_attn=*/true, + /*saved_kv=*/true, + /*same_embed_dim=*/false, + /*average_attn_weights=*/average_attn_weights), + "bias cannot be added to static key"); + + // test_multihead_attn_all_arguments3 + _multihead_attn_test_helper( + /*add_key_padding_mask=*/true, + /*add_bias_kv=*/false, + /*add_zero_attn=*/true, + /*saved_kv=*/true, + /*same_embed_dim=*/true, + /*average_attn_weights=*/average_attn_weights); + } +} + +TEST_F(ModulesTest, PrettyPrintIdentity) { + ASSERT_EQ(c10::str(Identity()), "torch::nn::Identity()"); +} + +TEST_F(ModulesTest, PrettyPrintFlatten) { + ASSERT_EQ(c10::str(Flatten()), "torch::nn::Flatten(start_dim=1, end_dim=-1)"); + ASSERT_EQ( + c10::str(Flatten(FlattenOptions().start_dim(2).end_dim(4))), + "torch::nn::Flatten(start_dim=2, end_dim=4)"); +} + +TEST_F(ModulesTest, PrettyPrintUnflatten) { + ASSERT_EQ( + c10::str(Unflatten(UnflattenOptions(0, {2, 2}))), + "torch::nn::Unflatten(dim=0, unflattened_size={2, 2})"); +} + +TEST_F(ModulesTest, ReflectionPad1d) { + { + ReflectionPad1d m(ReflectionPad1dOptions(2)); + auto input = torch::arange(8, torch::kFloat).reshape({1, 2, 4}); + auto output = m(input); + auto expected = torch::tensor( + {{{2., 1., 0., 1., 2., 3., 2., 1.}, {6., 5., 4., 5., 6., 7., 6., 5.}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ReflectionPad1d m(ReflectionPad1dOptions({3, 1})); + auto input = torch::arange(8, torch::kFloat).reshape({1, 2, 4}); + auto output = m(input); + auto expected = torch::tensor( + {{{3., 2., 1., 0., 1., 2., 3., 2.}, {7., 6., 5., 4., 5., 6., 7., 6.}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(ModulesTest, ReflectionPad2d) { + { + ReflectionPad2d m(ReflectionPad2dOptions(2)); + auto input = torch::arange(9, torch::kFloat).reshape({1, 1, 3, 3}); + auto output = m(input); + auto expected = torch::tensor( + {{{{8., 7., 6., 7., 8., 7., 6.}, + {5., 4., 3., 4., 5., 4., 3.}, + {2., 1., 0., 1., 2., 1., 0.}, + {5., 4., 3., 4., 5., 4., 3.}, + {8., 7., 6., 7., 8., 7., 6.}, + {5., 4., 3., 4., 5., 4., 3.}, + {2., 1., 0., 1., 2., 1., 0.}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ReflectionPad2d m(ReflectionPad2dOptions({1, 1, 2, 0})); + auto input = torch::arange(9, torch::kFloat).reshape({1, 1, 3, 3}); + auto output = m(input); + auto expected = torch::tensor( + {{{{7., 6., 7., 8., 7.}, + {4., 3., 4., 5., 4.}, + {1., 0., 1., 2., 1.}, + {4., 3., 4., 5., 4.}, + {7., 6., 7., 8., 7.}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(ModulesTest, ReflectionPad3d) { + { + ReflectionPad3d m(ReflectionPad3dOptions(1)); + auto input = torch::arange(8, torch::kFloat).reshape({1, 1, 2, 2, 2}); + auto output = m(input); + auto expected = torch::tensor( + {{{{{7., 6., 7., 6.}, + {5., 4., 5., 4.}, + {7., 6., 7., 6.}, + {5., 4., 5., 4.}}, + {{3., 2., 3., 2.}, + {1., 0., 1., 0.}, + {3., 2., 3., 2.}, + {1., 0., 1., 0.}}, + {{7., 6., 7., 6.}, + {5., 4., 5., 4.}, + {7., 6., 7., 6.}, + {5., 4., 5., 4.}}, + {{3., 2., 3., 2.}, + {1., 0., 1., 0.}, + {3., 2., 3., 2.}, + {1., 0., 1., 0.}}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ReflectionPad3d m(ReflectionPad3dOptions({0, 1, 1, 0, 1, 2})); + auto input = torch::arange(16, torch::kFloat).reshape({1, 1, 4, 2, 2}); + auto output = m(input); + auto expected = torch::tensor( + {{{{{6., 7., 6.}, {4., 5., 4.}, {6., 7., 6.}}, + {{2., 3., 2.}, {0., 1., 0.}, {2., 3., 2.}}, + {{6., 7., 6.}, {4., 5., 4.}, {6., 7., 6.}}, + {{10., 11., 10.}, {8., 9., 8.}, {10., 11., 10.}}, + {{14., 15., 14.}, {12., 13., 12.}, {14., 15., 14.}}, + {{10., 11., 10.}, {8., 9., 8.}, {10., 11., 10.}}, + {{6., 7., 6.}, {4., 5., 4.}, {6., 7., 6.}}}}}, + torch::kFloat); + ASSERT_EQ(output.sizes(), std::vector({1, 1, 7, 3, 3})); + ASSERT_TRUE(output.allclose(expected)); + } +} +TEST_F(ModulesTest, ReplicationPad1d) { + { + ReplicationPad1d m(ReplicationPad1dOptions(2)); + auto input = torch::arange(8, torch::kFloat).reshape({1, 2, 4}); + auto output = m(input); + auto expected = torch::tensor( + {{{0., 0., 0., 1., 2., 3., 3., 3.}, {4., 4., 4., 5., 6., 7., 7., 7.}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ReplicationPad1d m(ReplicationPad1dOptions({3, 1})); + auto input = torch::arange(8, torch::kFloat).reshape({1, 2, 4}); + auto output = m(input); + auto expected = torch::tensor( + {{{0., 0., 0., 0., 1., 2., 3., 3.}, {4., 4., 4., 4., 5., 6., 7., 7.}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(ModulesTest, ReplicationPad2d) { + { + ReplicationPad2d m(ReplicationPad2dOptions(2)); + auto input = torch::arange(9, torch::kFloat).reshape({1, 1, 3, 3}); + auto output = m(input); + auto expected = torch::tensor( + {{{{0., 0., 0., 1., 2., 2., 2.}, + {0., 0., 0., 1., 2., 2., 2.}, + {0., 0., 0., 1., 2., 2., 2.}, + {3., 3., 3., 4., 5., 5., 5.}, + {6., 6., 6., 7., 8., 8., 8.}, + {6., 6., 6., 7., 8., 8., 8.}, + {6., 6., 6., 7., 8., 8., 8.}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ReplicationPad2d m(ReplicationPad2dOptions({1, 1, 2, 0})); + auto input = torch::arange(9, torch::kFloat).reshape({1, 1, 3, 3}); + auto output = m(input); + auto expected = torch::tensor( + {{{{0., 0., 1., 2., 2.}, + {0., 0., 1., 2., 2.}, + {0., 0., 1., 2., 2.}, + {3., 3., 4., 5., 5.}, + {6., 6., 7., 8., 8.}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(ModulesTest, ReplicationPad3d) { + { + ReplicationPad3d m(ReplicationPad3dOptions(1)); + auto input = torch::arange(8, torch::kFloat).reshape({1, 1, 2, 2, 2}); + auto output = m(input); + auto expected = torch::tensor( + {{{{{0., 0., 1., 1.}, + {0., 0., 1., 1.}, + {2., 2., 3., 3.}, + {2., 2., 3., 3.}}, + {{0., 0., 1., 1.}, + {0., 0., 1., 1.}, + {2., 2., 3., 3.}, + {2., 2., 3., 3.}}, + {{4., 4., 5., 5.}, + {4., 4., 5., 5.}, + {6., 6., 7., 7.}, + {6., 6., 7., 7.}}, + {{4., 4., 5., 5.}, + {4., 4., 5., 5.}, + {6., 6., 7., 7.}, + {6., 6., 7., 7.}}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ReplicationPad3d m(ReplicationPad3dOptions({1, 2, 1, 2, 1, 2})); + auto input = torch::arange(8, torch::kFloat).reshape({1, 1, 2, 2, 2}); + auto output = m(input); + auto expected = torch::tensor( + {{{{{0., 0., 1., 1., 1.}, + {0., 0., 1., 1., 1.}, + {2., 2., 3., 3., 3.}, + {2., 2., 3., 3., 3.}, + {2., 2., 3., 3., 3.}}, + {{0., 0., 1., 1., 1.}, + {0., 0., 1., 1., 1.}, + {2., 2., 3., 3., 3.}, + {2., 2., 3., 3., 3.}, + {2., 2., 3., 3., 3.}}, + {{4., 4., 5., 5., 5.}, + {4., 4., 5., 5., 5.}, + {6., 6., 7., 7., 7.}, + {6., 6., 7., 7., 7.}, + {6., 6., 7., 7., 7.}}, + {{4., 4., 5., 5., 5.}, + {4., 4., 5., 5., 5.}, + {6., 6., 7., 7., 7.}, + {6., 6., 7., 7., 7.}, + {6., 6., 7., 7., 7.}}, + {{4., 4., 5., 5., 5.}, + {4., 4., 5., 5., 5.}, + {6., 6., 7., 7., 7.}, + {6., 6., 7., 7., 7.}, + {6., 6., 7., 7., 7.}}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(ModulesTest, ZeroPad1d) { + { + ZeroPad1d m(ZeroPad1dOptions(2)); + auto input = torch::arange(8, torch::kFloat).reshape({1, 2, 4}); + auto output = m(input); + auto expected = torch::tensor( + {{{0., 0., 0., 1., 2., 3., 0., 0.}, {0., 0., 4., 5., 6., 7., 0., 0.}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ZeroPad1d m(ZeroPad1dOptions({3, 1})); + auto input = torch::arange(6, torch::kFloat).reshape({1, 2, 3}); + auto output = m(input); + auto expected = torch::tensor( + {{{0., 0., 0., 0., 1., 2., 0.}, {0., 0., 0., 3., 4., 5., 0.}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(ModulesTest, ZeroPad2d) { + { + ZeroPad2d m(ZeroPad2dOptions(2)); + auto input = torch::arange(9, torch::kFloat).reshape({1, 1, 3, 3}); + auto output = m(input); + auto expected = torch::tensor( + {{{{0., 0., 0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0., 0., 0.}, + {0., 0., 0., 1., 2., 0., 0.}, + {0., 0., 3., 4., 5., 0., 0.}, + {0., 0., 6., 7., 8., 0., 0.}, + {0., 0., 0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0., 0., 0.}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ZeroPad2d m(ZeroPad2dOptions({1, 1, 2, 0})); + auto input = torch::arange(9, torch::kFloat).reshape({1, 1, 3, 3}); + auto output = m(input); + auto expected = torch::tensor( + {{{{0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 1., 2., 0.}, + {0., 3., 4., 5., 0.}, + {0., 6., 7., 8., 0.}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(ModulesTest, ZeroPad3d) { + { + ZeroPad3d m(ZeroPad3dOptions(1)); + auto input = torch::arange(8, torch::kFloat).reshape({1, 1, 2, 2, 2}); + auto output = m(input); + auto expected = torch::tensor( + {{{{{0., 0., 0., 0.}, + {0., 0., 0., 0.}, + {0., 0., 0., 0.}, + {0., 0., 0., 0.}}, + {{0., 0., 0., 0.}, + {0., 0., 1., 0.}, + {0., 2., 3., 0.}, + {0., 0., 0., 0.}}, + {{0., 0., 0., 0.}, + {0., 4., 5., 0.}, + {0., 6., 7., 0.}, + {0., 0., 0., 0.}}, + {{0., 0., 0., 0.}, + {0., 0., 0., 0.}, + {0., 0., 0., 0.}, + {0., 0., 0., 0.}}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ZeroPad3d m(ZeroPad3dOptions({1, 2, 1, 2, 1, 2})); + auto input = torch::arange(8, torch::kFloat).reshape({1, 1, 2, 2, 2}); + auto output = m(input); + auto expected = torch::tensor( + {{{{{0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}}, + {{0., 0., 0., 0., 0.}, + {0., 0., 1., 0., 0.}, + {0., 2., 3., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}}, + {{0., 0., 0., 0., 0.}, + {0., 4., 5., 0., 0.}, + {0., 6., 7., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}}, + {{0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}}, + {{0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}, + {0., 0., 0., 0., 0.}}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(ModulesTest, ConstantPad1d) { + { + ConstantPad1d m(ConstantPad1dOptions(2, 3.5)); + auto input = torch::arange(8, torch::kFloat).reshape({1, 2, 4}); + auto output = m(input); + auto expected = torch::tensor( + {{{3.5000, 3.5000, 0.0000, 1.0000, 2.0000, 3.0000, 3.5000, 3.5000}, + {3.5000, 3.5000, 4.0000, 5.0000, 6.0000, 7.0000, 3.5000, 3.5000}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ConstantPad1d m(ConstantPad1dOptions({3, 1}, 3.5)); + auto input = torch::arange(6, torch::kFloat).reshape({1, 2, 3}); + auto output = m(input); + auto expected = torch::tensor( + {{{3.5000, 3.5000, 3.5000, 0.0000, 1.0000, 2.0000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.0000, 4.0000, 5.0000, 3.5000}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(ModulesTest, ConstantPad2d) { + { + ConstantPad2d m(ConstantPad2dOptions(2, 3.5)); + auto input = torch::arange(4, torch::kFloat).reshape({1, 2, 2}); + auto output = m(input); + auto expected = torch::tensor( + {{{3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 0.0000, 1.0000, 3.5000, 3.5000}, + {3.5000, 3.5000, 2.0000, 3.0000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ConstantPad2d m(ConstantPad2dOptions({3, 0, 2, 1}, 3.5)); + auto input = torch::arange(4, torch::kFloat).reshape({1, 2, 2}); + auto output = m(input); + auto expected = torch::tensor( + {{{3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 0.0000, 1.0000}, + {3.5000, 3.5000, 3.5000, 2.0000, 3.0000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(ModulesTest, ConstantPad3d) { + { + ConstantPad3d m(ConstantPad3dOptions(1, 3.5)); + auto input = torch::arange(8, torch::kFloat).reshape({1, 1, 2, 2, 2}); + auto output = m(input); + auto expected = torch::tensor( + {{{{{3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000}}, + {{3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 0.0000, 1.0000, 3.5000}, + {3.5000, 2.0000, 3.0000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000}}, + {{3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 4.0000, 5.0000, 3.5000}, + {3.5000, 6.0000, 7.0000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000}}, + {{3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000}}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } + { + ConstantPad3d m(ConstantPad3dOptions({1, 2, 1, 2, 1, 2}, 3.5)); + auto input = torch::arange(8, torch::kFloat).reshape({1, 1, 2, 2, 2}); + auto output = m(input); + auto expected = torch::tensor( + {{{{{3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}}, + {{3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 0.0000, 1.0000, 3.5000, 3.5000}, + {3.5000, 2.0000, 3.0000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}}, + {{3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 4.0000, 5.0000, 3.5000, 3.5000}, + {3.5000, 6.0000, 7.0000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}}, + {{3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}}, + {{3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}, + {3.5000, 3.5000, 3.5000, 3.5000, 3.5000}}}}}, + torch::kFloat); + ASSERT_TRUE(output.allclose(expected)); + } +} + +TEST_F(ModulesTest, CrossMapLRN2d) { + /// size 3, default options + auto input = + torch::arange(9, torch::kFloat32).view({1, 1, 3, 3}).requires_grad_(true); + auto expected = torch::tensor( + {{{{0.00000000, 0.99997497, 1.99980010}, + {2.99932500, 3.99840070, 4.99687700}, + {5.99460600, 6.99143740, 7.98722360}}}}, + torch::kFloat32); + auto grad_expected = torch::tensor( + {{{{1.00000000, 0.99992496, 0.99970007}, + {0.99932520, 0.99880093, 0.99812720}, + {0.99730474, 0.99633380, 0.99521490}}}}, + torch::kFloat32); + auto crossmaplrn2d = CrossMapLRN2d(3); + auto output = crossmaplrn2d(input); + output.sum().backward(); + + ASSERT_TRUE(input.grad().allclose(grad_expected)); + ASSERT_TRUE(output.allclose(expected)); + + /// size change + crossmaplrn2d = + CrossMapLRN2d(CrossMapLRN2dOptions(4).alpha(1e-4).beta(0.75).k(1)); + output = crossmaplrn2d(input); + expected = torch::tensor( + {{{{0.00000000, 0.99998120, 1.99985000}, + {2.99949400, 3.99880050, 4.99765800}, + {5.99595300, 6.99357600, 7.99041300}}}}, + torch::kFloat32); + ASSERT_TRUE(output.allclose(expected)); + + /// alpha change + crossmaplrn2d = + CrossMapLRN2d(CrossMapLRN2dOptions(3).alpha(1e-3).beta(0.75).k(1)); + output = crossmaplrn2d(input); + expected = torch::tensor( + {{{{0.00000000, 0.99975010, 1.99800230}, + {2.99326750, 3.98407440, 4.96897600}, + {5.94656100, 6.91545720, 7.87434340}}}}, + torch::kFloat32); + ASSERT_TRUE(output.allclose(expected)); + + /// beta change + crossmaplrn2d = + CrossMapLRN2d(CrossMapLRN2dOptions(3).alpha(1e-4).beta(0.95).k(1)); + output = crossmaplrn2d(input); + expected = torch::tensor( + {{{{0.00000000, 0.99996830, 1.99974680}, + {2.99914500, 3.99797440, 4.99604460}, + {5.99316840, 6.98915600, 7.98382000}}}}, + torch::kFloat32); + ASSERT_TRUE(output.allclose(expected)); + + /// k change + crossmaplrn2d = + CrossMapLRN2d(CrossMapLRN2dOptions(3).alpha(1e-4).beta(0.75).k(2)); + output = crossmaplrn2d(input); + expected = torch::tensor( + {{{{0.00000000, 0.59459610, 1.18914770}, + {1.78361000, 2.37793870, 2.97208900}, + {3.56601700, 4.15967700, 4.75302650}}}}, + torch::kFloat32); + ASSERT_TRUE(output.allclose(expected)); +} + +TEST_F(ModulesTest, RNNCell) { + torch::manual_seed(0); + auto rnn = RNNCell(1, 2); + + auto input = torch::randn({3, 1}); + auto hx = torch::randn({3, 2}); + auto output = rnn(input, hx); + auto expected = + torch::tensor({{-0.5078, 0.4380}, {-0.7215, 0.2969}, {-0.1304, 0.0653}}); + ASSERT_TRUE(torch::allclose(output, expected, 1e-05, 2e-04)); + + output = rnn(input); + expected = + torch::tensor({{-0.0775, 0.6688}, {-0.0734, 0.4759}, {-0.0725, 0.4225}}); + ASSERT_TRUE(torch::allclose(output, expected, 1e-05, 2e-04)); + + input = torch::randn({1}); + hx = torch::randn({2}); + output = rnn(input, hx); + expected = torch::tensor({0.2808, 0.6505}); + ASSERT_TRUE(torch::allclose(output, expected, 1e-05, 2e-04)); + + { + auto input = torch::randn({3, 2}); + auto hx = torch::randn({3, 2}); + ASSERT_THROWS_WITH( + rnn(input, hx), "input has inconsistent input_size: got 2 expected 1"); + } + + { + auto input = torch::randn({3, 1}); + auto hx = torch::randn({3, 1}); + ASSERT_THROWS_WITH( + rnn(input, hx), + "hidden0 has inconsistent hidden_size: got 1, expected 2"); + } + + { + auto input = torch::randn({3, 1, 1, 1, 1}); + auto hx = torch::randn({3, 2}); + ASSERT_THROWS_WITH( + rnn(input, hx), "Expected input to be 1D or 2D, got 5D instead"); + } + + { + auto input = torch::randn({3, 1}); + auto hx = torch::randn({3, 1, 1, 1, 2}); + ASSERT_THROWS_WITH( + rnn(input, hx), "Expected hidden to be 1D or 2D, got 5D instead"); + } +} + +TEST_F(ModulesTest, LSTMCell) { + torch::manual_seed(0); + auto lstm = LSTMCell(1, 2); + + auto input = torch::randn({3, 1}); + auto hx = torch::randn({3, 2}); + auto cx = torch::randn({3, 2}); + auto output = lstm(input, std::make_tuple(hx, cx)); + auto output_hx = std::get<0>(output); + auto output_cx = std::get<1>(output); + auto expected_hx = + torch::tensor({{-0.2462, 0.0810}, {-0.2206, 0.1867}, {-0.0146, 0.0429}}); + auto expected_cx = + torch::tensor({{-0.4480, 0.1071}, {-0.6245, 0.2687}, {-0.0322, 0.0518}}); + ASSERT_TRUE(torch::allclose(output_hx, expected_hx, 1e-05, 2e-04)); + ASSERT_TRUE(torch::allclose(output_cx, expected_cx, 1e-05, 2e-04)); + + output = lstm(input); + output_hx = std::get<0>(output); + output_cx = std::get<1>(output); + expected_hx = + torch::tensor({{-0.1331, 0.1634}, {-0.1494, 0.2869}, {-0.1428, 0.2263}}); + expected_cx = + torch::tensor({{-0.2679, 0.2180}, {-0.3049, 0.3493}, {-0.2896, 0.2853}}); + ASSERT_TRUE(torch::allclose(output_hx, expected_hx, 1e-05, 2e-04)); + ASSERT_TRUE(torch::allclose(output_cx, expected_cx, 1e-05, 2e-04)); + + input = torch::randn({1}); + hx = torch::randn({2}); + cx = torch::randn({2}); + output = lstm(input, std::make_tuple(hx, cx)); + output_hx = std::get<0>(output); + output_cx = std::get<1>(output); + expected_hx = torch::tensor({-0.0443, 0.1537}); + expected_cx = torch::tensor({-0.1195, 0.2144}); + ASSERT_TRUE(torch::allclose(output_hx, expected_hx, 1e-05, 2e-04)); + ASSERT_TRUE(torch::allclose(output_cx, expected_cx, 1e-05, 2e-04)); + + { + auto input = torch::randn({3, 2}); + auto hx = torch::randn({3, 2}); + auto cx = torch::randn({3, 2}); + ASSERT_THROWS_WITH( + lstm(input, std::make_tuple(hx, cx)), + "input has inconsistent input_size: got 2 expected 1"); + } + + { + auto input = torch::randn({3, 1}); + auto hx = torch::randn({3, 1}); + auto cx = torch::randn({3, 2}); + ASSERT_THROWS_WITH( + lstm(input, std::make_tuple(hx, cx)), + "hidden0 has inconsistent hidden_size: got 1, expected 2"); + } + + { + auto input = torch::randn({3, 1}); + auto hx = torch::randn({3, 2}); + auto cx = torch::randn({3, 1}); + ASSERT_THROWS_WITH( + lstm(input, std::make_tuple(hx, cx)), + "hidden1 has inconsistent hidden_size: got 1, expected 2"); + } + + { + auto input = torch::randn({3, 1, 1, 1, 1}); + auto hx = torch::randn({3, 1}); + auto cx = torch::randn({3, 1}); + ASSERT_THROWS_WITH( + lstm(input, std::make_tuple(hx, cx)), + "Expected input to be 1D or 2D, got 5D instead"); + } + + { + auto input = torch::randn({3, 1}); + auto hx = torch::randn({3, 1, 1, 1, 2}); + auto cx = torch::randn({3, 2}); + ASSERT_THROWS_WITH( + lstm(input, std::make_tuple(hx, cx)), + "Expected hx[0] to be 1D or 2D, got 5D instead"); + } + + { + auto input = torch::randn({3, 1}); + auto hx = torch::randn({3, 2}); + auto cx = torch::randn({3, 1, 1, 1, 2}); + ASSERT_THROWS_WITH( + lstm(input, std::make_tuple(hx, cx)), + "Expected hx[1] to be 1D or 2D, got 5D instead"); + } +} + +TEST_F(ModulesTest, GRUCell) { + torch::manual_seed(0); + auto gru = GRUCell(1, 2); + + auto input = torch::randn({3, 1}); + auto hx = torch::randn({3, 2}); + auto output = gru(input, hx); + auto expected = + torch::tensor({{1.0243, 0.3227}, {-0.5659, 0.0330}, {-0.4030, -0.2800}}); + ASSERT_TRUE(torch::allclose(output, expected, 1e-05, 2e-04)); + + output = gru(input); + expected = + torch::tensor({{-0.0085, 0.1095}, {-0.1291, 0.2675}, {-0.1339, 0.2725}}); + ASSERT_TRUE(torch::allclose(output, expected, 1e-05, 2e-04)); + + input = torch::randn({1}); + hx = torch::randn({2}); + output = gru(input, hx); + expected = torch::tensor({-1.0058, -0.3025}); + ASSERT_TRUE(torch::allclose(output, expected, 1e-05, 2e-04)); + + { + auto input = torch::randn({3, 2}); + auto hx = torch::randn({3, 2}); + ASSERT_THROWS_WITH( + gru(input, hx), "input has inconsistent input_size: got 2 expected 1"); + } + + { + auto input = torch::randn({3, 1}); + auto hx = torch::randn({3, 1}); + ASSERT_THROWS_WITH( + gru(input, hx), + "hidden0 has inconsistent hidden_size: got 1, expected 2"); + } + + { + auto input = torch::randn({3, 1, 1, 1, 1}); + auto hx = torch::randn({3, 2}); + ASSERT_THROWS_WITH( + gru(input, hx), "Expected input to be 1D or 2D, got 5D instead"); + } + + { + auto input = torch::randn({3, 1}); + auto hx = torch::randn({3, 1, 1, 1, 2}); + ASSERT_THROWS_WITH( + gru(input, hx), "Expected hidden to be 1D or 2D, got 5D instead"); + } +} + +TEST_F(ModulesTest, PrettyPrintLinear) { + ASSERT_EQ( + c10::str(Linear(3, 4)), + "torch::nn::Linear(in_features=3, out_features=4, bias=true)"); +} + +TEST_F(ModulesTest, PrettyPrintBilinear) { + ASSERT_EQ( + c10::str(Bilinear(3, 2, 4)), + "torch::nn::Bilinear(in1_features=3, in2_features=2, out_features=4, bias=true)"); + ASSERT_EQ( + c10::str(Bilinear(BilinearOptions(3, 2, 4).bias(false))), + "torch::nn::Bilinear(in1_features=3, in2_features=2, out_features=4, bias=false)"); +} + +TEST_F(ModulesTest, PrettyPrintConv) { + ASSERT_EQ( + c10::str(Conv1d(3, 4, 5)), + "torch::nn::Conv1d(3, 4, kernel_size=5, stride=1)"); + { + auto options = Conv1dOptions(3, 4, 5); + ASSERT_EQ( + c10::str(Conv1d(options.padding(torch::kSame))), + "torch::nn::Conv1d(3, 4, kernel_size=5, stride=1, padding='same')"); + ASSERT_EQ( + c10::str(Conv1d(options.padding(torch::kValid))), + "torch::nn::Conv1d(3, 4, kernel_size=5, stride=1, padding='valid')"); + } + + ASSERT_EQ( + c10::str(Conv2d(3, 4, 5)), + "torch::nn::Conv2d(3, 4, kernel_size=[5, 5], stride=[1, 1])"); + ASSERT_EQ( + c10::str(Conv2d(Conv2dOptions(3, 4, 5).stride(2))), + "torch::nn::Conv2d(3, 4, kernel_size=[5, 5], stride=[2, 2])"); + { + const auto options = + Conv2dOptions(3, 4, std::vector{5, 6}).stride({1, 2}); + ASSERT_EQ( + c10::str(Conv2d(options)), + "torch::nn::Conv2d(3, 4, kernel_size=[5, 6], stride=[1, 2])"); + } + { + auto options = Conv2dOptions(3, 4, std::vector{5, 6}); + ASSERT_EQ( + c10::str(Conv2d(options.padding(torch::kSame))), + "torch::nn::Conv2d(3, 4, kernel_size=[5, 6], stride=[1, 1], padding='same')"); + ASSERT_EQ( + c10::str(Conv2d(options.padding(torch::kValid))), + "torch::nn::Conv2d(3, 4, kernel_size=[5, 6], stride=[1, 1], padding='valid')"); + } + + ASSERT_EQ( + c10::str(Conv3d(4, 4, std::vector{5, 6, 7})), + "torch::nn::Conv3d(4, 4, kernel_size=[5, 6, 7], stride=[1, 1, 1])"); + { + const auto options = Conv3dOptions(4, 4, std::vector{5, 6, 7}) + .stride({1, 2, 3}) + .padding(1) + .dilation(0) + .groups(2) + .bias(false) + .padding_mode(torch::kCircular); + ASSERT_EQ( + c10::str(Conv3d(options)), + "torch::nn::Conv3d(" + "4, " + "4, " + "kernel_size=[5, 6, 7], " + "stride=[1, 2, 3], " + "padding=[1, 1, 1], " + "dilation=[0, 0, 0], " + "groups=2, " + "bias=false, " + "padding_mode=kCircular)"); + } + { + auto options = Conv3dOptions(3, 4, std::vector{5, 6, 7}); + ASSERT_EQ( + c10::str(Conv3d(options.padding(torch::kSame))), + "torch::nn::Conv3d(3, 4, kernel_size=[5, 6, 7], stride=[1, 1, 1], padding='same')"); + ASSERT_EQ( + c10::str(Conv3d(options.padding(torch::kValid))), + "torch::nn::Conv3d(3, 4, kernel_size=[5, 6, 7], stride=[1, 1, 1], padding='valid')"); + } +} + +TEST_F(ModulesTest, PrettyPrintConvTranspose) { + ASSERT_EQ( + c10::str(ConvTranspose1d(3, 4, 5)), + "torch::nn::ConvTranspose1d(3, 4, kernel_size=5, stride=1)"); + + ASSERT_EQ( + c10::str(ConvTranspose2d(3, 4, 5)), + "torch::nn::ConvTranspose2d(3, 4, kernel_size=[5, 5], stride=[1, 1])"); + ASSERT_EQ( + c10::str(ConvTranspose2d(ConvTranspose2dOptions(3, 4, 5).stride(2))), + "torch::nn::ConvTranspose2d(3, 4, kernel_size=[5, 5], stride=[2, 2])"); + { + const auto options = + ConvTranspose2dOptions(3, 4, std::vector{5, 6}).stride({1, 2}); + ASSERT_EQ( + c10::str(ConvTranspose2d(options)), + "torch::nn::ConvTranspose2d(3, 4, kernel_size=[5, 6], stride=[1, 2])"); + } + + ASSERT_EQ( + c10::str(ConvTranspose3d(4, 4, std::vector{5, 6, 7})), + "torch::nn::ConvTranspose3d(4, 4, kernel_size=[5, 6, 7], stride=[1, 1, 1])"); + { + const auto options = + ConvTranspose3dOptions(4, 4, std::vector{5, 6, 7}) + .stride({1, 2, 3}) + .padding(1) + .dilation(0) + .groups(2) + .bias(false) + .padding_mode(torch::kCircular); + ASSERT_EQ( + c10::str(ConvTranspose3d(options)), + "torch::nn::ConvTranspose3d(" + "4, " + "4, " + "kernel_size=[5, 6, 7], " + "stride=[1, 2, 3], " + "padding=[1, 1, 1], " + "dilation=[0, 0, 0], " + "groups=2, " + "bias=false, " + "padding_mode=kCircular)"); + } +} + +TEST_F(ModulesTest, PrettyPrintUpsample) { + ASSERT_EQ( + c10::str( + Upsample(UpsampleOptions().size(std::vector({2, 4, 4})))), + "torch::nn::Upsample(size=[2, 4, 4], mode=kNearest)"); + ASSERT_EQ( + c10::str(Upsample(UpsampleOptions() + .scale_factor(std::vector({0.5, 1.5})) + .mode(torch::kBilinear))), + "torch::nn::Upsample(scale_factor=[0.5, 1.5], mode=kBilinear)"); +} + +TEST_F(ModulesTest, PrettyPrintFold) { + ASSERT_EQ( + c10::str(Fold(FoldOptions({2, 2}, {5, 5}))), + "torch::nn::Fold(output_size=[2, 2], kernel_size=[5, 5], dilation=[1, 1], padding=[0, 0], stride=[1, 1])"); + ASSERT_EQ( + c10::str(Fold( + FoldOptions({8, 8}, {3, 3}).dilation(2).padding({2, 1}).stride(2))), + "torch::nn::Fold(output_size=[8, 8], kernel_size=[3, 3], dilation=[2, 2], padding=[2, 1], stride=[2, 2])"); +} + +TEST_F(ModulesTest, PrettyPrintUnfold) { + ASSERT_EQ( + c10::str(Unfold(torch::IntArrayRef({2, 4}))), + "torch::nn::Unfold(kernel_size=[2, 4], dilation=[1, 1], padding=[0, 0], stride=[1, 1])"); + ASSERT_EQ( + c10::str( + Unfold(UnfoldOptions({2, 4}).dilation(2).padding({2, 1}).stride(2))), + "torch::nn::Unfold(kernel_size=[2, 4], dilation=[2, 2], padding=[2, 1], stride=[2, 2])"); +} + +TEST_F(ModulesTest, PrettyPrintMaxPool) { + ASSERT_EQ( + c10::str(MaxPool1d(5)), + "torch::nn::MaxPool1d(kernel_size=5, stride=5, padding=0, dilation=1, ceil_mode=false)"); + ASSERT_EQ( + c10::str(MaxPool2d(5)), + "torch::nn::MaxPool2d(kernel_size=[5, 5], stride=[5, 5], padding=[0, 0], dilation=[1, 1], ceil_mode=false)"); + ASSERT_EQ( + c10::str(MaxPool2d(MaxPool2dOptions(5).stride(2))), + "torch::nn::MaxPool2d(kernel_size=[5, 5], stride=[2, 2], padding=[0, 0], dilation=[1, 1], ceil_mode=false)"); + ASSERT_EQ( + c10::str(MaxPool3d(5)), + "torch::nn::MaxPool3d(kernel_size=[5, 5, 5], stride=[5, 5, 5], padding=[0, 0, 0], dilation=[1, 1, 1], ceil_mode=false)"); + ASSERT_EQ( + c10::str(MaxPool3d(MaxPool3dOptions(5).stride(2))), + "torch::nn::MaxPool3d(kernel_size=[5, 5, 5], stride=[2, 2, 2], padding=[0, 0, 0], dilation=[1, 1, 1], ceil_mode=false)"); + + const auto options = + MaxPool2dOptions(std::vector{5, 6}).stride({1, 2}); + ASSERT_EQ( + c10::str(MaxPool2d(options)), + "torch::nn::MaxPool2d(kernel_size=[5, 6], stride=[1, 2], padding=[0, 0], dilation=[1, 1], ceil_mode=false)"); +} + +TEST_F(ModulesTest, PrettyPrintAvgPool) { + ASSERT_EQ( + c10::str(AvgPool1d(5)), + "torch::nn::AvgPool1d(kernel_size=5, stride=5, padding=0)"); + ASSERT_EQ( + c10::str(AvgPool2d(5)), + "torch::nn::AvgPool2d(kernel_size=[5, 5], stride=[5, 5], padding=[0, 0])"); + ASSERT_EQ( + c10::str(AvgPool2d(AvgPool2dOptions(5).stride(2))), + "torch::nn::AvgPool2d(kernel_size=[5, 5], stride=[2, 2], padding=[0, 0])"); + ASSERT_EQ( + c10::str(AvgPool3d(5)), + "torch::nn::AvgPool3d(kernel_size=[5, 5, 5], stride=[5, 5, 5], padding=[0, 0, 0])"); + ASSERT_EQ( + c10::str(AvgPool3d(AvgPool3dOptions(5).stride(2))), + "torch::nn::AvgPool3d(kernel_size=[5, 5, 5], stride=[2, 2, 2], padding=[0, 0, 0])"); + + const auto options = + AvgPool2dOptions(std::vector{5, 6}).stride({1, 2}); + ASSERT_EQ( + c10::str(AvgPool2d(options)), + "torch::nn::AvgPool2d(kernel_size=[5, 6], stride=[1, 2], padding=[0, 0])"); +} + +TEST_F(ModulesTest, PrettyPrinFractionalMaxPool) { + ASSERT_EQ( + c10::str( + FractionalMaxPool2d(FractionalMaxPool2dOptions(5).output_size(1))), + "torch::nn::FractionalMaxPool2d()"); + ASSERT_EQ( + c10::str( + FractionalMaxPool3d(FractionalMaxPool3dOptions(5).output_size(1))), + "torch::nn::FractionalMaxPool3d()"); +} + +TEST_F(ModulesTest, PrettyPrintLPPool) { + ASSERT_EQ( + c10::str(LPPool1d(2, 5)), + "torch::nn::LPPool1d(norm_type=2, kernel_size=5, stride=5, ceil_mode=false)"); + ASSERT_EQ( + c10::str(LPPool1d(LPPool1dOptions(1, 2).stride(5).ceil_mode(true))), + "torch::nn::LPPool1d(norm_type=1, kernel_size=2, stride=5, ceil_mode=true)"); + ASSERT_EQ( + c10::str(LPPool2d(2, std::vector({1, 2}))), + "torch::nn::LPPool2d(norm_type=2, kernel_size=[1, 2], stride=[1, 2], ceil_mode=false)"); + ASSERT_EQ( + c10::str(LPPool2d(LPPool2dOptions(1, std::vector({3, 4})) + .stride({5, 6}) + .ceil_mode(true))), + "torch::nn::LPPool2d(norm_type=1, kernel_size=[3, 4], stride=[5, 6], ceil_mode=true)"); + ASSERT_EQ( + c10::str(LPPool3d(2, std::vector({1, 2, 3}))), + "torch::nn::LPPool3d(norm_type=2, kernel_size=[1, 2, 3], stride=[1, 2, 3], ceil_mode=false)"); + ASSERT_EQ( + c10::str(LPPool3d(LPPool3dOptions(1, std::vector({3, 4, 5})) + .stride({5, 6, 7}) + .ceil_mode(true))), + "torch::nn::LPPool3d(norm_type=1, kernel_size=[3, 4, 5], stride=[5, 6, 7], ceil_mode=true)"); +} + +TEST_F(ModulesTest, PrettyPrintAdaptiveMaxPool) { + ASSERT_EQ( + c10::str(AdaptiveMaxPool1d(5)), + "torch::nn::AdaptiveMaxPool1d(output_size=5)"); + + const auto options = AdaptiveMaxPool1dOptions(3); + ASSERT_EQ( + c10::str(AdaptiveMaxPool1d(options)), + "torch::nn::AdaptiveMaxPool1d(output_size=3)"); + + ASSERT_EQ( + c10::str(AdaptiveMaxPool2d(5)), + "torch::nn::AdaptiveMaxPool2d(output_size=[5, 5])"); + ASSERT_EQ( + c10::str(AdaptiveMaxPool2d(AdaptiveMaxPool2dOptions({5, 6}))), + "torch::nn::AdaptiveMaxPool2d(output_size=[5, 6])"); + ASSERT_EQ( + c10::str(AdaptiveMaxPool2d(AdaptiveMaxPool2dOptions({5, std::nullopt}))), + "torch::nn::AdaptiveMaxPool2d(output_size=[5, None])"); + ASSERT_EQ( + c10::str(AdaptiveMaxPool2d( + AdaptiveMaxPool2dOptions({std::nullopt, std::nullopt}))), + "torch::nn::AdaptiveMaxPool2d(output_size=[None, None])"); + + ASSERT_EQ( + c10::str(AdaptiveMaxPool3d(5)), + "torch::nn::AdaptiveMaxPool3d(output_size=[5, 5, 5])"); + ASSERT_EQ( + c10::str(AdaptiveMaxPool3d(AdaptiveMaxPool3dOptions({5, 6, 7}))), + "torch::nn::AdaptiveMaxPool3d(output_size=[5, 6, 7])"); + ASSERT_EQ( + c10::str( + AdaptiveMaxPool3d(AdaptiveMaxPool3dOptions({5, std::nullopt, 7}))), + "torch::nn::AdaptiveMaxPool3d(output_size=[5, None, 7])"); + ASSERT_EQ( + c10::str(AdaptiveMaxPool3d(AdaptiveMaxPool3dOptions( + {std::nullopt, std::nullopt, std::nullopt}))), + "torch::nn::AdaptiveMaxPool3d(output_size=[None, None, None])"); +} + +TEST_F(ModulesTest, PrettyPrintAdaptiveAvgPool) { + ASSERT_EQ( + c10::str(AdaptiveAvgPool1d(5)), + "torch::nn::AdaptiveAvgPool1d(output_size=5)"); + + ASSERT_EQ( + c10::str(AdaptiveAvgPool2d(5)), + "torch::nn::AdaptiveAvgPool2d(output_size=[5, 5])"); + ASSERT_EQ( + c10::str(AdaptiveAvgPool2d(AdaptiveAvgPool2dOptions({5, 6}))), + "torch::nn::AdaptiveAvgPool2d(output_size=[5, 6])"); + ASSERT_EQ( + c10::str(AdaptiveAvgPool2d(AdaptiveAvgPool2dOptions({5, std::nullopt}))), + "torch::nn::AdaptiveAvgPool2d(output_size=[5, None])"); + ASSERT_EQ( + c10::str(AdaptiveAvgPool2d( + AdaptiveAvgPool2dOptions({std::nullopt, std::nullopt}))), + "torch::nn::AdaptiveAvgPool2d(output_size=[None, None])"); + + ASSERT_EQ( + c10::str(AdaptiveAvgPool3d(5)), + "torch::nn::AdaptiveAvgPool3d(output_size=[5, 5, 5])"); + ASSERT_EQ( + c10::str(AdaptiveAvgPool3d(AdaptiveAvgPool3dOptions({5, 6, 7}))), + "torch::nn::AdaptiveAvgPool3d(output_size=[5, 6, 7])"); + ASSERT_EQ( + c10::str( + AdaptiveAvgPool3d(AdaptiveAvgPool3dOptions({5, std::nullopt, 7}))), + "torch::nn::AdaptiveAvgPool3d(output_size=[5, None, 7])"); + ASSERT_EQ( + c10::str(AdaptiveAvgPool3d(AdaptiveAvgPool3dOptions( + {std::nullopt, std::nullopt, std::nullopt}))), + "torch::nn::AdaptiveAvgPool3d(output_size=[None, None, None])"); +} + +TEST_F(ModulesTest, PrettyPrintMaxUnpool) { + ASSERT_EQ( + c10::str(MaxUnpool1d(5)), + "torch::nn::MaxUnpool1d(kernel_size=5, stride=5, padding=0)"); + ASSERT_EQ( + c10::str(MaxUnpool1d(MaxUnpool1dOptions(5).stride(3).padding(1))), + "torch::nn::MaxUnpool1d(kernel_size=5, stride=3, padding=1)"); + + ASSERT_EQ( + c10::str(MaxUnpool2d(5)), + "torch::nn::MaxUnpool2d(kernel_size=[5, 5], stride=[5, 5], padding=[0, 0])"); + ASSERT_EQ( + c10::str(MaxUnpool2d(std::vector{5, 6})), + "torch::nn::MaxUnpool2d(kernel_size=[5, 6], stride=[5, 6], padding=[0, 0])"); + ASSERT_EQ( + c10::str(MaxUnpool2d(MaxUnpool2dOptions(std::vector{5, 6}) + .stride({3, 4}) + .padding({1, 2}))), + "torch::nn::MaxUnpool2d(kernel_size=[5, 6], stride=[3, 4], padding=[1, 2])"); +} + +TEST_F(ModulesTest, PrettyPrintDropout) { + ASSERT_EQ(c10::str(Dropout()), "torch::nn::Dropout(p=0.5, inplace=false)"); + ASSERT_EQ( + c10::str(Dropout(0.42)), "torch::nn::Dropout(p=0.42, inplace=false)"); + ASSERT_EQ( + c10::str(Dropout(DropoutOptions().p(0.42).inplace(true))), + "torch::nn::Dropout(p=0.42, inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintDropout2d) { + ASSERT_EQ( + c10::str(Dropout2d()), "torch::nn::Dropout2d(p=0.5, inplace=false)"); + ASSERT_EQ( + c10::str(Dropout2d(0.42)), "torch::nn::Dropout2d(p=0.42, inplace=false)"); + ASSERT_EQ( + c10::str(Dropout2d(Dropout2dOptions().p(0.42).inplace(true))), + "torch::nn::Dropout2d(p=0.42, inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintDropout3d) { + ASSERT_EQ( + c10::str(Dropout3d()), "torch::nn::Dropout3d(p=0.5, inplace=false)"); + ASSERT_EQ( + c10::str(Dropout3d(0.42)), "torch::nn::Dropout3d(p=0.42, inplace=false)"); + ASSERT_EQ( + c10::str(Dropout3d(Dropout3dOptions().p(0.42).inplace(true))), + "torch::nn::Dropout3d(p=0.42, inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintFunctional) { + ASSERT_EQ(c10::str(Functional(torch::relu)), "torch::nn::Functional()"); +} + +TEST_F(ModulesTest, PrettyPrintBatchNorm1d) { + ASSERT_EQ( + c10::str(BatchNorm1d(BatchNorm1dOptions(4) + .eps(0.5) + .momentum(0.1) + .affine(false) + .track_running_stats(true))), + "torch::nn::BatchNorm1d(4, eps=0.5, momentum=0.1, affine=false, track_running_stats=true)"); +} + +TEST_F(ModulesTest, PrettyPrintBatchNorm2d) { + ASSERT_EQ( + c10::str(BatchNorm2d(BatchNorm2dOptions(4) + .eps(0.5) + .momentum(0.1) + .affine(false) + .track_running_stats(true))), + "torch::nn::BatchNorm2d(4, eps=0.5, momentum=0.1, affine=false, track_running_stats=true)"); +} + +TEST_F(ModulesTest, PrettyPrintBatchNorm3d) { + ASSERT_EQ( + c10::str(BatchNorm3d(BatchNorm3dOptions(4) + .eps(0.5) + .momentum(0.1) + .affine(false) + .track_running_stats(true))), + "torch::nn::BatchNorm3d(4, eps=0.5, momentum=0.1, affine=false, track_running_stats=true)"); +} + +TEST_F(ModulesTest, PrettyPrintInstanceNorm1d) { + ASSERT_EQ( + c10::str(InstanceNorm1d(InstanceNorm1dOptions(4) + .eps(0.5) + .momentum(0.1) + .affine(false) + .track_running_stats(true))), + "torch::nn::InstanceNorm1d(4, eps=0.5, momentum=0.1, affine=false, track_running_stats=true)"); +} + +TEST_F(ModulesTest, PrettyPrintInstanceNorm2d) { + ASSERT_EQ( + c10::str(InstanceNorm2d(InstanceNorm2dOptions(4) + .eps(0.5) + .momentum(0.1) + .affine(false) + .track_running_stats(true))), + "torch::nn::InstanceNorm2d(4, eps=0.5, momentum=0.1, affine=false, track_running_stats=true)"); +} + +TEST_F(ModulesTest, PrettyPrintInstanceNorm3d) { + ASSERT_EQ( + c10::str(InstanceNorm3d(InstanceNorm3dOptions(4) + .eps(0.5) + .momentum(0.1) + .affine(false) + .track_running_stats(true))), + "torch::nn::InstanceNorm3d(4, eps=0.5, momentum=0.1, affine=false, track_running_stats=true)"); +} + +TEST_F(ModulesTest, PrettyPrintLayerNorm) { + ASSERT_EQ( + c10::str(LayerNorm(LayerNormOptions({2, 2}))), + "torch::nn::LayerNorm([2, 2], eps=1e-05, elementwise_affine=true)"); + ASSERT_EQ( + c10::str(LayerNorm( + LayerNormOptions({2, 2}).elementwise_affine(false).eps(2e-5))), + "torch::nn::LayerNorm([2, 2], eps=2e-05, elementwise_affine=false)"); +} + +TEST_F(ModulesTest, PrettyPrintGroupNorm) { + ASSERT_EQ( + c10::str(GroupNorm(GroupNormOptions(2, 2))), + "torch::nn::GroupNorm(2, 2, eps=1e-05, affine=true)"); + ASSERT_EQ( + c10::str(GroupNorm(GroupNormOptions(2, 2).eps(2e-5).affine(false))), + "torch::nn::GroupNorm(2, 2, eps=2e-05, affine=false)"); +} + +TEST_F(ModulesTest, PrettyPrintLocalResponseNorm) { + ASSERT_EQ( + c10::str(LocalResponseNorm(LocalResponseNormOptions(2))), + "torch::nn::LocalResponseNorm(2, alpha=0.0001, beta=0.75, k=1)"); + ASSERT_EQ( + c10::str(LocalResponseNorm( + LocalResponseNormOptions(2).alpha(0.0002).beta(0.85).k(2.))), + "torch::nn::LocalResponseNorm(2, alpha=0.0002, beta=0.85, k=2)"); +} + +TEST_F(ModulesTest, PrettyPrintEmbedding) { + ASSERT_EQ( + c10::str(Embedding(EmbeddingOptions(10, 2))), + "torch::nn::Embedding(num_embeddings=10, embedding_dim=2)"); + ASSERT_EQ( + c10::str(Embedding(EmbeddingOptions(10, 2).padding_idx(3).max_norm(2))), + "torch::nn::Embedding(num_embeddings=10, embedding_dim=2, padding_idx=3, max_norm=2)"); + ASSERT_EQ( + c10::str(Embedding(EmbeddingOptions(10, 2) + .padding_idx(3) + .max_norm(2) + .norm_type(2.5) + .scale_grad_by_freq(true) + .sparse(true))), + "torch::nn::Embedding(num_embeddings=10, embedding_dim=2, padding_idx=3, max_norm=2, norm_type=2.5, scale_grad_by_freq=true, sparse=true)"); +} + +TEST_F(ModulesTest, PrettyPrintEmbeddingBag) { + ASSERT_EQ( + c10::str(EmbeddingBag(EmbeddingBagOptions(10, 2))), + "torch::nn::EmbeddingBag(num_embeddings=10, embedding_dim=2)"); + ASSERT_EQ( + c10::str(EmbeddingBag(EmbeddingBagOptions(10, 2).max_norm(2))), + "torch::nn::EmbeddingBag(num_embeddings=10, embedding_dim=2, max_norm=2)"); + ASSERT_EQ( + c10::str(EmbeddingBag(EmbeddingBagOptions(10, 2) + .max_norm(2) + .norm_type(2.5) + .scale_grad_by_freq(true) + .sparse(true))), + "torch::nn::EmbeddingBag(num_embeddings=10, embedding_dim=2, max_norm=2, norm_type=2.5, scale_grad_by_freq=true, sparse=true)"); + ASSERT_EQ( + c10::str(EmbeddingBag(EmbeddingBagOptions(10, 2) + .max_norm(2) + .norm_type(2.5) + .scale_grad_by_freq(true) + .sparse(true) + .mode(torch::kSum))), + "torch::nn::EmbeddingBag(num_embeddings=10, embedding_dim=2, max_norm=2, norm_type=2.5, scale_grad_by_freq=true, sparse=true, mode=kSum)"); + ASSERT_EQ( + c10::str(EmbeddingBag(EmbeddingBagOptions(10, 2) + .max_norm(2) + .norm_type(2.5) + .scale_grad_by_freq(true) + .sparse(true) + .mode(torch::kSum) + .padding_idx(5))), + "torch::nn::EmbeddingBag(num_embeddings=10, embedding_dim=2, max_norm=2, norm_type=2.5, scale_grad_by_freq=true, sparse=true, mode=kSum, padding_idx=5)"); +} + +TEST_F(ModulesTest, PrettyPrintL1Loss) { + ASSERT_EQ(c10::str(L1Loss()), "torch::nn::L1Loss()"); +} +TEST_F(ModulesTest, PrettyPrintKLDivLoss) { + ASSERT_EQ(c10::str(KLDivLoss()), "torch::nn::KLDivLoss()"); +} +TEST_F(ModulesTest, PrettyPrintMSELoss) { + ASSERT_EQ(c10::str(MSELoss()), "torch::nn::MSELoss()"); +} +TEST_F(ModulesTest, PrettyPrintBCELoss) { + ASSERT_EQ(c10::str(BCELoss()), "torch::nn::BCELoss()"); +} +TEST_F(ModulesTest, PrettyPrintHingeEmbeddingLoss) { + ASSERT_EQ( + c10::str(HingeEmbeddingLoss(HingeEmbeddingLossOptions().margin(4))), + "torch::nn::HingeEmbeddingLoss(margin=4)"); +} + +TEST_F(ModulesTest, PrettyPrintCosineEmbeddingLoss) { + ASSERT_EQ( + c10::str(CosineEmbeddingLoss(CosineEmbeddingLossOptions().margin(0.25))), + "torch::nn::CosineEmbeddingLoss(margin=0.25)"); +} + +TEST_F(ModulesTest, PrettyPrintTripletMarginLoss) { + ASSERT_EQ( + c10::str(TripletMarginLoss( + TripletMarginLossOptions().margin(3).p(2).eps(1e-06).swap(false))), + "torch::nn::TripletMarginLoss(margin=3, p=2, eps=1e-06, swap=false)"); +} + +TEST_F(ModulesTest, PrettyPrintTripletMarginWithDistanceLoss) { + auto distanceOptions = TripletMarginWithDistanceLossOptions() + .distance_function([&](const torch::Tensor& x, + const torch::Tensor& y) { + return torch::pairwise_distance(x, y, 2.0, 1e-6); + }) + .margin(1.5) + .swap(true) + .reduction(torch::kMean); + ASSERT_EQ( + c10::str(TripletMarginWithDistanceLoss(distanceOptions)), + "torch::nn::TripletMarginWithDistanceLoss(margin=1.5, swap=true)"); +} + +TEST_F(ModulesTest, PrettyPrintNLLLoss) { + ASSERT_EQ(c10::str(NLLLoss()), "torch::nn::NLLLoss()"); +} + +TEST_F(ModulesTest, PrettyPrinCrossEntropyLoss) { + ASSERT_EQ(c10::str(CrossEntropyLoss()), "torch::nn::CrossEntropyLoss()"); +} + +TEST_F(ModulesTest, PrettyPrintMultiLabelMarginLoss) { + ASSERT_EQ( + c10::str(MultiLabelMarginLoss()), "torch::nn::MultiLabelMarginLoss()"); +} + +TEST_F(ModulesTest, PrettyPrintMultiLabelSoftMarginLoss) { + ASSERT_EQ( + c10::str(MultiLabelSoftMarginLoss()), + "torch::nn::MultiLabelSoftMarginLoss()"); +} + +TEST_F(ModulesTest, PrettyPrintSoftMarginLoss) { + ASSERT_EQ(c10::str(SoftMarginLoss()), "torch::nn::SoftMarginLoss()"); +} + +TEST_F(ModulesTest, PrettyPrintCosineSimilarity) { + ASSERT_EQ( + c10::str(CosineSimilarity()), + "torch::nn::CosineSimilarity(dim=1, eps=1e-08)"); + ASSERT_EQ( + c10::str(CosineSimilarity(CosineSimilarityOptions().dim(0).eps(0.5))), + "torch::nn::CosineSimilarity(dim=0, eps=0.5)"); +} + +TEST_F(ModulesTest, PrettyPrintPairwiseDistance) { + ASSERT_EQ( + c10::str(PairwiseDistance()), + "torch::nn::PairwiseDistance(p=2, eps=1e-06, keepdim=false)"); + ASSERT_EQ( + c10::str(PairwiseDistance( + PairwiseDistanceOptions().p(3).eps(0.5).keepdim(true))), + "torch::nn::PairwiseDistance(p=3, eps=0.5, keepdim=true)"); +} + +TEST_F(ModulesTest, PrettyPrintReflectionPad) { + ASSERT_EQ( + c10::str(ReflectionPad1d(ReflectionPad1dOptions(2))), + "torch::nn::ReflectionPad1d(padding=[2, 2])"); + ASSERT_EQ( + c10::str(ReflectionPad1d(ReflectionPad1dOptions({3, 1}))), + "torch::nn::ReflectionPad1d(padding=[3, 1])"); + ASSERT_EQ( + c10::str(ReflectionPad2d(ReflectionPad2dOptions(2))), + "torch::nn::ReflectionPad2d(padding=[2, 2, 2, 2])"); + ASSERT_EQ( + c10::str(ReflectionPad2d(ReflectionPad2dOptions({1, 1, 2, 0}))), + "torch::nn::ReflectionPad2d(padding=[1, 1, 2, 0])"); +} + +TEST_F(ModulesTest, PrettyPrintReplicationPad) { + ASSERT_EQ( + c10::str(ReplicationPad1d(ReplicationPad1dOptions(2))), + "torch::nn::ReplicationPad1d(padding=[2, 2])"); + ASSERT_EQ( + c10::str(ReplicationPad1d(ReplicationPad1dOptions({3, 1}))), + "torch::nn::ReplicationPad1d(padding=[3, 1])"); + ASSERT_EQ( + c10::str(ReplicationPad2d(ReplicationPad2dOptions(2))), + "torch::nn::ReplicationPad2d(padding=[2, 2, 2, 2])"); + ASSERT_EQ( + c10::str(ReplicationPad2d(ReplicationPad2dOptions({1, 1, 2, 0}))), + "torch::nn::ReplicationPad2d(padding=[1, 1, 2, 0])"); + ASSERT_EQ( + c10::str(ReplicationPad3d(ReplicationPad3dOptions(1))), + "torch::nn::ReplicationPad3d(padding=[1, 1, 1, 1, 1, 1])"); + ASSERT_EQ( + c10::str(ReplicationPad3d(ReplicationPad3dOptions({1, 2, 1, 2, 1, 2}))), + "torch::nn::ReplicationPad3d(padding=[1, 2, 1, 2, 1, 2])"); +} + +TEST_F(ModulesTest, PrettyPrintZeroPad) { + ASSERT_EQ( + c10::str(ZeroPad1d(ZeroPad1dOptions(2))), + "torch::nn::ZeroPad1d(padding=[2, 2])"); + ASSERT_EQ( + c10::str(ZeroPad1d(ZeroPad1dOptions({3, 1}))), + "torch::nn::ZeroPad1d(padding=[3, 1])"); + ASSERT_EQ( + c10::str(ZeroPad2d(ZeroPad2dOptions(2))), + "torch::nn::ZeroPad2d(padding=[2, 2, 2, 2])"); + ASSERT_EQ( + c10::str(ZeroPad2d(ZeroPad2dOptions({1, 1, 2, 0}))), + "torch::nn::ZeroPad2d(padding=[1, 1, 2, 0])"); + ASSERT_EQ( + c10::str(ZeroPad3d(ZeroPad3dOptions(1))), + "torch::nn::ZeroPad3d(padding=[1, 1, 1, 1, 1, 1])"); + ASSERT_EQ( + c10::str(ZeroPad3d(ZeroPad3dOptions({1, 2, 1, 2, 1, 2}))), + "torch::nn::ZeroPad3d(padding=[1, 2, 1, 2, 1, 2])"); +} + +TEST_F(ModulesTest, PrettyPrintConstantPad) { + ASSERT_EQ( + c10::str(ConstantPad1d(ConstantPad1dOptions(2, 3.5))), + "torch::nn::ConstantPad1d(padding=[2, 2], value=3.5)"); + ASSERT_EQ( + c10::str(ConstantPad1d(ConstantPad1dOptions({3, 1}, 3.5))), + "torch::nn::ConstantPad1d(padding=[3, 1], value=3.5)"); + ASSERT_EQ( + c10::str(ConstantPad2d(ConstantPad2dOptions(2, 3.5))), + "torch::nn::ConstantPad2d(padding=[2, 2, 2, 2], value=3.5)"); + ASSERT_EQ( + c10::str(ConstantPad2d(ConstantPad2dOptions({3, 0, 2, 1}, 3.5))), + "torch::nn::ConstantPad2d(padding=[3, 0, 2, 1], value=3.5)"); + ASSERT_EQ( + c10::str(ConstantPad3d(ConstantPad3dOptions(1, 3.5))), + "torch::nn::ConstantPad3d(padding=[1, 1, 1, 1, 1, 1], value=3.5)"); + ASSERT_EQ( + c10::str(ConstantPad3d(ConstantPad3dOptions({1, 2, 1, 2, 1, 2}, 3.5))), + "torch::nn::ConstantPad3d(padding=[1, 2, 1, 2, 1, 2], value=3.5)"); +} + +TEST_F(ModulesTest, PrettyPrintNestedModel) { + struct InnerTestModule : torch::nn::Module { + InnerTestModule() + : torch::nn::Module("InnerTestModule"), + fc(register_module("fc", torch::nn::Linear(3, 4))), + table(register_module("table", torch::nn::Embedding(10, 2))) {} + + torch::nn::Linear fc; + torch::nn::Embedding table; + }; + + struct TestModule : torch::nn::Module { + TestModule() + : torch::nn::Module("TestModule"), + fc(register_module("fc", torch::nn::Linear(4, 5))), + table(register_module( + "table", + torch::nn::Embedding(EmbeddingOptions(10, 2)))), + inner(register_module("inner", std::make_shared())) { + } + + torch::nn::Linear fc; + torch::nn::Embedding table; + std::shared_ptr inner; + }; + + ASSERT_EQ( + c10::str(TestModule{}), + "TestModule(\n" + " (fc): torch::nn::Linear(in_features=4, out_features=5, bias=true)\n" + " (table): torch::nn::Embedding(num_embeddings=10, embedding_dim=2)\n" + " (inner): InnerTestModule(\n" + " (fc): torch::nn::Linear(in_features=3, out_features=4, bias=true)\n" + " (table): torch::nn::Embedding(num_embeddings=10, embedding_dim=2)\n" + " )\n" + ")"); +} + +TEST_F(ModulesTest, PrettyPrintELU) { + ASSERT_EQ(c10::str(ELU()), "torch::nn::ELU(alpha=1)"); + ASSERT_EQ( + c10::str(ELU(ELUOptions().alpha(42.42).inplace(true))), + "torch::nn::ELU(alpha=42.42, inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintSELU) { + ASSERT_EQ(c10::str(SELU()), "torch::nn::SELU()"); + ASSERT_EQ( + c10::str(SELU(SELUOptions().inplace(true))), + "torch::nn::SELU(inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintGLU) { + ASSERT_EQ(c10::str(GLU()), "torch::nn::GLU(dim=-1)"); + ASSERT_EQ(c10::str(GLU(1)), "torch::nn::GLU(dim=1)"); +} + +TEST_F(ModulesTest, PrettyPrintHardshrink) { + ASSERT_EQ(c10::str(Hardshrink()), "torch::nn::Hardshrink(0.5)"); + ASSERT_EQ( + c10::str(Hardshrink(HardshrinkOptions().lambda(42.42))), + "torch::nn::Hardshrink(42.42)"); +} + +TEST_F(ModulesTest, PrettyPrintHardtanh) { + ASSERT_EQ(c10::str(Hardtanh()), "torch::nn::Hardtanh(min_val=-1, max_val=1)"); + ASSERT_EQ( + c10::str(Hardtanh( + HardtanhOptions().min_val(-42.42).max_val(0.42).inplace(true))), + "torch::nn::Hardtanh(min_val=-42.42, max_val=0.42, inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintLeakyReLU) { + ASSERT_EQ(c10::str(LeakyReLU()), "torch::nn::LeakyReLU(negative_slope=0.01)"); + ASSERT_EQ( + c10::str( + LeakyReLU(LeakyReLUOptions().negative_slope(0.42).inplace(true))), + "torch::nn::LeakyReLU(negative_slope=0.42, inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintLogSigmoid) { + ASSERT_EQ(c10::str(LogSigmoid()), "torch::nn::LogSigmoid()"); +} + +TEST_F(ModulesTest, PrettyPrintSoftmax) { + ASSERT_EQ(c10::str(Softmax(SoftmaxOptions(1))), "torch::nn::Softmax(dim=1)"); +} + +TEST_F(ModulesTest, PrettyPrintSoftmin) { + ASSERT_EQ(c10::str(Softmin(SoftminOptions(1))), "torch::nn::Softmin(dim=1)"); +} + +TEST_F(ModulesTest, PrettyPrintLogSoftmax) { + ASSERT_EQ( + c10::str(LogSoftmax(LogSoftmaxOptions(1))), + "torch::nn::LogSoftmax(dim=1)"); +} + +TEST_F(ModulesTest, PrettyPrintSoftmax2d) { + ASSERT_EQ(c10::str(Softmax2d()), "torch::nn::Softmax2d()"); +} + +TEST_F(ModulesTest, PrettyPrintPReLU) { + ASSERT_EQ(c10::str(PReLU()), "torch::nn::PReLU(num_parameters=1)"); + ASSERT_EQ( + c10::str(PReLU(PReLUOptions().num_parameters(42))), + "torch::nn::PReLU(num_parameters=42)"); +} + +TEST_F(ModulesTest, PrettyPrintReLU) { + ASSERT_EQ(c10::str(ReLU()), "torch::nn::ReLU()"); + ASSERT_EQ( + c10::str(ReLU(ReLUOptions().inplace(true))), + "torch::nn::ReLU(inplace=true)"); + ASSERT_EQ(c10::str(ReLU(/*inplace=*/true)), "torch::nn::ReLU(inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintReLU6) { + ASSERT_EQ(c10::str(ReLU6()), "torch::nn::ReLU6()"); + ASSERT_EQ( + c10::str(ReLU6(ReLU6Options().inplace(true))), + "torch::nn::ReLU6(inplace=true)"); + ASSERT_EQ( + c10::str(ReLU6(/*inplace=*/true)), "torch::nn::ReLU6(inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintRReLU) { + ASSERT_EQ(c10::str(RReLU()), "torch::nn::RReLU(lower=0.125, upper=0.333333)"); + ASSERT_EQ( + c10::str(RReLU(RReLUOptions().lower(0.24).upper(0.42).inplace(true))), + "torch::nn::RReLU(lower=0.24, upper=0.42, inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintCELU) { + ASSERT_EQ(c10::str(CELU()), "torch::nn::CELU(alpha=1)"); + ASSERT_EQ( + c10::str(CELU(CELUOptions().alpha(42.42).inplace(true))), + "torch::nn::CELU(alpha=42.42, inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintSigmoid) { + ASSERT_EQ(c10::str(Sigmoid()), "torch::nn::Sigmoid()"); +} + +TEST_F(ModulesTest, PrettyPrintPixelShuffle) { + ASSERT_EQ( + c10::str(PixelShuffle(PixelShuffleOptions(5))), + "torch::nn::PixelShuffle(upscale_factor=5)"); +} + +TEST_F(ModulesTest, PrettyPrintPixelUnshuffle) { + ASSERT_EQ( + c10::str(PixelUnshuffle(PixelUnshuffleOptions(5))), + "torch::nn::PixelUnshuffle(downscale_factor=5)"); +} + +TEST_F(ModulesTest, PrettyPrintSoftplus) { + ASSERT_EQ(c10::str(Softplus()), "torch::nn::Softplus(beta=1, threshold=20)"); + ASSERT_EQ( + c10::str(Softplus(SoftplusOptions().beta(0.24).threshold(42.42))), + "torch::nn::Softplus(beta=0.24, threshold=42.42)"); +} + +TEST_F(ModulesTest, PrettyPrintSoftshrink) { + ASSERT_EQ(c10::str(Softshrink()), "torch::nn::Softshrink(0.5)"); + ASSERT_EQ( + c10::str(Softshrink(SoftshrinkOptions(42.42))), + "torch::nn::Softshrink(42.42)"); +} + +TEST_F(ModulesTest, PrettyPrintSoftsign) { + ASSERT_EQ(c10::str(Softsign()), "torch::nn::Softsign()"); +} + +TEST_F(ModulesTest, PrettyPrintTanh) { + ASSERT_EQ(c10::str(Tanh()), "torch::nn::Tanh()"); +} + +TEST_F(ModulesTest, PrettyPrintTanhshrink) { + ASSERT_EQ(c10::str(Tanhshrink()), "torch::nn::Tanhshrink()"); +} + +TEST_F(ModulesTest, PrettyPrintThreshold) { + ASSERT_EQ( + c10::str(Threshold(24.24, 42.42)), + "torch::nn::Threshold(threshold=24.24, value=42.42)"); + ASSERT_EQ( + c10::str(Threshold(ThresholdOptions(42.42, 24.24).inplace(true))), + "torch::nn::Threshold(threshold=42.42, value=24.24, inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintCTCLoss) { + ASSERT_EQ(c10::str(CTCLoss()), "torch::nn::CTCLoss()"); + ASSERT_EQ( + c10::str( + CTCLoss(CTCLossOptions().blank(42).zero_infinity(false).reduction( + torch::kSum))), + "torch::nn::CTCLoss()"); +} + +TEST_F(ModulesTest, PrettyPrintPoissonNLLLoss) { + ASSERT_EQ(c10::str(PoissonNLLLoss()), "torch::nn::PoissonNLLLoss()"); + ASSERT_EQ( + c10::str(PoissonNLLLoss(PoissonNLLLossOptions() + .log_input(false) + .full(true) + .eps(0.42) + .reduction(torch::kSum))), + "torch::nn::PoissonNLLLoss()"); +} + +TEST_F(ModulesTest, PrettyPrintMarginRankingLoss) { + ASSERT_EQ(c10::str(MarginRankingLoss()), "torch::nn::MarginRankingLoss()"); + ASSERT_EQ( + c10::str(MarginRankingLoss( + MarginRankingLossOptions().margin(0.5).reduction(torch::kSum))), + "torch::nn::MarginRankingLoss()"); +} + +TEST_F(ModulesTest, PrettyPrintCrossMapLRN2d) { + ASSERT_EQ( + c10::str(CrossMapLRN2d(4)), + "torch::nn::CrossMapLRN2d(4, alpha=0.0001, beta=0.75, k=1)"); + ASSERT_EQ( + c10::str( + CrossMapLRN2d(CrossMapLRN2dOptions(3).alpha(1e-5).beta(0.1).k(10))), + "torch::nn::CrossMapLRN2d(3, alpha=1e-05, beta=0.1, k=10)"); +} + +TEST_F(ModulesTest, PrettyPrintAlphaDropout) { + ASSERT_EQ( + c10::str(AlphaDropout()), + "torch::nn::AlphaDropout(p=0.5, inplace=false)"); + ASSERT_EQ( + c10::str(AlphaDropout(AlphaDropoutOptions(0.2))), + "torch::nn::AlphaDropout(p=0.2, inplace=false)"); + ASSERT_EQ( + c10::str(AlphaDropout(AlphaDropoutOptions(0.2).inplace(true))), + "torch::nn::AlphaDropout(p=0.2, inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintFeatureAlphaDropout) { + ASSERT_EQ( + c10::str(FeatureAlphaDropout()), + "torch::nn::FeatureAlphaDropout(p=0.5, inplace=false)"); + ASSERT_EQ( + c10::str(FeatureAlphaDropout(FeatureAlphaDropoutOptions(0.2))), + "torch::nn::FeatureAlphaDropout(p=0.2, inplace=false)"); + ASSERT_EQ( + c10::str( + FeatureAlphaDropout(FeatureAlphaDropoutOptions(0.2).inplace(true))), + "torch::nn::FeatureAlphaDropout(p=0.2, inplace=true)"); +} + +TEST_F(ModulesTest, PrettyPrintBCEWithLogitsLoss) { + ASSERT_EQ(c10::str(BCEWithLogitsLoss()), "torch::nn::BCEWithLogitsLoss()"); + ASSERT_EQ( + c10::str(BCEWithLogitsLoss(BCEWithLogitsLossOptions() + .weight(torch::ones({3, 3})) + .pos_weight(torch::ones({3, 3})) + .reduction(torch::kSum))), + "torch::nn::BCEWithLogitsLoss()"); +} + +TEST_F(ModulesTest, MultiheadAttentionRejectsZeroNumHeads) { + EXPECT_THROW(MultiheadAttention(MultiheadAttentionOptions(0, 0)), c10::Error); +} + +TEST_F(ModulesTest, PrettyPrintMultiheadAttention) { + ASSERT_EQ( + c10::str(MultiheadAttention(20, 10)), + "torch::nn::MultiheadAttention(\n (out_proj): torch::nn::Linear(in_features=20, out_features=20, bias=true)\n)"); + ASSERT_EQ( + c10::str( + MultiheadAttention(MultiheadAttentionOptions(20, 10).bias(false))), + "torch::nn::MultiheadAttention(\n (out_proj): torch::nn::Linear(in_features=20, out_features=20, bias=false)\n)"); +} + +TEST_F(ModulesTest, PrettyPrintRNNCell) { + ASSERT_EQ(c10::str(RNNCell(20, 10)), "torch::nn::RNNCell(20, 10)"); + ASSERT_EQ( + c10::str(RNNCell( + RNNCellOptions(20, 10).bias(false).nonlinearity(torch::kTanh))), + "torch::nn::RNNCell(20, 10, bias=false)"); + ASSERT_EQ( + c10::str(RNNCell( + RNNCellOptions(20, 10).bias(false).nonlinearity(torch::kReLU))), + "torch::nn::RNNCell(20, 10, bias=false, nonlinearity=kReLU)"); +} + +TEST_F(ModulesTest, PrettyPrintLSTMCell) { + ASSERT_EQ(c10::str(LSTMCell(20, 10)), "torch::nn::LSTMCell(20, 10)"); + ASSERT_EQ( + c10::str(LSTMCell(LSTMCellOptions(20, 10).bias(false))), + "torch::nn::LSTMCell(20, 10, bias=false)"); +} + +TEST_F(ModulesTest, PrettyPrintGRUCell) { + ASSERT_EQ(c10::str(GRUCell(20, 10)), "torch::nn::GRUCell(20, 10)"); + ASSERT_EQ( + c10::str(GRUCell(GRUCellOptions(20, 10).bias(false))), + "torch::nn::GRUCell(20, 10, bias=false)"); +} + +TEST_F(ModulesTest, PrettyPrintAdaptiveLogSoftmaxWithLoss) { + { + AdaptiveLogSoftmaxWithLoss asfm( + AdaptiveLogSoftmaxWithLossOptions(8, 4, {2}).div_value(2.)); + ASSERT_EQ( + c10::str(asfm), + "torch::nn::AdaptiveLogSoftmaxWithLoss(\n" + " (head): torch::nn::Linear(in_features=8, out_features=3, bias=false)\n" + " (tail): torch::nn::ModuleList(\n" + " (0): torch::nn::Sequential(\n" + " (0): torch::nn::Linear(in_features=8, out_features=4, bias=false)\n" + " (1): torch::nn::Linear(in_features=4, out_features=2, bias=false)\n" + " )\n" + " )\n" + ")"); + } + { + AdaptiveLogSoftmaxWithLoss asfm( + AdaptiveLogSoftmaxWithLossOptions(8, 10, {4, 8}) + .div_value(2.) + .head_bias(true)); + ASSERT_EQ( + c10::str(asfm), + "torch::nn::AdaptiveLogSoftmaxWithLoss(\n" + " (head): torch::nn::Linear(in_features=8, out_features=6, bias=true)\n" + " (tail): torch::nn::ModuleList(\n" + " (0): torch::nn::Sequential(\n" + " (0): torch::nn::Linear(in_features=8, out_features=4, bias=false)\n" + " (1): torch::nn::Linear(in_features=4, out_features=4, bias=false)\n" + " )\n" + " (1): torch::nn::Sequential(\n" + " (0): torch::nn::Linear(in_features=8, out_features=2, bias=false)\n" + " (1): torch::nn::Linear(in_features=2, out_features=2, bias=false)\n" + " )\n" + " )\n" + ")"); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/namespace.cpp b/western_ai/usa/pytorch/test/cpp/api/namespace.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7fc381b6d54555133360656ee4c253c8142f4d77 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/namespace.cpp @@ -0,0 +1,20 @@ +#include + +#include + +struct Node {}; + +// If `torch::autograd::Note` is leaked into the root namespace, the following +// compile error would throw: +// ``` +// void NotLeakingSymbolsFromTorchAutogradNamespace_test_func(Node *node) {} +// ^ +// error: reference to `Node` is ambiguous +// ``` +void NotLeakingSymbolsFromTorchAutogradNamespace_test_func(Node* node) {} + +TEST(NamespaceTests, NotLeakingSymbolsFromTorchAutogradNamespace) { + // Checks that we are not leaking symbols from the + // `torch::autograd` namespace to the root namespace + NotLeakingSymbolsFromTorchAutogradNamespace_test_func(nullptr); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/nested.cpp b/western_ai/usa/pytorch/test/cpp/api/nested.cpp new file mode 100644 index 0000000000000000000000000000000000000000..15ecc5c986e13678adcfe7e9ca7fc155faaadc4e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/nested.cpp @@ -0,0 +1,15 @@ +#include + +#include +#include + +#include + +// Simple test that verifies the nested namespace is registered properly +// properly in C++ +TEST(NestedTest, Nested) { + auto a = torch::randn({2, 3}); + auto b = torch::randn({4, 5}); + auto nt = torch::nested::nested_tensor({a, b}); + torch::nested::to_padded_tensor(nt, 0); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/nested_int.cpp b/western_ai/usa/pytorch/test/cpp/api/nested_int.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a04de402dd36a8ee8ec22d3e0354d860c95bc0f7 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/nested_int.cpp @@ -0,0 +1,105 @@ +#include + +#include +#include +#include +#include + +#include + +TEST(NestedIntTest, Comparisons) { + auto a = c10::SymInt( + c10::SymNode(c10::make_intrusive(1, 1))); + auto b = c10::SymInt( + c10::SymNode(c10::make_intrusive(1, 1))); + auto c = c10::SymInt( + c10::SymNode(c10::make_intrusive(2, 1))); + auto d = c10::SymInt(3); + + ASSERT_TRUE(a == a); + ASSERT_TRUE(a == b); + ASSERT_FALSE(a != a); + ASSERT_FALSE(a != b); + ASSERT_FALSE(a == c); + ASSERT_TRUE(a != c); + + ASSERT_FALSE(a == d); + ASSERT_TRUE(a != d); + ASSERT_FALSE(d == a); + ASSERT_TRUE(d != a); + + // ge + ASSERT_TRUE(a >= a); + ASSERT_TRUE(a >= b); + ASSERT_TRUE(b >= a); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(a >= c), c10::Error); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(c >= a), c10::Error); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(c >= 3), c10::Error); + ASSERT_TRUE(c >= 2); + ASSERT_TRUE(c >= 1); + ASSERT_FALSE(1 >= c); + + // lt + ASSERT_FALSE(a < a); + ASSERT_FALSE(a < b); + ASSERT_FALSE(b < a); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(a < c), c10::Error); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(c < a), c10::Error); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(3 < a), c10::Error); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(2 < a), c10::Error); + ASSERT_TRUE(1 < a); + + // le + ASSERT_TRUE(a <= a); + ASSERT_TRUE(b <= a); + ASSERT_TRUE(a <= b); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(a <= c), c10::Error); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(c <= a), c10::Error); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(3 <= c), c10::Error); + ASSERT_TRUE(2 <= c); + ASSERT_TRUE(1 <= c); + ASSERT_FALSE(c <= 1); + + // gt + ASSERT_FALSE(a > a); + ASSERT_FALSE(b > a); + ASSERT_FALSE(a > b); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(a > c), c10::Error); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(c > a), c10::Error); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(a > 3), c10::Error); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + EXPECT_THROW((void)(a > 2), c10::Error); + ASSERT_TRUE(a > 1); +} + +TEST(NestedIntTest, WithFactor) { + auto a = c10::SymInt( + c10::SymNode(c10::make_intrusive(1, 5))); + auto b = c10::SymInt( + c10::SymNode(c10::make_intrusive(1, 10))); + // eq + ASSERT_FALSE(a == b); + ASSERT_FALSE(a >= b); + ASSERT_TRUE(b >= a); + ASSERT_TRUE(a <= b); + ASSERT_FALSE(b <= a); + // ne + ASSERT_TRUE(a != b); + // mul + ASSERT_TRUE(a * 2 == b); + ASSERT_TRUE(a * 3 >= b); + ASSERT_TRUE(a * 2 == 2 * a); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/nn_utils.cpp b/western_ai/usa/pytorch/test/cpp/api/nn_utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bde925afb88e00f9ef2c6fe46d0fcb4abe403027 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/nn_utils.cpp @@ -0,0 +1,893 @@ +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include + +using namespace torch::nn; + +namespace rnn_utils = torch::nn::utils::rnn; + +struct NNUtilsTest : torch::test::SeedingFixture {}; +struct PackedSequenceTest : torch::test::SeedingFixture {}; + +TEST_F(NNUtilsTest, ClipGradNorm) { + auto l = Linear(10, 10); + float max_norm = 2; + auto compute_norm = [&](float norm_type) -> float { + float total_norm = 0.0; + if (norm_type != std::numeric_limits::infinity()) { + for (const auto& p : l->parameters()) { + total_norm += + p.grad().data().abs().pow(norm_type).sum().item().toFloat(); + } + return std::pow(total_norm, 1.0 / norm_type); + } else { + for (const auto& p : l->parameters()) { + auto param_max = p.grad().data().abs().max().item().toFloat(); + if (param_max > total_norm) { + total_norm = param_max; + } + } + return total_norm; + } + }; + auto compare_scaling = + [&](const std::vector& grads) -> torch::Tensor { + std::vector p_scale; + for (const auto i : c10::irange(grads.size())) { + auto param = l->parameters()[i]; + auto grad = grads[i]; + p_scale.push_back(param.grad().data().div(grad).view(-1)); + } + auto scale = torch::cat(p_scale); + return scale; // need to assert std is 0. + }; + + std::vector grads = { + torch::arange(1.0, 101).view({10, 10}), + torch::ones({10}).div(1000), + }; + std::vector norm_types = { + 0.5, + 1.5, + 2.0, + 4.0, + std::numeric_limits::infinity(), + }; + for (auto norm_type : norm_types) { + for (const auto i : c10::irange(grads.size())) { + l->parameters()[i].mutable_grad() = + grads[i].clone().view_as(l->parameters()[i].data()); + } + auto norm_before = compute_norm(norm_type); + auto norm = utils::clip_grad_norm_(l->parameters(), max_norm, norm_type); + auto norm_after = compute_norm(norm_type); + ASSERT_FLOAT_EQ(norm, norm_before); + ASSERT_NEAR(norm_after, max_norm, 1e-6); + ASSERT_LE(norm_after, max_norm); + auto scaled = compare_scaling(grads); + ASSERT_NEAR(0, scaled.std().item().toFloat(), 1e-7); + } + // Small gradients should be left unchanged + grads = { + torch::rand({10, 10}).div(10000), + torch::ones(10).div(500), + }; + for (auto norm_type : norm_types) { + for (const auto i : c10::irange(grads.size())) { + l->parameters()[i].grad().data().copy_(grads[i]); + } + auto norm_before = compute_norm(norm_type); + auto norm = utils::clip_grad_norm_(l->parameters(), max_norm, norm_type); + auto norm_after = compute_norm(norm_type); + ASSERT_FLOAT_EQ(norm, norm_before); + ASSERT_FLOAT_EQ(norm_before, norm_after); + ASSERT_LE(norm_after, max_norm); + auto scaled = compare_scaling(grads); + ASSERT_NEAR(0, scaled.std().item().toFloat(), 1e-7); + ASSERT_FLOAT_EQ(scaled[0].item().toFloat(), 1); + } + // should accept a single tensor as input + auto p1 = torch::randn({10, 10}); + auto p2 = torch::randn({10, 10}); + auto g = torch::arange(1., 101).view({10, 10}); + p1.mutable_grad() = g.clone(); + p2.mutable_grad() = g.clone(); + for (const auto norm_type : norm_types) { + utils::clip_grad_norm_(p1, max_norm, norm_type); + utils::clip_grad_norm_({p2}, max_norm, norm_type); + ASSERT_TRUE(torch::allclose(p1.grad(), p2.grad())); + } +} + +// Check that clip_grad_norm_ raises an error if the norm of a gradient +// is non-finite +TEST_F(NNUtilsTest, ClipGradNormErrorIfNonfinite) { + double inf = std::numeric_limits::infinity(); + double nan = std::numeric_limits::quiet_NaN(); + + using Vector = std::vector; + + Vector norms_pos = {0.1, 1, 2, 3.5, inf}; + Vector norms_neg = {-0.1, -1, -2, -3.5}; + Vector norms_neg_plus_0 = {0, -0.1, -1, -2, -3.5}; + Vector norms_except_0 = {0.1, 1, 2, 3.5, inf, -0.1, -1, -2, -3.5}; + Vector norms_all = {0, 0.1, 1, 2, 3.5, inf, -0.1, -1, -2, -3.5}; + + // Each entry in test_cases has the following values, in this order: + // + // grad_only_one_elem If True, only one element of the parameter's + // gradient is set to the scalar grad, and the + // rest of the elements are 0. If False, all grad + // elements are equal to the scalar. + // + // prefix_finite_grad_param If True, prefix a parameter that has a grad + // of 1. + // + // scalars Scalars to use as the parameter's grad, through + // multiplication + // + // norms_nonfinite Norm types that should produce nonfinite total norm + // + // norms_finite Norm types that should produce finite total norm + std::vector> test_cases({ + // Test errors from an infinite grad + std::make_tuple( + false, false, Vector({inf, -inf}), norms_except_0, Vector({0})), + std::make_tuple( + false, true, Vector({inf, -inf}), norms_pos, norms_neg_plus_0), + std::make_tuple( + true, false, Vector({inf, -inf}), norms_pos, norms_neg_plus_0), + std::make_tuple( + false, true, Vector({inf, -inf}), norms_pos, norms_neg_plus_0), + + // Test errors from a NaN grad + std::make_tuple(false, false, Vector({nan}), norms_except_0, Vector({0})), + std::make_tuple(false, true, Vector({nan}), norms_except_0, Vector({0})), + std::make_tuple(true, false, Vector({nan}), norms_except_0, Vector({0})), + std::make_tuple(true, true, Vector({nan}), norms_except_0, Vector({0})), + + // Test a grad that should never error + std::make_tuple(false, false, Vector({2e22, -2e22}), Vector(), norms_all), + std::make_tuple(false, true, Vector({2e22, -2e22}), Vector(), norms_all), + std::make_tuple(true, false, Vector({2e22, -2e22}), Vector(), norms_all), + std::make_tuple(true, true, Vector({2e22, -2e22}), Vector(), norms_all), + + // Test a grad that will overflow to inf for only some norm orders + std::make_tuple( + false, + false, + Vector({2e200, -2e200}), + Vector({3.5, 2, -2, -3.5}), + Vector({inf, 1, 0.1, 0, -1, -0.1})), + std::make_tuple( + false, + true, + Vector({2e200, -2e200}), + Vector({3.5, 2}), + Vector({inf, 1, 0.1, 0, -1, -0.1, -2, -3.5})), + std::make_tuple( + true, + false, + Vector({2e200, -2e200}), + Vector({3.5, 2}), + Vector({inf, 1, 0.1, 0, -1, -0.1, -2, -3.5})), + std::make_tuple( + false, + true, + Vector({2e200, -2e200}), + Vector({3.5, 2}), + Vector({inf, 1, 0.1, 0, -1, -0.1, -2, -3.5})), + }); + + auto gen_parameters = [](double scalar, + bool grad_only_one_elem, + bool prefix_finite_grad_param, + torch::DeviceType device_type) { + auto param = torch::ones( + 10, + torch::TensorOptions() + .dtype(torch::kDouble) + .device(device_type) + .requires_grad(true)); + if (grad_only_one_elem) { + param[1].mul(scalar).sum().backward(); + } else { + param.mul(scalar).sum().backward(); + } + + std::vector parameters; + if (prefix_finite_grad_param) { + auto prefix_param = torch::ones( + 1, + torch::TensorOptions() + .dtype(torch::kDouble) + .device(device_type) + .requires_grad(true)); + prefix_param.mul(1).sum().backward(); + parameters.push_back(prefix_param); + } + parameters.push_back(param); + + return parameters; + }; + + auto run_test_case = [&gen_parameters]( + double norm_type, + bool error_if_nonfinite, + double scalar, + bool grad_only_one_elem, + bool prefix_finite_grad_param, + bool is_norm_nonfinite, + torch::DeviceType device_type) { + std::stringstream ss; + ss << "device: " << device_type << ", norm_type: " << norm_type + << ", error_if_nonfinite: " << error_if_nonfinite + << ", scalar: " << scalar + << ", grad_only_one_elem: " << grad_only_one_elem + << ", prefix_finite_grad_param: " << prefix_finite_grad_param + << ", is_norm_nonfinite: " << is_norm_nonfinite; + std::string msg = std::move(ss).str(); + + auto parameters = gen_parameters( + scalar, grad_only_one_elem, prefix_finite_grad_param, device_type); + + if (is_norm_nonfinite && error_if_nonfinite) { + std::vector grads_before; + // NOLINTNEXTLINE(performance-for-range-copy) + for (auto p : parameters) { + // NOLINTNEXTLINE(performance-inefficient-vector-operation) + grads_before.push_back(p.grad().clone()); + } + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + EXPECT_THROW( + utils::clip_grad_norm_(parameters, 1., norm_type, true), + std::exception) + << msg; + // Grads should not change if error is thrown + for (const auto p_idx : c10::irange(parameters.size())) { + ASSERT_TRUE(torch::allclose( + parameters[p_idx].grad(), + grads_before[p_idx], + 1.0, + 0.0, + /*equal_nan*/ true)) + << msg; + } + } else { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + EXPECT_NO_THROW( + utils::clip_grad_norm_(parameters, 1., norm_type, error_if_nonfinite)) + << msg; + } + }; + + for (auto device_type : {torch::kCPU, torch::kCUDA}) { + if (device_type == torch::kCUDA && !torch::cuda::is_available()) { + continue; + } + for (auto test_case : test_cases) { + auto grad_only_one_elem = std::get<0>(test_case); + auto prefix_finite_grad_param = std::get<1>(test_case); + auto scalars = std::get<2>(test_case); + auto norms_nonfinite = std::get<3>(test_case); + auto norms_finite = std::get<4>(test_case); + + for (auto error_if_nonfinite : {false, true}) { + for (auto scalar : scalars) { + for (auto norm_type : norms_nonfinite) { + run_test_case( + norm_type, + error_if_nonfinite, + scalar, + grad_only_one_elem, + prefix_finite_grad_param, + true, + device_type); + } + + for (auto norm_type : norms_finite) { + run_test_case( + norm_type, + error_if_nonfinite, + scalar, + grad_only_one_elem, + prefix_finite_grad_param, + false, + device_type); + } + } + } + } + } +} + +TEST_F(NNUtilsTest, ClipGradValue) { + auto l = Linear(10, 10); + float clip_value = 2.5; + + torch::Tensor grad_w = torch::arange(-50., 50).view({10, 10}).div_(5); + torch::Tensor grad_b = torch::ones({10}).mul_(2); + std::vector> grad_lists = { + {grad_w, grad_b}, {grad_w, torch::Tensor()}}; + for (auto grad_list : grad_lists) { + for (const auto i : c10::irange(grad_list.size())) { + auto p = l->parameters()[i]; + auto g = grad_list[i]; + p.mutable_grad() = g.defined() ? g.clone().view_as(p.data()) : g; + } + + utils::clip_grad_value_(l->parameters(), clip_value); + for (const auto& p : l->parameters()) { + if (p.grad().defined()) { + ASSERT_LE(p.grad().data().max().item().toFloat(), clip_value); + ASSERT_GE(p.grad().data().min().item().toFloat(), -clip_value); + } + } + } + + // Should accept a single Tensor as input + auto p1 = torch::randn({10, 10}); + auto p2 = torch::randn({10, 10}); + auto g = torch::arange(-50., 50).view({10, 10}).div_(5); + p1.mutable_grad() = g.clone(); + p2.mutable_grad() = g.clone(); + utils::clip_grad_value_(p1, clip_value); + utils::clip_grad_value_({p2}, clip_value); + ASSERT_TRUE(torch::allclose(p1.grad(), p2.grad())); +} + +TEST_F(NNUtilsTest, ConvertParameters) { + std::vector parameters{ + torch::arange(9, torch::kFloat32), + torch::arange(9, torch::kFloat32).view({3, 3}), + torch::arange(8, torch::kFloat32).view({2, 2, 2})}; + + auto expected = torch::cat( + {torch::arange(9, torch::kFloat32), + torch::arange(9, torch::kFloat32).view(-1), + torch::arange(8, torch::kFloat32).view(-1)}); + auto vector = utils::parameters_to_vector(parameters); + ASSERT_TRUE(vector.allclose(expected)); + + std::vector zero_parameters{ + torch::zeros({9}, torch::kFloat32), + torch::zeros({9}, torch::kFloat32).view({3, 3}), + torch::zeros({8}, torch::kFloat32).view({2, 2, 2})}; + + utils::vector_to_parameters(vector, zero_parameters); + for (const auto i : c10::irange(zero_parameters.size())) { + ASSERT_TRUE(zero_parameters[i].allclose(parameters[i])); + } + + { + auto conv1 = Conv2d(3, 10, 5); + auto fc1 = Linear(10, 20); + auto model = Sequential(conv1, fc1); + + auto vec = utils::parameters_to_vector(model->parameters()); + ASSERT_EQ(vec.size(0), 980); + } + { + auto conv1 = Conv2d(3, 10, 5); + auto fc1 = Linear(10, 20); + auto model = Sequential(conv1, fc1); + + auto vec = torch::arange(0., 980); + utils::vector_to_parameters(vec, model->parameters()); + + auto sample = model->parameters()[0][0][0][0]; + ASSERT_TRUE(torch::equal(sample.data(), vec.data().slice(0, 0, 5))); + } +} + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,cppcoreguidelines-avoid-non-const-global-variables) +int64_t PackedSequenceTest_batch_size = 5; +// NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,cppcoreguidelines-avoid-non-const-global-variables) +int64_t PackedSequenceTest_max_length = 6; + +std::vector PackedSequenceTest_ordered_sequence( + torch::ScalarType tensor_type) { + std::vector seqs; + seqs.reserve(PackedSequenceTest_batch_size); + for ([[maybe_unused]] const auto i : + c10::irange(PackedSequenceTest_batch_size)) { + seqs.emplace_back(torch::empty( + {torch::randint(1, PackedSequenceTest_max_length, {1}).item()}, + tensor_type)); + } + for (auto& s : seqs) { + s.random_(-128, 128); + } + sort( + seqs.begin(), + seqs.end(), + [&](const torch::Tensor& t1, const torch::Tensor& t2) { + return t1.size(0) > t2.size(0); + }); + return seqs; +} + +std::tuple PackedSequenceTest_padded_sequence( + torch::ScalarType tensor_type) { + // Create Tensor of random padded sequences + auto ordered = PackedSequenceTest_ordered_sequence(tensor_type); + auto lengths = torch::empty({(int64_t)ordered.size()}, torch::kInt64); + for (const auto i : c10::irange(ordered.size())) { + lengths[i] = ordered[i].size(0); + } + auto padded_tensor = rnn_utils::pad_sequence(ordered); + return std::make_tuple(padded_tensor, lengths); +} + +void assert_is_equal_packed_sequence( + const rnn_utils::PackedSequence& a, + const rnn_utils::PackedSequence& b) { + ASSERT_TRUE(torch::allclose(a.data(), b.data())); + ASSERT_TRUE(torch::allclose(a.batch_sizes(), b.batch_sizes())); + ASSERT_TRUE( + (!a.sorted_indices().defined() && !b.sorted_indices().defined()) || + torch::allclose(a.sorted_indices(), b.sorted_indices())); + ASSERT_TRUE( + (!a.unsorted_indices().defined() && !b.unsorted_indices().defined()) || + torch::allclose(a.unsorted_indices(), b.unsorted_indices())); +} + +void assert_is_same_packed_sequence( + const rnn_utils::PackedSequence& a, + const rnn_utils::PackedSequence& b) { + ASSERT_TRUE(a.data().is_same(b.data())); + ASSERT_TRUE(a.batch_sizes().is_same(b.batch_sizes())); + ASSERT_TRUE(a.sorted_indices().is_same(b.sorted_indices())); + ASSERT_TRUE(a.unsorted_indices().is_same(b.unsorted_indices())); +} + +TEST_F(PackedSequenceTest, WrongOrder) { + auto a = torch::ones({25, 300}); + auto b = torch::ones({22, 300}); + auto b_a = rnn_utils::pad_sequence({b, a}); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW( + rnn_utils::pack_padded_sequence( + b_a, + torch::tensor({22, 25}), + /*batch_first=*/false, + /*enforce_sorted=*/true), + c10::Error); +} + +TEST_F(PackedSequenceTest, TotalLength) { + auto [padded, lengths] = PackedSequenceTest_padded_sequence(torch::kFloat); + int64_t max_length = torch::max(lengths).item(); + rnn_utils::PackedSequence packed = + rnn_utils::pack_padded_sequence(padded, lengths); + + // test ValueError if total_length < max_length + for (int64_t total_length : std::vector{-1, 0, max_length - 1}) { + for (bool batch_first : std::vector{true, false}) { + auto err_fn = [&]() { + rnn_utils::pad_packed_sequence( + packed, + /*batch_first=*/batch_first, + /*padding_value=*/0.0, + /*total_length=*/total_length); + }; + ASSERT_THROWS_WITH( + err_fn(), + "Expected total_length to be at least the length of the longest sequence in input"); + } + } + + // test that pad_packed_sequence returns results of correct length + for (bool batch_first : std::vector{true, false}) { + auto no_extra_pad = std::get<0>( + rnn_utils::pad_packed_sequence(packed, /*batch_first=*/batch_first)); + for (int64_t total_length_delta : std::vector{0, 1, 8}) { + int64_t total_length = max_length + total_length_delta; + auto [unpacked, lengths_out] = rnn_utils::pad_packed_sequence( + packed, + /*batch_first=*/batch_first, + /*padding_value=*/0.0, + /*total_length=*/total_length); + ASSERT_TRUE(torch::allclose(lengths, lengths_out)); + ASSERT_EQ(unpacked.size(batch_first ? 1 : 0), total_length); + torch::Tensor ref_output, extra_pad; + if (total_length_delta == 0) { + ref_output = no_extra_pad; + } else if (batch_first) { + extra_pad = torch::zeros( + {PackedSequenceTest_batch_size, total_length_delta}, + no_extra_pad.options()); + ref_output = torch::cat({no_extra_pad, extra_pad}, 1); + } else { + extra_pad = torch::zeros( + {total_length_delta, PackedSequenceTest_batch_size}, + no_extra_pad.options()); + ref_output = torch::cat({no_extra_pad, extra_pad}, 0); + } + ASSERT_TRUE(torch::allclose(unpacked, ref_output)); + } + } +} + +TEST_F(PackedSequenceTest, To) { + for (bool enforce_sorted : std::vector{true, false}) { + auto [padded, lengths] = PackedSequenceTest_padded_sequence(torch::kInt); + rnn_utils::PackedSequence a = rnn_utils::pack_padded_sequence( + padded, + lengths, + /*batch_first=*/false, + /*enforce_sorted=*/enforce_sorted) + .cpu(); + + assert_is_same_packed_sequence(a, a.to(torch::kCPU)); + assert_is_same_packed_sequence(a, a.cpu()); + assert_is_same_packed_sequence( + a, a.to(torch::device(torch::kCPU).dtype(torch::kInt32))); + + if (torch::cuda::is_available()) { + auto b = a.cuda(); + assert_is_same_packed_sequence(b, b.to(torch::kCUDA)); + assert_is_same_packed_sequence(b, b.cuda()); + assert_is_equal_packed_sequence(a, b.to(torch::kCPU)); + assert_is_equal_packed_sequence(b, a.to(torch::kCUDA)); + assert_is_equal_packed_sequence( + a, b.to(torch::device(torch::kCPU).dtype(torch::kInt32))); + assert_is_same_packed_sequence(b, b.to(torch::kInt32)); + } + } +} + +TEST_F(NNUtilsTest, PackSequence) { + auto _compatibility_test = [&](torch::ArrayRef sequences, + torch::Tensor lengths, + bool batch_first, + bool enforce_sorted = false) { + torch::Tensor padded = rnn_utils::pad_sequence(sequences, batch_first); + rnn_utils::PackedSequence packed = + rnn_utils::pack_sequence(sequences, enforce_sorted); + std::tuple unpacked = + rnn_utils::pad_packed_sequence(packed, batch_first); + ASSERT_TRUE(torch::allclose(padded, std::get<0>(unpacked))); + rnn_utils::PackedSequence pack_padded = rnn_utils::pack_padded_sequence( + padded, lengths, batch_first, enforce_sorted); + assert_is_equal_packed_sequence(packed, pack_padded); + }; + + // single dimensional + auto a = torch::tensor({1, 2, 3}); + auto b = torch::tensor({4, 5}); + auto c = torch::tensor({6}); + rnn_utils::PackedSequence packed = + rnn_utils::pack_sequence({a, b, c}, /*enforce_sorted=*/false); + auto expected = torch::tensor({1, 4, 6, 2, 5, 3}); + ASSERT_TRUE(torch::allclose(packed.batch_sizes(), torch::tensor({3, 2, 1}))); + ASSERT_TRUE(torch::allclose(packed.data(), expected)); + ASSERT_TRUE( + torch::allclose(packed.sorted_indices(), torch::tensor({0, 1, 2}))); + ASSERT_TRUE( + torch::allclose(packed.unsorted_indices(), torch::tensor({0, 1, 2}))); + + rnn_utils::PackedSequence packed_unsorted = + rnn_utils::pack_sequence({b, c, a}, /*enforce_sorted=*/false); + ASSERT_TRUE( + torch::allclose(packed_unsorted.batch_sizes(), torch::tensor({3, 2, 1}))); + ASSERT_TRUE(torch::allclose(packed_unsorted.data(), expected)); + ASSERT_TRUE(torch::allclose( + packed_unsorted.sorted_indices(), torch::tensor({2, 0, 1}))); + ASSERT_TRUE(torch::allclose( + packed_unsorted.unsorted_indices(), torch::tensor({1, 2, 0}))); + + // single dimensional, enforce_sorted = True + rnn_utils::PackedSequence packed_enforce_sorted = + rnn_utils::pack_sequence({a, b, c}, /*enforce_sorted=*/true); + ASSERT_TRUE(torch::allclose( + packed_enforce_sorted.batch_sizes(), torch::tensor({3, 2, 1}))); + ASSERT_TRUE(torch::allclose(packed_enforce_sorted.data(), expected)); + ASSERT_FALSE(packed_enforce_sorted.sorted_indices().defined()); + ASSERT_FALSE(packed_enforce_sorted.unsorted_indices().defined()); + + ASSERT_THROWS_WITH( + rnn_utils::pack_sequence({b, c, a}, /*enforce_sorted=*/true), + "must be sorted in decreasing order"); + + ASSERT_THROWS_WITH( + rnn_utils::pack_sequence({b, c, a}, /*enforce_sorted=*/true), + "You can pass `enforce_sorted=False`"); + + // more dimensions + int64_t maxlen = 9; + for (int64_t num_dim : std::vector{0, 1, 2, 3}) { + std::vector sequences; + std::vector lengths_vec; + std::vector trailing_dims(num_dim, 4); + for (int64_t i = maxlen; i > 0; i--) { + int64_t seq_len = i * i; + lengths_vec.emplace_back(seq_len); + std::vector tensor_sizes{seq_len, 5}; + tensor_sizes.insert( + tensor_sizes.end(), trailing_dims.begin(), trailing_dims.end()); + sequences.emplace_back(torch::rand(tensor_sizes)); + } + std::vector unsorted_sequences; + for (const auto& s : sequences) { + // NOLINTNEXTLINE(performance-inefficient-vector-operation) + unsorted_sequences.emplace_back(s.clone()); + } + std::shuffle( + std::begin(unsorted_sequences), + std::end(unsorted_sequences), + std::default_random_engine{}); + + std::vector unsorted_sequences_lengths_vec; + for (const auto& t : unsorted_sequences) { + // NOLINTNEXTLINE(performance-inefficient-vector-operation) + unsorted_sequences_lengths_vec.emplace_back(t.size(0)); + } + + // compatibility with other utilities + for (bool batch_first : std::vector{true, false}) { + for (bool enforce_sorted : std::vector{true, false}) { + _compatibility_test( + sequences, torch::tensor(lengths_vec), batch_first, enforce_sorted); + } + _compatibility_test( + unsorted_sequences, + torch::tensor(unsorted_sequences_lengths_vec), + batch_first); + } + } +} + +TEST_F(NNUtilsTest, PackPaddedSequence) { + auto generate_test_case = [&](torch::ArrayRef sorted_lengths, + bool should_shuffle) { + auto pad = [&](torch::Tensor tensor, int64_t length) { + std::vector tensor_sizes{length - tensor.size(0)}; + tensor_sizes.insert( + tensor_sizes.end(), + tensor.sizes().slice(1).begin(), + tensor.sizes().slice(1).end()); + return torch::cat({tensor, torch::zeros(tensor_sizes, tensor.options())}); + }; + int64_t max_length = sorted_lengths[0]; + torch::Tensor batch_sizes = torch::empty({max_length}, torch::kInt64); + for (int64_t i = 1; i < max_length + 1; i++) { + int64_t total = 0; + for (const auto& x : sorted_lengths) { + if (x >= i) { + total++; + } + } + batch_sizes[i - 1] = total; + } + std::vector tensors_to_be_cat; + for (int64_t i = 1; i < static_cast(sorted_lengths.size() + 1); + i++) { + int64_t l = sorted_lengths.at(i - 1); + tensors_to_be_cat.emplace_back(pad( + i * 100 + torch::arange(1., 5 * l + 1).view({l, 1, 5}), max_length)); + } + auto padded = torch::cat(tensors_to_be_cat, 1); + std::vector expected_data_vec; + for (const auto n : c10::irange(batch_sizes.size(0))) { + int64_t batch_size = batch_sizes[n].item(); + for (const auto i : c10::irange(batch_size)) { + expected_data_vec.emplace_back( + torch::arange(1., 6) + (i + 1) * 100 + 5 * n); + } + } + auto expected_data = torch::stack(expected_data_vec, /*dim=*/0); + + torch::Tensor unsorted_indices, lengths; + if (should_shuffle) { + // Shuffle the padded sequence to create an unsorted sequence + std::vector permutation; + for (const auto i : c10::irange(sorted_lengths.size())) { + permutation.emplace_back(i); + } + std::shuffle( + std::begin(permutation), + std::end(permutation), + std::default_random_engine{}); + + unsorted_indices = torch::tensor(permutation); + padded = padded.index_select(1, unsorted_indices); + lengths = torch::tensor(sorted_lengths).index_select(0, unsorted_indices); + } else { + unsorted_indices = torch::Tensor(); + lengths = torch::tensor(sorted_lengths); + } + + return std::make_tuple( + padded.requires_grad_(), + lengths, + expected_data, + batch_sizes, + unsorted_indices); + }; + + std::vector, bool>> test_cases = { + // sorted_lengths, should_shuffle + {{10, 8, 4, 2, 2, 2, 1}, false}, + {{11, 10, 8, 6, 4, 3, 1}, false}, + {{11, 10, 8, 6, 4, 3, 1}, true}}; + + for (const auto& test_case : test_cases) { + for (bool batch_first : std::vector{true, false}) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + std::vector sorted_lengths = std::get<0>(test_case); + bool should_shuffle = std::get<1>(test_case); + + auto [padded, lengths, expected_data, batch_sizes, unsorted_indices] = + generate_test_case(sorted_lengths, should_shuffle); + + auto src = padded; + if (batch_first) { + src = src.transpose(0, 1); + } + + // check output + rnn_utils::PackedSequence packed = rnn_utils::pack_padded_sequence( + src, + lengths, + /*batch_first=*/batch_first, + /*enforce_sorted=*/!should_shuffle); + ASSERT_TRUE(torch::allclose(packed.data(), expected_data)); + ASSERT_TRUE(torch::allclose(packed.batch_sizes(), batch_sizes)); + ASSERT_TRUE( + (!packed.unsorted_indices().defined() && + !unsorted_indices.defined()) || + torch::allclose(packed.unsorted_indices(), unsorted_indices)); + + // test inverse + auto [unpacked, unpacked_len] = + rnn_utils::pad_packed_sequence(packed, /*batch_first=*/batch_first); + ASSERT_TRUE(torch::allclose(unpacked, src)); + ASSERT_TRUE(torch::allclose(unpacked_len, lengths)); + + // check grad + if (padded.grad().defined()) { + torch::NoGradGuard no_grad; + padded.grad().zero_(); + } + torch::Tensor grad_output; + { + torch::NoGradGuard no_grad; + grad_output = unpacked.clone().normal_(); + } + unpacked.backward(grad_output); + if (batch_first) { + grad_output.transpose_(0, 1); + } + for (const auto i : c10::irange(lengths.size(0))) { + int64_t l = lengths[i].item(); + ASSERT_TRUE(torch::allclose( + padded.grad().narrow(0, 0, l).select(1, i), + grad_output.narrow(0, 0, l).select(1, i))); + if (l < 10) { + ASSERT_EQ( + padded.grad() + .narrow(0, l, padded.grad().size(0) - l) + .select(1, i) + .abs() + .sum() + .item(), + 0); + } + } + } + } + + // test error messages + ASSERT_THROWS_WITH( + rnn_utils::pack_padded_sequence( + torch::randn({3, 3}), torch::tensor({1, 3, 2})), + "You can pass `enforce_sorted=False`"); + ASSERT_THROWS_WITH( + rnn_utils::pack_padded_sequence(torch::randn({0, 0}), torch::tensor({})), + "empty tensor"); +} + +TEST_F(NNUtilsTest, PadSequence) { + auto pad = [&](const torch::Tensor& tensor, int64_t length) { + torch::NoGradGuard no_grad; + std::vector tensor_sizes{length - tensor.size(0)}; + tensor_sizes.insert( + tensor_sizes.end(), + tensor.sizes().slice(1).begin(), + tensor.sizes().slice(1).end()); + return torch::cat({tensor, torch::zeros(tensor_sizes, tensor.options())}); + }; + + // single dimensional + auto a = torch::tensor({1, 2, 3}); + auto b = torch::tensor({4, 5}); + auto c = torch::tensor({6}); + + torch::Tensor expected, padded; + + // batch_first = true + expected = torch::tensor({{4, 5, 0}, {1, 2, 3}, {6, 0, 0}}); + padded = rnn_utils::pad_sequence({b, a, c}, true); + ASSERT_TRUE(padded.allclose(expected)); + + // batch_first = false + padded = rnn_utils::pad_sequence({b, a, c}); + ASSERT_TRUE(padded.allclose(expected.transpose(0, 1))); + + // padding_side = "left", batch_first = true + expected = torch::tensor({{0, 4, 5}, {1, 2, 3}, {0, 0, 6}}); + padded = rnn_utils::pad_sequence({b, a, c}, true, 0, "left"); + ASSERT_TRUE(padded.allclose(expected)); + + // padding_side = "left", batch_first = false + padded = rnn_utils::pad_sequence({b, a, c}, false, 0, "left"); + ASSERT_TRUE(padded.allclose(expected.transpose(0, 1))); + + // pad with non-zero value + expected = torch::tensor({{4, 5, 1}, {1, 2, 3}, {6, 1, 1}}); + padded = rnn_utils::pad_sequence({b, a, c}, true, 1); + ASSERT_TRUE(padded.allclose(expected)); + + // Test pad sorted sequence + expected = torch::tensor({{1, 2, 3}, {4, 5, 0}, {6, 0, 0}}); + padded = rnn_utils::pad_sequence({a, b, c}, true); + ASSERT_TRUE(padded.allclose(expected)); + + // more dimensions + int64_t maxlen = 9; + for (int64_t num_dim : std::vector{0, 1, 2, 3}) { + std::vector sequences; + std::vector trailing_dims(num_dim, 4); + for (int64_t i = 1; i < maxlen + 1; i++) { + int64_t seq_len = i * i; + std::vector tensor_sizes{seq_len, 5}; + tensor_sizes.insert( + tensor_sizes.end(), trailing_dims.begin(), trailing_dims.end()); + sequences.emplace_back(torch::rand(tensor_sizes)); + } + std::shuffle( + std::begin(sequences), + std::end(sequences), + std::default_random_engine{}); + std::vector expected_tensors; + for (const torch::Tensor& seq : sequences) { + // NOLINTNEXTLINE(performance-inefficient-vector-operation) + expected_tensors.emplace_back(pad(seq, maxlen * maxlen)); + } + + // batch first = true + auto expected = torch::stack(expected_tensors); + auto padded = rnn_utils::pad_sequence(sequences, true); + ASSERT_TRUE(padded.allclose(expected)); + + // batch first = false + padded = rnn_utils::pad_sequence(sequences); + ASSERT_TRUE(padded.allclose(expected.transpose(0, 1))); + + // reset expected_tensors for padding_side + expected_tensors.clear(); + for (const torch::Tensor& seq : sequences) { + // NOLINTNEXTLINE(performance-inefficient-vector-operation) + expected_tensors.emplace_back( + torch::flip(pad(torch::flip(seq, {0}), maxlen * maxlen), {0})); + } + expected = torch::stack(expected_tensors); + // padding_side = "left", batch_first = true + padded = rnn_utils::pad_sequence(sequences, true, 0, "left"); + ASSERT_TRUE(padded.allclose(expected)); + + // padding_side = "left", batch_first = false + padded = rnn_utils::pad_sequence(sequences, false, 0, "left"); + ASSERT_TRUE(padded.allclose(expected.transpose(0, 1))); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/operations.cpp b/western_ai/usa/pytorch/test/cpp/api/operations.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a614e4fd8c988d4fdcf724c8b92c9b09976d4cb1 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/operations.cpp @@ -0,0 +1,88 @@ +#include + +#include +#include + +#include +struct OperationTest : torch::test::SeedingFixture { + protected: + void SetUp() override {} + + const int TEST_AMOUNT = 10; +}; + +TEST_F(OperationTest, Lerp) { + for ([[maybe_unused]] const auto i : c10::irange(TEST_AMOUNT)) { + // test lerp_kernel_scalar + auto start = torch::rand({3, 5}); + auto end = torch::rand({3, 5}); + auto scalar = 0.5; + // expected and actual + auto scalar_expected = start + scalar * (end - start); + auto out = torch::lerp(start, end, scalar); + // compare + ASSERT_EQ(out.dtype(), scalar_expected.dtype()); + ASSERT_TRUE(out.allclose(scalar_expected)); + + // test lerp_kernel_tensor + auto weight = torch::rand({3, 5}); + // expected and actual + auto tensor_expected = start + weight * (end - start); + out = torch::lerp(start, end, weight); + // compare + ASSERT_EQ(out.dtype(), tensor_expected.dtype()); + ASSERT_TRUE(out.allclose(tensor_expected)); + } +} + +TEST_F(OperationTest, Cross) { + for ([[maybe_unused]] const auto i : c10::irange(TEST_AMOUNT)) { + // input + auto a = torch::rand({10, 3}); + auto b = torch::rand({10, 3}); + // expected + auto exp = torch::empty({10, 3}); + for (const auto j : c10::irange(10)) { + auto u1 = a[j][0], u2 = a[j][1], u3 = a[j][2]; + auto v1 = b[j][0], v2 = b[j][1], v3 = b[j][2]; + exp[j][0] = u2 * v3 - v2 * u3; + exp[j][1] = v1 * u3 - u1 * v3; + exp[j][2] = u1 * v2 - v1 * u2; + } + // actual + auto out = torch::cross(a, b); + // compare + ASSERT_EQ(out.dtype(), exp.dtype()); + ASSERT_TRUE(out.allclose(exp)); + } +} + +TEST_F(OperationTest, Linear_out) { + { + const auto x = torch::arange(100., 118).resize_({3, 3, 2}); + const auto w = torch::arange(200., 206).resize_({3, 2}); + const auto b = torch::arange(300., 303); + auto y = torch::empty({3, 3, 3}); + at::linear_out(y, x, w, b); + const auto y_exp = torch::tensor( + {{{40601, 41004, 41407}, {41403, 41814, 42225}, {42205, 42624, 43043}}, + {{43007, 43434, 43861}, {43809, 44244, 44679}, {44611, 45054, 45497}}, + {{45413, 45864, 46315}, {46215, 46674, 47133}, {47017, 47484, 47951}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, y_exp)); + } + { + const auto x = torch::arange(100., 118).resize_({3, 3, 2}); + const auto w = torch::arange(200., 206).resize_({3, 2}); + auto y = torch::empty({3, 3, 3}); + at::linear_out(y, x, w); + ASSERT_EQ(y.ndimension(), 3); + ASSERT_EQ(y.sizes(), torch::IntArrayRef({3, 3, 3})); + const auto y_exp = torch::tensor( + {{{40301, 40703, 41105}, {41103, 41513, 41923}, {41905, 42323, 42741}}, + {{42707, 43133, 43559}, {43509, 43943, 44377}, {44311, 44753, 45195}}, + {{45113, 45563, 46013}, {45915, 46373, 46831}, {46717, 47183, 47649}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, y_exp)); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/optim.cpp b/western_ai/usa/pytorch/test/cpp/api/optim.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f1c75c60c06f578a785eaed2e97419de465d41de --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/optim.cpp @@ -0,0 +1,1071 @@ +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace torch::nn; +using namespace torch::optim; + +template +static bool test_optimizer_xor(Options options) { + torch::manual_seed(0); + + Sequential model( + Linear(2, 8), + Functional(torch::sigmoid), + Linear(8, 1), + Functional(torch::sigmoid)); + + const int64_t kBatchSize = 200; + const int64_t kMaximumNumberOfEpochs = 3000; + + OptimizerClass optimizer(model->parameters(), std::move(options)); + + double running_loss = 1; + int epoch = 0; + while (running_loss > 0.1) { + auto inputs = torch::empty({kBatchSize, 2}); + auto labels = torch::empty({kBatchSize}); + for (const auto i : c10::irange(kBatchSize)) { + inputs[i] = torch::randint(2, {2}, torch::kInt64); + labels[i] = inputs[i][0].item() ^ inputs[i][1].item(); + } + + inputs.set_requires_grad(true); + + auto step = [&](OptimizerClass& optimizer, + Sequential model, + const torch::Tensor& inputs, + const torch::Tensor& labels) { + auto closure = [&]() { + optimizer.zero_grad(); + auto x = model->forward(inputs); + auto loss = torch::binary_cross_entropy(x, labels); + loss.backward(); + return loss; + }; + return optimizer.step(closure); + }; + + torch::Tensor loss = step(optimizer, model, inputs, labels); + + running_loss = running_loss * 0.99 + loss.item() * 0.01; + if (epoch > kMaximumNumberOfEpochs) { + std::cout << "Loss is too high after epoch " << epoch << ": " + << running_loss << '\n'; + return false; + } + epoch++; + } + return true; +} + +template +static void assign_parameter( + const Parameters& parameters, + const char* name, + const torch::Tensor& new_tensor) { + auto parameter = parameters[name]; + parameter.set_requires_grad(false); + parameter.flatten().copy_(new_tensor); + parameter.set_requires_grad(true); +} + +template +static void check_exact_values( + Options options, + std::vector> expected_parameters) { + const size_t kIterations = 1001; + const size_t kSampleEvery = 100; + + torch::manual_seed(0); + + Sequential model( + Linear(2, 3), + Functional(torch::sigmoid), + Linear(3, 1), + Functional(torch::sigmoid)); + + model->to(torch::kFloat64); + + // Use exact input values because matching random values is hard. + auto parameters = model->named_parameters(); + assign_parameter( + parameters, + "0.weight", + torch::tensor( + {-0.2109, -0.4976, -0.1413, -0.3420, -0.2524, 0.6976}, + torch::kFloat64)); + assign_parameter( + parameters, + "0.bias", + torch::tensor({-0.1085, -0.2979, 0.6892}, torch::kFloat64)); + assign_parameter( + parameters, + "2.weight", + torch::tensor({-0.0508, -0.3941, -0.2843}, torch::kFloat64)); + assign_parameter( + parameters, "2.bias", torch::tensor({-0.0711}, torch::kFloat64)); + + auto optimizer = OptimizerClass(parameters.values(), std::move(options)); + torch::Tensor input = + torch::tensor({0.1, 0.2, 0.3, 0.4, 0.5, 0.6}, torch::kFloat64) + .reshape({3, 2}); + + for (const auto i : c10::irange(kIterations)) { + optimizer.zero_grad(); + auto output = model->forward(input); + auto loss = output.sum(); + loss.backward(); + + auto closure = []() { return torch::tensor({10}); }; + optimizer.step(closure); + + if (i % kSampleEvery == 0) { + ASSERT_TRUE( + expected_parameters.at(i / kSampleEvery).size() == parameters.size()); + for (const auto p : c10::irange(parameters.size())) { + ASSERT_TRUE(parameters[p]->defined()); + // Always compare using double dtype, regardless of the original dtype + // of the tensors + auto computed = parameters[p]->flatten().to(torch::kFloat64); + auto expected = + expected_parameters.at(i / kSampleEvery).at(p).to(torch::kFloat64); + if (!computed.allclose(expected, /*rtol=*/1e-3, /*atol=*/5e-4)) { + std::cout << "Iteration " << i << ": " << computed + << " != " << expected << " (parameter " << p << ')' << '\n'; + ASSERT_TRUE(false); + } + } + } + } +} + +TEST(OptimTest, OptimizerAccessors) { + auto options = AdagradOptions(1.0); + std::vector params; + for ([[maybe_unused]] const auto i : c10::irange(3)) { + params.push_back(torch::randn(10)); + } + auto optimizer = Adagrad(params, options); + // test for defaults() method with non-const reference + auto& options_ = static_cast(optimizer.defaults()); + ASSERT_TRUE(options == options_); + // test for param_groups() with non-const reference return + auto& params_groups = optimizer.param_groups(); + params_groups.emplace_back(params); + auto& params_1 = params_groups[1].params(); + for (const auto i : c10::irange(params_1.size())) { + torch::equal(params[i], params_1[i]); + } + + // test for add_param_group() when one or more params existing in another + // param_group are passed in the new param group to be added + ASSERT_THROWS_WITH( + optimizer.add_param_group(OptimizerParamGroup(params)), + "some parameters appear in more than one parameter group"); + + // test for state() with non-const reference return + auto& state_ = static_cast( + *(optimizer.state()[params_1[0].unsafeGetTensorImpl()])); + state_.step(state_.step() + 1); + + const auto& optimizer_ = Adagrad(params, options); + optimizer_.defaults(); + // test for param_groups() with const reference return + (void)optimizer_.param_groups(); + // test for state() with const reference return + optimizer_.state(); +} + +#define OLD_INTERFACE_WARNING_CHECK(func) \ + { \ + torch::test::WarningCapture warnings; \ + func; \ + ASSERT_EQ( \ + torch::test::count_substr_occurrences( \ + warnings.str(), "will be removed"), \ + 1); \ + } + +struct MyOptimizerOptions + : public OptimizerCloneableOptions { + MyOptimizerOptions(double lr = 1.0) : lr_(lr) {} + TORCH_ARG(double, lr) = 1.0; +}; + +TEST(OptimTest, OldInterface) { + struct MyOptimizer : Optimizer { + using Optimizer::Optimizer; + torch::Tensor step(LossClosure closure = nullptr) override { + return {}; + } + explicit MyOptimizer( + std::vector params, + const MyOptimizerOptions& defaults = {}) + : Optimizer( + std::move(params), + std::make_unique(defaults)) {} + }; + std::vector parameters = { + torch::ones({2, 3}), torch::zeros({2, 3}), torch::rand({2, 3})}; + { + MyOptimizer optimizer(parameters); + size_t size = 0; + OLD_INTERFACE_WARNING_CHECK(size = optimizer.size()); + ASSERT_EQ(size, parameters.size()); + } + { + std::vector params; + MyOptimizer optimizer(params); + + size_t size = 0; + OLD_INTERFACE_WARNING_CHECK(size = optimizer.size()); + ASSERT_EQ(size, 0); + + OLD_INTERFACE_WARNING_CHECK(optimizer.add_parameters(parameters)); + + OLD_INTERFACE_WARNING_CHECK(size = optimizer.size()); + ASSERT_EQ(size, parameters.size()); + + std::vector params_; + OLD_INTERFACE_WARNING_CHECK(params_ = optimizer.parameters()); + for (const auto p : c10::irange(size)) { + ASSERT_TRUE(params_[p].allclose(parameters[p])); + } + } + { + Linear linear(3, 4); + MyOptimizer optimizer(linear->parameters()); + + size_t size = 0; + OLD_INTERFACE_WARNING_CHECK(size = optimizer.size()); + ASSERT_EQ(size, linear->parameters().size()); + } +} + +TEST(OptimTest, XORConvergence_SGD) { + ASSERT_TRUE(test_optimizer_xor( + SGDOptions(0.1).momentum(0.9).nesterov(true).weight_decay(1e-6))); +} + +TEST(OptimTest, XORConvergence_LBFGS) { + ASSERT_TRUE(test_optimizer_xor(LBFGSOptions(1.0))); + ASSERT_TRUE(test_optimizer_xor( + LBFGSOptions(1.0).line_search_fn("strong_wolfe"))); +} + +TEST(OptimTest, XORConvergence_Adagrad) { + ASSERT_TRUE(test_optimizer_xor( + AdagradOptions(1.0).weight_decay(1e-6).lr_decay(1e-3))); +} + +TEST(OptimTest, XORConvergence_RMSprop) { + ASSERT_TRUE(test_optimizer_xor(RMSpropOptions(0.1).centered(true))); +} + +TEST(OptimTest, XORConvergence_RMSpropWithMomentum) { + ASSERT_TRUE(test_optimizer_xor( + RMSpropOptions(0.1).momentum(0.9).weight_decay(1e-6))); +} + +TEST(OptimTest, XORConvergence_Adam) { + ASSERT_TRUE(test_optimizer_xor(AdamOptions(0.1).weight_decay(1e-6))); +} + +TEST(OptimTest, XORConvergence_AdamWithAmsgrad) { + ASSERT_TRUE(test_optimizer_xor( + AdamOptions(0.1).weight_decay(1e-6).amsgrad(true))); +} + +TEST(OptimTest, ProducesPyTorchValues_Adam) { + check_exact_values(AdamOptions(1.0), expected_parameters::Adam()); +} + +TEST(OptimTest, ProducesPyTorchValues_AdamWithWeightDecay) { + check_exact_values( + AdamOptions(1.0).weight_decay(1e-2), + expected_parameters::Adam_with_weight_decay()); +} + +TEST(OptimTest, ProducesPyTorchValues_AdamWithWeightDecayAndAMSGrad) { + check_exact_values( + AdamOptions(1.0).weight_decay(1e-6).amsgrad(true), + expected_parameters::Adam_with_weight_decay_and_amsgrad()); +} + +TEST(OptimTest, XORConvergence_AdamW) { + ASSERT_TRUE(test_optimizer_xor(AdamWOptions(0.1))); +} + +TEST(OptimTest, XORConvergence_AdamWWithAmsgrad) { + ASSERT_TRUE(test_optimizer_xor(AdamWOptions(0.1).amsgrad(true))); +} + +TEST(OptimTest, ProducesPyTorchValues_AdamW) { + check_exact_values(AdamWOptions(1.0), expected_parameters::AdamW()); +} + +TEST(OptimTest, ProducesPyTorchValues_AdamWWithoutWeightDecay) { + check_exact_values( + AdamWOptions(1.0).weight_decay(0), + expected_parameters::AdamW_without_weight_decay()); +} + +TEST(OptimTest, ProducesPyTorchValues_AdamWWithAMSGrad) { + check_exact_values( + AdamWOptions(1.0).amsgrad(true), + expected_parameters::AdamW_with_amsgrad()); +} + +TEST(OptimTest, ProducesPyTorchValues_Adagrad) { + check_exact_values( + AdagradOptions(1.0), expected_parameters::Adagrad()); +} + +TEST(OptimTest, ProducesPyTorchValues_AdagradWithWeightDecay) { + check_exact_values( + AdagradOptions(1.0).weight_decay(1e-2), + expected_parameters::Adagrad_with_weight_decay()); +} + +TEST(OptimTest, ProducesPyTorchValues_AdagradWithWeightDecayAndLRDecay) { + check_exact_values( + AdagradOptions(1.0).weight_decay(1e-6).lr_decay(1e-3), + expected_parameters::Adagrad_with_weight_decay_and_lr_decay()); +} + +TEST(OptimTest, ProducesPyTorchValues_RMSprop) { + check_exact_values( + RMSpropOptions(0.1), expected_parameters::RMSprop()); +} + +TEST(OptimTest, ProducesPyTorchValues_RMSpropWithWeightDecay) { + check_exact_values( + RMSpropOptions(0.1).weight_decay(1e-2), + expected_parameters::RMSprop_with_weight_decay()); +} + +TEST(OptimTest, ProducesPyTorchValues_RMSpropWithWeightDecayAndCentered) { + check_exact_values( + RMSpropOptions(0.1).weight_decay(1e-6).centered(true), + expected_parameters::RMSprop_with_weight_decay_and_centered()); +} + +TEST( + OptimTest, + ProducesPyTorchValues_RMSpropWithWeightDecayAndCenteredAndMomentum) { + check_exact_values( + RMSpropOptions(0.1).weight_decay(1e-6).centered(true).momentum(0.9), + expected_parameters:: + RMSprop_with_weight_decay_and_centered_and_momentum()); +} + +TEST(OptimTest, ProducesPyTorchValues_SGD) { + check_exact_values(SGDOptions(0.1), expected_parameters::SGD()); +} + +TEST(OptimTest, ProducesPyTorchValues_SGDWithWeightDecay) { + check_exact_values( + SGDOptions(0.1).weight_decay(1e-2), + expected_parameters::SGD_with_weight_decay()); +} + +TEST(OptimTest, ProducesPyTorchValues_SGDWithWeightDecayAndMomentum) { + check_exact_values( + SGDOptions(0.1).weight_decay(1e-2).momentum(0.9), + expected_parameters::SGD_with_weight_decay_and_momentum()); +} + +TEST(OptimTest, ProducesPyTorchValues_SGDWithWeightDecayAndNesterovMomentum) { + check_exact_values( + SGDOptions(0.1).weight_decay(1e-6).momentum(0.9).nesterov(true), + expected_parameters::SGD_with_weight_decay_and_nesterov_momentum()); +} + +TEST(OptimTest, ProducesPyTorchValues_LBFGS) { + check_exact_values(LBFGSOptions(1.0), expected_parameters::LBFGS()); +} + +TEST(OptimTest, ProducesPyTorchValues_LBFGS_with_line_search) { + check_exact_values( + LBFGSOptions(1.0).line_search_fn("strong_wolfe"), + expected_parameters::LBFGS_with_line_search()); +} + +TEST(OptimTest, ZeroGrad) { + torch::manual_seed(0); + + Linear model(2, 8); + SGD optimizer(model->parameters(), 0.1); + + for (const auto& parameter : model->parameters()) { + ASSERT_FALSE(parameter.grad().defined()); + } + + auto output = model->forward(torch::ones({5, 2})); + auto loss = output.sum(); + loss.backward(); + + for (const auto& parameter : model->parameters()) { + ASSERT_TRUE(parameter.grad().defined()); + ASSERT_GT(parameter.grad().sum().item(), 0); + } + + optimizer.zero_grad(); + + for (const auto& parameter : model->parameters()) { + ASSERT_FALSE(parameter.grad().defined()); + } +} + +TEST(OptimTest, ExternalVectorOfParameters) { + torch::manual_seed(0); + + std::vector parameters = { + torch::randn({2, 2}), torch::randn({3, 3}), torch::randn({4, 4})}; + std::vector original_parameters = { + parameters[0].clone(), parameters[1].clone(), parameters[2].clone()}; + + // Set all gradients to one + for (auto& parameter : parameters) { + parameter.mutable_grad() = torch::ones_like(parameter); + } + + SGD optimizer(parameters, 1.0); + + optimizer.step(); + + ASSERT_TRUE(parameters[0].allclose(original_parameters[0] - 1.0)); + ASSERT_TRUE(parameters[1].allclose(original_parameters[1] - 1.0)); + ASSERT_TRUE(parameters[2].allclose(original_parameters[2] - 1.0)); +} + +TEST(OptimTest, AddParameter_LBFGS) { + torch::manual_seed(0); + + std::vector parameters = {torch::randn({5, 5})}; + std::vector original_parameters = {parameters[0].clone()}; + + // Set all gradients to one + for (auto& parameter : parameters) { + parameter.mutable_grad() = torch::ones_like(parameter); + } + + LBFGS optimizer(std::vector{}, 1.0); + OLD_INTERFACE_WARNING_CHECK(optimizer.add_parameters(parameters)); + + optimizer.step([]() { return torch::tensor(1); }); + + // REQUIRE this doesn't throw +} + +// Check whether the learning rate of the parameter groups in the optimizer are +// the same as the expected learning rates given in the epoch:learning rate map +static void check_lr_change( + Optimizer& optimizer, + LRScheduler& lr_scheduler, + std::map expected_epoch_lrs) { + // Find maximum epoch in map + unsigned kIterations = std::max_element( + expected_epoch_lrs.begin(), + expected_epoch_lrs.end(), + [](const std::pair& a, + const std::pair& b) -> bool { + return a.second > b.second; + }) + ->first; + + for (unsigned i = 0; i <= kIterations; i++) { + const auto epoch_iter = expected_epoch_lrs.find(i); + if (epoch_iter != expected_epoch_lrs.end()) { + // Compare the similarity of the two floating point learning rates + ASSERT_TRUE( + fabs( + epoch_iter->second - + optimizer.param_groups()[0].options().get_lr()) < + std::numeric_limits::epsilon()); + } + optimizer.step(); + lr_scheduler.step(); + } +} + +// Very similar to check_lr_change, but for ReduceLROnPlateauScheduler +// which does not inherit from LRScheduler and requires a metrics +// input to step(). +static void check_lr_change_for_reduce_on_plateau( + Optimizer& optimizer, + ReduceLROnPlateauScheduler& lr_scheduler, + std::map expected_epoch_lrs) { + // Find maximum epoch in map + unsigned kIterations = std::max_element( + expected_epoch_lrs.begin(), + expected_epoch_lrs.end(), + [](const std::pair& a, + const std::pair& b) -> bool { + return a.second > b.second; + }) + ->first; + + for (unsigned i = 0; i <= kIterations; i++) { + const auto epoch_iter = expected_epoch_lrs.find(i); + if (epoch_iter != expected_epoch_lrs.end()) { + // Compare the similarity of the two floating point learning rates + ASSERT_TRUE( + fabs( + epoch_iter->second - + optimizer.param_groups()[0].options().get_lr()) < + std::numeric_limits::epsilon()); + } + optimizer.step(); + lr_scheduler.step(5.0); + } +} + +TEST(OptimTest, CheckLRChange_StepLR_Adam) { + torch::Tensor parameters = torch::zeros({1}); + auto optimizer = Adam({parameters}, AdamOptions().lr(1e-3)); + + const unsigned step_size = 20; + const double gamma = 0.5; + StepLR step_lr_scheduler(optimizer, step_size, gamma); + + // The learning rate should have halved at epoch 20 + const std::map expected_epoch_lrs = {{1, 1e-3}, {25, 5e-4}}; + + check_lr_change(optimizer, step_lr_scheduler, expected_epoch_lrs); +} + +TEST(OptimTest, CheckLRChange_ReduceLROnPlateau_Adam) { + torch::Tensor parameters = torch::zeros({1}); + auto optimizer = Adam({parameters}, AdamOptions().lr(1e-3)); + const float factor = 0.5; + const int patience = 20; + ReduceLROnPlateauScheduler reduce_lr_on_plateau_scheduler( + optimizer, + ReduceLROnPlateauScheduler::SchedulerMode::min, + factor, + patience); + + // The learning rate should have halved at epoch 20 + const std::map expected_epoch_lrs = {{1, 1e-3}, {25, 5e-4}}; + + check_lr_change_for_reduce_on_plateau( + optimizer, reduce_lr_on_plateau_scheduler, expected_epoch_lrs); +} +// Tests for Issue 141884: Parameter group inheritance functionality +// Validates that partial options in parameter groups correctly inherit +// defaults from the optimizer while preserving explicitly set values +TEST(OptimTest, MergeWithDefaultOptions_Adam) { + // Create tensors for parameter groups + auto tensor1 = torch::randn({2, 2}).requires_grad_(true); + auto tensor2 = torch::randn({3, 3}).requires_grad_(true); + + // Create param groups with partial options + std::vector param_groups; + + // Group 1: Only weight_decay specified, should inherit lr, betas, eps, + // amsgrad + param_groups.emplace_back( + std::vector{tensor1}, + std::make_unique(AdamOptions().weight_decay(0.11))); + + // Group 2: Only eps specified, should inherit others + param_groups.emplace_back( + std::vector{tensor2}, + std::make_unique(AdamOptions().eps(1e-6))); + + // Create optimizer with specific defaults + AdamOptions defaults; + defaults.lr(0.002) + .betas(std::make_tuple(0.8, 0.88)) + .eps(1e-12) + .weight_decay(0.05) + .amsgrad(true); + + Adam optimizer(param_groups, defaults); + + // Check Group 1: weight_decay preserved, others inherited + auto& group1_opts = + static_cast(optimizer.param_groups()[0].options()); + ASSERT_EQ(group1_opts.lr(), 0.002); // Inherited + ASSERT_EQ(group1_opts.betas(), std::make_tuple(0.8, 0.88)); // Inherited + ASSERT_EQ(group1_opts.eps(), 1e-12); // Inherited + ASSERT_EQ(group1_opts.weight_decay(), 0.11); // Preserved + ASSERT_TRUE(group1_opts.amsgrad()); // Inherited + + // Check Group 2: eps preserved, others inherited + auto& group2_opts = + static_cast(optimizer.param_groups()[1].options()); + ASSERT_EQ(group2_opts.lr(), 0.002); // Inherited + ASSERT_EQ(group2_opts.betas(), std::make_tuple(0.8, 0.88)); // Inherited + ASSERT_EQ(group2_opts.eps(), 1e-6); // Preserved + ASSERT_EQ(group2_opts.weight_decay(), 0.05); // Inherited + ASSERT_TRUE(group2_opts.amsgrad()); // Inherited +} + +TEST(OptimTest, MergeWithDefaultOptions_SGD) { + // Create tensors for parameter groups + auto tensor1 = torch::randn({2, 2}).requires_grad_(true); + auto tensor2 = torch::randn({3, 3}).requires_grad_(true); + + // Create param groups with partial options + std::vector param_groups; + + // Group 1: Only lr and weight_decay specified, should inherit momentum, + // dampening, nesterov + param_groups.emplace_back( + std::vector{tensor1}, + std::make_unique(SGDOptions(0.01).weight_decay(0.22))); + + // Group 2: Only lr specified, should inherit all others + param_groups.emplace_back( + std::vector{tensor2}, + std::make_unique(SGDOptions(0.02))); + + // Create optimizer with specific defaults + SGDOptions defaults(0.001); // lr should be overridden by param groups + defaults.momentum(0.9) + .dampening(0.0) // Must be 0 for Nesterov + .weight_decay(0.05) + .nesterov(true); + + SGD optimizer(param_groups, defaults); + + // Check Group 1: lr and weight_decay preserved, others inherited + auto& group1_opts = + static_cast(optimizer.param_groups()[0].options()); + ASSERT_EQ(group1_opts.lr(), 0.01); // Preserved + ASSERT_EQ(group1_opts.momentum(), 0.9); // Inherited + ASSERT_EQ(group1_opts.dampening(), 0.0); // Inherited + ASSERT_EQ(group1_opts.weight_decay(), 0.22); // Preserved + ASSERT_TRUE(group1_opts.nesterov()); // Inherited + + // Check Group 2: lr preserved, others inherited + auto& group2_opts = + static_cast(optimizer.param_groups()[1].options()); + ASSERT_EQ(group2_opts.lr(), 0.02); // Preserved + ASSERT_EQ(group2_opts.momentum(), 0.9); // Inherited + ASSERT_EQ(group2_opts.dampening(), 0.0); // Inherited + ASSERT_EQ(group2_opts.weight_decay(), 0.05); // Inherited + ASSERT_TRUE(group2_opts.nesterov()); // Inherited +} + +TEST(OptimTest, MergeWithDefaultOptions_AdamW) { + // Create tensors for parameter groups + auto tensor1 = torch::randn({2, 2}).requires_grad_(true); + auto tensor2 = torch::randn({3, 3}).requires_grad_(true); + + // Create param groups with partial options + std::vector param_groups; + + // Group 1: Only eps specified, should inherit others + param_groups.emplace_back( + std::vector{tensor1}, + std::make_unique(AdamWOptions().eps(1e-6))); + + // Group 2: Only betas specified, should inherit others + param_groups.emplace_back( + std::vector{tensor2}, + std::make_unique( + AdamWOptions().betas(std::make_tuple(0.95, 0.999)))); + + // Create optimizer with specific defaults + AdamWOptions defaults; + defaults.lr(0.003) + .betas(std::make_tuple(0.9, 0.98)) + .eps(1e-8) + .weight_decay(0.02) + .amsgrad(false); + + AdamW optimizer(param_groups, defaults); + + // Check Group 1: eps preserved, others inherited + auto& group1_opts = + static_cast(optimizer.param_groups()[0].options()); + ASSERT_EQ(group1_opts.lr(), 0.003); // Inherited + ASSERT_EQ(group1_opts.betas(), std::make_tuple(0.9, 0.98)); // Inherited + ASSERT_EQ(group1_opts.eps(), 1e-6); // Preserved + ASSERT_EQ(group1_opts.weight_decay(), 0.02); // Inherited + ASSERT_FALSE(group1_opts.amsgrad()); // Inherited + + // Check Group 2: betas preserved, others inherited + auto& group2_opts = + static_cast(optimizer.param_groups()[1].options()); + ASSERT_EQ(group2_opts.lr(), 0.003); // Inherited + ASSERT_EQ(group2_opts.betas(), std::make_tuple(0.95, 0.999)); // Preserved + ASSERT_EQ(group2_opts.eps(), 1e-8); // Inherited + ASSERT_EQ(group2_opts.weight_decay(), 0.02); // Inherited + ASSERT_FALSE(group2_opts.amsgrad()); // Inherited +} + +TEST(OptimTest, MergeWithDefaultOptions_Adagrad) { + // Create tensors for parameter groups + auto tensor1 = torch::randn({2, 2}).requires_grad_(true); + auto tensor2 = torch::randn({3, 3}).requires_grad_(true); + + // Create param groups with partial options + std::vector param_groups; + + // Group 1: Only lr_decay specified, should inherit others + param_groups.emplace_back( + std::vector{tensor1}, + std::make_unique(AdagradOptions().lr_decay(0.001))); + + // Group 2: Only initial_accumulator_value specified, should inherit others + param_groups.emplace_back( + std::vector{tensor2}, + std::make_unique( + AdagradOptions().initial_accumulator_value(0.5))); + + // Create optimizer with specific defaults + AdagradOptions defaults; + defaults.lr(0.04) + .lr_decay(0.002) + .weight_decay(0.03) + .initial_accumulator_value(0.1) + .eps(1e-11); + + Adagrad optimizer(param_groups, defaults); + + // Check Group 1: lr_decay preserved, others inherited + auto& group1_opts = + static_cast(optimizer.param_groups()[0].options()); + ASSERT_EQ(group1_opts.lr(), 0.04); // Inherited + ASSERT_EQ(group1_opts.lr_decay(), 0.001); // Preserved + ASSERT_EQ(group1_opts.weight_decay(), 0.03); // Inherited + ASSERT_EQ(group1_opts.initial_accumulator_value(), 0.1); // Inherited + ASSERT_EQ(group1_opts.eps(), 1e-11); // Inherited + + // Check Group 2: initial_accumulator_value preserved, others inherited + auto& group2_opts = + static_cast(optimizer.param_groups()[1].options()); + ASSERT_EQ(group2_opts.lr(), 0.04); // Inherited + ASSERT_EQ(group2_opts.lr_decay(), 0.002); // Inherited + ASSERT_EQ(group2_opts.weight_decay(), 0.03); // Inherited + ASSERT_EQ(group2_opts.initial_accumulator_value(), 0.5); // Preserved + ASSERT_EQ(group2_opts.eps(), 1e-11); // Inherited +} + +TEST(OptimTest, MergeWithDefaultOptions_RMSprop) { + // Create tensors for parameter groups + auto tensor1 = torch::randn({2, 2}).requires_grad_(true); + auto tensor2 = torch::randn({3, 3}).requires_grad_(true); + + // Create param groups with partial options + std::vector param_groups; + + // Group 1: Only alpha specified, should inherit others + param_groups.emplace_back( + std::vector{tensor1}, + std::make_unique(RMSpropOptions().alpha(0.95))); + + // Group 2: Only momentum and centered specified, should inherit others + param_groups.emplace_back( + std::vector{tensor2}, + std::make_unique( + RMSpropOptions().momentum(0.8).centered(true))); + + // Create optimizer with specific defaults + RMSpropOptions defaults; + defaults.lr(0.015) + .alpha(0.98) + .eps(1e-9) + .weight_decay(0.01) + .momentum(0.7) + .centered(false); + + RMSprop optimizer(param_groups, defaults); + + // Check Group 1: alpha preserved, others inherited + auto& group1_opts = + static_cast(optimizer.param_groups()[0].options()); + ASSERT_EQ(group1_opts.lr(), 0.015); // Inherited + ASSERT_EQ(group1_opts.alpha(), 0.95); // Preserved + ASSERT_EQ(group1_opts.eps(), 1e-9); // Inherited + ASSERT_EQ(group1_opts.weight_decay(), 0.01); // Inherited + ASSERT_EQ(group1_opts.momentum(), 0.7); // Inherited + ASSERT_FALSE(group1_opts.centered()); // Inherited + + // Check Group 2: momentum and centered preserved, others inherited + auto& group2_opts = + static_cast(optimizer.param_groups()[1].options()); + ASSERT_EQ(group2_opts.lr(), 0.015); // Inherited + ASSERT_EQ(group2_opts.alpha(), 0.98); // Inherited + ASSERT_EQ(group2_opts.eps(), 1e-9); // Inherited + ASSERT_EQ(group2_opts.weight_decay(), 0.01); // Inherited + ASSERT_EQ(group2_opts.momentum(), 0.8); // Preserved + ASSERT_TRUE(group2_opts.centered()); // Preserved +} + +TEST(OptimTest, MergeWithDefaultOptions_LBFGS) { + // Create tensors for single parameter group (LBFGS limitation) + auto tensor1 = torch::randn({2, 2}).requires_grad_(true); + auto tensor2 = torch::randn({3, 3}).requires_grad_(true); + + // Create param group with partial options + std::vector param_groups; + + // Single group: Only max_iter specified, should inherit others + param_groups.emplace_back( + std::vector{ + tensor1, tensor2}, // Combine tensors in single group + std::make_unique(LBFGSOptions().max_iter(15))); + + // Create optimizer with specific defaults + LBFGSOptions defaults; + defaults.lr(0.8) + .max_iter(25) + .max_eval(31) // Use same value that appears to be auto-calculated + .tolerance_grad(1e-5) + .tolerance_change(1e-8) + .history_size(80) + .line_search_fn("strong_wolfe"); + + LBFGS optimizer(param_groups, defaults); + + // Check Group: max_iter preserved, others inherited + auto& group_opts = + static_cast(optimizer.param_groups()[0].options()); + ASSERT_EQ(group_opts.lr(), 0.8); // Inherited + ASSERT_EQ(group_opts.max_iter(), 15); // Preserved + ASSERT_EQ(group_opts.max_eval(), 31); // Inherited + ASSERT_EQ(group_opts.tolerance_grad(), 1e-5); // Inherited + ASSERT_EQ(group_opts.tolerance_change(), 1e-8); // Inherited + ASSERT_EQ(group_opts.history_size(), 80); // Inherited + ASSERT_EQ(group_opts.line_search_fn(), "strong_wolfe"); // Inherited +} + +TEST(OptimTest, MergeWithDefaultOptions_NoOptionsInheritance) { + // Test that param groups without options get full defaults + auto tensor1 = torch::randn({2, 2}).requires_grad_(true); + auto tensor2 = torch::randn({3, 3}).requires_grad_(true); + + std::vector param_groups; + + // Groups with no options - should inherit everything + param_groups.emplace_back(std::vector{tensor1}); + param_groups.emplace_back(std::vector{tensor2}); + + // Create optimizer with specific defaults + AdamOptions defaults; + defaults.lr(0.005) + .betas(std::make_tuple(0.85, 0.95)) + .eps(1e-7) + .weight_decay(0.08) + .amsgrad(true); + + Adam optimizer(param_groups, defaults); + + // Both groups should have exactly the default options + for (int i = 0; i < 2; i++) { + auto& group_opts = + static_cast(optimizer.param_groups()[i].options()); + ASSERT_EQ(group_opts.lr(), 0.005); + ASSERT_EQ(group_opts.betas(), std::make_tuple(0.85, 0.95)); + ASSERT_EQ(group_opts.eps(), 1e-7); + ASSERT_EQ(group_opts.weight_decay(), 0.08); + ASSERT_TRUE(group_opts.amsgrad()); + } +} + +// Test that field tracking survives serialization/deserialization cycles +TEST(OptimTest, SerializationPreservesFieldTracking_Adam) { + // Create tensors for parameter groups + auto tensor1 = torch::randn({2, 2}).requires_grad_(true); + auto tensor2 = torch::randn({3, 3}).requires_grad_(true); + + // Create param groups with partial options using fluent API (marks fields as + // explicit) + std::vector param_groups; + + // Group 1: Only weight_decay and amsgrad explicitly set via fluent API + param_groups.emplace_back( + std::vector{tensor1}, + std::make_unique( + AdamOptions().weight_decay(0.11).amsgrad(true))); + + // Group 2: Only eps explicitly set via fluent API + param_groups.emplace_back( + std::vector{tensor2}, + std::make_unique(AdamOptions().eps(1e-6))); + + // Create optimizer with specific defaults + AdamOptions defaults; + defaults.lr(0.002) + .betas(std::make_tuple(0.8, 0.88)) + .eps(1e-12) + .weight_decay(0.05) + .amsgrad(false); + + Adam original_optimizer(param_groups, defaults); + + // Capture original state for comparison + auto& orig_group1_opts = + static_cast(original_optimizer.param_groups()[0].options()); + auto& orig_group2_opts = + static_cast(original_optimizer.param_groups()[1].options()); + + // Verify original state (sanity check) + ASSERT_NEAR(orig_group1_opts.weight_decay(), 0.11, 1e-6); // Explicitly set + ASSERT_TRUE(orig_group1_opts.amsgrad()); // Explicitly set + ASSERT_NEAR(orig_group1_opts.lr(), 0.002, 1e-6); // Inherited + ASSERT_NEAR(orig_group2_opts.eps(), 1e-6, 1e-9); // Explicitly set + ASSERT_NEAR(orig_group2_opts.lr(), 0.002, 1e-6); // Inherited + + // Test serialization of the options objects (where field tracking lives) + std::stringstream ss1, ss2; + + // Serialize the parameter group options + { + torch::serialize::OutputArchive archive; + orig_group1_opts.serialize(archive); + archive.save_to(ss1); + } + { + torch::serialize::OutputArchive archive; + orig_group2_opts.serialize(archive); + archive.save_to(ss2); + } + + // Create new options objects and deserialize + AdamOptions loaded_group1_opts; + AdamOptions loaded_group2_opts; + + { + torch::serialize::InputArchive archive; + archive.load_from(ss1); + loaded_group1_opts.serialize(archive); + } + { + torch::serialize::InputArchive archive; + archive.load_from(ss2); + loaded_group2_opts.serialize(archive); + } + + // Verify that all parameter values are preserved after deserialization + + // Group 1: weight_decay and amsgrad should be preserved as explicitly set, + // others inherited + ASSERT_NEAR(loaded_group1_opts.lr(), 0.002, 1e-6); // Inherited + ASSERT_EQ( + loaded_group1_opts.betas(), std::make_tuple(0.8, 0.88)); // Inherited + ASSERT_NEAR(loaded_group1_opts.eps(), 1e-12, 1e-15); // Inherited + ASSERT_NEAR(loaded_group1_opts.weight_decay(), 0.11, 1e-6); // Explicitly set + ASSERT_TRUE(loaded_group1_opts.amsgrad()); // Explicitly set + + // Group 2: eps should be preserved as explicitly set, others inherited + ASSERT_NEAR(loaded_group2_opts.lr(), 0.002, 1e-6); // Inherited + ASSERT_EQ( + loaded_group2_opts.betas(), std::make_tuple(0.8, 0.88)); // Inherited + ASSERT_NEAR(loaded_group2_opts.eps(), 1e-6, 1e-9); // Explicitly set + ASSERT_NEAR(loaded_group2_opts.weight_decay(), 0.05, 1e-6); // Inherited + ASSERT_FALSE(loaded_group2_opts.amsgrad()); // Inherited + + // CRITICAL: Test that field tracking is preserved after serialization + // Create a new optimizer using the deserialized options to test inheritance + auto tensor3 = torch::randn({2, 2}).requires_grad_(true); + auto tensor4 = torch::randn({3, 3}).requires_grad_(true); + + std::vector test_param_groups; + test_param_groups.emplace_back( + std::vector{tensor3}, + std::make_unique(loaded_group1_opts)); + test_param_groups.emplace_back( + std::vector{tensor4}, + std::make_unique(loaded_group2_opts)); + + Adam test_optimizer(test_param_groups, defaults); + + // The field tracking should work correctly for inheritance + auto& final_group1_opts = + static_cast(test_optimizer.param_groups()[0].options()); + auto& final_group2_opts = + static_cast(test_optimizer.param_groups()[1].options()); + + // Group 1: weight_decay and amsgrad should still be preserved as explicitly + // set + ASSERT_NEAR( + final_group1_opts.weight_decay(), + 0.11, + 1e-6); // Explicitly set (preserved) + ASSERT_TRUE(final_group1_opts.amsgrad()); // Explicitly set (preserved) + ASSERT_NEAR(final_group1_opts.lr(), 0.002, 1e-6); // Inherited from defaults + + // Group 2: eps should still be preserved as explicitly set + ASSERT_NEAR( + final_group2_opts.eps(), 1e-6, 1e-9); // Explicitly set (preserved) + ASSERT_NEAR(final_group2_opts.lr(), 0.002, 1e-6); // Inherited from defaults +} + +// Test serialization with SGD (different parameter types) +TEST(OptimTest, SerializationPreservesFieldTracking_SGD) { + // Create tensors + auto tensor1 = torch::randn({2, 2}).requires_grad_(true); + + // Create param group with partial options using fluent API + std::vector param_groups; + param_groups.emplace_back( + std::vector{tensor1}, + std::make_unique( + SGDOptions(0.01).weight_decay(0.22).nesterov(true))); + + // Create optimizer with defaults + SGDOptions defaults(0.001); + defaults.momentum(0.9).dampening(0.0).weight_decay(0.05).nesterov(false); + + SGD original_optimizer(param_groups, defaults); + + // Test serialization of the SGD options (where field tracking lives) + auto& original_opts = + static_cast(original_optimizer.param_groups()[0].options()); + + std::stringstream ss; + { + torch::serialize::OutputArchive archive; + original_opts.serialize(archive); + archive.save_to(ss); + } + + SGDOptions loaded_opts(0.0); // Dummy initial value + { + torch::serialize::InputArchive archive; + archive.load_from(ss); + loaded_opts.serialize(archive); + } + ASSERT_NEAR(loaded_opts.lr(), 0.01, 1e-6); // Explicitly set + ASSERT_NEAR(loaded_opts.momentum(), 0.9, 1e-6); // Inherited + ASSERT_NEAR(loaded_opts.dampening(), 0.0, 1e-6); // Inherited + ASSERT_NEAR(loaded_opts.weight_decay(), 0.22, 1e-6); // Explicitly set + ASSERT_TRUE(loaded_opts.nesterov()); // Explicitly set + + // Test that field tracking still works after deserialization by creating new + // optimizer + auto tensor2 = torch::randn({3, 3}).requires_grad_(true); + std::vector test_param_groups; + test_param_groups.emplace_back( + std::vector{tensor2}, + std::make_unique(loaded_opts)); + + SGD test_optimizer(test_param_groups, defaults); + + auto& final_opts = + static_cast(test_optimizer.param_groups()[0].options()); + ASSERT_NEAR(final_opts.lr(), 0.01, 1e-6); // Explicitly set (preserved) + ASSERT_NEAR( + final_opts.weight_decay(), 0.22, 1e-6); // Explicitly set (preserved) + ASSERT_TRUE(final_opts.nesterov()); // Explicitly set (preserved) + ASSERT_NEAR(final_opts.momentum(), 0.9, 1e-6); // Inherited from defaults + ASSERT_NEAR(final_opts.dampening(), 0.0, 1e-6); // Inherited from defaults +} diff --git a/western_ai/usa/pytorch/test/cpp/api/optim_baseline.h b/western_ai/usa/pytorch/test/cpp/api/optim_baseline.h new file mode 100644 index 0000000000000000000000000000000000000000..a96ab9917a053d24926787c13c64ee0b6a72ab56 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/optim_baseline.h @@ -0,0 +1,3060 @@ +// @generated from optim_baseline.py + +#include + +#include + +namespace expected_parameters { + +inline std::vector> LBFGS() { + return { + { + torch::tensor( + {-0.20959197386869663, + -0.49580870398532073, + -0.1313442585372408, + -0.3287331939506787, + -0.24613947168465267, + 0.705889510763571}), + torch::tensor( + {-0.10412662274500666, -0.2644705062031845, 0.7102859961803084}), + torch::tensor( + {-0.19787984636009417, -0.5320223708266223, -0.5396083236337847}), + torch::tensor({-0.43108206822505857}), + }, + { + torch::tensor( + {0.4377600774755075, + 0.3828823919505896, + 0.5308031277873992, + 0.5752746453369446, + 0.23943592910168343, + 1.3739197373644627}), + torch::tensor( + {2.209263823172053, 2.154134023426646, 2.534834254325867}), + torch::tensor( + {-4.091952315741579, -4.67916063385269, -4.781279234594454}), + torch::tensor({-4.776742087865583}), + }, + { + torch::tensor( + {0.4377600774755075, + 0.3828823919505896, + 0.5308031277873992, + 0.5752746453369446, + 0.23943592910168343, + 1.3739197373644627}), + torch::tensor( + {2.209263823172053, 2.154134023426646, 2.534834254325867}), + torch::tensor( + {-4.091952315741579, -4.67916063385269, -4.781279234594454}), + torch::tensor({-4.776742087865583}), + }, + { + torch::tensor( + {0.4377600774755075, + 0.3828823919505896, + 0.5308031277873992, + 0.5752746453369446, + 0.23943592910168343, + 1.3739197373644627}), + torch::tensor( + {2.209263823172053, 2.154134023426646, 2.534834254325867}), + torch::tensor( + {-4.091952315741579, -4.67916063385269, -4.781279234594454}), + torch::tensor({-4.776742087865583}), + }, + { + torch::tensor( + {0.4377600774755075, + 0.3828823919505896, + 0.5308031277873992, + 0.5752746453369446, + 0.23943592910168343, + 1.3739197373644627}), + torch::tensor( + {2.209263823172053, 2.154134023426646, 2.534834254325867}), + torch::tensor( + {-4.091952315741579, -4.67916063385269, -4.781279234594454}), + torch::tensor({-4.776742087865583}), + }, + { + torch::tensor( + {0.4377600774755075, + 0.3828823919505896, + 0.5308031277873992, + 0.5752746453369446, + 0.23943592910168343, + 1.3739197373644627}), + torch::tensor( + {2.209263823172053, 2.154134023426646, 2.534834254325867}), + torch::tensor( + {-4.091952315741579, -4.67916063385269, -4.781279234594454}), + torch::tensor({-4.776742087865583}), + }, + { + torch::tensor( + {0.4377600774755075, + 0.3828823919505896, + 0.5308031277873992, + 0.5752746453369446, + 0.23943592910168343, + 1.3739197373644627}), + torch::tensor( + {2.209263823172053, 2.154134023426646, 2.534834254325867}), + torch::tensor( + {-4.091952315741579, -4.67916063385269, -4.781279234594454}), + torch::tensor({-4.776742087865583}), + }, + { + torch::tensor( + {0.4377600774755075, + 0.3828823919505896, + 0.5308031277873992, + 0.5752746453369446, + 0.23943592910168343, + 1.3739197373644627}), + torch::tensor( + {2.209263823172053, 2.154134023426646, 2.534834254325867}), + torch::tensor( + {-4.091952315741579, -4.67916063385269, -4.781279234594454}), + torch::tensor({-4.776742087865583}), + }, + { + torch::tensor( + {0.4377600774755075, + 0.3828823919505896, + 0.5308031277873992, + 0.5752746453369446, + 0.23943592910168343, + 1.3739197373644627}), + torch::tensor( + {2.209263823172053, 2.154134023426646, 2.534834254325867}), + torch::tensor( + {-4.091952315741579, -4.67916063385269, -4.781279234594454}), + torch::tensor({-4.776742087865583}), + }, + { + torch::tensor( + {0.4377600774755075, + 0.3828823919505896, + 0.5308031277873992, + 0.5752746453369446, + 0.23943592910168343, + 1.3739197373644627}), + torch::tensor( + {2.209263823172053, 2.154134023426646, 2.534834254325867}), + torch::tensor( + {-4.091952315741579, -4.67916063385269, -4.781279234594454}), + torch::tensor({-4.776742087865583}), + }, + { + torch::tensor( + {0.4377600774755075, + 0.3828823919505896, + 0.5308031277873992, + 0.5752746453369446, + 0.23943592910168343, + 1.3739197373644627}), + torch::tensor( + {2.209263823172053, 2.154134023426646, 2.534834254325867}), + torch::tensor( + {-4.091952315741579, -4.67916063385269, -4.781279234594454}), + torch::tensor({-4.776742087865583}), + }, + }; +} + +inline std::vector> LBFGS_with_line_search() { + return { + { + torch::tensor( + {-0.2108988568338871, + -0.4975560466422629, + -0.14129216202471762, + -0.3420288967865903, + -0.2523635082803723, + 0.6975570255493777}), + torch::tensor( + {-0.10853121966252377, -0.297948499687533, 0.6892015099955717}), + torch::tensor( + {-0.05080313011597659, + -0.39413518751058996, + -0.28433759745928844}), + torch::tensor({-0.07113812430174116}), + }, + { + torch::tensor( + {-0.2108988568338871, + -0.4975560466422629, + -0.14129216202471762, + -0.3420288967865903, + -0.2523635082803723, + 0.6975570255493777}), + torch::tensor( + {-0.10853121966252377, -0.297948499687533, 0.6892015099955717}), + torch::tensor( + {-0.05080313011597659, + -0.39413518751058996, + -0.28433759745928844}), + torch::tensor({-0.07113812430174116}), + }, + { + torch::tensor( + {-0.2108988568338871, + -0.4975560466422629, + -0.14129216202471762, + -0.3420288967865903, + -0.2523635082803723, + 0.6975570255493777}), + torch::tensor( + {-0.10853121966252377, -0.297948499687533, 0.6892015099955717}), + torch::tensor( + {-0.05080313011597659, + -0.39413518751058996, + -0.28433759745928844}), + torch::tensor({-0.07113812430174116}), + }, + { + torch::tensor( + {-0.2108988568338871, + -0.4975560466422629, + -0.14129216202471762, + -0.3420288967865903, + -0.2523635082803723, + 0.6975570255493777}), + torch::tensor( + {-0.10853121966252377, -0.297948499687533, 0.6892015099955717}), + torch::tensor( + {-0.05080313011597659, + -0.39413518751058996, + -0.28433759745928844}), + torch::tensor({-0.07113812430174116}), + }, + { + torch::tensor( + {-0.2108988568338871, + -0.4975560466422629, + -0.14129216202471762, + -0.3420288967865903, + -0.2523635082803723, + 0.6975570255493777}), + torch::tensor( + {-0.10853121966252377, -0.297948499687533, 0.6892015099955717}), + torch::tensor( + {-0.05080313011597659, + -0.39413518751058996, + -0.28433759745928844}), + torch::tensor({-0.07113812430174116}), + }, + { + torch::tensor( + {-0.2108988568338871, + -0.4975560466422629, + -0.14129216202471762, + -0.3420288967865903, + -0.2523635082803723, + 0.6975570255493777}), + torch::tensor( + {-0.10853121966252377, -0.297948499687533, 0.6892015099955717}), + torch::tensor( + {-0.05080313011597659, + -0.39413518751058996, + -0.28433759745928844}), + torch::tensor({-0.07113812430174116}), + }, + { + torch::tensor( + {-0.2108988568338871, + -0.4975560466422629, + -0.14129216202471762, + -0.3420288967865903, + -0.2523635082803723, + 0.6975570255493777}), + torch::tensor( + {-0.10853121966252377, -0.297948499687533, 0.6892015099955717}), + torch::tensor( + {-0.05080313011597659, + -0.39413518751058996, + -0.28433759745928844}), + torch::tensor({-0.07113812430174116}), + }, + { + torch::tensor( + {-0.2108988568338871, + -0.4975560466422629, + -0.14129216202471762, + -0.3420288967865903, + -0.2523635082803723, + 0.6975570255493777}), + torch::tensor( + {-0.10853121966252377, -0.297948499687533, 0.6892015099955717}), + torch::tensor( + {-0.05080313011597659, + -0.39413518751058996, + -0.28433759745928844}), + torch::tensor({-0.07113812430174116}), + }, + { + torch::tensor( + {-0.2108988568338871, + -0.4975560466422629, + -0.14129216202471762, + -0.3420288967865903, + -0.2523635082803723, + 0.6975570255493777}), + torch::tensor( + {-0.10853121966252377, -0.297948499687533, 0.6892015099955717}), + torch::tensor( + {-0.05080313011597659, + -0.39413518751058996, + -0.28433759745928844}), + torch::tensor({-0.07113812430174116}), + }, + { + torch::tensor( + {-0.2108988568338871, + -0.4975560466422629, + -0.14129216202471762, + -0.3420288967865903, + -0.2523635082803723, + 0.6975570255493777}), + torch::tensor( + {-0.10853121966252377, -0.297948499687533, 0.6892015099955717}), + torch::tensor( + {-0.05080313011597659, + -0.39413518751058996, + -0.28433759745928844}), + torch::tensor({-0.07113812430174116}), + }, + { + torch::tensor( + {-0.2108988568338871, + -0.4975560466422629, + -0.14129216202471762, + -0.3420288967865903, + -0.2523635082803723, + 0.6975570255493777}), + torch::tensor( + {-0.10853121966252377, -0.297948499687533, 0.6892015099955717}), + torch::tensor( + {-0.05080313011597659, + -0.39413518751058996, + -0.28433759745928844}), + torch::tensor({-0.07113812430174116}), + }, + }; +} + +inline std::vector> Adam() { + return { + { + torch::tensor( + {0.7890972864438472, + 0.5024410688121617, + 0.8587073313055582, + 0.6579707241208395, + 0.7476356819075531, + 1.697556420651692}), + torch::tensor( + {0.891467636010675, 0.7020513497567501, 1.6892012709428947}), + torch::tensor( + {-1.0508030958460797, -1.3941351509567657, -1.284337577714353}), + torch::tensor({-1.071138110298716}), + }, + { + torch::tensor( + {8.233039313231828, + 7.971150747377481, + 6.6436209506776, + 6.470977407900541, + 6.170125488259256, + 7.1507391033435015}), + torch::tensor( + {8.417695070103735, 6.597188212844593, 7.23175710827678}), + torch::tensor( + {-6.729624357635757, -7.09743493108154, -6.753301896575352}), + torch::tensor({-6.435639096011218}), + }, + { + torch::tensor( + {8.233424596059296, + 7.971537360032308, + 6.643920150720394, + 6.47127807553724, + 6.170405874224489, + 7.151021086137982}), + torch::tensor( + {8.418084791214294, 6.597493171180545, 7.232043740621598}), + torch::tensor( + {-6.729918250724671, -7.097730102046093, -6.753584809755359}), + torch::tensor({-6.4359165566974985}), + }, + { + torch::tensor( + {8.233424610557648, + 7.971537374586563, + 6.643920161995285, + 6.471278086877829, + 6.170405884785074, + 7.151021096766405}), + torch::tensor( + {8.418084805901902, 6.597493182713584, 7.2320437514477875}), + torch::tensor( + {-6.72991829363266, -7.097730147102975, -6.753584838821182}), + torch::tensor({-6.435916580217771}), + }, + { + torch::tensor( + {8.233424610575101, + 7.971537374611125, + 6.643920162027962, + 6.471278086923278, + 6.170405884809245, + 7.15102109680004}), + torch::tensor( + {8.418084805946389, 6.597493182796847, 7.232043751509309}), + torch::tensor( + {-6.729918332327653, -7.097730188349552, -6.753584861205486}), + torch::tensor({-6.435916596115672}), + }, + { + torch::tensor( + {8.233424610594858, + 7.971537374639166, + 6.643920162065571, + 6.471278086975759, + 6.170405884836981, + 7.1510210968387975}), + torch::tensor( + {8.418084805997614, 6.59749318289335, 7.232043751580523}), + torch::tensor( + {-6.72991837738045, -7.097730236373201, -6.753584887267492}), + torch::tensor({-6.43591661462546}), + }, + { + torch::tensor( + {8.233424610617288, + 7.971537374671012, + 6.643920162108285, + 6.471278087035362, + 6.170405884868481, + 7.151021096882811}), + torch::tensor( + {8.418084806055795, 6.59749318300295, 7.232043751661401}), + torch::tensor( + {-6.729918428547273, -7.09773029091405, -6.753584916866329}), + torch::tensor({-6.4359166356471755}), + }, + { + torch::tensor( + {8.233424610642352, + 7.9715373747065925, + 6.6439201621560064, + 6.471278087101955, + 6.1704058849036745, + 7.151021096931989}), + torch::tensor( + {8.418084806120799, 6.597493183125404, 7.232043751751764}), + torch::tensor( + {-6.729918485714688, -7.0977303518511805, -6.753584949936365}), + torch::tensor({-6.43591665913422}), + }, + { + torch::tensor( + {8.233424610670035, + 7.97153737474589, + 6.6439201622087145, + 6.471278087175502, + 6.170405884942545, + 7.151021096986302}), + torch::tensor( + {8.418084806192592, 6.597493183260647, 7.232043751851564}), + torch::tensor( + {-6.729918548853505, -7.097730419153473, -6.753584986460725}), + torch::tensor({-6.435916685074594}), + }, + { + torch::tensor( + {8.233424610700348, + 7.971537374788922, + 6.643920162266433, + 6.4712780872560405, + 6.17040588498511, + 7.151021097045779}), + torch::tensor( + {8.418084806271214, 6.597493183408747, 7.232043751960854}), + torch::tensor( + {-6.7299186179943, -7.097730492853521, -6.753585026457088}), + torch::tensor({-6.435916713480863}), + }, + { + torch::tensor( + {8.233424610733326, + 7.971537374835737, + 6.643920162329225, + 6.471278087343659, + 6.170405885031416, + 7.151021097110483}), + torch::tensor( + {8.418084806356743, 6.597493183569867, 7.232043752079749}), + torch::tensor( + {-6.729918693213275, -7.097730573032567, -6.753585069969552}), + torch::tensor({-6.43591674438434}), + }, + }; +} + +inline std::vector> Adam_with_weight_decay() { + return { + { + torch::tensor( + {0.7890990163499767, + 0.5024427688479549, + 0.858707365154099, + 0.65797076763247, + 0.7476358193232038, + 1.6975559791029715}), + torch::tensor( + {0.8914677624298939, 0.7020513562204098, 1.6892012237887575}), + torch::tensor( + {-1.050803095786311, -1.3941351504224309, -1.2843375776028747}), + torch::tensor({-1.0711381102847533}), + }, + { + torch::tensor( + {0.17835734655765323, + 0.2542117171890537, + 0.19681971909229715, + 0.23522651199260597, + 0.17806083719648957, + 0.22943655675307303}), + torch::tensor( + {0.6227676931552837, 0.6058596954431213, 0.6077176546857177}), + torch::tensor( + {-1.4259755901844118, -1.4333355461952704, -1.408545526635006}), + torch::tensor({-2.0710783081666215}), + }, + { + torch::tensor( + {0.17965695035191162, + 0.24254352340441693, + 0.17964663531482672, + 0.24250834976541322, + 0.17962893833698693, + 0.24249920074277215}), + torch::tensor( + {0.6287144967638043, 0.6286955805603279, 0.6286563093833837}), + torch::tensor( + {-1.4123887230853596, -1.4124007126659273, -1.4122701589749163}), + torch::tensor({-2.063357041247863}), + }, + { + torch::tensor( + {0.1796366651819, + 0.24250861931831874, + 0.17963731759793083, + 0.24250861142436989, + 0.1796372002681969, + 0.24250890248031373}), + torch::tensor( + {0.6287221269294724, 0.6287225821354421, 0.6287220274975922}), + torch::tensor( + {-1.4123466103044011, -1.4123465669572683, -1.4123462614739388}), + torch::tensor({-2.063368365143669}), + }, + { + torch::tensor( + {0.17963666103165563, + 0.24250882317446784, + 0.17963665831217887, + 0.24250882481082656, + 0.17963666029066117, + 0.24250882426223175}), + torch::tensor( + {0.6287216329900817, 0.6287216340515608, 0.6287216326960158}), + torch::tensor( + {-1.4123467542623926, -1.4123467542350234, -1.4123467478191443}), + torch::tensor({-2.0633690432440437}), + }, + { + torch::tensor( + {0.17963666098500394, + 0.24250882442377164, + 0.17963666099348902, + 0.2425088244120223, + 0.1796366609725109, + 0.24250882441058697}), + torch::tensor( + {0.6287216343798432, 0.6287216343800675, 0.6287216343742645}), + torch::tensor( + {-1.4123467490742723, -1.412346749072554, -1.4123467490678536}), + torch::tensor({-2.0633690434425396}), + }, + { + torch::tensor( + {0.17963666098407144, + 0.24250882442250174, + 0.17963666098407347, + 0.2425088244224233, + 0.17963666098409325, + 0.2425088244225157}), + torch::tensor( + {0.6287216343836609, 0.6287216343836147, 0.6287216343836255}), + torch::tensor( + {-1.412346749067226, -1.412346749067243, -1.412346749067169}), + torch::tensor({-2.063369043434909}), + }, + { + torch::tensor( + {0.17963666098406988, + 0.2425088244224408, + 0.17963666098407077, + 0.24250882442244073, + 0.17963666098407008, + 0.2425088244224409}), + torch::tensor( + {0.6287216343837067, 0.6287216343837065, 0.6287216343837069}), + torch::tensor( + {-1.4123467490671706, -1.412346749067171, -1.4123467490671713}), + torch::tensor({-2.0633690434349057}), + }, + { + torch::tensor( + {0.17963666098407038, + 0.24250882442244104, + 0.17963666098407027, + 0.24250882442244104, + 0.17963666098407025, + 0.24250882442244098}), + torch::tensor( + {0.6287216343837067, 0.628721634383707, 0.6287216343837067}), + torch::tensor( + {-1.4123467490671706, -1.4123467490671708, -1.4123467490671706}), + torch::tensor({-2.0633690434349052}), + }, + { + torch::tensor( + {0.1796366609840706, + 0.24250882442244143, + 0.17963666098407047, + 0.24250882442244096, + 0.17963666098407025, + 0.24250882442244098}), + torch::tensor( + {0.6287216343837069, 0.6287216343837067, 0.6287216343837067}), + torch::tensor( + {-1.4123467490671706, -1.4123467490671706, -1.4123467490671708}), + torch::tensor({-2.0633690434349052}), + }, + { + torch::tensor( + {0.1796366609840692, + 0.24250882442244046, + 0.17963666098407022, + 0.24250882442244082, + 0.17963666098407, + 0.24250882442244104}), + torch::tensor( + {0.6287216343837063, 0.6287216343837068, 0.6287216343837067}), + torch::tensor( + {-1.4123467490671708, -1.4123467490671706, -1.4123467490671708}), + torch::tensor({-2.0633690434349052}), + }, + }; +} + +inline std::vector> +Adam_with_weight_decay_and_amsgrad() { + return { + { + torch::tensor( + {0.7890972867575196, + 0.5024410692260988, + 0.8587073313091852, + 0.6579707241257546, + 0.7476356819241026, + 1.6975564206261673}), + torch::tensor( + {0.8914676360248869, 0.7020513497574256, 1.6892012709389561}), + torch::tensor( + {-1.050803095846074, -1.3941351509567128, -1.284337577714342}), + torch::tensor({-1.0711381102987145}), + }, + { + torch::tensor( + {6.790598887618061, + 6.914995398136696, + 6.41533478566264, + 6.297644005485053, + 5.845162499872375, + 6.862229173597117}), + torch::tensor( + {7.958707058914726, 6.511338624975532, 7.100969502256063}), + torch::tensor( + {-6.690689640539306, -7.056584601121166, -6.72114879738572}), + torch::tensor({-6.406608295022552}), + }, + { + torch::tensor( + {4.707506618354547, + 5.291519064582759, + 6.0451502264500006, + 6.024403702678936, + 5.309533822430375, + 6.388110918107735}), + torch::tensor( + {7.200495189000188, 6.398387074819269, 6.904125817198589}), + torch::tensor( + {-6.664150387053514, -7.026716194929788, -6.705821732490459}), + torch::tensor({-6.396310969025695}), + }, + { + torch::tensor( + {2.9508632109188633, + 3.7657755643994775, + 5.60741774331852, + 5.6957903028180565, + 4.70145185833677, + 5.835064148607041}), + torch::tensor( + {6.343524400109462, 6.258242740866945, 6.663022973860484}), + torch::tensor( + {-6.630461605133603, -6.988854886932907, -6.686194352796841}), + torch::tensor({-6.383010489575922}), + }, + { + torch::tensor( + {1.7128944635692829, + 2.536365345915568, + 5.140416924817106, + 5.33803266121343, + 4.083921806116116, + 5.254596369238127}), + torch::tensor( + {5.477917349690043, 6.100068192681452, 6.394747035239918}), + torch::tensor( + {-6.591742325353548, -6.945355504749947, -6.663584873152447}), + torch::tensor({-6.367675348676994}), + }, + { + torch::tensor( + {0.9341502247285258, + 1.6339620765410685, + 4.6679910940755835, + 4.967688979298023, + 3.4933141073198866, + 4.678195347615295}), + torch::tensor( + {4.655117321178743, 5.9294836450698245, 6.110061909503652}), + torch::tensor( + {-6.549116242899458, -6.897486328511809, -6.638629863638681}), + torch::tensor({-6.350731975696792}), + }, + { + torch::tensor( + {0.483518223008081, + 1.014261673266094, + 4.205928052060015, + 4.596195204751035, + 2.9502123780175378, + 4.125826973031755}), + torch::tensor( + {3.90359317709392, 5.750505227817309, 5.816617309738603}), + torch::tensor( + {-6.503371263778851, -6.846137347295816, -6.611773185243846}), + torch::tensor({-6.3324769650576656}), + }, + { + torch::tensor( + {0.2393576446248765, + 0.6100241101779533, + 3.764601561942264, + 4.231602962540335, + 2.4647709193637635, + 3.6096961114614476}), + torch::tensor( + {3.236721556734532, 5.566168160977344, 5.520085344708356}), + torch::tensor( + {-6.455100840665729, -6.791979259673106, -6.583347815648856}), + torch::tensor({-6.3131323905801136}), + }, + { + torch::tensor( + {0.11401265024593016, + 0.3570521972760832, + 3.350517925954931, + 3.8795333009419823, + 2.0402068661130683, + 3.136550602110189}), + torch::tensor( + {2.6579570162250215, 5.378834741966309, 5.224742933241745}), + torch::tensor( + {-6.4047717084756375, -6.7355397098532155, -6.55361495837462}), + torch::tensor({-6.2928722155069945}), + }, + { + torch::tensor( + {0.05251515193791458, + 0.20410212473600725, + 2.9673680881961273, + 3.543794405777883, + 1.6752677855061209, + 2.709287985107431}), + torch::tensor( + {2.164479166686583, 5.190372657839918, 4.9338240234040756}), + torch::tensor( + {-6.352761531270841, -6.6772456859648175, -6.52278570167088}), + torch::tensor({-6.271836898876738}), + }, + { + torch::tensor( + {0.023489947480426834, + 0.11428338573638941, + 2.616797245623764, + 3.226821571853439, + 1.3659994589608537, + 2.328136084816453}), + torch::tensor( + {1.74978620664146, 5.002269811977871, 4.649756802441968}), + torch::tensor( + {-6.299382007948917, -6.617449564196286, -6.491034254081261}), + torch::tensor({-6.250142306631259}), + }, + }; +} + +inline std::vector> AdamW() { + return { + { + torch::tensor( + {0.7912062750121864, + 0.5074166292785842, + 0.8601202529258052, + 0.6613910130887053, + 0.7501593169903569, + 1.6905808503961983}), + torch::tensor( + {0.8925529482073002, 0.7050308347536254, 1.682309255842939}), + torch::tensor( + {-1.05029506454492, -1.3901937990816595, -1.2814942017397601}), + torch::tensor({-1.0704267290556988}), + }, + { + torch::tensor( + {3.3165329599188507, + 3.223120441823618, + 2.665544565239194, + 2.6044341406663225, + 2.479859063483047, + 2.836831717112226}), + torch::tensor( + {3.3885192024669744, 2.6544147219174556, 2.8709245656887328}), + torch::tensor( + {-2.70172647102137, -2.836731459490802, -2.69652471546253}), + torch::tensor({-2.575239255076019}), + }, + { + torch::tensor( + {2.231471944853865, + 2.3549328325971755, + 1.5699078054795328, + 1.6160272935884685, + 1.5339085081403547, + 1.7397405105941612}), + torch::tensor( + {2.8552579170807926, 1.8369866847839356, 1.9735168512425862}), + torch::tensor( + {-2.6042083360293855, -2.6996673713262336, -1.8976087706977893}), + torch::tensor({-1.6180915942867784}), + }, + { + torch::tensor( + {2.084688381515552, + 2.3141612674892946, + 1.4850714710140511, + 1.5961047256668386, + 1.440300645879787, + 1.6065354941586025}), + torch::tensor( + {3.0111385685659444, 1.955556497153507, 1.9596562467797627}), + torch::tensor( + {-2.889337305884852, -2.965249100126337, -1.7721676671605975}), + torch::tensor({-1.4001341655590005}), + }, + { + torch::tensor( + {2.0465343456006604, + 2.311613891239368, + 1.4666717526896398, + 1.601383980913499, + 1.4223660595993763, + 1.5711552625612757}), + torch::tensor( + {3.07151984580744, 2.0112690538174802, 1.9592484602763875}), + torch::tensor( + {-3.0186469726426863, -3.093855445542849, -1.7367953899738784}), + torch::tensor({-1.3299011560804312}), + }, + { + torch::tensor( + {2.039659777412556, + 2.3178034179536273, + 1.4654302718412722, + 1.6094701969162322, + 1.4230510816446773, + 1.565168902852383}), + torch::tensor( + {3.1007583934270064, 2.039757113618415, 1.9652096140698696}), + torch::tensor( + {-3.0880626664330832, -3.166705422245348, -1.73538367534238}), + torch::tensor({-1.3130428735015893}), + }, + { + torch::tensor( + {2.0413773043991963, + 2.3251469369586366, + 1.4690808101517236, + 1.6174065798291044, + 1.4280274009117935, + 1.5682418226469732}), + torch::tensor( + {3.118843540209399, 2.057729936485249, 1.9742319629710936}), + torch::tensor( + {-3.1331019663177013, -3.2154332694373107, -1.7459831639793468}), + torch::tensor({-1.3148644134154366}), + }, + { + torch::tensor( + {2.0452604138357113, + 2.332074253989847, + 1.4738845773449165, + 1.6246403004735728, + 1.4335712611625357, + 1.573826630920094}), + torch::tensor( + {3.1324088069784093, 2.0711763619826575, 1.9841582498316732}), + torch::tensor( + {-3.16737058959847, -3.2529206463859146, -1.7602788393925501}), + torch::tensor({-1.32281766461531}), + }, + { + torch::tensor( + {2.0495243704493262, + 2.338413341249581, + 1.4787599440132637, + 1.631210274009555, + 1.438849155552895, + 1.5798736919537595}), + torch::tensor( + {3.1438209015414227, 2.0823943437659658, 1.9940075805973108}), + torch::tensor( + {-3.19628690363529, -3.2845941643030367, -1.7752333900055153}), + torch::tensor({-1.332456718314933}), + }, + { + torch::tensor( + {2.0536979895206295, + 2.3442520601250334, + 1.4834272584224222, + 1.6372462654983486, + 1.4437517398490174, + 1.585780877834892}), + torch::tensor( + {3.1540081461072447, 2.0923381262560454, 2.0034284957296107}), + torch::tensor( + {-3.222142968201519, -3.312867602521477, -1.7898220261118043}), + torch::tensor({-1.3422692037690986}), + }, + { + torch::tensor( + {2.0576784836825315, + 2.3496934395759377, + 1.4878413407927933, + 1.6428612479757005, + 1.4483225979568104, + 1.5914034339763325}), + torch::tensor( + {3.163383747232199, 2.101446878895216, 2.012344413569353}), + torch::tensor( + {-3.246000281299229, -3.338904166978488, -1.8037666936489785}), + torch::tensor({-1.3517884775416527}), + }, + }; +} + +inline std::vector> AdamW_without_weight_decay() { + return { + { + torch::tensor( + {0.7890972864438476, + 0.5024410688121617, + 0.858707331305558, + 0.6579707241208395, + 0.7476356819075531, + 1.6975564206516922}), + torch::tensor( + {0.891467636010675, 0.70205134975675, 1.689201270942895}), + torch::tensor( + {-1.0508030958460797, -1.3941351509567654, -1.284337577714353}), + torch::tensor({-1.071138110298716}), + }, + { + torch::tensor( + {8.233039313231831, + 7.971150747377481, + 6.643620950677599, + 6.47097740790054, + 6.170125488259256, + 7.150739103343502}), + torch::tensor( + {8.417695070103738, 6.597188212844593, 7.23175710827678}), + torch::tensor( + {-6.729624357635757, -7.09743493108154, -6.753301896575352}), + torch::tensor({-6.435639096011218}), + }, + { + torch::tensor( + {8.233424596059299, + 7.971537360032308, + 6.643920150720393, + 6.471278075537239, + 6.170405874224489, + 7.151021086137983}), + torch::tensor( + {8.418084791214298, 6.597493171180545, 7.232043740621598}), + torch::tensor( + {-6.729918250724671, -7.097730102046093, -6.753584809755359}), + torch::tensor({-6.4359165566974985}), + }, + { + torch::tensor( + {8.233424610557652, + 7.971537374586563, + 6.643920161995284, + 6.471278086877828, + 6.170405884785074, + 7.151021096766406}), + torch::tensor( + {8.418084805901906, 6.597493182713584, 7.2320437514477875}), + torch::tensor( + {-6.72991829363266, -7.097730147102975, -6.753584838821182}), + torch::tensor({-6.435916580217771}), + }, + { + torch::tensor( + {8.233424610575105, + 7.971537374611125, + 6.643920162027961, + 6.471278086923277, + 6.170405884809245, + 7.151021096800041}), + torch::tensor( + {8.418084805946393, 6.597493182796847, 7.232043751509309}), + torch::tensor( + {-6.729918332327653, -7.097730188349552, -6.753584861205486}), + torch::tensor({-6.435916596115672}), + }, + { + torch::tensor( + {8.233424610594861, + 7.971537374639166, + 6.64392016206557, + 6.471278086975758, + 6.170405884836981, + 7.151021096838798}), + torch::tensor( + {8.418084805997617, 6.59749318289335, 7.232043751580523}), + torch::tensor( + {-6.72991837738045, -7.097730236373201, -6.753584887267492}), + torch::tensor({-6.43591661462546}), + }, + { + torch::tensor( + {8.233424610617291, + 7.971537374671012, + 6.643920162108284, + 6.471278087035361, + 6.170405884868481, + 7.151021096882812}), + torch::tensor( + {8.418084806055798, 6.59749318300295, 7.232043751661401}), + torch::tensor( + {-6.729918428547273, -7.09773029091405, -6.753584916866329}), + torch::tensor({-6.4359166356471755}), + }, + { + torch::tensor( + {8.233424610642356, + 7.9715373747065925, + 6.643920162156006, + 6.471278087101954, + 6.1704058849036745, + 7.15102109693199}), + torch::tensor( + {8.418084806120802, 6.597493183125404, 7.232043751751764}), + torch::tensor( + {-6.729918485714688, -7.0977303518511805, -6.753584949936365}), + torch::tensor({-6.43591665913422}), + }, + { + torch::tensor( + {8.233424610670038, + 7.97153737474589, + 6.643920162208714, + 6.471278087175501, + 6.170405884942545, + 7.151021096986303}), + torch::tensor( + {8.418084806192596, 6.597493183260647, 7.232043751851564}), + torch::tensor( + {-6.729918548853505, -7.097730419153473, -6.753584986460725}), + torch::tensor({-6.435916685074594}), + }, + { + torch::tensor( + {8.233424610700352, + 7.971537374788922, + 6.643920162266432, + 6.47127808725604, + 6.17040588498511, + 7.1510210970457795}), + torch::tensor( + {8.418084806271217, 6.597493183408747, 7.232043751960854}), + torch::tensor( + {-6.7299186179943, -7.097730492853521, -6.753585026457088}), + torch::tensor({-6.435916713480863}), + }, + { + torch::tensor( + {8.23342461073333, + 7.971537374835737, + 6.643920162329224, + 6.471278087343658, + 6.170405885031416, + 7.151021097110484}), + torch::tensor( + {8.418084806356747, 6.597493183569867, 7.232043752079749}), + torch::tensor( + {-6.729918693213275, -7.097730573032567, -6.753585069969552}), + torch::tensor({-6.43591674438434}), + }, + }; +} + +inline std::vector> AdamW_with_amsgrad() { + return { + { + torch::tensor( + {0.7912062750121864, + 0.5074166292785842, + 0.8601202529258052, + 0.6613910130887053, + 0.7501593169903569, + 1.6905808503961983}), + torch::tensor( + {0.8925529482073002, 0.7050308347536254, 1.682309255842939}), + torch::tensor( + {-1.05029506454492, -1.3901937990816595, -1.2814942017397601}), + torch::tensor({-1.0704267290556988}), + }, + { + torch::tensor( + {3.3017259270507915, + 3.2082991753694565, + 2.653930978510442, + 2.5927674339810585, + 2.4689608790182933, + 2.825873703467739}), + torch::tensor( + {3.373698198112671, 2.6425942964586664, 2.8597930424244304}), + torch::tensor( + {-2.690360632302962, -2.8253191596069525, -2.6855499873057473}), + torch::tensor({-2.5644658591929406}), + }, + { + torch::tensor( + {2.222607725541013, + 2.3447188854637004, + 1.5614270655258826, + 1.606610018462357, + 1.5260497191448619, + 1.7309643622674138}), + torch::tensor( + {2.84137462783552, 1.824806600633721, 1.9620493659996037}), + torch::tensor( + {-2.576642773625787, -2.6706153846815766, -1.8799876863754623}), + torch::tensor({-1.6044722984810953}), + }, + { + torch::tensor( + {2.0739558768648205, + 2.3008338863863496, + 1.4738888208638767, + 1.5829485271829449, + 1.4296176764284294, + 1.5939984909850073}), + torch::tensor( + {2.9908013612792415, 1.936590940953305, 1.941691630199464}), + torch::tensor( + {-2.846562884997548, -2.9195962101501203, -1.746484716887341}), + torch::tensor({-1.381525131003179}), + }, + { + torch::tensor( + {2.0333926094256953, + 2.294977109171754, + 1.452870514716895, + 1.584853677999522, + 1.4086299433181402, + 1.5548201727855224}), + torch::tensor( + {3.0454817801193976, 1.9867169062383696, 1.935312753106444}), + torch::tensor( + {-2.9612116762746394, -3.0322275992001084, -1.7026114905180725}), + torch::tensor({-1.30563541393247}), + }, + { + torch::tensor( + {2.02392417168201, + 2.2977279587859, + 1.4488511120131309, + 1.5894646930743725, + 1.406253686759073, + 1.5450647949022756}), + torch::tensor( + {3.069025602955343, 2.0096872138967488, 1.935438546309299}), + torch::tensor( + {-3.016103148166836, -3.0893062953033583, -1.6925290685615872}), + torch::tensor({-1.282870120405012}), + }, + { + torch::tensor( + {2.0230257817348316, + 2.3016167065040647, + 1.4496901978629444, + 1.5939034289777392, + 1.4082421794430946, + 1.5444538756003756}), + torch::tensor( + {3.0814016132787954, 2.022150201844143, 1.9387429991308658}), + torch::tensor( + {-3.0466485946438406, -3.1223144611322446, -1.6944083009127773}), + torch::tensor({-1.2786980736911064}), + }, + { + torch::tensor( + {2.024305922065404, + 2.305101549510461, + 1.4516863420588493, + 1.5976447954882376, + 1.4108596097183552, + 1.5464236284303425}), + torch::tensor( + {3.0892574233545065, 2.0300944858242236, 1.943040321845021}), + torch::tensor( + {-3.0664189567897306, -3.1440820888166425, -1.6999750448893618}), + torch::tensor({-1.2806281811826203}), + }, + { + torch::tensor( + {2.0259854116237364, + 2.3080169152218293, + 1.4537680296813915, + 1.6007369432392426, + 1.4132529823064277, + 1.5489035046525346}), + torch::tensor( + {3.0949717180851337, 2.0358251379915764, 1.9473249654893}), + torch::tensor( + {-3.0808231377905426, -3.160021699873689, -1.7062031001273494}), + torch::tensor({-1.28423401369705}), + }, + { + torch::tensor( + {2.0275923948348638, + 2.3104512601389637, + 1.455657715721078, + 1.6033123357613526, + 1.4153003204463288, + 1.5512775896116622}), + torch::tensor( + {3.099479021299846, 2.0403012223048775, 1.9512285931847464}), + torch::tensor( + {-3.092151979336299, -3.1725453680885267, -1.7120689614428697}), + torch::tensor({-1.2880095517062655}), + }, + { + torch::tensor( + {2.029022468328371, + 2.3125066985045892, + 1.4573100228823295, + 1.605484259933419, + 1.417038245960655, + 1.5533932056240227}), + torch::tensor( + {3.103195011518616, 2.043964003458376, 1.9546640840748621}), + torch::tensor( + {-3.1014680843131184, -3.1828179298513968, -1.7172933346797972}), + torch::tensor({-1.2914899987134136}), + }, + }; +} + +inline std::vector> Adagrad() { + return { + { + torch::tensor( + {0.7891011045987429, + 0.502443924512199, + 0.8587078329085825, + 0.6579710994224826, + 0.7476364836215006, + 1.697557019500397}), + torch::tensor( + {0.8914687688941954, 0.7020514988069096, 1.6892015076050444}), + torch::tensor( + {-1.0508031297732776, -1.3941351871450518, -1.284337597261839}), + torch::tensor({-1.071138124161711}), + }, + { + torch::tensor( + {2.4079229696892583, + 2.2346803754764286, + 1.6967885588547365, + 1.552279695827649, + 1.2259044248443602, + 2.221279696180243}), + torch::tensor( + {2.9334079162217193, 1.7619824934767887, 2.3464577179091473}), + torch::tensor( + {-2.221396083069719, -2.549950976011168, -1.9709315957317095}), + torch::tensor({-1.5858816837541876}), + }, + { + torch::tensor( + {2.510404433941812, + 2.3522584510262887, + 1.7921695110761213, + 1.657755825836846, + 1.2891186618593045, + 2.291878516133922}), + torch::tensor( + {3.092171180776419, 1.8971624370952997, 2.438734251283465}), + torch::tensor( + {-2.437641633486504, -2.7704264590526573, -2.0949471699460225}), + torch::tensor({-1.6769121890401757}), + }, + { + torch::tensor( + {2.5652648968109415, + 2.4155313947260972, + 1.844241233613541, + 1.7156513351246399, + 1.3245206506797171, + 2.3315409972138825}), + torch::tensor( + {3.178399916514377, 1.9721945764936502, 2.4909037706250428}), + torch::tensor( + {-2.5658710403147933, -2.901921821645266, -2.168560672193225}), + torch::tensor({-1.7307903926154131}), + }, + { + torch::tensor( + {2.6021584494332592, + 2.4582101324909065, + 1.8796060082750778, + 1.7550965207414717, + 1.3489253597999988, + 2.3589345190118247}), + torch::tensor( + {3.2368674310041516, 2.0236468833666894, 2.52707132741292}), + torch::tensor( + {-2.6573969292994164, -2.9960731060650505, -2.2211375717304076}), + torch::tensor({-1.7692090167089707}), + }, + { + torch::tensor( + {2.629700772579208, + 2.4901377017698683, + 1.906173477530586, + 1.7847957161833832, + 1.3674517119505822, + 2.3797578857769905}), + torch::tensor( + {3.2807643102638546, 2.062561811940094, 2.5546379424362775}), + torch::tensor( + {-2.7286379977755035, -3.0695109399636236, -2.262081199960513}), + torch::tensor({-1.7990936323432214}), + }, + { + torch::tensor( + {2.6515471766995247, + 2.51550257362603, + 1.927341363452414, + 1.8084994719811576, + 1.3823309942932445, + 2.3964995243914373}), + torch::tensor( + {3.3157334001309473, 2.093728023484945, 2.5768468697402924}), + torch::tensor( + {-2.786981763434855, -3.129746439571402, -2.29562487034177}), + torch::tensor({-1.8235564908139104}), + }, + { + torch::tensor( + {2.6695780544837886, + 2.53646401614724, + 1.9448721033433505, + 1.828157582353901, + 1.3947329882074622, + 2.4104657178934947}), + torch::tensor( + {3.344694775590452, 2.1196465761628516, 2.5954050923596252}), + torch::tensor( + {-2.8363936812536537, -3.1808219609745194, -2.32404190866147}), + torch::tensor({-1.8442667636913117}), + }, + { + torch::tensor( + {2.684883801533072, + 2.5542762192735515, + 1.9597939532350015, + 1.844909608012419, + 1.4053459079217485, + 2.4224257790968386}), + torch::tensor( + {3.369349515259956, 2.1417845308976795, 2.611319989214332}), + torch::tensor( + {-2.879251075341889, -3.225165734647855, -2.3486956737228057}), + torch::tensor({-1.86222449978646}), + }, + { + torch::tensor( + {2.698151012423769, + 2.56972998600169, + 1.972757472697587, + 1.8594775691681182, + 1.4146081751022495, + 2.43287021079559}), + torch::tensor( + {3.390772758897601, 2.1610741754331757, 2.6252349489549824}), + torch::tensor( + {-2.917092322961074, -3.264351563375218, -2.370468664387175}), + torch::tensor({-1.8780765115117757}), + }, + { + torch::tensor( + {2.7098389356033783, + 2.5833548721723747, + 1.9841994925173085, + 1.8723468731726323, + 1.4228158926355312, + 2.4421305315945085}), + torch::tensor( + {3.4096859099156673, 2.178143852041279, 2.6375854547611364}), + torch::tensor( + {-2.9509704554208467, -3.2994581338995044, -2.3899651139415874}), + torch::tensor({-1.8922653655195538}), + }, + }; +} + +inline std::vector> Adagrad_with_weight_decay() { + return { + { + torch::tensor( + {0.7891011218979068, + 0.5024439415126254, + 0.8587078332470682, + 0.6579710998575992, + 0.7476364849956589, + 1.6975570150849029}), + torch::tensor( + {0.8914687701583902, 0.7020514988715463, 1.6892015071335027}), + torch::tensor( + {-1.0508031297726799, -1.3941351871397083, -1.2843375972607243}), + torch::tensor({-1.0711381241615712}), + }, + { + torch::tensor( + {0.1846116678522213, + 0.24944077103107917, + 0.18651745437755768, + 0.25219093533041764, + 0.18712037968446713, + 0.25289206444055234}), + torch::tensor( + {0.6482869597891656, 0.6580215784646755, 0.6581256007663537}), + torch::tensor( + {-1.454709711443681, -1.4748063405174818, -1.4811625946604765}), + torch::tensor({-1.905292836544363}), + }, + { + torch::tensor( + {0.18059895999281475, + 0.2438515539257779, + 0.18067177884778182, + 0.24397186395008694, + 0.18168388351830797, + 0.24533853846052017}), + torch::tensor( + {0.6325250261983028, 0.6331827793513023, 0.6366659383355598}), + torch::tensor( + {-1.420803333750877, -1.4215627240541653, -1.4320264544533396}), + torch::tensor({-2.030135641848322}), + }, + { + torch::tensor( + {0.17981392697398363, + 0.2427571544305695, + 0.17981150414451733, + 0.24275725992310523, + 0.18014798619115763, + 0.2432144956227816}), + torch::tensor( + {0.6294321320817985, 0.6294873737410742, 0.6306958589251878}), + torch::tensor( + {-1.4139253354785764, -1.413902680470981, -1.4173628530293867}), + torch::tensor({-2.056210117690093}), + }, + { + torch::tensor( + {0.17967006242163747, + 0.24255582734557277, + 0.17966873677301953, + 0.24255462870545766, + 0.1797588230898851, + 0.24267729072765756}), + torch::tensor( + {0.6288576295241085, 0.6288643132826753, 0.6291921485342001}), + torch::tensor( + {-1.4126465879787569, -1.4126335126907266, -1.4135586793353685}), + torch::tensor({-2.0618018405404825}), + }, + { + torch::tensor( + {0.17964321284685653, + 0.24251808241139364, + 0.17964291377171066, + 0.24251779104198598, + 0.1796651574178102, + 0.2425481059085456}), + torch::tensor( + {0.628748693136779, 0.6287498167193976, 0.6288312441271762}), + torch::tensor( + {-1.412405895385289, -1.4124029484481164, -1.4126313051315378}), + torch::tensor({-2.0630223163099304}), + }, + { + torch::tensor( + {0.1796379973927849, + 0.2425107191246215, + 0.1796379363134342, + 0.2425106592536174, + 0.17964321802205502, + 0.24251786094585864}), + torch::tensor( + {0.6287272170354161, 0.6287274414587727, 0.6287468362309863}), + torch::tensor( + {-1.4123588626342634, -1.412358263650784, -1.412412480569672}), + torch::tensor({-2.0632918101480255}), + }, + { + torch::tensor( + {0.17963694231402444, + 0.24250922426893615, + 0.1796369298071621, + 0.2425092121007451, + 0.17963815759528073, + 0.24251088666939838}), + torch::tensor( + {0.6287228195255881, 0.6287228675172439, 0.628727383936762}), + torch::tensor( + {-1.4123493065102781, -1.412349184462438, -1.4123617872243597}), + torch::tensor({-2.063351765096138}), + }, + { + torch::tensor( + {0.1796367215904632, + 0.24250891070978667, + 0.17963671897003045, + 0.24250890818318074, + 0.17963700091084855, + 0.24250929278196054}), + torch::tensor( + {0.6287218911936107, 0.6287219017313679, 0.6287229399204574}), + torch::tensor( + {-1.4123473011084142, -1.412347275640343, -1.41235016959507}), + torch::tensor({-2.0633651674043505}), + }, + { + torch::tensor( + {0.17963667424978783, + 0.24250884333120687, + 0.17963667368829764, + 0.24250884279379462, + 0.17963673796131557, + 0.24250893047794023}), + torch::tensor( + {0.6287216908150736, 0.6287216931558691, 0.6287219299749583}), + torch::tensor( + {-1.4123468700596724, -1.4123468646187736, -1.4123475243360133}), + torch::tensor({-2.0633681724342527}), + }, + { + torch::tensor( + {0.1796366639185348, + 0.24250882860835257, + 0.17963666379614568, + 0.24250882849182892, + 0.17963667838367053, + 0.2425088483939741}), + torch::tensor( + {0.6287216468984888, 0.6287216474215305, 0.6287217011907862}), + torch::tensor( + {-1.4123467758545658, -1.412346774671038, -1.4123469244007658}), + torch::tensor({-2.0633688474977467}), + }, + }; +} + +inline std::vector> +Adagrad_with_weight_decay_and_lr_decay() { + return { + { + torch::tensor( + {0.7891011046018798, + 0.5024439245163383, + 0.8587078329086189, + 0.6579710994225316, + 0.747636483621666, + 1.697557019500142}), + torch::tensor( + {0.8914687688943375, 0.7020514988069164, 1.6892015076050049}), + torch::tensor( + {-1.0508031297732776, -1.3941351871450511, -1.284337597261839}), + torch::tensor({-1.0711381241617108}), + }, + { + torch::tensor( + {2.346218944110103, + 2.191939439502003, + 1.683355201740813, + 1.5405520021635604, + 1.2137800230828062, + 2.205283463717303}), + torch::tensor( + {2.9090564593404, 1.7509657336815554, 2.336166413186925}), + torch::tensor( + {-2.206159683368316, -2.5344318233445415, -1.9622783535807609}), + torch::tensor({-1.5796101463783623}), + }, + { + torch::tensor( + {2.3889328781057233, + 2.2678221038007296, + 1.7667624725138267, + 1.6358015176639822, + 1.2655767687152566, + 2.261088056711282}), + torch::tensor( + {3.045569451994985, 1.8770196253823253, 2.4192707519566765}), + torch::tensor( + {-2.4079300017528613, -2.7399112002234305, -2.0780613510632375}), + torch::tensor({-1.664722108226537}), + }, + { + torch::tensor( + {2.3886137557806384, + 2.2922158071009178, + 1.8078384116424007, + 1.684352474440932, + 1.290353948335789, + 2.2870715509706496}), + torch::tensor( + {3.111110355394278, 1.9438501730282314, 2.4630249355872826}), + torch::tensor( + {-2.5226122034499263, -2.857315093916292, -2.143964860243905}), + torch::tensor({-1.7130685809905042}), + }, + { + torch::tensor( + {2.374703352203156, + 2.298804499257456, + 1.8330249458212446, + 1.7151661013307244, + 1.3048586226945842, + 2.3017650590464274}), + torch::tensor( + {3.150318222034133, 1.9877926185369321, 2.491399976401679}), + torch::tensor( + {-2.601415913361488, -2.938203895113964, -2.1892988334550028}), + torch::tensor({-1.7462964261966805}), + }, + { + torch::tensor( + {2.3553658567303812, + 2.297191758042688, + 1.8501154749072124, + 1.736836058688188, + 1.3141313000193942, + 2.3107452592153854}), + torch::tensor( + {3.1762315339155434, 2.0197585204578647, 2.5117041377790197}), + torch::tensor( + {-2.6606644002288697, -2.9991216074293856, -2.223413376189609}), + torch::tensor({-1.7712905233118807}), + }, + { + torch::tensor( + {2.3338052201696207, + 2.2913023710914993, + 1.8624163948044772, + 1.7530300731725454, + 1.3203313209234842, + 2.3163969478854747}), + torch::tensor( + {3.1943525925688934, 2.044447386769377, 2.527109724607397}), + torch::tensor( + {-2.7076634717294894, -3.047500808469036, -2.250495807208967}), + torch::tensor({-1.7911288238757486}), + }, + { + torch::tensor( + {2.3114979154644892, + 2.2830501835377808, + 1.871616142999356, + 1.765632597660841, + 1.324565631636651, + 2.319939234205203}), + torch::tensor( + {3.2074779925809085, 2.0642940833670544, 2.5392671301471235}), + torch::tensor( + {-2.7463093287485925, -3.087315541134716, -2.272780318857348}), + torch::tensor({-1.8074516661537263}), + }, + { + torch::tensor( + {2.2891841627387346, + 2.2734716995793693, + 1.8786818999895825, + 1.7757301317117602, + 1.3274682997719436, + 2.3220679353993825}), + torch::tensor( + {3.2172019619454075, 2.0807140893178175, 2.5491374815141876}), + torch::tensor( + {-2.7789204504423823, -3.1209351402429175, -2.2915969523376867}), + torch::tensor({-1.821234722948421}), + }, + { + torch::tensor( + {2.2672498238343066, + 2.2631678037928893, + 1.8842131287032622, + 1.7840007705383882, + 1.3294311820750493, + 2.323211243034543}), + torch::tensor( + {3.224507488068445, 2.094598223519413, 2.5573257155791715}), + torch::tensor( + {-2.8069849199086647, -3.1498826045022925, -2.3077996970997727}), + torch::tensor({-1.8331040438272388}), + }, + { + torch::tensor( + {2.2458961718688957, + 2.2525031725114775, + 1.8886034384961112, + 1.7908930341267952, + 1.3307102435291205, + 2.323647497600462}), + torch::tensor( + {3.230036385413078, 2.1065407459636134, 2.5642349249609664}), + torch::tensor( + {-2.83151249424399, -3.1751926295316566, -2.3219682378974036}), + torch::tensor({-1.8434843744626483}), + }, + }; +} + +inline std::vector> RMSprop() { + return { + { + torch::tensor( + {0.7890625772821005, + 0.502415108650816, + 0.8587027713011453, + 0.657967312300643, + 0.7476283936579036, + 1.6975509766054537}), + torch::tensor( + {0.8914573371873159, 0.7020499947573374, 1.6891991194739453}), + torch::tensor( + {-1.0508027874171133, -1.3941348219724659, -1.2843374000099703}), + torch::tensor({-1.0711379842715099}), + }, + { + torch::tensor( + {2.448571858277443, + 2.2809152044417678, + 1.7346424449151965, + 1.5940004770230667, + 1.250761131839982, + 2.248993270255382}), + torch::tensor( + {2.994661478530102, 1.8150485290864256, 2.382542610897819}), + torch::tensor( + {-2.3036981738757825, -2.6337299521275646, -2.018370122358821}), + torch::tensor({-1.620787559800898}), + }, + { + torch::tensor( + {2.5837582475607785, + 2.4365737242301537, + 1.8622886519354538, + 1.7357065282848232, + 1.3369695670141974, + 2.3454934716983695}), + torch::tensor( + {3.2061266499381618, 1.9981112525417788, 2.5092495986614}), + torch::tensor( + {-2.6110809365525958, -2.9484807193016787, -2.194898560798439}), + torch::tensor({-1.7501043480625826}), + }, + { + torch::tensor( + {2.669969051134511, + 2.536559412710799, + 1.9456091681389671, + 1.828914948091767, + 1.3952956766999587, + 2.4110816686341923}), + torch::tensor( + {3.343672975593657, 2.1204057198913002, 2.5961524902119497}), + torch::tensor( + {-2.8372329851331006, -3.1817729538857207, -2.3249971853996954}), + torch::tensor({-1.8450422173907486}), + }, + { + torch::tensor( + {2.7375365004059122, + 2.6153071545358633, + 2.0117493624534313, + 1.9033001982031035, + 1.4427501882445097, + 2.4646213743186127}), + torch::tensor( + {3.452912454199796, 2.2190451524127535, 2.667552790123282}), + torch::tensor( + {-3.0329479456731505, -3.384582488936652, -2.4377299824997136}), + torch::tensor({-1.9271014784118226}), + }, + { + torch::tensor( + {2.7952372917068753, + 2.682820220375722, + 2.0687223272686994, + 1.967654548778711, + 1.4844410726622166, + 2.5117888904510117}), + torch::tensor( + {3.5471904628565745, 2.305113548262141, 2.7307948248967304}), + torch::tensor( + {-3.2141190290332537, -3.572944633614449, -2.5421970206546827}), + torch::tensor({-2.0029976985219666}), + }, + { + torch::tensor( + {2.8467333937519483, + 2.7432785177110395, + 2.119898810135385, + 2.0256805416741255, + 1.5225256464280221, + 2.554983108087885}), + torch::tensor( + {3.632098323876194, 2.383289304179778, 2.7889864719999222}), + torch::tensor( + {-3.387579944167926, -3.7537658010839294, -2.6423123266260427}), + torch::tensor({-2.075616951445725}), + }, + { + torch::tensor( + {2.8938600498060203, + 2.798776984183615, + 2.16697381475156, + 2.079238538430203, + 1.5580820887115123, + 2.5954023023969692}), + torch::tensor( + {3.71043881435304, 2.455919099321953, 2.8436784441941008}), + torch::tensor( + {-3.5567368287146417, -3.930484868709691, -2.740026479434574}), + torch::tensor({-2.146398256871758}), + }, + { + torch::tensor( + {2.937649394399939, + 2.850492965312311, + 2.210902777232446, + 2.1293746183147633, + 1.5917084661873866, + 2.6337100421079533}), + torch::tensor( + {3.7837853328443516, 2.5243155701130604, 2.8957265009949373}), + torch::tensor( + {-3.7234268485210475, -4.104949193318518, -2.836390693799751}), + torch::tensor({-2.216118773360611}), + }, + { + torch::tensor( + {2.9787316798887558, + 2.8991451078473207, + 2.252272487010975, + 2.1767288729202767, + 1.6237627746697592, + 2.670302507579268}), + torch::tensor( + {3.8530980655045086, 2.589275531553025, 2.9456388178450936}), + torch::tensor( + {-3.8886856459619636, -4.278191888396593, -2.9319964928350313}), + torch::tensor({-2.285217699505124}), + }, + { + torch::tensor( + {3.017515620579049, + 2.9452004251453268, + 2.291469925522962, + 2.2217217025782916, + 1.6544730927621272, + 2.705431441204422}), + torch::tensor( + {3.9190041004420166, 2.651317624465938, 2.993736489599992}), + torch::tensor( + {-4.053111559341913, -4.450801801162238, -3.0271845519131957}), + torch::tensor({-2.3539498905973444}), + }, + }; +} + +inline std::vector> RMSprop_with_weight_decay() { + return { + { + torch::tensor( + {0.7890798754118442, + 0.5024321083861885, + 0.8587031097835685, + 0.6579677474141494, + 0.7476297677960806, + 1.6975465611838714}), + torch::tensor( + {0.891458601354904, 0.7020500593937647, 1.6891986479348047}), + torch::tensor( + {-1.0508027868194278, -1.3941348166291232, -1.2843373988951865}), + torch::tensor({-1.0711379841318796}), + }, + { + torch::tensor( + {0.2139892652405453, + 0.2779011713353896, + 0.18684802794665187, + 0.2507569370785562, + 0.19145335235130007, + 0.2557687813140708}), + torch::tensor( + {0.6720959116689083, 0.6480734848064099, 0.654263007004671}), + torch::tensor( + {-1.4357633640899097, -1.4493557950073235, -1.4619011018357073}), + torch::tensor({-1.9673083558727926}), + }, + { + torch::tensor( + {0.23961935744660673, + 0.30354236865888945, + 0.19567694278583514, + 0.2544696440133763, + 0.21982879020261814, + 0.27495711471979495}), + torch::tensor( + {0.6927895724658635, 0.638015535479105, 0.6523245375960234}), + torch::tensor( + {-1.413722583500382, -1.4170291001633526, -1.4166977298480703}), + torch::tensor({-2.0626651115437147}), + }, + { + torch::tensor( + {0.2506635865272117, + 0.314639511428635, + 0.25116892910818034, + 0.30431399579592144, + 0.25219625048710015, + 0.3160110008170742}), + torch::tensor( + {0.7051419232960522, 0.6699011906397543, 0.699097284678438}), + torch::tensor( + {-1.4206083241624232, -1.4257037444100107, -1.4171061826065132}), + torch::tensor({-2.075537874763694}), + }, + { + torch::tensor( + {0.23285924743063605, + 0.29652494304777544, + 0.2335322002738168, + 0.2969991261380461, + 0.23358272245229555, + 0.2973997498166104}), + torch::tensor( + {0.6855589594925036, 0.6796983775695974, 0.6864174803983276}), + torch::tensor( + {-1.43110762794651, -1.4334934742818164, -1.422739552145125}), + torch::tensor({-2.0842642493046184}), + }, + { + torch::tensor( + {0.23356397699389828, + 0.29737142391985355, + 0.23367622061822368, + 0.29749447597160267, + 0.23418481357395918, + 0.29818122925156104}), + torch::tensor( + {0.6866530583001205, 0.6858933385102559, 0.6883944045412603}), + torch::tensor( + {-1.4564955509607018, -1.4583548131500643, -1.4418225445708595}), + torch::tensor({-2.1064103749186183}), + }, + { + torch::tensor( + {0.2318717301174723, + 0.2952904159872858, + 0.23194024439476665, + 0.29537019824987687, + 0.2316421336904657, + 0.2951041425983894}), + torch::tensor( + {0.6834813130194509, 0.6834401711464199, 0.6837275457100463}), + torch::tensor( + {-1.4647835805276763, -1.4653452408179053, -1.4571142112777709}), + torch::tensor({-2.1209598505912086}), + }, + { + torch::tensor( + {0.2308683396504178, + 0.2940474629750448, + 0.23089067678260966, + 0.29407615110959306, + 0.23064069314214175, + 0.29379043611390243}), + torch::tensor( + {0.6815062281792611, 0.6815233687209215, 0.6812759203026146}), + torch::tensor( + {-1.4643013018530682, -1.4644523635284246, -1.4617493939684878}), + torch::tensor({-2.1247293635678854}), + }, + { + torch::tensor( + {0.23066464201462678, + 0.29376059273730426, + 0.23067069245857366, + 0.2937690399784267, + 0.23057551211606675, + 0.2936477517373107}), + torch::tensor( + {0.6809028781780304, 0.6809134105028244, 0.6807404613096301}), + torch::tensor( + {-1.4637927352177986, -1.4638374228010727, -1.46299287102643}), + torch::tensor({-2.1258082720638107}), + }, + { + torch::tensor( + {0.23062625079199173, + 0.2936990787425707, + 0.2306278924729115, + 0.2937014834661651, + 0.23059813368157003, + 0.29366073890476396}), + torch::tensor( + {0.6807251804689082, 0.6807295616357246, 0.6806523640328994}), + torch::tensor( + {-1.4635790398985618, -1.463592926902286, -1.4633272688236565}), + torch::tensor({-2.1261396358141798}), + }, + { + torch::tensor( + {0.23061701122700193, + 0.293683983817782, + 0.23061747865501653, + 0.29368467998690806, + 0.23060855595638208, + 0.2936719507340021}), + torch::tensor( + {0.6806714673830832, 0.6806730903175793, 0.6806434720800856}), + torch::tensor( + {-1.4635008778278134, -1.4635052859178375, -1.4634208375068285}), + torch::tensor({-2.1262432969587723}), + }, + }; +} + +inline std::vector> +RMSprop_with_weight_decay_and_centered() { + return { + { + torch::tensor( + {0.7941000061626792, + 0.507452636734552, + 0.8637405354185987, + 0.663005089317529, + 0.7526661272860107, + 1.7025887305065852}), + torch::tensor( + {0.8964950370033696, 0.7070877948157552, 1.6942369105467197}), + torch::tensor( + {-1.055840599214661, -1.3991726335388424, -1.2893752132746332}), + torch::tensor({-1.0761757981162612}), + }, + { + torch::tensor( + {2.3762999876885833, + 2.239095829416783, + 1.726175067071914, + 1.5891569459230444, + 1.2410074108588462, + 2.2345431036725723}), + torch::tensor( + {2.990896455635836, 1.8152108764849464, 2.377985429759037}), + torch::tensor( + {-2.3071822180635286, -2.636859516619699, -2.0198181394256642}), + torch::tensor({-1.622583045791722}), + }, + { + torch::tensor( + {2.372800588647971, + 2.3022753207224254, + 1.836028714221617, + 1.7190937269287105, + 1.3068955839895078, + 2.3035835673200364}), + torch::tensor( + {3.1656599892042343, 1.9942937608209466, 2.4947143457182657}), + torch::tensor( + {-2.6139790332516775, -2.9507738987695404, -2.1954425128779516}), + torch::tensor({-1.7513053380188806}), + }, + { + torch::tensor( + {2.2398453700818455, + 2.2513384246965904, + 1.8892176431436287, + 1.7921873754661686, + 1.3310951408713538, + 2.3236392222350397}), + torch::tensor( + {3.240166119454613, 2.109742813600189, 2.5651614461576973}), + torch::tensor( + {-2.8388734382997454, -3.1824200770676123, -2.324831397600949}), + torch::tensor({-1.8460315737386976}), + }, + { + torch::tensor( + {1.9829606312242465, + 2.097356567850692, + 1.9050263843525033, + 1.8325835415812346, + 1.3222762370713104, + 2.3024963133870147}), + torch::tensor( + {3.2465360572089974, 2.1967266045869915, 2.6091992649970672}), + torch::tensor( + {-3.0326878099587207, -3.3827004807595005, -2.436989182250496}), + torch::tensor({-1.928273216206344}), + }, + { + torch::tensor( + {1.6051175329080525, + 1.8332107491649114, + 1.8794767349053179, + 1.8403588051948856, + 1.273824111314107, + 2.2296571379436823}), + torch::tensor( + {3.1814362940910437, 2.263019214072847, 2.6273016977574013}), + torch::tensor( + {-3.210932646440219, -3.567153254014387, -2.541016943923914}), + torch::tensor({-2.0049155134617154}), + }, + { + torch::tensor( + {1.1588059349082709, + 1.477861379523226, + 1.7992410089026634, + 1.806460009198667, + 1.1739931551629919, + 2.08647960875392}), + torch::tensor( + {3.03843703712275, 2.308203068375877, 2.6125393914734083}), + torch::tensor( + {-3.379830678608588, -3.741970414470626, -2.6410082400846546}), + torch::tensor({-2.079294995910487}), + }, + { + torch::tensor( + {0.7701433312419088, + 1.1105026677424745, + 1.646507516936639, + 1.71625269098179, + 1.013748545414221, + 1.8532966501655352}), + torch::tensor( + {2.827176875885245, 2.327401948159928, 2.5535309398603405}), + torch::tensor( + {-3.54193329850986, -3.9096652952123145, -2.739408870192437}), + torch::tensor({-2.1537939241668997}), + }, + { + torch::tensor( + {0.5598923129351211, + 0.8460500042788701, + 1.4084175549165017, + 1.5547314210944563, + 0.8019580519338424, + 1.5258384663629627}), + torch::tensor( + {2.5774950379490265, 2.313101306699127, 2.4388695757441745}), + torch::tensor( + {-3.6974974230160087, -4.070190514312716, -2.8378932675718405}), + torch::tensor({-2.2307225014430423}), + }, + { + torch::tensor( + {0.5016784472836648, + 0.7258690889265433, + 1.0976902935953956, + 1.319949187972513, + 0.5853930356154851, + 1.1446978015944624}), + torch::tensor( + {2.3235249877284945, 2.2592840970420176, 2.2681461698609375}), + torch::tensor( + {-3.8444921272569115, -4.22021051361099, -2.9373192115434263}), + torch::tensor({-2.312733063937045}), + }, + { + torch::tensor( + {0.4875468895095056, + 0.6878747871467128, + 0.7787871237567606, + 1.0462592546102176, + 0.4416468896022397, + 0.8122992916762792}), + torch::tensor( + {2.1078734515587483, 2.17034337037527, 2.0666325968568535}), + torch::tensor( + {-3.9782695475825216, -4.352093055115415, -3.0377809502927033}), + torch::tensor({-2.403496388200805}), + }, + }; +} + +inline std::vector> +RMSprop_with_weight_decay_and_centered_and_momentum() { + return { + { + torch::tensor( + {0.7941000061626794, + 0.507452636734552, + 0.8637405354185985, + 0.663005089317529, + 0.7526661272860107, + 1.7025887305065852}), + torch::tensor( + {0.8964950370033699, 0.7070877948157552, 1.6942369105467197}), + torch::tensor( + {-1.055840599214661, -1.3991726335388424, -1.2893752132746332}), + torch::tensor({-1.0761757981162612}), + }, + { + torch::tensor( + {11.587263945492355, + 12.552112516667206, + 10.773002960161074, + 10.782117868337808, + 9.675467654064093, + 10.830689360054789}), + torch::tensor( + {15.298238342006444, 11.252244653209866, 11.423905295074075}), + torch::tensor( + {-11.287147147258441, -11.673871066494183, -11.143068139029769}), + torch::tensor({-10.744790465364126}), + }, + { + torch::tensor( + {5.993130757784388, + 7.778269455146452, + 9.705741295559012, + 9.974952848613889, + 8.171307305871647, + 9.551498426643077}), + torch::tensor( + {12.811268477045155, 10.912201832960703, 10.87477550647832}), + torch::tensor( + {-11.20842921856976, -11.58706973895515, -11.098172235374586}), + torch::tensor({-10.714110383698559}), + }, + { + torch::tensor( + {1.917316794757853, + 3.442098373003915, + 8.160846071267297, + 8.76673426856121, + 6.163892823252042, + 7.748894752821816}), + torch::tensor( + {9.52929937981379, 10.371703621802425, 10.02242566317017}), + torch::tensor( + {-11.07914626767133, -11.444639737948599, -11.02397978065452}), + torch::tensor({-10.663204622623406}), + }, + { + torch::tensor( + {0.24211162925745067, + 0.8235150923738451, + 6.109652191353378, + 7.070860554523037, + 3.8366635637770212, + 5.46037058418296}), + torch::tensor( + {5.7908039507441, 9.534309069066389, 8.752252906881251}), + torch::tensor( + {-10.868651889371552, -11.212965695734527, -10.90242744782103}), + torch::tensor({-10.579596899816439}), + }, + { + torch::tensor( + {0.0024206009020476234, + 0.05521740497689468, + 3.753606156332189, + 4.9331546064599685, + 1.7094621184709604, + 3.022224882400484}), + torch::tensor( + {2.4729429920325234, 8.290211439306459, 6.983317870704776}), + torch::tensor( + {-10.529133489023623, -10.839885990130032, -10.704345435808353}), + torch::tensor({-10.44279235413811}), + }, + { + torch::tensor( + {8.523664833406631e-06, + -0.00018498015809617104, + 1.6343074841140277, + 2.683608480982546, + 0.41425107807132744, + 1.092111816609512}), + torch::tensor( + {0.553119873538318, 6.566845593450314, 4.783317472190566}), + torch::tensor( + {-9.990101114696575, -10.24914448933998, -10.38447825909146}), + torch::tensor({-10.220382375374728}), + }, + { + torch::tensor( + {5.3669182339397725e-08, + -2.899704029399283e-07, + 0.37916783268568177, + 0.9399553431452395, + 0.02859528129337607, + 0.17650614337704745}), + torch::tensor( + {0.03166973497545419, 4.442846994093523, 2.5203464928754724}), + torch::tensor( + {-9.15653357178671, -9.339631853060773, -9.875729313751442}), + torch::tensor({-9.862669711962374}), + }, + { + torch::tensor( + {2.1133356499004335e-06, + 2.4524630407768025e-06, + 0.023655729923601883, + 0.14273709578291396, + -8.950192389690758e-05, + 0.004237697008964042}), + torch::tensor( + {-0.00012364097582548376, 2.291191859107928, 0.8331414409602524}), + torch::tensor( + {-7.922566174765117, -8.003055545094796, -9.086673634672907}), + torch::tensor({-9.297519364373224}), + }, + { + torch::tensor( + {0.0023497430294992434, + 0.0028611316714725037, + 0.0006998739627296072, + 0.003657156536057531, + 0.001654303471369622, + 0.0018171459470053366}), + torch::tensor( + {0.004569191565477355, 0.7292466599711233, 0.11475431260766135}), + torch::tensor( + {-6.223834483308681, -6.185383631607397, -7.912955414853613}), + torch::tensor({-8.430731662958186}), + }, + { + torch::tensor( + {0.10393820340367545, + 0.13982074666181732, + 0.0831407198272949, + 0.10183584198629944, + 0.13949594516972202, + 0.17822672100147108}), + torch::tensor( + {0.340394645020639, 0.24860888862359687, 0.3191404515531066}), + torch::tensor( + {-4.174294597914298, -4.037528929635062, -6.297198700024484}), + torch::tensor({-7.182093090194918}), + }, + }; +} + +inline std::vector> SGD() { + return { + { + torch::tensor( + {-0.21063957030131192, + -0.4972093725858961, + -0.13931849072410168, + -0.33939101965581686, + -0.25112865488453673, + 0.6992101966874735}), + torch::tensor( + {-0.1076573444246077, -0.2913064413859577, 0.6933846874181748}), + torch::tensor( + {-0.07998325778863398, + -0.42149210515421365, + -0.33498349553944556}), + torch::tensor({-0.14255126505509488}), + }, + { + torch::tensor( + {-0.15543131540224012, + -0.42351103963720343, + -0.04196796248622072, + -0.2095223178068499, + -0.16031407286541022, + 0.8209742464453325}), + torch::tensor( + {0.07724343607160136, 0.03387529472490231, 1.0028793648054941}), + torch::tensor( + {-0.8213382425894498, -1.1570800333254736, -1.615476033165743}), + torch::tensor({-1.8734090731084845}), + }, + { + torch::tensor( + {-0.13342791770744886, + -0.3941509709488104, + -0.011470356542661934, + -0.16885142516066962, + -0.13306680693528108, + 0.8576491729785701}), + torch::tensor( + {0.15081014600761677, 0.13560816175111742, 1.0971559708365837}), + torch::tensor( + {-0.9780975407869251, -1.3215153697157924, -1.876021387605152}), + torch::tensor({-2.2024413056528886}), + }, + { + torch::tensor( + {-0.11963097684681223, + -0.37573675130134543, + 0.0069987166413883715, + -0.14420855651125972, + -0.11733423659038758, + 0.8788673419128562}), + torch::tensor( + {0.1969829338759005, 0.1973461164047132, 1.1520119567305152}), + torch::tensor( + {-1.0677802792431819, -1.4166561260631119, -2.022033753216991}), + torch::tensor({-2.383452427292781}), + }, + { + torch::tensor( + {-0.10950806441156272, + -0.3622226699218595, + 0.02028489243523426, + -0.1264725422838007, + -0.10635775660996463, + 0.8936912722040982}), + torch::tensor( + {0.23089462331826793, 0.24184450074084418, 1.1904864598387046}), + torch::tensor( + {-1.1306213044009719, -1.4837186483578142, -2.122884602514208}), + torch::tensor({-2.5071352505158395}), + }, + { + torch::tensor( + {-0.10149090356585248, + -0.3515172115812867, + 0.030662536099764083, + -0.11261325211798616, + -0.09797248308626623, + 0.905027632401109}), + torch::tensor( + {0.25777759826689434, 0.2766609657536915, 1.2199973265718322}), + torch::tensor( + {-1.1789655573653979, -1.5355073692636774, -2.199612583884608}), + torch::tensor({-2.600529541471662}), + }, + { + torch::tensor( + {-0.09484472748389533, + -0.3426405023243085, + 0.03917399284640637, + -0.10124188994381228, + -0.09121264836307835, + 0.9141743475340721}), + torch::tensor( + {0.2800829300171032, 0.3052600200290069, 1.2438661306695873}), + torch::tensor( + {-1.2182324765944266, -1.5776851394085492, -2.2613704866316295}), + torch::tensor({-2.6752743361973184}), + }, + { + torch::tensor( + {-0.08916446117741175, + -0.33505233521798666, + 0.04638527943959316, + -0.09160422984057517, + -0.08556486270584644, + 0.9218219103015535}), + torch::tensor( + {0.2991619380154852, 0.3295237551295101, 1.2638639017720827}), + torch::tensor( + {-1.251282493526328, -1.6132564639504312, -2.3129529937213853}), + torch::tensor({-2.73741957239466}), + }, + { + torch::tensor( + {-0.08420245801272856, + -0.3284224385121882, + 0.05263847708646642, + -0.08324438788845245, + -0.08072424164719598, + 0.9283806476306355}), + torch::tensor( + {0.31584087342663564, 0.35059019818200393, 1.2810450644764015}), + torch::tensor( + {-1.2798091496372141, -1.6440072538210193, -2.357180462961105}), + torch::tensor({-2.7905023459395872}), + }, + { + torch::tensor( + {-0.07979600214534928, + -0.3225337978155753, + 0.0581562720006689, + -0.07586555700667826, + -0.07649523955108037, + 0.9341138824526719}), + torch::tensor( + {0.3306627217189733, 0.3692005578577212, 1.2960873917356066}), + torch::tensor( + {-1.3048976883823566, -1.6710855742501123, -2.395849898454614}), + torch::tensor({-2.836765085555123}), + }, + { + torch::tensor( + {-0.07583232846497832, + -0.3172360102461862, + 0.06309179259248046, + -0.06926361352067158, + -0.07274510848082802, + 0.9392004636935606}), + torch::tensor( + {0.3440038606091545, 0.3858647867996722, 1.3094518934419668}), + torch::tensor( + {-1.3272851146877218, -1.6952731308502653, -2.4301754289421598}), + torch::tensor({-2.8777164728823017}), + }, + }; +} + +inline std::vector> SGD_with_weight_decay() { + return { + { + torch::tensor( + {-0.21042867144447805, + -0.49671181653925384, + -0.13917719856207697, + -0.3390489907590303, + -0.2508762913762564, + 0.6985126396619242}), + torch::tensor( + {-0.10754881320494518, -0.2910084928862701, 0.6926954859081793}), + torch::tensor( + {-0.079932454658518, -0.42109796996670307, -0.33469915794198624}), + torch::tensor({-0.14248012693079315}), + }, + { + torch::tensor( + {-0.13579982290274883, + -0.3765456284475787, + -0.03166970700350034, + -0.18102559254681197, + -0.1373234786735746, + 0.7522156177001302}), + torch::tensor( + {0.08550003826014418, 0.051563225553454196, 0.9321399061276381}), + torch::tensor( + {-0.796312238882584, -1.1010063686038731, -1.5363716774172782}), + torch::tensor({-1.8045854907382846}), + }, + { + torch::tensor( + {-0.09659168723529124, + -0.30562076936588267, + 0.0067128671455129185, + -0.1166002367977548, + -0.09012083166238948, + 0.7264953102453368}), + torch::tensor( + {0.16531808496504802, 0.16488328577596398, 0.9610743966573317}), + torch::tensor( + {-0.9202466399245914, -1.2052829272891832, -1.7049756710541348}), + torch::tensor({-2.0415977924493043}), + }, + { + torch::tensor( + {-0.06728100597713035, + -0.2496589601654196, + 0.03186158526394668, + -0.07105441484407878, + -0.056478595544178806, + 0.6910758436366733}), + torch::tensor( + {0.21707768347081777, 0.23575238192099465, 0.9564382346520686}), + torch::tensor( + {-0.9788195039029999, -1.2447191597975946, -1.762020156061963}), + torch::tensor({-2.131504419683077}), + }, + { + torch::tensor( + {-0.04304955053155505, + -0.20206572730420902, + 0.050959513946324475, + -0.034700093557440984, + -0.02922465201167018, + 0.6547611705604361}), + torch::tensor( + {0.2563898231537708, 0.2878867158887637, 0.9414221685252802}), + torch::tensor( + {-1.0143969472996655, -1.2623288365082088, -1.7800471460065668}), + torch::tensor({-2.170255083720924}), + }, + { + torch::tensor( + {-0.022154717038262738, + -0.16036518660639862, + 0.06644401410758827, + -0.004183373274651896, + -0.005965877978527781, + 0.6200298215101535}), + torch::tensor( + {0.2886406829874717, 0.32924516791460257, 0.9230983700837223}), + torch::tensor( + {-1.0397895250773481, -1.2710914166240181, -1.7807758009603087}), + torch::tensor({-2.1862978976514738}), + }, + { + torch::tensor( + {-0.0037439139848317077, + -0.12328293308251938, + 0.07944696186805641, + 0.022100305718442022, + 0.014399113804332037, + 0.587697912745227}), + torch::tensor( + {0.3162871074692008, 0.36346293565421134, 0.9042402154310412}), + torch::tensor( + {-1.060234961430088, -1.2762264965487675, -1.7731268727630662}), + torch::tensor({-2.191253945056341}), + }, + { + torch::tensor( + {0.012675985938854726, + -0.09003711893222131, + 0.09059095692632844, + 0.04506778924310349, + 0.03247299240601001, + 0.5579755127260052}), + torch::tensor( + {0.3406226998933173, 0.3924947745885882, 0.8860121369119325}), + torch::tensor( + {-1.0781407849705034, -1.2800528898634018, -1.7613120374342215}), + torch::tensor({-2.190575043873577}), + }, + { + torch::tensor( + {0.027425440985777993, + -0.06008809958617219, + 0.10026092920861808, + 0.06531092947039244, + 0.048628754907931976, + 0.5308215072596255}), + torch::tensor( + {0.36239744520280553, 0.4175162387638887, 0.8688788105023479}), + torch::tensor( + {-1.0946579691370502, -1.283610342226948, -1.7474706191775764}), + torch::tensor({-2.1870021744944763}), + }, + { + torch::tensor( + {0.04073250980147411, + -0.03303024103555013, + 0.1087177047593139, + 0.08324870459183518, + 0.0631222868881554, + 0.5060892094042873}), + torch::tensor( + {0.38208249693950175, 0.4393002654989596, 0.8529817924677643}), + torch::tensor( + {-1.1103326127955466, -1.287332405916359, -1.73273866274852}), + torch::tensor({-2.1819672316721337}), + }, + { + torch::tensor( + {0.05277160918732605, + -0.008539186625351441, + 0.1161515444487197, + 0.09919929206676087, + 0.07614530177703588, + 0.48359250162323586}), + torch::tensor( + {0.3999968617221315, 0.45839442009256354, 0.8383132966805791}), + torch::tensor( + {-1.1254107858333455, -1.2913604197768889, -1.717739109221235}), + torch::tensor({-2.1762368071604308}), + }, + }; +} + +inline std::vector> +SGD_with_weight_decay_and_momentum() { + return { + { + torch::tensor( + {-0.21042867144447805, + -0.49671181653925384, + -0.13917719856207697, + -0.3390489907590303, + -0.2508762913762564, + 0.6985126396619242}), + torch::tensor( + {-0.10754881320494518, -0.2910084928862701, 0.6926954859081793}), + torch::tensor( + {-0.079932454658518, -0.42109796996670307, -0.33469915794198624}), + torch::tensor({-0.14248012693079315}), + }, + { + torch::tensor( + {0.0056118487251954775, + -0.0710915563059199, + 0.07701400891926036, + 0.047067327035013866, + 0.0428654052972598, + 0.4352977220593751}), + torch::tensor( + {0.23834837300214828, 0.32366382503704183, 0.7128321016634689}), + torch::tensor( + {-1.041947788394885, -1.1730950187020548, -1.7648205873351157}), + torch::tensor({-2.3359277661920594}), + }, + { + torch::tensor( + {0.11520007183759418, + 0.1289453768763286, + 0.14586845555951963, + 0.1775341535876219, + 0.15614155642578995, + 0.33379126147460536}), + torch::tensor( + {0.465853656413685, 0.520197917876909, 0.7274876508280723}), + torch::tensor( + {-1.2034746444882527, -1.286126969233868, -1.604528340632377}), + torch::tensor({-2.203215909196624}), + }, + { + torch::tensor( + {0.15331258730374997, + 0.197909036233604, + 0.16663814647374195, + 0.2183320498727895, + 0.1803274550482287, + 0.28362745794417826}), + torch::tensor( + {0.5532312776994917, 0.5834224152126115, 0.6903579410976886}), + torch::tensor( + {-1.3052171323471546, -1.3514190497186434, -1.5153574535010634}), + torch::tensor({-2.123181139806548}), + }, + { + torch::tensor( + {0.16814113185552507, + 0.22386572201448868, + 0.17413795101952864, + 0.23280515326261633, + 0.1839142207976228, + 0.2614499495870909}), + torch::tensor( + {0.592282876576759, 0.6083877519652824, 0.663438748699906}), + torch::tensor( + {-1.3591143274292896, -1.383673065830997, -1.467157893517277}), + torch::tensor({-2.087859547998447}), + }, + { + torch::tensor( + {0.1743742243877178, + 0.2343126153059798, + 0.17716942927642254, + 0.23838669643330088, + 0.18308461132092924, + 0.25149544624452974}), + torch::tensor( + {0.6108281747800746, 0.6192657661217672, 0.6475519545045926}), + torch::tensor( + {-1.3860527054444407, -1.398816664238087, -1.4412527948055516}), + torch::tensor({-2.0731939075659627}), + }, + { + torch::tensor( + {0.1771465478751462, + 0.23875859951719522, + 0.1784868271584857, + 0.2406786372566496, + 0.18181103291606765, + 0.24687877342069478}), + torch::tensor( + {0.6198586021174767, 0.6242349464856269, 0.638736845373371}), + torch::tensor( + {-1.3993307716862977, -1.4058965193851591, -1.42747775986796}), + torch::tensor({-2.0672675843404598}), + }, + { + torch::tensor( + {0.17843093585357683, + 0.24073954802700465, + 0.17908697027440873, + 0.2416675839909268, + 0.18088350526559058, + 0.24467193314356378}), + torch::tensor( + {0.6243071074374693, 0.6265628975677455, 0.6339840865876518}), + torch::tensor( + {-1.4058750036106915, -1.4092362337714568, -1.4202202926903085}), + torch::tensor({-2.0649062340635584}), + }, + { + torch::tensor( + {0.17904350645021613, + 0.24165496946247034, + 0.17936920658487726, + 0.24211164489776849, + 0.18031858582735988, + 0.2435923992630521}), + torch::tensor( + {0.626513445507806, 0.6276715667697311, 0.6314641991686346}), + torch::tensor( + {-1.409113940967948, -1.410830795235453, -1.4164247285253404}), + torch::tensor({-2.0639728292802046}), + }, + { + torch::tensor( + {0.17934167113683835, + 0.242089962404631, + 0.17950490408309286, + 0.24231745350706005, + 0.17999989292556767, + 0.2430557755257577}), + torch::tensor( + {0.6276131793232345, 0.6282062328090801, 0.6301427155170752}), + torch::tensor( + {-1.4107251789010826, -1.4116011824171857, -1.4144511767962422}), + torch::tensor({-2.0636056316673934}), + }, + { + torch::tensor( + {0.17948886155124505, + 0.24230096332204806, + 0.17957117450689372, + 0.242415213133214, + 0.17982712042628357, + 0.2427862039224869}), + torch::tensor( + {0.6281635672171683, 0.6284667582211864, 0.6294549191500093}), + torch::tensor( + {-1.4115305541843781, -1.4119772978756444, -1.4134296522818641}), + torch::tensor({-2.0634616066978615}), + }, + }; +} + +inline std::vector> +SGD_with_weight_decay_and_nesterov_momentum() { + return { + { + torch::tensor( + {-0.21040617235121148, + -0.49689727139951717, + -0.13754215970803657, + -0.33701686525263036, + -0.2500172388792182, + 0.700697918175925}), + torch::tensor( + {-0.1068708360895515, -0.2853285323043249, 0.6971494161502307}), + torch::tensor( + {-0.10624536304143092, -0.4461132561477894, -0.3805647497874434}), + torch::tensor({-0.2068230782168696}), + }, + { + torch::tensor( + {-0.1262387113548655, + -0.3844658218758334, + 0.03124406856508884, + -0.11170532152425781, + -0.09823268522398332, + 0.9040698525178972}), + torch::tensor( + {0.17551336074135096, 0.27976614792027166, 1.2138399680985128}), + torch::tensor( + {-1.592840413595591, -1.8986806244521564, -2.966181914454827}), + torch::tensor({-3.7728444542017687}), + }, + { + torch::tensor( + {-0.11614716303292183, + -0.3709539909720773, + 0.04307078045512772, + -0.09588329367245825, + -0.08795603365024904, + 0.9178771227283019}), + torch::tensor( + {0.20944042006388683, 0.3195483889401668, 1.2500270348310718}), + torch::tensor( + {-1.635011052494502, -1.9463243375558272, -3.035708036973984}), + torch::tensor({-3.8570351018212796}), + }, + { + torch::tensor( + {-0.10793942832760066, + -0.35995697973682966, + 0.05260329955808716, + -0.08312010825923577, + -0.07986326997915319, + 0.9287409473303162}), + torch::tensor( + {0.2370574459090396, 0.35168415020524857, 1.278618438127574}), + torch::tensor( + {-1.669141810658011, -1.984894370767313, -3.091259532917102}), + torch::tensor({-3.923827025320545}), + }, + { + torch::tensor( + {-0.1010142826857921, + -0.35067247612415425, + 0.06058642765135953, + -0.07242353828264116, + -0.07320722520220559, + 0.9376663294528951}), + torch::tensor( + {0.26037531373638517, 0.37864768429039036, 1.3021925174954938}), + torch::tensor( + {-1.6978623668013235, -2.017346013780729, -3.137511248751908}), + torch::tensor({-3.9791368472670334}), + }, + { + torch::tensor( + {-0.0950223925827384, + -0.34263425874631004, + 0.06744912149060932, + -0.0632219668955612, + -0.06756850374320933, + 0.9452179348012486}), + torch::tensor( + {0.2805629021730173, 0.40186559210837897, 1.322201974233735}), + torch::tensor( + {-1.7226667375672964, -2.0453651314263936, -3.177094625235675}), + torch::tensor({-4.02626353351958}), + }, + { + torch::tensor( + {-0.08974074058929343, + -0.3355446553621404, + 0.07346375443579244, + -0.055152336279101065, + -0.06268648871001672, + 0.9517469338705254}), + torch::tensor( + {0.2983667593362755, 0.42224471824689497, 1.3395523443811077}), + torch::tensor( + {-1.7445022249557358, -2.070022023204061, -3.2116640112699977}), + torch::tensor({-4.0672678681014025}), + }, + { + torch::tensor( + {-0.08501805567029425, + -0.32920173535901165, + 0.07881418855733362, + -0.04796950604202488, + -0.058388411064112675, + 0.9574862878804136}), + torch::tensor( + {0.314293480998118, 0.44039784591234016, 1.3548455497581404}), + torch::tensor( + {-1.7640079833055848, -2.092039516337239, -3.2423272727017984}), + torch::tensor({-4.1035226231344275}), + }, + { + torch::tensor( + {-0.08074691916762683, + -0.32346209125355385, + 0.08363041954091821, + -0.04150011326440421, + -0.05455400195824525, + 0.9625982669377223}), + torch::tensor( + {0.3287029554083447, 0.456758647445438, 1.3685016029692183}), + torch::tensor( + {-1.781635969965323, -2.1119291060463254, -3.2698625668054278}), + torch::tensor({-4.135987715622241}), + }, + { + torch::tensor( + {-0.07684825926741544, + -0.3182201487496606, + 0.08800775942949753, + -0.03561702050647377, + -0.05109626940874274, + 0.967200308063431}), + torch::tensor( + {0.3418602073865105, 0.47164537681262036, 1.3808249543962092}), + torch::tensor( + {-1.7977177360253929, -2.1300662313234127, -3.2948372910743537}), + torch::tensor({-4.165360368453936}), + }, + { + torch::tensor( + {-0.07326215742204903, + -0.31339589848358795, + 0.09201816976416921, + -0.030224217178854797, + -0.0479503174605994, + 0.9713800632469923}), + torch::tensor( + {0.35396614509118257, 0.485298524494989, 1.3920431643924076}), + torch::tensor( + {-1.8125038126190638, -2.146734711618823, -3.3176778240157505}), + torch::tensor({-4.192162739857097}), + }, + }; +} + +} // namespace expected_parameters diff --git a/western_ai/usa/pytorch/test/cpp/api/optim_baseline.py b/western_ai/usa/pytorch/test/cpp/api/optim_baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..de3ca862a42b30dcb2a81569fed945e6b18ca26c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/optim_baseline.py @@ -0,0 +1,143 @@ +"""Script to generate baseline values from PyTorch optimization algorithms""" + +import argparse +import math +import sys + +import torch +import torch.optim + + +HEADER = """ +#include + +#include + +namespace expected_parameters { +""" + +FOOTER = "} // namespace expected_parameters" + +PARAMETERS = "inline std::vector> {}() {{" + +OPTIMIZERS = { + "LBFGS": lambda p: torch.optim.LBFGS(p, 1.0), + "LBFGS_with_line_search": lambda p: torch.optim.LBFGS( + p, 1.0, line_search_fn="strong_wolfe" + ), + "Adam": lambda p: torch.optim.Adam(p, 1.0), + "Adam_with_weight_decay": lambda p: torch.optim.Adam(p, 1.0, weight_decay=1e-2), + "Adam_with_weight_decay_and_amsgrad": lambda p: torch.optim.Adam( + p, 1.0, weight_decay=1e-6, amsgrad=True + ), + "AdamW": lambda p: torch.optim.AdamW(p, 1.0), + "AdamW_without_weight_decay": lambda p: torch.optim.AdamW(p, 1.0, weight_decay=0), + "AdamW_with_amsgrad": lambda p: torch.optim.AdamW(p, 1.0, amsgrad=True), + "Adagrad": lambda p: torch.optim.Adagrad(p, 1.0), + "Adagrad_with_weight_decay": lambda p: torch.optim.Adagrad( + p, 1.0, weight_decay=1e-2 + ), + "Adagrad_with_weight_decay_and_lr_decay": lambda p: torch.optim.Adagrad( + p, 1.0, weight_decay=1e-6, lr_decay=1e-3 + ), + "RMSprop": lambda p: torch.optim.RMSprop(p, 0.1), + "RMSprop_with_weight_decay": lambda p: torch.optim.RMSprop( + p, 0.1, weight_decay=1e-2 + ), + "RMSprop_with_weight_decay_and_centered": lambda p: torch.optim.RMSprop( + p, 0.1, weight_decay=1e-6, centered=True + ), + "RMSprop_with_weight_decay_and_centered_and_momentum": lambda p: torch.optim.RMSprop( + p, 0.1, weight_decay=1e-6, centered=True, momentum=0.9 + ), + "SGD": lambda p: torch.optim.SGD(p, 0.1), + "SGD_with_weight_decay": lambda p: torch.optim.SGD(p, 0.1, weight_decay=1e-2), + "SGD_with_weight_decay_and_momentum": lambda p: torch.optim.SGD( + p, 0.1, momentum=0.9, weight_decay=1e-2 + ), + "SGD_with_weight_decay_and_nesterov_momentum": lambda p: torch.optim.SGD( + p, 0.1, momentum=0.9, weight_decay=1e-6, nesterov=True + ), +} + + +def weight_init(module): + if isinstance(module, torch.nn.Linear): + stdev = 1.0 / math.sqrt(module.weight.size(1)) + for p in module.parameters(): + p.data.uniform_(-stdev, stdev) + + +def run(optimizer_name, iterations, sample_every): + torch.manual_seed(0) + model = torch.nn.Sequential( + torch.nn.Linear(2, 3), + torch.nn.Sigmoid(), + torch.nn.Linear(3, 1), + torch.nn.Sigmoid(), + ) + model = model.to(torch.float64).apply(weight_init) + + optimizer = OPTIMIZERS[optimizer_name](model.parameters()) + + input = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]], dtype=torch.float64) + + values = [] + for i in range(iterations): + optimizer.zero_grad() + + output = model.forward(input) + loss = output.sum() + loss.backward() + + def closure(): + return torch.tensor([10.0]) + + optimizer.step(closure) + + if i % sample_every == 0: + values.append( + [p.clone().flatten().data.numpy() for p in model.parameters()] + ) + + return values + + +def emit(optimizer_parameter_map): + # Don't write generated with an @ in front, else this file is recognized as generated. + print("// @{} from {}".format("generated", __file__)) + print(HEADER) + for optimizer_name, parameters in optimizer_parameter_map.items(): + print(PARAMETERS.format(optimizer_name)) + print(" return {") + for sample in parameters: + print(" {") + for parameter in sample: + parameter_values = "{{{}}}".format(", ".join(map(str, parameter))) + print(f" torch::tensor({parameter_values}),") + print(" },") + print(" };") + print("}\n") + print(FOOTER) + + +def main(): + parser = argparse.ArgumentParser( + "Produce optimization output baseline from PyTorch" + ) + parser.add_argument("-i", "--iterations", default=1001, type=int) + parser.add_argument("-s", "--sample-every", default=100, type=int) + options = parser.parse_args() + + optimizer_parameter_map = {} + for optimizer in OPTIMIZERS: + sys.stderr.write(f"Evaluating {optimizer} ...\n") + optimizer_parameter_map[optimizer] = run( + optimizer, options.iterations, options.sample_every + ) + + emit(optimizer_parameter_map) + + +if __name__ == "__main__": + main() diff --git a/western_ai/usa/pytorch/test/cpp/api/ordered_dict.cpp b/western_ai/usa/pytorch/test/cpp/api/ordered_dict.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8f4511862816dd43829767f8956bbb92852dfe6c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/ordered_dict.cpp @@ -0,0 +1,234 @@ +#include + +#include +#include + +template +using OrderedDict = torch::OrderedDict; + +TEST(OrderedDictTest, IsEmptyAfterDefaultConstruction) { + OrderedDict dict; + ASSERT_EQ(dict.key_description(), "Key"); + ASSERT_TRUE(dict.is_empty()); + ASSERT_EQ(dict.size(), 0); +} + +TEST(OrderedDictTest, InsertAddsElementsWhenTheyAreYetNotPresent) { + OrderedDict dict; + dict.insert("a", 1); + dict.insert("b", 2); + ASSERT_EQ(dict.size(), 2); +} + +TEST(OrderedDictTest, GetReturnsValuesWhenTheyArePresent) { + OrderedDict dict; + dict.insert("a", 1); + dict.insert("b", 2); + ASSERT_EQ(dict["a"], 1); + ASSERT_EQ(dict["b"], 2); +} + +TEST(OrderedDictTest, GetThrowsWhenPassedKeysThatAreNotPresent) { + OrderedDict dict; + dict.insert("a", 1); + dict.insert("b", 2); + ASSERT_THROWS_WITH(dict["foo"], "Key 'foo' is not defined"); + ASSERT_THROWS_WITH(dict[""], "Key '' is not defined"); +} + +TEST(OrderedDictTest, CanInitializeFromList) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_EQ(dict.size(), 2); + ASSERT_EQ(dict["a"], 1); + ASSERT_EQ(dict["b"], 2); +} + +TEST(OrderedDictTest, InsertThrowsWhenPassedElementsThatArePresent) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_THROWS_WITH(dict.insert("a", 1), "Key 'a' already defined"); + ASSERT_THROWS_WITH(dict.insert("b", 1), "Key 'b' already defined"); +} + +TEST(OrderedDictTest, FrontReturnsTheFirstItem) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_EQ(dict.front().key(), "a"); + ASSERT_EQ(dict.front().value(), 1); +} + +TEST(OrderedDictTest, FrontThrowsWhenEmpty) { + OrderedDict dict; + ASSERT_THROWS_WITH(dict.front(), "Called front() on an empty OrderedDict"); +} + +TEST(OrderedDictTest, BackReturnsTheLastItem) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_EQ(dict.back().key(), "b"); + ASSERT_EQ(dict.back().value(), 2); +} + +TEST(OrderedDictTest, BackThrowsWhenEmpty) { + OrderedDict dict; + ASSERT_THROWS_WITH(dict.back(), "Called back() on an empty OrderedDict"); +} + +TEST(OrderedDictTest, FindReturnsPointersToValuesWhenPresent) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_NE(dict.find("a"), nullptr); + ASSERT_EQ(*dict.find("a"), 1); + ASSERT_NE(dict.find("b"), nullptr); + ASSERT_EQ(*dict.find("b"), 2); +} + +TEST(OrderedDictTest, FindReturnsNullPointersWhenPasesdKeysThatAreNotPresent) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_EQ(dict.find("bar"), nullptr); + ASSERT_EQ(dict.find(""), nullptr); +} + +TEST(OrderedDictTest, SubscriptOperatorThrowsWhenPassedKeysThatAreNotPresent) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_EQ(dict["a"], 1); + ASSERT_EQ(dict["b"], 2); +} + +TEST( + OrderedDictTest, + SubscriptOperatorReturnsItemsPositionallyWhenPassedIntegers) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_EQ(dict[0].key(), "a"); + ASSERT_EQ(dict[0].value(), 1); + ASSERT_EQ(dict[1].key(), "b"); + ASSERT_EQ(dict[1].value(), 2); +} + +TEST(OrderedDictTest, SubscriptOperatorsThrowswhenPassedKeysThatAreNotPresent) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_THROWS_WITH(dict["foo"], "Key 'foo' is not defined"); + ASSERT_THROWS_WITH(dict[""], "Key '' is not defined"); +} + +TEST(OrderedDictTest, UpdateInsertsAllItemsFromAnotherOrderedDict) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + OrderedDict dict2 = {{"c", 3}}; + dict2.update(dict); + ASSERT_EQ(dict2.size(), 3); + ASSERT_NE(dict2.find("a"), nullptr); + ASSERT_NE(dict2.find("b"), nullptr); + ASSERT_NE(dict2.find("c"), nullptr); +} + +TEST(OrderedDictTest, UpdateAlsoChecksForDuplicates) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + OrderedDict dict2 = {{"a", 1}}; + ASSERT_THROWS_WITH(dict2.update(dict), "Key 'a' already defined"); +} + +TEST(OrderedDictTest, CanIterateItems) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + auto iterator = dict.begin(); + ASSERT_NE(iterator, dict.end()); + ASSERT_EQ(iterator->key(), "a"); + ASSERT_EQ(iterator->value(), 1); + ++iterator; + ASSERT_NE(iterator, dict.end()); + ASSERT_EQ(iterator->key(), "b"); + ASSERT_EQ(iterator->value(), 2); + ++iterator; + ASSERT_EQ(iterator, dict.end()); +} + +TEST(OrderedDictTest, EraseWorks) { + OrderedDict dict = {{"a", 1}, {"b", 2}, {"c", 3}}; + dict.erase("b"); + ASSERT_FALSE(dict.contains("b")); + ASSERT_EQ(dict["a"], 1); + ASSERT_EQ(dict["c"], 3); + dict.erase("a"); + ASSERT_FALSE(dict.contains("a")); + ASSERT_EQ(dict["c"], 3); + dict.erase("c"); + ASSERT_FALSE(dict.contains("c")); + ASSERT_TRUE(dict.is_empty()); +} + +TEST(OrderedDictTest, ClearMakesTheDictEmpty) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_FALSE(dict.is_empty()); + dict.clear(); + ASSERT_TRUE(dict.is_empty()); +} + +TEST(OrderedDictTest, CanCopyConstruct) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + OrderedDict copy = dict; + ASSERT_EQ(copy.size(), 2); + ASSERT_EQ(*copy[0], 1); + ASSERT_EQ(*copy[1], 2); +} + +TEST(OrderedDictTest, CanCopyAssign) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + OrderedDict copy = {{"c", 1}}; + ASSERT_NE(copy.find("c"), nullptr); + copy = dict; + ASSERT_EQ(copy.size(), 2); + ASSERT_EQ(*copy[0], 1); + ASSERT_EQ(*copy[1], 2); + ASSERT_EQ(copy.find("c"), nullptr); +} + +TEST(OrderedDictTest, CanMoveConstruct) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + OrderedDict copy = std::move(dict); + ASSERT_EQ(copy.size(), 2); + ASSERT_EQ(*copy[0], 1); + ASSERT_EQ(*copy[1], 2); +} + +TEST(OrderedDictTest, CanMoveAssign) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + OrderedDict copy = {{"c", 1}}; + ASSERT_NE(copy.find("c"), nullptr); + copy = std::move(dict); + ASSERT_EQ(copy.size(), 2); + ASSERT_EQ(*copy[0], 1); + ASSERT_EQ(*copy[1], 2); + ASSERT_EQ(copy.find("c"), nullptr); +} + +TEST(OrderedDictTest, CanInsertWithBraces) { + OrderedDict> dict; + dict.insert("a", {1, 2}); + ASSERT_FALSE(dict.is_empty()); + ASSERT_EQ(dict["a"].first, 1); + ASSERT_EQ(dict["a"].second, 2); +} + +TEST(OrderedDictTest, ErrorMessagesIncludeTheKeyDescription) { + OrderedDict dict("Penguin"); + ASSERT_EQ(dict.key_description(), "Penguin"); + dict.insert("a", 1); + ASSERT_FALSE(dict.is_empty()); + ASSERT_THROWS_WITH(dict["b"], "Penguin 'b' is not defined"); + ASSERT_THROWS_WITH(dict.insert("a", 1), "Penguin 'a' already defined"); +} + +TEST(OrderedDictTest, KeysReturnsAllKeys) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_EQ(dict.keys(), std::vector({"a", "b"})); +} + +TEST(OrderedDictTest, ValuesReturnsAllValues) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + ASSERT_EQ(dict.values(), std::vector({1, 2})); +} + +TEST(OrderedDictTest, ItemsReturnsAllItems) { + OrderedDict dict = {{"a", 1}, {"b", 2}}; + std::vector::Item> items = dict.items(); + ASSERT_EQ(items.size(), 2); + ASSERT_EQ(items[0].key(), "a"); + ASSERT_EQ(items[0].value(), 1); + ASSERT_EQ(items[1].key(), "b"); + ASSERT_EQ(items[1].value(), 2); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/parallel.cpp b/western_ai/usa/pytorch/test/cpp/api/parallel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..57d795417001ada616230c20061e0e7c6f0c495a --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/parallel.cpp @@ -0,0 +1,294 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +using namespace torch::autograd; +using namespace torch::nn; + +struct ParallelTest : torch::test::SeedingFixture {}; + +TEST_F(ParallelTest, DifferentiableScatter_MultiCUDA) { + Scatter scatter( + {torch::Device(torch::kCUDA, 0), torch::Device(torch::kCUDA, 1)}); + + auto input = torch::ones(10, torch::requires_grad(true)); + auto output = scatter.apply({input}); + + ASSERT_EQ(output.size(), 2); + ASSERT_EQ(output[0].size(0), 5); + ASSERT_EQ(output[1].size(0), 5); + + ASSERT_TRUE(torch::cat({output[0].to(torch::kCPU), output[1].to(torch::kCPU)}) + .allclose(input)); + + torch::Tensor sum = output[0].to({torch::kCUDA, 1}) + output[1]; + sum.backward(torch::ones_like(sum)); + + ASSERT_TRUE(input.grad().defined()); + ASSERT_TRUE(input.grad().device().is_cpu()); + ASSERT_EQ(input.grad().sum().item(), 10); +} + +TEST_F(ParallelTest, DifferentiableGather_MultiCUDA) { + Gather gather(torch::Device(torch::kCUDA, 1)); + + auto a = torch::ones(5, torch::requires_grad(true).device(torch::kCUDA, 0)); + auto b = torch::ones(5, torch::requires_grad(true).device(torch::kCUDA, 1)); + + auto outputs = gather.apply({a, b}); + ASSERT_EQ(outputs.size(), 1); + torch::Tensor output = outputs.front(); + + ASSERT_EQ(output.size(0), 10); + ASSERT_EQ(output.device(), torch::Device(torch::kCUDA, 1)); + + auto chunks = output.chunk(2); + ASSERT_TRUE(chunks[0].to({torch::kCUDA, 0}).allclose(a)); + ASSERT_TRUE(chunks[1].allclose(b)); + + output.backward(torch::ones_like(output)); + + ASSERT_TRUE(a.grad().defined()); + ASSERT_EQ(a.grad().device(), torch::Device(torch::kCUDA, 0)); + ASSERT_EQ(a.grad().sum().item(), 5); + + ASSERT_TRUE(b.grad().defined()); + ASSERT_EQ(b.grad().device(), torch::Device(torch::kCUDA, 1)); + ASSERT_EQ(b.grad().sum().item(), 5); +} + +TEST_F(ParallelTest, Replicate_MultiCUDA) { + Linear linear(3, 4); + auto replicas = parallel::replicate( + linear, {torch::Device(torch::kCUDA, 0), torch::Device(torch::kCUDA, 1)}); + ASSERT_EQ(replicas.size(), 2); + + auto original_parameters = linear->parameters(); + + auto replica1_parameters = replicas[0]->parameters(); + for (auto& parameter : replica1_parameters) { + ASSERT_EQ(parameter.device(), torch::Device(torch::kCUDA, 0)); + } + replicas[0]->to(torch::kCPU); + ASSERT_EQ(replica1_parameters.size(), original_parameters.size()); + for (const auto i : c10::irange(original_parameters.size())) { + ASSERT_TRUE(replica1_parameters[i].allclose(original_parameters[i])); + ASSERT_TRUE( + replica1_parameters[i].data_ptr() != + original_parameters[i].data_ptr()); + } + + auto replica2_parameters = replicas[1]->parameters(); + for (auto& parameter : replica2_parameters) { + ASSERT_EQ(parameter.device(), torch::Device(torch::kCUDA, 1)); + } + replicas[1]->to(torch::kCPU); + ASSERT_EQ(replica2_parameters.size(), original_parameters.size()); + for (const auto i : c10::irange(original_parameters.size())) { + ASSERT_TRUE(replica2_parameters[i].allclose(original_parameters[i])); + ASSERT_TRUE( + replica2_parameters[i].data_ptr() != + original_parameters[i].data_ptr()); + } +} + +TEST_F(ParallelTest, ParallelApply_MultiCUDA) { + Linear a(3, 4); + + Linear b(std::dynamic_pointer_cast(a->clone())); + b->to({torch::kCUDA, 0}); + + Linear c(std::dynamic_pointer_cast(a->clone())); + c->to({torch::kCUDA, 1}); + + std::vector modules = {a, b, c}; + std::vector inputs = { + torch::ones({2, 3}), + torch::ones({2, 3}, torch::device({torch::kCUDA, 0})), + torch::ones({2, 3}, torch::device({torch::kCUDA, 1}))}; + + auto outputs = parallel::parallel_apply(modules, inputs); + + ASSERT_EQ(outputs.size(), 3); + ASSERT_TRUE(outputs[0].device().is_cpu()); + + ASSERT_EQ(outputs[1].device(), torch::Device(torch::kCUDA, 0)); + ASSERT_TRUE(outputs[1].to(torch::kCPU).allclose(outputs[0])); + + ASSERT_EQ(outputs[2].device(), torch::Device(torch::kCUDA, 1)); + ASSERT_TRUE(outputs[2].to(torch::kCPU).allclose(outputs[0])); +} + +TEST_F(ParallelTest, ParallelApplyWithDifferentOutputDevice_MultiCUDA) { + struct M : torch::nn::Module { + torch::Tensor forward(torch::Tensor input) { + return torch::ones(5, torch::kInt32); + } + }; + + std::vector> modules = { + std::make_shared(), std::make_shared(), std::make_shared()}; + std::vector inputs = { + torch::empty({}), torch::empty({}), torch::empty({})}; + std::vector devices = { + {torch::kCUDA, 1}, {torch::kCUDA, 0}, {torch::kCPU}}; + + auto outputs = parallel::parallel_apply(modules, inputs, devices); + + ASSERT_EQ(outputs.size(), 3); + ASSERT_TRUE(outputs[0].device().is_cuda()); + ASSERT_EQ(outputs[0].device(), torch::Device(torch::kCUDA, 1)); + + ASSERT_TRUE(outputs[1].device().is_cuda()); + ASSERT_EQ(outputs[1].device(), torch::Device(torch::kCUDA, 0)); + + ASSERT_TRUE(outputs[2].device().is_cpu()); +} + +TEST_F(ParallelTest, ParallelApplyRethrowsException_MultiCUDA) { + struct M : torch::nn::Cloneable { + void reset() override {} + torch::Tensor forward(torch::Tensor input) { + throw std::runtime_error("Badness!"); + } + }; + + auto m = std::make_shared(); + auto input = torch::ones({10, 3}); + ASSERT_THROWS_WITH(parallel::data_parallel(m, input), "Badness!"); +} + +TEST_F( + ParallelTest, + DataParallelPlacesTheOutputOnTheRequestedDevice_MultiCUDA) { + struct M : torch::nn::Cloneable { + void reset() override {} + torch::Tensor forward(torch::Tensor input) { + // The returned tensor should be on the output device. + return torch::ones(3); + } + }; + auto m = std::make_shared(); + auto input = torch::ones({10, 3}); + { + auto output = parallel::data_parallel( + m, + input, + /*devices=*/std::nullopt, + /*output_device=*/torch::Device(torch::kCUDA, 1)); + ASSERT_TRUE(output.defined()); + ASSERT_TRUE(output.device().is_cuda()); + ASSERT_EQ(output.device().index(), 1); + } + { + // Verify for the single-device case (where we don't scatter/gather). + auto output = parallel::data_parallel( + m, + input, + /*devices=*/std::vector{torch::Device(torch::kCUDA, 0)}, + /*output_device=*/torch::Device(torch::kCUDA, 1)); + ASSERT_TRUE(output.defined()); + ASSERT_TRUE(output.device().is_cuda()); + ASSERT_EQ(output.device().index(), 1); + } +} + +TEST_F(ParallelTest, DataParallelUsesAllAvailableCUDADevices_CUDA) { + struct M : torch::nn::Cloneable { + void reset() override {} + torch::Tensor forward(torch::Tensor input) { + return torch::tensor({input.device().index()}); + } + }; + + auto m = std::make_shared(); + const auto device_count = torch::cuda::device_count(); + auto input = torch::ones({std::max(10, int(2 * device_count)), 3}); + auto output = parallel::data_parallel(m, input); + + ASSERT_EQ(output.numel(), device_count); + for (const auto i : c10::irange(device_count)) { + ASSERT_EQ(output[i].item(), i); + } +} + +TEST_F(ParallelTest, DataParallelNumericalEquivalence_MultiCUDA) { + struct M : torch::nn::Cloneable { + M() { + reset(); + } + + void reset() override { + conv = register_module( + "conv", + torch::nn::Conv2d(torch::nn::Conv2dOptions(2, 2, /*kernel_size=*/2))); + fc = register_module("fc", torch::nn::Linear(8, 2)); + } + + torch::Tensor forward(torch::Tensor x) { + x = conv->forward(x); + x = torch::relu(x); + x = x.view({-1, 8}); + x = fc->forward(x); + return torch::log_softmax(x, /*dim=*/1); + } + + torch::nn::Conv2d conv{nullptr}; + torch::nn::Linear fc{nullptr}; + }; + + // prepare modules and inputs + auto input = torch::ones({16, 2, 3, 3}); + auto input_dp = torch::ones({16, 2, 3, 3}); + auto model = std::make_shared(); + auto model_dp = std::dynamic_pointer_cast(model->clone()); + + // run 3 training iterations + for (const auto i : c10::irange(3)) { + input += i; + input_dp += i; + + // non-parallel training + torch::optim::SGD optim(model->parameters(), torch::optim::SGDOptions(0.1)); + auto output = model->forward(input); + auto loss = torch::mse_loss(output, torch::zeros_like(output)); + loss.backward(); + optim.step(); + + // data-parallel training + torch::optim::SGD optim_dp( + model_dp->parameters(), torch::optim::SGDOptions(0.1)); + auto output_dp = parallel::data_parallel(model_dp, input_dp); + auto loss_dp = torch::mse_loss(output_dp, torch::zeros_like(output_dp)); + loss_dp.backward(); + optim_dp.step(); + + // make sure that weights are the same + model->to(torch::kCPU); + model_dp->to(torch::kCPU); + auto params = model->parameters(); + auto params_dp = model_dp->parameters(); + ASSERT_EQ(params.size(), params_dp.size()); + for (auto it = params.begin(), it_dp = params_dp.begin(); + it != params.end() && it_dp != params.end(); + ++it, ++it_dp) { + ASSERT_TRUE(torch::allclose(*it, *it_dp)); + } + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/parallel_benchmark.cpp b/western_ai/usa/pytorch/test/cpp/api/parallel_benchmark.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2a7587fb3d2f3b4ccd7cd22cbb3c0c4e3dbf8411 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/parallel_benchmark.cpp @@ -0,0 +1,88 @@ +#include +#include +#include +#include + +class Baton { + public: + void post() { + std::unique_lock l(lock_); + done_ = true; + cv_.notify_all(); + } + void wait() { + std::unique_lock l(lock_); + while (!done_) { + cv_.wait(l); + } + } + + private: + std::mutex lock_; + std::condition_variable cv_; + bool done_{false}; +}; + +void AtLaunch_Base(int32_t numIters) { + struct Helper { + explicit Helper(int32_t lim) : limit_(lim) {} + void operator()() { + if (++val_ == limit_) { + done.post(); + } else { + at::launch([this]() { (*this)(); }); + } + } + int val_{0}; + int limit_; + Baton done; + }; + Helper h(numIters); + auto start = std::chrono::system_clock::now(); + h(); + h.done.wait(); + std::cout << "NoData " + << static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now() - start) + .count()) / + static_cast(numIters) + << " usec/each\n"; +} + +void AtLaunch_WithData(int32_t numIters, int32_t vecSize) { + struct Helper { + explicit Helper(int32_t lim) : limit_(lim) {} + void operator()(std::vector v) { + if (++val_ == limit_) { + done.post(); + } else { + at::launch([this, v = std::move(v)]() { (*this)(v); }); + } + } + int val_{0}; + int limit_; + Baton done; + }; + Helper h(numIters); + std::vector v(vecSize, 0); + auto start = std::chrono::system_clock::now(); + h(v); + h.done.wait(); + std::cout << "WithData(" << vecSize << "): " + << static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now() - start) + .count()) / + static_cast(numIters) + << " usec/each\n"; +} + +int main(int argc, char** argv) { + int32_t N = 1000000; + AtLaunch_Base(N); + AtLaunch_WithData(N, 0); + AtLaunch_WithData(N, 4); + AtLaunch_WithData(N, 256); + return 0; +} diff --git a/western_ai/usa/pytorch/test/cpp/api/parameterdict.cpp b/western_ai/usa/pytorch/test/cpp/api/parameterdict.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0d0fd0a5e8e31160713b2fdd00f1ff0c8520a742 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/parameterdict.cpp @@ -0,0 +1,144 @@ +#include +#include +#include +#include +#include + +#include + +using namespace torch::nn; +using namespace torch::test; + +struct ParameterDictTest : torch::test::SeedingFixture {}; + +TEST_F(ParameterDictTest, ConstructFromTensor) { + ParameterDict dict; + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + torch::Tensor tb = torch::randn({1, 2}, torch::requires_grad(false)); + torch::Tensor tc = torch::randn({1, 2}); + ASSERT_TRUE(ta.requires_grad()); + ASSERT_FALSE(tb.requires_grad()); + dict->insert("A", ta); + dict->insert("B", tb); + dict->insert("C", tc); + ASSERT_EQ(dict->size(), 3); + ASSERT_TRUE(torch::all(torch::eq(dict["A"], ta)).item()); + ASSERT_TRUE(dict["A"].requires_grad()); + ASSERT_TRUE(torch::all(torch::eq(dict["B"], tb)).item()); + ASSERT_FALSE(dict["B"].requires_grad()); +} + +TEST_F(ParameterDictTest, ConstructFromOrderedDict) { + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + torch::Tensor tb = torch::randn({1, 2}, torch::requires_grad(false)); + torch::Tensor tc = torch::randn({1, 2}); + torch::OrderedDict params = { + {"A", ta}, {"B", tb}, {"C", tc}}; + auto dict = torch::nn::ParameterDict(params); + ASSERT_EQ(dict->size(), 3); + ASSERT_TRUE(torch::all(torch::eq(dict["A"], ta)).item()); + ASSERT_TRUE(dict["A"].requires_grad()); + ASSERT_TRUE(torch::all(torch::eq(dict["B"], tb)).item()); + ASSERT_FALSE(dict["B"].requires_grad()); +} + +TEST_F(ParameterDictTest, InsertAndContains) { + ParameterDict dict; + dict->insert("A", torch::tensor({1.0})); + ASSERT_EQ(dict->size(), 1); + ASSERT_TRUE(dict->contains("A")); + ASSERT_FALSE(dict->contains("C")); +} + +TEST_F(ParameterDictTest, InsertAndClear) { + ParameterDict dict; + dict->insert("A", torch::tensor({1.0})); + ASSERT_EQ(dict->size(), 1); + dict->clear(); + ASSERT_EQ(dict->size(), 0); +} + +TEST_F(ParameterDictTest, InsertAndPop) { + ParameterDict dict; + dict->insert("A", torch::tensor({1.0})); + ASSERT_EQ(dict->size(), 1); + ASSERT_THROWS_WITH(dict->pop("B"), "Parameter 'B' is not defined"); + torch::Tensor p = dict->pop("A"); + ASSERT_EQ(dict->size(), 0); + ASSERT_TRUE(torch::eq(p, torch::tensor({1.0})).item()); +} + +TEST_F(ParameterDictTest, SimpleUpdate) { + ParameterDict dict; + ParameterDict wrongDict; + ParameterDict rightDict; + dict->insert("A", torch::tensor({1.0})); + dict->insert("B", torch::tensor({2.0})); + dict->insert("C", torch::tensor({3.0})); + wrongDict->insert("A", torch::tensor({5.0})); + wrongDict->insert("D", torch::tensor({5.0})); + ASSERT_THROWS_WITH(dict->update(*wrongDict), "Parameter 'D' is not defined"); + rightDict->insert("A", torch::tensor({5.0})); + dict->update(*rightDict); + ASSERT_EQ(dict->size(), 3); + ASSERT_TRUE(torch::eq(dict["A"], torch::tensor({5.0})).item()); +} + +TEST_F(ParameterDictTest, Keys) { + torch::OrderedDict params = { + {"a", torch::tensor({1.0})}, + {"b", torch::tensor({2.0})}, + {"c", torch::tensor({1.0, 2.0})}}; + auto dict = torch::nn::ParameterDict(params); + std::vector keys = dict->keys(); + std::vector true_keys{"a", "b", "c"}; + ASSERT_EQ(keys, true_keys); +} + +TEST_F(ParameterDictTest, Values) { + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + torch::Tensor tb = torch::randn({1, 2}, torch::requires_grad(false)); + torch::Tensor tc = torch::randn({1, 2}); + torch::OrderedDict params = { + {"a", ta}, {"b", tb}, {"c", tc}}; + auto dict = torch::nn::ParameterDict(params); + std::vector values = dict->values(); + std::vector true_values{ta, tb, tc}; + for (auto i = 0U; i < values.size(); i += 1) { + ASSERT_TRUE(torch::all(torch::eq(values[i], true_values[i])).item()); + } +} + +TEST_F(ParameterDictTest, Get) { + ParameterDict dict; + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + torch::Tensor tb = torch::randn({1, 2}, torch::requires_grad(false)); + torch::Tensor tc = torch::randn({1, 2}); + ASSERT_TRUE(ta.requires_grad()); + ASSERT_FALSE(tb.requires_grad()); + dict->insert("A", ta); + dict->insert("B", tb); + dict->insert("C", tc); + ASSERT_EQ(dict->size(), 3); + ASSERT_TRUE(torch::all(torch::eq(dict->get("A"), ta)).item()); + ASSERT_TRUE(dict->get("A").requires_grad()); + ASSERT_TRUE(torch::all(torch::eq(dict->get("B"), tb)).item()); + ASSERT_FALSE(dict->get("B").requires_grad()); +} + +TEST_F(ParameterDictTest, PrettyPrintParameterDict) { + torch::OrderedDict params = { + {"a", torch::tensor({1.0})}, + {"b", torch::tensor({2.0, 1.0})}, + {"c", torch::tensor({{3.0}, {2.1}})}, + {"d", torch::tensor({{3.0, 1.3}, {1.2, 2.1}})}}; + auto dict = torch::nn::ParameterDict(params); + ASSERT_EQ( + c10::str(dict), + "torch::nn::ParameterDict(\n" + "(a): Parameter containing: [Float of size [1]]\n" + "(b): Parameter containing: [Float of size [2]]\n" + "(c): Parameter containing: [Float of size [2, 1]]\n" + "(d): Parameter containing: [Float of size [2, 2]]\n" + ")"); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/parameterlist.cpp b/western_ai/usa/pytorch/test/cpp/api/parameterlist.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d472bf8c3da59975891380d3205c1449b3ac84ea --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/parameterlist.cpp @@ -0,0 +1,163 @@ +#include + +#include +#include + +#include +#include +#include + +#include + +using namespace torch::nn; +using namespace torch::test; + +struct ParameterListTest : torch::test::SeedingFixture {}; + +TEST_F(ParameterListTest, ConstructsFromSharedPointer) { + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + torch::Tensor tb = torch::randn({1, 2}, torch::requires_grad(false)); + torch::Tensor tc = torch::randn({1, 2}); + ASSERT_TRUE(ta.requires_grad()); + ASSERT_FALSE(tb.requires_grad()); + ParameterList list(ta, tb, tc); + ASSERT_EQ(list->size(), 3); +} + +TEST_F(ParameterListTest, isEmpty) { + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + ParameterList list; + ASSERT_TRUE(list->is_empty()); + list->append(ta); + ASSERT_FALSE(list->is_empty()); + ASSERT_EQ(list->size(), 1); +} + +TEST_F(ParameterListTest, PushBackAddsAnElement) { + ParameterList list; + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + torch::Tensor tb = torch::randn({1, 2}, torch::requires_grad(false)); + torch::Tensor tc = torch::randn({1, 2}); + torch::Tensor td = torch::randn({1, 2, 3}); + ASSERT_EQ(list->size(), 0); + ASSERT_TRUE(list->is_empty()); + list->append(ta); + ASSERT_EQ(list->size(), 1); + list->append(tb); + ASSERT_EQ(list->size(), 2); + list->append(tc); + ASSERT_EQ(list->size(), 3); + list->append(td); + ASSERT_EQ(list->size(), 4); +} +TEST_F(ParameterListTest, ForEachLoop) { + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + torch::Tensor tb = torch::randn({1, 2}, torch::requires_grad(false)); + torch::Tensor tc = torch::randn({1, 2}); + torch::Tensor td = torch::randn({1, 2, 3}); + ParameterList list(ta, tb, tc, td); + std::vector params = {ta, tb, tc, td}; + ASSERT_EQ(list->size(), 4); + int idx = 0; + for (const auto& pair : *list) { + ASSERT_TRUE( + torch::all(torch::eq(pair.value(), params[idx++])).item()); + } +} + +TEST_F(ParameterListTest, AccessWithAt) { + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + torch::Tensor tb = torch::randn({1, 2}, torch::requires_grad(false)); + torch::Tensor tc = torch::randn({1, 2}); + torch::Tensor td = torch::randn({1, 2, 3}); + std::vector params = {ta, tb, tc, td}; + + ParameterList list; + for (auto& param : params) { + list->append(param); + } + ASSERT_EQ(list->size(), 4); + + // returns the correct module for a given index + for (const auto i : c10::irange(params.size())) { + ASSERT_TRUE(torch::all(torch::eq(list->at(i), params[i])).item()); + } + + for (const auto i : c10::irange(params.size())) { + ASSERT_TRUE(torch::all(torch::eq(list[i], params[i])).item()); + } + + // throws for a bad index + ASSERT_THROWS_WITH(list->at(params.size() + 100), "Index out of range"); + ASSERT_THROWS_WITH(list->at(params.size() + 1), "Index out of range"); + ASSERT_THROWS_WITH(list[params.size() + 1], "Index out of range"); +} + +TEST_F(ParameterListTest, ExtendPushesParametersFromOtherParameterList) { + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + torch::Tensor tb = torch::randn({1, 2}, torch::requires_grad(false)); + torch::Tensor tc = torch::randn({1, 2}); + torch::Tensor td = torch::randn({1, 2, 3}); + torch::Tensor te = torch::randn({1, 2}); + torch::Tensor tf = torch::randn({1, 2, 3}); + ParameterList a(ta, tb); + ParameterList b(tc, td); + a->extend(*b); + + ASSERT_EQ(a->size(), 4); + ASSERT_TRUE(torch::all(torch::eq(a[0], ta)).item()); + ASSERT_TRUE(torch::all(torch::eq(a[1], tb)).item()); + ASSERT_TRUE(torch::all(torch::eq(a[2], tc)).item()); + ASSERT_TRUE(torch::all(torch::eq(a[3], td)).item()); + + ASSERT_EQ(b->size(), 2); + ASSERT_TRUE(torch::all(torch::eq(b[0], tc)).item()); + ASSERT_TRUE(torch::all(torch::eq(b[1], td)).item()); + + std::vector c = {te, tf}; + b->extend(c); + + ASSERT_EQ(b->size(), 4); + ASSERT_TRUE(torch::all(torch::eq(b[0], tc)).item()); + ASSERT_TRUE(torch::all(torch::eq(b[1], td)).item()); + ASSERT_TRUE(torch::all(torch::eq(b[2], te)).item()); + ASSERT_TRUE(torch::all(torch::eq(b[3], tf)).item()); +} + +TEST_F(ParameterListTest, PrettyPrintParameterList) { + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + torch::Tensor tb = torch::randn({1, 2}, torch::requires_grad(false)); + torch::Tensor tc = torch::randn({1, 2}); + ParameterList list(ta, tb, tc); + ASSERT_EQ( + c10::str(list), + "torch::nn::ParameterList(\n" + "(0): Parameter containing: [Float of size [1, 2]]\n" + "(1): Parameter containing: [Float of size [1, 2]]\n" + "(2): Parameter containing: [Float of size [1, 2]]\n" + ")"); +} + +TEST_F(ParameterListTest, IncrementAdd) { + torch::Tensor ta = torch::randn({1, 2}, torch::requires_grad(true)); + torch::Tensor tb = torch::randn({1, 2}, torch::requires_grad(false)); + torch::Tensor tc = torch::randn({1, 2}); + torch::Tensor td = torch::randn({1, 2, 3}); + torch::Tensor te = torch::randn({1, 2}); + torch::Tensor tf = torch::randn({1, 2, 3}); + ParameterList listA(ta, tb, tc); + ParameterList listB(td, te, tf); + std::vector tensors{ta, tb, tc, td, te, tf}; + int idx = 0; + *listA += *listB; + ASSERT_TRUE(torch::all(torch::eq(listA[0], ta)).item()); + ASSERT_TRUE(torch::all(torch::eq(listA[1], tb)).item()); + ASSERT_TRUE(torch::all(torch::eq(listA[2], tc)).item()); + ASSERT_TRUE(torch::all(torch::eq(listA[3], td)).item()); + ASSERT_TRUE(torch::all(torch::eq(listA[4], te)).item()); + ASSERT_TRUE(torch::all(torch::eq(listA[5], tf)).item()); + for (const auto& P : listA->named_parameters(false)) + ASSERT_TRUE(torch::all(torch::eq(P.value(), tensors[idx++])).item()); + + ASSERT_EQ(idx, 6); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/printing.cpp b/western_ai/usa/pytorch/test/cpp/api/printing.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ff1e3c1137e16f1eee476fa7679dd0ded16ef1af --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/printing.cpp @@ -0,0 +1,28 @@ +#include +#include +#include +#include + +TEST(PrintSciModeTest, ToggleScientificNotation) { + auto t = torch::tensor({0.00001, 100000.0}); + + // Test with scientific notation enabled + torch::set_printoption_sci_mode(true); + std::ostringstream oss1; + oss1 << t; + auto out1 = oss1.str(); + std::cout << "With sci_mode=true: '" << out1 << '\'' << std::endl; + EXPECT_TRUE( + out1.find("e-") != std::string::npos || + out1.find("e+") != std::string::npos); + + // Test with scientific notation disabled + torch::set_printoption_sci_mode(false); + std::ostringstream oss2; + oss2 << t; + auto out2 = oss2.str(); + std::cout << "With sci_mode=false: '" << out2 << '\'' << std::endl; + EXPECT_TRUE( + out2.find("e-") == std::string::npos && + out2.find("e+") == std::string::npos); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/rnn.cpp b/western_ai/usa/pytorch/test/cpp/api/rnn.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c072df42851598d95013e20ff01bf828c561cdbd --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/rnn.cpp @@ -0,0 +1,845 @@ +#include + +#include + +#include +#ifdef USE_CUDA +#include +#endif + +using namespace torch::nn; +using namespace torch::test; + +template +bool test_RNN_xor(Func&& model_maker, bool cuda = false) { + torch::manual_seed(0); + + auto nhid = 32; + auto model = std::make_shared(); + auto l1 = model->add(Linear(1, nhid), "l1"); + auto rnn_model = model_maker(nhid); + auto rnn = model->add(rnn_model, "rnn"); + auto nout = nhid; + if (rnn_model.get()->options_base.proj_size() > 0) { + nout = rnn_model.get()->options_base.proj_size(); + } + auto lo = model->add(Linear(nout, 1), "lo"); + + torch::optim::Adam optimizer(model->parameters(), 1e-2); + auto forward_op = [&](torch::Tensor x) { + auto T = x.size(0); + auto B = x.size(1); + x = x.view({T * B, 1}); + x = l1->forward(x).view({T, B, nhid}).tanh_(); + x = std::get<0>(rnn->forward(x))[T - 1]; + x = lo->forward(x); + return x; + }; + + if (cuda) { + model->to(torch::kCUDA); + } + + float running_loss = 1; + int epoch = 0; + auto max_epoch = 1500; + while (running_loss > 1e-2) { + auto bs = 16U; + auto nlen = 5U; + + const auto backend = cuda ? torch::kCUDA : torch::kCPU; + auto inputs = + torch::rand({nlen, bs, 1}, backend).round().to(torch::kFloat32); + auto labels = inputs.sum(0).detach(); + inputs.set_requires_grad(true); + auto outputs = forward_op(inputs); + torch::Tensor loss = torch::mse_loss(outputs, labels); + + optimizer.zero_grad(); + loss.backward(); + optimizer.step(); + + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,cppcoreguidelines-avoid-magic-numbers,bugprone-narrowing-conversions) + running_loss = running_loss * 0.99 + loss.item() * 0.01; + if (epoch > max_epoch) { + return false; + } + epoch++; + } + return true; +}; + +void check_lstm_sizes( + std::tuple> + lstm_output) { + // Expect the LSTM to have 64 outputs and 3 layers, with an input of batch + // 10 and 16 time steps (10 x 16 x n) + + torch::Tensor output = std::get<0>(lstm_output); + std::tuple state = std::get<1>(lstm_output); + torch::Tensor hx = std::get<0>(state); + torch::Tensor cx = std::get<1>(state); + + ASSERT_EQ(output.ndimension(), 3); + ASSERT_EQ(output.size(0), 10); + ASSERT_EQ(output.size(1), 16); + ASSERT_EQ(output.size(2), 64); + + ASSERT_EQ(hx.ndimension(), 3); + ASSERT_EQ(hx.size(0), 3); // layers + ASSERT_EQ(hx.size(1), 16); // Batchsize + ASSERT_EQ(hx.size(2), 64); // 64 hidden dims + + ASSERT_EQ(cx.ndimension(), 3); + ASSERT_EQ(cx.size(0), 3); // layers + ASSERT_EQ(cx.size(1), 16); // Batchsize + ASSERT_EQ(cx.size(2), 64); // 64 hidden dims + + // Something is in the hiddens + ASSERT_GT(hx.norm().item(), 0); + ASSERT_GT(cx.norm().item(), 0); +} + +void check_lstm_sizes_proj( + std::tuple> + lstm_output) { + // Expect the LSTM to have 32 outputs and 3 layers, with an input of batch + // 10 and 16 time steps (10 x 16 x n) + + torch::Tensor output = std::get<0>(lstm_output); + std::tuple state = std::get<1>(lstm_output); + torch::Tensor hx = std::get<0>(state); + torch::Tensor cx = std::get<1>(state); + + ASSERT_EQ(output.ndimension(), 3); + ASSERT_EQ(output.size(0), 10); + ASSERT_EQ(output.size(1), 16); + ASSERT_EQ(output.size(2), 32); + + ASSERT_EQ(hx.ndimension(), 3); + ASSERT_EQ(hx.size(0), 3); // layers + ASSERT_EQ(hx.size(1), 16); // Batchsize + ASSERT_EQ(hx.size(2), 32); // 32 hidden dims + + ASSERT_EQ(cx.ndimension(), 3); + ASSERT_EQ(cx.size(0), 3); // layers + ASSERT_EQ(cx.size(1), 16); // Batchsize + ASSERT_EQ(cx.size(2), 64); // 64 cell dims + + // Something is in the hiddens + ASSERT_GT(hx.norm().item(), 0); + ASSERT_GT(cx.norm().item(), 0); +} + +struct RNNTest : torch::test::SeedingFixture {}; + +TEST_F(RNNTest, CheckOutputSizes) { + LSTM model(LSTMOptions(128, 64).num_layers(3).dropout(0.2)); + // Input size is: sequence length, batch size, input size + auto x = torch::randn({10, 16, 128}, torch::requires_grad()); + auto output = model->forward(x); + auto y = x.mean(); + + y.backward(); + check_lstm_sizes(output); + + auto next = model->forward(x, std::get<1>(output)); + + check_lstm_sizes(next); + + auto output_hx = std::get<0>(std::get<1>(output)); + auto output_cx = std::get<1>(std::get<1>(output)); + + auto next_hx = std::get<0>(std::get<1>(next)); + auto next_cx = std::get<1>(std::get<1>(next)); + + torch::Tensor diff = + torch::cat({next_hx, next_cx}, 0) - torch::cat({output_hx, output_cx}, 0); + + // Hiddens changed + ASSERT_GT(diff.abs().sum().item(), 1e-3); +} + +TEST_F(RNNTest, CheckOutputSizesProj) { + LSTM model(LSTMOptions(128, 64).num_layers(3).dropout(0.2).proj_size(32)); + // Input size is: sequence length, batch size, input size + auto x = torch::randn({10, 16, 128}, torch::requires_grad()); + auto output = model->forward(x); + auto y = x.mean(); + + y.backward(); + check_lstm_sizes_proj(output); + + auto next = model->forward(x, std::get<1>(output)); + + check_lstm_sizes_proj(next); + + auto output_hx = std::get<0>(std::get<1>(output)); + auto output_cx = std::get<1>(std::get<1>(output)); + + auto next_hx = std::get<0>(std::get<1>(next)); + auto next_cx = std::get<1>(std::get<1>(next)); + + torch::Tensor diff = next_hx - output_hx; + // Hiddens changed + ASSERT_GT(diff.abs().sum().item(), 1e-3); + diff = next_cx - output_cx; + ASSERT_GT(diff.abs().sum().item(), 1e-3); +} + +TEST_F(RNNTest, CheckOutputValuesMatchPyTorch) { + torch::manual_seed(0); + // Make sure the outputs match pytorch outputs + LSTM model(2, 2); + for (auto& v : model->parameters()) { + float size = v.numel(); + auto p = static_cast(v.storage().mutable_data()); + for (size_t i = 0; i < size; i++) { + p[i] = i / size; + } + } + + auto x = torch::empty({3, 4, 2}, torch::requires_grad()); + float size = x.numel(); + auto p = static_cast(x.storage().mutable_data()); + for (size_t i = 0; i < size; i++) { + p[i] = (size - i) / size; + } + + auto out = model->forward(x); + ASSERT_EQ(std::get<0>(out).ndimension(), 3); + ASSERT_EQ(std::get<0>(out).size(0), 3); + ASSERT_EQ(std::get<0>(out).size(1), 4); + ASSERT_EQ(std::get<0>(out).size(2), 2); + + auto flat = std::get<0>(out).view(3 * 4 * 2); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + float c_out[] = {0.4391, 0.5402, 0.4330, 0.5324, 0.4261, 0.5239, + 0.4183, 0.5147, 0.6822, 0.8064, 0.6726, 0.7968, + 0.6620, 0.7860, 0.6501, 0.7741, 0.7889, 0.9003, + 0.7769, 0.8905, 0.7635, 0.8794, 0.7484, 0.8666}; + for (size_t i = 0; i < 3 * 4 * 2; i++) { + ASSERT_LT(std::abs(flat[i].item() - c_out[i]), 1e-3); + } + + auto hx = std::get<0>(std::get<1>(out)); + auto cx = std::get<1>(std::get<1>(out)); + + ASSERT_EQ(hx.ndimension(), 3); // layers x B x 2 + ASSERT_EQ(hx.size(0), 1); + ASSERT_EQ(hx.size(1), 4); + ASSERT_EQ(hx.size(2), 2); + + ASSERT_EQ(cx.ndimension(), 3); // layers x B x 2 + ASSERT_EQ(cx.size(0), 1); + ASSERT_EQ(cx.size(1), 4); + ASSERT_EQ(cx.size(2), 2); + + flat = torch::cat({hx, cx}, 0).view(16); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + float h_out[] = { + 0.7889, + 0.9003, + 0.7769, + 0.8905, + 0.7635, + 0.8794, + 0.7484, + 0.8666, + 1.1647, + 1.6106, + 1.1425, + 1.5726, + 1.1187, + 1.5329, + 1.0931, + 1.4911}; + for (size_t i = 0; i < 16; i++) { + ASSERT_LT(std::abs(flat[i].item() - h_out[i]), 1e-3); + } +} + +TEST_F(RNNTest, EndToEndLSTM) { + ASSERT_TRUE(test_RNN_xor( + [](int s) { return LSTM(LSTMOptions(s, s).num_layers(2)); })); +} + +TEST_F(RNNTest, EndToEndLSTMProj) { + ASSERT_TRUE(test_RNN_xor([](int s) { + return LSTM(LSTMOptions(s, s).num_layers(2).proj_size(s / 2)); + })); +} + +TEST_F(RNNTest, EndToEndGRU) { + ASSERT_TRUE(test_RNN_xor( + [](int s) { return GRU(GRUOptions(s, s).num_layers(2)); })); +} + +TEST_F(RNNTest, EndToEndRNNRelu) { + ASSERT_TRUE(test_RNN_xor([](int s) { + return RNN(RNNOptions(s, s).nonlinearity(torch::kReLU).num_layers(2)); + })); +} + +TEST_F(RNNTest, EndToEndRNNTanh) { + ASSERT_TRUE(test_RNN_xor([](int s) { + return RNN(RNNOptions(s, s).nonlinearity(torch::kTanh).num_layers(2)); + })); +} + +TEST_F(RNNTest, Sizes_CUDA) { + torch::manual_seed(0); + LSTM model(LSTMOptions(128, 64).num_layers(3).dropout(0.2)); + model->to(torch::kCUDA); + auto x = + torch::randn({10, 16, 128}, torch::requires_grad().device(torch::kCUDA)); + auto output = model->forward(x); + auto y = x.mean(); + + y.backward(); + check_lstm_sizes(output); + + auto next = model->forward(x, std::get<1>(output)); + + check_lstm_sizes(next); + + auto output_hx = std::get<0>(std::get<1>(output)); + auto output_cx = std::get<1>(std::get<1>(output)); + + auto next_hx = std::get<0>(std::get<1>(next)); + auto next_cx = std::get<1>(std::get<1>(next)); + + torch::Tensor diff = + torch::cat({next_hx, next_cx}, 0) - torch::cat({output_hx, output_cx}, 0); + + // Hiddens changed + ASSERT_GT(diff.abs().sum().item(), 1e-3); +} + +TEST_F(RNNTest, SizesProj_CUDA) { + torch::manual_seed(0); + LSTM model(LSTMOptions(128, 64).num_layers(3).dropout(0.2).proj_size(32)); + model->to(torch::kCUDA); + auto x = + torch::randn({10, 16, 128}, torch::requires_grad().device(torch::kCUDA)); + auto output = model->forward(x); + auto y = x.mean(); + + y.backward(); + check_lstm_sizes_proj(output); + + auto next = model->forward(x, std::get<1>(output)); + + check_lstm_sizes_proj(next); + + auto output_hx = std::get<0>(std::get<1>(output)); + auto output_cx = std::get<1>(std::get<1>(output)); + + auto next_hx = std::get<0>(std::get<1>(next)); + auto next_cx = std::get<1>(std::get<1>(next)); + + torch::Tensor diff = next_hx - output_hx; + // Hiddens changed + ASSERT_GT(diff.abs().sum().item(), 1e-3); + diff = next_cx - output_cx; + ASSERT_GT(diff.abs().sum().item(), 1e-3); +} + +TEST_F(RNNTest, EndToEndLSTM_CUDA) { + ASSERT_TRUE(test_RNN_xor( + [](int s) { return LSTM(LSTMOptions(s, s).num_layers(2)); }, true)); +} + +TEST_F(RNNTest, EndToEndLSTMProj_CUDA) { + ASSERT_TRUE(test_RNN_xor( + [](int s) { + return LSTM(LSTMOptions(s, s).num_layers(2).proj_size(s / 2)); + }, + true)); +} + +TEST_F(RNNTest, EndToEndGRU_CUDA) { + ASSERT_TRUE(test_RNN_xor( + [](int s) { return GRU(GRUOptions(s, s).num_layers(2)); }, true)); +} + +TEST_F(RNNTest, EndToEndRNNRelu_CUDA) { + ASSERT_TRUE(test_RNN_xor( + [](int s) { + return RNN(RNNOptions(s, s).nonlinearity(torch::kReLU).num_layers(2)); + }, + true)); +} +TEST_F(RNNTest, EndToEndRNNTanh_CUDA) { + ASSERT_TRUE(test_RNN_xor( + [](int s) { + return RNN(RNNOptions(s, s).nonlinearity(torch::kTanh).num_layers(2)); + }, + true)); +} + +TEST_F(RNNTest, PrettyPrintRNNs) { + ASSERT_EQ( + c10::str(LSTM(LSTMOptions(128, 64).num_layers(3).dropout(0.2))), + "torch::nn::LSTM(input_size=128, hidden_size=64, num_layers=3, bias=true, batch_first=false, dropout=0.2, bidirectional=false)"); + ASSERT_EQ( + c10::str( + LSTM(LSTMOptions(128, 64).num_layers(3).dropout(0.2).proj_size(32))), + "torch::nn::LSTM(input_size=128, hidden_size=64, num_layers=3, bias=true, batch_first=false, dropout=0.2, bidirectional=false, proj_size=32)"); + ASSERT_EQ( + c10::str(GRU(GRUOptions(128, 64).num_layers(3).dropout(0.5))), + "torch::nn::GRU(input_size=128, hidden_size=64, num_layers=3, bias=true, batch_first=false, dropout=0.5, bidirectional=false)"); + ASSERT_EQ( + c10::str(RNN(RNNOptions(128, 64).num_layers(3).dropout(0.2).nonlinearity( + torch::kTanh))), + "torch::nn::RNN(input_size=128, hidden_size=64, num_layers=3, bias=true, batch_first=false, dropout=0.2, bidirectional=false)"); +} + +// This test assures that flatten_parameters does not crash, +// when bidirectional is set to true +// https://github.com/pytorch/pytorch/issues/19545 +TEST_F(RNNTest, BidirectionalFlattenParameters) { + GRU gru(GRUOptions(100, 256).num_layers(2).bidirectional(true)); + gru->flatten_parameters(); +} + +template +void copyParameters( + torch::nn::ModuleHolder& target, + std::string t_suffix, + const torch::nn::ModuleHolder& source, + std::string s_suffix) { + at::NoGradGuard guard; + target->named_parameters()["weight_ih_l" + t_suffix].copy_( + source->named_parameters()["weight_ih_l" + s_suffix]); + target->named_parameters()["weight_hh_l" + t_suffix].copy_( + source->named_parameters()["weight_hh_l" + s_suffix]); + target->named_parameters()["bias_ih_l" + t_suffix].copy_( + source->named_parameters()["bias_ih_l" + s_suffix]); + target->named_parameters()["bias_hh_l" + t_suffix].copy_( + source->named_parameters()["bias_hh_l" + s_suffix]); +} + +std::tuple gru_output_to_device( + std::tuple gru_output, + torch::Device device) { + return std::make_tuple( + std::get<0>(gru_output).to(device), std::get<1>(gru_output).to(device)); +} + +std::tuple> +lstm_output_to_device( + std::tuple> + lstm_output, + torch::Device device) { + auto hidden_states = std::get<1>(lstm_output); + return std::make_tuple( + std::get<0>(lstm_output).to(device), + std::make_tuple( + std::get<0>(hidden_states).to(device), + std::get<1>(hidden_states).to(device))); +} + +// This test is a port of python code introduced here: +// https://medium.com/data-science/understanding-bidirectional-rnn-in-pytorch-5bd25a5dd66 +// Reverse forward of bidirectional GRU should act +// as regular forward of unidirectional GRU +void BidirectionalGRUReverseForward(bool cuda) { + auto opt = torch::TensorOptions() + .dtype(torch::kFloat32) + .requires_grad(false) + .device(cuda ? torch::kCUDA : torch::kCPU); + auto input = torch::tensor({1, 2, 3, 4, 5}, opt).reshape({5, 1, 1}); + auto input_reversed = torch::tensor({5, 4, 3, 2, 1}, opt).reshape({5, 1, 1}); + + auto gru_options = GRUOptions(1, 1).num_layers(1).batch_first(false); + GRU bi_grus{gru_options.bidirectional(true)}; + GRU reverse_gru{gru_options.bidirectional(false)}; + + if (cuda) { + bi_grus->to(torch::kCUDA); + reverse_gru->to(torch::kCUDA); + } + + // Now make sure the weights of the reverse gru layer match + // ones of the (reversed) bidirectional's: + copyParameters(reverse_gru, "0", bi_grus, "0_reverse"); + + auto bi_output = bi_grus->forward(input); + auto reverse_output = reverse_gru->forward(input_reversed); + + if (cuda) { + bi_output = gru_output_to_device(bi_output, torch::kCPU); + reverse_output = gru_output_to_device(reverse_output, torch::kCPU); + } + + ASSERT_EQ( + std::get<0>(bi_output).size(0), std::get<0>(reverse_output).size(0)); + auto size = std::get<0>(bi_output).size(0); + for (int i = 0; i < size; i++) { + ASSERT_EQ( + std::get<0>(bi_output)[i][0][1].item(), + std::get<0>(reverse_output)[size - 1 - i][0][0].item()); + } + // The hidden states of the reversed GRUs sits + // in the odd indices in the first dimension. + ASSERT_EQ( + std::get<1>(bi_output)[1][0][0].item(), + std::get<1>(reverse_output)[0][0][0].item()); +} + +TEST_F(RNNTest, BidirectionalGRUReverseForward) { + BidirectionalGRUReverseForward(false); +} + +TEST_F(RNNTest, BidirectionalGRUReverseForward_CUDA) { + BidirectionalGRUReverseForward(true); +} + +// Reverse forward of bidirectional LSTM should act +// as regular forward of unidirectional LSTM +void BidirectionalLSTMReverseForwardTest(bool cuda) { + // ROCm 6.3 had a regression in RNN behavior requiring ASSERT_NEAR + constexpr auto tolerance = 1e-5; + auto opt = torch::TensorOptions() + .dtype(torch::kFloat32) + .requires_grad(false) + .device(cuda ? torch::kCUDA : torch::kCPU); + auto input = torch::tensor({1, 2, 3, 4, 5}, opt).reshape({5, 1, 1}); + auto input_reversed = torch::tensor({5, 4, 3, 2, 1}, opt).reshape({5, 1, 1}); + + auto lstm_opt = LSTMOptions(1, 1).num_layers(1).batch_first(false); + + LSTM bi_lstm{lstm_opt.bidirectional(true)}; + LSTM reverse_lstm{lstm_opt.bidirectional(false)}; + + if (cuda) { + bi_lstm->to(torch::kCUDA); + reverse_lstm->to(torch::kCUDA); + } + + // Now make sure the weights of the reverse lstm layer match + // ones of the (reversed) bidirectional's: + copyParameters(reverse_lstm, "0", bi_lstm, "0_reverse"); + + auto bi_output = bi_lstm->forward(input); + auto reverse_output = reverse_lstm->forward(input_reversed); + + if (cuda) { + bi_output = lstm_output_to_device(bi_output, torch::kCPU); + reverse_output = lstm_output_to_device(reverse_output, torch::kCPU); + } + + ASSERT_EQ( + std::get<0>(bi_output).size(0), std::get<0>(reverse_output).size(0)); + auto size = std::get<0>(bi_output).size(0); + for (int i = 0; i < size; i++) { + ASSERT_NEAR( + std::get<0>(bi_output)[i][0][1].item(), + std::get<0>(reverse_output)[size - 1 - i][0][0].item(), + tolerance); + } + // The hidden states of the reversed LSTM sits + // in the odd indices in the first dimension. + ASSERT_EQ( + std::get<0>(std::get<1>(bi_output))[1][0][0].item(), + std::get<0>(std::get<1>(reverse_output))[0][0][0].item()); + ASSERT_EQ( + std::get<1>(std::get<1>(bi_output))[1][0][0].item(), + std::get<1>(std::get<1>(reverse_output))[0][0][0].item()); +} + +TEST_F(RNNTest, BidirectionalLSTMReverseForward) { + BidirectionalLSTMReverseForwardTest(false); +} + +TEST_F(RNNTest, BidirectionalLSTMReverseForward_CUDA) { + BidirectionalLSTMReverseForwardTest(true); +} + +TEST_F(RNNTest, BidirectionalMultilayerGRU_CPU_vs_CUDA) { +#ifdef USE_CUDA + // Get device properties + const auto prop = at::cuda::getCurrentDeviceProperties(); + // TODO: Investigate why results on sm89 are much less accurate + // See https://github.com/pytorch/pytorch/issues/141915 + const auto tolerance = prop->major == 8 && prop->minor == 9 ? 2e-4 : 1e-5; +#else + constexpr auto tolerance = 1e-5; +#endif + // Create two GRUs with the same options + auto opt = + GRUOptions(2, 4).num_layers(3).batch_first(false).bidirectional(true); + GRU gru_cpu{opt}; + GRU gru_cuda{opt}; + + // Copy weights and biases from CPU GRU to CUDA GRU + { + at::NoGradGuard guard; + for (const auto& param : gru_cpu->named_parameters(/*recurse=*/false)) { + gru_cuda->named_parameters()[param.key()].copy_( + gru_cpu->named_parameters()[param.key()]); + } + } + + gru_cpu->flatten_parameters(); + gru_cuda->flatten_parameters(); + + // Move GRU to CUDA + gru_cuda->to(torch::kCUDA); + + // Create the same inputs + auto input_opt = + torch::TensorOptions().dtype(torch::kFloat32).requires_grad(false); + auto input_cpu = + torch::tensor({1, 2, 3, 4, 5, 6}, input_opt).reshape({3, 1, 2}); + auto input_cuda = torch::tensor({1, 2, 3, 4, 5, 6}, input_opt) + .reshape({3, 1, 2}) + .to(torch::kCUDA); + + // Call forward on both GRUs + auto output_cpu = gru_cpu->forward(input_cpu); + auto output_cuda = gru_cuda->forward(input_cuda); + + output_cpu = gru_output_to_device(output_cpu, torch::kCPU); + + // Assert that the output and state are equal on CPU and CUDA + ASSERT_EQ(std::get<0>(output_cpu).dim(), std::get<0>(output_cuda).dim()); + for (int i = 0; i < std::get<0>(output_cpu).dim(); i++) { + ASSERT_EQ( + std::get<0>(output_cpu).size(i), std::get<0>(output_cuda).size(i)); + } + for (int i = 0; i < std::get<0>(output_cpu).size(0); i++) { + for (int j = 0; j < std::get<0>(output_cpu).size(1); j++) { + for (int k = 0; k < std::get<0>(output_cpu).size(2); k++) { + ASSERT_NEAR( + std::get<0>(output_cpu)[i][j][k].item(), + std::get<0>(output_cuda)[i][j][k].item(), + tolerance); + } + } + } +} + +TEST_F(RNNTest, BidirectionalMultilayerLSTM_CPU_vs_CUDA) { +#ifdef USE_CUDA + // Get device properties + const auto prop = at::cuda::getCurrentDeviceProperties(); + // TODO: Investigate why results on sm89 are much less accurate + // See https://github.com/pytorch/pytorch/issues/141915 + const auto tolerance = prop->major == 8 && prop->minor == 9 ? 2e-4 : 1e-5; +#else + constexpr auto tolerance = 1e-5; +#endif + // Create two LSTMs with the same options + auto opt = + LSTMOptions(2, 4).num_layers(3).batch_first(false).bidirectional(true); + LSTM lstm_cpu{opt}; + LSTM lstm_cuda{opt}; + + // Copy weights and biases from CPU LSTM to CUDA LSTM + { + at::NoGradGuard guard; + for (const auto& param : lstm_cpu->named_parameters(/*recurse=*/false)) { + lstm_cuda->named_parameters()[param.key()].copy_( + lstm_cpu->named_parameters()[param.key()]); + } + } + + lstm_cpu->flatten_parameters(); + lstm_cuda->flatten_parameters(); + + // Move LSTM to CUDA + lstm_cuda->to(torch::kCUDA); + + auto options = + torch::TensorOptions().dtype(torch::kFloat32).requires_grad(false); + auto input_cpu = + torch::tensor({1, 2, 3, 4, 5, 6}, options).reshape({3, 1, 2}); + auto input_cuda = torch::tensor({1, 2, 3, 4, 5, 6}, options) + .reshape({3, 1, 2}) + .to(torch::kCUDA); + + // Call forward on both LSTMs + auto output_cpu = lstm_cpu->forward(input_cpu); + auto output_cuda = lstm_cuda->forward(input_cuda); + + output_cpu = lstm_output_to_device(output_cpu, torch::kCPU); + + // Assert that the output and state are equal on CPU and CUDA + ASSERT_EQ(std::get<0>(output_cpu).dim(), std::get<0>(output_cuda).dim()); + for (int i = 0; i < std::get<0>(output_cpu).dim(); i++) { + ASSERT_EQ( + std::get<0>(output_cpu).size(i), std::get<0>(output_cuda).size(i)); + } + for (int i = 0; i < std::get<0>(output_cpu).size(0); i++) { + for (int j = 0; j < std::get<0>(output_cpu).size(1); j++) { + for (int k = 0; k < std::get<0>(output_cpu).size(2); k++) { + ASSERT_NEAR( + std::get<0>(output_cpu)[i][j][k].item(), + std::get<0>(output_cuda)[i][j][k].item(), + tolerance); + } + } + } +} + +TEST_F(RNNTest, BidirectionalMultilayerLSTMProj_CPU_vs_CUDA) { +#ifdef USE_CUDA + // Get device properties + const auto prop = at::cuda::getCurrentDeviceProperties(); + // TODO: Investigate why results on sm89 are much less accurate + // See https://github.com/pytorch/pytorch/issues/141915 + const auto tolerance = prop->major == 8 && prop->minor == 9 ? 2e-4 : 1e-5; +#else + constexpr auto tolerance = 1e-5; +#endif + // Create two LSTMs with the same options + auto opt = LSTMOptions(2, 4) + .num_layers(3) + .batch_first(false) + .bidirectional(true) + .proj_size(2); + LSTM lstm_cpu{opt}; + LSTM lstm_cuda{opt}; + + // Copy weights and biases from CPU LSTM to CUDA LSTM + { + at::NoGradGuard guard; + for (const auto& param : lstm_cpu->named_parameters(/*recurse=*/false)) { + lstm_cuda->named_parameters()[param.key()].copy_( + lstm_cpu->named_parameters()[param.key()]); + } + } + + lstm_cpu->flatten_parameters(); + lstm_cuda->flatten_parameters(); + + // Move LSTM to CUDA + lstm_cuda->to(torch::kCUDA); + + auto options = + torch::TensorOptions().dtype(torch::kFloat32).requires_grad(false); + auto input_cpu = + torch::tensor({1, 2, 3, 4, 5, 6}, options).reshape({3, 1, 2}); + auto input_cuda = torch::tensor({1, 2, 3, 4, 5, 6}, options) + .reshape({3, 1, 2}) + .to(torch::kCUDA); + + // Call forward on both LSTMs + auto output_cpu = lstm_cpu->forward(input_cpu); + auto output_cuda = lstm_cuda->forward(input_cuda); + + output_cpu = lstm_output_to_device(output_cpu, torch::kCPU); + + // Assert that the output and state are equal on CPU and CUDA + ASSERT_EQ(std::get<0>(output_cpu).dim(), std::get<0>(output_cuda).dim()); + for (int i = 0; i < std::get<0>(output_cpu).dim(); i++) { + ASSERT_EQ( + std::get<0>(output_cpu).size(i), std::get<0>(output_cuda).size(i)); + } + for (int i = 0; i < std::get<0>(output_cpu).size(0); i++) { + for (int j = 0; j < std::get<0>(output_cpu).size(1); j++) { + for (int k = 0; k < std::get<0>(output_cpu).size(2); k++) { + ASSERT_NEAR( + std::get<0>(output_cpu)[i][j][k].item(), + std::get<0>(output_cuda)[i][j][k].item(), + tolerance); + } + } + } +} + +TEST_F(RNNTest, UsePackedSequenceAsInput) { + { + torch::manual_seed(0); + auto m = RNN(2, 3); + torch::nn::utils::rnn::PackedSequence packed_input = + torch::nn::utils::rnn::pack_sequence({torch::ones({3, 2})}); + auto rnn_output = m->forward_with_packed_input(packed_input); + auto expected_output = torch::tensor( + {{-0.0645, -0.7274, 0.4531}, + {-0.3970, -0.6950, 0.6009}, + {-0.3877, -0.7310, 0.6806}}); + ASSERT_TRUE(torch::allclose( + std::get<0>(rnn_output).data(), expected_output, 1e-05, 2e-04)); + + // Test passing optional argument to `RNN::forward_with_packed_input` + rnn_output = m->forward_with_packed_input(packed_input, torch::Tensor()); + ASSERT_TRUE(torch::allclose( + std::get<0>(rnn_output).data(), expected_output, 1e-05, 2e-04)); + } + { + torch::manual_seed(0); + auto m = LSTM(2, 3); + torch::nn::utils::rnn::PackedSequence packed_input = + torch::nn::utils::rnn::pack_sequence({torch::ones({3, 2})}); + auto rnn_output = m->forward_with_packed_input(packed_input); + auto expected_output = torch::tensor( + {{-0.2693, -0.1240, 0.0744}, + {-0.3889, -0.1919, 0.1183}, + {-0.4425, -0.2314, 0.1386}}); + ASSERT_TRUE(torch::allclose( + std::get<0>(rnn_output).data(), expected_output, 1e-05, 2e-04)); + + // Test passing optional argument to `LSTM::forward_with_packed_input` + rnn_output = m->forward_with_packed_input(packed_input, std::nullopt); + ASSERT_TRUE(torch::allclose( + std::get<0>(rnn_output).data(), expected_output, 1e-05, 2e-04)); + } + { + torch::manual_seed(0); + auto m = GRU(2, 3); + torch::nn::utils::rnn::PackedSequence packed_input = + torch::nn::utils::rnn::pack_sequence({torch::ones({3, 2})}); + auto rnn_output = m->forward_with_packed_input(packed_input); + auto expected_output = torch::tensor( + {{-0.1134, 0.0467, 0.2336}, + {-0.1189, 0.0502, 0.2960}, + {-0.1138, 0.0484, 0.3110}}); + ASSERT_TRUE(torch::allclose( + std::get<0>(rnn_output).data(), expected_output, 1e-05, 2e-04)); + + // Test passing optional argument to `GRU::forward_with_packed_input` + rnn_output = m->forward_with_packed_input(packed_input, torch::Tensor()); + ASSERT_TRUE(torch::allclose( + std::get<0>(rnn_output).data(), expected_output, 1e-05, 2e-04)); + } +} + +TEST_F(RNNTest, CheckErrorInfos) { + { + auto options = torch::nn::RNNOptions(1, 0).num_layers(1); + ASSERT_THROWS_WITH(RNN(options), "hidden_size must be greater than zero"); + + options = torch::nn::RNNOptions(1, 1).num_layers(0); + ASSERT_THROWS_WITH(RNN(options), "num_layers must be greater than zero"); + } + { + auto options = torch::nn::LSTMOptions(1, 0).num_layers(1); + ASSERT_THROWS_WITH(LSTM(options), "hidden_size must be greater than zero"); + + options = torch::nn::LSTMOptions(1, 1).num_layers(0); + ASSERT_THROWS_WITH(LSTM(options), "num_layers must be greater than zero"); + } + { + auto options = torch::nn::GRUOptions(1, 0).num_layers(1); + ASSERT_THROWS_WITH(GRU(options), "hidden_size must be greater than zero"); + + options = torch::nn::GRUOptions(1, 1).num_layers(0); + ASSERT_THROWS_WITH(GRU(options), "num_layers must be greater than zero"); + } +} + +// This test assures that pad_packed_sequence does not crash when packed with +// cuda tensors, https://github.com/pytorch/pytorch/issues/115027 +TEST_F(RNNTest, CheckPadPackedSequenceWithCudaTensors_CUDA) { + // Create input on the GPU, sample 5x5 + auto input = torch::randn({5, 5}).to(at::ScalarType::Float).cuda(); + auto lengths = torch::full({5}, 5); + + auto packed = + torch::nn::utils::rnn::pack_padded_sequence(input, lengths, false, false); + + auto error = torch::nn::utils::rnn::pad_packed_sequence(packed); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/sequential.cpp b/western_ai/usa/pytorch/test/cpp/api/sequential.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7207971307545895fe7d1be834d3f34f7e86ff0c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/sequential.cpp @@ -0,0 +1,673 @@ +#include + +#include +#include + +#include +#include +#include + +#include + +using namespace torch::nn; +using namespace torch::test; + +struct SequentialTest : torch::test::SeedingFixture {}; + +TEST_F(SequentialTest, CanContainThings) { + Sequential sequential(Linear(3, 4), ReLU(), BatchNorm1d(3)); +} + +TEST_F(SequentialTest, ConstructsFromSharedPointer) { + struct M : torch::nn::Module { + explicit M(int value_) : value(value_) {} + int value; + int forward() { + return value; + } + }; + Sequential sequential( + std::make_shared(1), std::make_shared(2), std::make_shared(3)); + ASSERT_EQ(sequential->size(), 3); + + Sequential sequential_named( + {{"m1", std::make_shared(1)}, + {std::string("m2"), std::make_shared(2)}, + {"m3", std::make_shared(3)}}); + ASSERT_EQ(sequential->size(), 3); +} + +TEST_F(SequentialTest, ConstructsFromConcreteType) { + static int copy_count; + + struct M : torch::nn::Module { + explicit M(int value_) : value(value_) {} + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) + M(const M& other) : torch::nn::Module(other) { + copy_count++; + } + int value; + int forward() { + return value; + } + }; + + copy_count = 0; + Sequential sequential(M(1), M(2), M(3)); + ASSERT_EQ(sequential->size(), 3); + // NOTE: The current implementation expects each module to be copied exactly + // once, which happens when the module is passed into `std::make_shared()`. + // TODO: Find a way to avoid copying, and then delete the copy constructor of + // `M`. + ASSERT_EQ(copy_count, 3); + + copy_count = 0; + Sequential sequential_named( + {{"m1", M(1)}, {std::string("m2"), M(2)}, {"m3", M(3)}}); + ASSERT_EQ(sequential->size(), 3); + ASSERT_EQ(copy_count, 3); +} + +TEST_F(SequentialTest, ConstructsFromModuleHolder) { + struct MImpl : torch::nn::Module { + explicit MImpl(int value_) : value(value_) {} + int forward() { + return value; + } + int value; + }; + + struct M : torch::nn::ModuleHolder { + using torch::nn::ModuleHolder::ModuleHolder; + using torch::nn::ModuleHolder::get; + }; + + Sequential sequential(M(1), M(2), M(3)); + ASSERT_EQ(sequential->size(), 3); + + Sequential sequential_named( + {{"m1", M(1)}, {std::string("m2"), M(2)}, {"m3", M(3)}}); + ASSERT_EQ(sequential->size(), 3); +} + +TEST_F(SequentialTest, PushBackAddsAnElement) { + struct M : torch::nn::Module { + explicit M(int value_) : value(value_) {} + int forward() { + return value; + } + int value; + }; + + // Test unnamed submodules + Sequential sequential; + ASSERT_EQ(sequential->size(), 0); + ASSERT_TRUE(sequential->is_empty()); + sequential->push_back(Linear(3, 4)); + ASSERT_EQ(sequential->size(), 1); + sequential->push_back(std::make_shared(1)); + ASSERT_EQ(sequential->size(), 2); + sequential->push_back(M(2)); + ASSERT_EQ(sequential->size(), 3); + + // Mix named and unnamed submodules + Sequential sequential_named; + ASSERT_EQ(sequential_named->size(), 0); + ASSERT_TRUE(sequential_named->is_empty()); + + sequential_named->push_back(Linear(3, 4)); + ASSERT_EQ(sequential_named->size(), 1); + ASSERT_EQ(sequential_named->named_children()[0].key(), "0"); + sequential_named->push_back(std::string("linear2"), Linear(3, 4)); + ASSERT_EQ(sequential_named->size(), 2); + ASSERT_EQ(sequential_named->named_children()[1].key(), "linear2"); + + sequential_named->push_back("shared_m1", std::make_shared(1)); + ASSERT_EQ(sequential_named->size(), 3); + ASSERT_EQ(sequential_named->named_children()[2].key(), "shared_m1"); + sequential_named->push_back(std::make_shared(1)); + ASSERT_EQ(sequential_named->size(), 4); + ASSERT_EQ(sequential_named->named_children()[3].key(), "3"); + + sequential_named->push_back(M(1)); + ASSERT_EQ(sequential_named->size(), 5); + ASSERT_EQ(sequential_named->named_children()[4].key(), "4"); + sequential_named->push_back(std::string("m2"), M(1)); + ASSERT_EQ(sequential_named->size(), 6); + ASSERT_EQ(sequential_named->named_children()[5].key(), "m2"); + + // named and unnamed AnyModule's + Sequential sequential_any; + auto a = torch::nn::AnyModule(torch::nn::Linear(1, 2)); + ASSERT_EQ(sequential_any->size(), 0); + ASSERT_TRUE(sequential_any->is_empty()); + sequential_any->push_back(a); + ASSERT_EQ(sequential_any->size(), 1); + ASSERT_EQ(sequential_any->named_children()[0].key(), "0"); + sequential_any->push_back("fc", a); + ASSERT_EQ(sequential_any->size(), 2); + ASSERT_EQ(sequential_any->named_children()[1].key(), "fc"); +} + +TEST_F(SequentialTest, AccessWithAt) { + struct M : torch::nn::Module { + explicit M(int value_) : value(value_) {} + int forward() { + return value; + } + int value; + }; + std::vector> modules = { + std::make_shared(1), std::make_shared(2), std::make_shared(3)}; + + Sequential sequential; + for (auto& module : modules) { + sequential->push_back(module); + } + ASSERT_EQ(sequential->size(), 3); + + // returns the correct module for a given index + for (const auto i : c10::irange(modules.size())) { + ASSERT_EQ(&sequential->at(i), modules[i].get()); + } + + // throws for a bad index + ASSERT_THROWS_WITH( + sequential->at(modules.size() + 1), "Index out of range"); + ASSERT_THROWS_WITH( + sequential->at(modules.size() + 1000000), "Index out of range"); +} + +TEST_F(SequentialTest, AccessWithPtr) { + struct M : torch::nn::Module { + explicit M(int value_) : value(value_) {} + int forward() { + return value; + } + int value; + }; + std::vector> modules = { + std::make_shared(1), std::make_shared(2), std::make_shared(3)}; + + Sequential sequential; + for (auto& module : modules) { + sequential->push_back(module); + } + ASSERT_EQ(sequential->size(), 3); + + // returns the correct module for a given index + for (const auto i : c10::irange(modules.size())) { + ASSERT_EQ(sequential->ptr(i).get(), modules[i].get()); + ASSERT_EQ(sequential[i].get(), modules[i].get()); + ASSERT_EQ(sequential->ptr(i).get(), modules[i].get()); + } + + // throws for a bad index + ASSERT_THROWS_WITH(sequential->ptr(modules.size() + 1), "Index out of range"); + ASSERT_THROWS_WITH( + sequential->ptr(modules.size() + 1000000), "Index out of range"); +} + +TEST_F(SequentialTest, CallingForwardOnEmptySequentialIsDisallowed) { + Sequential empty; + ASSERT_THROWS_WITH( + empty->forward(), "Cannot call forward() on an empty Sequential"); +} + +TEST_F(SequentialTest, CallingForwardChainsCorrectly) { + struct MockModule : torch::nn::Module { + explicit MockModule(int value) : expected(value) {} + int expected; + int forward(int value) { + assert(value == expected); + return value + 1; + } + }; + + Sequential sequential(MockModule{1}, MockModule{2}, MockModule{3}); + + ASSERT_EQ(sequential->forward(1), 4); +} + +TEST_F(SequentialTest, CallingForwardWithTheWrongReturnTypeThrows) { + struct M : public torch::nn::Module { + int forward() { + return 5; + } + }; + + Sequential sequential(M{}); + ASSERT_EQ(sequential->forward(), 5); + ASSERT_THROWS_WITH( + sequential->forward(), + "The type of the return value is int, but you asked for type float"); +} + +TEST_F(SequentialTest, TheReturnTypeOfForwardDefaultsToTensor) { + struct M : public torch::nn::Module { + torch::Tensor forward(torch::Tensor v) { + return v; + } + }; + + Sequential sequential(M{}); + auto variable = torch::ones({3, 3}, torch::requires_grad()); + ASSERT_TRUE(sequential->forward(variable).equal(variable)); +} + +TEST_F(SequentialTest, ForwardReturnsTheLastValue) { + torch::manual_seed(0); + Sequential sequential(Linear(10, 3), Linear(3, 5), Linear(5, 100)); + + auto x = torch::randn({1000, 10}, torch::requires_grad()); + auto y = sequential->forward(x); + ASSERT_EQ(y.ndimension(), 2); + ASSERT_EQ(y.size(0), 1000); + ASSERT_EQ(y.size(1), 100); +} + +TEST_F(SequentialTest, SanityCheckForHoldingStandardModules) { + Sequential sequential( + Linear(10, 3), + Conv2d(1, 2, 3), + Dropout(0.5), + BatchNorm2d(5), + Embedding(4, 10), + LSTM(4, 5)); +} + +TEST_F(SequentialTest, ExtendPushesModulesFromOtherSequential) { + struct A : torch::nn::Module { + int forward(int x) { + return x; + } + }; + struct B : torch::nn::Module { + int forward(int x) { + return x; + } + }; + struct C : torch::nn::Module { + int forward(int x) { + return x; + } + }; + struct D : torch::nn::Module { + int forward(int x) { + return x; + } + }; + Sequential a(A{}, B{}); + Sequential b(C{}, D{}); + a->extend(*b); + + ASSERT_EQ(a->size(), 4); + ASSERT_TRUE(a[0]->as()); + ASSERT_TRUE(a[1]->as()); + ASSERT_TRUE(a[2]->as()); + ASSERT_TRUE(a[3]->as()); + + ASSERT_EQ(b->size(), 2); + ASSERT_TRUE(b[0]->as()); + ASSERT_TRUE(b[1]->as()); + + std::vector> c = { + std::make_shared(), std::make_shared()}; + b->extend(c); + + ASSERT_EQ(b->size(), 4); + ASSERT_TRUE(b[0]->as()); + ASSERT_TRUE(b[1]->as()); + ASSERT_TRUE(b[2]->as()); + ASSERT_TRUE(b[3]->as()); +} + +TEST_F(SequentialTest, HasReferenceSemantics) { + Sequential first(Linear(2, 3), Linear(4, 4), Linear(4, 5)); + Sequential second(first); + + ASSERT_EQ(first.get(), second.get()); + ASSERT_EQ(first->size(), second->size()); + ASSERT_TRUE(std::equal( + first->begin(), + first->end(), + second->begin(), + [](const AnyModule& first, const AnyModule& second) { + return &first == &second; + })); +} + +TEST_F(SequentialTest, IsCloneable) { + Sequential sequential(Linear(3, 4), Functional(torch::relu), BatchNorm1d(3)); + Sequential clone = + std::dynamic_pointer_cast(sequential->clone()); + ASSERT_EQ(sequential->size(), clone->size()); + + for (size_t i = 0; i < sequential->size(); ++i) { + // The modules should be the same kind (type). + ASSERT_EQ(sequential[i]->name(), clone[i]->name()); + // But not pointer-equal (distinct objects). + ASSERT_NE(sequential[i], clone[i]); + } + + // Verify that the clone is deep, i.e. parameters of modules are cloned too. + + torch::NoGradGuard no_grad; + + auto params1 = sequential->named_parameters(); + auto params2 = clone->named_parameters(); + ASSERT_EQ(params1.size(), params2.size()); + for (auto& param : params1) { + ASSERT_FALSE(pointer_equal(param.value(), params2[param.key()])); + ASSERT_EQ(param->device(), params2[param.key()].device()); + ASSERT_TRUE(param->allclose(params2[param.key()])); + param->add_(2); + } + for (auto& param : params1) { + ASSERT_FALSE(param->allclose(params2[param.key()])); + } +} + +TEST_F(SequentialTest, RegistersElementsAsSubmodules) { + Sequential sequential(Linear(10, 3), Conv2d(1, 2, 3), Dropout2d(0.5)); + + auto modules = sequential->children(); + ASSERT_TRUE(modules[0]->as()); + ASSERT_TRUE(modules[1]->as()); + ASSERT_TRUE(modules[2]->as()); +} + +TEST_F(SequentialTest, CloneToDevice_CUDA) { + Sequential sequential(Linear(3, 4), Functional(torch::relu), BatchNorm1d(3)); + torch::Device device(torch::kCUDA, 0); + Sequential clone = + std::dynamic_pointer_cast(sequential->clone(device)); + for (const auto& p : clone->parameters()) { + ASSERT_EQ(p.device(), device); + } + for (const auto& b : clone->buffers()) { + ASSERT_EQ(b.device(), device); + } +} + +TEST_F(SequentialTest, PrettyPrintSequential) { + Sequential sequential( + Linear(10, 3), + Conv2d(1, 2, 3), + Dropout(0.5), + BatchNorm2d(5), + Embedding(4, 10), + LSTM(4, 5)); + ASSERT_EQ( + c10::str(sequential), + "torch::nn::Sequential(\n" + " (0): torch::nn::Linear(in_features=10, out_features=3, bias=true)\n" + " (1): torch::nn::Conv2d(1, 2, kernel_size=[3, 3], stride=[1, 1])\n" + " (2): torch::nn::Dropout(p=0.5, inplace=false)\n" + " (3): torch::nn::BatchNorm2d(5, eps=1e-05, momentum=0.1, affine=true, track_running_stats=true)\n" + " (4): torch::nn::Embedding(num_embeddings=4, embedding_dim=10)\n" + " (5): torch::nn::LSTM(input_size=4, hidden_size=5, num_layers=1, bias=true, batch_first=false, dropout=0, bidirectional=false)\n" + ")"); + + Sequential sequential_named( + {{"linear", Linear(10, 3)}, + {"conv2d", Conv2d(1, 2, 3)}, + {"dropout", Dropout(0.5)}, + {"batchnorm2d", BatchNorm2d(5)}, + {"embedding", Embedding(4, 10)}, + {"lstm", LSTM(4, 5)}}); + ASSERT_EQ( + c10::str(sequential_named), + "torch::nn::Sequential(\n" + " (linear): torch::nn::Linear(in_features=10, out_features=3, bias=true)\n" + " (conv2d): torch::nn::Conv2d(1, 2, kernel_size=[3, 3], stride=[1, 1])\n" + " (dropout): torch::nn::Dropout(p=0.5, inplace=false)\n" + " (batchnorm2d): torch::nn::BatchNorm2d(5, eps=1e-05, momentum=0.1, affine=true, track_running_stats=true)\n" + " (embedding): torch::nn::Embedding(num_embeddings=4, embedding_dim=10)\n" + " (lstm): torch::nn::LSTM(input_size=4, hidden_size=5, num_layers=1, bias=true, batch_first=false, dropout=0, bidirectional=false)\n" + ")"); +} + +TEST_F(SequentialTest, ModuleForwardMethodOptionalArg) { + { + Sequential sequential( + Identity(), + ConvTranspose1d(ConvTranspose1dOptions(3, 2, 3).stride(1).bias(false))); + std::dynamic_pointer_cast(sequential[1]) + ->weight.set_data(torch::arange(18.).reshape({3, 2, 3})); + auto x = torch::arange(30.).reshape({2, 3, 5}); + auto y = sequential->forward(x); + auto expected = torch::tensor( + {{{150., 333., 552., 615., 678., 501., 276.}, + {195., 432., 714., 804., 894., 654., 357.}}, + {{420., 918., 1497., 1560., 1623., 1176., 636.}, + {600., 1287., 2064., 2154., 2244., 1599., 852.}}}); + ASSERT_TRUE(torch::allclose(y, expected)); + } + { + Sequential sequential( + Identity(), + ConvTranspose2d(ConvTranspose2dOptions(3, 2, 3).stride(1).bias(false))); + std::dynamic_pointer_cast(sequential[1]) + ->weight.set_data(torch::arange(54.).reshape({3, 2, 3, 3})); + auto x = torch::arange(75.).reshape({1, 3, 5, 5}); + auto y = sequential->forward(x); + auto expected = torch::tensor( + {{{{2250., 4629., 7140., 7311., 7482., 5133., 2640.}, + {4995., 10272., 15837., 16206., 16575., 11364., 5841.}, + {8280., 17019., 26226., 26820., 27414., 18783., 9648.}, + {9225., 18954., 29196., 29790., 30384., 20808., 10683.}, + {10170., 20889., 32166., 32760., 33354., 22833., 11718.}, + {7515., 15420., 23721., 24144., 24567., 16800., 8613.}, + {4140., 8487., 13044., 13269., 13494., 9219., 4722.}}, + {{2925., 6006., 9246., 9498., 9750., 6672., 3423.}, + {6480., 13296., 20454., 20985., 21516., 14712., 7542.}, + {10710., 21960., 33759., 34596., 35433., 24210., 12402.}, + {12060., 24705., 37944., 38781., 39618., 27045., 13842.}, + {13410., 27450., 42129., 42966., 43803., 29880., 15282.}, + {9810., 20064., 30768., 31353., 31938., 21768., 11124.}, + {5355., 10944., 16770., 17076., 17382., 11838., 6045.}}}}); + ASSERT_TRUE(torch::allclose(y, expected)); + } + { + Sequential sequential( + Identity(), + ConvTranspose3d(ConvTranspose3dOptions(2, 2, 2).stride(1).bias(false))); + std::dynamic_pointer_cast(sequential[1]) + ->weight.set_data(torch::arange(32.).reshape({2, 2, 2, 2, 2})); + auto x = torch::arange(16.).reshape({1, 2, 2, 2, 2}); + auto y = sequential->forward(x); + auto expected = torch::tensor( + {{{{{128., 280., 154.}, {304., 664., 364.}, {184., 400., 218.}}, + {{352., 768., 420.}, {832., 1808., 984.}, {496., 1072., 580.}}, + {{256., 552., 298.}, {592., 1272., 684.}, {344., 736., 394.}}}, + {{{192., 424., 234.}, {464., 1016., 556.}, {280., 608., 330.}}, + {{544., 1184., 644.}, {1280., 2768., 1496.}, {752., 1616., 868.}}, + {{384., 824., 442.}, {880., 1880., 1004.}, {504., 1072., 570.}}}}}); + ASSERT_TRUE(torch::allclose(y, expected)); + } + { + auto weight = torch::tensor({{1., 2.3, 3.}, {4., 5.1, 6.3}}); + Sequential sequential(Identity(), EmbeddingBag::from_pretrained(weight)); + auto x = torch::tensor({{1, 0}}, torch::kLong); + auto y = sequential->forward(x); + auto expected = torch::tensor({2.5000, 3.7000, 4.6500}); + ASSERT_TRUE(torch::allclose(y, expected)); + } + { + torch::manual_seed(0); + + int64_t embed_dim = 8; + int64_t num_heads = 4; + int64_t batch_size = 8; + int64_t src_len = 3; + int64_t tgt_len = 1; + + auto query = torch::ones({batch_size, tgt_len, embed_dim}); + auto key = torch::ones({batch_size, src_len, embed_dim}); + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto value = key; + + Sequential sequential(MultiheadAttention(embed_dim, num_heads)); + auto output = sequential->forward>( + query.transpose(0, 1), key.transpose(0, 1), value.transpose(0, 1)); + + auto attn_output = std::get<0>(output); + auto attn_output_expected = torch::tensor( + {{{0.0674, -0.0056, 0.1324, 0.0922, 0.0160, -0.0934, -0.1700, 0.1663}, + {0.0674, -0.0056, 0.1324, 0.0922, 0.0160, -0.0934, -0.1700, 0.1663}, + {0.0674, -0.0056, 0.1324, 0.0922, 0.0160, -0.0934, -0.1700, 0.1663}, + {0.0674, -0.0056, 0.1324, 0.0922, 0.0160, -0.0934, -0.1700, 0.1663}, + {0.0674, -0.0056, 0.1324, 0.0922, 0.0160, -0.0934, -0.1700, 0.1663}, + {0.0674, -0.0056, 0.1324, 0.0922, 0.0160, -0.0934, -0.1700, 0.1663}, + {0.0674, -0.0056, 0.1324, 0.0922, 0.0160, -0.0934, -0.1700, 0.1663}, + {0.0674, + -0.0056, + 0.1324, + 0.0922, + 0.0160, + -0.0934, + -0.1700, + 0.1663}}}); + ASSERT_TRUE( + torch::allclose(attn_output, attn_output_expected, 1e-05, 2e-04)); + + auto attn_output_weights = std::get<1>(output); + auto attn_output_weights_expected = torch::tensor( + {{{0.3333, 0.3333, 0.3333}}, + {{0.3333, 0.3333, 0.3333}}, + {{0.3333, 0.3333, 0.3333}}, + {{0.3333, 0.3333, 0.3333}}, + {{0.3333, 0.3333, 0.3333}}, + {{0.3333, 0.3333, 0.3333}}, + {{0.3333, 0.3333, 0.3333}}, + {{0.3333, 0.3333, 0.3333}}}); + ASSERT_TRUE(torch::allclose( + attn_output_weights, attn_output_weights_expected, 1e-05, 2e-04)); + } + { + auto indices = torch::tensor({{{1, 3, 4}}}, torch::kLong); + auto x = torch::tensor({{{2, 4, 5}}}, torch::dtype(torch::kFloat)); + Sequential sequential(MaxUnpool1d(3)); + auto y = sequential->forward(x, indices); + auto expected = + torch::tensor({{{0, 2, 0, 4, 5, 0, 0, 0, 0}}}, torch::kFloat); + ASSERT_TRUE(torch::allclose(y, expected)); + } + { + auto indices = torch::tensor( + {{{{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}}, + {{{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}}}, + torch::kLong); + auto x = torch::tensor( + {{{{6, 8, 9}, {16, 18, 19}, {21, 23, 24}}}, + {{{31, 33, 34}, {41, 43, 44}, {46, 48, 49}}}}, + torch::dtype(torch::kFloat)); + Sequential sequential( + MaxUnpool2d(MaxUnpool2dOptions(3).stride(2).padding(1))); + auto y = sequential->forward(x, indices); + auto expected = torch::tensor( + {{{{0, 0, 0, 0, 0}, + {0, 6, 0, 8, 9}, + {0, 0, 0, 0, 0}, + {0, 16, 0, 18, 19}, + {0, 21, 0, 23, 24}}}, + {{{0, 0, 0, 0, 0}, + {0, 31, 0, 33, 34}, + {0, 0, 0, 0, 0}, + {0, 41, 0, 43, 44}, + {0, 46, 0, 48, 49}}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, expected)); + } + { + auto indices = torch::tensor({{{{{26}}}}}, torch::kLong); + auto x = torch::tensor( + {{{{{26}}}}}, torch::dtype(torch::kFloat).requires_grad(true)); + Sequential sequential(MaxUnpool3d(3)); + auto y = sequential->forward(x, indices); + auto expected = torch::tensor( + {{{{{0, 0, 0}, {0, 0, 0}, {0, 0, 0}}, + {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}}, + {{0, 0, 0}, {0, 0, 0}, {0, 0, 26}}}}}, + torch::kFloat); + ASSERT_TRUE(torch::allclose(y, expected)); + } + { + torch::manual_seed(0); + Sequential sequential(Identity(), RNN(2, 3)); + auto x = torch::ones({2, 3, 2}); + auto rnn_output = + sequential->forward>(x); + auto expected_output = torch::tensor( + {{{-0.0645, -0.7274, 0.4531}, + {-0.0645, -0.7274, 0.4531}, + {-0.0645, -0.7274, 0.4531}}, + {{-0.3970, -0.6950, 0.6009}, + {-0.3970, -0.6950, 0.6009}, + {-0.3970, -0.6950, 0.6009}}}); + ASSERT_TRUE(torch::allclose( + std::get<0>(rnn_output), expected_output, 1e-05, 2e-04)); + } + { + torch::manual_seed(0); + Sequential sequential(Identity(), LSTM(2, 3)); + auto x = torch::ones({2, 3, 2}); + auto rnn_output = sequential->forward< + std::tuple>>(x); + auto expected_output = torch::tensor( + {{{-0.2693, -0.1240, 0.0744}, + {-0.2693, -0.1240, 0.0744}, + {-0.2693, -0.1240, 0.0744}}, + {{-0.3889, -0.1919, 0.1183}, + {-0.3889, -0.1919, 0.1183}, + {-0.3889, -0.1919, 0.1183}}}); + ASSERT_TRUE(torch::allclose( + std::get<0>(rnn_output), expected_output, 1e-05, 2e-04)); + } + { + torch::manual_seed(0); + Sequential sequential(Identity(), GRU(2, 3)); + auto x = torch::ones({2, 3, 2}); + auto rnn_output = + sequential->forward>(x); + auto expected_output = torch::tensor( + {{{-0.1134, 0.0467, 0.2336}, + {-0.1134, 0.0467, 0.2336}, + {-0.1134, 0.0467, 0.2336}}, + {{-0.1189, 0.0502, 0.2960}, + {-0.1189, 0.0502, 0.2960}, + {-0.1189, 0.0502, 0.2960}}}); + ASSERT_TRUE(torch::allclose( + std::get<0>(rnn_output), expected_output, 1e-05, 2e-04)); + } + { + torch::manual_seed(0); + Sequential sequential(Identity(), RNNCell(2, 3)); + auto x = torch::ones({2, 2}); + auto rnn_output = sequential->forward(x); + auto expected_output = + torch::tensor({{-0.0645, -0.7274, 0.4531}, {-0.0645, -0.7274, 0.4531}}); + ASSERT_TRUE(torch::allclose(rnn_output, expected_output, 1e-05, 2e-04)); + } + { + torch::manual_seed(0); + Sequential sequential(Identity(), LSTMCell(2, 3)); + auto x = torch::ones({2, 2}); + auto rnn_output = + sequential->forward>(x); + auto expected_output = + torch::tensor({{-0.2693, -0.1240, 0.0744}, {-0.2693, -0.1240, 0.0744}}); + ASSERT_TRUE(torch::allclose( + std::get<0>(rnn_output), expected_output, 1e-05, 2e-04)); + } + { + torch::manual_seed(0); + Sequential sequential(Identity(), GRUCell(2, 3)); + auto x = torch::ones({2, 2}); + auto rnn_output = sequential->forward(x); + auto expected_output = + torch::tensor({{-0.1134, 0.0467, 0.2336}, {-0.1134, 0.0467, 0.2336}}); + ASSERT_TRUE(torch::allclose(rnn_output, expected_output, 1e-05, 2e-04)); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/serialize.cpp b/western_ai/usa/pytorch/test/cpp/api/serialize.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8608b6ab14ad7bdf027c303763378882d69352aa --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/serialize.cpp @@ -0,0 +1,1094 @@ +#include + +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include + +using namespace torch::test; +using namespace torch::nn; +using namespace torch::optim; + +namespace { +Sequential xor_model() { + return Sequential( + Linear(2, 8), + Functional(at::sigmoid), + Linear(8, 1), + Functional(at::sigmoid)); +} + +torch::Tensor save_and_load(torch::Tensor input) { + std::stringstream stream; + torch::save(input, stream); + torch::Tensor tensor; + torch::load(tensor, stream); + return tensor; +} +} // namespace + +template +void is_optimizer_param_group_equal( + const OptimizerParamGroup& lhs, + const OptimizerParamGroup& rhs) { + const auto& lhs_params = lhs.params(); + const auto& rhs_params = rhs.params(); + + ASSERT_TRUE(lhs_params.size() == rhs_params.size()); + for (const auto j : c10::irange(lhs_params.size())) { + ASSERT_TRUE(torch::equal(lhs_params[j], rhs_params[j])); + } + ASSERT_TRUE( + static_cast(lhs.options()) == + static_cast(rhs.options())); +} + +template +void is_optimizer_state_equal( + const ska::flat_hash_map>& + lhs_state, + const ska::flat_hash_map>& + rhs_state) { + ASSERT_TRUE(lhs_state.size() == rhs_state.size()); + for (const auto& value : lhs_state) { + auto found = rhs_state.find(value.first); + ASSERT_TRUE(found != rhs_state.end()); + const DerivedOptimizerParamState& lhs_curr_state = + static_cast(*(value.second.get())); + const DerivedOptimizerParamState& rhs_curr_state = + static_cast(*(found->second.get())); + ASSERT_TRUE(lhs_curr_state == rhs_curr_state); + } +} + +template < + typename OptimizerClass, + typename DerivedOptimizerOptions, + typename DerivedOptimizerParamState> +void test_serialize_optimizer( + DerivedOptimizerOptions options, + bool only_has_global_state = false) { + torch::manual_seed(0); + auto model1 = Linear(5, 2); + auto model2 = Linear(5, 2); + auto model3 = Linear(5, 2); + + // Models 1, 2, 3 will have the same parameters. + auto model_tempfile = c10::make_tempfile(); + torch::save(model1, model_tempfile.name); + torch::load(model2, model_tempfile.name); + torch::load(model3, model_tempfile.name); + + auto param1 = model1->named_parameters(); + auto param2 = model2->named_parameters(); + auto param3 = model3->named_parameters(); + for (const auto& p : param1) { + ASSERT_TRUE(p->allclose(param2[p.key()])); + ASSERT_TRUE(param2[p.key()].allclose(param3[p.key()])); + } + // Make some optimizers + auto optim1 = OptimizerClass( + {torch::optim::OptimizerParamGroup(model1->parameters())}, options); + auto optim2 = OptimizerClass(model2->parameters(), options); + auto optim2_2 = OptimizerClass(model2->parameters(), options); + auto optim3 = OptimizerClass(model3->parameters(), options); + auto optim3_2 = OptimizerClass(model3->parameters(), options); + for (auto& param_group : optim3_2.param_groups()) { + const double lr = param_group.options().get_lr(); + // change the learning rate, which will be overwritten by the loading + // otherwise, test cannot check if options are saved and loaded correctly + param_group.options().set_lr(lr + 0.01); + } + + auto x = torch::ones({10, 5}); + + auto step = [&x](torch::optim::Optimizer& optimizer, Linear model) { + optimizer.zero_grad(); + auto y = model->forward(x).sum(); + y.backward(); + auto closure = []() { return torch::tensor({10}); }; + optimizer.step(closure); + }; + + // Do 2 steps of model1 + step(optim1, model1); + step(optim1, model1); + + // Do 2 steps of model 2 without saving the optimizer + step(optim2, model2); + step(optim2_2, model2); + + // Do 1 step of model 3 + step(optim3, model3); + + // save the optimizer + auto optim_tempfile = c10::make_tempfile(); + torch::save(optim3, optim_tempfile.name); + torch::load(optim3_2, optim_tempfile.name); + + auto& optim3_2_param_groups = optim3_2.param_groups(); + auto& optim3_param_groups = optim3.param_groups(); + auto& optim3_2_state = optim3_2.state(); + auto& optim3_state = optim3.state(); + + // optim3_2 and optim1 should have param_groups and state of size 1 and + // state_size respectively + ASSERT_TRUE(optim3_2_param_groups.size() == 1); + // state_size = 2 for all optimizers except LBFGS as LBFGS only maintains one + // global state + unsigned state_size = only_has_global_state ? 1 : 2; + ASSERT_TRUE(optim3_2_state.size() == state_size); + + // optim3_2 and optim1 should have param_groups and state of same size + ASSERT_TRUE(optim3_2_param_groups.size() == optim3_param_groups.size()); + ASSERT_TRUE(optim3_2_state.size() == optim3_state.size()); + + // checking correctness of serialization logic for optimizer.param_groups_ and + // optimizer.state_ + for (const auto i : c10::irange(optim3_2_param_groups.size())) { + is_optimizer_param_group_equal( + optim3_2_param_groups[i], optim3_param_groups[i]); + is_optimizer_state_equal( + optim3_2_state, optim3_state); + } + + // Do step2 for model 3 + step(optim3_2, model3); + + param1 = model1->named_parameters(); + param2 = model2->named_parameters(); + param3 = model3->named_parameters(); + for (const auto& p : param1) { + const auto& name = p.key(); + // Model 1 and 3 should be the same + ASSERT_TRUE( + param1[name].norm().item() == param3[name].norm().item()); + ASSERT_TRUE( + param1[name].norm().item() != param2[name].norm().item()); + } +} + +/// Utility function to save a value of `int64_t` type. +void write_int_value( + torch::serialize::OutputArchive& archive, + const std::string& key, + const int64_t& value) { + archive.write(key, c10::IValue(value)); +} +// Utility function to save a vector of buffers. +template +void write_tensors_to_archive( + torch::serialize::OutputArchive& archive, + const std::string& key, + const BufferContainer& buffers) { + archive.write( + key + "/size", torch::tensor(static_cast(buffers.size()))); + for (const auto index : c10::irange(buffers.size())) { + archive.write( + key + "/" + std::to_string(index), buffers[index], /*is_buffer=*/true); + } +} + +// Utility function to save a vector of step buffers. +void write_step_buffers( + torch::serialize::OutputArchive& archive, + const std::string& key, + const std::vector& steps) { + std::vector tensors; + tensors.reserve(steps.size()); + for (const auto& step : steps) { + tensors.push_back(torch::tensor(static_cast(step))); + } + write_tensors_to_archive(archive, key, tensors); +} + +#define OLD_SERIALIZATION_LOGIC_WARNING_CHECK(funcname, optimizer, filename) \ + { \ + WarningCapture warnings; \ + funcname(optimizer, filename); \ + ASSERT_EQ( \ + count_substr_occurrences(warnings.str(), "old serialization"), 1); \ + } + +TEST(SerializeTest, KeysFunc) { + auto tempfile = c10::make_tempfile(); + torch::serialize::OutputArchive output_archive; + for (const auto i : c10::irange(3)) { + output_archive.write( + "element/" + std::to_string(i), c10::IValue(static_cast(i))); + } + output_archive.save_to(tempfile.name); + torch::serialize::InputArchive input_archive; + input_archive.load_from(tempfile.name); + std::vector keys = input_archive.keys(); + ASSERT_EQ(keys.size(), 3); + for (const auto i : c10::irange(keys.size())) { + ASSERT_EQ(keys[i], "element/" + std::to_string(i)); + } +} + +TEST(SerializeTest, TryReadFunc) { + auto tempfile = c10::make_tempfile(); + torch::serialize::OutputArchive output_archive; + for (const auto i : c10::irange(3)) { + output_archive.write( + "element/" + std::to_string(i), c10::IValue(static_cast(i))); + } + output_archive.save_to(tempfile.name); + torch::serialize::InputArchive input_archive; + input_archive.load_from(tempfile.name); + c10::IValue ivalue; + ASSERT_FALSE(input_archive.try_read("1", ivalue)); + ASSERT_TRUE(input_archive.try_read("element/1", ivalue)); + ASSERT_EQ(ivalue.toInt(), 1); +} + +TEST(SerializeTest, Basic) { + torch::manual_seed(0); + + auto x = torch::randn({5, 5}); + auto y = save_and_load(x); + + ASSERT_TRUE(y.defined()); + ASSERT_EQ(x.sizes().vec(), y.sizes().vec()); + ASSERT_TRUE(x.allclose(y)); +} + +TEST(SerializeTest, MathBits) { + torch::manual_seed(0); + + auto options = torch::TensorOptions{}.dtype(torch::kComplexFloat); + auto x = torch::randn({5, 5}, options); + { + auto expected = torch::conj(x); + auto actual = save_and_load(expected); + + ASSERT_TRUE(actual.defined()); + ASSERT_EQ(actual.sizes().vec(), expected.sizes().vec()); + ASSERT_TRUE(actual.allclose(expected)); + } + + { + auto expected = torch::_neg_view(x); + auto actual = save_and_load(expected); + + ASSERT_TRUE(actual.defined()); + ASSERT_EQ(actual.sizes().vec(), expected.sizes().vec()); + ASSERT_TRUE(actual.allclose(expected)); + } + + { + auto expected = torch::conj(torch::_neg_view(x)); + auto actual = save_and_load(expected); + + ASSERT_TRUE(actual.defined()); + ASSERT_EQ(actual.sizes().vec(), expected.sizes().vec()); + ASSERT_TRUE(actual.allclose(expected)); + } + + { + // We don't support serializing `ZeroTensor` as it is not public facing yet. + // If in future, `ZeroTensor` serialization is supported, this test should + // start failing! + auto t = torch::_efficientzerotensor({5, 5}); + ASSERT_THROWS_WITH(save_and_load(t), "ZeroTensor is not serializable,"); + } +} + +TEST(SerializeTest, BasicToFile) { + torch::manual_seed(0); + + auto x = torch::randn({5, 5}); + + auto tempfile = c10::make_tempfile(); + torch::save(x, tempfile.name); + + torch::Tensor y; + torch::load(y, tempfile.name); + + ASSERT_TRUE(y.defined()); + ASSERT_EQ(x.sizes().vec(), y.sizes().vec()); + ASSERT_TRUE(x.allclose(y)); +} + +TEST(SerializeTest, BasicViaFunc) { + torch::manual_seed(0); + + auto x = torch::randn({5, 5}); + + std::string serialized; + torch::save(x, [&](const void* buf, size_t n) { + serialized.append(reinterpret_cast(buf), n); + return n; + }); + torch::Tensor y; + torch::load(y, serialized.data(), serialized.size()); + + ASSERT_TRUE(y.defined()); + ASSERT_EQ(x.sizes().vec(), y.sizes().vec()); + ASSERT_TRUE(x.allclose(y)); + + torch::Tensor z; + torch::load( + z, + [&](uint64_t pos, void* buf, size_t n) -> size_t { + if (pos >= serialized.size()) + return 0; + size_t nbytes = + std::min(static_cast(pos) + n, serialized.size()) - pos; + memcpy(buf, serialized.data() + pos, nbytes); + return nbytes; + }, + [&]() -> size_t { return serialized.size(); }); + ASSERT_TRUE(z.defined()); + ASSERT_EQ(x.sizes().vec(), z.sizes().vec()); + ASSERT_TRUE(x.allclose(z)); +} + +TEST(SerializeTest, Resized) { + torch::manual_seed(0); + + auto x = torch::randn({11, 5}); + x.resize_({5, 5}); + auto y = save_and_load(x); + + ASSERT_TRUE(y.defined()); + ASSERT_EQ(x.sizes().vec(), y.sizes().vec()); + ASSERT_TRUE(x.allclose(y)); +} + +TEST(SerializeTest, Sliced) { + torch::manual_seed(0); + + auto x = torch::randn({11, 5}); + x = x.slice(0, 1, 5); + auto y = save_and_load(x); + + ASSERT_TRUE(y.defined()); + ASSERT_EQ(x.sizes().vec(), y.sizes().vec()); + ASSERT_TRUE(x.allclose(y)); +} + +TEST(SerializeTest, NonContiguous) { + torch::manual_seed(0); + + auto x = torch::randn({11, 5}); + x = x.slice(1, 1, 4); + auto y = save_and_load(x); + + ASSERT_TRUE(y.defined()); + ASSERT_EQ(x.sizes().vec(), y.sizes().vec()); + ASSERT_TRUE(x.allclose(y)); +} + +TEST(SerializeTest, ErrorOnMissingKey) { + struct B : torch::nn::Module { + B(const std::string& name_c) { + register_buffer(name_c, torch::ones(5, torch::kFloat)); + } + }; + struct A : torch::nn::Module { + A(const std::string& name_b, const std::string& name_c) { + register_module(name_b, std::make_shared(name_c)); + } + }; + struct M : torch::nn::Module { + M(const std::string& name_a, + const std::string& name_b, + const std::string& name_c) { + register_module(name_a, std::make_shared(name_b, name_c)); + } + }; + + // create a hierarchy of models with names differing below the top level + auto model1 = std::make_shared("a", "b", "c"); + auto model2 = std::make_shared("a", "b", "x"); + auto model3 = std::make_shared("a", "x", "c"); + + std::stringstream stream; + torch::save(model1, stream); + // We want the errors to contain hierarchy information, too. + ASSERT_THROWS_WITH( + torch::load(model2, stream), "No such serialized tensor 'a.b.x'"); + stream.seekg(0, stream.beg); + ASSERT_THROWS_WITH( + torch::load(model3, stream), "No such serialized submodule: 'a.x'"); +} + +TEST(SerializeTest, XOR) { + // We better be able to save and load an XOR model! + auto getLoss = [](Sequential model, uint32_t batch_size) { + auto inputs = torch::empty({batch_size, 2}); + auto labels = torch::empty({batch_size}); + for (const auto i : c10::irange(batch_size)) { + inputs[i] = torch::randint(2, {2}, torch::kInt64); + labels[i] = inputs[i][0].item() ^ inputs[i][1].item(); + } + auto x = model->forward(inputs); + return torch::binary_cross_entropy(x, labels); + }; + + auto model = xor_model(); + auto model2 = xor_model(); + auto model3 = xor_model(); + auto optimizer = torch::optim::SGD( + model->parameters(), + torch::optim::SGDOptions(1e-1).momentum(0.9).nesterov(true).weight_decay( + 1e-6)); + + float running_loss = 1; + int epoch = 0; + while (running_loss > 0.1) { + torch::Tensor loss = getLoss(model, 4); + optimizer.zero_grad(); + loss.backward(); + optimizer.step(); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + running_loss = running_loss * 0.99 + loss.sum().item() * 0.01; + ASSERT_LT(epoch, 3000); + epoch++; + } + + auto tempfile = c10::make_tempfile(); + torch::save(model, tempfile.name); + torch::load(model2, tempfile.name); + + auto loss = getLoss(model2, 100); + ASSERT_LT(loss.item(), 0.1); +} + +TEST(SerializeTest, Optim) { + auto model1 = Linear(5, 2); + auto model2 = Linear(5, 2); + auto model3 = Linear(5, 2); + + // Models 1, 2, 3 will have the same parameters. + auto model_tempfile = c10::make_tempfile(); + torch::save(model1, model_tempfile.name); + torch::load(model2, model_tempfile.name); + torch::load(model3, model_tempfile.name); + + auto param1 = model1->named_parameters(); + auto param2 = model2->named_parameters(); + auto param3 = model3->named_parameters(); + for (const auto& p : param1) { + ASSERT_TRUE(p->allclose(param2[p.key()])); + ASSERT_TRUE(param2[p.key()].allclose(param3[p.key()])); + } + + // Make some optimizers with momentum (and thus state) + auto optim1 = torch::optim::SGD( + model1->parameters(), torch::optim::SGDOptions(1e-1).momentum(0.9)); + auto optim2 = torch::optim::SGD( + model2->parameters(), torch::optim::SGDOptions(1e-1).momentum(0.9)); + auto optim2_2 = torch::optim::SGD( + model2->parameters(), torch::optim::SGDOptions(1e-1).momentum(0.9)); + auto optim3 = torch::optim::SGD( + model3->parameters(), torch::optim::SGDOptions(1e-1).momentum(0.9)); + auto optim3_2 = torch::optim::SGD( + model3->parameters(), torch::optim::SGDOptions(1e-1).momentum(0.9)); + + auto x = torch::ones({10, 5}); + + auto step = [&x](torch::optim::Optimizer& optimizer, Linear model) { + optimizer.zero_grad(); + auto y = model->forward(x).sum(); + y.backward(); + optimizer.step(); + }; + + // Do 2 steps of model1 + step(optim1, model1); + step(optim1, model1); + + // Do 2 steps of model 2 without saving the optimizer + step(optim2, model2); + step(optim2_2, model2); + + // Do 2 steps of model 3 while saving the optimizer + step(optim3, model3); + + auto optim_tempfile = c10::make_tempfile(); + torch::save(optim3, optim_tempfile.name); + torch::load(optim3_2, optim_tempfile.name); + step(optim3_2, model3); + + param1 = model1->named_parameters(); + param2 = model2->named_parameters(); + param3 = model3->named_parameters(); + for (const auto& p : param1) { + const auto& name = p.key(); + // Model 1 and 3 should be the same + ASSERT_TRUE( + param1[name].norm().item() == param3[name].norm().item()); + ASSERT_TRUE( + param1[name].norm().item() != param2[name].norm().item()); + } +} + +TEST(SerializeTest, Optim_Adagrad) { + test_serialize_optimizer( + AdagradOptions(1e-1)); + + // bc compatibility check + auto model1 = Linear(5, 2); + auto optim1 = torch::optim::Adagrad( + model1->parameters(), torch::optim::AdagradOptions(1e-1)); + + auto x = torch::ones({10, 5}); + auto step = [&x](torch::optim::Optimizer& optimizer, Linear model) { + optimizer.zero_grad(); + auto y = model->forward(x).sum(); + y.backward(); + optimizer.step(); + }; + step(optim1, model1); + auto optim1_2 = + Adagrad(model1->parameters(), torch::optim::AdagradOptions(1e-1)); + + // fill up with optim1 sum_buffers + std::vector sum_buffers; + // fill up with optim1 state_buffers + std::vector step_buffers; + const auto& params_ = optim1.param_groups()[0].params(); + const auto& optim1_state = optim1.state(); + for (const auto& param : params_) { + auto key_ = param.unsafeGetTensorImpl(); + const AdagradParamState& curr_state_ = + static_cast(*(optim1_state.at(key_).get())); + sum_buffers.emplace_back(curr_state_.sum()); + step_buffers.emplace_back(curr_state_.step()); + } + // write sum_buffers and step_buffers to the file + auto optim_tempfile_old_format = c10::make_tempfile(); + torch::serialize::OutputArchive output_archive; + write_tensors_to_archive(output_archive, "sum_buffers", sum_buffers); + write_step_buffers(output_archive, "step_buffers", step_buffers); + output_archive.save_to(optim_tempfile_old_format.name); + OLD_SERIALIZATION_LOGIC_WARNING_CHECK( + torch::load, optim1_2, optim_tempfile_old_format.name); + is_optimizer_state_equal(optim1.state(), optim1_2.state()); +} + +TEST(SerializeTest, Optim_SGD) { + test_serialize_optimizer( + SGDOptions(1e-1).momentum(0.9)); + + // bc compatibility check + auto model1 = Linear(5, 2); + auto model1_params = model1->parameters(); + // added a tensor for lazy init check - when all params do not have a momentum + // buffer entry + model1_params.emplace_back(torch::randn({2, 3})); + auto optim1 = torch::optim::SGD( + model1_params, torch::optim::SGDOptions(0.01).momentum(0.9)); + + auto x = torch::ones({10, 5}); + auto step = [&x](torch::optim::Optimizer& optimizer, Linear model) { + optimizer.zero_grad(); + auto y = model->forward(x).sum(); + y.backward(); + optimizer.step(); + }; + step(optim1, model1); + + std::vector momentum_buffers; + int64_t iteration_{0}; + const auto& params_ = optim1.param_groups()[0].params(); + const auto& optim1_state = optim1.state(); + for (const auto i : c10::irange(params_.size())) { + if (i != (params_.size() - 1)) { + auto key_ = params_[i].unsafeGetTensorImpl(); + const SGDParamState& curr_state_ = + static_cast(*(optim1_state.at(key_).get())); + momentum_buffers.emplace_back(curr_state_.momentum_buffer()); + } + } + ASSERT_TRUE(momentum_buffers.size() == (params_.size() - 1)); + // write momentum_buffers to the file + auto optim_tempfile_old_format = c10::make_tempfile(); + torch::serialize::OutputArchive output_archive; + write_tensors_to_archive( + output_archive, "momentum_buffers", momentum_buffers); + write_int_value(output_archive, "iteration_", iteration_); + output_archive.save_to(optim_tempfile_old_format.name); + auto optim1_2 = + SGD(model1_params, torch::optim::SGDOptions(1e-1).momentum(0.9)); + OLD_SERIALIZATION_LOGIC_WARNING_CHECK( + torch::load, optim1_2, optim_tempfile_old_format.name); + is_optimizer_state_equal(optim1.state(), optim1_2.state()); +} + +TEST(SerializeTest, Optim_Adam) { + test_serialize_optimizer( + AdamOptions().lr(0.99999).amsgrad(true).weight_decay(0.5)); + + // bc compatibility check + auto model1 = Linear(5, 2); + auto model1_params = model1->parameters(); + // added a tensor for lazy init check - when all params do not have entry in + // buffers + model1_params.emplace_back(torch::randn({2, 3})); + auto optim1 = torch::optim::Adam( + model1_params, torch::optim::AdamOptions().weight_decay(0.5)); + + auto x = torch::ones({10, 5}); + auto step = [&x](torch::optim::Optimizer& optimizer, Linear model) { + optimizer.zero_grad(); + auto y = model->forward(x).sum(); + y.backward(); + optimizer.step(); + }; + step(optim1, model1); + + std::vector step_buffers; + std::vector exp_average_buffers; + std::vector exp_average_sq_buffers; + std::vector max_exp_average_sq_buffers; + const auto& params_ = optim1.param_groups()[0].params(); + const auto& optim1_state = optim1.state(); + for (const auto i : c10::irange(params_.size())) { + if (i != (params_.size() - 1)) { + auto key_ = params_[i].unsafeGetTensorImpl(); + const AdamParamState& curr_state_ = + static_cast(*(optim1_state.at(key_).get())); + step_buffers.emplace_back(curr_state_.step()); + exp_average_buffers.emplace_back(curr_state_.exp_avg()); + exp_average_sq_buffers.emplace_back(curr_state_.exp_avg_sq()); + if (curr_state_.max_exp_avg_sq().defined()) { + max_exp_average_sq_buffers.emplace_back(curr_state_.max_exp_avg_sq()); + } + } + } + // write buffers to the file + auto optim_tempfile_old_format = c10::make_tempfile(); + torch::serialize::OutputArchive output_archive; + write_step_buffers(output_archive, "step_buffers", step_buffers); + write_tensors_to_archive( + output_archive, "exp_average_buffers", exp_average_buffers); + write_tensors_to_archive( + output_archive, "exp_average_sq_buffers", exp_average_sq_buffers); + write_tensors_to_archive( + output_archive, "max_exp_average_sq_buffers", max_exp_average_sq_buffers); + output_archive.save_to(optim_tempfile_old_format.name); + auto optim1_2 = Adam(model1_params, torch::optim::AdamOptions()); + OLD_SERIALIZATION_LOGIC_WARNING_CHECK( + torch::load, optim1_2, optim_tempfile_old_format.name); + is_optimizer_state_equal(optim1.state(), optim1_2.state()); +} + +TEST(SerializeTest, Optim_AdamW) { + test_serialize_optimizer( + AdamWOptions().lr(0.99999).amsgrad(true).betas( + std::make_tuple(0.999, 0.1))); + + // bc compatibility check + auto model1 = Linear(5, 2); + auto model1_params = model1->parameters(); + // added a tensor for lazy init check - when all params do not have entry in + // buffers + model1_params.emplace_back(torch::randn({2, 3})); + auto optim1 = torch::optim::AdamW( + model1_params, torch::optim::AdamWOptions().weight_decay(0.5)); + + auto x = torch::ones({10, 5}); + auto step = [&x](torch::optim::Optimizer& optimizer, Linear model) { + optimizer.zero_grad(); + auto y = model->forward(x).sum(); + y.backward(); + optimizer.step(); + }; + step(optim1, model1); + + std::vector step_buffers; + std::vector exp_average_buffers; + std::vector exp_average_sq_buffers; + std::vector max_exp_average_sq_buffers; + const auto& params_ = optim1.param_groups()[0].params(); + const auto& optim1_state = optim1.state(); + for (const auto i : c10::irange(params_.size())) { + if (i != (params_.size() - 1)) { + auto key_ = params_[i].unsafeGetTensorImpl(); + const AdamWParamState& curr_state_ = + static_cast(*(optim1_state.at(key_).get())); + step_buffers.emplace_back(curr_state_.step()); + exp_average_buffers.emplace_back(curr_state_.exp_avg()); + exp_average_sq_buffers.emplace_back(curr_state_.exp_avg_sq()); + if (curr_state_.max_exp_avg_sq().defined()) { + max_exp_average_sq_buffers.emplace_back(curr_state_.max_exp_avg_sq()); + } + } + } + // write buffers to the file + auto optim_tempfile_old_format = c10::make_tempfile(); + torch::serialize::OutputArchive output_archive; + write_step_buffers(output_archive, "step_buffers", step_buffers); + write_tensors_to_archive( + output_archive, "exp_average_buffers", exp_average_buffers); + write_tensors_to_archive( + output_archive, "exp_average_sq_buffers", exp_average_sq_buffers); + write_tensors_to_archive( + output_archive, "max_exp_average_sq_buffers", max_exp_average_sq_buffers); + output_archive.save_to(optim_tempfile_old_format.name); + auto optim1_2 = AdamW(model1_params, torch::optim::AdamWOptions()); + OLD_SERIALIZATION_LOGIC_WARNING_CHECK( + torch::load, optim1_2, optim_tempfile_old_format.name); + is_optimizer_state_equal(optim1.state(), optim1_2.state()); +} + +TEST(SerializeTest, Optim_RMSprop) { + auto options = RMSpropOptions(0.1).momentum(0.9).centered(true); + test_serialize_optimizer(options); + + // bc compatibility check + auto model1 = Linear(5, 2); + auto model1_params = model1->parameters(); + + // added a tensor for lazy init check - when all params do not have a momentum + // buffer entry + model1_params.emplace_back(torch::randn({2, 3})); + auto optim1 = torch::optim::RMSprop(model1_params, options); + + auto x = torch::ones({10, 5}); + auto step = [&x](torch::optim::Optimizer& optimizer, Linear model) { + optimizer.zero_grad(); + auto y = model->forward(x).sum(); + y.backward(); + optimizer.step(); + }; + step(optim1, model1); + + std::vector square_average_buffers; + std::vector momentum_buffers; + std::vector grad_average_buffers; + const auto& params_ = optim1.param_groups()[0].params(); + const auto& optim1_state = optim1.state(); + for (const auto i : c10::irange(params_.size())) { + if (i != (params_.size() - 1)) { + auto key_ = params_[i].unsafeGetTensorImpl(); + const RMSpropParamState& curr_state_ = + static_cast(*(optim1_state.at(key_).get())); + square_average_buffers.emplace_back(curr_state_.square_avg()); + if (curr_state_.momentum_buffer().defined()) { + momentum_buffers.emplace_back(curr_state_.momentum_buffer()); + } + if (curr_state_.grad_avg().defined()) { + grad_average_buffers.emplace_back(curr_state_.grad_avg()); + } + } + } + // write buffers to the file + auto optim_tempfile_old_format = c10::make_tempfile(); + torch::serialize::OutputArchive output_archive; + write_tensors_to_archive( + output_archive, "square_average_buffers", square_average_buffers); + write_tensors_to_archive( + output_archive, "momentum_buffers", momentum_buffers); + write_tensors_to_archive( + output_archive, "grad_average_buffers", grad_average_buffers); + output_archive.save_to(optim_tempfile_old_format.name); + auto optim1_2 = RMSprop(model1_params, options); + OLD_SERIALIZATION_LOGIC_WARNING_CHECK( + torch::load, optim1_2, optim_tempfile_old_format.name); + const auto& params1_2_ = optim1_2.param_groups()[0].params(); + auto& optim1_2_state = optim1_2.state(); + // old RMSprop didn't track step value + for (const auto i : c10::irange(params1_2_.size())) { + if (i != (params1_2_.size() - 1)) { + auto key_ = params_[i].unsafeGetTensorImpl(); + const RMSpropParamState& curr_state_ = + static_cast(*(optim1_state.at(key_).get())); + RMSpropParamState& curr_state1_2_ = + static_cast(*(optim1_2_state.at(key_).get())); + curr_state1_2_.step(curr_state_.step()); + } + } + is_optimizer_state_equal(optim1.state(), optim1_2.state()); +} + +TEST(SerializeTest, Optim_LBFGS) { + test_serialize_optimizer( + LBFGSOptions(), true); + // bc compatibility check + auto model1 = Linear(5, 2); + auto model1_params = model1->parameters(); + // added a tensor for lazy init check - when all params do not have entry in + // buffers + model1_params.emplace_back(torch::randn({2, 3})); + auto optim1 = + torch::optim::LBFGS(model1_params, torch::optim::LBFGSOptions()); + + auto x = torch::ones({10, 5}); + auto step = [&x](torch::optim::Optimizer& optimizer, Linear model) { + optimizer.zero_grad(); + auto y = model->forward(x).sum(); + y.backward(); + auto closure = []() { return torch::tensor({10}); }; + optimizer.step(closure); + }; + + step(optim1, model1); + + at::Tensor d, t, H_diag, prev_flat_grad, prev_loss; + std::deque old_dirs, old_stps; + + const auto& params_ = optim1.param_groups()[0].params(); + auto key_ = params_[0].unsafeGetTensorImpl(); + const auto& optim1_state = + static_cast(*(optim1.state().at(key_).get())); + d = optim1_state.d(); + t = at::tensor(optim1_state.t()); + H_diag = optim1_state.H_diag(); + prev_flat_grad = optim1_state.prev_flat_grad(); + prev_loss = at::tensor(optim1_state.prev_loss()); + old_dirs = optim1_state.old_dirs(); + + // write buffers to the file + auto optim_tempfile_old_format = c10::make_tempfile(); + torch::serialize::OutputArchive output_archive; + output_archive.write("d", d, /*is_buffer=*/true); + output_archive.write("t", t, /*is_buffer=*/true); + output_archive.write("H_diag", H_diag, /*is_buffer=*/true); + output_archive.write("prev_flat_grad", prev_flat_grad, /*is_buffer=*/true); + output_archive.write("prev_loss", prev_loss, /*is_buffer=*/true); + write_tensors_to_archive(output_archive, "old_dirs", old_dirs); + write_tensors_to_archive(output_archive, "old_stps", old_stps); + output_archive.save_to(optim_tempfile_old_format.name); + + auto optim1_2 = LBFGS(model1_params, torch::optim::LBFGSOptions()); + OLD_SERIALIZATION_LOGIC_WARNING_CHECK( + torch::load, optim1_2, optim_tempfile_old_format.name); + + const auto& params1_2_ = optim1_2.param_groups()[0].params(); + auto param_key = params1_2_[0].unsafeGetTensorImpl(); + auto& optim1_2_state = + static_cast(*(optim1_2.state().at(param_key).get())); + + // old LBFGS didn't track func_evals, n_iter, ro, al values + optim1_2_state.func_evals(optim1_state.func_evals()); + optim1_2_state.n_iter(optim1_state.n_iter()); + optim1_2_state.ro(optim1_state.ro()); + optim1_2_state.al(optim1_state.al()); + + is_optimizer_state_equal(optim1.state(), optim1_2.state()); +} + +TEST(SerializeTest, XOR_CUDA) { + torch::manual_seed(0); + // We better be able to save and load a XOR model! + auto getLoss = [](Sequential model, + uint32_t batch_size, + bool is_cuda = false) { + auto inputs = torch::empty({batch_size, 2}); + auto labels = torch::empty({batch_size}); + if (is_cuda) { + inputs = inputs.cuda(); + labels = labels.cuda(); + } + for (const auto i : c10::irange(batch_size)) { + inputs[i] = torch::randint(2, {2}, torch::kInt64); + labels[i] = inputs[i][0].item() ^ inputs[i][1].item(); + } + auto x = model->forward(inputs); + return torch::binary_cross_entropy(x, labels); + }; + + auto model = xor_model(); + auto model2 = xor_model(); + auto model3 = xor_model(); + auto optimizer = torch::optim::SGD( + model->parameters(), + torch::optim::SGDOptions(1e-1).momentum(0.9).nesterov(true).weight_decay( + 1e-6)); + + float running_loss = 1; + int epoch = 0; + while (running_loss > 0.1) { + torch::Tensor loss = getLoss(model, 4); + optimizer.zero_grad(); + loss.backward(); + optimizer.step(); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + running_loss = running_loss * 0.99 + loss.sum().item() * 0.01; + ASSERT_LT(epoch, 3000); + epoch++; + } + + auto tempfile = c10::make_tempfile(); + torch::save(model, tempfile.name); + torch::load(model2, tempfile.name); + + auto loss = getLoss(model2, 100); + ASSERT_LT(loss.item(), 0.1); + + model2->to(torch::kCUDA); + loss = getLoss(model2, 100, true); + ASSERT_LT(loss.item(), 0.1); + + auto tempfile2 = c10::make_tempfile(); + torch::save(model2, tempfile2.name); + torch::load(model3, tempfile2.name); + + loss = getLoss(model3, 100, true); + ASSERT_LT(loss.item(), 0.1); +} + +TEST( + SerializeTest, + CanSerializeModulesWithIntermediateModulesWithoutParametersOrBuffers) { + struct C : torch::nn::Module { + C() { + register_buffer("foo", torch::ones(5, torch::kInt32)); + } + }; + struct B : torch::nn::Module {}; + struct A : torch::nn::Module { + A() { + register_module("b", std::make_shared()); + register_module("c", std::make_shared()); + } + }; + struct M : torch::nn::Module { + M() { + register_module("a", std::make_shared()); + } + }; + + auto out = std::make_shared(); + std::stringstream ss; + torch::save(out, ss); + auto in = std::make_shared(); + torch::load(in, ss); + + const int output = in->named_buffers()["a.c.foo"].sum().item(); + ASSERT_EQ(output, 5); +} + +TEST(SerializeTest, VectorOfTensors) { + torch::manual_seed(0); + + std::vector x_vec = { + torch::randn({1, 2}), torch::randn({3, 4})}; + + std::stringstream stream; + torch::save(x_vec, stream); + + std::vector y_vec; + torch::load(y_vec, stream); + + for (const auto i : c10::irange(x_vec.size())) { + auto& x = x_vec[i]; + auto& y = y_vec[i]; + ASSERT_TRUE(y.defined()); + ASSERT_EQ(x.sizes().vec(), y.sizes().vec()); + ASSERT_TRUE(x.allclose(y)); + } +} + +TEST(SerializeTest, IValue) { + c10::IValue ivalue(1); + auto tempfile = c10::make_tempfile(); + torch::serialize::OutputArchive output_archive; + output_archive.write("value", ivalue); + output_archive.save_to(tempfile.name); + + torch::serialize::InputArchive input_archive; + input_archive.load_from(tempfile.name); + c10::IValue ivalue_out; + input_archive.read("value", ivalue_out); + ASSERT_EQ(ivalue_out.toInt(), 1); + + ASSERT_THROWS_WITH( + input_archive.read("bad_key", ivalue_out), + "does not have a field with name"); +} + +// NOTE: if a `Module` contains unserializable submodules (e.g. +// `nn::Functional`), we expect those submodules to be skipped when the `Module` +// is being serialized. +TEST(SerializeTest, UnserializableSubmoduleIsSkippedWhenSavingModule) { + struct A : torch::nn::Module { + A() { + register_module("relu", torch::nn::Functional(torch::relu)); + } + }; + + auto out = std::make_shared(); + std::stringstream ss; + torch::save(out, ss); + + torch::serialize::InputArchive archive; + archive.load_from(ss); + torch::serialize::InputArchive relu_archive; + + // Submodule with name "relu" should not exist in the `InputArchive`, + // because the "relu" submodule is an `nn::Functional` and is not + // serializable. + ASSERT_FALSE(archive.try_read("relu", relu_archive)); +} + +// NOTE: If a `Module` contains unserializable submodules (e.g. +// `nn::Functional`), we don't check the existence of those submodules in the +// `InputArchive` when deserializing. +TEST(SerializeTest, UnserializableSubmoduleIsIgnoredWhenLoadingModule) { + struct B : torch::nn::Module { + B() { + register_module("relu1", torch::nn::Functional(torch::relu)); + register_buffer("foo", torch::zeros(5, torch::kInt32)); + } + }; + struct A : torch::nn::Module { + A() { + register_module("b", std::make_shared()); + register_module("relu2", torch::nn::Functional(torch::relu)); + } + }; + + auto out = std::make_shared(); + // Manually change the values of "b.foo", so that we can check whether the + // buffer contains these values after deserialization. + out->named_buffers()["b.foo"].fill_(1); + auto tempfile = c10::make_tempfile(); + torch::save(out, tempfile.name); + + torch::serialize::InputArchive archive; + archive.load_from(tempfile.name); + torch::serialize::InputArchive archive_b; + torch::serialize::InputArchive archive_relu; + torch::Tensor tensor_foo; + + ASSERT_TRUE(archive.try_read("b", archive_b)); + ASSERT_TRUE(archive_b.try_read("foo", tensor_foo, /*is_buffer=*/true)); + + // Submodule with name "relu1" should not exist in `archive_b`, because the + // "relu1" submodule is an `nn::Functional` and is not serializable. + ASSERT_FALSE(archive_b.try_read("relu1", archive_relu)); + + // Submodule with name "relu2" should not exist in `archive`, because the + // "relu2" submodule is an `nn::Functional` and is not serializable. + ASSERT_FALSE(archive.try_read("relu2", archive_relu)); + + auto in = std::make_shared(); + // `torch::load(...)` works without error, even though `A` contains the + // `nn::Functional` submodules while the serialized file doesn't, because the + // `nn::Functional` submodules are not serializable and thus ignored when + // deserializing. + torch::load(in, tempfile.name); + + // Check that the "b.foo" buffer is correctly deserialized from the file. + const int output = in->named_buffers()["b.foo"].sum().item(); + // `output` should equal to the sum of the values we manually assigned to + // "b.foo" before serialization. + ASSERT_EQ(output, 5); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/special.cpp b/western_ai/usa/pytorch/test/cpp/api/special.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5850e27c1120abcd3d8eb4effaaa64be3e9b4e4c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/special.cpp @@ -0,0 +1,13 @@ +#include + +#include +#include + +#include + +// Simple test that verifies the special namespace is registered properly +// properly in C++ +TEST(SpecialTest, special) { + auto t = torch::randn(128, torch::kDouble); + torch::special::gammaln(t); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/static.cpp b/western_ai/usa/pytorch/test/cpp/api/static.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bdb3e85c96e2561a6b94818f3653e1bc879fd8b2 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/static.cpp @@ -0,0 +1,90 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include + +template < + typename T, + typename = std::enable_if_t::value>> +bool f(T&& m) { + return false; +} + +template +torch::detail::enable_if_module_t f(T&& m) { + return true; +} + +TEST(TestStatic, EnableIfModule) { + ASSERT_TRUE(f(torch::nn::LinearImpl(1, 2))); + ASSERT_FALSE(f(5)); + ASSERT_TRUE(torch::detail::check_not_lvalue_references()); + ASSERT_TRUE((torch::detail::check_not_lvalue_references())); + ASSERT_FALSE( + (torch::detail::check_not_lvalue_references())); + ASSERT_TRUE(torch::detail::check_not_lvalue_references()); + ASSERT_FALSE(torch::detail::check_not_lvalue_references()); +} + +namespace { + +struct A : torch::nn::Module { + int forward() { + return 5; + } +}; + +struct B : torch::nn::Module { + std::string forward(torch::Tensor tensor) { + return ""; + } +}; + +struct C : torch::nn::Module { + float forward(torch::Tensor& tensor) { + return 5.0; + } +}; + +struct D : torch::nn::Module { + char forward(torch::Tensor&& tensor) { + return 'x'; + } +}; + +struct E : torch::nn::Module {}; + +} // anonymous namespace + +// Put in a function because macros don't handle the comma between arguments to +// is_same well ... +template +void assert_has_expected_type() { + using ReturnType = + typename torch::detail::return_type_of_forward::type; + constexpr bool is_expected_type = std::is_same_v; + ASSERT_TRUE(is_expected_type) << Module().name(); +} + +TEST(TestStatic, ReturnTypeOfForward) { + assert_has_expected_type(); + assert_has_expected_type(); + assert_has_expected_type(); + assert_has_expected_type(); + assert_has_expected_type(); +} + +TEST(TestStatic, Apply) { + std::vector v; + torch::apply([&v](int x) { v.push_back(x); }, 1, 2, 3, 4, 5); + ASSERT_EQ(v.size(), 5); + for (const auto i : c10::irange(v.size())) { + ASSERT_EQ(v.at(i), i + 1); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/support.cpp b/western_ai/usa/pytorch/test/cpp/api/support.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9d34ae48b050a01e90ed264d1adb6c6f11979f77 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/support.cpp @@ -0,0 +1,9 @@ +#include + +namespace torch { +namespace test { + +std::mutex AutoDefaultDtypeMode::default_dtype_mutex; + +} // namespace test +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/api/support.h b/western_ai/usa/pytorch/test/cpp/api/support.h new file mode 100644 index 0000000000000000000000000000000000000000..c907dff90c2a03ad0845e115312c177a064ffc5f --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/support.h @@ -0,0 +1,196 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace torch { +namespace test { + +// Lets you use a container without making a new class, +// for experimental implementations +class SimpleContainer : public nn::Cloneable { + public: + void reset() override {} + + template + ModuleHolder add( + ModuleHolder module_holder, + std::string name = std::string()) { + return Module::register_module(std::move(name), module_holder); + } +}; + +struct SeedingFixture : public ::testing::Test { + SeedingFixture() { + torch::manual_seed(0); + } +}; + +struct WarningCapture : public WarningHandler { + WarningCapture() : prev_(WarningUtils::get_warning_handler()) { + WarningUtils::set_warning_handler(this); + } + + ~WarningCapture() override { + WarningUtils::set_warning_handler(prev_); + } + + const std::vector& messages() { + return messages_; + } + + std::string str() { + return c10::Join("\n", messages_); + } + + void process(const c10::Warning& warning) override { + messages_.push_back(warning.msg()); + } + + private: + WarningHandler* prev_; + std::vector messages_; +}; + +inline bool pointer_equal(at::Tensor first, at::Tensor second) { + return first.data_ptr() == second.data_ptr(); +} + +// This mirrors the `isinstance(x, torch.Tensor) and isinstance(y, +// torch.Tensor)` branch in `TestCase.assertEqual` in +// torch/testing/_internal/common_utils.py +inline void assert_tensor_equal( + at::Tensor a, + at::Tensor b, + bool allow_inf = false) { + ASSERT_TRUE(a.sizes() == b.sizes()); + if (a.numel() > 0) { + if (a.device().type() == torch::kCPU && + (a.scalar_type() == torch::kFloat16 || + a.scalar_type() == torch::kBFloat16)) { + // CPU half and bfloat16 tensors don't have the methods we need below + a = a.to(torch::kFloat32); + } + if (a.device().type() == torch::kCUDA && + a.scalar_type() == torch::kBFloat16) { + // CUDA bfloat16 tensors don't have the methods we need below + a = a.to(torch::kFloat32); + } + b = b.to(a); + + if ((a.scalar_type() == torch::kBool) != + (b.scalar_type() == torch::kBool)) { + TORCH_CHECK(false, "Was expecting both tensors to be bool type."); + } else { + if (a.scalar_type() == torch::kBool && b.scalar_type() == torch::kBool) { + // we want to respect precision but as bool doesn't support subtraction, + // boolean tensor has to be converted to int + a = a.to(torch::kInt); + b = b.to(torch::kInt); + } + + auto diff = a - b; + if (a.is_floating_point()) { + // check that NaNs are in the same locations + auto nan_mask = torch::isnan(a); + ASSERT_TRUE(torch::equal(nan_mask, torch::isnan(b))); + diff.index_put_({nan_mask}, 0); + // inf check if allow_inf=true + if (allow_inf) { + auto inf_mask = torch::isinf(a); + auto inf_sign = inf_mask.sign(); + ASSERT_TRUE(torch::equal(inf_sign, torch::isinf(b).sign())); + diff.index_put_({inf_mask}, 0); + } + } + // TODO: implement abs on CharTensor (int8) + if (diff.is_signed() && diff.scalar_type() != torch::kInt8) { + diff = diff.abs(); + } + auto max_err = diff.max().item(); + ASSERT_LE(max_err, 1e-5); + } + } +} + +// This mirrors the `isinstance(x, torch.Tensor) and isinstance(y, +// torch.Tensor)` branch in `TestCase.assertNotEqual` in +// torch/testing/_internal/common_utils.py +inline void assert_tensor_not_equal(at::Tensor x, at::Tensor y) { + if (x.sizes() != y.sizes()) { + return; + } + ASSERT_GT(x.numel(), 0); + y = y.type_as(x); + y = x.is_cuda() ? y.to({torch::kCUDA, x.get_device()}) : y.cpu(); + auto nan_mask = x != x; + if (torch::equal(nan_mask, y != y)) { + auto diff = x - y; + if (diff.is_signed()) { + diff = diff.abs(); + } + diff.index_put_({nan_mask}, 0); + // Use `item()` to work around: + // https://github.com/pytorch/pytorch/issues/22301 + auto max_err = diff.max().item(); + ASSERT_GE(max_err, 1e-5); + } +} + +inline int count_substr_occurrences( + const std::string& str, + const std::string& substr) { + int count = 0; + size_t pos = str.find(substr); + + while (pos != std::string::npos) { + count++; + pos = str.find(substr, pos + substr.size()); + } + + return count; +} + +// A RAII, thread local (!) guard that changes default dtype upon +// construction, and sets it back to the original dtype upon destruction. +// +// Usage of this guard is synchronized across threads, so that at any given +// time, only one guard can take effect. +struct AutoDefaultDtypeMode { + static std::mutex default_dtype_mutex; + + AutoDefaultDtypeMode(c10::ScalarType default_dtype) + : prev_default_dtype( + torch::typeMetaToScalarType(torch::get_default_dtype())) { + default_dtype_mutex.lock(); + torch::set_default_dtype(torch::scalarTypeToTypeMeta(default_dtype)); + } + ~AutoDefaultDtypeMode() { + default_dtype_mutex.unlock(); + torch::set_default_dtype(torch::scalarTypeToTypeMeta(prev_default_dtype)); + } + c10::ScalarType prev_default_dtype; +}; + +inline void assert_tensor_creation_meta( + torch::Tensor& x, + torch::autograd::CreationMeta creation_meta) { + auto autograd_meta = x.unsafeGetTensorImpl()->autograd_meta(); + TORCH_CHECK(autograd_meta); + auto view_meta = + static_cast(autograd_meta); + TORCH_CHECK(view_meta->has_bw_view()); + ASSERT_EQ(view_meta->get_creation_meta(), creation_meta); +} +} // namespace test +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/api/tensor.cpp b/western_ai/usa/pytorch/test/cpp/api/tensor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..77edac857449fcddf052570240c562b55276bc6e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/tensor.cpp @@ -0,0 +1,1270 @@ +#include +#include + +#include +#include + +#include +#include +#include + +#include + +using namespace torch::test; + +template +bool exactly_equal(at::Tensor left, T right) { + return left.item() == right; +} + +template +bool almost_equal(at::Tensor left, T right, double tolerance = 1e-4) { + return std::abs(left.item() - right) < tolerance; +} + +#define REQUIRE_TENSOR_OPTIONS(device_, index_, type_, layout_) \ + ASSERT_TRUE( \ + tensor.device().type() == at::Device((device_), (index_)).type()); \ + ASSERT_TRUE( \ + tensor.device().index() == at::Device((device_), (index_)).index()); \ + ASSERT_EQ(tensor.dtype(), (type_)); \ + ASSERT_TRUE(tensor.layout() == (layout_)) + +TEST(TensorTest, ToDtype) { + auto tensor = at::empty({3, 4}); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kStrided); + + tensor = tensor.to(at::kInt); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kInt, at::kStrided); + + tensor = tensor.to(at::kChar); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kChar, at::kStrided); + + tensor = tensor.to(at::kDouble); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kDouble, at::kStrided); + + tensor = tensor.to(at::TensorOptions(at::kInt)); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kInt, at::kStrided); + + tensor = tensor.to(at::TensorOptions(at::kChar)); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kChar, at::kStrided); + + tensor = tensor.to(at::TensorOptions(at::kDouble)); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kDouble, at::kStrided); +} + +TEST(TensorTest, ToTensorAndTensorAttributes) { + auto tensor = at::empty({3, 4}); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kStrided); + + auto other = at::empty({3, 4}, at::kInt); + tensor = tensor.to(other); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kInt, at::kStrided); + + other = at::empty({3, 4}, at::kDouble); + tensor = tensor.to(other.dtype()); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kDouble, at::kStrided); + tensor = tensor.to(other.device()); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kDouble, at::kStrided); + + other = at::empty({3, 4}, at::kLong); + tensor = tensor.to(other.device(), other.dtype()); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kLong, at::kStrided); + + other = at::empty({3, 4}, at::kInt); + tensor = tensor.to(other.options()); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kInt, at::kStrided); +} + +TEST(TensorTest, TensorAttributes) { + std::vector v = {1.0, 2.0, 3.0}; + auto options = c10::requires_grad(true); + auto tensor = at::from_blob(v.data(), {3}, options); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kStrided); + ASSERT_TRUE(tensor.requires_grad()); +} + +// Not currently supported. +// TEST(TensorTest, ToLayout) { +// auto tensor = at::empty({3, 4}); +// REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kStrided); +// +// tensor = tensor.to(at::kSparse); +// REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kSparse); +// +// tensor = tensor.to(at::kStrided); +// REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kStrided); +// } + +TEST(TensorTest, ToOptionsWithRequiresGrad) { + { + // Respects requires_grad + auto tensor = torch::empty({3, 4}, at::requires_grad()); + ASSERT_TRUE(tensor.requires_grad()); + + tensor = tensor.to(at::kDouble); + ASSERT_TRUE(tensor.requires_grad()); + + // Throws if requires_grad is set in TensorOptions + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW( + tensor.to(at::TensorOptions().requires_grad(true)), c10::Error); + + // Doesn't throw if requires_grad is not set + tensor.to(at::TensorOptions()); + tensor.to(at::TensorOptions().requires_grad(false)); + } + { + auto tensor = torch::empty({3, 4}); + ASSERT_FALSE(tensor.requires_grad()); + + // Respects requires_grad + tensor = tensor.to(at::kDouble); + ASSERT_FALSE(tensor.requires_grad()); + + // Throws if requires_grad is set in TensorOptions + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW( + tensor.to(at::TensorOptions().requires_grad(true)), c10::Error); + + // Doesn't throw if requires_grad is not set + tensor.to(at::TensorOptions()); + tensor.to(at::TensorOptions().requires_grad(false)); + } +} + +TEST(TensorTest, ToDoesNotCopyWhenOptionsAreAllTheSame) { + { + auto tensor = at::empty({3, 4}, at::kFloat); + auto hopefully_not_copy = tensor.to(at::kFloat); + ASSERT_EQ(hopefully_not_copy.data_ptr(), tensor.data_ptr()); + } + { + auto tensor = at::empty({3, 4}, at::kFloat); + auto hopefully_not_copy = tensor.to(tensor.options()); + ASSERT_EQ(hopefully_not_copy.data_ptr(), tensor.data_ptr()); + } + { + auto tensor = at::empty({3, 4}, at::kFloat); + auto hopefully_not_copy = tensor.to(tensor.dtype()); + ASSERT_EQ(hopefully_not_copy.data_ptr(), tensor.data_ptr()); + } + { + auto tensor = at::empty({3, 4}, at::kFloat); + auto hopefully_not_copy = tensor.to(tensor.device()); + ASSERT_EQ(hopefully_not_copy.data_ptr(), tensor.data_ptr()); + } + { + auto tensor = at::empty({3, 4}, at::kFloat); + auto hopefully_not_copy = tensor.to(tensor); + ASSERT_EQ(hopefully_not_copy.data_ptr(), tensor.data_ptr()); + } +} + +TEST(TensorTest, AtTensorCtorScalar) { + auto tensor = at::tensor(123); + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.dtype(), at::kInt); + ASSERT_EQ(tensor[0].item(), 123); + + tensor = at::tensor(123.456f); + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.dtype(), at::kFloat); + ASSERT_TRUE(almost_equal(tensor[0], 123.456f)); + + tensor = at::tensor(123.456); + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.dtype(), at::kDouble); + ASSERT_TRUE(almost_equal(tensor[0], 123.456)); + + tensor = at::tensor(123, at::dtype(at::kFloat)) + 0.5; + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.dtype(), at::kFloat); + ASSERT_TRUE(almost_equal(tensor[0], 123.5)); + + tensor = at::tensor(c10::complex(1.0, 2.0)) + 0.5; + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.dtype(), at::kComplexFloat); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5, 2.0))); + + tensor = + at::tensor(c10::complex(1.0, 2.0), at::dtype(at::kComplexFloat)) + + 0.5; + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.dtype(), at::kComplexFloat); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5, 2.0))); + + tensor = at::tensor(c10::complex(1.0, 2.0)) + 0.5; + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.dtype(), at::kComplexDouble); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5, 2.0))); + + tensor = + at::tensor(c10::complex(1.0, 2.0), at::dtype(at::kComplexDouble)) + + 0.5; + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.dtype(), at::kComplexDouble); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5, 2.0))); +} + +TEST(TensorTest, AtTensorCtorSingleDim) { + auto tensor = at::tensor({1, 2, 3}); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), at::kInt); + ASSERT_TRUE(exactly_equal(tensor[0], 1)); + ASSERT_TRUE(exactly_equal(tensor[1], 2)); + ASSERT_TRUE(exactly_equal(tensor[2], 3)); + + tensor = at::tensor(std::vector({1, 2, 3})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), at::kInt); + ASSERT_TRUE(exactly_equal(tensor[0], 1)); + ASSERT_TRUE(exactly_equal(tensor[1], 2)); + ASSERT_TRUE(exactly_equal(tensor[2], 3)); + + tensor = at::tensor({1.5, 2.25, 3.125}); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), at::kDouble); + ASSERT_TRUE(almost_equal(tensor[0], 1.5)); + ASSERT_TRUE(almost_equal(tensor[1], 2.25)); + ASSERT_TRUE(almost_equal(tensor[2], 3.125)); + + tensor = at::tensor( + {c10::complex(1.5, 0.15), + c10::complex(1.5, 0.15), + c10::complex(3.125, 0.3125)}); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), at::kComplexFloat); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5, 0.15))); + ASSERT_TRUE(almost_equal(tensor[1], c10::complex(1.5, 0.15))); + ASSERT_TRUE(almost_equal(tensor[2], c10::complex(3.125, 0.3125))); + + tensor = at::tensor( + {c10::complex(1.5, 0.15), + c10::complex(1.5, 0.15), + c10::complex(3.125, 0.3125)}); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), at::kComplexDouble); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5, 0.15))); + ASSERT_TRUE(almost_equal(tensor[1], c10::complex(1.5, 0.15))); + ASSERT_TRUE(almost_equal(tensor[2], c10::complex(3.125, 0.3125))); + + tensor = at::tensor({1.1, 2.2, 3.3}, at::dtype(at::kInt)); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), at::kInt); + ASSERT_EQ(tensor.layout(), at::kStrided); + ASSERT_TRUE(exactly_equal(tensor[0], 1)); + ASSERT_TRUE(exactly_equal(tensor[1], 2)); + ASSERT_TRUE(exactly_equal(tensor[2], 3)); + + tensor = at::tensor(std::vector({1.5, 2.25, 3.125})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), at::kDouble); + ASSERT_TRUE(almost_equal(tensor[0], 1.5)); + ASSERT_TRUE(almost_equal(tensor[1], 2.25)); + ASSERT_TRUE(almost_equal(tensor[2], 3.125)); + + tensor = at::tensor(std::vector>( + {c10::complex(1.5, 0.15), + c10::complex(1.5, 0.15), + c10::complex(3.125, 0.3125)})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), at::kComplexFloat); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5, 0.15))); + ASSERT_TRUE(almost_equal(tensor[1], c10::complex(1.5, 0.15))); + ASSERT_TRUE(almost_equal(tensor[2], c10::complex(3.125, 0.3125))); + + tensor = at::tensor(std::vector>( + {c10::complex(1.5, 0.15), + c10::complex(1.5, 0.15), + c10::complex(3.125, 0.3125)})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), at::kComplexDouble); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5, 0.15))); + ASSERT_TRUE(almost_equal(tensor[1], c10::complex(1.5, 0.15))); + ASSERT_TRUE(almost_equal(tensor[2], c10::complex(3.125, 0.3125))); + + std::vector v = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + tensor = at::tensor(v); + ASSERT_EQ(tensor.numel(), v.size()); + ASSERT_EQ(tensor.dtype(), at::kInt); + for (const auto i : c10::irange(v.size())) { + ASSERT_TRUE(exactly_equal(tensor[i], v.at(i))); + } + + std::vector w = {1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 10.0}; + tensor = at::tensor(w); + ASSERT_EQ(tensor.numel(), w.size()); + ASSERT_EQ(tensor.dtype(), at::kDouble); + for (const auto i : c10::irange(w.size())) { + ASSERT_TRUE(almost_equal(tensor[i], w.at(i))); + } + + std::vector> x = { + {1.1, -1.1}, + {2.2, -2.2}, + {3.3, -3.3}, + {4.4, -4.4}, + {5.5, -5.5}, + {6.6, -6.6}, + {7.7, -7.7}, + {8.8, -8.8}, + {9.9, -9.9}, + {10.0, -10.0}}; + tensor = at::tensor(x); + ASSERT_EQ(tensor.numel(), x.size()); + ASSERT_EQ(tensor.dtype(), at::kComplexDouble); + for (const auto i : c10::irange(x.size())) { + ASSERT_TRUE(almost_equal(tensor[i], x.at(i))); + } +} + +TEST(TensorTest, AtTensorCastRealToComplex) { + auto tensor = + at::tensor(std::vector({1.5, 2.5, 3.5}), at::kComplexDouble); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), at::kComplexDouble); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5))); + ASSERT_TRUE(almost_equal(tensor[1], c10::complex(2.5))); + ASSERT_TRUE(almost_equal(tensor[2], c10::complex(3.5))); + + tensor = at::tensor({1.5, 2.5, 3.5}, at::kComplexDouble); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), at::kComplexDouble); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5))); + ASSERT_TRUE(almost_equal(tensor[1], c10::complex(2.5))); + ASSERT_TRUE(almost_equal(tensor[2], c10::complex(3.5))); + + tensor = at::tensor(1.5, at::kComplexDouble); + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.dtype(), at::kComplexDouble); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5))); +} + +TEST(TensorTest, AtTensorCastComplexToRealErrorChecks) { + { + ASSERT_THROWS_WITH( + at::tensor(c10::complex(0.1, 0.2), at::kFloat), + "\"tensor_cpu\" not implemented for 'Float'"); + } + { + ASSERT_THROWS_WITH( + at::tensor({c10::complex(0.1, 0.2)}, at::kFloat), + "\"tensor_cpu\" not implemented for 'Float'"); + } + { + ASSERT_THROWS_WITH( + at::tensor( + std::vector>{c10::complex(0.1, 0.2)}, + at::kFloat), + "\"tensor_cpu\" not implemented for 'Float'"); + } +} + +TEST(TensorTest, TorchTensorCtorScalarIntegralType) { + auto tensor = torch::tensor(123); + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.sizes(), std::vector({})); + ASSERT_EQ(tensor.dtype(), at::kLong); + ASSERT_EQ(tensor.item(), 123); +} + +void test_TorchTensorCtorScalarFloatingType_expected_dtype( + c10::ScalarType default_dtype) { + AutoDefaultDtypeMode dtype_mode(default_dtype); + + auto tensor = torch::tensor(123.456f); + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.sizes(), std::vector({})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_TRUE(almost_equal(tensor, 123.456f)); + + tensor = torch::tensor(123.456); + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.sizes(), std::vector({})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_TRUE(almost_equal(tensor, 123.456)); + + tensor = torch::tensor({123.456}); + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.sizes(), std::vector({1})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_TRUE(almost_equal(tensor[0], 123.456)); +} + +TEST(TensorTest, TorchTensorCtorScalarFloatingType) { + test_TorchTensorCtorScalarFloatingType_expected_dtype( + /*default_dtype=*/torch::kFloat); + test_TorchTensorCtorScalarFloatingType_expected_dtype( + /*default_dtype=*/torch::kDouble); +} + +TEST(TensorTest, TorchTensorCtorScalarBoolType) { + auto tensor = torch::tensor(true); + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.sizes(), std::vector({})); + ASSERT_EQ(tensor.dtype(), at::kBool); + ASSERT_TRUE(exactly_equal(tensor, true)); + + tensor = torch::tensor({true}); + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.sizes(), std::vector({1})); + ASSERT_EQ(tensor.dtype(), at::kBool); + ASSERT_TRUE(exactly_equal(tensor[0], true)); +} + +TEST(TensorTest, TorchTensorCtorSingleDimIntegralType) { + auto tensor = torch::tensor({1, 2, 3}); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.sizes(), std::vector({3})); + ASSERT_EQ(tensor.dtype(), at::kLong); + ASSERT_TRUE(exactly_equal(tensor[0], 1)); + ASSERT_TRUE(exactly_equal(tensor[1], 2)); + ASSERT_TRUE(exactly_equal(tensor[2], 3)); + + tensor = torch::tensor(at::ArrayRef({1, 2, 3})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.sizes(), std::vector({3})); + ASSERT_EQ(tensor.dtype(), at::kLong); + ASSERT_TRUE(exactly_equal(tensor[0], 1)); + ASSERT_TRUE(exactly_equal(tensor[1], 2)); + ASSERT_TRUE(exactly_equal(tensor[2], 3)); + + tensor = torch::tensor(std::vector({1, 2, 3})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.sizes(), std::vector({3})); + ASSERT_EQ(tensor.dtype(), at::kLong); + ASSERT_TRUE(exactly_equal(tensor[0], 1)); + ASSERT_TRUE(exactly_equal(tensor[1], 2)); + ASSERT_TRUE(exactly_equal(tensor[2], 3)); + + tensor = torch::tensor(at::ArrayRef({1, 2, 3})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.sizes(), std::vector({3})); + ASSERT_EQ(tensor.dtype(), at::kLong); + ASSERT_TRUE(exactly_equal(tensor[0], 1)); + ASSERT_TRUE(exactly_equal(tensor[1], 2)); + ASSERT_TRUE(exactly_equal(tensor[2], 3)); + + tensor = torch::tensor(std::vector({1, 2, 3})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.sizes(), std::vector({3})); + ASSERT_EQ(tensor.dtype(), at::kLong); + ASSERT_TRUE(exactly_equal(tensor[0], 1)); + ASSERT_TRUE(exactly_equal(tensor[1], 2)); + ASSERT_TRUE(exactly_equal(tensor[2], 3)); +} + +void test_TorchTensorCtorSingleDimFloatingType_expected_dtype( + c10::ScalarType default_dtype) { + AutoDefaultDtypeMode dtype_mode(default_dtype); + + auto tensor = torch::tensor({1.5, 2.25, 3.125}); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.sizes(), std::vector({3})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_TRUE(almost_equal(tensor[0], 1.5)); + ASSERT_TRUE(almost_equal(tensor[1], 2.25)); + ASSERT_TRUE(almost_equal(tensor[2], 3.125)); + + tensor = torch::tensor({1.5f, 2.25f, 3.125f}); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.sizes(), std::vector({3})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_TRUE(almost_equal(tensor[0], 1.5f)); + ASSERT_TRUE(almost_equal(tensor[1], 2.25f)); + ASSERT_TRUE(almost_equal(tensor[2], 3.125f)); + + tensor = torch::tensor(at::ArrayRef({1.5f, 2.25f, 3.125f})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_TRUE(almost_equal(tensor[0], 1.5)); + ASSERT_TRUE(almost_equal(tensor[1], 2.25)); + ASSERT_TRUE(almost_equal(tensor[2], 3.125)); + + tensor = torch::tensor(std::vector({1.5f, 2.25f, 3.125f})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.sizes(), std::vector({3})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_TRUE(almost_equal(tensor[0], 1.5)); + ASSERT_TRUE(almost_equal(tensor[1], 2.25)); + ASSERT_TRUE(almost_equal(tensor[2], 3.125)); + + tensor = torch::tensor(at::ArrayRef({1.5, 2.25, 3.125})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_TRUE(almost_equal(tensor[0], 1.5)); + ASSERT_TRUE(almost_equal(tensor[1], 2.25)); + ASSERT_TRUE(almost_equal(tensor[2], 3.125)); + + tensor = torch::tensor(std::vector({1.5, 2.25, 3.125})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.sizes(), std::vector({3})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_TRUE(almost_equal(tensor[0], 1.5)); + ASSERT_TRUE(almost_equal(tensor[1], 2.25)); + ASSERT_TRUE(almost_equal(tensor[2], 3.125)); +} + +TEST(TensorTest, TorchTensorCtorSingleDimFloatingType) { + test_TorchTensorCtorSingleDimFloatingType_expected_dtype( + /*default_dtype=*/torch::kFloat); + test_TorchTensorCtorSingleDimFloatingType_expected_dtype( + /*default_dtype=*/torch::kDouble); +} + +TEST(TensorTest, TorchTensorCtorSingleDimBoolType) { + auto tensor = torch::tensor({true, false, true}); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.sizes(), std::vector({3})); + ASSERT_EQ(tensor.dtype(), at::kBool); + ASSERT_TRUE(exactly_equal(tensor[0], true)); + ASSERT_TRUE(exactly_equal(tensor[1], false)); + ASSERT_TRUE(exactly_equal(tensor[2], true)); + + tensor = torch::tensor(at::ArrayRef({true, false, true})); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.sizes(), std::vector({3})); + ASSERT_EQ(tensor.dtype(), at::kBool); + ASSERT_TRUE(exactly_equal(tensor[0], true)); + ASSERT_TRUE(exactly_equal(tensor[1], false)); + ASSERT_TRUE(exactly_equal(tensor[2], true)); +} + +TEST(TensorTest, TorchTensorCtorMultiDimIntegralType) { + { + auto tensor = torch::tensor({{1, 2}}); + ASSERT_EQ(tensor.dtype(), torch::kLong); + ASSERT_EQ(tensor.sizes(), std::vector({1, 2})); + ASSERT_TRUE(torch::allclose( + tensor, torch::arange(1, 3, torch::kLong).view(tensor.sizes()))); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{1}, {2}}); + ASSERT_EQ(tensor.dtype(), torch::kLong); + ASSERT_EQ(tensor.sizes(), std::vector({2, 1})); + ASSERT_TRUE(torch::allclose( + tensor, torch::arange(1, 3, torch::kLong).view(tensor.sizes()))); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{{1, 2}}}); + ASSERT_EQ(tensor.dtype(), torch::kLong); + ASSERT_EQ(tensor.sizes(), std::vector({1, 1, 2})); + ASSERT_TRUE(torch::allclose( + tensor, torch::arange(1, 3, torch::kLong).view(tensor.sizes()))); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{{1}, {2}}}); + ASSERT_EQ(tensor.dtype(), torch::kLong); + ASSERT_EQ(tensor.sizes(), std::vector({1, 2, 1})); + ASSERT_TRUE(torch::allclose( + tensor, torch::arange(1, 3, torch::kLong).view(tensor.sizes()))); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{1, 2}, {3, 4}}); + ASSERT_EQ(tensor.dtype(), torch::kLong); + ASSERT_EQ(tensor.sizes(), std::vector({2, 2})); + ASSERT_TRUE(torch::allclose( + tensor, torch::arange(1, 5, torch::kLong).view(tensor.sizes()))); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{{{{{{{{{1}}}}}}}}}}); + ASSERT_EQ(tensor.dtype(), torch::kLong); + ASSERT_EQ( + tensor.sizes(), std::vector({1, 1, 1, 1, 1, 1, 1, 1, 1, 1})); + ASSERT_TRUE(torch::allclose( + tensor, torch::full({1}, 1, torch::kLong).view(tensor.sizes()))); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{{{{{{{{{1, 2}}}}}}}}}}); + ASSERT_EQ(tensor.dtype(), torch::kLong); + ASSERT_EQ( + tensor.sizes(), std::vector({1, 1, 1, 1, 1, 1, 1, 1, 1, 2})); + ASSERT_TRUE(torch::allclose( + tensor, torch::arange(1, 3, torch::kLong).view(tensor.sizes()))); + ASSERT_FALSE(tensor.requires_grad()); + } +} + +void test_TorchTensorCtorMultiDimFloatingType_expected_dtype( + c10::ScalarType default_dtype) { + AutoDefaultDtypeMode dtype_mode(default_dtype); + { + auto tensor = torch::tensor({{1.0, 2.0}}); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_EQ(tensor.sizes(), std::vector({1, 2})); + ASSERT_TRUE(torch::allclose( + tensor, torch::arange(1, 3, default_dtype).view(tensor.sizes()))); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor( + {{{{{{{{1.0, 2.0, 3.0}}}}}, + {{{{{4.0, 5.0, 6.0}}}}}, + {{{{{7.0, 8.0, 9.0}}}}}}}}); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_EQ(tensor.sizes(), std::vector({1, 1, 3, 1, 1, 1, 1, 3})); + ASSERT_TRUE(torch::allclose( + tensor, torch::arange(1, 10, default_dtype).view(tensor.sizes()))); + ASSERT_FALSE(tensor.requires_grad()); + } +} + +TEST(TensorTest, TorchTensorCtorMultiDimFloatingType) { + test_TorchTensorCtorMultiDimFloatingType_expected_dtype( + /*default_dtype=*/torch::kFloat); + test_TorchTensorCtorMultiDimFloatingType_expected_dtype( + /*default_dtype=*/torch::kDouble); +} + +TEST(TensorTest, TorchTensorCtorMultiDimBoolType) { + { + auto tensor = torch::tensor({{true, false}}); + ASSERT_EQ(tensor.dtype(), torch::kBool); + ASSERT_EQ(tensor.sizes(), std::vector({1, 2})); + auto expected = torch::empty(tensor.sizes(), torch::kBool); + expected[0][0] = true; + expected[0][1] = false; + ASSERT_TRUE(torch::equal(tensor, expected)); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{true}, {false}}); + ASSERT_EQ(tensor.dtype(), torch::kBool); + ASSERT_EQ(tensor.sizes(), std::vector({2, 1})); + auto expected = torch::empty(tensor.sizes(), torch::kBool); + expected[0][0] = true; + expected[1][0] = false; + ASSERT_TRUE(torch::equal(tensor, expected)); + ASSERT_FALSE(tensor.requires_grad()); + } +} + +TEST(TensorTest, TorchTensorCtorMultiDimWithOptions) { + { + auto tensor = torch::tensor({{1, 2}}, torch::dtype(torch::kInt)); + ASSERT_EQ(tensor.dtype(), torch::kInt); + ASSERT_EQ(tensor.sizes(), std::vector({1, 2})); + ASSERT_TRUE(torch::allclose( + tensor, torch::arange(1, 3, torch::kInt).view(tensor.sizes()))); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor( + {{1, 2}, {3, 4}}, torch::dtype(torch::kFloat).requires_grad(true)); + ASSERT_EQ(tensor.dtype(), torch::kFloat); + ASSERT_EQ(tensor.sizes(), std::vector({2, 2})); + ASSERT_TRUE(torch::allclose( + tensor, torch::arange(1, 5, torch::kFloat).view(tensor.sizes()))); + ASSERT_TRUE(tensor.requires_grad()); + } +} + +TEST(TensorTest, TorchTensorCtorMultiDimErrorChecks) { + { + ASSERT_THROWS_WITH( + torch::tensor({{{2, 3, 4}, {{5, 6}, {7}}}}), + "Expected all sub-lists to have sizes: 2 (e.g. {5, 6}), but got sub-list {7} with sizes: 1"); + } + { + ASSERT_THROWS_WITH( + torch::tensor({{{1, 2.0}, {1, 2.0}}}), + "Expected all elements of the tensor to have the same scalar type: Int, but got element of scalar type: Double"); + } + { + ASSERT_THROWS_WITH( + torch::tensor({{{true, 2.0, 3}, {true, 2.0, 3}}}), + "Expected all elements of the tensor to have the same scalar type: Bool, but got element of scalar type: Double"); + } + { + ASSERT_THROWS_WITH( + torch::tensor({{{true}, {2}}}), + "Expected all elements of the tensor to have the same scalar type: Bool, but got element of scalar type: Int"); + } + { + ASSERT_THROWS_WITH( + torch::tensor({{{true, 2}}}), + "Expected all elements of the tensor to have the same scalar type: Bool, but got element of scalar type: Int"); + } +} + +TEST(TensorTest, TorchTensorCastRealToComplex) { + auto tensor = torch::tensor( + std::vector({1.5, 2.5, 3.5}), torch::kComplexDouble); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), torch::kComplexDouble); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5))); + ASSERT_TRUE(almost_equal(tensor[1], c10::complex(2.5))); + ASSERT_TRUE(almost_equal(tensor[2], c10::complex(3.5))); + + tensor = torch::tensor({1.5, 2.5, 3.5}, torch::kComplexDouble); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor.dtype(), torch::kComplexDouble); + ASSERT_TRUE(almost_equal(tensor[0], c10::complex(1.5))); + ASSERT_TRUE(almost_equal(tensor[1], c10::complex(2.5))); + ASSERT_TRUE(almost_equal(tensor[2], c10::complex(3.5))); + + tensor = torch::tensor(1.5, torch::kComplexDouble); + ASSERT_EQ(tensor.numel(), 1); + ASSERT_EQ(tensor.dtype(), torch::kComplexDouble); + ASSERT_TRUE(almost_equal(tensor, c10::complex(1.5))); +} + +TEST(TensorTest, TorchTensorCastComplexToRealErrorChecks) { + { + ASSERT_THROWS_WITH( + torch::tensor(c10::complex(0.1, 0.2), torch::kFloat), + "value cannot be converted to type float without overflow"); + } + { + ASSERT_THROWS_WITH( + torch::tensor( + {c10::complex(0.1, 0.2), c10::complex(0.3, 0.4)}, + torch::kFloat), + "value cannot be converted to type float without overflow"); + } + { + ASSERT_THROWS_WITH( + torch::tensor( + std::vector>{ + c10::complex(0.1, 0.2), c10::complex(0.3, 0.4)}, + torch::kFloat), + "can not do torch::tensor(complex, dtype=non-complex) because complex can not be casted to real number without loss of information"); + } +} + +void test_TorchTensorCtorMultiDim_CUDA_expected_dtype( + c10::ScalarType default_dtype) { + AutoDefaultDtypeMode dtype_mode(default_dtype); + + auto tensor = torch::tensor( + {{{{{{{{1.0, 2.0, 3.0}}}}}, + {{{{{4.0, 5.0, 6.0}}}}}, + {{{{{7.0, 8.0, 9.0}}}}}}}}, + torch::dtype(default_dtype).device(torch::kCUDA)); + ASSERT_TRUE(tensor.device().is_cuda()); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_EQ(tensor.sizes(), std::vector({1, 1, 3, 1, 1, 1, 1, 3})); + ASSERT_TRUE(torch::allclose( + tensor, + torch::arange(1, 10, default_dtype) + .view(tensor.sizes()) + .to(torch::kCUDA))); + ASSERT_FALSE(tensor.requires_grad()); +} + +TEST(TensorTest, TorchTensorCtorMultiDim_CUDA) { + test_TorchTensorCtorMultiDim_CUDA_expected_dtype( + /*default_dtype=*/torch::kFloat); + test_TorchTensorCtorMultiDim_CUDA_expected_dtype( + /*default_dtype=*/torch::kDouble); +} + +void test_TorchTensorCtorZeroSizedDim_expected_dtype( + c10::ScalarType default_dtype) { + AutoDefaultDtypeMode dtype_mode(default_dtype); + { + auto tensor = torch::tensor({}); + ASSERT_EQ(tensor.numel(), 0); + ASSERT_EQ(tensor.sizes(), std::vector({0})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{}, {}}); + ASSERT_EQ(tensor.numel(), 0); + ASSERT_EQ(tensor.sizes(), std::vector({2, 0})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{{}, {}}}); + ASSERT_EQ(tensor.numel(), 0); + ASSERT_EQ(tensor.sizes(), std::vector({1, 2, 0})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{{}}}); + ASSERT_EQ(tensor.numel(), 0); + ASSERT_EQ(tensor.sizes(), std::vector({1, 1, 0})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{{{{{{{}}}}}}}}); + ASSERT_EQ(tensor.numel(), 0); + ASSERT_EQ(tensor.sizes(), std::vector({1, 1, 1, 1, 1, 1, 1, 0})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{{{{{{{}}}}, {{{{}}}}}}}}); + ASSERT_EQ(tensor.numel(), 0); + ASSERT_EQ(tensor.sizes(), std::vector({1, 1, 1, 2, 1, 1, 1, 0})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_FALSE(tensor.requires_grad()); + } + { + auto tensor = torch::tensor({{{{{{{{{{}}}}}}}}}}); + ASSERT_EQ(tensor.numel(), 0); + ASSERT_EQ( + tensor.sizes(), std::vector({1, 1, 1, 1, 1, 1, 1, 1, 1, 0})); + ASSERT_EQ(tensor.dtype(), default_dtype); + ASSERT_FALSE(tensor.requires_grad()); + } +} + +TEST(TensorTest, TorchTensorCtorZeroSizedDim) { + test_TorchTensorCtorZeroSizedDim_expected_dtype( + /*default_dtype=*/torch::kFloat); + test_TorchTensorCtorZeroSizedDim_expected_dtype( + /*default_dtype=*/torch::kDouble); +} + +void test_TorchTensorCtorWithoutSpecifyingDtype_expected_dtype( + c10::ScalarType default_dtype) { + AutoDefaultDtypeMode dtype_mode(default_dtype); + + ASSERT_EQ(torch::tensor({1., 2., 3.}).dtype(), default_dtype); + ASSERT_EQ(torch::tensor({{1., 2., 3.}}).dtype(), default_dtype); + ASSERT_EQ( + torch::tensor({1., 2., 3.}, torch::TensorOptions()).dtype(), + default_dtype); + ASSERT_EQ( + torch::tensor({{1., 2., 3.}}, torch::TensorOptions()).dtype(), + default_dtype); +} + +TEST(TensorTest, TorchTensorCtorWithoutSpecifyingDtype) { + ASSERT_EQ(torch::tensor({1, 2, 3}).dtype(), torch::kLong); + ASSERT_EQ(torch::tensor({{1, 2, 3}}).dtype(), torch::kLong); + ASSERT_EQ( + torch::tensor({1, 2, 3}, torch::TensorOptions()).dtype(), torch::kLong); + ASSERT_EQ( + torch::tensor({{1, 2, 3}}, torch::TensorOptions()).dtype(), torch::kLong); + + test_TorchTensorCtorWithoutSpecifyingDtype_expected_dtype( + /*default_dtype=*/torch::kFloat); + test_TorchTensorCtorWithoutSpecifyingDtype_expected_dtype( + /*default_dtype=*/torch::kDouble); +} + +void test_TorchTensorCtorWithNonDtypeOptions_expected_dtype( + c10::ScalarType default_dtype) { + AutoDefaultDtypeMode dtype_mode(default_dtype); + + ASSERT_EQ( + torch::tensor({1, 2, 3}, torch::TensorOptions()).dtype(), torch::kLong); + ASSERT_EQ( + torch::tensor(at::ArrayRef({1, 2, 3}), torch::TensorOptions()) + .dtype(), + torch::kLong); + ASSERT_EQ( + torch::tensor(std::vector({1, 2, 3}), torch::TensorOptions()) + .dtype(), + torch::kLong); + + ASSERT_EQ( + torch::tensor({1., 2., 3.}, torch::TensorOptions()).dtype(), + default_dtype); + ASSERT_EQ( + torch::tensor(at::ArrayRef({1., 2., 3.}), torch::TensorOptions()) + .dtype(), + default_dtype); + ASSERT_EQ( + torch::tensor(std::vector({1., 2., 3.}), torch::TensorOptions()) + .dtype(), + default_dtype); + + ASSERT_EQ( + torch::tensor({1.f, 2.f, 3.f}, torch::TensorOptions()).dtype(), + default_dtype); + ASSERT_EQ( + torch::tensor( + at::ArrayRef({1.f, 2.f, 3.f}), torch::TensorOptions()) + .dtype(), + default_dtype); + ASSERT_EQ( + torch::tensor(std::vector({1.f, 2.f, 3.f}), torch::TensorOptions()) + .dtype(), + default_dtype); +} + +TEST(TensorTest, TorchTensorCtorWithNonDtypeOptions) { + test_TorchTensorCtorWithNonDtypeOptions_expected_dtype( + /*default_dtype=*/torch::kFloat); + test_TorchTensorCtorWithNonDtypeOptions_expected_dtype( + /*default_dtype=*/torch::kDouble); +} + +void test_Arange_expected_dtype(c10::ScalarType default_dtype) { + AutoDefaultDtypeMode dtype_mode(default_dtype); + + ASSERT_EQ(torch::arange(0., 5).dtype(), default_dtype); +} + +TEST(TensorTest, Arange) { + { + auto x = torch::arange(0, 5); + ASSERT_EQ(x.dtype(), torch::kLong); + } + test_Arange_expected_dtype(torch::kFloat); + test_Arange_expected_dtype(torch::kDouble); +} + +TEST(TensorTest, PrettyPrintTensorDataContainer) { + { + ASSERT_EQ(c10::str(torch::detail::TensorDataContainer(1.1)), "1.1"); + } + { + ASSERT_EQ( + c10::str(torch::detail::TensorDataContainer({1.1, 2.2})), "{1.1, 2.2}"); + } + { + ASSERT_EQ( + c10::str(torch::detail::TensorDataContainer({{1, 2}, {3, 4}})), + "{{1, 2}, {3, 4}}"); + } + { + ASSERT_EQ( + c10::str(torch::detail::TensorDataContainer( + {{{{{{{{1.1, 2.2, 3.3}}}}}, + {{{{{4.4, 5.5, 6.6}}}}}, + {{{{{7.7, 8.8, 9.9}}}}}}}})), + "{{{{{{{{1.1, 2.2, 3.3}}}}}, {{{{{4.4, 5.5, 6.6}}}}}, {{{{{7.7, 8.8, 9.9}}}}}}}}"); + } + { + ASSERT_EQ( + c10::str(torch::detail::TensorDataContainer({{{{{{{{{{1}}}}}}}}}})), + "{{{{{{{{{{1}}}}}}}}}}"); + } + { + ASSERT_EQ( + c10::str(torch::detail::TensorDataContainer({{{{{{{{{{}}}}}}}}}})), + "{{{{{{{{{{}}}}}}}}}}"); + } + { + ASSERT_EQ( + c10::str(torch::detail::TensorDataContainer({{{{{{{{{{1, 2}}}}}}}}}})), + "{{{{{{{{{{1, 2}}}}}}}}}}"); + } + { + ASSERT_EQ( + c10::str(torch::detail::TensorDataContainer( + at::ArrayRef({1.1, 2.2}))), + "{1.1, 2.2}"); + } + { + ASSERT_EQ( + c10::str(torch::detail::TensorDataContainer( + std::vector({1.1, 2.2}))), + "{1.1, 2.2}"); + } +} + +TEST(TensorTest, TensorDataContainerCallingAccessorOfWrongType) { + { + ASSERT_THROWS_WITH( + torch::detail::TensorDataContainer(1.1).init_list(), + "Can only call `init_list()` on a TensorDataContainer that has `is_init_list() == true`"); + ASSERT_THROWS_WITH( + torch::detail::TensorDataContainer(1.1).tensor(), + "Can only call `tensor()` on a TensorDataContainer that has `is_tensor() == true`"); + } + { + ASSERT_THROWS_WITH( + torch::detail::TensorDataContainer({1.1, 2.2}).scalar(), + "Can only call `scalar()` on a TensorDataContainer that has `is_scalar() == true`"); + ASSERT_THROWS_WITH( + torch::detail::TensorDataContainer({1.1, 2.2}).tensor(), + "Can only call `tensor()` on a TensorDataContainer that has `is_tensor() == true`"); + } + { + ASSERT_THROWS_WITH( + torch::detail::TensorDataContainer(at::ArrayRef({1.1, 2.2})) + .scalar(), + "Can only call `scalar()` on a TensorDataContainer that has `is_scalar() == true`"); + ASSERT_THROWS_WITH( + torch::detail::TensorDataContainer(at::ArrayRef({1.1, 2.2})) + .init_list(), + "Can only call `init_list()` on a TensorDataContainer that has `is_init_list() == true`"); + } +} + +TEST(TensorTest, FromBlob) { + std::vector v = {1.0, 2.0, 3.0}; + auto tensor = torch::from_blob( + v.data(), v.size(), torch::dtype(torch::kFloat64).requires_grad(true)); + ASSERT_TRUE(tensor.requires_grad()); + ASSERT_EQ(tensor.dtype(), torch::kFloat64); + ASSERT_EQ(tensor.numel(), 3); + ASSERT_EQ(tensor[0].item(), 1); + ASSERT_EQ(tensor[1].item(), 2); + ASSERT_EQ(tensor[2].item(), 3); + // Above syntax did not copy the data, and has nullptr deleter context. + ASSERT_EQ(tensor.storage().data_ptr().get_context(), nullptr); +} + +TEST(TensorTest, FromBlobUsesDeleter) { + bool called = false; + { + std::vector v = {1, 2, 3}; + auto tensor = torch::from_blob( + v.data(), + v.size(), + /*deleter=*/[&called](void* data) { called = true; }, + torch::kInt32); + } + ASSERT_TRUE(called); +} + +TEST(TensorTest, FromBlobWithStrides) { + // clang-format off + std::vector v = { + 1, 2, 3, + 4, 5, 6, + 7, 8, 9 + }; + // clang-format on + auto tensor = torch::from_blob( + v.data(), + /*sizes=*/{3, 3}, + /*strides=*/{1, 3}, + torch::kInt32); + ASSERT_EQ(tensor.dtype(), torch::kInt32); + ASSERT_EQ(tensor.numel(), 9); + const std::vector expected_strides = {1, 3}; + ASSERT_EQ(tensor.strides(), expected_strides); + for (const auto i : c10::irange(tensor.size(0))) { + for (const auto j : c10::irange(tensor.size(1))) { + // NOTE: This is column major because the strides are swapped. + EXPECT_EQ(tensor[i][j].item(), 1 + (j * tensor.size(1)) + i); + } + } +} + +TEST(TensorTest, Item) { + { + torch::Tensor tensor = torch::tensor(3.14); + torch::Scalar scalar = tensor.item(); + ASSERT_NEAR(scalar.to(), 3.14, 1e-5); + } + { + torch::Tensor tensor = torch::tensor(123); + torch::Scalar scalar = tensor.item(); + ASSERT_EQ(scalar.to(), 123); + } +} + +TEST(TensorTest, Item_CUDA) { + { + torch::Tensor tensor = torch::tensor(3.14, torch::kCUDA); + torch::Scalar scalar = tensor.item(); + ASSERT_NEAR(scalar.to(), 3.14, 1e-5); + } + { + torch::Tensor tensor = torch::tensor(123, torch::kCUDA); + torch::Scalar scalar = tensor.item(); + ASSERT_EQ(scalar.to(), 123); + } +} + +TEST(TensorTest, DataPtr) { + auto tensor = at::empty({3, 4}, at::kFloat); + auto tensor_not_copy = tensor.to(tensor.options()); + ASSERT_EQ(tensor_not_copy.data_ptr(), tensor.data_ptr()); + ASSERT_EQ(tensor_not_copy.data_ptr(), tensor.data_ptr()); +} + +TEST(TensorTest, Data) { + const auto tensor = torch::rand({3, 3}); + ASSERT_TRUE(torch::equal(tensor, tensor.data())); +} + +TEST(TensorTest, BackwardAndGrad) { + auto x = torch::tensor({5}, torch::dtype(torch::kFloat).requires_grad(true)); + auto y = x * x; + y.backward(); + ASSERT_EQ(x.grad().item(), 10.0); +} + +TEST(TensorTest, BackwardCreatesOnesGrad) { + const auto x = + torch::tensor({5}, torch::dtype(torch::kFloat).requires_grad(true)); + x.backward(); + ASSERT_TRUE(torch::equal(x.grad(), torch::ones_like(x))); +} + +TEST(TensorTest, BackwardNonScalarOutputs) { + auto x = torch::randn({5, 5}, torch::requires_grad()); + auto y = x * x; + ASSERT_THROWS_WITH( + y.backward(), "grad can be implicitly created only for scalar outputs"); +} + +TEST(TensorTest, BackwardComplexScalarOutput) { + auto x = torch::randn({5, 5}, torch::requires_grad()); + auto y = (x * c10::Scalar(c10::complex(0, 0.5))).sum(); + ASSERT_THROWS_WITH( + y.backward(), "grad can be computed only for real scalar outputs"); +} + +TEST(TensorTest, IsLeaf) { + auto x = torch::tensor({5}, torch::dtype(torch::kFloat).requires_grad(true)); + auto y = x * x; + ASSERT_TRUE(x.is_leaf()); + ASSERT_FALSE(y.is_leaf()); +} + +TEST(TensorTest, OutputNr) { + auto x = torch::tensor({5}, torch::dtype(torch::kFloat).requires_grad(true)); + auto y = x * x; + ASSERT_EQ(x.output_nr(), 0); + ASSERT_EQ(y.output_nr(), 0); +} + +TEST(TensorTest, Version) { + auto x = torch::ones(3); + ASSERT_EQ(x._version(), 0); + x.mul_(2); + ASSERT_EQ(x._version(), 1); + x.add_(1); + ASSERT_EQ(x._version(), 2); +} + +TEST(TensorTest, Detach) { + auto x = torch::tensor({5}, torch::dtype(torch::kFloat).requires_grad(true)); + auto y = x * x; + const auto y_detached = y.detach(); + ASSERT_FALSE(y.is_leaf()); + ASSERT_TRUE(y_detached.is_leaf()); + ASSERT_FALSE(y_detached.requires_grad()); +} + +TEST(TensorTest, DetachInplace) { + auto x = torch::tensor({5}, torch::dtype(torch::kFloat).requires_grad(true)); + auto y = x * x; + auto y_detached = y.detach_(); + ASSERT_TRUE(y.is_leaf()); + ASSERT_FALSE(y.requires_grad()); + ASSERT_TRUE(y_detached.is_leaf()); + ASSERT_FALSE(y_detached.requires_grad()); +} + +TEST(TensorTest, SetData) { + auto x = torch::randn({5}); + auto y = torch::randn({5}); + ASSERT_FALSE(torch::equal(x, y)); + ASSERT_NE(x.data_ptr(), y.data_ptr()); + + x.set_data(y); + ASSERT_TRUE(torch::equal(x, y)); + ASSERT_EQ(x.data_ptr(), y.data_ptr()); +} + +TEST(TensorTest, RequiresGradInplace) { + auto x = torch::tensor({5.0}); + x.requires_grad_(true); + ASSERT_TRUE(x.requires_grad()); + + auto y = x * x; + ASSERT_THROWS_WITH( + y.requires_grad_(false), + "you can only change requires_grad flags of leaf variables."); + + x.requires_grad_(false); + ASSERT_FALSE(x.requires_grad()); + + const auto int_tensor = + torch::tensor({5}, at::TensorOptions().dtype(torch::kInt)); + ASSERT_THROWS_WITH( + int_tensor.requires_grad_(true), + "Only Tensors of floating point and complex dtype can require gradients"); +} + +TEST(TensorTest, StdDimension) { + // Test that std(0) doesn't select the std(unbiased=False) overload (gh-40287) + auto x = torch::randn({4, 3}); + auto std = x.std(0); + + ASSERT_EQ(x.var(0).numel(), 3); + ASSERT_EQ(x.std(0).numel(), 3); + + ASSERT_EQ(x.var(0, /*unbiased=*/true).numel(), 3); + ASSERT_EQ(x.std(0, /*unbiased=*/true).numel(), 3); + + ASSERT_EQ(torch::var(x, 0).numel(), 3); + ASSERT_EQ(std::get<0>(torch::var_mean(x, 0)).numel(), 3); + ASSERT_EQ(torch::std(x, 0).numel(), 3); + ASSERT_EQ(std::get<0>(torch::std_mean(x, 0)).numel(), 3); + + ASSERT_EQ(torch::var(x, 0, /*unbiased=*/true).numel(), 3); + ASSERT_EQ(std::get<0>(torch::var_mean(x, 0, /*unbiased=*/true)).numel(), 3); + ASSERT_EQ(torch::std(x, 0, /*unbiased=*/true).numel(), 3); + ASSERT_EQ(std::get<0>(torch::std_mean(x, 0, /*unbiased=*/true)).numel(), 3); +} + +TEST(TensorTest, ReshapeAlias) { + // Tests the behavior of the _reshape_alias private operator so + // that it matches the behavior of as_strided and view. + auto x = torch::randn({3, 3}); + ASSERT_TRUE(torch::equal( + torch::_reshape_alias(x, {2, 2}, {1, 2}), + torch::as_strided(x, {2, 2}, {1, 2}))); + ASSERT_TRUE(torch::equal(torch::_reshape_alias(x, {9}, {1}), x.view({-1}))); + + // Test that the backward works fine. + auto y = torch::randn({3, 3}, torch::requires_grad(true)); + auto z = torch::clone(y).detach().requires_grad_(true); + (y * y).view({-1}).mean().backward(); + torch::_reshape_alias((z * z), {9}, {1}).mean().backward(); + ASSERT_TRUE(torch::equal(y.grad(), z.grad())); +} + +TEST(TensorTest, BackendMetadata) { + // Tests ability to assign custom backend metadata to tensor. + + struct CustomBackendMetadata : public c10::BackendMeta { + mutable bool cloned_{false}; // for testing this field will mutate when + // clone() is called by shallow_copy_from. + c10::intrusive_ptr clone( + const c10::intrusive_ptr& ptr) const override { + cloned_ = true; + return c10::BackendMeta::clone(ptr); + } + }; + + at::Tensor y; + c10::intrusive_ptr tmeta{}; + CustomBackendMetadata* custom_tmeta{nullptr}; + + { + auto x = torch::ones({3, 3}); + auto impl{x.unsafeGetTensorImpl()}; + ASSERT_TRUE(impl != nullptr); + + tmeta = impl->get_backend_meta_intrusive_ptr(); + ASSERT_TRUE(tmeta == nullptr); + c10::intrusive_ptr new_tmeta{ + std::unique_ptr(new CustomBackendMetadata())}; + impl->set_backend_meta(new_tmeta); + tmeta = impl->get_backend_meta_intrusive_ptr(); + ASSERT_TRUE(tmeta == new_tmeta); + custom_tmeta = dynamic_cast(tmeta.get()); + ASSERT_TRUE(custom_tmeta != nullptr); + ASSERT_TRUE(custom_tmeta->cloned_ == false); + y.unsafeGetTensorImpl()->shallow_copy_from(x.getIntrusivePtr()); + } + + ASSERT_TRUE( + tmeta == y.unsafeGetTensorImpl()->get_backend_meta_intrusive_ptr()); + ASSERT_TRUE(tmeta.get() == y.unsafeGetTensorImpl()->get_backend_meta()); + ASSERT_TRUE(custom_tmeta->cloned_ == true); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/tensor_cuda.cpp b/western_ai/usa/pytorch/test/cpp/api/tensor_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..16891bbd501d2da39bc9c8d0e2a5d84503de665c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/tensor_cuda.cpp @@ -0,0 +1,188 @@ +#include + +#include +#include +#include + +#include + +#define REQUIRE_TENSOR_OPTIONS(device_, index_, type_, layout_) \ + ASSERT_TRUE( \ + tensor.device().type() == at::Device((device_), (index_)).type()); \ + ASSERT_TRUE( \ + tensor.device().index() == at::Device((device_), (index_)).index()); \ + ASSERT_EQ(tensor.dtype(), (type_)); \ + ASSERT_TRUE(tensor.layout() == (layout_)) + +TEST(TensorTest, AllocatesTensorOnTheCorrectDevice_MultiCUDA) { + auto tensor = at::tensor({1, 2, 3}, at::device({at::kCUDA, 1})); + ASSERT_EQ(tensor.device().type(), at::Device::Type::CUDA); + ASSERT_EQ(tensor.device().index(), 1); +} + +TEST(TensorTest, ToDevice_MultiCUDA) { + auto tensor = at::empty({3, 4}); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kStrided); + + tensor = tensor.to({at::kCUDA, 1}); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 1, at::kFloat, at::kStrided); + + tensor = tensor.to({at::kCUDA, 0}); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 0, at::kFloat, at::kStrided); + + tensor = tensor.to({at::kCUDA, 1}); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 1, at::kFloat, at::kStrided); + + tensor = tensor.to(at::Device(at::kCPU)); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kStrided); + + tensor = tensor.to(at::kCUDA); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 0, at::kFloat, at::kStrided); + + tensor = tensor.to(at::TensorOptions({at::kCUDA, 1})); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 1, at::kFloat, at::kStrided); + + tensor = tensor.to(at::TensorOptions({at::kCUDA, 0})); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 0, at::kFloat, at::kStrided); + + tensor = tensor.to(at::TensorOptions(at::kDouble)); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 0, at::kDouble, at::kStrided); + + tensor = tensor.to(at::TensorOptions({at::kCUDA, 1})); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 1, at::kDouble, at::kStrided); + + tensor = tensor.to(at::TensorOptions(at::kInt)); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 1, at::kInt, at::kStrided); + + tensor = tensor.to(at::TensorOptions(at::Device(at::kCPU))); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kInt, at::kStrided); + + tensor = tensor.to(at::TensorOptions(at::kCUDA)); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 0, at::kInt, at::kStrided); +} + +TEST(TensorTest, ToTensorAndTensorAttributes_MultiCUDA) { + auto tensor = at::empty({3, 4}); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kStrided); + + auto other = at::empty({3, 4}, at::kFloat); + tensor = tensor.to(other); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kStrided); + + other = at::empty({3, 4}, at::TensorOptions(at::kCUDA).dtype(at::kDouble)); + tensor = tensor.to(other.dtype()); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kDouble, at::kStrided); + tensor = tensor.to(other.device()); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 0, at::kDouble, at::kStrided); + + other = at::empty({3, 4}, at::TensorOptions({at::kCUDA, 1}).dtype(at::kLong)); + tensor = tensor.to(other.device(), other.dtype()); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 1, at::kLong, at::kStrided); + + other = at::empty({3, 4}, at::kFloat); + tensor = tensor.to(other.options()); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kStrided); +} + +TEST(TensorTest, ToDoesNotCopyWhenOptionsAreAllTheSame_CUDA) { + auto tensor = at::empty( + {3, 4}, at::TensorOptions(at::kFloat).device(at::Device("cuda"))); + auto hopefully_not_copy = tensor.to(tensor.options()); + ASSERT_EQ(hopefully_not_copy.data_ptr(), tensor.data_ptr()); + hopefully_not_copy = tensor.to(at::kFloat); + ASSERT_EQ(hopefully_not_copy.data_ptr(), tensor.data_ptr()); + hopefully_not_copy = tensor.to("cuda"); + ASSERT_EQ(hopefully_not_copy.data_ptr(), tensor.data_ptr()); + hopefully_not_copy = tensor.to(at::TensorOptions("cuda")); + ASSERT_EQ(hopefully_not_copy.data_ptr(), tensor.data_ptr()); + hopefully_not_copy = tensor.to(at::TensorOptions(at::kFloat)); + ASSERT_EQ(hopefully_not_copy.data_ptr(), tensor.data_ptr()); +} + +TEST(TensorTest, ToDeviceAndDtype_MultiCUDA) { + auto tensor = at::empty({3, 4}); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kFloat, at::kStrided); + + tensor = tensor.to({at::kCUDA, 1}, at::kInt); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 1, at::kInt, at::kStrided); + + tensor = tensor.to(at::TensorOptions({at::kCUDA, 0}).dtype(at::kLong)); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 0, at::kLong, at::kStrided); + + tensor = tensor.to(at::TensorOptions({at::kCUDA, 1}).dtype(at::kDouble)); + REQUIRE_TENSOR_OPTIONS(at::kCUDA, 1, at::kDouble, at::kStrided); + + tensor = tensor.to(at::kCPU, at::kInt); + REQUIRE_TENSOR_OPTIONS(at::kCPU, -1, at::kInt, at::kStrided); +} + +TEST(TensorTest, MagmaInitializesCorrectly_CUDA) { + // Any tensor will work here as long as it's invertible + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) + float data[] = {1, 1, 1, 0, 0, 3, 1, 2, 2, 3, 1, 0, 1, 0, 2, 1}; + auto tensor = + at::from_blob(data, {4, 4}, at::TensorOptions(at::kFloat)).cuda(); + if (at::hasMAGMA()) { + at::inverse(tensor); + } +} + +#ifdef USE_CUDA +#include +#if AT_CUDNN_ENABLED() +TEST(CuDNNBatchNormTest, OutVariantMatchesFunctional) { + if (!torch::cuda::is_available()) { + GTEST_SKIP() << "CUDA is not available"; + } + if (!at::Context::hasCuDNN()) { + GTEST_SKIP() << "cuDNN is not available"; + } + + auto device = torch::device(torch::kCUDA); + + auto input = torch::rand({2, 3, 4, 4}, device); + auto weight = torch::randn({3}, device); + auto bias = torch::randn({3}, device); + auto running_mean = torch::zeros({3}, device); + auto running_var = torch::ones({3}, device); + + bool training = true; + double exponential_average_factor = 0.1; + double epsilon = 1e-5; + + auto output = torch::empty_like(input); + auto save_mean = torch::empty({3}, device); + auto save_var = torch::empty({3}, device); + auto reserve = torch::empty({0}, device.dtype(torch::kByte)); + + at::native::cudnn_batch_norm_out( + input, + weight, + bias, + running_mean, + running_var, + training, + exponential_average_factor, + epsilon, + output, + save_mean, + save_var, + reserve); + + auto ref_outputs = at::native::cudnn_batch_norm( + input, + weight, + bias, + running_mean, + running_var, + training, + exponential_average_factor, + epsilon); + + ASSERT_TRUE(torch::allclose(output, std::get<0>(ref_outputs))); + ASSERT_TRUE(torch::allclose(save_mean, std::get<1>(ref_outputs))); + ASSERT_TRUE(torch::allclose(save_var, std::get<2>(ref_outputs))); + ASSERT_TRUE(torch::equal(reserve, std::get<3>(ref_outputs))); +} +#endif // AT_CUDNN_ENABLED() +#endif // USE_CUDA diff --git a/western_ai/usa/pytorch/test/cpp/api/tensor_flatten.cpp b/western_ai/usa/pytorch/test/cpp/api/tensor_flatten.cpp new file mode 100644 index 0000000000000000000000000000000000000000..08b245752eafa721001f6843cebc8a17789527df --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/tensor_flatten.cpp @@ -0,0 +1,43 @@ +#include +#include + +#include +#include +#include + +using namespace torch::test; + +TEST(UnflattenDenseTensorTest, TestEmptyTensor) { + auto emptyTensor1 = at::tensor(std::vector()); + auto emptyTensor2 = at::tensor(std::vector()); + auto tensor1 = at::tensor({1, 2, 3}); + auto tensor2 = at::tensor({4, 5}); + auto tensorList = + std::vector({tensor1, emptyTensor1, emptyTensor2, tensor2}); + auto flatTensor = at::tensor({1, 2, 3, 4, 5}); + auto unflatten_results = + torch::utils::unflatten_dense_tensors(flatTensor, tensorList); + ASSERT_EQ(unflatten_results.size(), 4); + ASSERT_EQ(unflatten_results.at(0).numel(), 3); + ASSERT_EQ(unflatten_results.at(1).numel(), 0); + ASSERT_EQ(unflatten_results.at(2).numel(), 0); + ASSERT_EQ(unflatten_results.at(3).numel(), 2); + + // empty tensor address is 0 as memory is not allocated yet + ASSERT_EQ(unflatten_results.at(1).data_ptr(), nullptr); + ASSERT_EQ(unflatten_results.at(2).data_ptr(), nullptr); + // without fix in unflatten_dense_tensors() for empty tensors, + // unflattend empty tensor unflatten_results.at(1) will share the same storage + // as other non-empty tensor like unflatten_results.at(3). + // after fix, the empty tensor and non-empty tensor do not share the same + // storage. + ASSERT_NE( + unflatten_results.at(1).data_ptr(), unflatten_results.at(3).data_ptr()); + unflatten_results.at(1).resize_(1); + unflatten_results.at(2).resize_(1); + // after resizing the two empty tensors, the resized tensors do not share + // the same storage. without fix in unflatten_dense_tensors() for empty + // tensors, the resized tensors will share the same storage. + ASSERT_NE( + unflatten_results.at(1).data_ptr(), unflatten_results.at(2).data_ptr()); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/tensor_indexing.cpp b/western_ai/usa/pytorch/test/cpp/api/tensor_indexing.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b861e98e015cecca26b97dc6d4af26a850a2b580 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/tensor_indexing.cpp @@ -0,0 +1,1003 @@ +#include + +#include + +#include + +using namespace torch::indexing; +using namespace torch::test; + +TEST(TensorIndexingTest, Slice) { + Slice slice(1, 2, 3); + ASSERT_EQ(slice.start(), 1); + ASSERT_EQ(slice.stop(), 2); + ASSERT_EQ(slice.step(), 3); + + ASSERT_EQ(c10::str(slice), "1:2:3"); +} + +TEST(TensorIndexingTest, TensorIndex) { + { + std::vector indices = { + None, + "...", + Ellipsis, + 0, + true, + Slice(1, None, 2), + torch::tensor({1, 2})}; + ASSERT_TRUE(indices[0].is_none()); + ASSERT_TRUE(indices[1].is_ellipsis()); + ASSERT_TRUE(indices[2].is_ellipsis()); + ASSERT_TRUE(indices[3].is_integer()); + ASSERT_TRUE(indices[3].integer() == 0); + ASSERT_TRUE(indices[4].is_boolean()); + ASSERT_TRUE(indices[4].boolean() == true); + ASSERT_TRUE(indices[5].is_slice()); + ASSERT_TRUE(indices[5].slice().start() == 1); + ASSERT_TRUE(indices[5].slice().stop() == INDEX_MAX); + ASSERT_TRUE(indices[5].slice().step() == 2); + ASSERT_TRUE(indices[6].is_tensor()); + ASSERT_TRUE(torch::equal(indices[6].tensor(), torch::tensor({1, 2}))); + } + + ASSERT_THROWS_WITH( + TensorIndex(".."), + "Expected \"...\" to represent an ellipsis index, but got \"..\""); + + { + std::vector indices = { + None, "...", Ellipsis, 0, true, Slice(1, None, 2)}; + ASSERT_EQ( + c10::str(indices), + c10::str("(None, ..., ..., 0, true, 1:", INDEX_MAX, ":2)")); + ASSERT_EQ(c10::str(indices[0]), "None"); + ASSERT_EQ(c10::str(indices[1]), "..."); + ASSERT_EQ(c10::str(indices[2]), "..."); + ASSERT_EQ(c10::str(indices[3]), "0"); + ASSERT_EQ(c10::str(indices[4]), "true"); + ASSERT_EQ(c10::str(indices[5]), c10::str("1:", INDEX_MAX, ":2")); + } + + ASSERT_EQ( + c10::str(std::vector({Slice()})), + c10::str("(0:", INDEX_MAX, ":1)")); + ASSERT_EQ( + c10::str(std::vector({Slice(None, None)})), + c10::str("(0:", INDEX_MAX, ":1)")); + ASSERT_EQ( + c10::str(std::vector({Slice(None, None, None)})), + c10::str("(0:", INDEX_MAX, ":1)")); + + ASSERT_EQ( + c10::str(std::vector({Slice(1, None)})), + c10::str("(1:", INDEX_MAX, ":1)")); + ASSERT_EQ( + c10::str(std::vector({Slice(1, None, None)})), + c10::str("(1:", INDEX_MAX, ":1)")); + ASSERT_EQ( + c10::str(std::vector({Slice(None, 3)})), + c10::str("(0:3:1)")); + ASSERT_EQ( + c10::str(std::vector({Slice(None, 3, None)})), + c10::str("(0:3:1)")); + ASSERT_EQ( + c10::str(std::vector({Slice(None, None, 2)})), + c10::str("(0:", INDEX_MAX, ":2)")); + ASSERT_EQ( + c10::str(std::vector({Slice(None, None, -1)})), + c10::str("(", INDEX_MAX, ":", INDEX_MIN, ":-1)")); + + ASSERT_EQ( + c10::str(std::vector({Slice(1, 3)})), c10::str("(1:3:1)")); + ASSERT_EQ( + c10::str(std::vector({Slice(1, None, 2)})), + c10::str("(1:", INDEX_MAX, ":2)")); + ASSERT_EQ( + c10::str(std::vector({Slice(1, None, -1)})), + c10::str("(1:", INDEX_MIN, ":-1)")); + ASSERT_EQ( + c10::str(std::vector({Slice(None, 3, 2)})), + c10::str("(0:3:2)")); + ASSERT_EQ( + c10::str(std::vector({Slice(None, 3, -1)})), + c10::str("(", INDEX_MAX, ":3:-1)")); + + ASSERT_EQ( + c10::str(std::vector({Slice(1, 3, 2)})), + c10::str("(1:3:2)")); +} + +TEST(TensorIndexingTest, TestNoIndices) { + torch::Tensor tensor = torch::randn({20, 20}); + torch::Tensor value = torch::randn({20, 20}); + std::vector indices; + + ASSERT_THROWS_WITH( + tensor.index({}), + "Passing an empty index list to Tensor::index() is not valid syntax"); + ASSERT_THROWS_WITH( + tensor.index_put_({}, 1), + "Passing an empty index list to Tensor::index_put_() is not valid syntax"); + ASSERT_THROWS_WITH( + tensor.index_put_({}, value), + "Passing an empty index list to Tensor::index_put_() is not valid syntax"); + + ASSERT_THROWS_WITH( + tensor.index(indices), + "Passing an empty index list to Tensor::index() is not valid syntax"); + ASSERT_THROWS_WITH( + tensor.index_put_(indices, 1), + "Passing an empty index list to Tensor::index_put_() is not valid syntax"); + ASSERT_THROWS_WITH( + tensor.index_put_(indices, value), + "Passing an empty index list to Tensor::index_put_() is not valid syntax"); +} + +TEST(TensorIndexingTest, TestAdvancedIndexingWithListOfTensor) { + { + torch::Tensor tensor = torch::randn({20, 20}); + torch::Tensor index = torch::arange(10, torch::kLong).cpu(); + torch::Tensor result = at::index(tensor, {index}); + torch::Tensor result_with_init_list = tensor.index({index}); + ASSERT_TRUE(result.equal(result_with_init_list)); + } + { + torch::Tensor tensor = torch::randn({20, 20}); + torch::Tensor index = torch::arange(10, torch::kLong).cpu(); + torch::Tensor result = at::index_put_(tensor, {index}, torch::ones({20})); + torch::Tensor result_with_init_list = + tensor.index_put_({index}, torch::ones({20})); + ASSERT_TRUE(result.equal(result_with_init_list)); + } + { + torch::Tensor tensor = torch::randn({20, 20}); + torch::Tensor index = torch::arange(10, torch::kLong).cpu(); + torch::Tensor result = + at::index_put_(tensor, {index}, torch::ones({1, 20})); + torch::Tensor result_with_init_list = + tensor.index_put_({index}, torch::ones({1, 20})); + ASSERT_TRUE(result.equal(result_with_init_list)); + } +} + +TEST(TensorIndexingTest, TestSingleInt) { + auto v = torch::randn({5, 7, 3}); + ASSERT_EQ(v.index({4}).sizes(), torch::IntArrayRef({7, 3})); +} + +TEST(TensorIndexingTest, TestMultipleInt) { + auto v = torch::randn({5, 7, 3}); + ASSERT_EQ(v.index({4}).sizes(), torch::IntArrayRef({7, 3})); + ASSERT_EQ(v.index({4, Slice(), 1}).sizes(), torch::IntArrayRef({7})); + + // To show that `.index_put_` works + v.index_put_({4, 3, 1}, 0); + ASSERT_EQ(v.index({4, 3, 1}).item(), 0); +} + +TEST(TensorIndexingTest, TestNone) { + auto v = torch::randn({5, 7, 3}); + ASSERT_EQ(v.index({None}).sizes(), torch::IntArrayRef({1, 5, 7, 3})); + ASSERT_EQ(v.index({Slice(), None}).sizes(), torch::IntArrayRef({5, 1, 7, 3})); + ASSERT_EQ( + v.index({Slice(), None, None}).sizes(), + torch::IntArrayRef({5, 1, 1, 7, 3})); + ASSERT_EQ(v.index({"...", None}).sizes(), torch::IntArrayRef({5, 7, 3, 1})); +} + +TEST(TensorIndexingTest, TestStep) { + auto v = torch::arange(10); + assert_tensor_equal(v.index({Slice(None, None, 1)}), v); + assert_tensor_equal( + v.index({Slice(None, None, 2)}), torch::tensor({0, 2, 4, 6, 8})); + assert_tensor_equal( + v.index({Slice(None, None, 3)}), torch::tensor({0, 3, 6, 9})); + assert_tensor_equal(v.index({Slice(None, None, 11)}), torch::tensor({0})); + assert_tensor_equal(v.index({Slice(1, 6, 2)}), torch::tensor({1, 3, 5})); +} + +TEST(TensorIndexingTest, TestStepAssignment) { + auto v = torch::zeros({4, 4}); + v.index_put_({0, Slice(1, None, 2)}, torch::tensor({3., 4.})); + assert_tensor_equal(v.index({0}), torch::tensor({0., 3., 0., 4.})); + assert_tensor_equal(v.index({Slice(1, None)}).sum(), torch::tensor(0)); +} + +TEST(TensorIndexingTest, TestBoolIndices) { + { + auto v = torch::randn({5, 7, 3}); + auto boolIndices = + torch::tensor({true, false, true, true, false}, torch::kBool); + ASSERT_EQ(v.index({boolIndices}).sizes(), torch::IntArrayRef({3, 7, 3})); + assert_tensor_equal( + v.index({boolIndices}), + torch::stack({v.index({0}), v.index({2}), v.index({3})})); + } + { + auto v = torch::tensor({true, false, true}, torch::kBool); + auto boolIndices = torch::tensor({true, false, false}, torch::kBool); + auto uint8Indices = torch::tensor({1, 0, 0}, torch::kUInt8); + + { + WarningCapture warnings; + + ASSERT_EQ( + v.index({boolIndices}).sizes(), v.index({uint8Indices}).sizes()); + assert_tensor_equal(v.index({boolIndices}), v.index({uint8Indices})); + assert_tensor_equal( + v.index({boolIndices}), torch::tensor({true}, torch::kBool)); + + ASSERT_EQ( + count_substr_occurrences( + warnings.str(), + "indexing with dtype torch.uint8 is now deprecated"), + 2); + } + } +} + +TEST(TensorIndexingTest, TestBoolIndicesAccumulate) { + auto mask = torch::zeros({10}, torch::kBool); + auto y = torch::ones({10, 10}); + y.index_put_({mask}, {y.index({mask})}, /*accumulate=*/true); + assert_tensor_equal(y, torch::ones({10, 10})); +} + +TEST(TensorIndexingTest, TestMultipleBoolIndices) { + auto v = torch::randn({5, 7, 3}); + // note: these broadcast together and are transposed to the first dim + auto mask1 = torch::tensor({1, 0, 1, 1, 0}, torch::kBool); + auto mask2 = torch::tensor({1, 1, 1}, torch::kBool); + ASSERT_EQ( + v.index({mask1, Slice(), mask2}).sizes(), torch::IntArrayRef({3, 7})); +} + +TEST(TensorIndexingTest, TestByteMask) { + { + auto v = torch::randn({5, 7, 3}); + auto mask = torch::tensor({1, 0, 1, 1, 0}, torch::kByte); + { + WarningCapture warnings; + + ASSERT_EQ(v.index({mask}).sizes(), torch::IntArrayRef({3, 7, 3})); + assert_tensor_equal(v.index({mask}), torch::stack({v[0], v[2], v[3]})); + + ASSERT_EQ( + count_substr_occurrences( + warnings.str(), + "indexing with dtype torch.uint8 is now deprecated"), + 2); + } + } + { + auto v = torch::tensor({1.}); + assert_tensor_equal(v.index({v == 0}), torch::randn({0})); + } +} + +TEST(TensorIndexingTest, TestByteMaskAccumulate) { + auto mask = torch::zeros({10}, torch::kUInt8); + auto y = torch::ones({10, 10}); + { + WarningCapture warnings; + + y.index_put_({mask}, y.index({mask}), /*accumulate=*/true); + assert_tensor_equal(y, torch::ones({10, 10})); + + ASSERT_EQ( + count_substr_occurrences( + warnings.str(), + "indexing with dtype torch.uint8 is now deprecated"), + 2); + } +} + +TEST(TensorIndexingTest, TestMultipleByteMask) { + auto v = torch::randn({5, 7, 3}); + // note: these broadcast together and are transposed to the first dim + auto mask1 = torch::tensor({1, 0, 1, 1, 0}, torch::kByte); + auto mask2 = torch::tensor({1, 1, 1}, torch::kByte); + { + WarningCapture warnings; + + ASSERT_EQ( + v.index({mask1, Slice(), mask2}).sizes(), torch::IntArrayRef({3, 7})); + + ASSERT_EQ( + count_substr_occurrences( + warnings.str(), + "indexing with dtype torch.uint8 is now deprecated"), + 2); + } +} + +TEST(TensorIndexingTest, TestByteMask2d) { + auto v = torch::randn({5, 7, 3}); + auto c = torch::randn({5, 7}); + int64_t num_ones = (c > 0).sum().item().to(); + auto r = v.index({c > 0}); + ASSERT_EQ(r.sizes(), torch::IntArrayRef({num_ones, 3})); +} + +TEST(TensorIndexingTest, TestIntIndices) { + auto v = torch::randn({5, 7, 3}); + ASSERT_EQ( + v.index({torch::tensor({0, 4, 2})}).sizes(), + torch::IntArrayRef({3, 7, 3})); + ASSERT_EQ( + v.index({Slice(), torch::tensor({0, 4, 2})}).sizes(), + torch::IntArrayRef({5, 3, 3})); + ASSERT_EQ( + v.index({Slice(), torch::tensor({{0, 1}, {4, 3}})}).sizes(), + torch::IntArrayRef({5, 2, 2, 3})); +} + +TEST(TensorIndexingTest, TestIntIndices2d) { + // From the NumPy indexing example + auto x = torch::arange(0, 12, torch::kLong).view({4, 3}); + auto rows = torch::tensor({{0, 0}, {3, 3}}); + auto columns = torch::tensor({{0, 2}, {0, 2}}); + assert_tensor_equal( + x.index({rows, columns}), torch::tensor({{0, 2}, {9, 11}})); +} + +TEST(TensorIndexingTest, TestIntIndicesBroadcast) { + // From the NumPy indexing example + auto x = torch::arange(0, 12, torch::kLong).view({4, 3}); + auto rows = torch::tensor({0, 3}); + auto columns = torch::tensor({0, 2}); + auto result = x.index({rows.index({Slice(), None}), columns}); + assert_tensor_equal(result, torch::tensor({{0, 2}, {9, 11}})); +} + +TEST(TensorIndexingTest, TestEmptyIndex) { + auto x = torch::arange(0, 12).view({4, 3}); + auto idx = torch::tensor({}, torch::kLong); + ASSERT_EQ(x.index({idx}).numel(), 0); + + // empty assignment should have no effect but not throw an exception + auto y = x.clone(); + y.index_put_({idx}, -1); + assert_tensor_equal(x, y); + + auto mask = torch::zeros({4, 3}, torch::kBool); + y.index_put_({mask}, -1); + assert_tensor_equal(x, y); +} + +TEST(TensorIndexingTest, TestEmptyNdimIndex) { + torch::Device device(torch::kCPU); + { + auto x = torch::randn({5}, device); + assert_tensor_equal( + torch::empty({0, 2}, device), + x.index({torch::empty( + {0, 2}, torch::TensorOptions(torch::kInt64).device(device))})); + } + { + auto x = torch::randn({2, 3, 4, 5}, device); + assert_tensor_equal( + torch::empty({2, 0, 6, 4, 5}, device), + x.index( + {Slice(), + torch::empty( + {0, 6}, torch::TensorOptions(torch::kInt64).device(device))})); + } + { + auto x = torch::empty({10, 0}); + ASSERT_EQ( + x.index({torch::tensor({1, 2})}).sizes(), torch::IntArrayRef({2, 0})); + ASSERT_EQ( + x.index( + {torch::tensor({}, torch::kLong), torch::tensor({}, torch::kLong)}) + .sizes(), + torch::IntArrayRef({0})); + ASSERT_THROWS_WITH( + x.index({Slice(), torch::tensor({0, 1})}), "for dimension with size 0"); + } +} + +TEST(TensorIndexingTest, TestEmptyNdimIndex_CUDA) { + torch::Device device(torch::kCUDA); + { + auto x = torch::randn({5}, device); + assert_tensor_equal( + torch::empty({0, 2}, device), + x.index({torch::empty( + {0, 2}, torch::TensorOptions(torch::kInt64).device(device))})); + } + { + auto x = torch::randn({2, 3, 4, 5}, device); + assert_tensor_equal( + torch::empty({2, 0, 6, 4, 5}, device), + x.index( + {Slice(), + torch::empty( + {0, 6}, torch::TensorOptions(torch::kInt64).device(device))})); + } +} + +TEST(TensorIndexingTest, TestEmptyNdimIndexBool) { + torch::Device device(torch::kCPU); + auto x = torch::randn({5}, device); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW( + x.index({torch::empty( + {0, 2}, torch::TensorOptions(torch::kUInt8).device(device))}), + c10::Error); +} + +TEST(TensorIndexingTest, TestEmptyNdimIndexBool_CUDA) { + torch::Device device(torch::kCUDA); + auto x = torch::randn({5}, device); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW( + x.index({torch::empty( + {0, 2}, torch::TensorOptions(torch::kUInt8).device(device))}), + c10::Error); +} + +TEST(TensorIndexingTest, TestEmptySlice) { + torch::Device device(torch::kCPU); + auto x = torch::randn({2, 3, 4, 5}, device); + auto y = x.index({Slice(), Slice(), Slice(), 1}); + auto z = y.index({Slice(), Slice(1, 1), Slice()}); + ASSERT_EQ(z.sizes(), torch::IntArrayRef({2, 0, 4})); + // this isn't technically necessary, but matches NumPy stride calculations. + ASSERT_EQ(z.strides(), torch::IntArrayRef({60, 20, 5})); + ASSERT_TRUE(z.is_contiguous()); +} + +TEST(TensorIndexingTest, TestEmptySlice_CUDA) { + torch::Device device(torch::kCUDA); + auto x = torch::randn({2, 3, 4, 5}, device); + auto y = x.index({Slice(), Slice(), Slice(), 1}); + auto z = y.index({Slice(), Slice(1, 1), Slice()}); + ASSERT_EQ(z.sizes(), torch::IntArrayRef({2, 0, 4})); + // this isn't technically necessary, but matches NumPy stride calculations. + ASSERT_EQ(z.strides(), torch::IntArrayRef({60, 20, 5})); + ASSERT_TRUE(z.is_contiguous()); +} + +TEST(TensorIndexingTest, TestIndexGetitemCopyBoolsSlices) { + auto true_tensor = torch::tensor(1, torch::kUInt8); + auto false_tensor = torch::tensor(0, torch::kUInt8); + + std::vector tensors = {torch::randn({2, 3}), torch::tensor(3)}; + + for (auto& a : tensors) { + ASSERT_NE(a.data_ptr(), a.index({true}).data_ptr()); + { + std::vector sizes = {0}; + sizes.insert(sizes.end(), a.sizes().begin(), a.sizes().end()); + assert_tensor_equal(torch::empty(sizes), a.index({false})); + } + ASSERT_NE(a.data_ptr(), a.index({true_tensor}).data_ptr()); + { + std::vector sizes = {0}; + sizes.insert(sizes.end(), a.sizes().begin(), a.sizes().end()); + assert_tensor_equal(torch::empty(sizes), a.index({false_tensor})); + } + ASSERT_EQ(a.data_ptr(), a.index({None}).data_ptr()); + ASSERT_EQ(a.data_ptr(), a.index({"..."}).data_ptr()); + } +} + +TEST(TensorIndexingTest, TestIndexSetitemBoolsSlices) { + auto true_tensor = torch::tensor(1, torch::kUInt8); + auto false_tensor = torch::tensor(0, torch::kUInt8); + + std::vector tensors = {torch::randn({2, 3}), torch::tensor(3)}; + + for (auto& a : tensors) { + // prefix with a 1,1, to ensure we are compatible with numpy which cuts off + // prefix 1s (some of these ops already prefix a 1 to the size) + auto neg_ones = torch::ones_like(a) * -1; + auto neg_ones_expanded = neg_ones.unsqueeze(0).unsqueeze(0); + a.index_put_({true}, neg_ones_expanded); + assert_tensor_equal(a, neg_ones); + a.index_put_({false}, 5); + assert_tensor_equal(a, neg_ones); + a.index_put_({true_tensor}, neg_ones_expanded * 2); + assert_tensor_equal(a, neg_ones * 2); + a.index_put_({false_tensor}, 5); + assert_tensor_equal(a, neg_ones * 2); + a.index_put_({None}, neg_ones_expanded * 3); + assert_tensor_equal(a, neg_ones * 3); + a.index_put_({"..."}, neg_ones_expanded * 4); + assert_tensor_equal(a, neg_ones * 4); + if (a.dim() == 0) { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(a.index_put_({Slice()}, neg_ones_expanded * 5), c10::Error); + } + } +} + +TEST(TensorIndexingTest, TestIndexScalarWithBoolMask) { + torch::Device device(torch::kCPU); + + auto a = torch::tensor(1, device); + auto uintMask = + torch::tensor(true, torch::TensorOptions(torch::kUInt8).device(device)); + auto boolMask = + torch::tensor(true, torch::TensorOptions(torch::kBool).device(device)); + assert_tensor_equal(a.index({uintMask}), a.index({boolMask})); + ASSERT_EQ(a.index({uintMask}).dtype(), a.index({boolMask}).dtype()); + + a = torch::tensor(true, torch::TensorOptions(torch::kBool).device(device)); + assert_tensor_equal(a.index({uintMask}), a.index({boolMask})); + ASSERT_EQ(a.index({uintMask}).dtype(), a.index({boolMask}).dtype()); +} + +TEST(TensorIndexingTest, TestIndexScalarWithBoolMask_CUDA) { + torch::Device device(torch::kCUDA); + + auto a = torch::tensor(1, device); + auto uintMask = + torch::tensor(true, torch::TensorOptions(torch::kUInt8).device(device)); + auto boolMask = + torch::tensor(true, torch::TensorOptions(torch::kBool).device(device)); + assert_tensor_equal(a.index({uintMask}), a.index({boolMask})); + ASSERT_EQ(a.index({uintMask}).dtype(), a.index({boolMask}).dtype()); + + a = torch::tensor(true, torch::TensorOptions(torch::kBool).device(device)); + assert_tensor_equal(a.index({uintMask}), a.index({boolMask})); + ASSERT_EQ(a.index({uintMask}).dtype(), a.index({boolMask}).dtype()); +} + +TEST(TensorIndexingTest, TestSetitemExpansionError) { + auto true_tensor = torch::tensor(true); + auto a = torch::randn({2, 3}); + // check prefix with non-1s doesn't work + std::vector tensor_sizes{5, 1}; + tensor_sizes.insert(tensor_sizes.end(), a.sizes().begin(), a.sizes().end()); + auto a_expanded = a.expand(tensor_sizes); + // NumPy: ValueError + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(a.index_put_({true}, a_expanded), c10::Error); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(a.index_put_({true_tensor}, a_expanded), c10::Error); +} + +TEST(TensorIndexingTest, TestGetitemScalars) { + auto zero = torch::tensor(0, torch::kInt64); + auto one = torch::tensor(1, torch::kInt64); + + // non-scalar indexed with scalars + auto a = torch::randn({2, 3}); + assert_tensor_equal(a.index({0}), a.index({zero})); + assert_tensor_equal(a.index({0}).index({1}), a.index({zero}).index({one})); + assert_tensor_equal(a.index({0, 1}), a.index({zero, one})); + assert_tensor_equal(a.index({0, one}), a.index({zero, 1})); + + // indexing by a scalar should slice (not copy) + ASSERT_EQ(a.index({0, 1}).data_ptr(), a.index({zero, one}).data_ptr()); + ASSERT_EQ(a.index({1}).data_ptr(), a.index({one.to(torch::kInt)}).data_ptr()); + ASSERT_EQ( + a.index({1}).data_ptr(), a.index({one.to(torch::kShort)}).data_ptr()); + + // scalar indexed with scalar + auto r = torch::randn({}); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(r.index({Slice()}), c10::Error); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(r.index({zero}), c10::Error); + assert_tensor_equal(r, r.index({"..."})); +} + +TEST(TensorIndexingTest, TestSetitemScalars) { + auto zero = torch::tensor(0, torch::kInt64); + + // non-scalar indexed with scalars + auto a = torch::randn({2, 3}); + auto a_set_with_number = a.clone(); + auto a_set_with_scalar = a.clone(); + auto b = torch::randn({3}); + + a_set_with_number.index_put_({0}, b); + a_set_with_scalar.index_put_({zero}, b); + assert_tensor_equal(a_set_with_number, a_set_with_scalar); + a.index_put_({1, zero}, 7.7); + ASSERT_TRUE(a.index({1, 0}).allclose(torch::tensor(7.7))); + + // scalar indexed with scalars + auto r = torch::randn({}); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(r.index_put_({Slice()}, 8.8), c10::Error); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(r.index_put_({zero}, 8.8), c10::Error); + r.index_put_({"..."}, 9.9); + ASSERT_TRUE(r.allclose(torch::tensor(9.9))); +} + +TEST(TensorIndexingTest, TestBasicAdvancedCombined) { + // From the NumPy indexing example + auto x = torch::arange(0, 12).to(torch::kLong).view({4, 3}); + assert_tensor_equal( + x.index({Slice(1, 2), Slice(1, 3)}), + x.index({Slice(1, 2), torch::tensor({1, 2})})); + assert_tensor_equal( + x.index({Slice(1, 2), Slice(1, 3)}), torch::tensor({{4, 5}})); + + // Check that it is a copy + { + auto unmodified = x.clone(); + x.index({Slice(1, 2), torch::tensor({1, 2})}).zero_(); + assert_tensor_equal(x, unmodified); + } + + // But assignment should modify the original + { + auto unmodified = x.clone(); + x.index_put_({Slice(1, 2), torch::tensor({1, 2})}, 0); + assert_tensor_not_equal(x, unmodified); + } +} + +TEST(TensorIndexingTest, TestIntAssignment) { + { + auto x = torch::arange(0, 4).to(torch::kLong).view({2, 2}); + x.index_put_({1}, 5); + assert_tensor_equal(x, torch::tensor({{0, 1}, {5, 5}})); + } + + { + auto x = torch::arange(0, 4).to(torch::kLong).view({2, 2}); + x.index_put_({1}, torch::arange(5, 7).to(torch::kLong)); + assert_tensor_equal(x, torch::tensor({{0, 1}, {5, 6}})); + } +} + +TEST(TensorIndexingTest, TestByteTensorAssignment) { + auto x = torch::arange(0., 16).to(torch::kFloat).view({4, 4}); + auto b = torch::tensor({true, false, true, false}, torch::kByte); + auto value = torch::tensor({3., 4., 5., 6.}); + + { + WarningCapture warnings; + + x.index_put_({b}, value); + + ASSERT_EQ( + count_substr_occurrences( + warnings.str(), + "indexing with dtype torch.uint8 is now deprecated"), + 1); + } + + assert_tensor_equal(x.index({0}), value); + assert_tensor_equal(x.index({1}), torch::arange(4, 8).to(torch::kLong)); + assert_tensor_equal(x.index({2}), value); + assert_tensor_equal(x.index({3}), torch::arange(12, 16).to(torch::kLong)); +} + +TEST(TensorIndexingTest, TestVariableSlicing) { + auto x = torch::arange(0, 16).view({4, 4}); + auto indices = torch::tensor({0, 1}, torch::kInt); + int i = indices[0].item(); + int j = indices[1].item(); + assert_tensor_equal(x.index({Slice(i, j)}), x.index({Slice(0, 1)})); +} + +TEST(TensorIndexingTest, TestEllipsisTensor) { + auto x = torch::arange(0, 9).to(torch::kLong).view({3, 3}); + auto idx = torch::tensor({0, 2}); + assert_tensor_equal( + x.index({"...", idx}), torch::tensor({{0, 2}, {3, 5}, {6, 8}})); + assert_tensor_equal( + x.index({idx, "..."}), torch::tensor({{0, 1, 2}, {6, 7, 8}})); +} + +TEST(TensorIndexingTest, TestOutOfBoundIndex) { + auto x = torch::arange(0, 100).view({2, 5, 10}); + ASSERT_THROWS_WITH( + x.index({0, 5}), "index 5 is out of bounds for dimension 1 with size 5"); + ASSERT_THROWS_WITH( + x.index({4, 5}), "index 4 is out of bounds for dimension 0 with size 2"); + ASSERT_THROWS_WITH( + x.index({0, 1, 15}), + "index 15 is out of bounds for dimension 2 with size 10"); + ASSERT_THROWS_WITH( + x.index({Slice(), Slice(), 12}), + "index 12 is out of bounds for dimension 2 with size 10"); +} + +TEST(TensorIndexingTest, TestZeroDimIndex) { + auto x = torch::tensor(10); + + auto runner = [&]() -> torch::Tensor { + std::cout << x.index({0}) << std::endl; + return x.index({0}); + }; + + ASSERT_THROWS_WITH(runner(), "invalid index"); +} + +// The tests below are from NumPy test_indexing.py with some modifications to +// make them compatible with libtorch. It's licensed under the BDS license +// below: +// +// Copyright (c) 2005-2017, NumPy Developers. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of the NumPy Developers nor the names of any +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +TEST(NumpyTests, TestNoneIndex) { + // `None` index adds newaxis + auto a = torch::tensor({1, 2, 3}); + ASSERT_EQ(a.index({None}).dim(), a.dim() + 1); +} + +TEST(NumpyTests, TestEmptyFancyIndex) { + // Empty list index creates an empty array + auto a = torch::tensor({1, 2, 3}); + assert_tensor_equal( + a.index({torch::tensor({}, torch::kLong)}), torch::tensor({})); + + auto b = torch::tensor({}).to(torch::kLong); + assert_tensor_equal( + a.index({torch::tensor({}, torch::kLong)}), + torch::tensor({}, torch::kLong)); + + b = torch::tensor({}).to(torch::kFloat); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(a.index({b}), c10::Error); +} + +TEST(NumpyTests, TestEllipsisIndex) { + auto a = torch::tensor({{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}); + ASSERT_FALSE(a.index({"..."}).is_same(a)); + assert_tensor_equal(a.index({"..."}), a); + // `a[...]` was `a` in numpy <1.9. + ASSERT_EQ(a.index({"..."}).data_ptr(), a.data_ptr()); + + // Slicing with ellipsis can skip an + // arbitrary number of dimensions + assert_tensor_equal(a.index({0, "..."}), a.index({0})); + assert_tensor_equal(a.index({0, "..."}), a.index({0, Slice()})); + assert_tensor_equal(a.index({"...", 0}), a.index({Slice(), 0})); + + // In NumPy, slicing with ellipsis results in a 0-dim array. In PyTorch + // we don't have separate 0-dim arrays and scalars. + assert_tensor_equal(a.index({0, "...", 1}), torch::tensor(2)); + + // Assignment with `Ellipsis` on 0-d arrays + auto b = torch::tensor(1); + b.index_put_({Ellipsis}, 2); + ASSERT_EQ(b.item(), 2); +} + +TEST(NumpyTests, TestSingleIntIndex) { + // Single integer index selects one row + auto a = torch::tensor({{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}); + + assert_tensor_equal(a.index({0}), torch::tensor({1, 2, 3})); + assert_tensor_equal(a.index({-1}), torch::tensor({7, 8, 9})); + + // Index out of bounds produces IndexError + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(a.index({1 << 30}), c10::Error); + // NOTE: According to the standard + // (http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2017/p0543r0.html), for + // signed integers, if during the evaluation of an expression, the result is + // not mathematically defined or not in the range of representable values for + // its type, the behavior is undefined. Therefore, there is no way to check + // for index overflow case because it might not throw exception. + // ASSERT_THROW(a(1 << 64), c10::Error); +} + +TEST(NumpyTests, TestSingleBoolIndex) { + // Single boolean index + auto a = torch::tensor({{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}); + + assert_tensor_equal(a.index({true}), a.index({None})); + assert_tensor_equal(a.index({false}), a.index({None}).index({Slice(0, 0)})); +} + +TEST(NumpyTests, TestBooleanShapeMismatch) { + auto arr = torch::ones({5, 4, 3}); + + auto index = torch::tensor({true}); + ASSERT_THROWS_WITH(arr.index({index}), "mask"); + + index = torch::tensor({false, false, false, false, false, false}); + ASSERT_THROWS_WITH(arr.index({index}), "mask"); + + { + WarningCapture warnings; + + index = torch::empty({4, 4}, torch::kByte).zero_(); + ASSERT_THROWS_WITH(arr.index({index}), "mask"); + ASSERT_THROWS_WITH(arr.index({Slice(), index}), "mask"); + + ASSERT_EQ( + count_substr_occurrences( + warnings.str(), + "indexing with dtype torch.uint8 is now deprecated"), + 2); + } +} + +TEST(NumpyTests, TestBooleanIndexingOnedim) { + // Indexing a 2-dimensional array with + // boolean array of length one + auto a = torch::tensor({{0., 0., 0.}}); + auto b = torch::tensor({true}); + assert_tensor_equal(a.index({b}), a); + // boolean assignment + a.index_put_({b}, 1.); + assert_tensor_equal(a, torch::tensor({{1., 1., 1.}})); +} + +TEST(NumpyTests, TestBooleanAssignmentValueMismatch) { + // A boolean assignment should fail when the shape of the values + // cannot be broadcast to the subscription. (see also gh-3458) + auto a = torch::arange(0, 4); + + auto f = [](torch::Tensor a, std::vector v) -> void { + a.index_put_({a > -1}, torch::tensor(v)); + }; + + ASSERT_THROWS_WITH(f(a, {}), "shape mismatch"); + ASSERT_THROWS_WITH(f(a, {1, 2, 3}), "shape mismatch"); + ASSERT_THROWS_WITH(f(a.index({Slice(None, 1)}), {1, 2, 3}), "shape mismatch"); +} + +TEST(NumpyTests, TestBooleanIndexingTwodim) { + // Indexing a 2-dimensional array with + // 2-dimensional boolean array + auto a = torch::tensor({{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}); + auto b = torch::tensor( + {{true, false, true}, {false, true, false}, {true, false, true}}); + assert_tensor_equal(a.index({b}), torch::tensor({1, 3, 5, 7, 9})); + assert_tensor_equal(a.index({b.index({1})}), torch::tensor({{4, 5, 6}})); + assert_tensor_equal(a.index({b.index({0})}), a.index({b.index({2})})); + + // boolean assignment + a.index_put_({b}, 0); + assert_tensor_equal(a, torch::tensor({{0, 2, 0}, {4, 0, 6}, {0, 8, 0}})); +} + +TEST(NumpyTests, TestBooleanIndexingWeirdness) { + // Weird boolean indexing things + auto a = torch::ones({2, 3, 4}); + ASSERT_EQ( + a.index({false, true, "..."}).sizes(), torch::IntArrayRef({0, 2, 3, 4})); + assert_tensor_equal( + torch::ones({1, 2}), + a.index( + {true, + torch::tensor({0, 1}), + true, + true, + torch::tensor({1}), + torch::tensor({{2}})})); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(a.index({false, torch::tensor({0, 1}), "..."}), c10::Error); +} + +TEST(NumpyTests, TestBooleanIndexingWeirdnessTensors) { + // Weird boolean indexing things + auto false_tensor = torch::tensor(false); + auto true_tensor = torch::tensor(true); + auto a = torch::ones({2, 3, 4}); + ASSERT_EQ( + a.index({false, true, "..."}).sizes(), torch::IntArrayRef({0, 2, 3, 4})); + assert_tensor_equal( + torch::ones({1, 2}), + a.index( + {true_tensor, + torch::tensor({0, 1}), + true_tensor, + true_tensor, + torch::tensor({1}), + torch::tensor({{2}})})); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW( + a.index({false_tensor, torch::tensor({0, 1}), "..."}), c10::Error); +} + +TEST(NumpyTests, TestBooleanIndexingAlldims) { + auto true_tensor = torch::tensor(true); + auto a = torch::ones({2, 3}); + ASSERT_EQ(a.index({true, true}).sizes(), torch::IntArrayRef({1, 2, 3})); + ASSERT_EQ( + a.index({true_tensor, true_tensor}).sizes(), + torch::IntArrayRef({1, 2, 3})); +} + +TEST(NumpyTests, TestBooleanListIndexing) { + // Indexing a 2-dimensional array with + // boolean lists + auto a = torch::tensor({{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}); + auto b = torch::tensor({true, false, false}); + auto c = torch::tensor({true, true, false}); + assert_tensor_equal(a.index({b}), torch::tensor({{1, 2, 3}})); + assert_tensor_equal(a.index({b, b}), torch::tensor({1})); + assert_tensor_equal(a.index({c}), torch::tensor({{1, 2, 3}, {4, 5, 6}})); + assert_tensor_equal(a.index({c, c}), torch::tensor({1, 5})); +} + +TEST(NumpyTests, TestEverythingReturnsViews) { + // Before `...` would return a itself. + auto a = torch::tensor({5}); + + ASSERT_FALSE(a.is_same(a.index({"..."}))); + ASSERT_FALSE(a.is_same(a.index({Slice()}))); +} + +TEST(NumpyTests, TestBroaderrorsIndexing) { + auto a = torch::zeros({5, 5}); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW( + a.index({torch::tensor({0, 1}), torch::tensor({0, 1, 2})}), c10::Error); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW( + a.index_put_({torch::tensor({0, 1}), torch::tensor({0, 1, 2})}, 0), + c10::Error); +} + +TEST(NumpyTests, TestTrivialFancyOutOfBounds) { + auto a = torch::zeros({5}); + auto ind = torch::ones({20}, torch::kInt64); + ind.index_put_({-1}, 10); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(a.index({ind}), c10::Error); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(a.index_put_({ind}, 0), c10::Error); + ind = torch::ones({20}, torch::kInt64); + ind.index_put_({0}, 11); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(a.index({ind}), c10::Error); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_THROW(a.index_put_({ind}, 0), c10::Error); +} + +TEST(NumpyTests, TestIndexIsLarger) { + // Simple case of fancy index broadcasting of the index. + auto a = torch::zeros({5, 5}); + a.index_put_( + {torch::tensor({{0}, {1}, {2}}), torch::tensor({0, 1, 2})}, + torch::tensor({2., 3., 4.})); + + ASSERT_TRUE( + (a.index({Slice(None, 3), Slice(None, 3)}) == torch::tensor({2., 3., 4.})) + .all() + .item()); +} + +TEST(NumpyTests, TestBroadcastSubspace) { + auto a = torch::zeros({100, 100}); + auto v = torch::arange(0., 100).index({Slice(), None}); + auto b = torch::arange(99, -1, -1).to(torch::kLong); + a.index_put_({b}, v); + auto expected = b.to(torch::kDouble).unsqueeze(1).expand({100, 100}); + assert_tensor_equal(a, expected); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/tensor_options.cpp b/western_ai/usa/pytorch/test/cpp/api/tensor_options.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d9f4d281d479898a91cb64534e0cbdacce4cae61 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/tensor_options.cpp @@ -0,0 +1,168 @@ +#include +#include + +#include + +#include +#include + +using namespace at; +using namespace torch::test; + +// A macro so we don't lose location information when an assertion fails. +#define REQUIRE_OPTIONS(device_, index_, type_, layout_) \ + ASSERT_EQ(options.device().type(), Device((device_), (index_)).type()); \ + ASSERT_TRUE( \ + options.device().index() == Device((device_), (index_)).index()); \ + ASSERT_EQ(options.dtype(), (type_)); \ + ASSERT_TRUE(options.layout() == (layout_)) + +#define REQUIRE_TENSOR_OPTIONS(device_, index_, type_, layout_) \ + ASSERT_EQ(tensor.device().type(), Device((device_), (index_)).type()); \ + ASSERT_EQ(tensor.device().index(), Device((device_), (index_)).index()); \ + ASSERT_EQ(tensor.scalar_type(), (type_)); \ + ASSERT_TRUE(tensor.options().layout() == (layout_)) + +TEST(TensorOptionsTest, DefaultsToTheRightValues) { + TensorOptions options; + REQUIRE_OPTIONS(kCPU, -1, kFloat, kStrided); +} + +TEST(TensorOptionsTest, UtilityFunctionsReturnTheRightTensorOptions) { + auto options = dtype(kInt); + REQUIRE_OPTIONS(kCPU, -1, kInt, kStrided); + + options = layout(kSparse); + REQUIRE_OPTIONS(kCPU, -1, kFloat, kSparse); + + options = device({kCUDA, 1}); + REQUIRE_OPTIONS(kCUDA, 1, kFloat, kStrided); + + options = device_index(1); + REQUIRE_OPTIONS(kCUDA, 1, kFloat, kStrided); + + options = dtype(kByte).layout(kSparse).device(kCUDA, 2).device_index(3); + REQUIRE_OPTIONS(kCUDA, 3, kByte, kSparse); +} + +TEST(TensorOptionsTest, ConstructsWellFromCPUTypes) { + TensorOptions options; + REQUIRE_OPTIONS(kCPU, -1, kFloat, kStrided); + + options = TensorOptions({kCPU, 0}); + REQUIRE_OPTIONS(kCPU, 0, kFloat, kStrided); + + options = TensorOptions("cpu:0"); + REQUIRE_OPTIONS(kCPU, 0, kFloat, kStrided); + + options = TensorOptions(kInt); + REQUIRE_OPTIONS(kCPU, -1, kInt, kStrided); + + options = + TensorOptions(getDeprecatedTypeProperties(Backend::SparseCPU, kFloat)); + REQUIRE_OPTIONS(kCPU, -1, kFloat, kSparse); + + options = + TensorOptions(getDeprecatedTypeProperties(Backend::SparseCPU, kByte)); + REQUIRE_OPTIONS(kCPU, -1, kByte, kSparse); +} + +TEST(TensorOptionsTest, ConstructsWellFromCPUTensors) { + auto options = empty(5, kDouble).options(); + REQUIRE_OPTIONS(kCPU, -1, kDouble, kStrided); + + options = empty(5, getDeprecatedTypeProperties(Backend::SparseCPU, kByte)) + .options(); + REQUIRE_OPTIONS(kCPU, -1, kByte, kSparse); +} + +TEST(TensorOptionsTest, ConstructsWellFromVariables) { + auto options = torch::empty(5).options(); + REQUIRE_OPTIONS(kCPU, -1, kFloat, kStrided); + ASSERT_FALSE(options.requires_grad()); + + options = torch::empty(5, at::requires_grad()).options(); + REQUIRE_OPTIONS(kCPU, -1, kFloat, kStrided); + ASSERT_FALSE(options.requires_grad()); +} + +TEST(TensorOptionsTest, MkldnnDeviceAllocationFailsCleanly) { + ASSERT_THROWS_WITH( + torch::empty({2}, TensorOptions().device(Device(DeviceType::MKLDNN))), + "The 'mkldnn' device type is deprecated and cannot be used to allocate " + "dense tensors"); +} + +TEST(DeviceTest, ParsesCorrectlyFromString) { + Device device("cpu:0"); + ASSERT_EQ(device, Device(DeviceType::CPU, 0)); + + device = Device("cpu"); + ASSERT_EQ(device, Device(DeviceType::CPU)); + + device = Device("cuda:123"); + ASSERT_EQ(device, Device(DeviceType::CUDA, 123)); + + device = Device("cuda"); + ASSERT_EQ(device, Device(DeviceType::CUDA)); + + device = Device("mkldnn"); + ASSERT_EQ(device, Device(DeviceType::MKLDNN)); + + device = Device("opengl"); + ASSERT_EQ(device, Device(DeviceType::OPENGL)); + + device = Device("opencl"); + ASSERT_EQ(device, Device(DeviceType::OPENCL)); + + device = Device("ideep"); + ASSERT_EQ(device, Device(DeviceType::IDEEP)); + + device = Device("hip"); + ASSERT_EQ(device, Device(DeviceType::HIP)); + + device = Device("hip:123"); + ASSERT_EQ(device, Device(DeviceType::HIP, 123)); + + std::vector badnesses = { + "", "cud:1", "cuda:", "cpu::1", ":1", "3", "tpu:4", "??"}; + for (const auto& badness : badnesses) { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_ANY_THROW({ Device d(badness); }); + } +} + +TEST(DefaultDtypeTest, CanSetAndGetDefaultDtype) { + AutoDefaultDtypeMode dtype_mode(kFloat); + + ASSERT_EQ(at::get_default_dtype(), kFloat); + set_default_dtype(caffe2::TypeMeta::Make()); + ASSERT_EQ(at::get_default_dtype(), kInt); +} + +TEST(DefaultDtypeTest, NewTensorOptionsHasCorrectDefault) { + AutoDefaultDtypeMode dtype_mode(kFloat); + + set_default_dtype(caffe2::TypeMeta::Make()); + ASSERT_EQ(at::get_default_dtype(), kInt); + TensorOptions options; + ASSERT_EQ(options.dtype(), kInt); +} + +TEST(DefaultDtypeTest, NewTensorsHaveCorrectDefaultDtype) { + AutoDefaultDtypeMode dtype_mode(kFloat); + set_default_dtype(caffe2::TypeMeta::Make()); + { + auto tensor = torch::ones(5); + ASSERT_EQ(tensor.dtype(), kInt); + } + set_default_dtype(caffe2::TypeMeta::Make()); + { + auto tensor = torch::ones(5); + ASSERT_EQ(tensor.dtype(), kDouble); + } + { + auto tensor = torch::ones(5, kFloat); + ASSERT_EQ(tensor.dtype(), kFloat); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/tensor_options_cuda.cpp b/western_ai/usa/pytorch/test/cpp/api/tensor_options_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..53d58ee5a1b27f2ce81e898b5db68d43afe24e08 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/tensor_options_cuda.cpp @@ -0,0 +1,82 @@ +#include + +#include + +#include + +// NB: This file is compiled even in CPU build (for some reason), so +// make sure you don't include any CUDA only headers. + +using namespace at; + +// TODO: This might be generally helpful aliases elsewhere. +at::Device CPUDevice() { + return at::Device(at::kCPU); +} +at::Device CUDADevice(DeviceIndex index) { + return at::Device(at::kCUDA, index); +} + +// A macro so we don't lose location information when an assertion fails. +#define REQUIRE_OPTIONS(device_, index_, type_, layout_) \ + ASSERT_EQ(options.device().type(), Device((device_), (index_)).type()); \ + ASSERT_TRUE( \ + options.device().index() == Device((device_), (index_)).index()); \ + ASSERT_EQ(typeMetaToScalarType(options.dtype()), (type_)); \ + ASSERT_TRUE(options.layout() == (layout_)) + +#define REQUIRE_TENSOR_OPTIONS(device_, index_, type_, layout_) \ + ASSERT_EQ(tensor.device().type(), Device((device_), (index_)).type()); \ + ASSERT_EQ(tensor.device().index(), Device((device_), (index_)).index()); \ + ASSERT_EQ(tensor.scalar_type(), (type_)); \ + ASSERT_TRUE(tensor.options().layout() == (layout_)) + +TEST(TensorOptionsTest, ConstructsWellFromCUDATypes_CUDA) { + auto options = CUDA(kFloat).options(); + REQUIRE_OPTIONS(kCUDA, -1, kFloat, kStrided); + + options = CUDA(kInt).options(); + REQUIRE_OPTIONS(kCUDA, -1, kInt, kStrided); + + options = getDeprecatedTypeProperties(Backend::SparseCUDA, kFloat).options(); + REQUIRE_OPTIONS(kCUDA, -1, kFloat, kSparse); + + options = getDeprecatedTypeProperties(Backend::SparseCUDA, kByte).options(); + REQUIRE_OPTIONS(kCUDA, -1, kByte, kSparse); + + // NOLINTNEXTLINE(bugprone-argument-comment,cppcoreguidelines-avoid-magic-numbers) + options = CUDA(kFloat).options(/*device=*/5); + REQUIRE_OPTIONS(kCUDA, 5, kFloat, kStrided); + + options = + // NOLINTNEXTLINE(bugprone-argument-comment,cppcoreguidelines-avoid-magic-numbers) + getDeprecatedTypeProperties(Backend::SparseCUDA, kFloat) + .options(/*device=*/5); + REQUIRE_OPTIONS(kCUDA, 5, kFloat, kSparse); +} + +TEST(TensorOptionsTest, ConstructsWellFromCUDATensors_MultiCUDA) { + auto options = empty(5, device(kCUDA).dtype(kDouble)).options(); + REQUIRE_OPTIONS(kCUDA, 0, kDouble, kStrided); + + options = empty(5, getDeprecatedTypeProperties(Backend::SparseCUDA, kByte)) + .options(); + REQUIRE_OPTIONS(kCUDA, 0, kByte, kSparse); + + if (torch::cuda::device_count() > 1) { + Tensor tensor; + { + DeviceGuard guard(CUDADevice(1)); + tensor = empty(5, device(kCUDA)); + } + options = tensor.options(); + REQUIRE_OPTIONS(kCUDA, 1, kFloat, kStrided); + + { + DeviceGuard guard(CUDADevice(1)); + tensor = empty(5, device(kCUDA).layout(kSparse)); + } + options = tensor.options(); + REQUIRE_OPTIONS(kCUDA, 1, kFloat, kSparse); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/api/torch_include.cpp b/western_ai/usa/pytorch/test/cpp/api/torch_include.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5afc790f4591c5dc35d55f82f7b7559bef393642 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/torch_include.cpp @@ -0,0 +1,14 @@ +#include + +#include + +// NOTE: This test suite exists to make sure that common `torch::` functions +// can be used without additional includes beyond `torch/torch.h`. + +TEST(TorchIncludeTest, GetSetNumThreads) { + torch::init_num_threads(); + torch::set_num_threads(2); + torch::set_num_interop_threads(2); + torch::get_num_threads(); + torch::get_num_interop_threads(); +} diff --git a/western_ai/usa/pytorch/test/cpp/api/transformer.cpp b/western_ai/usa/pytorch/test/cpp/api/transformer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9df4600bb62ee62775587ed716fd036d1807896c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/api/transformer.cpp @@ -0,0 +1,1523 @@ +#include + +#include + +#include + +using namespace torch::nn; + +struct TransformerTest : torch::test::SeedingFixture {}; + +// a generic function to set constants for parameters so we have fixed result +// for deterministic test +template +void set_parameter_to_constants( + Model& model, + const torch::TensorOptions& tensor_options) { + torch::NoGradGuard guard; + for (auto& p : model->parameters()) { + auto sz = p.view(-1).size(0); + p.copy_(torch::cos(torch::arange(0, sz, tensor_options).view(p.sizes()))); + } +} + +// a generic function to provide consistent encoder/decoder layer for all the +// transformer tests +template +T_LAYER get_a_test_layer( + const torch::TensorOptions& tensor_options, + bool use_callable_activation) { + int64_t d_model = 4; + int64_t nhead = 2; + int64_t dim_feedforward = 16; + double dropout = 0.0; + + // activation is always ReLU here and it can be adjusted later depending on + // the usage + T_LAYER layer(T_OPTIONS(d_model, nhead) + .dim_feedforward(dim_feedforward) + .dropout(dropout)); + if (tensor_options.device() == torch::kCUDA) { + layer->to(torch::kCUDA); + } + if (use_callable_activation) { + layer.get()->options.activation( + [&](const torch::Tensor& t) { return torch::nn::functional::relu(t); }); + } + + // set constant weights of the model + set_parameter_to_constants(layer, tensor_options); + + return layer; +} + +void transformer_encoder_layer_test_helper( + bool is_cuda, + bool use_callable_activation) { + // this is a deterministic test for TransformerEncoderLayer + torch::Device device = is_cuda ? torch::kCUDA : torch::kCPU; + torch::TensorOptions tensor_options = + torch::TensorOptions().dtype(torch::kFloat32).device(device); + + TransformerEncoderLayer model = + get_a_test_layer( + tensor_options, use_callable_activation); + + // relu test case 1 + torch::Tensor encoder_input = + torch::tensor({{{20, 30, 40, 50}}}, tensor_options); + torch::Tensor result = model(encoder_input).detach(); + torch::Tensor ref_output = torch::tensor( + {{{2.258703, 0.127985, -0.697881, 0.170862}}}, tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // all 0 values are NOT masked. This shouldn't mask anything + torch::Tensor mask = torch::tensor({{0}}, tensor_options) == 1; + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // all 1 values are masked. Since there is only 1 input embedding this will + // result in nan. + mask = torch::tensor({{1}}, tensor_options) == 1; + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ASSERT_TRUE(torch::isnan(result).all().item().to()); + + // relu test case 2 + encoder_input = + torch::tensor({{{1, 2, 3, 4}}, {{5, 6, 7, 8}}}, tensor_options); + result = model(encoder_input).detach(); + ref_output = torch::tensor( + {{{2.272644, 0.119035, -0.691669, 0.153486}}, + {{2.272644, 0.119035, -0.691669, 0.153486}}}, + tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // all 0 values are NOT masked + mask = torch::tensor({{0, 0}}, tensor_options) == 1; + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // mask with 1 and 0 + mask = torch::tensor({{1, 0}}, tensor_options) == 1; + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ref_output = torch::tensor( + {{{2.301516, 0.092249, -0.679101, 0.103088}}, + {{2.301516, 0.092249, -0.679101, 0.103088}}}, + tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // relu test case 3 + encoder_input = torch::tensor( + {{{0.7462, 0.6653, 0.5679, 0.4891}, {0.5387, 0.1655, 0.3565, 0.0471}}, + {{0.8335, 0.2799, 0.5031, 0.2947}, {0.1402, 0.0318, 0.7636, 0.1346}}, + {{0.6333, 0.9344, 0.1376, 0.9938}, {0.8924, 0.2872, 0.6692, 0.2944}}, + {{0.9897, 0.6915, 0.3154, 0.1733}, {0.8645, 0.3513, 0.3064, 0.0767}}, + {{0.8117, 0.2366, 0.4838, 0.7881}, {0.3718, 0.4945, 0.9511, 0.0864}}}, + tensor_options); + result = model(encoder_input).detach(); + ref_output = torch::tensor( + {{{2.428589, 0.020835, -0.602055, -0.085249}, + {2.427987, 0.021213, -0.602496, -0.084103}}, + {{2.424689, 0.019155, -0.604793, -0.085672}, + {2.413863, 0.022211, -0.612486, -0.072490}}, + {{2.433774, 0.021598, -0.598343, -0.087548}, + {2.425104, 0.019748, -0.604515, -0.084839}}, + {{2.436185, 0.022682, -0.596625, -0.087261}, + {2.433556, 0.021891, -0.598509, -0.086832}}, + {{2.416246, 0.017512, -0.610712, -0.082961}, + {2.422901, 0.024187, -0.606178, -0.074929}}}, + tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // all 0 values are NOT masked + mask = torch::zeros({2, 5}, tensor_options) == 1; + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // mask with 0s and 1s + mask[0][1] = 1; + mask[1][3] = 1; + mask[1][4] = 1; + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ref_output = torch::tensor( + {{{2.429026, 0.020793, -0.601741, -0.085642}, + {2.428811, 0.021445, -0.601912, -0.084252}}, + {{2.425009, 0.019155, -0.604566, -0.085899}, + {2.415408, 0.02249, -0.611415, -0.073}}, + {{2.434199, 0.021682, -0.598039, -0.087699}, + {2.42598, 0.019941, -0.603896, -0.085091}}, + {{2.436457, 0.022736, -0.59643, -0.08736}, + {2.434021, 0.022093, -0.598179, -0.08679}}, + {{2.416531, 0.017498, -0.610513, -0.083181}, + {2.4242, 0.024653, -0.605266, -0.074959}}}, + tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // gelu test case 1 + model.get()->options.activation(torch::kGELU); + encoder_input = torch::tensor({{{20, 30, 40, 50}}}, tensor_options); + result = model(encoder_input).detach(); + ref_output = torch::tensor( + {{{2.249815, 0.131006, -0.702199, 0.177868}}}, tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // gelu test case 2 + encoder_input = torch::tensor( + {{{0.7462, 0.6653, 0.5679, 0.4891}, {0.5387, 0.1655, 0.3565, 0.0471}}, + {{0.8335, 0.2799, 0.5031, 0.2947}, {0.1402, 0.0318, 0.7636, 0.1346}}, + {{0.6333, 0.9344, 0.1376, 0.9938}, {0.8924, 0.2872, 0.6692, 0.2944}}, + {{0.9897, 0.6915, 0.3154, 0.1733}, {0.8645, 0.3513, 0.3064, 0.0767}}, + {{0.8117, 0.2366, 0.4838, 0.7881}, {0.3718, 0.4945, 0.9511, 0.0864}}}, + tensor_options); + result = model(encoder_input); + ref_output = torch::tensor( + {{{2.42163188, 0.03227153, -0.60714219, -0.05908082}, + {2.42151276, 0.03302179, -0.60722523, -0.05762651}}, + {{2.41926761, 0.02974034, -0.60879519, -0.0621269}, + {2.41626395, 0.03539356, -0.61087842, -0.04978623}}, + {{2.42382808, 0.03218872, -0.6055963, -0.06073591}, + {2.41983477, 0.03085259, -0.60840145, -0.06046414}}, + {{2.42500749, 0.03328855, -0.60476388, -0.0595334}, + {2.4237977, 0.03290575, -0.60561789, -0.05940082}}, + {{2.41383916, 0.02686345, -0.61256377, -0.06380707}, + {2.42000277, 0.03800944, -0.60824798, -0.04754947}}}, + tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); +} + +TEST_F(TransformerTest, TransformerEncoderLayer) { + transformer_encoder_layer_test_helper( + /*is_cuda=*/false, /*use_callable_activation=*/false); + transformer_encoder_layer_test_helper( + /*is_cuda=*/false, /*use_callable_activation=*/true); +} + +TEST_F(TransformerTest, TransformerEncoderLayer_CUDA) { + transformer_encoder_layer_test_helper( + /*is_cuda=*/true, /*use_callable_activation=*/false); + transformer_encoder_layer_test_helper( + /*is_cuda=*/true, /*use_callable_activation=*/true); +} + +void transformer_decoder_layer_test_helper( + bool is_cuda, + bool use_callable_activation) { + torch::Device device = is_cuda ? torch::kCUDA : torch::kCPU; + torch::TensorOptions tensor_options = + torch::TensorOptions().dtype(torch::kFloat32).device(device); + + TransformerDecoderLayer model = + get_a_test_layer( + tensor_options, use_callable_activation); + + // deterministic input + torch::Tensor decoder_input = + torch::tensor({{{20, 30, 40, 50}}}, tensor_options); + torch::Tensor memory_input = + torch::tensor({{{60, 70, 80, 90}}}, tensor_options); + torch::Tensor result = model(decoder_input, memory_input).detach(); + torch::Tensor ref_output = torch::tensor( + {{{2.314351, 0.094805, -0.671322, 0.101977}}}, tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = + torch::tensor({{{9, 10, 11, 12}}, {{11, 12, 13, 14}}}, tensor_options); + memory_input = torch::tensor({{{1, 2, 3, 4}}}, tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.422245, 0.051716, -0.606338, -0.024756}}, + {{2.422245, 0.051716, -0.606338, -0.024756}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = + torch::tensor({{{1, 2, 3, 4}}, {{5, 6, 7, 8}}}, tensor_options); + memory_input = + torch::tensor({{{9, 10, 11, 12}}, {{11, 12, 13, 14}}}, tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.343536, 0.085561, -0.654954, 0.074991}}, + {{2.343536, 0.085561, -0.654954, 0.074991}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = torch::tensor( + {{{0.4517, 0.6793, 0.5313, 0.0034}, {0.2678, 0.3677, 0.4459, 0.7166}}, + {{0.8100, 0.3716, 0.4096, 0.1976}, {0.6958, 0.8844, 0.6081, 0.8315}}, + {{0.0494, 0.9343, 0.5955, 0.3830}, {0.5404, 0.3464, 0.9378, 0.6200}}}, + tensor_options); + memory_input = torch::tensor( + {{{0.7462, 0.6653, 0.5679, 0.4891}, {0.5387, 0.1655, 0.3565, 0.0471}}, + {{0.8335, 0.2799, 0.5031, 0.2947}, {0.1402, 0.0318, 0.7636, 0.1346}}, + {{0.6333, 0.9344, 0.1376, 0.9938}, {0.8924, 0.2872, 0.6692, 0.2944}}, + {{0.9897, 0.6915, 0.3154, 0.1733}, {0.8645, 0.3513, 0.3064, 0.0767}}, + {{0.8117, 0.2366, 0.4838, 0.7881}, {0.3718, 0.4945, 0.9511, 0.0864}}}, + tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.430065, 0.027862, -0.601136, -0.073096}, + {2.431935, 0.028907, -0.599809, -0.072488}}, + {{2.428457, 0.027053, -0.602275, -0.073462}, + {2.431970, 0.029387, -0.599789, -0.071621}}, + {{2.431934, 0.028196, -0.599802, -0.073809}, + {2.432306, 0.028858, -0.599542, -0.072846}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // key_padding_mask + torch::Tensor t_mask = {}; + torch::Tensor m_mask = {}; + torch::Tensor key_padding_mask = torch::zeros({2, 3}, tensor_options) == 1; + result = model(decoder_input, memory_input, t_mask, m_mask, key_padding_mask) + .detach(); + ref_output = torch::tensor( + {{{2.430065, 0.027862, -0.601136, -0.073096}, + {2.431935, 0.028907, -0.599809, -0.072488}}, + {{2.428457, 0.027053, -0.602275, -0.073462}, + {2.431970, 0.029387, -0.599789, -0.071621}}, + {{2.431934, 0.028196, -0.599802, -0.073809}, + {2.432306, 0.028858, -0.599542, -0.072846}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // key_padding_mask + key_padding_mask[0][2] = 1; + key_padding_mask[1][1] = 1; + key_padding_mask[1][2] = 1; + result = model(decoder_input, memory_input, t_mask, m_mask, key_padding_mask) + .detach(); + ref_output = torch::tensor( + {{{2.430025, 0.027643, -0.601164, -0.073476}, + {2.4323, 0.029375, -0.599553, -0.071881}}, + {{2.428523, 0.026838, -0.602226, -0.07391}, + {2.432634, 0.029842, -0.599318, -0.071253}}, + {{2.432278, 0.028152, -0.599555, -0.074139}, + {2.432659, 0.029244, -0.599294, -0.072382}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // memory_key_padding_mask + torch::Tensor t_key_padding_mask = {}; + key_padding_mask = torch::zeros({2, 5}, tensor_options) == 1; + result = model( + decoder_input, + memory_input, + t_mask, + m_mask, + t_key_padding_mask, + key_padding_mask) + .detach(); + ref_output = torch::tensor( + {{{2.430065, 0.027862, -0.601136, -0.073096}, + {2.431935, 0.028907, -0.599809, -0.072488}}, + {{2.428457, 0.027053, -0.602275, -0.073462}, + {2.431970, 0.029387, -0.599789, -0.071621}}, + {{2.431934, 0.028196, -0.599802, -0.073809}, + {2.432306, 0.028858, -0.599542, -0.072846}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // memory_key_padding_mask + key_padding_mask[0][4] = 1; + key_padding_mask[1][3] = 1; + key_padding_mask[1][4] = 1; + result = model( + decoder_input, + memory_input, + t_mask, + m_mask, + t_key_padding_mask, + key_padding_mask) + .detach(); + ref_output = torch::tensor( + {{{2.429757, 0.027358, -0.601351, -0.073816}, + {2.432692, 0.028583, -0.599263, -0.073634}}, + {{2.428247, 0.02662, -0.602419, -0.074123}, + {2.432657, 0.029055, -0.599293, -0.072732}}, + {{2.431515, 0.027687, -0.600096, -0.074459}, + {2.433075, 0.028543, -0.598987, -0.073985}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); +} + +TEST_F(TransformerTest, TransformerDecoderLayer) { + transformer_decoder_layer_test_helper( + /*is_cuda=*/false, /*use_callable_activation=*/false); + transformer_decoder_layer_test_helper( + /*is_cuda=*/false, /*use_callable_activation=*/true); +} + +TEST_F(TransformerTest, TransformerDecoderLayer_CUDA) { + transformer_decoder_layer_test_helper( + /*is_cuda=*/true, /*use_callable_activation=*/false); + transformer_decoder_layer_test_helper( + /*is_cuda=*/true, /*use_callable_activation=*/true); +} + +void transformer_decoder_layer_test_helper_gelu( + bool is_cuda, + bool use_callable_activation) { + torch::Device device = is_cuda ? torch::kCUDA : torch::kCPU; + torch::TensorOptions tensor_options = + torch::TensorOptions().dtype(torch::kFloat32).device(device); + + TransformerDecoderLayer model = + get_a_test_layer( + tensor_options, use_callable_activation); + if (use_callable_activation) { + model.get()->options.activation( + [&](const torch::Tensor& t) { return torch::nn::functional::gelu(t); }); + } else { + model.get()->options.activation(torch::kGELU); + } + + // deterministic input + torch::Tensor decoder_input = + torch::tensor({{{20, 30, 40, 50}}}, tensor_options); + torch::Tensor memory_input = + torch::tensor({{{60, 70, 80, 90}}}, tensor_options); + torch::Tensor result = model(decoder_input, memory_input).detach(); + torch::Tensor ref_output = torch::tensor( + {{{2.306435, 0.095946, -0.675796, 0.10687}}}, tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = + torch::tensor({{{9, 10, 11, 12}}, {{11, 12, 13, 14}}}, tensor_options); + memory_input = torch::tensor({{{1, 2, 3, 4}}}, tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.415448, 0.054389, -0.610932, -0.0156613}}, + {{2.415448, 0.054389, -0.610932, -0.0156613}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = + torch::tensor({{{1, 2, 3, 4}}, {{5, 6, 7, 8}}}, tensor_options); + memory_input = + torch::tensor({{{9, 10, 11, 12}}, {{11, 12, 13, 14}}}, tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.338531, 0.087709, -0.65776, 0.080646}}, + {{2.338531, 0.087709, -0.65776, 0.080646}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = torch::tensor( + {{{0.4517, 0.6793, 0.5313, 0.0034}, {0.2678, 0.3677, 0.4459, 0.7166}}, + {{0.8100, 0.3716, 0.4096, 0.1976}, {0.6958, 0.8844, 0.6081, 0.8315}}, + {{0.0494, 0.9343, 0.5955, 0.3830}, {0.5404, 0.3464, 0.9378, 0.6200}}}, + tensor_options); + memory_input = torch::tensor( + {{{0.7462, 0.6653, 0.5679, 0.4891}, {0.5387, 0.1655, 0.3565, 0.0471}}, + {{0.8335, 0.2799, 0.5031, 0.2947}, {0.1402, 0.0318, 0.7636, 0.1346}}, + {{0.6333, 0.9344, 0.1376, 0.9938}, {0.8924, 0.2872, 0.6692, 0.2944}}, + {{0.9897, 0.6915, 0.3154, 0.1733}, {0.8645, 0.3513, 0.3064, 0.0767}}, + {{0.8117, 0.2366, 0.4838, 0.7881}, {0.3718, 0.4945, 0.9511, 0.0864}}}, + tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.42049104, 0.03443088, -0.60793706, -0.05436271}, + {2.42210631, 0.03546578, -0.60679895, -0.05357488}}, + {{2.41907674, 0.0336104, -0.60892977, -0.05490462}, + {2.42216881, 0.03586554, -0.6067524, -0.05289126}}, + {{2.42205716, 0.03488046, -0.60683681, -0.05460596}, + {2.42240309, 0.0354595, -0.60659063, -0.05378816}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); +} + +TEST_F(TransformerTest, TransformerDecoderLayer_gelu) { + transformer_decoder_layer_test_helper_gelu( + /*is_cuda=*/false, /*use_callable_activation=*/false); + transformer_decoder_layer_test_helper_gelu( + /*is_cuda=*/false, /*use_callable_activation=*/true); +} + +TEST_F(TransformerTest, TransformerDecoderLayer_gelu_CUDA) { + transformer_decoder_layer_test_helper_gelu( + /*is_cuda=*/true, /*use_callable_activation=*/false); + transformer_decoder_layer_test_helper_gelu( + /*is_cuda=*/true, /*use_callable_activation=*/true); +} + +void transformer_encoder_test_helper( + bool is_cuda, + bool use_callable_activation) { + // this is a deterministic test for TransformerEncoderLayer + torch::Device device = is_cuda ? torch::kCUDA : torch::kCPU; + torch::TensorOptions tensor_options = + torch::TensorOptions().dtype(torch::kFloat32).device(device); + + TransformerEncoderLayer encoder_layer = + get_a_test_layer( + tensor_options, use_callable_activation); + + TransformerEncoder model(TransformerEncoderOptions(encoder_layer, 1)); + if (is_cuda) { + model->to(torch::kCUDA); + } + + torch::Tensor encoder_input = torch::tensor( + {{{0.7462, 0.6653, 0.5679, 0.4891}, {0.5387, 0.1655, 0.3565, 0.0471}}, + {{0.8335, 0.2799, 0.5031, 0.2947}, {0.1402, 0.0318, 0.7636, 0.1346}}, + {{0.6333, 0.9344, 0.1376, 0.9938}, {0.8924, 0.2872, 0.6692, 0.2944}}, + {{0.9897, 0.6915, 0.3154, 0.1733}, {0.8645, 0.3513, 0.3064, 0.0767}}, + {{0.8117, 0.2366, 0.4838, 0.7881}, {0.3718, 0.4945, 0.9511, 0.0864}}}, + tensor_options); + torch::Tensor result = model(encoder_input).detach(); + torch::Tensor ref_output = torch::tensor( + {{{2.428589, 0.020835, -0.602055, -0.085249}, + {2.427987, 0.021213, -0.602496, -0.084103}}, + {{2.424689, 0.019155, -0.604793, -0.085672}, + {2.413863, 0.022211, -0.612486, -0.072490}}, + {{2.433774, 0.021598, -0.598343, -0.087548}, + {2.425104, 0.019748, -0.604515, -0.084839}}, + {{2.436185, 0.022682, -0.596625, -0.087261}, + {2.433556, 0.021891, -0.598509, -0.086832}}, + {{2.416246, 0.017512, -0.610712, -0.082961}, + {2.422901, 0.024187, -0.606178, -0.074929}}}, + tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // all 0 values are NOT masked + torch::Tensor mask = torch::zeros({2, 5}, tensor_options) == 1; + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // mask with 0s and 1s + mask[0][1] = 1; + mask[1][3] = 1; + mask[1][4] = 1; + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ref_output = torch::tensor( + {{{2.429026, 0.020793, -0.601741, -0.085642}, + {2.428811, 0.021445, -0.601912, -0.084252}}, + {{2.425009, 0.019155, -0.604566, -0.085899}, + {2.415408, 0.02249, -0.611415, -0.073}}, + {{2.434199, 0.021682, -0.598039, -0.087699}, + {2.42598, 0.019941, -0.603896, -0.085091}}, + {{2.436457, 0.022736, -0.59643, -0.08736}, + {2.434021, 0.022093, -0.598179, -0.08679}}, + {{2.416531, 0.017498, -0.610513, -0.083181}, + {2.4242, 0.024653, -0.605266, -0.074959}}}, + tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // test case 2, multiple layers no norm + model = TransformerEncoder(TransformerEncoderOptions(encoder_layer, 2)); + if (is_cuda) { + model->to(torch::kCUDA); + } + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ref_output = torch::tensor( + {{{2.419051, 0.017446, -0.608738, -0.085003}, + {2.419102, 0.017452, -0.608703, -0.085026}}, + {{2.419043, 0.017445, -0.608744, -0.084999}, + {2.419052, 0.017446, -0.608738, -0.085004}}, + {{2.419067, 0.017448, -0.608727, -0.085010}, + {2.419098, 0.017452, -0.608706, -0.085024}}, + {{2.419072, 0.017449, -0.608724, -0.085012}, + {2.419119, 0.017455, -0.608691, -0.085034}}, + {{2.419019, 0.017442, -0.608761, -0.084989}, + {2.419075, 0.017449, -0.608722, -0.085014}}}, + tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + model = TransformerEncoder(TransformerEncoderOptions(encoder_layer, 6)); + if (is_cuda) { + model->to(torch::kCUDA); + } + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ref_output = torch::tensor( + {{{2.419101, 0.017453, -0.608703, -0.085025}, + {2.419101, 0.017453, -0.608704, -0.085025}}, + {{2.419101, 0.017453, -0.608703, -0.085025}, + {2.419101, 0.017453, -0.608704, -0.085025}}, + {{2.419101, 0.017453, -0.608703, -0.085025}, + {2.419101, 0.017453, -0.608704, -0.085025}}, + {{2.419101, 0.017453, -0.608703, -0.085025}, + {2.419101, 0.017453, -0.608704, -0.085025}}, + {{2.419101, 0.017453, -0.608703, -0.085025}, + {2.419101, 0.017453, -0.608704, -0.085025}}}, + tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + // test case 3, multiple layers with norm + LayerNorm norm(LayerNormOptions({encoder_layer.get()->options.d_model()})); + model = TransformerEncoder( + TransformerEncoderOptions(encoder_layer, 2).norm(AnyModule(norm))); + if (is_cuda) { + model->to(torch::kCUDA); + } + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ref_output = torch::tensor( + {{{1.695949, -0.357635, -0.893077, -0.445238}, + {1.695955, -0.357639, -0.893050, -0.445266}}, + {{1.695948, -0.357634, -0.893082, -0.445233}, + {1.695950, -0.357635, -0.893077, -0.445238}}, + {{1.695951, -0.357636, -0.893069, -0.445246}, + {1.695955, -0.357639, -0.893052, -0.445264}}, + {{1.695952, -0.357636, -0.893066, -0.445249}, + {1.695957, -0.357641, -0.893041, -0.445276}}, + {{1.695946, -0.357632, -0.893095, -0.445220}, + {1.695952, -0.357637, -0.893065, -0.445251}}}, + tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + model = TransformerEncoder( + TransformerEncoderOptions(encoder_layer, 6).norm(AnyModule(norm))); + if (is_cuda) { + model->to(torch::kCUDA); + } + result = model( + encoder_input, + /*src_mask=*/torch::Tensor{}, + /*src_key_padding_mask=*/mask) + .detach(); + ref_output = torch::tensor( + {{{1.695955, -0.357639, -0.893051, -0.445265}, + {1.695955, -0.357639, -0.893051, -0.445265}}, + {{1.695955, -0.357639, -0.893051, -0.445265}, + {1.695955, -0.357639, -0.893051, -0.445265}}, + {{1.695955, -0.357639, -0.893051, -0.445265}, + {1.695955, -0.357639, -0.893051, -0.445265}}, + {{1.695955, -0.357639, -0.893051, -0.445265}, + {1.695955, -0.357639, -0.893051, -0.445265}}, + {{1.695955, -0.357639, -0.893051, -0.445265}, + {1.695955, -0.357639, -0.893051, -0.445265}}}, + tensor_options); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); +} + +TEST_F(TransformerTest, TransformerEncoder) { + transformer_encoder_test_helper( + /*is_cuda=*/false, /*use_callable_activation=*/false); + transformer_encoder_test_helper( + /*is_cuda=*/false, /*use_callable_activation=*/true); +} + +TEST_F(TransformerTest, TransformerEncoder_CUDA) { + transformer_encoder_test_helper( + /*is_cuda=*/true, /*use_callable_activation=*/false); + transformer_encoder_test_helper( + /*is_cuda=*/true, /*use_callable_activation=*/true); +} + +TEST_F(TransformerTest, PrettyPrintTransformerEncoderLayer) { + ASSERT_EQ( + c10::str(TransformerEncoderLayer(4, 2)), + "torch::nn::TransformerEncoderLayerImpl(\n" + " (self_attn): torch::nn::MultiheadAttention(\n" + " (out_proj): torch::nn::Linear(in_features=4, out_features=4, bias=true)\n" + " )\n" + " (linear1): torch::nn::Linear(in_features=4, out_features=2048, bias=true)\n" + " (dropout): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (linear2): torch::nn::Linear(in_features=2048, out_features=4, bias=true)\n" + " (norm1): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (norm2): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (dropout1): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (dropout2): torch::nn::Dropout(p=0.1, inplace=false)\n" + ")"); +} + +TEST_F(TransformerTest, PrettyPrintTransformerEncoder) { + LayerNorm norm = LayerNorm(LayerNormOptions({4})); + TransformerEncoderOptions options( + TransformerEncoderOptions(TransformerEncoderLayerOptions(4, 2), 2) + .norm(AnyModule(norm))); + ASSERT_EQ( + c10::str(TransformerEncoder(options)), + "torch::nn::TransformerEncoderImpl(\n" + " (layers): torch::nn::ModuleList(\n" + " (0): torch::nn::TransformerEncoderLayerImpl(\n" + " (self_attn): torch::nn::MultiheadAttention(\n" + " (out_proj): torch::nn::Linear(in_features=4, out_features=4, bias=true)\n" + " )\n" + " (linear1): torch::nn::Linear(in_features=4, out_features=2048, bias=true)\n" + " (dropout): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (linear2): torch::nn::Linear(in_features=2048, out_features=4, bias=true)\n" + " (norm1): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (norm2): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (dropout1): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (dropout2): torch::nn::Dropout(p=0.1, inplace=false)\n" + " )\n" + " (1): torch::nn::TransformerEncoderLayerImpl(\n" + " (self_attn): torch::nn::MultiheadAttention(\n" + " (out_proj): torch::nn::Linear(in_features=4, out_features=4, bias=true)\n" + " )\n" + " (linear1): torch::nn::Linear(in_features=4, out_features=2048, bias=true)\n" + " (dropout): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (linear2): torch::nn::Linear(in_features=2048, out_features=4, bias=true)\n" + " (norm1): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (norm2): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (dropout1): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (dropout2): torch::nn::Dropout(p=0.1, inplace=false)\n" + " )\n" + " )\n" + " (norm): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + ")"); +} + +TEST_F(TransformerTest, PrettyPrintTransformerDecoderLayer) { + ASSERT_EQ( + c10::str(TransformerDecoderLayer(4, 2)), + "torch::nn::TransformerDecoderLayerImpl(\n" + " (self_attn): torch::nn::MultiheadAttention(\n" + " (out_proj): torch::nn::Linear(in_features=4, out_features=4, bias=true)\n" + " )\n" + " (multihead_attn): torch::nn::MultiheadAttention(\n" + " (out_proj): torch::nn::Linear(in_features=4, out_features=4, bias=true)\n" + " )\n" + " (linear1): torch::nn::Linear(in_features=4, out_features=2048, bias=true)\n" + " (dropout): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (linear2): torch::nn::Linear(in_features=2048, out_features=4, bias=true)\n" + " (norm1): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (norm2): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (norm3): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (dropout1): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (dropout2): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (dropout3): torch::nn::Dropout(p=0.1, inplace=false)\n" + ")"); +} + +void transformer_decoder_test_helper( + bool is_cuda, + bool use_callable_activation) { + // this is a deterministic test for TransformerDecoder + torch::Device device = is_cuda ? torch::kCUDA : torch::kCPU; + torch::TensorOptions tensor_options = + torch::TensorOptions().dtype(torch::kFloat32).device(device); + + TransformerDecoderLayer decoder_layer = + get_a_test_layer( + tensor_options, use_callable_activation); + + TransformerDecoder model(TransformerDecoderOptions(decoder_layer, 1)); + if (is_cuda) { + model->to(torch::kCUDA); + } + + torch::Tensor decoder_input = + torch::tensor({{{20, 30, 40, 50}}}, tensor_options); + torch::Tensor memory_input = + torch::tensor({{{60, 70, 80, 90}}}, tensor_options); + torch::Tensor result = model(decoder_input, memory_input).detach(); + torch::Tensor ref_output = torch::tensor( + {{{2.314351, 0.094805, -0.671322, 0.101977}}}, tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = + torch::tensor({{{9, 10, 11, 12}}, {{11, 12, 13, 14}}}, tensor_options); + memory_input = torch::tensor({{{1, 2, 3, 4}}}, tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.422245, 0.051716, -0.606338, -0.024756}}, + {{2.422245, 0.051716, -0.606338, -0.024756}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = + torch::tensor({{{1, 2, 3, 4}}, {{5, 6, 7, 8}}}, tensor_options); + memory_input = + torch::tensor({{{9, 10, 11, 12}}, {{11, 12, 13, 14}}}, tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.343536, 0.085561, -0.654954, 0.074991}}, + {{2.343536, 0.085561, -0.654954, 0.074991}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = torch::tensor( + {{{0.4517, 0.6793, 0.5313, 0.0034}, {0.2678, 0.3677, 0.4459, 0.7166}}, + {{0.8100, 0.3716, 0.4096, 0.1976}, {0.6958, 0.8844, 0.6081, 0.8315}}, + {{0.0494, 0.9343, 0.5955, 0.3830}, {0.5404, 0.3464, 0.9378, 0.6200}}}, + tensor_options); + memory_input = torch::tensor( + {{{0.7462, 0.6653, 0.5679, 0.4891}, {0.5387, 0.1655, 0.3565, 0.0471}}, + {{0.8335, 0.2799, 0.5031, 0.2947}, {0.1402, 0.0318, 0.7636, 0.1346}}, + {{0.6333, 0.9344, 0.1376, 0.9938}, {0.8924, 0.2872, 0.6692, 0.2944}}, + {{0.9897, 0.6915, 0.3154, 0.1733}, {0.8645, 0.3513, 0.3064, 0.0767}}, + {{0.8117, 0.2366, 0.4838, 0.7881}, {0.3718, 0.4945, 0.9511, 0.0864}}}, + tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.430065, 0.027862, -0.601136, -0.073096}, + {2.431935, 0.028907, -0.599809, -0.072488}}, + {{2.428457, 0.027053, -0.602275, -0.073462}, + {2.431970, 0.029387, -0.599789, -0.071621}}, + {{2.431934, 0.028196, -0.599802, -0.073809}, + {2.432306, 0.028858, -0.599542, -0.072846}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // key_padding_mask + torch::Tensor t_mask = {}; + torch::Tensor m_mask = {}; + torch::Tensor key_padding_mask = torch::zeros({2, 3}, tensor_options) == 1; + result = model(decoder_input, memory_input, t_mask, m_mask, key_padding_mask) + .detach(); + ref_output = torch::tensor( + {{{2.430065, 0.027862, -0.601136, -0.073096}, + {2.431935, 0.028907, -0.599809, -0.072488}}, + {{2.428457, 0.027053, -0.602275, -0.073462}, + {2.431970, 0.029387, -0.599789, -0.071621}}, + {{2.431934, 0.028196, -0.599802, -0.073809}, + {2.432306, 0.028858, -0.599542, -0.072846}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // key_padding_mask + key_padding_mask[0][2] = 1; + key_padding_mask[1][1] = 1; + key_padding_mask[1][2] = 1; + result = model(decoder_input, memory_input, t_mask, m_mask, key_padding_mask) + .detach(); + ref_output = torch::tensor( + {{{2.430025, 0.027643, -0.601164, -0.073476}, + {2.4323, 0.029375, -0.599553, -0.071881}}, + {{2.428523, 0.026838, -0.602226, -0.07391}, + {2.432634, 0.029842, -0.599318, -0.071253}}, + {{2.432278, 0.028152, -0.599555, -0.074139}, + {2.432659, 0.029244, -0.599294, -0.072382}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // memory_key_padding_mask + torch::Tensor t_key_padding_mask = {}; + key_padding_mask = torch::zeros({2, 5}, tensor_options) == 1; + result = model( + decoder_input, + memory_input, + t_mask, + m_mask, + t_key_padding_mask, + key_padding_mask) + .detach(); + ref_output = torch::tensor( + {{{2.430065, 0.027862, -0.601136, -0.073096}, + {2.431935, 0.028907, -0.599809, -0.072488}}, + {{2.428457, 0.027053, -0.602275, -0.073462}, + {2.431970, 0.029387, -0.599789, -0.071621}}, + {{2.431934, 0.028196, -0.599802, -0.073809}, + {2.432306, 0.028858, -0.599542, -0.072846}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // memory_key_padding_mask + key_padding_mask[0][4] = 1; + key_padding_mask[1][3] = 1; + key_padding_mask[1][4] = 1; + result = model( + decoder_input, + memory_input, + t_mask, + m_mask, + t_key_padding_mask, + key_padding_mask) + .detach(); + ref_output = torch::tensor( + {{{2.429757, 0.027358, -0.601351, -0.073816}, + {2.432692, 0.028583, -0.599263, -0.073634}}, + {{2.428247, 0.02662, -0.602419, -0.074123}, + {2.432657, 0.029055, -0.599293, -0.072732}}, + {{2.431515, 0.027687, -0.600096, -0.074459}, + {2.433075, 0.028543, -0.598987, -0.073985}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // multiple layers no norm + model = TransformerDecoder(TransformerDecoderOptions(decoder_layer, 2)); + if (is_cuda) { + model->to(torch::kCUDA); + } + + decoder_input = torch::tensor({{{20, 30, 40, 50}}}, tensor_options); + memory_input = torch::tensor({{{60, 70, 80, 90}}}, tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.31316, 0.0950293, -0.671995, 0.102802}}}, tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // multiple layers no norm + model = TransformerDecoder(TransformerDecoderOptions(decoder_layer, 6)); + if (is_cuda) { + model->to(torch::kCUDA); + } + // deterministic input + decoder_input = torch::tensor( + {{{0.4517, 0.6793, 0.5313, 0.0034}, {0.2678, 0.3677, 0.4459, 0.7166}}, + {{0.8100, 0.3716, 0.4096, 0.1976}, {0.6958, 0.8844, 0.6081, 0.8315}}, + {{0.0494, 0.9343, 0.5955, 0.3830}, {0.5404, 0.3464, 0.9378, 0.6200}}}, + tensor_options); + memory_input = torch::tensor( + {{{0.7462, 0.6653, 0.5679, 0.4891}, {0.5387, 0.1655, 0.3565, 0.0471}}, + {{0.8335, 0.2799, 0.5031, 0.2947}, {0.1402, 0.0318, 0.7636, 0.1346}}, + {{0.6333, 0.9344, 0.1376, 0.9938}, {0.8924, 0.2872, 0.6692, 0.2944}}, + {{0.9897, 0.6915, 0.3154, 0.1733}, {0.8645, 0.3513, 0.3064, 0.0767}}, + {{0.8117, 0.2366, 0.4838, 0.7881}, {0.3718, 0.4945, 0.9511, 0.0864}}}, + tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.42794, 0.026164, -0.60263, -0.0747591}, + {2.43113, 0.0279516, -0.600376, -0.0736896}}, + {{2.42794, 0.026164, -0.60263, -0.0747591}, + {2.43113, 0.0279516, -0.600376, -0.0736896}}, + {{2.42794, 0.026164, -0.60263, -0.0747591}, + {2.43113, 0.0279516, -0.600376, -0.0736896}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // multiple layers with norm + LayerNorm norm(LayerNormOptions({decoder_layer.get()->options.d_model()})); + model = TransformerDecoder( + TransformerDecoderOptions(decoder_layer, 2).norm(AnyModule(norm))); + if (is_cuda) { + model->to(torch::kCUDA); + } + + decoder_input = torch::tensor({{{20, 30, 40, 50}}}, tensor_options); + memory_input = torch::tensor({{{60, 70, 80, 90}}}, tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{1.66166, -0.326986, -1.01466, -0.320017}}}, tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // multiple layers with norm + model = TransformerDecoder( + TransformerDecoderOptions(decoder_layer, 6).norm(AnyModule(norm))); + if (is_cuda) { + model->to(torch::kCUDA); + } + // deterministic input + decoder_input = torch::tensor( + {{{0.4517, 0.6793, 0.5313, 0.0034}, {0.2678, 0.3677, 0.4459, 0.7166}}, + {{0.8100, 0.3716, 0.4096, 0.1976}, {0.6958, 0.8844, 0.6081, 0.8315}}, + {{0.0494, 0.9343, 0.5955, 0.3830}, {0.5404, 0.3464, 0.9378, 0.6200}}}, + tensor_options); + memory_input = torch::tensor( + {{{0.7462, 0.6653, 0.5679, 0.4891}, {0.5387, 0.1655, 0.3565, 0.0471}}, + {{0.8335, 0.2799, 0.5031, 0.2947}, {0.1402, 0.0318, 0.7636, 0.1346}}, + {{0.6333, 0.9344, 0.1376, 0.9938}, {0.8924, 0.2872, 0.6692, 0.2944}}, + {{0.9897, 0.6915, 0.3154, 0.1733}, {0.8645, 0.3513, 0.3064, 0.0767}}, + {{0.8117, 0.2366, 0.4838, 0.7881}, {0.3718, 0.4945, 0.9511, 0.0864}}}, + tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{1.69559, -0.357291, -0.894741, -0.443553}, + {1.69571, -0.357363, -0.894154, -0.444196}}, + {{1.69559, -0.357291, -0.894741, -0.443553}, + {1.69571, -0.357363, -0.894154, -0.444196}}, + {{1.69559, -0.357291, -0.894741, -0.443553}, + {1.69571, -0.357363, -0.894154, -0.444196}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // gelu activation test cases + decoder_layer.get()->options.activation(torch::kGELU); + model = TransformerDecoder(TransformerDecoderOptions(decoder_layer, 1)); + if (is_cuda) { + model->to(torch::kCUDA); + } + + // deterministic input + decoder_input = torch::tensor({{{20, 30, 40, 50}}}, tensor_options); + memory_input = torch::tensor({{{60, 70, 80, 90}}}, tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.306435, 0.095946, -0.675796, 0.10687}}}, tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = + torch::tensor({{{9, 10, 11, 12}}, {{11, 12, 13, 14}}}, tensor_options); + memory_input = torch::tensor({{{1, 2, 3, 4}}}, tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.415448, 0.054389, -0.610932, -0.0156613}}, + {{2.415448, 0.054389, -0.610932, -0.0156613}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = + torch::tensor({{{1, 2, 3, 4}}, {{5, 6, 7, 8}}}, tensor_options); + memory_input = + torch::tensor({{{9, 10, 11, 12}}, {{11, 12, 13, 14}}}, tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.338531, 0.087709, -0.65776, 0.080646}}, + {{2.338531, 0.087709, -0.65776, 0.080646}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // deterministic input + decoder_input = torch::tensor( + {{{0.4517, 0.6793, 0.5313, 0.0034}, {0.2678, 0.3677, 0.4459, 0.7166}}, + {{0.8100, 0.3716, 0.4096, 0.1976}, {0.6958, 0.8844, 0.6081, 0.8315}}, + {{0.0494, 0.9343, 0.5955, 0.3830}, {0.5404, 0.3464, 0.9378, 0.6200}}}, + tensor_options); + memory_input = torch::tensor( + {{{0.7462, 0.6653, 0.5679, 0.4891}, {0.5387, 0.1655, 0.3565, 0.0471}}, + {{0.8335, 0.2799, 0.5031, 0.2947}, {0.1402, 0.0318, 0.7636, 0.1346}}, + {{0.6333, 0.9344, 0.1376, 0.9938}, {0.8924, 0.2872, 0.6692, 0.2944}}, + {{0.9897, 0.6915, 0.3154, 0.1733}, {0.8645, 0.3513, 0.3064, 0.0767}}, + {{0.8117, 0.2366, 0.4838, 0.7881}, {0.3718, 0.4945, 0.9511, 0.0864}}}, + tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.42049104, 0.03443088, -0.60793706, -0.05436271}, + {2.42210631, 0.03546578, -0.60679895, -0.05357488}}, + {{2.41907674, 0.0336104, -0.60892977, -0.05490462}, + {2.42216881, 0.03586554, -0.6067524, -0.05289126}}, + {{2.42205716, 0.03488046, -0.60683681, -0.05460596}, + {2.42240309, 0.0354595, -0.60659063, -0.05378816}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // Multiple layers no norm + model = TransformerDecoder(TransformerDecoderOptions(decoder_layer, 6)); + if (is_cuda) { + model->to(torch::kCUDA); + } + decoder_input = torch::tensor( + {{{0.4517, 0.6793, 0.5313, 0.0034}, {0.2678, 0.3677, 0.4459, 0.7166}}, + {{0.8100, 0.3716, 0.4096, 0.1976}, {0.6958, 0.8844, 0.6081, 0.8315}}, + {{0.0494, 0.9343, 0.5955, 0.3830}, {0.5404, 0.3464, 0.9378, 0.6200}}}, + tensor_options); + memory_input = torch::tensor( + {{{0.7462, 0.6653, 0.5679, 0.4891}, {0.5387, 0.1655, 0.3565, 0.0471}}, + {{0.8335, 0.2799, 0.5031, 0.2947}, {0.1402, 0.0318, 0.7636, 0.1346}}, + {{0.6333, 0.9344, 0.1376, 0.9938}, {0.8924, 0.2872, 0.6692, 0.2944}}, + {{0.9897, 0.6915, 0.3154, 0.1733}, {0.8645, 0.3513, 0.3064, 0.0767}}, + {{0.8117, 0.2366, 0.4838, 0.7881}, {0.3718, 0.4945, 0.9511, 0.0864}}}, + tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{2.41859, 0.0328114, -0.609269, -0.0560386}, + {2.42138, 0.034598, -0.607316, -0.0546574}}, + {{2.41859, 0.0328114, -0.609269, -0.0560386}, + {2.42138, 0.034598, -0.607316, -0.0546574}}, + {{2.41859, 0.0328114, -0.609269, -0.0560386}, + {2.42138, 0.034598, -0.607316, -0.0546574}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); + + // Multiple layers with norm + norm = LayerNorm(LayerNormOptions({decoder_layer.get()->options.d_model()})); + model = TransformerDecoder( + TransformerDecoderOptions(decoder_layer, 6).norm(AnyModule(norm))); + if (is_cuda) { + model->to(torch::kCUDA); + } + + decoder_input = torch::tensor( + {{{0.4517, 0.6793, 0.5313, 0.0034}, {0.2678, 0.3677, 0.4459, 0.7166}}, + {{0.8100, 0.3716, 0.4096, 0.1976}, {0.6958, 0.8844, 0.6081, 0.8315}}, + {{0.0494, 0.9343, 0.5955, 0.3830}, {0.5404, 0.3464, 0.9378, 0.6200}}}, + tensor_options); + memory_input = torch::tensor( + {{{0.7462, 0.6653, 0.5679, 0.4891}, {0.5387, 0.1655, 0.3565, 0.0471}}, + {{0.8335, 0.2799, 0.5031, 0.2947}, {0.1402, 0.0318, 0.7636, 0.1346}}, + {{0.6333, 0.9344, 0.1376, 0.9938}, {0.8924, 0.2872, 0.6692, 0.2944}}, + {{0.9897, 0.6915, 0.3154, 0.1733}, {0.8645, 0.3513, 0.3064, 0.0767}}, + {{0.8117, 0.2366, 0.4838, 0.7881}, {0.3718, 0.4945, 0.9511, 0.0864}}}, + tensor_options); + result = model(decoder_input, memory_input).detach(); + ref_output = torch::tensor( + {{{1.69298, -0.355163, -0.906375, -0.431439}, + {1.69305, -0.355195, -0.906062, -0.431791}}, + {{1.69298, -0.355163, -0.906375, -0.431439}, + {1.69305, -0.355195, -0.906062, -0.431791}}, + {{1.69298, -0.355163, -0.906375, -0.431439}, + {1.69305, -0.355195, -0.906062, -0.431791}}}, + tensor_options); + ASSERT_EQ(result.sizes().size(), ref_output.sizes().size()); + ASSERT_TRUE(torch::allclose( + result, + ref_output, + 1e-7, + 1e-5, + /*equal_nan=*/true)); +} + +TEST_F(TransformerTest, TransformerDecoder) { + transformer_decoder_test_helper( + /*is_cuda=*/false, /*use_callable_activation=*/false); + transformer_decoder_test_helper( + /*is_cuda=*/false, /*use_callable_activation=*/true); +} + +TEST_F(TransformerTest, TransformerDecoder_CUDA) { + transformer_decoder_test_helper( + /*is_cuda=*/true, /*use_callable_activation=*/false); + transformer_decoder_test_helper( + /*is_cuda=*/true, /*use_callable_activation=*/true); +} + +TEST_F(TransformerTest, PrettyPrintTransformerDecoder) { + LayerNorm norm = LayerNorm(LayerNormOptions({4})); + TransformerDecoderOptions options( + TransformerDecoderOptions(TransformerDecoderLayerOptions(4, 2), 2) + .norm(AnyModule(norm))); + ASSERT_EQ( + c10::str(TransformerDecoder(options)), + "torch::nn::TransformerDecoderImpl(\n" + " (layers): torch::nn::ModuleList(\n" + " (0): torch::nn::TransformerDecoderLayerImpl(\n" + " (self_attn): torch::nn::MultiheadAttention(\n" + " (out_proj): torch::nn::Linear(in_features=4, out_features=4, bias=true)\n" + " )\n" + " (multihead_attn): torch::nn::MultiheadAttention(\n" + " (out_proj): torch::nn::Linear(in_features=4, out_features=4, bias=true)\n" + " )\n" + " (linear1): torch::nn::Linear(in_features=4, out_features=2048, bias=true)\n" + " (dropout): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (linear2): torch::nn::Linear(in_features=2048, out_features=4, bias=true)\n" + " (norm1): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (norm2): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (norm3): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (dropout1): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (dropout2): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (dropout3): torch::nn::Dropout(p=0.1, inplace=false)\n" + " )\n" + " (1): torch::nn::TransformerDecoderLayerImpl(\n" + " (self_attn): torch::nn::MultiheadAttention(\n" + " (out_proj): torch::nn::Linear(in_features=4, out_features=4, bias=true)\n" + " )\n" + " (multihead_attn): torch::nn::MultiheadAttention(\n" + " (out_proj): torch::nn::Linear(in_features=4, out_features=4, bias=true)\n" + " )\n" + " (linear1): torch::nn::Linear(in_features=4, out_features=2048, bias=true)\n" + " (dropout): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (linear2): torch::nn::Linear(in_features=2048, out_features=4, bias=true)\n" + " (norm1): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (norm2): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (norm3): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + " (dropout1): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (dropout2): torch::nn::Dropout(p=0.1, inplace=false)\n" + " (dropout3): torch::nn::Dropout(p=0.1, inplace=false)\n" + " )\n" + " )\n" + " (norm): torch::nn::LayerNorm([4], eps=1e-05, elementwise_affine=true)\n" + ")"); +} + +void transformer_test_helper(bool is_cuda, bool use_callable_activation) { + // this is a deterministic test for Transformere + torch::Device device = is_cuda ? torch::kCUDA : torch::kCPU; + torch::TensorOptions tensor_options = + torch::TensorOptions().dtype(torch::kFloat32).device(device); + + // transformer created encoder/decoder + auto options = TransformerOptions() + .d_model(4) + .nhead(2) + .num_encoder_layers(2) + .num_decoder_layers(1) + .dim_feedforward(16) + .dropout(0.0) + .activation(torch::kReLU); + if (use_callable_activation) { + options.activation( + [&](const torch::Tensor& t) { return torch::nn::functional::relu(t); }); + } + Transformer model(options); + + set_parameter_to_constants(model, tensor_options); + if (tensor_options.device() == torch::kCUDA) { + model->to(torch::kCUDA); + } + + // transformer with customized encoder/decoder + LayerNorm enorm(LayerNormOptions({4})); + TransformerEncoder encoder( + TransformerEncoderOptions( + TransformerEncoderLayerOptions(4, 2).dim_feedforward(16).dropout(0.0), + 2) + .norm(AnyModule(enorm))); + + LayerNorm dnorm(LayerNormOptions({4})); + TransformerDecoder decoder( + TransformerDecoderOptions( + TransformerDecoderLayerOptions(4, 2).dim_feedforward(16).dropout(0.0), + 1) + .norm(AnyModule(dnorm))); + + Transformer model_cus(TransformerOptions() + .d_model(4) + .nhead(2) + .custom_encoder(AnyModule(encoder)) + .custom_decoder(AnyModule(decoder))); + + set_parameter_to_constants(model_cus, tensor_options); + if (tensor_options.device() == torch::kCUDA) { + model_cus->to(torch::kCUDA); + } + + // test cases + torch::Tensor src = torch::tensor( + {{{1.0, 2.0, 3.0, 4.0}, {5.0, 6.0, 7.0, 8.0}}, + {{9.0, 10.0, 11.0, 12.0}, {13.0, 14.0, 15.0, 16.0}}, + {{17.0, 18.0, 19.0, 20.0}, {21.0, 22.0, 23.0, 24.0}}}, + tensor_options); + + torch::Tensor tgt = torch::tensor( + {{{1.0, 2.0, 3.0, 4.0}, {5.0, 6.0, 7.0, 8.0}}, + {{9.0, 10.0, 11.0, 12.0}, {13.0, 14.0, 15.0, 16.0}}}, + tensor_options); + + torch::Tensor ref_output = torch::tensor( + {{{2.695875, 0.347114, -0.044355, -0.549541}, + {2.696091, 0.347015, -0.044770, -0.548522}}, + {{2.695875, 0.347114, -0.044355, -0.549541}, + {2.696091, 0.347015, -0.044770, -0.548522}}}, + tensor_options); + torch::Tensor result = model(src, tgt); + torch::Tensor result_cus = model_cus(src, tgt); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE(result.equal(result_cus)); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + torch::Tensor src_mask = + Transformer::Impl::generate_square_subsequent_mask(src.size(0)) + .to(tensor_options); + ref_output = torch::tensor( + {{{2.695875, 0.347114, -0.044355, -0.549541}, + {2.696091, 0.347015, -0.044770, -0.548522}}, + {{2.695875, 0.347114, -0.044355, -0.549541}, + {2.696091, 0.347015, -0.044770, -0.548522}}}, + tensor_options); + result = model(src, tgt, src_mask); + result_cus = model_cus(src, tgt, src_mask); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE(result.equal(result_cus)); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); + + torch::Tensor tgt_key_padding_mask = + torch::zeros({tgt.size(1), tgt.size(0)}, tensor_options) == 1; + tgt_key_padding_mask[0][0] = 1; + tgt_key_padding_mask[1][1] = 1; + ref_output = torch::tensor( + {{{2.696114, 0.347004, -0.044813, -0.548417}, + {2.696091, 0.347015, -0.044770, -0.548522}}, + {{2.696114, 0.347004, -0.044813, -0.548417}, + {2.696091, 0.347015, -0.044770, -0.548522}}}, + tensor_options); + result = model( + src, + tgt, + src_mask, + torch::Tensor(), + torch::Tensor(), + torch::Tensor(), + tgt_key_padding_mask); + result_cus = model_cus( + src, + tgt, + src_mask, + torch::Tensor(), + torch::Tensor(), + torch::Tensor(), + tgt_key_padding_mask); + ASSERT_EQ(result.sizes(), ref_output.sizes()); + ASSERT_TRUE(result.equal(result_cus)); + ASSERT_TRUE( + torch::allclose(result, ref_output, 1e-7, 1e-5, /*equal_nan=*/true)); +} + +TEST_F(TransformerTest, Transformer) { + transformer_test_helper(/*is_cuda=*/false, /*use_callable_activation=*/false); + transformer_test_helper(/*is_cuda=*/false, /*use_callable_activation=*/true); +} + +TEST_F(TransformerTest, Transformer_CUDA) { + transformer_test_helper(/*is_cuda=*/true, /*use_callable_activation=*/false); + transformer_test_helper(/*is_cuda=*/true, /*use_callable_activation=*/true); +} + +TEST_F(TransformerTest, TransformerArgsCorrectness) { + Transformer model(TransformerOptions() + .d_model(4) + .nhead(2) + .num_encoder_layers(2) + .num_decoder_layers(1) + .dim_feedforward(16) + .dropout(0.0) + .activation(torch::kReLU)); + + torch::Tensor src = torch::randn({2, 3, 4}); + torch::Tensor tgt = torch::randn({3, 2, 4}); + + ASSERT_THROWS_WITH( + model(src, tgt), "src and tgt should have equal batch size"); + + tgt = torch::randn({2, 3, 3}); + ASSERT_THROWS_WITH( + model(src, tgt), "src and tgt should have same feature size as d_model"); + + src = torch::randn({2, 3}); + ASSERT_THROWS_WITH(model(src, tgt), "src and tgt should have 3 dimensions"); +} diff --git a/western_ai/usa/pytorch/test/cpp/c10d/BackoffTest.cpp b/western_ai/usa/pytorch/test/cpp/c10d/BackoffTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4efef0a464a42e1694c86016a873145c084f63e9 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/BackoffTest.cpp @@ -0,0 +1,65 @@ +#include +#include "StoreTestCommon.hpp" + +#include + +TEST(BackoffTest, exponentialBackoffDefaults) { + c10d::ExponentialBackoffWithJitter backoff; + EXPECT_EQ(backoff.initialInterval, std::chrono::milliseconds(500)); + EXPECT_EQ(backoff.maxInterval, std::chrono::milliseconds(60000)); + EXPECT_EQ(backoff.multiplier, 1.5); + EXPECT_EQ(backoff.randomizationFactor, 0.5); +} + +TEST(BackoffTest, exponentialBackoff) { + c10d::ExponentialBackoffWithJitter backoff; + backoff.randomizationFactor = 0.0; + backoff.multiplier = 2.0; + backoff.maxInterval = std::chrono::milliseconds(5000); + + EXPECT_EQ(backoff.nextBackoff(), std::chrono::milliseconds(500)); + EXPECT_EQ(backoff.nextBackoff(), std::chrono::milliseconds(1000)); + EXPECT_EQ(backoff.nextBackoff(), std::chrono::milliseconds(2000)); + EXPECT_EQ(backoff.nextBackoff(), std::chrono::milliseconds(4000)); + EXPECT_EQ(backoff.nextBackoff(), std::chrono::milliseconds(5000)); + EXPECT_EQ(backoff.nextBackoff(), std::chrono::milliseconds(5000)); + + backoff.reset(); + EXPECT_EQ(backoff.nextBackoff(), std::chrono::milliseconds(500)); + EXPECT_EQ(backoff.nextBackoff(), std::chrono::milliseconds(1000)); +} + +TEST(BackoffTest, expontentialBackoffRandomization) { + c10d::ExponentialBackoffWithJitter backoff; + backoff.initialInterval = std::chrono::milliseconds(1000); + backoff.randomizationFactor = 0.5; + backoff.multiplier = 1.0; + backoff.maxInterval = std::chrono::milliseconds(5000); + + for (int i = 0; i < 100; i++) { + auto backoffDur = backoff.nextBackoff(); + EXPECT_GE(backoffDur, std::chrono::milliseconds(500)); + EXPECT_LE(backoffDur, std::chrono::milliseconds(1500)); + } +} + +TEST(BackoffTest, fixedBackoff) { + c10d::FixedBackoff backoff{std::chrono::milliseconds(1000)}; + + EXPECT_EQ(backoff.nextBackoff(), std::chrono::milliseconds(1000)); + EXPECT_EQ(backoff.nextBackoff(), std::chrono::milliseconds(1000)); + backoff.reset(); + EXPECT_EQ(backoff.nextBackoff(), std::chrono::milliseconds(1000)); +} + +TEST(BackoffTest, sleep) { + std::chrono::milliseconds sleepTime{10}; + c10d::FixedBackoff backoff{sleepTime}; + + EXPECT_EQ(backoff.nextBackoff(), sleepTime); + + auto start = std::chrono::high_resolution_clock::now(); + backoff.sleepBackoff(); + auto dur = std::chrono::high_resolution_clock::now() - start; + EXPECT_GE(dur, sleepTime); +} diff --git a/western_ai/usa/pytorch/test/cpp/c10d/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/c10d/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..8632c0614e14914d3047ee40d7abeabf76614eec --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/CMakeLists.txt @@ -0,0 +1,146 @@ +if(USE_CUDA) + add_library(c10d_cuda_test CUDATest.cu) + target_include_directories(c10d_cuda_test PRIVATE $) + target_link_libraries(c10d_cuda_test torch_cuda) + add_dependencies(c10d_cuda_test torch_cuda) +elseif(USE_ROCM) + set_source_files_properties(hip/HIPTest.hip PROPERTIES LANGUAGE HIP) + add_library(c10d_hip_test hip/HIPTest.hip) + set_target_properties(c10d_hip_test PROPERTIES LINKER_LANGUAGE CXX) + target_include_directories(c10d_hip_test PRIVATE $) + target_link_libraries(c10d_hip_test torch_hip) + add_dependencies(c10d_hip_test torch_hip) +endif() + +function(c10d_add_test test_src) + set(prefix ARG) + set(noValues) + set(singleValues INSTALL_TEST) + set(multiValues LINK_LIBRARIES) + + include(CMakeParseArguments) + cmake_parse_arguments(${prefix} "${noValues}" "${singleValues}" "${multiValues}" ${ARGN}) + + get_filename_component(test_name ${test_src} NAME_WE) + add_executable(${test_name} "${test_src}") + target_include_directories(${test_name} PRIVATE + $ + $ + $ + ) + target_link_libraries(${test_name} PRIVATE + fmt::fmt-header-only + ${ARG_LINK_LIBRARIES} + ) + add_test(NAME ${test_name} COMMAND $) + + if(ARG_INSTALL_TEST) + set_target_properties(${test_name} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS ${test_name} DESTINATION bin) + endif() +endfunction() + +if(USE_ROCM) + c10d_add_test(hip/BackoffTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST OFF) + c10d_add_test(hip/FileStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST ${INSTALL_TEST}) + c10d_add_test(hip/TCPStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST ${INSTALL_TEST}) + if(NOT WIN32) + c10d_add_test(hip/HashStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST ${INSTALL_TEST}) + endif() + +else() + c10d_add_test(BackoffTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST OFF) + c10d_add_test(FileStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST ${INSTALL_TEST}) + c10d_add_test(TCPStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST ${INSTALL_TEST}) + if(NOT WIN32) + c10d_add_test(HashStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST ${INSTALL_TEST}) + endif() +endif() + +if(USE_CUDA) + if(USE_GLOO AND USE_C10D_GLOO) + c10d_add_test(ProcessGroupGlooTest.cpp LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main INSTALL_TEST ${INSTALL_TEST}) + c10d_add_test(ProcessGroupGlooAsyncTest.cpp LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main INSTALL_TEST ${INSTALL_TEST}) + endif() + if(USE_NCCL AND USE_C10D_NCCL) + # NCCL is a private dependency of libtorch, but the tests include some + # private headers of libtorch, which in turn include NCCL. As a hacky + # alternative to making NCCL a public dependency of libtorch, we make it + # a private dependency of the tests as well. + c10d_add_test( + ProcessGroupNCCLTest.cpp + LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main __caffe2_nccl INSTALL_TEST ${INSTALL_TEST}) + target_compile_definitions(ProcessGroupNCCLTest PRIVATE USE_CUDA USE_NCCL) + c10d_add_test( + ProcessGroupNCCLErrorsTest.cpp + LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main __caffe2_nccl INSTALL_TEST ${INSTALL_TEST}) + target_compile_definitions(ProcessGroupNCCLErrorsTest PRIVATE USE_CUDA USE_NCCL) + c10d_add_test( + NCCLDevCommManagerTest.cpp + LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main __caffe2_nccl INSTALL_TEST ${INSTALL_TEST}) + target_compile_definitions(NCCLDevCommManagerTest PRIVATE USE_CUDA USE_NCCL) + if(INSTALL_TEST) + install(TARGETS c10d_cuda_test DESTINATION lib) + endif() + endif() + if(USE_UCC AND USE_C10D_UCC) + # UCC is a private dependency of libtorch, but the tests include some + # private headers of libtorch, which in turn include UCC. As a hacky + # alternative to making UCC a public dependency of libtorch, we make it + # a private dependency of the tests as well. + c10d_add_test( + ProcessGroupUCCTest.cpp + LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main __caffe2_ucc INSTALL_TEST ${INSTALL_TEST}) + if(INSTALL_TEST) + install(TARGETS c10d_cuda_test DESTINATION lib) + endif() + endif() +elseif(USE_ROCM) + if(USE_GLOO AND USE_C10D_GLOO) + c10d_add_test(hip/ProcessGroupGlooTest.cpp LINK_LIBRARIES torch_cpu c10d_hip_test gtest_main INSTALL_TEST ${INSTALL_TEST}) + c10d_add_test(hip/ProcessGroupGlooAsyncTest.cpp LINK_LIBRARIES torch_cpu c10d_hip_test gtest_main INSTALL_TEST ${INSTALL_TEST}) + endif() + if(USE_NCCL AND USE_C10D_NCCL) + c10d_add_test( + hip/ProcessGroupNCCLTest.cpp + LINK_LIBRARIES torch_cpu c10d_hip_test gtest_main __caffe2_nccl INSTALL_TEST ${INSTALL_TEST}) + c10d_add_test( + hip/ProcessGroupNCCLErrorsTest.cpp + LINK_LIBRARIES torch_cpu c10d_hip_test gtest_main __caffe2_nccl INSTALL_TEST ${INSTALL_TEST}) + if(INSTALL_TEST) + install(TARGETS c10d_hip_test DESTINATION lib) + endif() + endif() + if(USE_UCC AND USE_C10D_UCC) + c10d_add_test( + hip/ProcessGroupUCCTest.cpp + LINK_LIBRARIES torch_cpu c10d_hip_test gtest_main __caffe2_ucc INSTALL_TEST ${INSTALL_TEST}) + if(INSTALL_TEST) + install(TARGETS c10d_hip_test DESTINATION lib) + endif() + endif() +else() + if(USE_GLOO AND USE_C10D_GLOO) + c10d_add_test(ProcessGroupGlooTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST OFF) + endif() +endif() + +if(USE_MPI AND USE_C10D_MPI) + add_definitions(-DMPIEXEC=${MPIEXEC}) + # MPI is a private dependency of libtorch, but the tests include some + # private headers of libtorch, which in turn include MPI. As a hacky + # alternative to making MPI a public dependency of libtorch, we make it + # a private dependency of the tests as well. + c10d_add_test(ProcessGroupMPITest.cpp LINK_LIBRARIES torch_cpu MPI::MPI_CXX INSTALL_TEST ${INSTALL_TEST}) +endif() + +if(LINUX AND USE_GLOO AND USE_C10D_GLOO) + add_executable(example_allreduce example/allreduce.cpp) + target_include_directories(example_allreduce PRIVATE $) + target_link_libraries(example_allreduce torch_cpu) + if(USE_CUDA) + target_link_libraries(example_allreduce torch_cuda) + elseif(USE_ROCM) + target_link_libraries(example_allreduce torch_hip) + endif() +endif() diff --git a/western_ai/usa/pytorch/test/cpp/c10d/CUDATest.cu b/western_ai/usa/pytorch/test/cpp/c10d/CUDATest.cu new file mode 100644 index 0000000000000000000000000000000000000000..9f34e46acc8652ed4205a0964c5e0cd81ae67272 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/CUDATest.cu @@ -0,0 +1,35 @@ +#include "CUDATest.hpp" +#include + +namespace c10d { +namespace test { + +namespace { +__global__ void waitClocks(const uint64_t count) { + // Few AMD specific GPUs have different clock intrinsic +#if defined(__GFX11__) && defined(USE_ROCM) && !defined(__CUDA_ARCH__) + clock_t start = wall_clock64(); +#else + clock_t start = clock64(); +#endif + clock_t offset = 0; + while (offset < count) { + offset = clock() - start; + } +} + +} // namespace + +void cudaSleep(at::cuda::CUDAStream& stream, uint64_t clocks) { + waitClocks<<<1, 1, 0, stream.stream()>>>(clocks); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +int cudaNumDevices() { + int n = 0; + C10_CUDA_CHECK_WARN(cudaGetDeviceCount(&n)); + return n; +} + +} // namespace test +} // namespace c10d diff --git a/western_ai/usa/pytorch/test/cpp/c10d/CUDATest.hpp b/western_ai/usa/pytorch/test/cpp/c10d/CUDATest.hpp new file mode 100644 index 0000000000000000000000000000000000000000..16fee061057a8e94eac03cdcbac71da970da718b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/CUDATest.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace c10d { +namespace test { + +#ifdef _WIN32 +#define EXPORT_TEST_API __declspec(dllexport) +#else +#define EXPORT_TEST_API +#endif + +EXPORT_TEST_API void cudaSleep(at::cuda::CUDAStream& stream, uint64_t clocks); + +EXPORT_TEST_API int cudaNumDevices(); + +} // namespace test +} // namespace c10d diff --git a/western_ai/usa/pytorch/test/cpp/c10d/FileStoreTest.cpp b/western_ai/usa/pytorch/test/cpp/c10d/FileStoreTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..325301f66e94fe178a12c9a0df56409859f2ec8c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/FileStoreTest.cpp @@ -0,0 +1,157 @@ + +#include +#include "StoreTestCommon.hpp" + +#ifndef _WIN32 +#include +#endif + +#include +#include + +#include + +#include +#include + +#ifdef _WIN32 +std::string tmppath() { + return c10d::test::autoGenerateTmpFilePath(); +} +#else +std::string tmppath() { + const char* tmpdir = getenv("TMPDIR"); + if (tmpdir == nullptr) { + tmpdir = "/tmp"; + } + + // Create template + std::vector tmp(256); + auto len = snprintf(tmp.data(), tmp.size(), "%s/testXXXXXX", tmpdir); + tmp.resize(len); + + // Create temporary file + auto fd = mkstemp(&tmp[0]); + if (fd == -1) { + throw std::system_error(errno, std::system_category()); + } + close(fd); + return std::string(tmp.data(), tmp.size()); +} +#endif + +void testGetSet(const std::string& path, const std::string& prefix = "") { + // Basic Set/Get on File Store + { + auto fileStore = c10::make_intrusive(path, 2); + c10d::PrefixStore store(prefix, fileStore); + c10d::test::set(store, "key0", "value0"); + c10d::test::set(store, "key1", "value1"); + c10d::test::set(store, "key2", "value2"); + c10d::test::check(store, "key0", "value0"); + c10d::test::check(store, "key1", "value1"); + c10d::test::check(store, "key2", "value2"); + auto numKeys = fileStore->getNumKeys(); + EXPECT_EQ(numKeys, 4); + + // Check compareSet, does not check return value + c10d::test::compareSet(store, "key0", "wrongExpectedValue", "newValue"); + c10d::test::check(store, "key0", "value0"); + c10d::test::compareSet(store, "key0", "value0", "newValue"); + c10d::test::check(store, "key0", "newValue"); + + // Check deleteKey + c10d::test::deleteKey(store, "key1"); + numKeys = fileStore->getNumKeys(); + EXPECT_EQ(numKeys, 3); + c10d::test::check(store, "key0", "newValue"); + c10d::test::check(store, "key2", "value2"); + + c10d::test::set(store, "-key0", "value-"); + c10d::test::check(store, "key0", "newValue"); + c10d::test::check(store, "-key0", "value-"); + numKeys = fileStore->getNumKeys(); + EXPECT_EQ(numKeys, 4); + c10d::test::deleteKey(store, "-key0"); + numKeys = fileStore->getNumKeys(); + EXPECT_EQ(numKeys, 3); + c10d::test::check(store, "key0", "newValue"); + c10d::test::check(store, "key2", "value2"); + } + + // Perform get on new instance + { + auto fileStore = c10::make_intrusive(path, 2); + c10d::PrefixStore store(prefix, fileStore); + c10d::test::check(store, "key0", "newValue"); + auto numKeys = fileStore->getNumKeys(); + // There will be 4 keys since we still use the same underlying file as the + // other store above. + EXPECT_EQ(numKeys, 4); + } +} + +void stressTestStore(std::string path, std::string prefix = "") { + // Hammer on FileStore::add + const auto numThreads = 4; + const auto numIterations = 100; + + std::vector threads; + c10d::test::Semaphore sem1, sem2; + + for ([[maybe_unused]] const auto i : c10::irange(numThreads)) { + threads.emplace_back([&] { + auto fileStore = + c10::make_intrusive(path, numThreads + 1); + c10d::PrefixStore store(prefix, fileStore); + sem1.post(); + sem2.wait(); + for ([[maybe_unused]] const auto j : c10::irange(numIterations)) { + store.add("counter", 1); + } + }); + } + + sem1.wait(numThreads); + sem2.post(numThreads); + for (auto& thread : threads) { + thread.join(); + } + + // Check that the counter has the expected value + { + auto fileStore = c10::make_intrusive(path, numThreads + 1); + c10d::PrefixStore store(prefix, fileStore); + std::string expected = std::to_string(numThreads * numIterations); + c10d::test::check(store, "counter", expected); + } +} + +class FileStoreTest : public ::testing::Test { + protected: + void SetUp() override { + path_ = tmppath(); + } + + void TearDown() override { + unlink(path_.c_str()); + } + + std::string path_; +}; + +TEST_F(FileStoreTest, testGetAndSet) { + testGetSet(path_); +} + +TEST_F(FileStoreTest, testGetAndSetWithPrefix) { + testGetSet(path_, "testPrefix"); +} + +TEST_F(FileStoreTest, testStressStore) { + stressTestStore(path_); +} + +TEST_F(FileStoreTest, testStressStoreWithPrefix) { + stressTestStore(path_, "testPrefix"); +} diff --git a/western_ai/usa/pytorch/test/cpp/c10d/HashStoreTest.cpp b/western_ai/usa/pytorch/test/cpp/c10d/HashStoreTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d3e0a48f066a19b08f34af111b3f2ec0c7c428ce --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/HashStoreTest.cpp @@ -0,0 +1,99 @@ +#include +#include "StoreTestCommon.hpp" + +#include + +#include + +#include +#include +#include + +constexpr int64_t kShortStoreTimeoutMillis = 100; + +void testGetSet(const std::string& prefix = "") { + // Basic set/get + { + auto hashStore = c10::make_intrusive(); + c10d::PrefixStore store(prefix, hashStore); + c10d::test::set(store, "key0", "value0"); + c10d::test::set(store, "key1", "value1"); + c10d::test::set(store, "key2", "value2"); + c10d::test::check(store, "key0", "value0"); + c10d::test::check(store, "key1", "value1"); + c10d::test::check(store, "key2", "value2"); + + // Check compareSet, does not check return value + c10d::test::compareSet(store, "key0", "wrongExpectedValue", "newValue"); + c10d::test::check(store, "key0", "value0"); + c10d::test::compareSet(store, "key0", "value0", "newValue"); + c10d::test::check(store, "key0", "newValue"); + + auto numKeys = store.getNumKeys(); + EXPECT_EQ(numKeys, 3); + auto delSuccess = store.deleteKey("key0"); + EXPECT_TRUE(delSuccess); + numKeys = store.getNumKeys(); + EXPECT_EQ(numKeys, 2); + auto delFailure = store.deleteKey("badKeyName"); + EXPECT_FALSE(delFailure); + auto timeout = std::chrono::milliseconds(kShortStoreTimeoutMillis); + store.setTimeout(timeout); + EXPECT_THROW(store.get("key0"), c10::DistStoreError); + } + + // get() waits up to timeout_. + { + auto hashStore = c10::make_intrusive(); + c10d::PrefixStore store(prefix, hashStore); + std::thread th([&]() { c10d::test::set(store, "key0", "value0"); }); + c10d::test::check(store, "key0", "value0"); + th.join(); + } +} + +void stressTestStore(std::string prefix = "") { + // Hammer on HashStore::add + const auto numThreads = 4; + const auto numIterations = 100; + + std::vector threads; + c10d::test::Semaphore sem1, sem2; + auto hashStore = c10::make_intrusive(); + c10d::PrefixStore store(std::move(prefix), hashStore); + + for ([[maybe_unused]] const auto i : c10::irange(numThreads)) { + threads.emplace_back([&] { + sem1.post(); + sem2.wait(); + for ([[maybe_unused]] const auto j : c10::irange(numIterations)) { + store.add("counter", 1); + } + }); + } + + sem1.wait(numThreads); + sem2.post(numThreads); + + for (auto& thread : threads) { + thread.join(); + } + std::string expected = std::to_string(numThreads * numIterations); + c10d::test::check(store, "counter", expected); +} + +TEST(HashStoreTest, testGetAndSet) { + testGetSet(); +} + +TEST(HashStoreTest, testGetAndSetWithPrefix) { + testGetSet("testPrefix"); +} + +TEST(HashStoreTest, testStressStore) { + stressTestStore(); +} + +TEST(HashStoreTest, testStressStoreWithPrefix) { + stressTestStore("testPrefix"); +} diff --git a/western_ai/usa/pytorch/test/cpp/c10d/NCCLDevCommManagerTest.cpp b/western_ai/usa/pytorch/test/cpp/c10d/NCCLDevCommManagerTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2558fd8252935c1d0408a698fadb7c09cdc38c88 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/NCCLDevCommManagerTest.cpp @@ -0,0 +1,277 @@ +#include + +// Include nccl_dev_cap.hpp first to define NCCL_HAS_SYMMEM_DEVICE_SUPPORT +#include + +#ifdef NCCL_HAS_SYMMEM_DEVICE_SUPPORT + +#include +#include +#include +#include +#include + +using namespace c10d::symmetric_memory; + +// Note: This test requires NCCL with symmetric memory device support. +// The ncclDevComm type is defined in nccl_device.h (included via +// nccl_dev_cap.hpp). For testing without full NCCL initialization, we use +// uninitialized values. In a real scenario, you would use ncclDevCommCreate to +// create valid communicators. + +// Test register_comm and get_comm +void test_register_and_get_comm() { + const std::string group_name = "test_group_comm"; + c10::Device device(c10::DeviceType::CUDA, 0); + + auto& manager = NCCLDevCommManager::get(device); + + // Register a host communicator + ncclComm_t mock_comm = nullptr; // In real test, this would be a valid comm + manager.register_comm(group_name, mock_comm); + + // Get the communicator back + ncclComm_t retrieved_comm = manager.get_comm(group_name); + EXPECT_EQ(retrieved_comm, mock_comm); + + // Registering the same communicator again should not throw + manager.register_comm(group_name, mock_comm); + + // Getting from non-existent group should throw + EXPECT_THROW(manager.get_comm("nonexistent_group"), c10::Error); +} + +// Test basic registration and retrieval with default function name key +void test_register_and_get_default_key() { + const std::string group_name = "test_group"; + c10::Device device(c10::DeviceType::CUDA, 0); + + auto& manager = NCCLDevCommManager::get(device); + + // Create a test device communicator (uninitialized for testing) + // In real usage, this would be created via ncclDevCommCreate + ncclDevComm devcomm0 = {}; + + // Register with default key (function name) + auto devcomm_opt = manager.register_devcomm(group_name, devcomm0); + EXPECT_TRUE(devcomm_opt.has_value()); + + // Get with default key + auto retrieved_opt = manager.get_devcomm(group_name); + EXPECT_TRUE(retrieved_opt.has_value()); + // Note: We can't easily compare ncclDevComm values without proper + // initialization This test verifies the registration/retrieval logic works +} + +// Test registration and retrieval with custom key +void test_register_and_get_custom_key() { + const std::string group_name = "test_group_custom"; + c10::Device device(c10::DeviceType::CUDA, 0); + + auto& manager = NCCLDevCommManager::get(device); + + // Create test device communicators (uninitialized for testing) + ncclDevComm devcomm0 = {}; + ncclDevComm devcomm1 = {}; + + // Register with custom keys + auto devcomm0_opt = + manager.register_devcomm(group_name, devcomm0, "custom_key0"); + EXPECT_TRUE(devcomm0_opt.has_value()); + + auto devcomm1_opt = + manager.register_devcomm(group_name, devcomm1, "custom_key1"); + EXPECT_TRUE(devcomm1_opt.has_value()); + + // Get with custom keys + auto retrieved0_opt = manager.get_devcomm(group_name, "custom_key0"); + EXPECT_TRUE(retrieved0_opt.has_value()); + + auto retrieved1_opt = manager.get_devcomm(group_name, "custom_key1"); + EXPECT_TRUE(retrieved1_opt.has_value()); +} + +// Test that registering the same key twice throws an error +void test_duplicate_registration_throws() { + const std::string group_name = "test_group_duplicate"; + c10::Device device(c10::DeviceType::CUDA, 0); + + auto& manager = NCCLDevCommManager::get(device); + + // Create test device communicators (uninitialized for testing) + ncclDevComm devcomm0 = {}; + ncclDevComm devcomm1 = {}; + + // Register first time - should succeed + auto devcomm0_opt = + manager.register_devcomm(group_name, devcomm0, "duplicate_key"); + EXPECT_TRUE(devcomm0_opt.has_value()); + + // Register second time with same key - should throw + EXPECT_THROW( + manager.register_devcomm(group_name, devcomm1, "duplicate_key"), + c10::Error); +} + +// Test that getting a non-existent key returns nullopt +void test_get_nonexistent_key() { + const std::string group_name = "test_group_nonexistent"; + c10::Device device(c10::DeviceType::CUDA, 0); + + auto& manager = NCCLDevCommManager::get(device); + + // Try to get a key that doesn't exist + auto retrieved_opt = manager.get_devcomm(group_name, "nonexistent_key"); + EXPECT_FALSE(retrieved_opt.has_value()); + + // Try to get from a group that doesn't exist + auto retrieved_group_opt = + manager.get_devcomm("nonexistent_group", "any_key"); + EXPECT_FALSE(retrieved_group_opt.has_value()); +} + +// Test multiple groups with different keys +void test_multiple_groups() { + const std::string group1 = "group1"; + const std::string group2 = "group2"; + c10::Device device(c10::DeviceType::CUDA, 0); + + auto& manager = NCCLDevCommManager::get(device); + + // Create test device communicators (uninitialized for testing) + ncclDevComm devcomm1a = {}; + ncclDevComm devcomm1b = {}; + ncclDevComm devcomm2a = {}; + + // Register in group1 + auto devcomm1a_opt = manager.register_devcomm(group1, devcomm1a, "key1a"); + EXPECT_TRUE(devcomm1a_opt.has_value()); + auto devcomm1b_opt = manager.register_devcomm(group1, devcomm1b, "key1b"); + EXPECT_TRUE(devcomm1b_opt.has_value()); + + // Register in group2 + auto devcomm2a_opt = manager.register_devcomm(group2, devcomm2a, "key2a"); + EXPECT_TRUE(devcomm2a_opt.has_value()); + + // Verify we can retrieve from both groups + auto retrieved1a_opt = manager.get_devcomm(group1, "key1a"); + EXPECT_TRUE(retrieved1a_opt.has_value()); + + auto retrieved1b_opt = manager.get_devcomm(group1, "key1b"); + EXPECT_TRUE(retrieved1b_opt.has_value()); + + auto retrieved2a_opt = manager.get_devcomm(group2, "key2a"); + EXPECT_TRUE(retrieved2a_opt.has_value()); + + // Verify keys from one group don't affect the other + auto cross_group_opt = manager.get_devcomm(group1, "key2a"); + EXPECT_FALSE(cross_group_opt.has_value()); +} + +// Test the example pattern from the comments +void test_example_pattern() { + const std::string group_name = "example_group"; + c10::Device device(c10::DeviceType::CUDA, 0); + + auto& manager = NCCLDevCommManager::get(device); + + // Example: foo function pattern + auto foo = [&manager, &group_name]() { + // Try to get first + auto devcomm_opt = manager.get_devcomm(group_name); + if (!devcomm_opt) { + // Not found, create then register + ncclDevComm devcomm = {}; // In real code: ncclDevCommCreate(...) + devcomm_opt = manager.register_devcomm(group_name, devcomm); + } + ncclDevComm& devcomm_ref = devcomm_opt->get(); + (void)devcomm_ref; // In real code, used for NCCL operations + EXPECT_TRUE(devcomm_opt.has_value()); + }; + + foo(); + + // Example: bar function pattern with multiple keys + auto bar = [&manager, &group_name]() { + ncclDevComm devcomm0 = {}; // In real code: ncclDevCommCreate(...) + ncclDevComm devcomm1 = {}; // In real code: ncclDevCommCreate(...) + // Register with custom keys + manager.register_devcomm(group_name, devcomm0, "bar0"); + manager.register_devcomm(group_name, devcomm1, "bar1"); + + // Verify both are registered + auto bar0_opt = manager.get_devcomm(group_name, "bar0"); + EXPECT_TRUE(bar0_opt.has_value()); + + auto bar1_opt = manager.get_devcomm(group_name, "bar1"); + EXPECT_TRUE(bar1_opt.has_value()); + }; + + bar(); +} + +// Note: This test file requires CUDA and NCCL to be available. +// The actual ncclDevComm type would be used in real scenarios. +// For unit testing, we're using a simple integer as a placeholder. +// In a real test environment, you would need to: +// 1. Initialize CUDA +// 2. Initialize NCCL +// 3. Create actual ncclDevComm objects using ncclDevCommCreate +// 4. Clean up properly + +TEST(NCCLDevCommManagerTest, RegisterAndGetComm) { + if (!at::cuda::is_available()) { + GTEST_SKIP() << "CUDA not available, skipping test"; + } + test_register_and_get_comm(); +} + +TEST(NCCLDevCommManagerTest, RegisterAndGetDefaultKey) { + if (!at::cuda::is_available()) { + GTEST_SKIP() << "CUDA not available, skipping test"; + } + test_register_and_get_default_key(); +} + +TEST(NCCLDevCommManagerTest, RegisterAndGetCustomKey) { + if (!at::cuda::is_available()) { + GTEST_SKIP() << "CUDA not available, skipping test"; + } + test_register_and_get_custom_key(); +} + +TEST(NCCLDevCommManagerTest, DuplicateRegistrationThrows) { + if (!at::cuda::is_available()) { + GTEST_SKIP() << "CUDA not available, skipping test"; + } + test_duplicate_registration_throws(); +} + +TEST(NCCLDevCommManagerTest, GetNonexistentKey) { + if (!at::cuda::is_available()) { + GTEST_SKIP() << "CUDA not available, skipping test"; + } + test_get_nonexistent_key(); +} + +TEST(NCCLDevCommManagerTest, MultipleGroups) { + if (!at::cuda::is_available()) { + GTEST_SKIP() << "CUDA not available, skipping test"; + } + test_multiple_groups(); +} + +TEST(NCCLDevCommManagerTest, ExamplePattern) { + if (!at::cuda::is_available()) { + GTEST_SKIP() << "CUDA not available, skipping test"; + } + test_example_pattern(); +} + +#else // NCCL_HAS_SYMMEM_DEVICE_SUPPORT + +TEST(NCCLDevCommManagerTest, Skipped) { + GTEST_SKIP() << "NCCL symmetric memory device support not available"; +} + +#endif // NCCL_HAS_SYMMEM_DEVICE_SUPPORT diff --git a/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupGlooAsyncTest.cpp b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupGlooAsyncTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..542b68cd5f96e49a5bf1dcc79f3b13ac4da3e3fd --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupGlooAsyncTest.cpp @@ -0,0 +1,273 @@ +#include +#include + +#include +#include +#include +#include +#include "CUDATest.hpp" +#include "TestUtils.hpp" + +using namespace c10d::test; + +using at::cuda::CUDAStream; + +template +std::vector initialize(const std::string& path, size_t N, Args&&... args) { + std::vector tests; + for ([[maybe_unused]] const auto i : c10::irange(N)) { + tests.push_back(std::move(T(path, std::forward(args)...))); + } + + std::vector threads; + for ([[maybe_unused]] const auto i : c10::irange(N)) { + threads.push_back(std::thread([i, N, &tests] { tests[i].start(i, N); })); + } + + for (auto& thread : threads) { + thread.join(); + } + + return tests; +} + +class AsyncTest { + public: + AsyncTest(std::string path) : path_(std::move(path)) {} + + AsyncTest(AsyncTest&& other) noexcept = default; + + ::c10d::ProcessGroupGloo& getProcessGroup() { + return *pg_; + } + + void start(int rank, int size) { + auto store = c10::make_intrusive<::c10d::FileStore>(path_, size); + + // Use tiny timeout to make this test run fast + auto options = ::c10d::ProcessGroupGloo::Options::create(); + options->timeout = std::chrono::milliseconds(50); + options->devices.push_back( + ::c10d::ProcessGroupGloo::createDeviceForHostname("127.0.0.1")); + + pg_ = + std::make_unique<::c10d::ProcessGroupGloo>(store, rank, size, options); + } + + protected: + std::string path_; + std::unique_ptr<::c10d::ProcessGroupGloo> pg_; +}; + +class AsyncInputIsOutputTest : public AsyncTest { + public: + AsyncInputIsOutputTest(const std::string& path, int numTensors) + : AsyncTest(path), + numTensors_(numTensors), + numDevices_(cudaNumDevices()) { + // Allocate inputs on available devices in a round robin fashion. + ::at::globalContext().lazyInitDevice(c10::DeviceType::CUDA); + inputs_.resize(numTensors_); + for (const auto i : c10::irange(numTensors_)) { + inputs_[i] = at::empty( + {16, 16}, + at::device( + {at::kCUDA, static_cast(i % numDevices_)})); + } + + // Allocate a stream per device. + // + // The "current stream" is set globally per device in THC, so we + // can't make two tensors on the same device use different streams + // and pass this along to the collective (since it uses the THC + // getters to retrieve the current stream). + // + at::cuda::OptionalCUDAGuard deviceGuard; + streams_.reserve(numDevices_); + for (const auto i : c10::irange(numDevices_)) { + deviceGuard.set_index(static_cast(i)); + streams_.push_back(at::cuda::getStreamFromPool()); + } + } + + void wait(c10::intrusive_ptr& work) { + c10::cuda::CUDAMultiStreamGuard guard(streams_); + work->wait(); + } + + std::vector getCpuTensors( + const std::vector& gpu_tensors) { + std::vector outputs(gpu_tensors.size()); + + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + + // Copy inputs to outputs + for (unsigned i = 0; i < gpu_tensors.size(); i++) { + outputs[i] = gpu_tensors[i].cpu(); + } + + return outputs; + } + + std::vector getTensors() { + return getCpuTensors(inputs_); + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const int numTensors_; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const int numDevices_; + std::vector inputs_; + std::vector streams_; +}; + +class AsyncAllreduceTest : public AsyncInputIsOutputTest { + public: + AsyncAllreduceTest(const std::string& path, int numTensors) + : AsyncInputIsOutputTest(path, numTensors) {} + + c10::intrusive_ptr run() { + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + + // Launch sleep on every stream + at::cuda::OptionalCUDAGuard deviceGuard; + for (const auto i : c10::irange(numDevices_)) { + deviceGuard.set_index(static_cast(i)); + cudaSleep(streams_[i], 10ull * 1000 * 1000); + } + + // Launch value initialization for every tensor + for (const auto i : c10::irange(numTensors_)) { + deviceGuard.set_index(static_cast(i % numDevices_)); + inputs_[i].fill_(pg_->getRank() * numTensors_ + i); + } + + return pg_->allreduce(inputs_); + } +}; + +class AsyncBroadcastTest : public AsyncInputIsOutputTest { + public: + AsyncBroadcastTest(const std::string& path, int numTensors) + : AsyncInputIsOutputTest(path, numTensors) {} + + c10::intrusive_ptr run(size_t rootRank, size_t rootTensor) { + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + + // Launch sleep on every stream + at::cuda::OptionalCUDAGuard deviceGuard; + for (const auto i : c10::irange(numDevices_)) { + deviceGuard.set_index(static_cast(i)); + cudaSleep(streams_[i], 10ull * 1000 * 1000); + } + + // Launch value initialization for every tensor + for (const auto i : c10::irange(numTensors_)) { + deviceGuard.set_index(static_cast(i % numDevices_)); + inputs_[i].fill_(pg_->getRank() * numTensors_ + i); + } + + ::c10d::BroadcastOptions options; + options.rootRank = static_cast(rootRank); + options.rootTensor = static_cast(rootTensor); + return pg_->broadcast(inputs_, options); + } +}; + +void runAsyncAllreduceTest( + const std::string& path, + size_t numProcesses = 4, + size_t numTensors = 2) { + auto tests = initialize(path, numProcesses, numTensors); + std::vector> work(numProcesses); + for (const auto i : c10::irange(numProcesses)) { + work[i] = tests[i].run(); + } + + // Wait for work to complete + for (const auto i : c10::irange(numProcesses)) { + tests[i].wait(work[i]); + } + + // Check results + for (const auto i : c10::irange(numProcesses)) { + const auto size = numProcesses * numTensors; + const auto expected = (size * (size - 1)) / 2; + auto tensors = tests[i].getTensors(); + auto results = tests[i].getCpuTensors(work[i]->result()); + EXPECT_EQ(tensors.size(), results.size()); + + for (const auto j : c10::irange(tensors.size())) { + auto& tensor = tensors[j]; + auto data = tensor.data_ptr(); + + auto& result_tensor = results[j]; + auto result_data = result_tensor.data_ptr(); + + EXPECT_EQ(tensor.numel(), result_tensor.numel()); + + for (const auto k : c10::irange(tensor.numel())) { + EXPECT_EQ(data[k], expected); + EXPECT_EQ(result_data[k], expected); + } + } + } +} + +void runAsyncBroadcastTest( + const std::string& path, + size_t numProcesses = 4, + size_t numTensors = 1) { + auto tests = initialize(path, numProcesses, numTensors); + + // Try every permutation of root rank and root tensor + for (const auto rootRank : c10::irange(numProcesses)) { + for (const auto rootTensor : c10::irange(numTensors)) { + std::vector> work(numProcesses); + for (const auto i : c10::irange(numProcesses)) { + work[i] = tests[i].run(rootRank, rootTensor); + } + + // Wait for work to complete + for (const auto i : c10::irange(numProcesses)) { + tests[i].wait(work[i]); + } + + // Check results + const auto expected = (rootRank * numTensors + rootTensor); + for (const auto i : c10::irange(numProcesses)) { + auto tensors = tests[i].getTensors(); + for (const auto& tensor : tensors) { + const auto* const data = tensor.const_data_ptr(); + for (const auto k : c10::irange(tensor.numel())) { + EXPECT_EQ(data[k], expected); + } + } + } + } + } +} + +#ifdef USE_CUDA +TEST(ProcessGroupGlooAsyncTest, testAsyncAllreduce) { + if (!at::cuda::is_available()) { + LOG(INFO) << "CUDA not available, skipping testAsyncAllreduce"; + return; + } + TemporaryFile file; + runAsyncAllreduceTest(file.path); +} + +TEST(ProcessGroupGlooAsyncTest, testAsyncBroadcast) { + if (!at::cuda::is_available()) { + LOG(INFO) << "CUDA not available, skipping testAsyncBroadcast"; + return; + } + TemporaryFile file; + runAsyncBroadcastTest(file.path); +} +#endif diff --git a/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupGlooTest.cpp b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupGlooTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5c745dee8d425cdc9c7518843d1fd93aa7177403 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupGlooTest.cpp @@ -0,0 +1,848 @@ +#ifndef _WIN32 +#include +#include +#include +#endif + +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include "TestUtils.hpp" + +using namespace c10d::test; +using namespace torch::autograd::profiler; + +constexpr auto kSendDelay = std::chrono::milliseconds(100); +constexpr auto kWaitTimeout = std::chrono::milliseconds(1); + +#ifndef _WIN32 +class SignalTest { + public: + SignalTest(std::string path) : path_(std::move(path)) {} + + ~SignalTest() { + if (arm_.joinable()) { + arm_.join(); + } + } + + // Arms test to send signal to PID when the semaphore unlocks. This + // happens as soon as the first collective completes successfully. + void arm(int pid, int signal) { + arm_ = std::thread([this, pid, signal] { + sem_.wait(); + kill(pid, signal); + }); + } + + c10::intrusive_ptr<::c10d::Work> run(int rank, int size) { + auto store = c10::make_intrusive<::c10d::FileStore>(path_, size); + + auto options = ::c10d::ProcessGroupGloo::Options::create(); + // Set a timeout that is small enough to make this test run fast, but also + // make sure that we don't get timeouts in the ProcessGroupGloo constructor. + options->timeout = std::chrono::milliseconds(1000); + options->devices.push_back( + ::c10d::ProcessGroupGloo::createDeviceForHostname("127.0.0.1")); + + ::c10d::ProcessGroupGloo pg(store, rank, size, options); + + // Initialize tensor list + std::vector tensors = { + at::ones({16, 16}), + }; + + // Loop until an exception happens + c10::intrusive_ptr<::c10d::Work> work; + while (true) { + work = pg.allreduce(tensors); + try { + work->wait(); + } catch (const std::exception&) { + break; + } + sem_.post(); + } + + return work; + } + + protected: + std::string path_; + std::thread arm_; + Semaphore sem_; +}; + +c10::intrusive_ptr<::c10d::Work> testSignal( + const std::string& path, + int signal) { + Fork fork; + if (fork.isChild()) { + SignalTest test(path); + test.run(1, 2); + exit(1); + } + + SignalTest test(path); + test.arm(fork.pid, signal); + return test.run(0, 2); +} +#endif + +class ProcessGroupGlooDelayed : public ::c10d::ProcessGroupGloo { + public: + ProcessGroupGlooDelayed( + const c10::intrusive_ptr<::c10d::Store>& store, + int rank, + int size, + c10::intrusive_ptr options) + : ProcessGroupGloo(store, rank, size, std::move(options)) {} + + c10::intrusive_ptr<::c10d::Work> send( + std::vector& tensors, + int dstRank, + int tag) override { + std::this_thread::sleep_for(kSendDelay); + return ::c10d::ProcessGroupGloo::send(tensors, dstRank, tag); + } +}; + +class CollectiveTest { + public: + static std::vector initialize( + const std::string& path, + int num, + bool delayed = false) { + std::vector tests; + for ([[maybe_unused]] const auto i : c10::irange(num)) { + tests.emplace_back(path); + } + + std::vector threads; + for (const auto i : c10::irange(num)) { + threads.emplace_back( + [i, &tests, delayed] { tests[i].start(i, tests.size(), delayed); }); + } + for (auto& thread : threads) { + thread.join(); + } + + return tests; + } + + CollectiveTest(std::string path) : path_(std::move(path)) {} + + CollectiveTest(CollectiveTest&& other) noexcept = default; + + ::c10d::ProcessGroupGloo& getProcessGroup() { + return *pg_; + } + + void start(int rank, size_t size, bool delayed) { + auto store = c10::make_intrusive<::c10d::FileStore>(path_, size); + + // Set a timeout that is small enough to make this test run fast, but also + // make sure that we don't get timeouts in the ProcessGroupGloo constructor. + auto options = ::c10d::ProcessGroupGloo::Options::create(); + options->timeout = std::chrono::milliseconds(1000); + options->devices.push_back( + ::c10d::ProcessGroupGloo::createDeviceForHostname("127.0.0.1")); + + if (!delayed) { + pg_ = std::make_unique<::c10d::ProcessGroupGloo>( + store, rank, size, options); + } else { + pg_ = + std::make_unique(store, rank, size, options); + } + } + + protected: + std::string path_; + std::unique_ptr<::c10d::ProcessGroupGloo> pg_; +}; + +std::vector> copyTensors( + const std::vector>& inputs) { + std::vector> outputs(inputs.size()); + for (const auto i : c10::irange(inputs.size())) { + const auto& input = inputs[i]; + std::vector output(input.size()); + for (const auto j : c10::irange(input.size())) { + output[j] = input[j].cpu(); + } + outputs[i] = output; + } + return outputs; +} + +std::vector> waitWork( + const std::vector>& works) { + std::vector> outputTensors; + for (auto& work : works) { + try { + work->wait(); + } catch (const std::exception& ex) { + LOG(ERROR) << "Exception received: " << ex.what() << '\n'; + } + outputTensors.emplace_back(work->result()); + } + return copyTensors(outputTensors); +} + +std::vector> waitFuture( + const std::vector>& works) { + std::vector> outputTensors; + for (auto& work : works) { + auto fut = work->getFuture(); + try { + fut->wait(); + } catch (const std::exception& ex) { + LOG(ERROR) << "Exception received: " << ex.what() << '\n'; + } + auto result = fut->value(); + if (result.isNone()) { + outputTensors.emplace_back(); + } else if (result.isTensorList()) { + outputTensors.emplace_back(result.toTensorVector()); + } else { + TORCH_CHECK(false, "future result should be tensor list or none"); + } + } + return copyTensors(outputTensors); +} + +void checkProfiledEvents( + const thread_event_lists& event_lists, + const char* expected_profile_str, + int expected_count, + std::vector> expected_shapes, + bool verify_shapes = true) { + if (verify_shapes) { + EXPECT_EQ(expected_count, expected_shapes.size()); + } + std::vector matched_shapes(expected_count); + for (const auto& li : event_lists) { + for (const auto& evt : li) { + auto match = !strcmp(evt.name(), expected_profile_str); + if (verify_shapes && match) { + auto shapesVec = evt.shapes(); + for (const auto i : c10::irange(expected_count)) { + // Assumptions: no two expected shapes are the same + if (shapesVec[0] == expected_shapes[i]) { + matched_shapes[i] = true; + } + } + } + } + } + if (verify_shapes) { + for (bool match : matched_shapes) { + EXPECT_TRUE(match); + } + } +} + +void testAllreduce( + const std::string& path, + const at::DeviceType b, + const at::ScalarType dtype = at::kFloat) { + const auto size = 4; + auto tests = CollectiveTest::initialize(path, size); + + // Generate inputs + std::vector> inputs(size); + std::vector> allShapes; + std::vector shapes = {16, 16}; + for (const auto i : c10::irange(size)) { + auto tensor = at::ones(shapes, at::dtype(dtype).device(b)) * i; + std::vector shapesVec = shapes; + allShapes.emplace_back(std::move(shapesVec)); + inputs[i] = std::vector({tensor}); + } + + // Kick off work + std::vector> work(size); + const char* GLOO_ALLREDUCE_STR = "gloo:all_reduce"; + enableProfilerLegacy(ProfilerConfig( + ProfilerState::CPU, /* report_input_shapes */ true, false)); + for (const auto i : c10::irange(size)) { + work[i] = tests[i].getProcessGroup().allreduce(inputs[i]); + } + // Wait for work to complete + auto outputs = waitFuture(work); + + auto event_lists = disableProfilerLegacy(); + checkProfiledEvents(event_lists, GLOO_ALLREDUCE_STR, size, allShapes); + + // Verify outputs + const auto expected = (size * (size - 1)) / 2; + for (const auto i : c10::irange(size)) { + auto tensor = outputs[i][0].to(at::kFloat); + auto data = tensor.data_ptr(); + for (const auto j : c10::irange(tensor.numel())) { + EXPECT_EQ(data[j], expected); + } + } +} + +// UsingWorkAPI tests are to make sure we still properly support work API. +// This should go away as we deprecate it. +void testAllreduceUsingWorkAPI( + const std::string& path, + const at::DeviceType b, + const at::ScalarType dtype = at::kFloat) { + const auto size = 4; + auto tests = CollectiveTest::initialize(path, size); + + // Generate inputs + std::vector> inputs(size); + std::vector> allShapes; + std::vector shapes = {16, 16}; + for (const auto i : c10::irange(size)) { + auto tensor = at::ones(shapes, at::dtype(dtype).device(b)) * i; + std::vector shapesVec = shapes; + allShapes.emplace_back(std::move(shapesVec)); + inputs[i] = std::vector({tensor}); + } + + // Kick off work + std::vector> work(size); + const char* GLOO_ALLREDUCE_STR = "gloo:all_reduce"; + enableProfilerLegacy(ProfilerConfig( + ProfilerState::CPU, /* report_input_shapes */ true, false)); + for (const auto i : c10::irange(size)) { + work[i] = tests[i].getProcessGroup().allreduce(inputs[i]); + } + // Wait for work to complete + auto outputs = waitWork(work); + + auto event_lists = disableProfilerLegacy(); + checkProfiledEvents(event_lists, GLOO_ALLREDUCE_STR, size, allShapes); + + // Verify outputs + const auto expected = (size * (size - 1)) / 2; + for (const auto i : c10::irange(size)) { + auto tensor = outputs[i][0].to(at::kFloat); + auto data = tensor.data_ptr(); + for (const auto j : c10::irange(tensor.numel())) { + EXPECT_EQ(data[j], expected); + } + } +} + +void testBroadcast( + const std::string& path, + const at::DeviceType b, + const at::ScalarType dtype = at::kFloat) { + const auto size = 2; + const auto stride = 2; + auto tests = CollectiveTest::initialize(path, size); + + std::vector> inputs(size); + std::vector shapes = {16, 16}; + // Try every permutation of root rank and root tensor + for (const auto i : c10::irange(size)) { + for (const auto j : c10::irange(stride)) { + std::vector> allShapes; + // Initialize inputs + for (const auto k : c10::irange(size)) { + std::vector shapesVec = shapes; + allShapes.emplace_back(std::move(shapesVec)); + inputs[k].resize(stride); + // This won't work if we ever support sparse CUDA + at::OptionalDeviceGuard deviceGuard; + for (const auto l : c10::irange(stride)) { + if (b == at::DeviceType::CUDA) { + deviceGuard.reset_device( + at::Device(at::kCUDA, static_cast(l))); + } + inputs[k][l] = + at::ones(shapes, at::dtype(dtype).device(b)) * (k * stride + l); + } + } + + ::c10d::BroadcastOptions options; + options.rootRank = i; + options.rootTensor = j; + + // Kick off work + const char* GLOO_BROADCAST_STR = "gloo:broadcast"; + enableProfilerLegacy(ProfilerConfig( + ProfilerState::CPU, /* report_input_shapes */ true, false)); + std::vector> work(size); + + for (const auto i : c10::irange(size)) { + work[i] = tests[i].getProcessGroup().broadcast(inputs[i], options); + } + + // Wait for work to complete + auto outputs = waitFuture(work); + + auto event_lists = disableProfilerLegacy(); + checkProfiledEvents(event_lists, GLOO_BROADCAST_STR, size, allShapes); + + // Verify outputs + const auto expected = (i * stride + j); + for (const auto k : c10::irange(size)) { + for (const auto l : c10::irange(stride)) { + auto tensor = outputs[k][l].to(at::kFloat); + auto data = tensor.data_ptr(); + for (const auto n : c10::irange(tensor.numel())) { + EXPECT_EQ(data[n], expected); + } + } + } + } + } +} + +void testAlltoall(const std::string& path, const at::DeviceType b) { + const auto size = 4; + auto tests = CollectiveTest::initialize(path, size); + + // Generate inputs + std::vector inputs(size); + std::vector> blobs = { + {0, 1, 2, 3, 4, 5}, + {10, 11, 12, 13, 14, 15, 16, 17, 18}, + {20, 21, 22, 23, 24}, + {30, 31, 32, 33, 34, 35, 36}, + }; + for (const auto rank : c10::irange(size)) { + std::vector& blob = blobs[rank]; + inputs[rank] = + at::from_blob(blob.data(), static_cast(blob.size())).to(b); + } + + // Allocate outputs + std::vector outputs(size); + std::vector outputLengths = {9, 7, 6, 5}; + for (const auto rank : c10::irange(size)) { + outputs[rank] = + at::empty(outputLengths[rank], c10::TensorOptions(at::kInt).device(b)); + } + + // Generate splits + std::vector> inputSplits = { + {2, 2, 1, 1}, + {3, 2, 2, 2}, + {2, 1, 1, 1}, + {2, 2, 2, 1}, + }; + std::vector> outputSplits = { + {2, 3, 2, 2}, + {2, 2, 1, 2}, + {1, 2, 1, 2}, + {1, 2, 1, 1}, + }; + + // Kick off work + std::vector> work(size); + const char* GLOO_A2A_STR = "gloo:all_to_all"; + std::vector> allShapes; + for (const auto& vec : inputSplits) { + // Due to concatenation of tensors, shape will actually be the sum + int64_t sum = 0; + for (const auto& s : vec) { + sum += s; + } + allShapes.push_back({sum}); + } + enableProfilerLegacy(ProfilerConfig( + ProfilerState::CPU, /* report_input_shapes */ true, false)); + for (const auto rank : c10::irange(size)) { + work[rank] = tests[rank].getProcessGroup().all_to_all_single( + outputs[rank], inputs[rank], outputSplits[rank], inputSplits[rank]); + } + + // Wait for work to complete + for (const auto i : c10::irange(size)) { + work[i]->wait(); + } + + auto event_lists = disableProfilerLegacy(); + checkProfiledEvents(event_lists, GLOO_A2A_STR, size, allShapes); + // Verify outputs + std::vector> expected = { + {0, 1, 10, 11, 12, 20, 21, 30, 31}, + {2, 3, 13, 14, 22, 32, 33}, + {4, 15, 16, 23, 34, 35}, + {5, 17, 18, 24, 36}, + }; + for (const auto rank : c10::irange(size)) { + at::Tensor tensor = outputs[rank].cpu(); + EXPECT_EQ(tensor.numel(), expected[rank].size()); + auto data = tensor.data_ptr(); + for (const auto j : c10::irange(tensor.numel())) { + EXPECT_EQ(data[j], expected[rank][j]); + } + } +} + +void testBarrier(const std::string& path) { + const auto size = 2; + auto tests = CollectiveTest::initialize(path, size); + + // Kick off work + enableProfilerLegacy(ProfilerConfig( + ProfilerState::CPU, /* report_input_shapes */ true, false)); + std::vector> work(size); + for (const auto i : c10::irange(size)) { + work[i] = tests[i].getProcessGroup().barrier(); + } + + // Wait for work to complete + waitFuture(work); + + auto event_lists = disableProfilerLegacy(); + const char* GLOO_STR = "gloo:barrier"; + std::vector> allShapes; + // Barrier does not use tensors, so skip shape checking. + checkProfiledEvents( + event_lists, + GLOO_STR, + size, + allShapes, + /* verify_shapes */ false); +} + +void testMonitoredBarrier(const std::string& path) { + const auto size = 2; + auto tests = CollectiveTest::initialize(path, size); + // Non-failure case: all ranks pass the blocking monitored barrier. + auto runMonitoredBarrier = [&](int i) { + tests[i].getProcessGroup().monitoredBarrier(); + }; + std::vector threads; + threads.reserve(size); + for (const auto r : c10::irange(size)) { + threads.emplace_back([=]() { runMonitoredBarrier(r); }); + } + for (auto& t : threads) { + t.join(); + } + // Failure case: Only rank 0 calls into monitored barrier, should result in + // error + auto runMonitoredBarrierWithException = [&](int i) { + if (i != 0) { + return; + } + + try { + tests[i].getProcessGroup().monitoredBarrier(); + FAIL() << "Exception should have been thrown."; + } catch (const std::exception& e) { + auto pos = std::string(e.what()).find("Rank 1"); + EXPECT_TRUE(pos != std::string::npos); + } + }; + threads.clear(); + for (const auto r : c10::irange(size)) { + threads.emplace_back([=]() { runMonitoredBarrierWithException(r); }); + } + for (auto& t : threads) { + t.join(); + } +} + +void testWaitDelay(const std::string& path) { + const auto size = 2; + auto tests = CollectiveTest::initialize(path, size, /* delay */ true); + + constexpr uint64_t tag = 0x1337; + // test that waiting for work to be sent can be aborted successfully. + auto selfRank = 0; + auto dstRank = 1; + std::vector tensors = { + at::ones({16, 16}), + }; + auto& pg = tests[selfRank].getProcessGroup(); + auto sendWork = pg.send(tensors, dstRank, tag); + EXPECT_THROW(sendWork->wait(kWaitTimeout), std::exception); +} + +void testSend(const std::string& path) { + const auto size = 2; + auto tests = CollectiveTest::initialize(path, size); + + constexpr uint64_t tag = 0x1337; + // test that waiting for work to be sent can be aborted successfully. + auto selfRank = 0; + auto dstRank = 1; + std::vector shapes{16, 16}; + std::vector> allShapes; + allShapes.push_back(shapes); + std::vector tensors = { + at::ones(shapes), + }; + auto& pg = tests[selfRank].getProcessGroup(); + const char* GLOO_SEND_STR = "gloo:send"; + enableProfilerLegacy(ProfilerConfig( + ProfilerState::CPU, /* report_input_shapes */ true, false)); + auto sendWork = pg.send(tensors, dstRank, tag); + bool sendCompleted = false; + std::thread waitSendThreadAbort([&]() { sendCompleted = sendWork->wait(); }); + sendWork->abort(); + // Block until the sendWork gets successfully aborted + waitSendThreadAbort.join(); + EXPECT_FALSE(sendCompleted); + auto event_lists = disableProfilerLegacy(); + checkProfiledEvents(event_lists, GLOO_SEND_STR, 1, allShapes); + + // Now create a separate sender thread to ensure that future waitsends can + // complete successfully. + + // Helper receiver to simulate a real recv/send pair + std::thread recvThread([&]() { + auto selfRank = 1; + auto srcRank = 0; + auto& pg = tests[selfRank].getProcessGroup(); + std::vector tensors = { + at::ones({16, 16}), + }; + + auto recvWork = pg.recv(tensors, srcRank, tag); + recvWork->wait(); + }); + + // Sender thread + std::thread sendThread([&]() { sendCompleted = sendWork->wait(); }); + sendThread.join(); + recvThread.join(); + EXPECT_TRUE(sendCompleted); +} + +void testRecv(const std::string& path) { + const auto size = 2; + auto tests = CollectiveTest::initialize(path, size); + constexpr uint64_t tag = 0x1337; + // test that waiting for work to be received can be aborted successfully. + auto selfRank = 0; + auto srcRank = 1; + std::vector shapes = {16, 16}; + std::vector> allShapes; + allShapes.push_back(shapes); + std::vector tensors = { + at::ones(shapes), + }; + const char* GLOO_RECV_STR = "gloo:recv"; + auto& pg = tests[selfRank].getProcessGroup(); + enableProfilerLegacy(ProfilerConfig( + ProfilerState::CPU, /* report_input_shapes */ true, false)); + auto recvWork = pg.recv(tensors, srcRank, tag); + bool recvCompleted = false; + std::thread waitRecvThreadAbort([&]() { recvCompleted = recvWork->wait(); }); + recvWork->abort(); + // Block until the first recv gets successfully aborted + waitRecvThreadAbort.join(); + EXPECT_FALSE(recvCompleted); + auto event_lists = disableProfilerLegacy(); + checkProfiledEvents(event_lists, GLOO_RECV_STR, 1, allShapes); + + // Now create a separate receiver thread to ensure that future waits can + // complete successfully. + + // Helper sender thread to simulate a real recv/send pair. + std::thread senderThread([&]() { + auto selfRank = 1; + auto destRank = 0; + + auto& pg = tests[selfRank].getProcessGroup(); + + std::vector tensors = { + at::ones({16, 16}), + }; + auto sendWork = pg.send(tensors, destRank, tag); + sendWork->wait(); + }); + // Receiver thread. + std::thread receiverThread([&]() { recvCompleted = recvWork->wait(); }); + senderThread.join(); + receiverThread.join(); + EXPECT_TRUE(recvCompleted); +} + +void testStoreSetGet(const std::string& path) { + const auto size = 2; + auto tests = CollectiveTest::initialize(path, size); + // test that get() gets the same value as the one that was set() + std::vector testVector = {1, 1, 1, 1}; + // Cast to ProcessGroupGloo::GlooStore to test specific GlooStore APIs. + auto rank_0_glooStore = static_cast( + tests[0].getProcessGroup()._getStore().get()); + auto rank_1_glooStore = static_cast( + tests[1].getProcessGroup()._getStore().get()); + + rank_0_glooStore->setUint("testKey", testVector); + auto value = rank_1_glooStore->getUint("testKey"); + EXPECT_TRUE(value == testVector); +} + +#ifndef _WIN32 +TEST(ProcessGroupGlooTest, testSIGSTOPException) { + // test SIGSTOP + // Fork() and TSAN don't play well together, so skip the test if we're testing + // with TSAN. + if (isTSANEnabled()) { + LOG(INFO) << "Skipping test since Fork() + TSAN is broken"; + return; + } + + TemporaryFile file; + auto work = testSignal(file.path, SIGSTOP); + EXPECT_FALSE(work->isSuccess()); + EXPECT_THROW(std::rethrow_exception(work->exception()), std::exception); +} + +TEST(ProcessGroupGlooTest, testSIGKILLException) { + // test SIGKILL + // Fork() and TSAN don't play well together, so skip the test if we're testing + // with TSAN. + if (isTSANEnabled()) { + LOG(INFO) << "Skipping test since Fork() + TSAN is broken"; + return; + } + + TemporaryFile file; + auto work = testSignal(file.path, SIGKILL); + EXPECT_FALSE(work->isSuccess()); + EXPECT_THROW(std::rethrow_exception(work->exception()), std::exception); +} +#endif + +TEST(ProcessGroupGlooTest, testAllReduceCPU) { + { + TemporaryFile file; + testAllreduce(file.path, at::DeviceType::CPU); + testAllreduceUsingWorkAPI(file.path, at::DeviceType::CPU); + } +} + +TEST(ProcessGroupGlooTest, testAllReduceBfloatCPU) { + { + TemporaryFile file; + testAllreduce(file.path, at::DeviceType::CPU, at::kBFloat16); + testAllreduceUsingWorkAPI(file.path, at::DeviceType::CPU); + } +} + +TEST(ProcessGroupGlooTest, testBroadcastCPU) { + { + TemporaryFile file; + testBroadcast(file.path, at::DeviceType::CPU); + } +} + +TEST(ProcessGroupGlooTest, testBroadcastBfloatCPU) { + { + TemporaryFile file; + testBroadcast(file.path, at::DeviceType::CPU, at::kBFloat16); + } +} + +TEST(ProcessGroupGlooTest, testAllToAllCPU) { + { + TemporaryFile file; + testAlltoall(file.path, at::DeviceType::CPU); + } +} + +TEST(ProcessGroupGlooTest, testBarrier) { + { + TemporaryFile file; + testBarrier(file.path); + } +} + +TEST(ProcessGroupGlooTest, testMonitoredBarrier) { + TemporaryFile file; + testMonitoredBarrier(file.path); +} + +TEST(ProcessGroupGlooTest, testSend) { + { + TemporaryFile file; + testSend(file.path); + } +} + +TEST(ProcessGroupGlooTest, testRecv) { + { + TemporaryFile file; + testRecv(file.path); + } +} + +TEST(ProcessGroupGlooTest, testStoreSetGet) { + TemporaryFile file; + testStoreSetGet(file.path); +} + +TEST(ProcessGroupGlooTest, testWaitDelay) { + { + TemporaryFile file; + testWaitDelay(file.path); + } +} + +#ifdef USE_CUDA +// CUDA-only tests +TEST(ProcessGroupGlooTest, testAllReduceCUDA) { + if (!torch::cuda::is_available()) { + LOG(INFO) << "Skipping test - requires CUDA"; + return; + } + { + TemporaryFile file; + testAllreduce(file.path, at::DeviceType::CUDA); + testAllreduceUsingWorkAPI(file.path, at::DeviceType::CUDA); + } +} + +TEST(ProcessGroupGlooTest, testBroadcastCUDA) { + if (torch::cuda::device_count() <= 1) { + LOG(INFO) << "Skipping test - requires multiple CUDA devices"; + return; + } + { + TemporaryFile file; + testBroadcast(file.path, at::DeviceType::CUDA); + } +} + +TEST(ProcessGroupGlooTest, testAlltoallCUDA) { + if (!torch::cuda::is_available()) { + LOG(INFO) << "Skipping test - requires CUDA"; + return; + } + { + TemporaryFile file; + testAlltoall(file.path, at::DeviceType::CUDA); + } +} + +TEST(ProcessGroupGlooTest, testBackendName) { + { + TemporaryFile file; + const auto size = 2; + auto tests = CollectiveTest::initialize(file.path, size); + + for (const auto i : c10::irange(size)) { + EXPECT_EQ( + tests[i].getProcessGroup().getBackendName(), + std::string(c10d::GLOO_BACKEND_NAME)); + } + } +} + +#endif diff --git a/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupMPITest.cpp b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupMPITest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1d8d64c5f4340bc513950c6fcb754be726bca14a --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupMPITest.cpp @@ -0,0 +1,480 @@ +#include + +#include +#include + +#include +#include +#include + +#define STR_HELPER(x) #x +#define STR(x) STR_HELPER(x) + +// Wait for work to complete +std::vector> waitWork( + const c10::intrusive_ptr<::c10d::ProcessGroupMPI>& pg, + const std::vector>& works) { + std::vector> outputTensors; + for (auto& work : works) { + try { + work->wait(); + } catch (const std::exception& ex) { + std::cerr << "Exception received: " << ex.what() << '\n'; + pg->abort(); + } + outputTensors.emplace_back(work->result()); + } + return outputTensors; +} + +// Wait using Futures +std::vector> waitFuture( + const c10::intrusive_ptr<::c10d::ProcessGroupMPI>& pg, + const std::vector>& works) { + std::vector> outputTensors; + for (auto& work : works) { + auto fut = work->getFuture(); + try { + fut->wait(); + } catch (const std::exception& ex) { + std::cerr << "Exception received: " << ex.what() << '\n'; + pg->abort(); + } + auto result = fut->value(); + if (result.isNone()) { + outputTensors.emplace_back(); + } else if (result.isTensorList()) { + outputTensors.emplace_back(result.toTensorVector()); + } else { + TORCH_CHECK(false, "future result should be tensor list or none"); + } + } + return outputTensors; +} + +void testAllreduce(int iter = 1000) { + auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); + + // Generate inputs + std::vector> works; + for (const auto i : c10::irange(iter)) { + auto tensor = at::ones({16, 16}) * i; + std::vector tensors = {tensor}; + + // Queue the work. + c10::intrusive_ptr<::c10d::Work> work = pg->allreduce(tensors); + works.push_back(std::move(work)); + } + + auto outputTensors = waitFuture(pg, works); + + // Get the world size + auto worldSize = pg->getSize(); + + // Verify outputs + for (const auto i : c10::irange(iter)) { + const auto expected = worldSize * i; + auto data = outputTensors[i][0].data_ptr(); + for (auto j = 0; j < outputTensors[i][0].numel(); ++j) { + if (data[j] != static_cast(expected)) { + TORCH_CHECK(false, "BOOM!"); + } + } + } +} + +void testBroadcast(int iter = 10000) { + auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); + std::vector> works; + for (const auto i : c10::irange(iter)) { + auto tensors = std::vector(); + if (pg->getRank() == 0) { + auto tensor = at::ones({16, 16}) * i; + tensors = std::vector({tensor}); + } else { + auto tensor = at::zeros({16, 16}); + tensors = std::vector({tensor}); + } + + // Queue the work. + c10::intrusive_ptr<::c10d::Work> work = pg->broadcast(tensors); + works.push_back(std::move(work)); + } + + auto outputTensors = waitFuture(pg, works); + + // Verify outputs + for (const auto i : c10::irange(iter)) { + const auto expected = i; + auto data = outputTensors[i][0].data_ptr(); + for (auto j = 0; j < outputTensors[i][0].numel(); ++j) { + if (data[j] != static_cast(expected)) { + TORCH_CHECK(false, "BOOM!"); + } + } + } +} + +void testReduce(int iter = 10000) { + auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); + std::vector> works; + for (const auto i : c10::irange(iter)) { + auto tensor = at::ones({16, 16}) * i; + auto tensors = std::vector({tensor}); + + // Queue the work. + c10::intrusive_ptr<::c10d::Work> work = pg->reduce(tensors); + works.push_back(std::move(work)); + } + + auto outputTensors = waitFuture(pg, works); + + // Get the world size + auto worldSize = pg->getSize(); + + if (pg->getRank() == 0) { + // Verify outputs + for (const auto i : c10::irange(iter)) { + const auto expected = worldSize * i; + auto data = outputTensors[i][0].data_ptr(); + for (auto j = 0; j < outputTensors[i][0].numel(); ++j) { + if (data[j] != static_cast(expected)) { + TORCH_CHECK(false, "BOOM!"); + } + } + } + } +} + +void testAllgather(int iter = 10000) { + auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); + std::vector> works; + + // Get the world size + auto worldSize = pg->getSize(); + auto rank = pg->getRank(); + + // Generate inputs + for (const auto i : c10::irange(iter)) { + auto tensor = at::ones({16, 16}) * i * rank; + auto tensors = std::vector({tensor}); + auto outputs = std::vector>(1); + outputs[0].resize(worldSize); + for (const auto j : c10::irange(worldSize)) { + outputs[0][j] = at::zeros({16, 16}); + } + + // Queue the work. + c10::intrusive_ptr<::c10d::Work> work = pg->allgather(outputs, tensors); + works.push_back(std::move(work)); + } + + auto outputTensors = waitFuture(pg, works); + + // Verify outputs + for (const auto i : c10::irange(iter)) { + for (const auto j : c10::irange(worldSize)) { + const auto expected = i * j; + auto data = outputTensors[i][j].data_ptr(); + for (auto k = 0; k < outputTensors[i][j].numel(); ++k) { + if (data[k] != static_cast(expected)) { + TORCH_CHECK(false, "BOOM!"); + } + } + } + } +} + +void testAllgatherBase(int iter = 10000) { + auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); + std::vector> works; + + // Get the world size + auto worldSize = pg->getSize(); + auto rank = pg->getRank(); + + // Generate inputs + for (const auto i : c10::irange(iter)) { + auto tensor = at::ones({16, 16}) * i * rank; + auto output = at::zeros({worldSize, 16, 16}); + + // Queue the work. + c10::intrusive_ptr<::c10d::Work> work = + pg->all_gather_single(output, tensor); + works.push_back(std::move(work)); + } + + auto outputTensors = waitFuture(pg, works); + + // Verify outputs + for (const auto i : c10::irange(iter)) { + for (const auto j : c10::irange(worldSize)) { + const auto expected = i * j; + auto data = outputTensors[i][0][j].data_ptr(); + for (auto k = 0; k < outputTensors[i][0][j].numel(); ++k) { + if (data[k] != static_cast(expected)) { + TORCH_CHECK(false, "BOOM!"); + } + } + } + } +} + +void testReduceScatter(int iter = 10000) { + auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); + std::vector> works; + + // Get the world size + auto worldSize = pg->getSize(); + auto rank = pg->getRank(); + + // Generate inputs + int count = 2; + for (const auto i : c10::irange(iter)) { + auto tensors = std::vector>(1); + tensors[0].resize(worldSize); + for (const auto j : c10::irange(worldSize)) { + tensors[0][j] = at::ones({count, count}) * i * rank; + } + auto output = at::zeros({count, count}); + auto outputs = std::vector({output}); + + // Queue the work. + c10::intrusive_ptr<::c10d::Work> work = + pg->reduce_scatter(outputs, tensors); + works.push_back(std::move(work)); + } + + auto outputTensors = waitFuture(pg, works); + + // Verify outputs + for (const auto i : c10::irange(iter)) { + const auto expected = i * (worldSize * (worldSize - 1)) / 2.0; + auto data = outputTensors[i][0].data_ptr(); + for (auto j = 0; j < outputTensors[i][0].numel(); ++j) { + if (data[j] != static_cast(expected)) { + TORCH_CHECK(false, "BOOM!"); + } + } + } +} + +void testReduceScatterBase(int iter = 10000) { + auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); + std::vector> works; + + // Get the world size + auto worldSize = pg->getSize(); + auto rank = pg->getRank(); + + // Generate inputs + for (const auto i : c10::irange(iter)) { + auto tensor = at::ones({worldSize, 16, 16}) * i * rank; + auto output = at::zeros({16, 16}); + auto outputs = std::vector({output}); + + // Queue the work. + c10::intrusive_ptr<::c10d::Work> work = + pg->reduce_scatter_single(output, tensor); + works.push_back(std::move(work)); + } + + auto outputTensors = waitFuture(pg, works); + + // Verify outputs + for (const auto i : c10::irange(iter)) { + const auto expected = i * (worldSize * (worldSize - 1)) / 2.0; + auto data = outputTensors[i][0].data_ptr(); + for (auto j = 0; j < outputTensors[i][0].numel(); ++j) { + if (data[j] != static_cast(expected)) { + TORCH_CHECK(false, "BOOM!"); + } + } + } +} + +void testGather(int iter = 10000) { + auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); + std::vector> works; + + // Get the world size + auto worldSize = pg->getSize(); + auto rank = pg->getRank(); + + // Generate inputs + for (const auto i : c10::irange(iter)) { + auto tensor = at::ones({16, 16}) * i * rank; + auto tensors = std::vector({tensor}); + auto outputs = std::vector>(0); + if (rank == 0) { + outputs = std::vector>(1); + outputs[0].resize(worldSize); + for (const auto j : c10::irange(worldSize)) { + outputs[0][j] = at::zeros({16, 16}); + } + } + + // Queue the work. + c10::intrusive_ptr<::c10d::Work> work = pg->gather(outputs, tensors); + works.push_back(std::move(work)); + } + + auto outputTensors = waitFuture(pg, works); + + // Verify outputs + if (rank == 0) { + for (const auto i : c10::irange(iter)) { + for (const auto j : c10::irange(worldSize)) { + const auto expected = i * j; + auto data = outputTensors[i][j].data_ptr(); + for (auto k = 0; k < outputTensors[i][j].numel(); ++k) { + if (data[k] != static_cast(expected)) { + TORCH_CHECK(false, "BOOM!"); + } + } + } + } + } else { + for (const auto i : c10::irange(iter)) { + if (!outputTensors[i].empty()) { + TORCH_CHECK(false, "BOOM!"); + } + } + } +} + +void testScatter(int iter = 1) { + auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); + std::vector> works; + + // Get the world size + auto worldSize = pg->getSize(); + auto rank = pg->getRank(); + + // Generate inputs + for (const auto i : c10::irange(iter)) { + auto tensor = at::zeros({16, 16}); + auto tensors = std::vector({tensor}); + auto inputs = std::vector>(0); + if (rank == 0) { + inputs = std::vector>(1); + inputs[0].resize(worldSize); + for (const auto j : c10::irange(worldSize)) { + inputs[0][j] = at::ones({16, 16}) * i * j; + } + } + + // Queue the work. + c10::intrusive_ptr<::c10d::Work> work = pg->scatter(tensors, inputs); + works.push_back(std::move(work)); + } + + auto outputTensors = waitFuture(pg, works); + + // Verify outputs + for (const auto i : c10::irange(iter)) { + for (const auto j : c10::irange(worldSize)) { + const auto expected = i * j; + auto data = outputTensors[i][0].data_ptr(); + for (auto k = 0; k < outputTensors[i][0].numel(); ++k) { + if (data[k] != static_cast(expected)) { + TORCH_CHECK(false, "BOOM!"); + } + } + } + } +} + +void testSendRecv(bool recvAnysource, int iter = 10000) { + auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); + // Generate inputs + std::vector> works; + + // pg->send does not keep sent tensors alive, so we need to. + std::vector> sendTensors(iter); + auto rank = pg->getRank(); + for (const auto i : c10::irange(iter)) { + if (rank == 0) { + auto tensor = at::ones({16, 16}) * i; + sendTensors[i] = std::vector({tensor}); + + // Queue the work. + c10::intrusive_ptr<::c10d::Work> work = pg->send(sendTensors[i], 1, 0); + works.push_back(std::move(work)); + } else { + auto tensor = at::zeros({16, 16}); + auto recvTensors = std::vector({tensor}); + + // Queue the work. + if (!recvAnysource) { + c10::intrusive_ptr<::c10d::Work> work = pg->recv(recvTensors, 0, 0); + works.push_back(std::move(work)); + } else { + c10::intrusive_ptr<::c10d::Work> work = + pg->recvAnysource(recvTensors, 0); + works.push_back(std::move(work)); + } + } + } + + auto outputTensors = waitWork(pg, works); + if (rank == 0) { + return; + } + + std::vector srcRanks; + if (recvAnysource) { + for (const auto& work : works) { + srcRanks.push_back(work->sourceRank()); + } + } + + // Verify outputs + for (const auto i : c10::irange(iter)) { + if (recvAnysource && srcRanks[i] != 0) { + TORCH_CHECK(false, "src rank is wrong for recvAnysource"); + } + const auto expected = i; + auto data = outputTensors[i][0].data_ptr(); + for (auto j = 0; j < outputTensors[i][0].numel(); ++j) { + if (data[j] != static_cast(expected)) { + TORCH_CHECK(false, "BOOM!"); + } + } + } +} + +void testBackendName() { + auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); + if (pg->getBackendName() != std::string(c10d::MPI_BACKEND_NAME)) { + TORCH_CHECK(false, "BOOM!"); + } +} + +int main(int argc, char** argv) { +#ifdef MPIEXEC + // If we are within an openmpi mpirun, then skip the exec + if (!std::getenv("OMPI_COMM_WORLD_SIZE")) { + std::cout << "Execute mpiexec from: " << STR(MPIEXEC) << '\n'; + execl(STR(MPIEXEC), "-np 2", argv[0], (char*)nullptr); + } + + testAllreduce(); + testBroadcast(); + testReduce(); + testAllgather(); + testAllgatherBase(); + testReduceScatter(); + testReduceScatterBase(); + testGather(); + testScatter(); + testSendRecv(false); + testSendRecv(true); + testBackendName(); + + std::cout << "Test successful" << '\n'; +#else + std::cout << "MPI executable not found, skipping test" << std::endl; +#endif + return EXIT_SUCCESS; +} diff --git a/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupNCCLErrorsTest.cpp b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupNCCLErrorsTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..88ac5f86fdd9139bfa4e1574ab9e77d6fadda19c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupNCCLErrorsTest.cpp @@ -0,0 +1,502 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include "CUDATest.hpp" +#include "TestUtils.hpp" + +#include + +using namespace c10d::test; + +constexpr int kNcclErrorHandlingVersion = 2400; + +class WorkNCCLSimulateErrors : public c10d::ProcessGroupNCCL::WorkNCCL { + public: + WorkNCCLSimulateErrors( + at::Device& device, + bool simulate_error, + int rank, + c10d::OpType opType, + uint64_t seq, + bool isP2P) + : WorkNCCL("0", "default_pg", device, rank, opType, seq, isP2P), + simulateError_(simulate_error) {} + + std::exception_ptr checkForNCCLErrors() override { + if (simulateError_) { + return std::make_exception_ptr(std::runtime_error("Error")); + } + return c10d::ProcessGroupNCCL::WorkNCCL::checkForNCCLErrors(); + } + + private: + bool simulateError_; +}; + +class ProcessGroupNCCLSimulateErrors : public c10d::ProcessGroupNCCL { + public: + ProcessGroupNCCLSimulateErrors( + const c10::intrusive_ptr& store, + int rank, + int size, + c10::intrusive_ptr opts) + : ProcessGroupNCCL(store, rank, size, std::move(opts)) {} + + std::exception_ptr checkForNCCLErrors( + std::shared_ptr& ncclComm) override { + if (simulateError_) { + return std::make_exception_ptr(std::runtime_error("Error")); + } + return c10d::ProcessGroupNCCL::checkForNCCLErrors(ncclComm); + } + + std::chrono::duration getWatchdogSleepInterval() { + return std::chrono::milliseconds( + ProcessGroupNCCLSimulateErrors::kWatchdogThreadSleepMillis); + } + + c10::intrusive_ptr initWork( + at::Device& device, + int rank, + c10d::OpType opType, + bool isP2P, + const char* profilingTitle, + const std::vector& inputs = {}, + const std::vector& outputs = {}, + bool record = false) override { + return c10::make_intrusive( + device, + simulateError_, + rank, + opType, + isP2P ? seqP2P_ : seqCollective_, + isP2P); + } + + size_t getNCCLCommCacheSize() { + return devNCCLCommMap_.size(); + } + + void simulateError() { + simulateError_ = true; + } + + void resetError() { + simulateError_ = false; + } + + private: + bool simulateError_{false}; +}; + +class WorkNCCLTimedoutErrors : public c10d::ProcessGroupNCCL::WorkNCCL { + public: + WorkNCCLTimedoutErrors( + at::Device& device, + bool set_timedout_error, + int rank, + c10d::OpType opType, + uint64_t seq, + bool isP2P) + : WorkNCCL("0", "default_pg", device, rank, opType, seq, isP2P), + setTimedoutError_(set_timedout_error) {} + + private: + bool isCompleted() override { + if (setTimedoutError_) { + return false; + } + return c10d::ProcessGroupNCCL::WorkNCCL::isCompleted(); + } + + private: + bool setTimedoutError_; +}; + +class ProcessGroupNCCLTimedOutErrors : public ProcessGroupNCCLSimulateErrors { + public: + ProcessGroupNCCLTimedOutErrors( + const c10::intrusive_ptr& store, + int rank, + int size, + c10::intrusive_ptr opts) + : ProcessGroupNCCLSimulateErrors(store, rank, size, std::move(opts)) {} + + c10::intrusive_ptr initWork( + at::Device& device, + int rank, + c10d::OpType opType, + bool isP2P, + const char* profilingTitle, + const std::vector& inputs = {}, + const std::vector& outputs = {}, + bool record = false) override { + return c10::make_intrusive( + device, + setTimedoutError_, + rank, + opType, + isP2P ? seqP2P_ : seqCollective_, + isP2P); + } + + void setTimedoutError() { + setTimedoutError_ = true; + } + + void resetTimedoutError() { + setTimedoutError_ = false; + } + + // In the constructor of ProcessGroupNCCL. We don't allow the watchdog thread + // to run any handling or desync report when the main thread is block wait. + // Even if users set handling and turn on desyncDebug flag, they will get + // reset. For the ease of unit test, we want the main thread to be block wait, + // so we have this hack to manually set the desync debug flag after PG + // creation. + void forceSetDesyncDebugFlag() { + watchdog_->setDesyncDebug(true); + } + + private: + bool setTimedoutError_{false}; +}; + +class ProcessGroupNCCLNoHeartbeatCaught + : public ProcessGroupNCCLTimedOutErrors { + public: + ProcessGroupNCCLNoHeartbeatCaught( + const c10::intrusive_ptr& store, + int rank, + int size, + c10::intrusive_ptr opts) + : ProcessGroupNCCLTimedOutErrors(store, rank, size, std::move(opts)) { + // Override the heartbeat monitor function to make sure that we capture + // the exception in the monitor thread because we cannot try-catch it in + // the main thread and we set a flag for the main thread to check. + heartbeatMonitor_ = std::make_unique(this); + } + + std::mutex& getWatchdogMutex() { + return workMetaListMutex_; + } + + bool getErrorCaughtFlag() { + return hasMonitorThreadCaughtError_; + } + + void forceTryWriteDebugInfo() { + std::future asyncDebugDump = std::async( + std::launch::async, [this]() { return this->dumpDebuggingInfo(); }); + asyncDebugDump.wait(); + } + + class TestHeartbeatMonitor : public c10d::ProcessGroupNCCL::HeartbeatMonitor { + public: + using HeartbeatMonitor::HeartbeatMonitor; + + void runLoop() override { + try { + c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop(); + } catch (const std::runtime_error&) { + // Safe cast because we know it's a ProcessGroupNCCLNoHeartbeatCaught + auto* pg = static_cast(pg_); + pg->hasMonitorThreadCaughtError_ = true; + } + } + }; + + protected: + // It's really hard to unit test std::abort. So we override it instead. + // Commented this override, we do see process aborted with core dump without + // this override. + void terminateProcess(const std::string& errMsg) override { + throw std::runtime_error(errMsg); + } + + bool hasMonitorThreadCaughtError_{false}; +}; + +class ProcessGroupNCCLDebugInfoStuck + : public ProcessGroupNCCLNoHeartbeatCaught { + public: + ProcessGroupNCCLDebugInfoStuck( + const c10::intrusive_ptr& store, + int rank, + int size, + c10::intrusive_ptr opts) + : ProcessGroupNCCLNoHeartbeatCaught(store, rank, size, std::move(opts)) {} +}; + +class ProcessGroupNCCLErrorsTest : public ::testing::Test { + protected: + bool skipTest() { + if (cudaNumDevices() == 0) { + LOG(INFO) << "Skipping test since CUDA is not available"; + return true; + } +#ifdef USE_C10D_NCCL + if (torch::cuda::nccl::version() < kNcclErrorHandlingVersion) { + LOG(INFO) << "Skipping test since NCCL version is too old"; + return true; + } +#endif + return false; + } + + void SetUp() override { + // Enable LOG(INFO) messages. + c10::initLogging(); + // Need to have this check for at SetUp to make sure we only run the test -- + // including the init -- when there are GPUs available. + if (skipTest()) { + GTEST_SKIP() << "Skipping ProcessGroupNCCLErrorsTest because system " + << "requirement is not met (no CUDA or GPU)."; + } + + size_t numDevices = 1; // One device per rank (thread) + TemporaryFile file; + store_ = c10::make_intrusive<::c10d::FileStore>(file.path, 1); + + tensors_.resize(numDevices); + tensors_[0] = at::empty({3, 3}, at::kCUDA); + } + + void TearDown() override { + ASSERT_TRUE(setenv(c10d::TORCH_NCCL_BLOCKING_WAIT[0].c_str(), "0", 1) == 0); + } + + std::vector tensors_; + c10::intrusive_ptr<::c10d::FileStore> store_; +}; + +TEST_F(ProcessGroupNCCLErrorsTest, testNCCLErrorsBlocking) { + ASSERT_TRUE(setenv(c10d::TORCH_NCCL_BLOCKING_WAIT[0].c_str(), "1", 1) == 0); + auto options = c10d::ProcessGroupNCCL::Options::create(); + options->timeout = std::chrono::milliseconds(1000); + ProcessGroupNCCLSimulateErrors pg(store_, 0, 1, options); + + auto work = pg.allreduce(tensors_); + work->wait(); + EXPECT_EQ(1, pg.getNCCLCommCacheSize()); + + // Now run all reduce with errors. + pg.simulateError(); + work = pg.allreduce(tensors_); + // Verify the work item failed. + EXPECT_THROW(work->wait(), std::runtime_error); +} + +TEST_F(ProcessGroupNCCLErrorsTest, testNCCLTimedoutErrorsBlocking) { + ASSERT_TRUE(setenv(c10d::TORCH_NCCL_BLOCKING_WAIT[0].c_str(), "1", 1) == 0); + auto options = c10d::ProcessGroupNCCL::Options::create(); + options->timeout = std::chrono::milliseconds(3000); + ProcessGroupNCCLTimedOutErrors pg(store_, 0, 1, options); + + auto work = pg.allreduce(tensors_); + work->wait(); + EXPECT_EQ(1, pg.getNCCLCommCacheSize()); + + // Now run all reduce with errors. + pg.setTimedoutError(); + work = pg.allreduce(tensors_); + EXPECT_THROW(work->wait(), c10::DistBackendError); + + // Communicators might be aborted here, further operations would fail. +} + +TEST_F(ProcessGroupNCCLErrorsTest, testNCCLErrorsNonBlocking) { + // Avoid watchdog thread to throw the exception and FR dumps to test the + // barrier throw behavior. + ASSERT_TRUE( + setenv(c10d::TORCH_NCCL_ASYNC_ERROR_HANDLING[0].c_str(), "0", 1) == 0); + ASSERT_TRUE(setenv(c10d::TORCH_NCCL_PROPAGATE_ERROR[0].c_str(), "1", 1) == 0); + auto options = c10d::ProcessGroupNCCL::Options::create(); + options->timeout = std::chrono::milliseconds(3000); + ProcessGroupNCCLSimulateErrors pg(store_, 0, 1, options); + + auto work = pg.allreduce(tensors_); + pg.barrier()->wait(); + EXPECT_EQ(1, pg.getNCCLCommCacheSize()); + + // Now run all reduce with errors. + pg.simulateError(); + work = pg.allreduce(tensors_); + + work->wait(); + // a NCCL ERROR happened before should stop the thread from passing the + // barrier. + EXPECT_THROW(pg.barrier()->wait(), std::runtime_error); +} + +// Function to read what we wrote to the local disk for validation. +std::string readTraceFromFile(const std::string& filename, size_t size) { + std::ifstream file(filename, std::ios::binary); + // Read the strings from the file + if (file) { // While the file stream is in good state + std::string str(size, '\0'); + file.read(&str[0], static_cast(size)); + if (file) { + return str; + } + } + return ""; +} + +// Extend the nested class outside the parent class +class TestDebugInfoWriter : public c10d::DebugInfoWriter { + public: + TestDebugInfoWriter(const std::string& namePrefix) + : DebugInfoWriter(namePrefix, 0) {} + + void write(const std::string& ncclTrace) override { + traces_.assign(ncclTrace.begin(), ncclTrace.end()); + c10d::DebugInfoWriter::write(ncclTrace); + } + + std::vector& getTraces() { + return traces_; + } + + private: + std::vector traces_; +}; + +TEST_F(ProcessGroupNCCLErrorsTest, testNCCLErrorsNoHeartbeat) { + // Note (kwen2501) 03/07/2025 + // TODO: re-enable + GTEST_SKIP() << "Skipping test as the trace write seems unstable."; + int heartBeatIntervalInSec = 2; + std::string timeInterval = std::to_string(heartBeatIntervalInSec); + ASSERT_TRUE(setenv(c10d::TORCH_NCCL_BLOCKING_WAIT[0].c_str(), "0", 1) == 0); + ASSERT_TRUE( + setenv( + c10d::TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC[0].c_str(), + timeInterval.c_str(), + 1) == 0); + ASSERT_TRUE( + setenv(c10d::TORCH_NCCL_ENABLE_MONITORING[0].c_str(), "1", 1) == 0); + auto tempFilename = c10::str( + std::filesystem::temp_directory_path().string(), "/comm_lib_trace_rank_"); + ASSERT_TRUE( + setenv("TORCH_NCCL_DEBUG_INFO_TEMP_FILE", tempFilename.c_str(), 1) == 0); + // Enable nccl flight recorder. + ASSERT_TRUE(setenv("TORCH_NCCL_TRACE_BUFFER_SIZE", "10", 1) == 0); + ASSERT_TRUE(setenv(c10d::TORCH_NCCL_DUMP_ON_TIMEOUT[0].c_str(), "1", 1) == 0); + auto options = c10d::ProcessGroupNCCL::Options::create(); + // Set a long watchdog timeout, so that we have enough time to lock the + // watchdog and let the heartbeat monitor thread to kick in. + options->timeout = std::chrono::milliseconds(30000); + ProcessGroupNCCLNoHeartbeatCaught pg(store_, 0, 1, options); + // The storer here is very similar to the fallback storer. + // The only difference is that we are storing traces also in memory for + // validation. + std::string fileNamePrefix = c10d::getCvarString( + {"TORCH_NCCL_DEBUG_INFO_TEMP_FILE"}, "/tmp/comm_lib_trace_rank_"); + std::unique_ptr wrterForTestPtr = + std::make_unique(fileNamePrefix); + std::vector& traces = wrterForTestPtr->getTraces(); + c10d::DebugInfoWriter::registerWriter(std::move(wrterForTestPtr)); + + // Normal collective case. + auto work = pg.allreduce(tensors_); + work->wait(); + + work = pg.allreduce(tensors_); + { + // Now run all reduce with errors. + std::lock_guard lock(pg.getWatchdogMutex()); + LOG(INFO) << "Lock watchdog thread."; + // Wait long enough before monitor thread throws exceptions. + std::this_thread::sleep_for( + std::chrono::seconds(heartBeatIntervalInSec * 3)); + // Check the monitoring thread launched and exception thrown. + EXPECT_TRUE(pg.getErrorCaughtFlag()); + } + work->wait(); + EXPECT_TRUE(!traces.empty()); + auto filename = c10::str(tempFilename, 0); + auto traceFromStorage = readTraceFromFile(filename, traces.size()); + // Check the traces read from storage match with the original nccl trace. + EXPECT_TRUE(traceFromStorage == std::string(traces.begin(), traces.end())); + std::filesystem::remove(filename); +} + +class ProcessGroupNCCLWatchdogTimeoutTest : public ProcessGroupNCCLErrorsTest { + protected: + void SetUp() override { + // TODO (kwen2501) + GTEST_SKIP() << "Skipping tests under ProcessGroupNCCLWatchdogTimeoutTest; " + << "will rewrite them after refactoring Work queues."; + ProcessGroupNCCLErrorsTest::SetUp(); + std::string timeInterval = std::to_string(heartBeatIntervalInSec); + ASSERT_TRUE(setenv(c10d::TORCH_NCCL_BLOCKING_WAIT[0].c_str(), "1", 1) == 0); + ASSERT_TRUE( + setenv( + c10d::TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC[0].c_str(), + timeInterval.c_str(), + 1) == 0); + ASSERT_TRUE( + setenv(c10d::TORCH_NCCL_ENABLE_MONITORING[0].c_str(), "1", 1) == 0); + ASSERT_TRUE(setenv(c10d::TORCH_NCCL_DESYNC_DEBUG[0].c_str(), "1", 1) == 0); + // We cannot capture the exception thrown in watchdog thread without making + // lots of changes to the code. So we don't let the watchdog throw + // exception. + ASSERT_TRUE( + setenv(c10d::TORCH_NCCL_ASYNC_ERROR_HANDLING[0].c_str(), "0", 1) == 0); + options_ = c10d::ProcessGroupNCCL::Options::create(); + // Set a super short watchdog timeout. + options_->timeout = std::chrono::milliseconds(100); + } + + void watchdogTimeoutTestCommon( + ProcessGroupNCCLNoHeartbeatCaught& pg, + int multiplier) { + pg.forceSetDesyncDebugFlag(); + pg.setTimedoutError(); + auto work = pg.allreduce(tensors_); + std::this_thread::sleep_for( + std::chrono::seconds(heartBeatIntervalInSec * multiplier)); + EXPECT_THROW(work->wait(), c10::DistBackendError); + } + + const int heartBeatIntervalInSec = 2; + c10::intrusive_ptr options_; +}; + +TEST_F(ProcessGroupNCCLWatchdogTimeoutTest, testNCCLTimedoutDebugInfoFinished) { + ProcessGroupNCCLNoHeartbeatCaught pg(store_, 0, 1, options_); + // Write debug info will lead to watchdog thread to wait for 30 seconds. + // And this is hard to override, so we just call it before hand. Otherwise, + // we need to set a long heartbeat timeout which will make the test way + // slower. + pg.forceTryWriteDebugInfo(); + watchdogTimeoutTestCommon(pg, 2); + + // The flag is false shows that the heartbeat monitor thread does not + // trigger process abort if getting debug info and destroy PG is fast. + EXPECT_FALSE(pg.getErrorCaughtFlag()); + + // Communicators might be aborted here, further operations would fail. +} + +TEST_F(ProcessGroupNCCLWatchdogTimeoutTest, testNCCLTimedoutDebugInfoStuck) { + ProcessGroupNCCLDebugInfoStuck pg(store_, 0, 1, options_); + // Need to keep main thread sleep longer so that we can let heartbeat monitor + // thread to finish the extra wait and flip the flag. + watchdogTimeoutTestCommon(pg, 4); + // The flag is true shows that the heartbeat monitor thread does trigger + // process abort if getting debug info gets stuck. + EXPECT_TRUE(pg.getErrorCaughtFlag()); + + // Communicators might be aborted here, further operations would fail. +} diff --git a/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupNCCLTest.cpp b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupNCCLTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..aeffc7875e14370bdd977690480601402c353a6a --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupNCCLTest.cpp @@ -0,0 +1,862 @@ +#include +#include + +#include +#include +#include "CUDATest.hpp" +#include "TestUtils.hpp" +#include "c10d/Types.hpp" + +#include +#include +#include + +#include +#include + +using namespace c10d::test; + +using at::cuda::CUDAStream; + +class NCCLTestBase { + public: + NCCLTestBase( + std::string path, + const std::chrono::milliseconds pgTimeout = + c10d::kProcessGroupNCCLDefaultTimeout) + : path_(std::move(path)), pgTimeout_(pgTimeout) {} + + NCCLTestBase(NCCLTestBase&& other) noexcept = default; + + ::c10::intrusive_ptr<::c10d::ProcessGroupNCCL> getProcessGroup() { + return pg_; + } + + ::c10::intrusive_ptr<::c10d::Store>& getProcessGroupStore() { + return store_; + } + + void initialize( + int rank, + size_t size, + std::optional<::c10::intrusive_ptr<::c10d::ProcessGroupNCCL>> split_from = + std::nullopt) { + store_ = c10::make_intrusive<::c10d::FileStore>(path_, size); + + c10::intrusive_ptr opts = + c10::make_intrusive(); + opts->timeout = pgTimeout_; + if (split_from) { + opts->split_from = *split_from; + opts->split_color = ++color_; + } + pg_ = c10::make_intrusive<::c10d::ProcessGroupNCCL>( + store_, rank, size, std::move(opts)); + } + + protected: + std::string path_; + ::c10::intrusive_ptr<::c10d::ProcessGroupNCCL> pg_; + std::chrono::milliseconds pgTimeout_; + ::c10::intrusive_ptr<::c10d::Store> store_; + int color_{1}; +}; + +class NCCLTest : public NCCLTestBase { + public: + NCCLTest( + const std::string& path, + int rank, + int worldSize, + std::chrono::milliseconds pgTimeout = + c10d::kProcessGroupNCCLDefaultTimeout, + int inputDim = 3) + : NCCLTestBase(path, pgTimeout), rank_(rank), worldSize_(worldSize) { + // Each device has a single tensor to perf the NCCL op + ::at::globalContext().lazyInitDevice(c10::DeviceType::CUDA); + tensors_.resize(numDevices_); + inputs_.resize(numDevices_); + outputs_.resize(numDevices_); + at::cuda::OptionalCUDAGuard deviceGuard; + assert(numDevices_ == 1); + for (const auto i : c10::irange(numDevices_)) { + deviceGuard.set_index(static_cast(rank_)); + tensors_[i] = at::empty({inputDim, inputDim}, at::kCUDA); + inputs_[i].resize(static_cast(worldSize_) * numDevices_); + outputs_[i].resize(static_cast(worldSize_) * numDevices_); + for (auto j = 0; j < worldSize_ * numDevices_; ++j) { + inputs_[i][j] = at::empty({inputDim, inputDim}, at::kCUDA); + outputs_[i][j] = at::empty({inputDim, inputDim}, at::kCUDA); + } + } + + // Allocate a stream per device. + // + // The "current stream" is set globally per device in THC, so we + // can't make two tensors on the same device use different streams + // and pass this along to the collective (since it uses the THC + // getters to retrieve the current stream). + // + // 1 device only, hence 1 stream only + deviceGuard.set_index(static_cast(rank_)); + streams_.push_back(at::cuda::getStreamFromPool()); + } + + void wait( + c10::intrusive_ptr& work, + std::chrono::milliseconds timeout = kNoTimeout) { + c10::cuda::CUDAMultiStreamGuard guard(streams_); + work->wait(timeout); + } + + std::vector getTensors() { + std::vector outputs(numDevices_); + + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + + // Copy inputs to outputs + for (const auto i : c10::irange(numDevices_)) { + C10_CUDA_CHECK(cudaStreamSynchronize(streams_[i].stream())); + outputs[i] = tensors_[i].cpu(); + } + + return outputs; + } + + std::vector> getInputTensors() { + return getTensorLists(inputs_); + } + std::vector> getOutputTensors() { + return getTensorLists(outputs_); + } + + int numDevices() const { + return numDevices_; + } + + private: + std::vector> getTensorLists( + std::vector>& tensor_lists) { + std::vector> outputs(numDevices_); + for (auto& output : outputs) { + output = std::vector( + static_cast(worldSize_ * numDevices_)); + } + + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + + // Copy inputs to outputs + for (const auto i : c10::irange(numDevices_)) { + C10_CUDA_CHECK(cudaStreamSynchronize(streams_[i].stream())); + for (auto j = 0; j < worldSize_ * numDevices_; ++j) { + outputs[i][j] = tensor_lists[i][j].cpu(); + } + } + return outputs; + } + + protected: + // Launches sleep on every CUDA device + void launchDeviceSleep() { + at::cuda::OptionalCUDAGuard deviceGuard; + for (const auto i : c10::irange(numDevices_)) { + deviceGuard.set_index(static_cast(rank_)); + cudaSleep(streams_[i], 2000ull * 1000 * 1000); + } + } + + // Launches value initialization for every tensor + void valueInitialization() { + at::cuda::OptionalCUDAGuard deviceGuard; + for (const auto i : c10::irange(numDevices_)) { + deviceGuard.set_index(static_cast(rank_)); + tensors_[i].fill_(pg_->getRank() * numDevices_ + i); + } + } + + at::Tensor to_sparse_row_indices_format(at::Tensor& tensor) { + // Get the indices of all non-zero elements in the dense tensor + // Get the unique row indices of the non-zero elements + auto row_indices = std::get<0>( + at::_unique(tensor.nonzero().select(/*dim=*/1, /*index=*/0))); + at::Tensor sparse_values = tensor.index_select( + /*dim=*/0, row_indices); // get the values at the non-zero indices + return at::sparse_coo_tensor( + row_indices.unsqueeze(0), sparse_values, tensor.sizes()) + .to(tensor.device()); + } + + // Launches value initialization for every sparse tensor + void valueInitializationForSparse() { + at::cuda::OptionalCUDAGuard deviceGuard; + for (const auto i : c10::irange(numDevices_)) { + deviceGuard.set_index(static_cast(rank_)); + tensors_[i].fill_(pg_->getRank() * numDevices_ + i + 1); + // Convert the dense tensor to a sparse tensor in COO row format + tensors_[i] = to_sparse_row_indices_format(tensors_[i]); + } + } + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const int numDevices_{1}; // one device per rank (thread) + int rank_; + int worldSize_; + std::vector tensors_; + std::vector> inputs_; + std::vector> outputs_; + std::vector streams_; +}; + +class AllreduceNCCLTest : public NCCLTest { + public: + AllreduceNCCLTest(const std::string& path, int rank, int worldSize) + : NCCLTest(path, rank, worldSize) {} + + c10::intrusive_ptr run() { + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + + launchDeviceSleep(); + valueInitialization(); + + using namespace torch::autograd::profiler; + // Make sure enabling profile does not make any issue. Note, in single + // process multi-device mode we do not expect any events be populated for + // collective operations, since profiling for that mode is not supported. + enableProfilerLegacy(ProfilerConfig(ProfilerState::CPU)); + auto results = pg_->allreduce(tensors_); + disableProfilerLegacy(); + return results; + } +}; + +class SparseAllreduceNCCLTest : public NCCLTest { + public: + SparseAllreduceNCCLTest( + const std::string& path, + int rank, + int worldSize, + int inputDim) + : NCCLTest( + path, + rank, + worldSize, + c10d::kProcessGroupNCCLDefaultTimeout, + inputDim) {} + + c10::intrusive_ptr run() { + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + launchDeviceSleep(); + valueInitializationForSparse(); + auto results = pg_->allreduce_sparse(tensors_); + return results; + } +}; + +class BroadcastNCCLTest : public NCCLTest { + public: + BroadcastNCCLTest(const std::string& path, int rank, int worldSize) + : NCCLTest(path, rank, worldSize) {} + + c10::intrusive_ptr run(int rootRank, int rootTensor) { + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + + launchDeviceSleep(); + valueInitialization(); + + ::c10d::BroadcastOptions options; + options.rootRank = rootRank; + options.rootTensor = rootTensor; + return pg_->broadcast(tensors_, options); + } +}; + +class ReduceNCCLTest : public NCCLTest { + public: + ReduceNCCLTest(const std::string& path, int rank, int worldSize) + : NCCLTest(path, rank, worldSize) {} + + c10::intrusive_ptr run(int rootRank, int rootTensor) { + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + + launchDeviceSleep(); + valueInitialization(); + + ::c10d::ReduceOptions options; + options.rootRank = rootRank; + options.rootTensor = rootTensor; + return pg_->reduce(tensors_, options); + } +}; + +class AllgatherNCCLTest : public NCCLTest { + public: + AllgatherNCCLTest(const std::string& path, int rank, int worldSize) + : NCCLTest(path, rank, worldSize) {} + + c10::intrusive_ptr run() { + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + + launchDeviceSleep(); + valueInitialization(); + + return pg_->allgather(outputs_, tensors_); + } +}; + +class AllgatherBaseNCCLTest : public NCCLTest { + public: + AllgatherBaseNCCLTest(const std::string& path, int rank, int worldSize) + : NCCLTest(path, rank, worldSize) { + output_tensor_ = at::empty({worldSize_, 3, 3}, at::kCUDA); + } + + c10::intrusive_ptr run() { + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + + launchDeviceSleep(); + valueInitialization(); + // contains at least one element otherwise wouldn't run. + // this is a flattened allgather, hence one rank contributes + // only 1 tensor, regardless of number of devices + return pg_->all_gather_single(output_tensor_, tensors_[0]); + } + + at::Tensor getOutputTensor() { + c10::cuda::CUDAMultiStreamGuard guard(streams_); + return output_tensor_.cpu(); + } + + at::Tensor getInputTensor() { + c10::cuda::CUDAMultiStreamGuard guard(streams_); + return tensors_[0].cpu(); + } + + private: + at::Tensor output_tensor_; +}; + +struct ReduceScatterNCCLTest : NCCLTest { + ReduceScatterNCCLTest(const std::string& path, int rank, int worldSize) + : NCCLTest(path, rank, worldSize) {} + + c10::intrusive_ptr run() { + // For the duration of this function, make THC use our streams + c10::cuda::CUDAMultiStreamGuard guard(streams_); + + at::cuda::OptionalCUDAGuard deviceGuard; + launchDeviceSleep(); + + // Launch value initialization for every tensor + for (auto j = 0; j < worldSize_; ++j) { + inputs_[0][j].fill_(rank_ * worldSize_ + j); + } + + return pg_->reduce_scatter(tensors_, inputs_); + } +}; + +class ReduceScatterBaseNCCLTest : public NCCLTest { + public: + ReduceScatterBaseNCCLTest(const std::string& path, int rank, int worldSize) + : NCCLTest(path, rank, worldSize) { + at::cuda::OptionalCUDAGuard deviceGuard; + deviceGuard.set_index(static_cast(rank_)); + output_tensor_ = at::empty({1}, at::kCUDA); + input_tensor_ = at::empty({worldSize}, at::kCUDA); + for (const auto i : c10::irange(worldSize)) { + input_tensor_[i] = i; + } + } + + c10::intrusive_ptr run() { + // For the duration of this function, make THC use our streams + at::cuda::CUDAMultiStreamGuard guard(streams_); + + launchDeviceSleep(); + return pg_->reduce_scatter_single(output_tensor_, input_tensor_); + } + + at::Tensor getOutputTensor() { + at::cuda::CUDAMultiStreamGuard guard(streams_); + return output_tensor_.cpu(); + } + + at::Tensor getInputTensor() { + at::cuda::CUDAMultiStreamGuard guard(streams_); + return input_tensor_.cpu(); + } + + private: + at::Tensor output_tensor_; + at::Tensor input_tensor_; +}; + +void testAllreduce(const std::string& path, int rank, int size) { + auto test = AllreduceNCCLTest(path, rank, size); + test.initialize(rank, size); + auto work = test.run(); + // Wait for work to finish + test.wait(work); + + // Validation + const int totalNumGPUs = test.numDevices() * size; + const auto expected = (totalNumGPUs * (totalNumGPUs - 1)) / 2; + const auto tensors = test.getTensors(); + for (const auto& tensor : tensors) { + const auto* const data = tensor.const_data_ptr(); + for (const auto k : c10::irange(tensor.numel())) { + EXPECT_EQ(data[k], expected) + << "Allreduce outputs do not match expected outputs"; + } + } +} + +void testSparseAllreduce(const std::string& path, int rank, int size) { + const int inputDim = 3; + auto test = SparseAllreduceNCCLTest(path, rank, size, inputDim); + test.initialize(rank, size); + auto work = test.run(); + // Wait for work to finish + test.wait(work); + + const auto input_tensors = test.getTensors(); + + // validate the work output is same as tensor + auto output_tensor = work->result(); + // Validation + int totalNumGPUs = test.numDevices() * size; + // Add one since we are seeding with an additional 1 to prevent empty tensors + totalNumGPUs++; + const auto expected = (totalNumGPUs * (totalNumGPUs - 1)) / 2; + for (const auto i : c10::irange(input_tensors.size())) { + const auto& tensor = input_tensors[i]; + + // validate the tensor is sparse + EXPECT_EQ(tensor.is_sparse(), true); + + auto indices = tensor._indices(); + auto values = tensor._values(); + + // validate indices are expected size + auto sizes = indices.sizes(); + EXPECT_EQ(sizes.size(), 2); + if (sizes[0] == 1) { + // row indices + EXPECT_EQ(sizes[1], inputDim); + } else if (sizes[0] == 2) { + // coordinate indices + EXPECT_EQ(sizes[1], inputDim * inputDim); + } + + // validate all tensor values are expected value + const auto* const data = values.const_data_ptr(); + for (const auto k : c10::irange(values.numel())) { + EXPECT_EQ(data[k], expected) + << "Allreduce outputs do not match expected outputs"; + } + + // expect the input and output tensors should be the same + auto input_dense = tensor.to_dense(); + auto output_dense = output_tensor[i].to(input_dense.device()).to_dense(); + EXPECT_TRUE(input_dense.allclose(output_dense)); + } +} + +void testSparseAllreduceLarge(const std::string& path, int rank, int size) { + const int inputDim = 2500; + auto test = SparseAllreduceNCCLTest(path, rank, size, inputDim); + test.initialize(rank, size); + auto work = test.run(); + // Wait for work to finish + test.wait(work); + + const auto input_tensors = test.getTensors(); + + // validate the work output is same as tensor + auto output_tensor = work->result(); + // Validation + int totalNumGPUs = test.numDevices() * size; + // Add one since we are seeding with an additional 1 to prevent empty tensors + totalNumGPUs++; + const auto expected = (totalNumGPUs * (totalNumGPUs - 1)) / 2; + for (const auto i : c10::irange(input_tensors.size())) { + const auto& tensor = input_tensors[i]; + + // validate the tensor is sparse + EXPECT_EQ(tensor.is_sparse(), true); + + auto indices = tensor._indices(); + auto values = tensor._values(); + + // validate indices are expected size + auto sizes = indices.sizes(); + EXPECT_EQ(sizes.size(), 2); + if (sizes[0] == 1) { + // row indices + EXPECT_EQ(sizes[1], inputDim); + } else if (sizes[0] == 2) { + // coordinate indices + EXPECT_EQ(sizes[1], inputDim * inputDim); + } + + // validate all tensor values are expected value + const auto* const data = values.const_data_ptr(); + for (const auto k : c10::irange(values.numel())) { + EXPECT_EQ(data[k], expected) + << "Allreduce outputs do not match expected outputs"; + } + + // expect the input and output tensors should be the same + auto input_dense = tensor.to_dense(); + auto output_dense = output_tensor[i].to(input_dense.device()).to_dense(); + EXPECT_TRUE(input_dense.allclose(output_dense)); + } +} + +void testBroadcast(const std::string& path, int rank, int size) { + auto test = BroadcastNCCLTest(path, rank, size); + test.initialize(rank, size); + + const int numDevices = test.numDevices(); + // try every permutation of root rank and root tensor + for (const auto rootRank : c10::irange(size)) { + for (const auto rootTensor : c10::irange(numDevices)) { + auto work = test.run(rootRank, rootTensor); + + // wait for work to complete + test.wait(work); + + // Check results + const auto expected = (rootRank * numDevices + rootTensor); + const auto tensors = test.getTensors(); + for (const auto& tensor : tensors) { + const auto* const data = tensor.const_data_ptr(); + for (const auto k : c10::irange(tensor.numel())) { + EXPECT_EQ(data[k], expected) + << "Broadcast outputs do not match expected outputs"; + } + } + } + } +} + +void testReduce(const std::string& path, int rank, int size) { + auto test = ReduceNCCLTest(path, rank, size); + test.initialize(rank, size); + + const int numDevices = test.numDevices(); + // try every permutation of root rank and root tensor + for (const auto rootRank : c10::irange(size)) { + for (const auto rootTensor : c10::irange(numDevices)) { + auto work = test.run(rootRank, rootTensor); + + // wait for work to complete + test.wait(work); + + // Validation + const int totalNumGPUs = numDevices * size; + const auto expected = (totalNumGPUs * (totalNumGPUs - 1)) / 2; + auto tensors = test.getTensors(); + if (rank == rootRank) { + auto& tensor = tensors[rootTensor]; + auto data = tensor.data_ptr(); + for (const auto k : c10::irange(tensor.numel())) { + EXPECT_EQ(data[k], expected) + << "Reduce outputs do not match expected outputs"; + } + } + } + } +} + +void testAllgather(const std::string& path, int rank, int size) { + auto test = AllgatherNCCLTest(path, rank, size); + test.initialize(rank, size); + auto work = test.run(); + // Wait for work to finish + test.wait(work); + + // Validation + auto tensors = test.getOutputTensors(); + // device index + for (auto& device : tensors) { + // rank index + for (const auto j : c10::irange(device.size())) { + const auto expected = j; + auto& tensor = device[j]; + auto data = tensor.data_ptr(); + for (const auto k : c10::irange(tensor.numel())) { + EXPECT_EQ(data[k], expected) + << "Allgather outputs do not match expected outputs"; + } + } + } +} + +void testAllgatherBase(const std::string& path, int rank, int size) { + auto test = AllgatherBaseNCCLTest(path, rank, size); + test.initialize(rank, size); + auto work = test.run(); + // Wait for work to finish + test.wait(work); + // Validation + auto output_tensor = test.getOutputTensor(); + auto input_tensor = test.getInputTensor(); + + auto data = output_tensor.data_ptr(); + + // Rank index + for (const auto i : c10::irange(output_tensor.numel())) { + // expected is i // input.numel() <- rank, and each rank contributed rank * + // num_gpu + const auto expected = (i / input_tensor.numel()) * test.numDevices(); + EXPECT_EQ(data[i], expected) + << "Allgather_base outputs do not match expected outputs"; + } +} +void testReduceScatterBase(const std::string& path, int rank, int size) { + auto test = ReduceScatterBaseNCCLTest(path, rank, size); + test.initialize(rank, size); + auto work = test.run(); + // Wait for work to finish + test.wait(work); + // Validation + auto output_tensor = test.getOutputTensor(); + auto input_tensor = test.getInputTensor(); + + auto data = output_tensor.data_ptr(); + + // Rank index + for (const auto i : c10::irange(output_tensor.numel())) { + // expected is i * input.numel() <- rank, and each rank contributed rank * + // num_gpu + const auto expected = size * rank * test.numDevices(); + EXPECT_EQ(data[i], expected) + << "Reducescatter_base outputs do not match expected outputs"; + } +} + +void testReduceScatter(const std::string& path, int rank, int size) { + auto test = ReduceScatterNCCLTest(path, rank, size); + test.initialize(rank, size); + auto work = test.run(); + // Wait for work to finish + test.wait(work); + + const auto participants = size; + const auto base = (participants * (participants - 1)) / 2; + + // Validation + auto tensors = test.getTensors(); + const auto modifier = rank * participants; + const auto expected = base * participants + modifier; + auto& tensor = tensors[0]; + auto data = tensor.data_ptr(); + for (const auto j : c10::irange(tensor.numel())) { + EXPECT_EQ(data[j], expected) + << "ReduceScatter outputs do not match expected outputs!"; + } +} + +void testSplittingCommunicator(const std::string& path, int rank, int size) { + auto test1 = BroadcastNCCLTest(path, rank, size); + test1.initialize(rank, size); + + auto test2 = BroadcastNCCLTest(path, rank, size); + test2.initialize(rank, size, test1.getProcessGroup()); + + // Steal the broadcast test and issue it for both of our groups. + // This ensures consistent full collective communication. TODO: + // maybe refactor the guts rather than copy-pasta, but it may not be + // worth it. + for (auto test : {&test1, &test2}) { + const int numDevices = test->numDevices(); + // try every permutation of root rank and root tensor + for (const auto rootRank : c10::irange(size)) { + for (const auto rootTensor : c10::irange(numDevices)) { + auto work = test->run(rootRank, rootTensor); + test->wait(work); + + // Check results + const auto expected = (rootRank * numDevices + rootTensor); + const auto tensors = test->getTensors(); + for (const auto& tensor : tensors) { + const auto* const data = tensor.const_data_ptr(); + for (const auto k : c10::irange(tensor.numel())) { + EXPECT_EQ(data[k], expected) + << "Broadcast outputs do not match expected outputs"; + } + } + } + } + } + + // Now that we've run full operations on both the original and split process + // group, ensure we saw exactly as many splits as we expected: 0 in the + // original process group, and one per device in the second. + EXPECT_EQ(test2.getProcessGroup()->getCommSplitCounter(), 0); + EXPECT_EQ(test1.getProcessGroup()->getCommSplitCounter(), test1.numDevices()); +} + +// All testAbc's use this signature +using FuncType = void (*)(const std::string&, int, int); + +class ProcessGroupNCCLTest : public ::testing::Test { + protected: + void SetUp() override { + c10::initLogging(); + // Use WORLD_SIZE and RANK environmental variables to do multi-node + // distributed testing + auto sizeEnv = std::getenv("WORLD_SIZE"); + if (sizeEnv) { + size_ = std::stoi(std::string(sizeEnv)); + } + LOG(INFO) << "ProcessGroupNCCLTest world size: " << size_; + } + + void TearDown() override { + // Reset TORCH_NCCL_BLOCKING_WAIT environment variable after each run. + ASSERT_TRUE(setenv(c10d::TORCH_NCCL_BLOCKING_WAIT[0].c_str(), "0", 1) == 0); + } + + bool skipTest() { + // Skip tests if CUDA is not available. + if (!at::cuda::is_available()) { + LOG(INFO) << "CUDA not available, skipping test"; + return true; + } + return false; + } + + void multiThreadRun(FuncType testFunc) { + TemporaryFile file; + std::vector threads; + threads.reserve(size_); + for (const auto rank : c10::irange(size_)) { + threads.emplace_back(testFunc, file.path, rank, size_); + } + for (const auto rank : c10::irange(size_)) { + threads[rank].join(); + } + } + + int size_{1}; +}; + +TEST_F(ProcessGroupNCCLTest, CUDAEventCache) { + if (skipTest()) { + return; + } + + // Test that the CUDAEventCache can be used to create CUDA events and reuse. + auto event1 = c10d::CUDAEventCache::get(1)->create(true); + auto event2 = c10d::CUDAEventCache::get(1)->create(false); + + auto event1_ptr = event1.get(); + auto event2_ptr = event2.get(); + // Mimic the behavior of the destroy of events. + event1 = nullptr; + event2 = nullptr; + + // Test that the CUDAEventCache is indeed reused. + auto event3 = c10d::CUDAEventCache::get(2)->create(true); + auto event4 = c10d::CUDAEventCache::get(2)->create(false); + // The cache has been used up, new events should be created. + auto event5 = c10d::CUDAEventCache::get(1)->create(true); + auto event6 = c10d::CUDAEventCache::get(1)->create(false); + // The cache has been used up, new events should be created. + auto event7 = c10d::CUDAEventCache::get(1)->create(true); + auto event8 = c10d::CUDAEventCache::get(1)->create(false); + EXPECT_NE(event1_ptr, event3.get()); + EXPECT_NE(event2_ptr, event4.get()); + EXPECT_EQ(event1_ptr, event5.get()); + EXPECT_EQ(event2_ptr, event6.get()); + EXPECT_NE(event1_ptr, event7.get()); + EXPECT_NE(event2_ptr, event8.get()); +} + +TEST_F(ProcessGroupNCCLTest, testAllreduce) { + if (skipTest()) { + return; + } + multiThreadRun(testAllreduce); +} + +TEST_F(ProcessGroupNCCLTest, testBroadcast) { + if (skipTest()) { + return; + } + multiThreadRun(testBroadcast); +} + +TEST_F(ProcessGroupNCCLTest, testReduce) { + if (skipTest()) { + return; + } + multiThreadRun(testReduce); +} + +TEST_F(ProcessGroupNCCLTest, testAllgather) { + if (skipTest()) { + return; + } + multiThreadRun(testAllgather); +} + +TEST_F(ProcessGroupNCCLTest, testAllgatherBase) { + if (skipTest()) { + return; + } + multiThreadRun(testAllgatherBase); +} + +TEST_F(ProcessGroupNCCLTest, testReduceScatter) { + if (skipTest()) { + return; + } + multiThreadRun(testReduceScatter); +} + +TEST_F(ProcessGroupNCCLTest, testReduceScatterBase) { + if (skipTest()) { + return; + } + multiThreadRun(testReduceScatterBase); +} + +TEST_F(ProcessGroupNCCLTest, testBackendName) { + if (skipTest()) { + return; + } + TemporaryFile file; + auto test = NCCLTestBase(file.path); + test.initialize(/*rank=*/0, /*size=*/1); + EXPECT_EQ( + test.getProcessGroup()->getBackendName(), + std::string(c10d::NCCL_BACKEND_NAME)); +} + +TEST_F(ProcessGroupNCCLTest, testSplittingCommunicator) { + if (skipTest()) { + return; + } + multiThreadRun(testSplittingCommunicator); +} + +#ifdef IS_NCCLX +TEST_F(ProcessGroupNCCLTest, testSparseAllreduce) { + if (skipTest()) { + return; + } + multiThreadRun(testSparseAllreduce); + multiThreadRun(testSparseAllreduceLarge); +} +#endif diff --git a/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupUCCTest.cpp b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupUCCTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e0c74a1b97a0900a8c94bc45ad80dd8c5349f264 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/ProcessGroupUCCTest.cpp @@ -0,0 +1,34 @@ +#ifdef USE_C10D_UCC +#include +#include + +#include +#include +TEST(ProcessGroupUCCTest, testTrim) { + std::vector> tests = { + {" allreduce ", "allreduce"}, + {"\tallgather", "allgather"}, + {"send\n", "send"}, + }; + for (auto entry : tests) { + ASSERT_EQ(c10d::trim(entry.first), entry.second); + } +} + +TEST(ProcessGroupUCCTest, testToLower) { + std::vector> tests = { + {"AllReduce", "allreduce"}, + {"ALLGATHER", "allgather"}, + {"send", "send"}, + }; + for (auto entry : tests) { + ASSERT_EQ(c10d::tolower(entry.first), entry.second); + } +} + +TEST(ProcessGroupUCCTest, testParseList) { + std::string input = "\tAllReduce, ALLGATHER, send\n"; + std::vector expect{"allreduce", "allgather", "send"}; + ASSERT_EQ(c10d::parse_list(input), expect); +} +#endif diff --git a/western_ai/usa/pytorch/test/cpp/c10d/StoreTestCommon.hpp b/western_ai/usa/pytorch/test/cpp/c10d/StoreTestCommon.hpp new file mode 100644 index 0000000000000000000000000000000000000000..72458de0942b3a013d4d30976ea2970dbc8ccb8f --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/StoreTestCommon.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include "TestUtils.hpp" + +#include + +namespace c10d { +namespace test { + +inline void set( + Store& store, + const std::string& key, + const std::string& value) { + std::vector data(value.begin(), value.end()); + store.set(key, data); +} + +inline std::vector compareSet( + Store& store, + const std::string& key, + const std::string& expectedValue, + const std::string& desiredValue) { + std::vector expectedData(expectedValue.begin(), expectedValue.end()); + std::vector desiredData(desiredValue.begin(), desiredValue.end()); + return store.compareSet(key, expectedData, desiredData); +} + +inline void check( + Store& store, + const std::string& key, + const std::string& expected) { + auto tmp = store.get(key); + auto actual = std::string((const char*)tmp.data(), tmp.size()); + EXPECT_EQ(actual, expected); +} + +inline void deleteKey( + Store& store, + const std::string& key) { + store.deleteKey(key); +} + +} // namespace test +} // namespace c10d diff --git a/western_ai/usa/pytorch/test/cpp/c10d/TCPStoreTest.cpp b/western_ai/usa/pytorch/test/cpp/c10d/TCPStoreTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b4e3570cb6e20b758edeabfc87464ef4c788fecf --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/TCPStoreTest.cpp @@ -0,0 +1,367 @@ +#include +#include "StoreTestCommon.hpp" + +#include +#include +#include + +#include + +#include +#include + +constexpr int64_t kShortStoreTimeoutMillis = 100; +constexpr int defaultTimeout = 20; + +c10::intrusive_ptr _createServer( + bool useLibUV, + int numWorkers = 1, + int timeout = defaultTimeout) { + return c10::make_intrusive( + "127.0.0.1", + c10d::TCPStoreOptions{ + /* port */ 0, + /* isServer */ true, + numWorkers, + /* waitWorkers */ false, + /* timeout */ std::chrono::seconds(timeout), + /* multiTenant */ false, + /* masterListenFd */ std::nullopt, + /* useLibUV*/ useLibUV}); +} + +// Different ports for different tests. +void testHelper(bool useLibUV, const std::string& prefix = "") { + constexpr auto numThreads = 16; + constexpr auto numWorkers = numThreads + 1; + + auto serverTCPStore = _createServer(useLibUV, numWorkers); + + auto serverStore = + c10::make_intrusive(prefix, serverTCPStore); + // server store + auto serverThread = std::thread([&serverStore, &serverTCPStore] { + // Wait for all workers to join. + serverTCPStore->waitForWorkers(); + + // Basic set/get on the server store + c10d::test::set(*serverStore, "key0", "value0"); + c10d::test::set(*serverStore, "key1", "value1"); + c10d::test::set(*serverStore, "key2", "value2"); + c10d::test::check(*serverStore, "key0", "value0"); + c10d::test::check(*serverStore, "key1", "value1"); + c10d::test::check(*serverStore, "key2", "value2"); + serverStore->add("counter", 1); + auto numKeys = serverStore->getNumKeys(); + // We expect 5 keys since 3 are added above, 'counter' is added by the + // helper thread, and the init key to coordinate workers. + EXPECT_EQ(numKeys, 5); + + // Check compareSet, does not check return value + c10d::test::compareSet( + *serverStore, "key0", "wrongExpectedValue", "newValue"); + c10d::test::check(*serverStore, "key0", "value0"); + c10d::test::compareSet(*serverStore, "key0", "value0", "newValue"); + c10d::test::check(*serverStore, "key0", "newValue"); + + auto delSuccess = serverStore->deleteKey("key0"); + // Ensure that the key was successfully deleted + EXPECT_TRUE(delSuccess); + auto delFailure = serverStore->deleteKey("badKeyName"); + // The key was not in the store so the delete operation should have failed + // and returned false. + EXPECT_FALSE(delFailure); + numKeys = serverStore->getNumKeys(); + EXPECT_EQ(numKeys, 4); + auto timeout = std::chrono::milliseconds(kShortStoreTimeoutMillis); + serverStore->setTimeout(timeout); + EXPECT_THROW(serverStore->get("key0"), c10::Error); + }); + + // Hammer on TCPStore + std::vector threads; + constexpr auto numIterations = 1000; + c10d::test::Semaphore sem1, sem2; + + c10d::TCPStoreOptions opts{}; + opts.port = serverTCPStore->getPort(); + opts.numWorkers = numWorkers; + + // Each thread will have a client store to send/recv data + std::vector> clientTCPStores; + std::vector> clientStores; + for (const auto i : c10::irange(numThreads)) { + clientTCPStores.push_back( + c10::make_intrusive("127.0.0.1", opts)); + clientStores.push_back( + c10::make_intrusive(prefix, clientTCPStores[i])); + } + + std::string expectedCounterRes = + std::to_string(numThreads * numIterations + 1); + + for (const auto i : c10::irange(numThreads)) { + threads.emplace_back([=, &sem1, &sem2, &clientStores, &expectedCounterRes] { + for ([[maybe_unused]] const auto j : c10::irange(numIterations)) { + clientStores[i]->add("counter", 1); + } + // Let each thread set and get key on its client store + std::string key = "thread_" + std::to_string(i); + for (const auto j : c10::irange(numIterations)) { + std::string val = "thread_val_" + std::to_string(j); + c10d::test::set(*clientStores[i], key, val); + c10d::test::check(*clientStores[i], key, val); + } + + sem1.post(); + sem2.wait(); + // Check the counter results + c10d::test::check(*clientStores[i], "counter", expectedCounterRes); + // Now check other threads' written data + for (const auto j : c10::irange(numThreads)) { + if (j == i) { + continue; + } + std::string key = "thread_" + std::to_string(i); + std::string val = "thread_val_" + std::to_string(numIterations - 1); + c10d::test::check(*clientStores[i], key, val); + } + }); + } + + sem1.wait(numThreads); + sem2.post(numThreads); + + for (auto& thread : threads) { + thread.join(); + } + + serverThread.join(); + + // Clear the store to test that client disconnect won't shutdown the store + clientStores.clear(); + clientTCPStores.clear(); + + // Check that the counter has the expected value + c10d::test::check(*serverStore, "counter", expectedCounterRes); + + // Check that each threads' written data from the main thread + for (const auto i : c10::irange(numThreads)) { + std::string key = "thread_" + std::to_string(i); + std::string val = "thread_val_" + std::to_string(numIterations - 1); + c10d::test::check(*serverStore, key, val); + } +} + +TEST(TCPStoreTest, testHelper) { + testHelper(false); +} + +TEST(TCPStoreTest, testHelperUV) { + testHelper(true); +} + +TEST(TCPStoreTest, testHelperPrefix) { + testHelper(false, "testPrefixNoUV"); +} + +TEST(TCPStoreTest, testHelperPrefixUV) { + testHelper(true, "testPrefixUV"); +} + +TEST(TCPStoreTest, testCleanShutdown) { + int numWorkers = 2; + + auto serverTCPStore = std::make_unique( + "127.0.0.1", + c10d::TCPStoreOptions{ + /* port */ 0, + /* isServer */ true, + numWorkers, + /* waitWorkers */ false, + /* timeout */ std::chrono::seconds(defaultTimeout)}); + c10d::test::set(*serverTCPStore, "key", "val"); + + auto clientTCPStore = c10::make_intrusive( + "127.0.0.1", + c10d::TCPStoreOptions{ + /* port */ serverTCPStore->getPort(), + /* isServer */ false, + numWorkers, + /* waitWorkers */ false, + /* timeout */ std::chrono::seconds(defaultTimeout)}); + clientTCPStore->get("key"); + + auto clientThread = std::thread([&clientTCPStore] { + EXPECT_THROW(clientTCPStore->get("invalid_key"), c10::DistNetworkError); + }); + + // start server shutdown during a client request + serverTCPStore = nullptr; + + clientThread.join(); +} + +TEST(TCPStoreTest, testLibUVPartialRead) { + int numWorkers = 2; // thread 0 creates both server and client + + // server part + c10d::TCPStoreOptions server_opts{ + 0, + true, // is master + numWorkers, + false, // don't wait otherwise client thread won't spawn + std::chrono::seconds(defaultTimeout)}; + server_opts.useLibUV = true; + + auto serverTCPStore = + std::make_unique("127.0.0.1", server_opts); + + // client part + c10d::TCPStoreOptions client_opts{ + serverTCPStore->getPort(), + false, // is master + numWorkers, + false, // wait workers + std::chrono::seconds(defaultTimeout)}; + client_opts.useLibUV = true; + auto clientTCPStore = + c10::make_intrusive("127.0.0.1", client_opts); + auto clientThread = std::thread([&clientTCPStore] { + std::string keyPrefix( + "/default_pg/0//b7dc24de75e482ba2ceb9f9ee20732c25c0166d8//cuda//"); + std::string value("v"); + std::vector valueBuf(value.begin(), value.end()); + + // split store->set(key, valueBuf) into two requests + for (int i = 0; i < 10; ++i) { + std::string key = keyPrefix + std::to_string(i); + clientTCPStore->_splitSet(key, valueBuf); + + // check the result on server + c10d::test::check(*clientTCPStore, key, "v"); + } + }); + + clientThread.join(); +} + +TEST(TCPStoreTest, testLibUVSetAndWait) { + int numWorkers = 128; // thread 0 creates both server and client + std::vector threads; + + // server part + c10d::TCPStoreOptions server_opts{ + 0, + true, // is master + numWorkers, + false, // don't wait otherwise client thread won't spawn + std::chrono::seconds(defaultTimeout)}; + server_opts.useLibUV = true; + + auto serverTCPStore = + std::make_unique("127.0.0.1", server_opts); + + // client part + c10d::TCPStoreOptions client_opts{ + serverTCPStore->getPort(), + false, // is master + numWorkers, + false, // wait workers + std::chrono::seconds(defaultTimeout)}; + client_opts.useLibUV = true; + + for (const auto i : c10::irange(numWorkers)) { + threads.emplace_back([=, &client_opts] { + auto clientTCPStore = + c10::make_intrusive("127.0.0.1", client_opts); + std::string key("k_" + std::to_string(i)); + std::string value("v_" + std::to_string(i)); + std::vector valueBuf(value.begin(), value.end()); + clientTCPStore->set(key, valueBuf); + std::vector all_keys; + for (const auto j : c10::irange(numWorkers)) { + all_keys.push_back("k_" + std::to_string(j)); + } + clientTCPStore->wait(all_keys); + }); + } + + for (auto& thread : threads) { + thread.join(); + } +} + +void testMultiTenantStores(bool libUV) { + c10d::TCPStoreOptions opts{}; + opts.isServer = true; + opts.multiTenant = true; + opts.useLibUV = libUV; + + // Construct two server stores on the same port. + auto store1 = c10::make_intrusive("localhost", opts); + auto store2 = c10::make_intrusive("localhost", opts); + + // Assert that the two stores share the same server. + c10d::test::set(*store1, "key0", "value0"); + c10d::test::check(*store2, "key0", "value0"); + + // Dispose the second instance and assert that the server is still alive. + store2.reset(); + + c10d::test::set(*store1, "key0", "value0"); + c10d::test::check(*store1, "key0", "value0"); +} + +TEST(TCPStoreTest, testMultiTenantStores) { + testMultiTenantStores(false); +} + +TEST(TCPStoreTest, testMultiTenantStoresUV) { + testMultiTenantStores(true); +} + +void testBarrier(bool useLibUV) { + constexpr int numWorkers = 4; + auto serverTCPStore = _createServer(useLibUV, numWorkers); + + std::vector threads; + c10d::TCPStoreOptions client_opts{}; + client_opts.port = serverTCPStore->getPort(); + client_opts.numWorkers = numWorkers; + client_opts.useLibUV = useLibUV; + + for (int i = 0; i < numWorkers; ++i) { + threads.emplace_back([=, &client_opts] { + auto store = + c10::make_intrusive("127.0.0.1", client_opts); + store->barrier( + "test_barrier", numWorkers, std::chrono::seconds(defaultTimeout)); + }); + } + for (auto& t : threads) { + t.join(); + } +} + +TEST(TCPStoreTest, testBarrier) { + testBarrier(false); +} + +TEST(TCPStoreTest, testBarrierUV) { + testBarrier(true); +} + +void testBarrierTimeout(bool useLibUV) { + auto store = _createServer(useLibUV, 1); + auto timeout = std::chrono::milliseconds(100); + EXPECT_THROW(store->barrier("timeout_key", 2, timeout), c10::DistStoreError); +} + +TEST(TCPStoreTest, testBarrierTimeout) { + testBarrierTimeout(false); +} + +TEST(TCPStoreTest, testBarrierTimeoutUV) { + testBarrierTimeout(true); +} diff --git a/western_ai/usa/pytorch/test/cpp/c10d/TestUtils.hpp b/western_ai/usa/pytorch/test/cpp/c10d/TestUtils.hpp new file mode 100644 index 0000000000000000000000000000000000000000..13c31709217572a5fca213717275b8f96ef3e81c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/TestUtils.hpp @@ -0,0 +1,134 @@ +#pragma once + +#ifndef _WIN32 +#include +#include +#include +#endif + +#include +#include + +#include +#include +#include +#include +#include + +namespace c10d { +namespace test { + +class Semaphore { + public: + void post(int n = 1) { + std::unique_lock lock(m_); + n_ += n; + cv_.notify_all(); + } + + void wait(int n = 1) { + std::unique_lock lock(m_); + while (n_ < n) { + cv_.wait(lock); + } + n_ -= n; + } + + protected: + int n_ = 0; + std::mutex m_; + std::condition_variable cv_; +}; + +#ifdef _WIN32 +std::string autoGenerateTmpFilePath() { + char tmp[L_tmpnam_s]; + errno_t err; + err = tmpnam_s(tmp, L_tmpnam_s); + if (err != 0) + { + throw std::system_error(errno, std::system_category()); + } + return std::string(tmp); +} + +std::string tmppath() { + const char* tmpfile = getenv("TMPFILE"); + if (tmpfile) { + return std::string(tmpfile); + } + else { + return autoGenerateTmpFilePath(); + } +} +#else +std::string tmppath() { + // TMPFILE is for manual test execution during which the user will specify + // the full temp file path using the environmental variable TMPFILE + const char* tmpfile = getenv("TMPFILE"); + if (tmpfile) { + return std::string(tmpfile); + } + + const char* tmpdir = getenv("TMPDIR"); + if (tmpdir == nullptr) { + tmpdir = "/tmp"; + } + + // Create template + std::vector tmp(256); + auto len = snprintf(tmp.data(), tmp.size(), "%s/testXXXXXX", tmpdir); + tmp.resize(len); + + // Create temporary file + auto fd = mkstemp(&tmp[0]); + if (fd == -1) { + throw std::system_error(errno, std::system_category()); + } + close(fd); + return std::string(tmp.data(), tmp.size()); +} +#endif + +bool isTSANEnabled() { + auto s = std::getenv("PYTORCH_TEST_WITH_TSAN"); + return s && strcmp(s, "1") == 0; +} +struct TemporaryFile { + std::string path; + + TemporaryFile() { + path = tmppath(); + } + + ~TemporaryFile() { + unlink(path.c_str()); + } +}; + +#ifndef _WIN32 +struct Fork { + pid_t pid; + + Fork() { + pid = fork(); + if (pid < 0) { + throw std::system_error(errno, std::system_category(), "fork"); + } + } + + ~Fork() { + if (pid > 0) { + kill(pid, SIGKILL); + waitpid(pid, nullptr, 0); + } + } + + bool isChild() { + return pid == 0; + } +}; +#endif + +} // namespace test +} // namespace c10d diff --git a/western_ai/usa/pytorch/test/cpp/c10d/example/allreduce.cpp b/western_ai/usa/pytorch/test/cpp/c10d/example/allreduce.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a74f43a50232ad6741bced2fe797eb1bfd8dabcf --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/c10d/example/allreduce.cpp @@ -0,0 +1,33 @@ +#include +#include +#include + +using namespace ::c10d; + +int main(int argc, char** argv) { + int rank = atoi(getenv("RANK")); + int size = atoi(getenv("SIZE")); + auto store = c10::make_intrusive("/tmp/c10d_example", size); + ProcessGroupGloo pg(store, rank, size); + + // Create some tensors + const auto ntensors = 10; + std::vector tensors; + for (const auto i : c10::irange(ntensors)) { + auto x = + at::ones({1000, 16 * (i + 1)}, at::TensorOptions(at::CPU(at::kFloat))); + tensors.push_back(x); + } + + // Kick off work + std::vector> pending; + for (const auto i : c10::irange(ntensors)) { + std::vector tmp = {tensors[i]}; + pending.push_back(pg.allreduce(tmp)); + } + + // Wait for work to complete + for (auto& work : pending) { + work->wait(); + } +} diff --git a/western_ai/usa/pytorch/test/cpp/common/main.cpp b/western_ai/usa/pytorch/test/cpp/common/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d79910ca5ceeebc64f261a455c5d297828232f07 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/common/main.cpp @@ -0,0 +1,33 @@ +#include + +#include + +#include +#include + +std::string add_negative_flag(const std::string& flag) { + std::string filter = ::testing::GTEST_FLAG(filter); + if (filter.find('-') == std::string::npos) { + filter.push_back('-'); + } else { + filter.push_back(':'); + } + filter += flag; + return filter; +} + +int main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + + if (!torch::cuda::is_available()) { + std::cout << "CUDA not available. Disabling CUDA and MultiCUDA tests" + << std::endl; + ::testing::GTEST_FLAG(filter) = add_negative_flag("*_CUDA:*_MultiCUDA"); + } else if (torch::cuda::device_count() < 2) { + std::cout << "Only one CUDA device detected. Disabling MultiCUDA tests" + << std::endl; + ::testing::GTEST_FLAG(filter) = add_negative_flag("*_MultiCUDA"); + } + + return RUN_ALL_TESTS(); +} diff --git a/western_ai/usa/pytorch/test/cpp/common/support.h b/western_ai/usa/pytorch/test/cpp/common/support.h new file mode 100644 index 0000000000000000000000000000000000000000..b9aea2ceb536c9622f5ae6990c1f9779101738c7 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/common/support.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace torch { +namespace test { +#define ASSERT_THROWS_WITH(statement, substring) \ + { \ + std::string assert_throws_with_error_message; \ + try { \ + (void)statement; \ + FAIL() << "Expected statement `" #statement \ + "` to throw an exception, but it did not"; \ + } catch (const c10::Error& e) { \ + assert_throws_with_error_message = e.what_without_backtrace(); \ + } catch (const std::exception& e) { \ + assert_throws_with_error_message = e.what(); \ + } \ + if (assert_throws_with_error_message.find(substring) == \ + std::string::npos) { \ + FAIL() << "Error message \"" << assert_throws_with_error_message \ + << "\" did not contain expected substring \"" << substring \ + << '"'; \ + } \ + } + +} // namespace test +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/dist_autograd/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/dist_autograd/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a2c976cdff8167302b8133c6a911884ebc9b6233 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/dist_autograd/CMakeLists.txt @@ -0,0 +1,24 @@ +if(USE_DISTRIBUTED AND NOT WIN32) + set(DIST_AUTOGRAD_TEST_DIR "${TORCH_ROOT}/test/cpp/dist_autograd") + set(DIST_AUTOGRAD_TEST_SOURCES + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${DIST_AUTOGRAD_TEST_DIR}/test_dist_autograd.cpp + ) + + add_executable(test_dist_autograd ${DIST_AUTOGRAD_TEST_SOURCES}) + target_include_directories(test_dist_autograd PRIVATE ${ATen_CPU_INCLUDE}) + target_link_libraries(test_dist_autograd PRIVATE torch gtest_main) + + if(USE_CUDA) + target_compile_definitions(test_dist_autograd PRIVATE "USE_CUDA") + endif() + + if(INSTALL_TEST) + set_target_properties(test_dist_autograd PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS test_dist_autograd DESTINATION bin) + # Install PDB files for MSVC builds + if(MSVC AND BUILD_SHARED_LIBS) + install(FILES $ DESTINATION bin OPTIONAL) + endif() + endif() +endif() diff --git a/western_ai/usa/pytorch/test/cpp/dist_autograd/test_dist_autograd.cpp b/western_ai/usa/pytorch/test/cpp/dist_autograd/test_dist_autograd.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b229d119e4c1919c381cde5e3c32f41270fa0079 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/dist_autograd/test_dist_autograd.cpp @@ -0,0 +1,108 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace distributed { +namespace autograd { + +class DistAutogradTest : public ::testing::Test { + protected: + static void SetUpTestCase() { + autogradContainer_ = &DistAutogradContainer::init(0); + } + + void TearDown() override { + autogradContainer_->releaseContext( + autogradContainer_->currentContext()->contextId()); + } + + static DistAutogradContainer* autogradContainer_; +}; + +DistAutogradContainer* DistAutogradTest::autogradContainer_ = nullptr; + +TEST_F(DistAutogradTest, TestSendFunctionInvalidInputs) { + auto options = at::TensorOptions().requires_grad(true); + auto in1 = torch::ones({3, 3}, options); + auto in2 = torch::ones({3, 3}, options); + + autogradContainer_->newContext(); + auto autogradContext = autogradContainer_->currentContext(); + // Attach the send autograd function to tensors. + std::vector tensors = {in1, in2}; + rpc::worker_id_t worker_id = 1; + addSendRpcBackward(autogradContext, AutogradMetadata(1, 1), tensors); + autogradContext->addKnownWorkerId(worker_id); + auto send_function = autogradContext->sendFunctions()[1]; + + // ensure that the worker_ids are recorded + auto knownWorkerIds = autogradContext->getKnownWorkerIds(); + ASSERT_TRUE(knownWorkerIds.find(worker_id) != knownWorkerIds.end()); + ASSERT_EQ(knownWorkerIds.size(), 1); + + // This should fail since the SendRpcBackward function shouldn't receive any + // inputs grad. + EXPECT_THROW(send_function->apply({in1, in2}), c10::Error); + + // This should fail since the SendRpcBackward function encounters an undefined + // grad. + send_function->setGrads({in1, torch::autograd::Variable()}); + EXPECT_THROW(send_function->apply({}), c10::Error); +} + +TEST_F(DistAutogradTest, TestInitializedContextCleanup) { + autogradContainer_->newContext(); + auto contextId = autogradContainer_->currentContext()->contextId(); + auto& engine = DistEngine::getInstance(); + ASSERT_EQ(0, engine.numBackwardPasses()); + + // Build autograd graph + auto x = torch::randn({2, 2}, torch::requires_grad()); + auto y = torch::randn({2, 2}, torch::requires_grad()); + auto z = (x * x + y * y).sum(); + ASSERT_NE(nullptr, z.grad_fn()); + + // Execute engine. + engine.execute(contextId, {z}, /* retainGraph */ false); + + // Validate appropriate cleanup. + ASSERT_EQ(0, engine.numBackwardPasses()); +} + +TEST_F(DistAutogradTest, TestInitializedContextCleanupSendFunction) { + autogradContainer_->newContext(); + auto context = autogradContainer_->currentContext(); + auto& engine = DistEngine::getInstance(); + ASSERT_EQ(0, engine.numBackwardPasses()); + + // Attach send function. + auto options = at::TensorOptions().requires_grad(true); + auto t = torch::ones({1}, options); + auto tensors = std::vector{t}; + addSendRpcBackward( + context, AutogradMetadata(context->contextId(), 0), tensors); + + auto sendFunction = context->retrieveSendFunction(0); + sendFunction->setGrads({t}); + + // Execute engine. + engine + .executeSendFunctionAsync(context, sendFunction, /*retrainGraph*/ false) + ->wait(); + + // Validate appropriate cleanup. + ASSERT_EQ(0, engine.numBackwardPasses()); +} + +} // namespace autograd +} // namespace distributed +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/jit/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..1d57895ac752754598fcf9fa3e78eb2c03126b63 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/CMakeLists.txt @@ -0,0 +1,171 @@ +set(JIT_TEST_ROOT ${TORCH_ROOT}/test/cpp/jit) + +# Build separate libraries the define custom classes/operators used from our Python tests. +# These are intended to be used with torch.ops.load_library() in our Python test suite. +add_library(torchbind_test SHARED + ${JIT_TEST_ROOT}/test_custom_class_registrations.h + ${JIT_TEST_ROOT}/test_custom_class_registrations.cpp +) +target_link_libraries(torchbind_test torch) + +add_library(jitbackend_test SHARED ${JIT_TEST_ROOT}/test_backend_lib.cpp) +target_link_libraries(jitbackend_test torch) + +set(BACKEND_WITH_COMPILER_SRCS + ${JIT_TEST_ROOT}/test_backend_compiler_lib.cpp + ${JIT_TEST_ROOT}/test_backend_compiler_preprocess.cpp +) +if(USE_KINETO) + # Testing edge profiler for backend use + # profiler_edge should only be added when USE_KINETO flag is on + list(APPEND BACKEND_WITH_COMPILER_SRCS + ${TORCH_SRC_DIR}/csrc/jit/mobile/profiler_edge.cpp) +endif() + +add_library(backend_with_compiler SHARED + ${BACKEND_WITH_COMPILER_SRCS} + ) +if(USE_KINETO) + set_target_properties(backend_with_compiler PROPERTIES COMPILE_FLAGS + "-DUSE_KINETO") +endif() +target_link_libraries(backend_with_compiler torch) + +if(INSTALL_TEST) + install(TARGETS torchbind_test DESTINATION lib) + install(TARGETS jitbackend_test DESTINATION lib) + install(TARGETS backend_with_compiler DESTINATION lib) +endif() + +# Build the cpp gtest binary containing the cpp-only tests. +set(JIT_TEST_SRCS + ${JIT_TEST_ROOT}/source_range_test.cpp + ${JIT_TEST_ROOT}/test_add_if_then_else.cpp + ${JIT_TEST_ROOT}/test_alias_analysis.cpp + ${JIT_TEST_ROOT}/test_argument_spec.cpp + ${JIT_TEST_ROOT}/test_autodiff.cpp + ${JIT_TEST_ROOT}/test_load_upgraders.cpp + ${JIT_TEST_ROOT}/test_op_replacement.cpp + ${JIT_TEST_ROOT}/test_upgrader_utils.cpp + ${JIT_TEST_ROOT}/test_backend.cpp + ${JIT_TEST_ROOT}/test_class_import.cpp + ${JIT_TEST_ROOT}/test_class_parser.cpp + ${JIT_TEST_ROOT}/test_class_type.cpp + ${JIT_TEST_ROOT}/test_code_template.cpp + ${JIT_TEST_ROOT}/test_concat_opt.cpp + ${JIT_TEST_ROOT}/test_constant_pooling.cpp + ${JIT_TEST_ROOT}/test_cleanup_passes.cpp + ${JIT_TEST_ROOT}/test_create_autodiff_subgraphs.cpp + ${JIT_TEST_ROOT}/test_custom_class.cpp + ${JIT_TEST_ROOT}/test_custom_class_registrations.h + ${JIT_TEST_ROOT}/test_custom_class_registrations.cpp + ${JIT_TEST_ROOT}/test_custom_operators.cpp + ${JIT_TEST_ROOT}/test_dce.cpp + ${JIT_TEST_ROOT}/test_fuser.cpp + ${JIT_TEST_ROOT}/test_graph_executor.cpp + ${JIT_TEST_ROOT}/test_graph_iterator.cpp + ${JIT_TEST_ROOT}/test_cs_debug_info_serialization.cpp + ${JIT_TEST_ROOT}/test_inliner.cpp + ${JIT_TEST_ROOT}/test_interface.cpp + ${JIT_TEST_ROOT}/test_interpreter.cpp + ${JIT_TEST_ROOT}/test_ir.cpp + ${JIT_TEST_ROOT}/test_irparser.cpp + ${JIT_TEST_ROOT}/test_jit_type.cpp + ${JIT_TEST_ROOT}/test_lexer.cpp + ${JIT_TEST_ROOT}/test_lite_interpreter.cpp + ${JIT_TEST_ROOT}/test_lite_interpreter_direct.cpp + ${JIT_TEST_ROOT}/test_lite_trainer.cpp + ${JIT_TEST_ROOT}/test_memory_dag.cpp + ${JIT_TEST_ROOT}/test_misc.cpp + ${JIT_TEST_ROOT}/test_mobile_type_parser.cpp + ${JIT_TEST_ROOT}/test_module_api.cpp + ${JIT_TEST_ROOT}/test_peephole_optimize.cpp + ${JIT_TEST_ROOT}/test_qualified_name.cpp + ${JIT_TEST_ROOT}/test_save_load.cpp + ${JIT_TEST_ROOT}/test_schema_info.cpp + ${JIT_TEST_ROOT}/test_schema_matching.cpp + ${JIT_TEST_ROOT}/test_stack_opt.cpp + ${JIT_TEST_ROOT}/test_subgraph_matcher.cpp + ${JIT_TEST_ROOT}/test_subgraph_rewriter.cpp + ${JIT_TEST_ROOT}/test_subgraph_utils.cpp + ${JIT_TEST_ROOT}/test_te.cpp + ${JIT_TEST_ROOT}/test_union.cpp + ${JIT_TEST_ROOT}/test_utils.cpp + ${JIT_TEST_ROOT}/test_script_profile.cpp + ${JIT_TEST_ROOT}/test_shape_analysis.cpp + ${JIT_TEST_ROOT}/test_jit_logging_levels.cpp + ${JIT_TEST_ROOT}/test_file_format.cpp + ${JIT_TEST_ROOT}/test_flatbuffer.cpp +) + +add_executable(test_jit + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${JIT_TEST_SRCS} +) + +# We also build with UBSAN flag in build_asan.h +# Skip UBSan on ROCm: combining ASAN + UBSan on ROCm Clang causes +# misaligned ASAN global metadata, triggering spurious ASAN alignment +# check aborts at startup. +if(USE_ASAN AND NOT USE_ROCM) + target_compile_options(test_jit PRIVATE "-fsanitize=undefined") + target_link_libraries(test_jit PRIVATE "-fsanitize=undefined") +endif() + +target_link_libraries( + test_jit PRIVATE flatbuffers) + + +# TODO temporary until we can delete the old gtest polyfills. +target_compile_definitions(test_jit PRIVATE USE_GTEST) + +if(USE_SYSTEM_ONNX) + target_link_libraries(test_jit PRIVATE onnx_proto onnx) +else() + target_link_libraries(test_jit PRIVATE onnx_library) +endif() + +if(USE_MKLDNN) + target_link_libraries(test_jit PRIVATE caffe2::mkldnn) +endif() + +set(JIT_TEST_DEPENDENCIES torch gtest_main jitbackend_test backend_with_compiler gmock) + +if(MSVC) + list(APPEND JIT_TEST_DEPENDENCIES onnx_library) +endif(MSVC) + +target_link_libraries(test_jit PRIVATE ${JIT_TEST_DEPENDENCIES}) +target_include_directories(test_jit PRIVATE ${ATen_CPU_INCLUDE}) + +if(LINUX) + #Update to target_link_options when CMake version can be upgraded + target_link_libraries(test_jit PRIVATE "-Wl,--no-as-needed,$,$,--as-needed") +endif() + +if(USE_CUDA) + target_compile_definitions(test_jit PRIVATE USE_CUDA) + # Suppress sign compare checks for NVFUSER JIT tests + if(NOT MSVC) + target_compile_options(test_jit PRIVATE $<$:-Wno-sign-compare>) + endif() +elseif(USE_ROCM) + target_link_libraries(test_jit PRIVATE + hiprtc::hiprtc + hip::amdhip64 + ${TORCH_CUDA_LIBRARIES}) + + target_compile_definitions(test_jit PRIVATE USE_ROCM) +endif() + +if(INSTALL_TEST) + set_target_properties(test_jit PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS test_jit DESTINATION bin) + install(FILES ${JIT_TEST_ROOT}/test_interpreter_async.pt DESTINATION bin) + install(FILES ${JIT_TEST_ROOT}/script_module_v4.ptl DESTINATION bin) + install(DIRECTORY ${JIT_TEST_ROOT}/upgrader_models DESTINATION bin) + # Install PDB files for MSVC builds + if(MSVC AND BUILD_SHARED_LIBS) + install(FILES $ DESTINATION bin OPTIONAL) + endif() +endif() diff --git a/western_ai/usa/pytorch/test/cpp/jit/README.md b/western_ai/usa/pytorch/test/cpp/jit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6c800c50a9d7a2db7639cdda4bf10bb39f80f93d --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/README.md @@ -0,0 +1,44 @@ +# JIT C++ Tests + +## Adding a new test +First, create a new test file. Test files should have be placed in this +directory, with a name that starts with `test_`, like `test_foo.cpp`. + +In general a single test suite + +Add your test file to the `JIT_TEST_SRCS` list in `test/cpp/jit/CMakeLists.txt`. + +A test file may look like: +```cpp +#include + +using namespace ::torch::jit + +TEST(FooTest, BarBaz) { + // ... +} + +// Append '_CUDA' to the test case name will automatically filter it out if CUDA +// is not compiled. +TEST(FooTest, NeedsAGpu_CUDA) { + // ... +} + +// Similarly, if only one GPU is detected, tests with `_MultiCUDA` at the end +// will not be run. +TEST(FooTest, NeedsMultipleGpus_MultiCUDA) { + // ... +} +``` + +## Building and running the tests +The following commands assume you are in PyTorch root. + +```bash +# ... Build PyTorch from source, e.g. +python -m pip install --no-build-isolation -v -e . +# (re)build just the binary +ninja -C build bin/test_jit +# run tests +build/bin/test_jit --gtest_filter='glob_style_filter*' +``` diff --git a/western_ai/usa/pytorch/test/cpp/jit/__init__.py b/western_ai/usa/pytorch/test/cpp/jit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/cpp/jit/script_module_v4.ptl b/western_ai/usa/pytorch/test/cpp/jit/script_module_v4.ptl new file mode 100644 index 0000000000000000000000000000000000000000..7ce36eed3e4698f187d6c24f57f5c7b48e850b52 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/script_module_v4.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/script_module_v5.ptl b/western_ai/usa/pytorch/test/cpp/jit/script_module_v5.ptl new file mode 100644 index 0000000000000000000000000000000000000000..b256b737c0b789c125fa6636399e87025c5a908b Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/script_module_v5.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/script_module_v6.ptl b/western_ai/usa/pytorch/test/cpp/jit/script_module_v6.ptl new file mode 100644 index 0000000000000000000000000000000000000000..a5945910fbd192658b0836213d6ebabbb0989912 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/script_module_v6.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/source_range_test.cpp b/western_ai/usa/pytorch/test/cpp/jit/source_range_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..14fd54682667a5b5060ba989f8d05e6b3eeb0112 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/source_range_test.cpp @@ -0,0 +1,77 @@ +#include + +#include +#include + +using namespace ::testing; +using namespace ::torch::jit; + +std::vector sampleStringCordViews() { + std::vector result; + + std::vector> strings; + strings.push_back(std::make_shared("hello world")); + strings.push_back(std::make_shared("nihaoma")); + + std::vector pieces{*strings[0], *strings[1]}; + + result.emplace_back(std::move(pieces), std::move(strings)); + pieces = {"hello worldnihaoma"}; + strings.clear(); + result.emplace_back(std::move(pieces), std::move(strings)); + return result; +} + +TEST(SourceRangeTest, test_find) { + for (const auto& view : sampleStringCordViews()) { + auto x = view.find("rldni", 0); + EXPECT_EQ(x, 8) << view.str(); + EXPECT_EQ(view.find("ello", 0), 1); + } +} + +TEST(SourceRangeTest, test_substr) { + for (const auto& view : sampleStringCordViews()) { + auto x = view.substr(4, 10).str(); + EXPECT_EQ(x, view.str().substr(4, 10)); + EXPECT_EQ(view.substr(0, view.size()).str(), view.str()); + for (const auto start : c10::irange(view.size())) { + for (const auto size : c10::irange(view.size())) { + EXPECT_EQ( + view.substr(start, size).str(), view.str().substr(start, size)); + } + } + } +} + +TEST(SourceRangeTest, test_iter_simple) { + for (const auto& view : sampleStringCordViews()) { + EXPECT_NE(view.begin(), view.end()); + EXPECT_TRUE(view.begin().has_next()); + EXPECT_EQ(view.str(), std::string(view.begin(), view.end())); + } +} + +TEST(SourceRangeTest, test_iter) { + int idx = 0; + for (const auto& view : sampleStringCordViews()) { + auto iter = view.iter_for_pos(5); + EXPECT_EQ(*iter, ' '); + if (idx++ == 0) { + EXPECT_EQ(iter.rest_line(), " world"); + } else { + EXPECT_EQ(iter.rest_line(), " worldnihaoma"); + } + EXPECT_EQ(*iter.next_iter(), 'w'); + EXPECT_EQ(iter.pos(), 5); + iter = view.iter_for_pos(13); + EXPECT_EQ(iter.pos(), 13); + } +} + +TEST(SourceRangeTest, SimpleString) { + Source src("hello"); + EXPECT_EQ(src.num_lines(), 1); + EXPECT_EQ(src.get_line(0), "hello"); + EXPECT_EQ(src.text_str().str(), "hello"); +} diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_add_if_then_else.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_add_if_then_else.cpp new file mode 100644 index 0000000000000000000000000000000000000000..03d3e48f1f61895b15ea0ec7abb5d29d7379bb10 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_add_if_then_else.cpp @@ -0,0 +1,53 @@ +#include + +#include +#include +#include + +namespace torch { +namespace jit { + +TEST(AddIfThenElseOpTest, AddIfThenElseOpSimple) { + const auto src = R"IR( + graph(%cond: bool, %a: Tensor, %b: Tensor): + %result: Tensor = prim::If(%cond) + block0(): + -> (%a) + block1(): + -> (%b) + return (%result) + )IR"; + + auto graph = std::make_shared(); + parseIR(src, graph.get()); + EXPECT_TRUE(AddIfThenElseOp(graph)); + + testing::FileCheck() + .check_count("= prim::IfThenElse", 1, /*exactly*/ true) + ->check_count("= prim::If", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(AddIfThenElseOpTest, NoIfThenElseOpMultipleOutputs) { + const auto src = R"IR( + graph(%cond: bool, %a: Tensor, %b: Tensor): + %result1: Tensor, %result2: Tensor = prim::If(%cond) + block0(): + -> (%a, %b) + block1(): + -> (%b, %a) + return (%result1, %result2) + )IR"; + + auto graph = std::make_shared(); + parseIR(src, graph.get()); + EXPECT_FALSE(AddIfThenElseOp(graph)); + + testing::FileCheck() + .check_count("= prim::IfThenElse", 0, /*exactly*/ true) + ->check_count("= prim::If", 1, /*exactly*/ true) + ->run(*graph); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_alias_analysis.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_alias_analysis.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e46a16b3a0855228a5cc5f7bb2f06ee7d71a6b95 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_alias_analysis.cpp @@ -0,0 +1,1706 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace torch { +namespace jit { + +inline c10::AliasAnalysisKind aliasAnalysisFromSchema() { + return c10::AliasAnalysisKind::FROM_SCHEMA; +} + +// Fixture to set up a graph and make assertions clearer +class TopologicalMoveTest : public ::testing::Test { + protected: + TopologicalMoveTest() { + createGraph(); + aliasDb = std::make_unique(graph); + } + + // Nodes are named after their output. + // e.g. "a" is an alias for "the node that outputs the value `a`" + void createGraph() { + graph = std::make_shared(); + createNode("a", {}); + createNode("b", {"a"}); + createNode("c", {}); + createNode("d", {"a", "b"}); + createNode("e", {"c", "b"}); + createNode("f", {"e"}); + createNode("g", {"e"}); + createNode("h", {"g"}); + createNode("i", {"g"}); + createNode("j", {"i"}); + createNode("k", {"i"}); + createNode("l", {"a"}); + createNode("m", {}, {"l"}); // block depends on l + createNode("n", {"m"}); + createNode("o", {"n"}); + createNode("p", {}); + createNode("q", {}); + createNode("r", {"q"}); + createNode("s", {"q"}); + + graph->lint(); + } + + void createNode( + const std::string& name, + const std::vector& inputNames, + const std::vector& blockInputNames = {}) { + std::vector inputs; + for (const auto& name_ : inputNames) { + // NOLINTNEXTLINE(performance-inefficient-vector-operation) + inputs.push_back(nodes.at(name_)->output()); + } + auto node = graph->appendNode(graph->create(prim::AutogradZero, inputs)); + node->output()->setDebugName(name); + nodes[name] = node; + + if (blockInputNames.size() != 0) { + node->addBlock(); + std::vector blockDeps; + for (const auto& name_ : blockInputNames) { + // NOLINTNEXTLINE(performance-inefficient-vector-operation) + blockDeps.push_back(nodes.at(name_)->output()); + } + + auto block = node->blocks().at(0); + block->appendNode(graph->create(prim::AutogradZero, blockDeps)); + } + } + + bool moveBeforeTopologicallyValid( + const std::string& toInsert, + const std::string& insertPoint) { + std::function func = + [this](Node* toInsert, Node* insertPoint) { + return aliasDb->moveBeforeTopologicallyValid(toInsert, insertPoint); + }; + return moveWithChecks(toInsert, insertPoint, func); + } + + bool moveAfterTopologicallyValid( + const std::string& toInsert, + const std::string& insertPoint) { + std::function func = + [this](Node* toInsert, Node* insertPoint) { + return aliasDb->moveAfterTopologicallyValid(toInsert, insertPoint); + }; + return moveWithChecks(toInsert, insertPoint, func); + } + + bool moveWithChecks( + const std::string& toInsert, + const std::string& insertPoint, + std::function func) { + auto n = nodes.at(toInsert); + auto insert = nodes.at(insertPoint); + bool isAfter = n->isAfter(insert); + + std::vector originalOrdering; + Node* original = isAfter ? n->next() : n->prev(); + + auto curNode = original; + while (curNode != n->owningBlock()->return_node()) { + originalOrdering.push_back(curNode); + if (isAfter) { + curNode = curNode->next(); + } else { + curNode = curNode->prev(); + } + } + + const auto couldMove = func(n, insert); + // Check the graph is okay + graph->lint(); + + // If this is the picture of nodes + // ... toInsert ... ... insertPoint + // ^----------^ check that these nodes haven't moved + curNode = original; + size_t idx = 0; + while (curNode != n->owningBlock()->return_node()) { + EXPECT_TRUE(originalOrdering[idx] == curNode); + if (isAfter) { + curNode = curNode->next(); + } else { + curNode = curNode->prev(); + } + idx++; + } + + return couldMove; + } + + void checkPostCondition( + const std::string& toInsert, + const std::string& insertPoint, + bool after) { + if (after) { + EXPECT_EQ(nodes.at(toInsert)->prev(), nodes.at(insertPoint)); + } else { + EXPECT_EQ(nodes.at(toInsert)->next(), nodes.at(insertPoint)); + } + } + + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::shared_ptr graph; + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::unique_ptr aliasDb; + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::unordered_map nodes; +}; + +TEST_F(TopologicalMoveTest, SplitsDeps) { + // Check that we are removing `this`'s deps properly when we need to split + // `this` and deps (see code for what the hell that means) + EXPECT_TRUE(moveBeforeTopologicallyValid("q", "s")); + checkPostCondition("q", "s", false); +} + +// Move after +TEST_F(TopologicalMoveTest, MoveAfterBackwardSimple) { + // Simple move backward + EXPECT_TRUE(moveAfterTopologicallyValid("c", "a")); + checkPostCondition("c", "a", true); +} +TEST_F(TopologicalMoveTest, MoveAfterBackwardInvalid) { + // simple invalid move backward + EXPECT_FALSE(moveAfterTopologicallyValid("d", "a")); +} + +TEST_F(TopologicalMoveTest, MoveAfterNoOp) { + // doesn't actually move anything + EXPECT_TRUE(moveAfterTopologicallyValid("f", "e")); + checkPostCondition("f", "e", true); +} + +TEST_F(TopologicalMoveTest, MoveAfterBackwardMultipleDeps) { + // move backward with multiple dependencies + EXPECT_TRUE(moveAfterTopologicallyValid("e", "c")); + checkPostCondition("e", "c", true); +} + +TEST_F(TopologicalMoveTest, MoveAfterBackwardNonZeroWorkingSet) { + // Move backward with non-zero working set + EXPECT_TRUE(moveAfterTopologicallyValid("k", "f")); + checkPostCondition("k", "f", true); +} + +TEST_F(TopologicalMoveTest, MoveAfterForwardSimple) { + // Simple move forward + EXPECT_TRUE(moveAfterTopologicallyValid("c", "d")); + checkPostCondition("c", "d", true); +} + +TEST_F(TopologicalMoveTest, MoveAfterForwardNonZeroWorkingSet) { + // Move forward with non-zero working set + EXPECT_TRUE(moveAfterTopologicallyValid("f", "l")); + checkPostCondition("f", "l", true); +} + +// Move before +TEST_F(TopologicalMoveTest, MoveBeforeForwardSimple) { + // Simple move forward + EXPECT_TRUE(moveBeforeTopologicallyValid("b", "d")); + checkPostCondition("b", "d", false); +} + +TEST_F(TopologicalMoveTest, MoveBeforeBackwardSimple) { + // Simple move backward + EXPECT_TRUE(moveBeforeTopologicallyValid("c", "a")); + checkPostCondition("c", "a", false); +} + +TEST_F(TopologicalMoveTest, MoveBeforeNoOp) { + // doesn't actually move anything + EXPECT_TRUE(moveBeforeTopologicallyValid("a", "b")); + checkPostCondition("a", "b", false); +} + +TEST_F(TopologicalMoveTest, MoveBeforeForwardWithDeps) { + // move forward with deps + EXPECT_TRUE(moveBeforeTopologicallyValid("f", "m")); + checkPostCondition("f", "m", false); +} + +TEST_F(TopologicalMoveTest, MoveBeforeBackwardWithDeps) { + // move backward with deps + EXPECT_TRUE(moveBeforeTopologicallyValid("l", "f")); + checkPostCondition("l", "f", false); +} + +// check that dependencies in blocks are recognized +TEST_F(TopologicalMoveTest, DepsDisallowMove) { + EXPECT_FALSE(moveAfterTopologicallyValid("l", "m")); + EXPECT_FALSE(moveBeforeTopologicallyValid("m", "l")); + EXPECT_FALSE(moveAfterTopologicallyValid("n", "l")); + EXPECT_FALSE(moveBeforeTopologicallyValid("l", "n")); +} + +// Test that moveAfter(n) and moveBefore(n->next()) are not necessarily +// equivalent. Here, the dependency ordering is n -> o -> p. So we can't +// move `n` after `o`, but we can move `n` before `p` (which pushes `o` after +// `p`) +TEST_F(TopologicalMoveTest, MoveAfterBeforeWithDeps) { + EXPECT_FALSE(moveAfterTopologicallyValid("n", "o")); + EXPECT_TRUE(moveBeforeTopologicallyValid("o", "p")); + checkPostCondition("o", "p", false); +} + +namespace { +Node* insertIf( + Graph& g, + Value* condValue, + std::function()> trueInst, + std::function()> falseInst) { + auto if_ = g.insertNode(g.create(prim::If, 0)); + if_->addInput(condValue); // condition value + auto trueBlock = if_->addBlock(); + auto falseBlock = if_->addBlock(); + { + // Mutate in true block + WithInsertPoint g(trueBlock); + auto outputs = trueInst(); + for (auto output : outputs) { + trueBlock->registerOutput(output); + } + } + { + WithInsertPoint g(falseBlock); + auto outputs = falseInst(); + for (auto output : outputs) { + falseBlock->registerOutput(output); + } + } + + EXPECT_TRUE(trueBlock->outputs().size() == falseBlock->outputs().size()); + for (auto output : trueBlock->outputs()) { + if_->addOutput()->setType(output->type()); + } + return if_; +} + +template +inline void expectThrows(Functor&& functor, const char* expectMessageContains) { + try { + std::forward(functor)(); + } catch (const Exception& e) { + if (std::string(e.what()).find(expectMessageContains) == + std::string::npos) { + TORCH_CHECK( + false, + "Expected error message to contain \"", + expectMessageContains, + "\" but error message was: ", + e.what()); + } + return; + } + TORCH_CHECK( + false, + "Expected to throw exception containing \"", + expectMessageContains, + "\" but didn't throw"); +} + +} // namespace + +TEST(AliasAnalysisTest, AliasingMutationBlocksMoves) { + auto graph = std::make_shared(); + auto a = graph->addInput(); + auto b = graph->addInput(); + + // addsB = b + b + // c = a + b + // a += b + // d = c + c + auto addsB = graph->insert(aten::add, {b, b}); + auto c = graph->insert(aten::add, {a, b}); + auto aMut = graph->insert(aten::add_, {a, b}); + auto d = graph->insert(aten::add, {c, c}); + + graph->lint(); + + AliasDb aliasDb(graph); + // Can't move past a mutation of a used value + EXPECT_FALSE(aliasDb.moveAfterTopologicallyValid(c->node(), aMut->node())); + EXPECT_TRUE(aliasDb.moveAfterTopologicallyValid(d->node(), c->node())); + + // b should alias to a (since they are both inputs) + EXPECT_FALSE( + aliasDb.moveAfterTopologicallyValid(addsB->node(), aMut->node())); + EXPECT_TRUE(aliasDb.moveAfterTopologicallyValid(addsB->node(), c->node())); + + graph->lint(); +} + +TEST(AliasAnalysisTest, AliasingMutationBlocksMoves2) { + auto graph = std::make_shared(); + auto a = graph->addInput(); + auto b = graph->addInput(); + + auto constant = graph->insertConstant(1); + auto fresh = graph->insert(aten::rand, {constant}); + auto usesB = graph->insert(aten::add, {b, fresh}); + auto aliasesB = graph->insert(aten::select, {a, constant, constant}); + auto mutatesAliasOfB = graph->insert(aten::add_, {aliasesB, fresh}); + graph->insert(aten::add, {fresh, aliasesB}); + graph->lint(); + + AliasDb aliasDb(graph); + EXPECT_FALSE(aliasDb.moveAfterTopologicallyValid( + aliasesB->node(), mutatesAliasOfB->node())); + EXPECT_FALSE(aliasDb.moveAfterTopologicallyValid( + usesB->node(), mutatesAliasOfB->node())); +} + +TEST(AliasAnalysisTest, SideEffectsBlockMoves) { + // Test moves across side effectful nodes + auto graph = std::make_shared(); + auto a = graph->addInput(); + auto print1 = graph->insertNode(graph->create(prim::Print, {a}, 0)); + WithInsertPoint guard(print1); + auto print2 = graph->insertNode(graph->create(prim::Print, {a, a}, 0)); + AliasDb aliasDb(graph); + + // def foo(a): + // print2(a, a) + // print1(a) + + // test moving across each other + EXPECT_FALSE(aliasDb.moveAfterTopologicallyValid(print2, print1)); + EXPECT_FALSE(aliasDb.moveBeforeTopologicallyValid(print1, print2)); + + // test moving where they already are + EXPECT_TRUE(aliasDb.moveBeforeTopologicallyValid(print2, print1)); + EXPECT_TRUE(aliasDb.moveAfterTopologicallyValid(print1, print2)); + + graph->insertNode(graph->create(prim::MakeTestTensor, {}, 1)); + AliasDb aliasDb2(graph); + + // def foo(a): + // print2(a, a) + // non_side_effectful = makeTestTensor() + // print1(a) + + // test moving with a side effectful node between + EXPECT_FALSE(aliasDb2.moveAfterTopologicallyValid(print2, print1)); + EXPECT_FALSE(aliasDb2.moveBeforeTopologicallyValid(print2, print1)); + EXPECT_FALSE(aliasDb2.moveAfterTopologicallyValid(print1, print2)); + EXPECT_FALSE(aliasDb2.moveBeforeTopologicallyValid(print1, print2)); +} + +TEST(AliasAnalysisTest, MovingAcrossInnerBlocks) { + // Test moves across inner blocks + + // a = rand(1) + // b = rand(1) + // if True: + // a.add_(b) + // c = a + b + auto graph = std::make_shared(); + auto constant = graph->insertConstant(1); + auto a = graph->insert(aten::rand, {constant}); + auto b = graph->insert(aten::rand, {constant}); + + auto if_ = insertIf( + *graph, + constant, + [&]() -> std::vector { + auto aMut = graph->insert(aten::add_, {a, b}); + return {aMut}; + }, + [&]() -> std::vector { return {a}; }); + + auto c = graph->insert(aten::add, {a, b}); + + graph->lint(); + + // we should not be able to move `c` before the if statement, since it + // may write to `a`. + AliasDb aliasDb(graph); + EXPECT_FALSE(aliasDb.moveBeforeTopologicallyValid(c->node(), if_)); +} + +TEST(AliasAnalysisTest, NoneHasNoWriters) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( + graph(): + %opt : Tensor? = prim::Constant() + %out : Tensor = prim::unchecked_unwrap_optional(%opt) + %ret.2 : Tensor = aten::div(%out, %out, %out) + return (%opt, %out, %ret.2) + )IR", + &*graph, + vmap); + + AliasDb aliasDb(graph); + EXPECT_FALSE(aliasDb.hasWriters(vmap["opt"]->node())); +} + +TEST(AliasAnalysisTest, SafeToChangeAliasingRelationship) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( + graph(%x : Tensor): + %3 : int = prim::Constant[value=1]() + %2 : int = prim::Constant[value=0]() + %b : Tensor = aten::add(%x, %2, %3) + %c : Tensor = aten::add(%x, %2, %3) + %d : Tensor = aten::add(%x, %2, %3) + %e : Tensor = aten::add(%x, %2, %3) + %f : Tensor[] = prim::ListConstruct(%e) + %14 : (Tensor, Tensor) = prim::TupleConstruct(%b, %c) + return (%14) + )IR", + &*graph, + vmap); + + AliasDb aliasDb(graph); + // x, b, c escape scope, so we can't introduce an aliasing relationship + EXPECT_FALSE(aliasDb.safeToChangeAliasingRelationship(vmap["x"], vmap["b"])); + EXPECT_FALSE(aliasDb.safeToChangeAliasingRelationship(vmap["b"], vmap["x"])); + EXPECT_FALSE(aliasDb.safeToChangeAliasingRelationship(vmap["b"], vmap["c"])); + EXPECT_FALSE(aliasDb.safeToChangeAliasingRelationship(vmap["c"], vmap["b"])); + + // e aliases the wildcard set because it's contained in a list + EXPECT_FALSE(aliasDb.safeToChangeAliasingRelationship(vmap["e"], vmap["x"])); + EXPECT_FALSE(aliasDb.safeToChangeAliasingRelationship(vmap["x"], vmap["e"])); + + // d is a temporary with no writers, safe to change aliasing relationship + // here + EXPECT_TRUE(aliasDb.safeToChangeAliasingRelationship(vmap["c"], vmap["d"])); + EXPECT_TRUE(aliasDb.safeToChangeAliasingRelationship(vmap["d"], vmap["c"])); +} + +class BatchAndInstanceNormFixture + : public ::testing::TestWithParam> { +}; + +TEST_P(BatchAndInstanceNormFixture, BatchAndInstanceNorm) { + auto param = GetParam(); + auto fnName = std::get<0>(param); + auto nodeKind = std::get<1>(param); + auto isTraining = std::get<2>(param); + std::string isTrainingStr = std::to_string((int)isTraining); + + auto graph = std::make_shared(); + + parseIR( + R"IR( + graph(%input : Tensor, %running_mean : Tensor, %running_var : Tensor): + %none : NoneType = prim::Constant() + %training : bool = prim::Constant[value=)IR" + + isTrainingStr + R"IR(]() + %momentum : float = prim::Constant[value=1.0]() + %eps : float = prim::Constant[value=1.0e-9]() + %cudnn_enabled : bool = prim::Constant[value=0]() + %res : Tensor = )IR" + + fnName + + R"IR((%input, %none, %none, %running_mean, %running_var, %training, %momentum, %eps, %cudnn_enabled) + return (%res) + )IR", + &*graph); + + graph->lint(); + DepthFirstGraphNodeIterator it(graph); + + Node* n = nullptr; + while ((n = it.next()) != nullptr) { + if (n->kind() == nodeKind) { + break; + } + } + EXPECT_TRUE(n != nullptr); + + AliasDb aliasDb(graph); + EXPECT_TRUE(aliasDb.hasWriters(n) == isTraining); +} + +TEST_P(BatchAndInstanceNormFixture, BatchAndInstanceNormTrainingUnknown) { + auto param = GetParam(); + auto fnName = std::get<0>(param); + auto nodeKind = std::get<1>(param); + + auto graph = std::make_shared(); + + parseIR( + R"IR( + graph(%input : Tensor, %running_mean : Tensor, %running_var : Tensor, %training : bool): + %none : NoneType = prim::Constant() + %momentum : float = prim::Constant[value=1.0]() + %eps : float = prim::Constant[value=1.0e-9]() + %cudnn_enabled : bool = prim::Constant[value=0]() + %res : Tensor = )IR" + + fnName + + R"IR((%input, %none, %none, %running_mean, %running_var, %training, %momentum, %eps, %cudnn_enabled) + return (%res) + )IR", + &*graph); + + graph->lint(); + DepthFirstGraphNodeIterator it(graph); + + Node* n = nullptr; + while ((n = it.next()) != nullptr) { + if (n->kind() == nodeKind) { + break; + } + } + EXPECT_TRUE(n != nullptr); + + AliasDb aliasDb(graph); + EXPECT_TRUE(aliasDb.hasWriters(n)); +} + +TEST_P(BatchAndInstanceNormFixture, BatchNormTrainingWithNoMeanOrVar) { + auto param = GetParam(); + auto fnName = std::get<0>(param); + auto nodeKind = std::get<1>(param); + auto isTraining = std::get<2>(param); + std::string isTrainingStr = std::to_string((int)isTraining); + + auto graph = std::make_shared(); + + parseIR( + R"IR( + graph(%input : Tensor): + %none : NoneType = prim::Constant() + %training : bool = prim::Constant[value=)IR" + + isTrainingStr + R"IR(]() + %momentum : float = prim::Constant[value=1.0]() + %eps : float = prim::Constant[value=1.0e-9]() + %cudnn_enabled : bool = prim::Constant[value=0]() + %res : Tensor = )IR" + + fnName + + R"IR((%input, %none, %none, %none, %none, %training, %momentum, %eps, %cudnn_enabled) + return (%res) + )IR", + &*graph); + + graph->lint(); + DepthFirstGraphNodeIterator it(graph); + + Node* n = nullptr; + while ((n = it.next()) != nullptr) { + if (n->kind() == nodeKind) { + break; + } + } + EXPECT_TRUE(n != nullptr); + + AliasDb aliasDb(graph); + EXPECT_FALSE(aliasDb.hasWriters(n)); +} + +INSTANTIATE_TEST_SUITE_P( + AliasAnalysisTest, + BatchAndInstanceNormFixture, + ::testing::Values( + std::make_tuple("aten::batch_norm", aten::batch_norm, false), + std::make_tuple("aten::instance_norm", aten::instance_norm, false), + std::make_tuple("aten::batch_norm", aten::batch_norm, true), + std::make_tuple("aten::instance_norm", aten::instance_norm, true))); + +TEST(WriteTrackingTest, Basic) { + RegisterOperators reg({Operator( + "prim::creates_alias(Tensor(a) x) -> Tensor(a)", + [](Stack&) {}, + aliasAnalysisFromSchema())}); + const auto creates_alias = Symbol::fromQualString("prim::creates_alias"); + auto graph = std::make_shared(); + auto a = graph->addInput(); + auto b = graph->addInput(); + + // aten::add(%b, %b) + // aten::add_(%a, %b) + // foo::creates_alias(%a) + auto pureNode = graph->insert(aten::add, {b, b})->node(); + auto writingNode = graph->insert(aten::add_, {a, b})->node(); + auto node3 = graph->insert(creates_alias, {a})->node(); + auto aAlias = node3->output(); + + graph->lint(); + + AliasDb aliasDb(graph); + EXPECT_TRUE(aliasDb.mayAlias(aAlias, a)); + EXPECT_TRUE(aliasDb.mayAlias(a, b)); + EXPECT_FALSE( + aliasDb.writesToAlias(pureNode, std::unordered_set{a})); + EXPECT_FALSE( + aliasDb.writesToAlias(pureNode, std::unordered_set{b})); + EXPECT_TRUE( + aliasDb.writesToAlias(writingNode, std::unordered_set{a})); + EXPECT_TRUE(aliasDb.writesToAlias( + writingNode, std::unordered_set{a, b})); + EXPECT_TRUE(aliasDb.writesToAlias( + writingNode, std::unordered_set{aAlias})); +} + +TEST(WriteTrackingTest, IsMutable) { + auto graph = std::make_shared(); + parseIR( + R"IR( + graph(%x: Tensor): + %b : Tensor = aten::relu_(%x) + return (%b) + )IR", + &*graph); + auto node_iter = graph->block()->nodes().begin(); + auto relu = *node_iter; + AliasDb aliasDb(graph); + EXPECT_TRUE(aliasDb.isMutable(relu)); +} + +TEST(WriteTrackingTest, IsImmutable) { + auto graph = std::make_shared(); + parseIR( + R"IR( + graph(%x: Tensor, %y : Tensor): + %b : Tensor = aten::mul(%x, %y) + return (%b) + )IR", + &*graph); + auto node_iter = graph->block()->nodes().begin(); + auto mul = *node_iter; + AliasDb aliasDb(graph); + EXPECT_FALSE(aliasDb.isMutable(mul)); +} + +TEST(WriteTrackingTest, HasWriters) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( + graph(%x: Tensor, %y : Tensor): + %c1 : int = prim::Constant[value=1]() + %b : Tensor = aten::add_(%x, %y, %c1) + return (%b) + )IR", + &*graph, + vmap); + auto add = vmap["b"]->node(); + AliasDb aliasDb(graph); + EXPECT_TRUE(aliasDb.hasWriters(add)); + EXPECT_TRUE(aliasDb.isMutable(add)); +} + +TEST(ContainerAliasingTest, MayContainAlias) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( + graph(%inp: Tensor[]): + %x : str = prim::Constant[value="a"]() + %y : Tensor = prim::Constant() + %z : Tensor = prim::Constant() + %a : (Tensor) = prim::TupleConstruct(%y) + %b : Dict(str, Tensor) = prim::DictConstruct(%x, %y) + %c : Tensor[] = prim::ListConstruct(%y) + return (%a, %b, %c) + )IR", + &*graph, + vmap); + + auto str_output = vmap["x"]; + auto ten_output = vmap["y"]; + auto local_var = vmap["z"]; + AliasDb aliasDb(graph); + + EXPECT_TRUE(graph->outputs().size() == 3); + for (auto out : graph->outputs()) { + EXPECT_TRUE(aliasDb.mayContainAlias(ten_output, out)); + EXPECT_FALSE(aliasDb.mayContainAlias(local_var, out)); + } + + EXPECT_TRUE(aliasDb.mayContainAlias(ten_output, graph->inputs())); + EXPECT_FALSE(aliasDb.mayContainAlias(local_var, graph->inputs())); + + EXPECT_TRUE(aliasDb.mayContainAlias(ten_output, graph->outputs())); + EXPECT_TRUE(aliasDb.mayContainAlias( + at::ArrayRef{ten_output}, graph->outputs())); + EXPECT_FALSE(aliasDb.mayContainAlias(str_output, graph->outputs())); +} + +TEST(ContainerAliasingTest, MayContainAlias_cast) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( + graph(%input.1 : Tensor): + %2 : NoneType = prim::Constant() + %3 : bool = prim::Constant[value=0]() + %4 : int = prim::Constant[value=6]() + %5 : int = prim::Constant[value=1]() + %a.1 : Tensor = aten::add(%input.1, %input.1, %5) + %b.1 : Tensor = aten::to(%a.1, %4, %3, %3, %2) + %c.1 : Tensor = aten::mul(%b.1, %b.1) + return (%c.1) + )IR", + &*graph, + vmap); + + auto a = vmap["a.1"]; + auto b = vmap["b.1"]; + auto c = vmap["c.1"]; + AliasDb aliasDb(graph); + + EXPECT_TRUE(graph->outputs().size() == 1); + for (auto out : graph->outputs()) { + EXPECT_TRUE(aliasDb.mayContainAlias(c, out)); + } + + EXPECT_TRUE(aliasDb.mayContainAlias(a, b)); + EXPECT_FALSE(aliasDb.mayContainAlias(b, graph->inputs())); + + EXPECT_TRUE(aliasDb.mayContainAlias(c, graph->outputs())); + EXPECT_TRUE( + aliasDb.mayContainAlias(at::ArrayRef{c}, graph->outputs())); + EXPECT_FALSE(aliasDb.mayContainAlias(b, graph->outputs())); +} + +TEST(ContainerAliasingTest, PrimitveValuesDontAliasContainers) { + auto graph = std::make_shared(); + parseIR( + R"IR( + graph(): + %x : str = prim::Constant[value="a"]() + %y : int = prim::Constant[value=1]() + %a : (int) = prim::TupleConstruct(%y) + %b : Dict(str, int) = prim::DictConstruct(%x, %y) + %c : int[] = prim::ListConstruct(%y) + return (%a, %b, %c) + )IR", + &*graph); + + auto node_iter = graph->block()->nodes().begin(); + node_iter++; // string + Node* int_node = *node_iter++; + AliasDb aliasDb(graph); + + EXPECT_TRUE(graph->outputs().size() == 3); + // primitive values don't need to alias container + for (auto out : graph->outputs()) { + EXPECT_FALSE(aliasDb.mayContainAlias(int_node->output(), out)); + } +} + +TEST(ContainerAliasingTest, UnionAliasing) { + auto graph = std::make_shared(); + parseIR( + R"IR( + graph(%a : Dict(str, Tensor), + %b : Tensor[], + %c : Union(Dict(str, Tensor), Tensor[])): + return (%a, %b, %c) + )IR", + &*graph); + + AliasDb aliasDb(graph); + auto a = graph->outputs().at(0); + auto b = graph->outputs().at(1); + auto c = graph->outputs().at(2); + + EXPECT_TRUE(aliasDb.mayAlias(a, c)); + EXPECT_TRUE(aliasDb.mayAlias(b, c)); + EXPECT_TRUE(aliasDb.mayAlias(c, c)); + EXPECT_FALSE(aliasDb.mayAlias(a, b)); + EXPECT_TRUE(aliasDb.mayContainAlias(a, b)); + EXPECT_TRUE(aliasDb.mayContainAlias(a, c)); + EXPECT_TRUE(aliasDb.mayContainAlias(b, c)); +} + +TEST(ContainerAliasingTest, InputsCanAliasOutputs) { + // Test input aliasing + auto graph = std::make_shared(); + parseIR( + R"IR( + graph(%x: Tensor, %y: Tensor): + %a : (Tensor) = prim::TupleConstruct(%x) + return (%a) + )IR", + &*graph); + + auto node_iter = graph->block()->nodes().begin(); + auto tuple_node = *node_iter; + AliasDb aliasDb(graph); + + for (auto input : graph->inputs()) { + EXPECT_TRUE(aliasDb.mayContainAlias(input, tuple_node->output())); + } + EXPECT_TRUE(aliasDb.mayContainAlias(graph->inputs(), graph->outputs())); +} + +// Test tuple that doesn't come from construct +TEST(ContainerAliasingTest, NestedTupleConstruct) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%x : int, + %y : Tensor, + %z : Tensor): + %3 : int = prim::Constant[value=1]() + %4 : bool = aten::eq(%x, %3) + %a : (Tensor) = prim::If(%4) + block0(): + %a.1 : (Tensor) = prim::TupleConstruct(%y) + -> (%a.1) + block1(): + %a.2 : (Tensor) = prim::TupleConstruct(%z) + -> (%a.2) + return (%a) + )IR", + &*graph); + + AliasDb aliasDb(graph); + + for (auto input : graph->inputs()) { + if (input->type() == IntType::get()) { + continue; + } + + EXPECT_TRUE(aliasDb.mayContainAlias(input, graph->outputs().at(0))); + } +} + +// test nested types +TEST(ContainerAliasingTest, NestedTypes) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(): + %a : Tensor = prim::MakeTestTensor() + %a_list : Tensor[] = prim::ListConstruct(%a) + %b : Tensor = prim::MakeTestTensor() + %b_list : Tensor[] = prim::ListConstruct(%b) + %13 : (Tensor[], Tensor[]) = prim::TupleConstruct(%a_list, %b_list) + return (%13) +)IR", + &*graph); + AliasDb aliasDb(graph); + auto g_output = graph->outputs().at(0); + auto list_2 = g_output->node()->inputs().at(0); + auto list_1 = g_output->node()->inputs().at(1); + + // TODO FIX assume conservatively for now + EXPECT_TRUE(aliasDb.mayContainAlias(list_1, list_2)); + EXPECT_TRUE(aliasDb.mayContainAlias(list_2, list_1)); + + EXPECT_TRUE(aliasDb.mayContainAlias(list_1, g_output)); + EXPECT_TRUE(aliasDb.mayContainAlias(list_2, g_output)); +} + +// simple example +TEST(ContainerAliasingTest, Simple) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(): + %0 : Tensor = prim::Constant() + %1 : Tensor = prim::Constant() + %13 : (Tensor) = prim::TupleConstruct(%0) + return (%13) +)IR", + &*graph); + AliasDb aliasDb(graph); + + auto node_iter = graph->block()->nodes().begin(); + auto first_ten = *node_iter++; + auto second_ten = *node_iter++; + auto tup_node = *node_iter; + + EXPECT_TRUE(aliasDb.mayContainAlias(first_ten->output(), tup_node->output())); + EXPECT_TRUE( + !aliasDb.mayContainAlias(second_ten->output(), tup_node->output())); + + std::vector first_st = {first_ten->output()}; + std::vector second_st = {second_ten->output()}; + std::vector tup_st = {tup_node->output()}; + EXPECT_TRUE(aliasDb.mayContainAlias(first_st, tup_st)); + EXPECT_FALSE(aliasDb.mayContainAlias(first_st, second_st)); + EXPECT_FALSE(aliasDb.mayContainAlias(second_st, tup_st)); +} + +TEST(ContainerAliasingTest, Lists) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( + graph(): + %x : str = prim::Constant[value="a"]() + %y : Tensor = prim::Constant() + %c : Tensor[] = prim::ListConstruct(%y) + %d : Tensor[] = prim::ListConstruct(%y) + return (%c, %d) + )IR", + &*graph, + vmap); + + AliasDb aliasDb(graph); + auto x = vmap["x"]; + auto c = vmap["c"]; + EXPECT_FALSE(aliasDb.mayContainAlias(x, c)); + EXPECT_FALSE(aliasDb.mayContainAlias(c, x)); + + auto d = vmap["d"]; + + EXPECT_TRUE(aliasDb.mayContainAlias(d, c)); + EXPECT_TRUE(aliasDb.mayContainAlias(c, d)); +} + +TEST(ContainerAliasingTest, Lists2) { + // Test list container aliasing + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(): + %0 : int = prim::Constant[value=2]() + %1 : int = prim::Constant[value=3]() + %2 : int[] = prim::ListConstruct(%0, %1) + %x : Tensor = prim::MakeTestTensor() + %12 : int[] = prim::ListConstruct(%0, %1) + %y : Tensor = prim::MakeTestTensor() + %22 : int[] = prim::ListConstruct(%0, %1) + %z : Tensor = prim::MakeTestTensor() + %32 : int[] = prim::ListConstruct(%0, %1) + %fresh : Tensor = prim::MakeTestTensor() + %foo : Tensor[] = prim::ListConstruct(%x, %y) + %43 : Tensor[] = aten::append(%foo, %z) + return () +)IR", + graph.get(), + vmap); + AliasDb aliasDb(graph); + auto x = vmap["x"]; + auto y = vmap["y"]; + auto z = vmap["z"]; + // Tensors x, y, and z went into a list, so they all may alias each other. + EXPECT_TRUE(aliasDb.mayAlias(x, y)); + EXPECT_TRUE(aliasDb.mayAlias(y, z)); + EXPECT_TRUE(aliasDb.mayAlias(x, z)); + + // But we know `fresh` didn't go into a list, so x, y, and z should not + // alias it. + auto fresh = vmap["fresh"]; + EXPECT_FALSE(aliasDb.mayAlias(x, fresh)); + EXPECT_FALSE(aliasDb.mayAlias(y, fresh)); + EXPECT_FALSE(aliasDb.mayAlias(z, fresh)); +} + +TEST(ContainerAliasingTest, Conservative) { + // test "conservative" analysis writes to the inside of a container. + auto ops = torch::RegisterOperators( + "custom::conservative", [](torch::List in) { return in; }); + + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(): + %0 : int = prim::Constant[value=2]() + %1 : int = prim::Constant[value=3]() + %2 : int[] = prim::ListConstruct(%0, %1) + %11 : Tensor = prim::MakeTestTensor() + %12 : Tensor[] = prim::ListConstruct(%11) + %out : Tensor[] = custom::conservative(%12) + %ret.2 : Tensor = aten::div(%11, %11) + return () +)IR", + graph.get(), + vmap); + AliasDb aliasDb(graph); + auto conservativeOp = vmap["out"]->node(); + auto tensor = vmap["11"]; + EXPECT_TRUE(aliasDb.writesToAlias(conservativeOp, ValueSet{tensor})); +} + +TEST(ContainerAliasingTest, MovesAcrossContainedWrites) { + auto ops = torch::RegisterOperators().op( + "uses::list", + torch::RegisterOperators::options() + .catchAllKernel( + [](torch::List in) { return torch::rand({2, 3}); }) + .aliasAnalysis(AliasAnalysisKind::PURE_FUNCTION)); + // Write to the inside of a list. Check that we can't reorder a + // print across it. + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(): + %35 : int = prim::Constant[value=1]() + %0 : int = prim::Constant[value=2]() + %1 : int = prim::Constant[value=3]() + %23 : int = prim::Constant[value=0]() + %2 : int[] = prim::ListConstruct(%0, %1) + %11 : Tensor = prim::MakeTestTensor() + %12 : int[] = prim::ListConstruct(%0, %1) + %21 : Tensor = prim::MakeTestTensor() + %l : Tensor[] = prim::ListConstruct(%11, %21) + %24 : Tensor = aten::select(%l, %23) + %25 : int[] = prim::ListConstruct(%0, %1) + %34 : Tensor = prim::MakeTestTensor() + %36 : Tensor = aten::add_(%24, %34, %35) + %37 : Tensor = uses::list(%l) + return (%37) +)IR", + graph.get(), + vmap); + AliasDb aliasDb(graph); + auto listUse = vmap["37"]->node(); + auto internalWrite = vmap["36"]->node(); + EXPECT_FALSE(aliasDb.moveBeforeTopologicallyValid(listUse, internalWrite)); +} + +TEST(ContainerAliasingTest, MovesAcrossContainedWritesNested) { + // The same as above, but with a nested list + auto ops = torch::RegisterOperators().op( + "uses::list", + torch::RegisterOperators::options() + .catchAllKernel( + [](torch::List in) { return torch::rand({2, 3}); }) + .aliasAnalysis(AliasAnalysisKind::PURE_FUNCTION)); + // Write to the inside of a list. Check that we can't reorder a + // print across it. + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(): + %38 : int = prim::Constant[value=1]() + %0 : int = prim::Constant[value=2]() + %1 : int = prim::Constant[value=3]() + %24 : int = prim::Constant[value=0]() + %2 : int[] = prim::ListConstruct(%0, %1) + %11 : Tensor = prim::MakeTestTensor() + %12 : int[] = prim::ListConstruct(%0, %1) + %21 : Tensor = prim::MakeTestTensor() + %l : Tensor[] = prim::ListConstruct(%11, %21) + %25 : Tensor = aten::select(%l, %24) + %27 : Tensor = aten::select(%25, %24, %24) + %28 : int[] = prim::ListConstruct(%0, %1) + %37 : Tensor = prim::MakeTestTensor() + %39 : Tensor = aten::add_(%27, %37, %38) + %40 : Tensor = uses::list(%l) + return (%40) +)IR", + graph.get(), + vmap); + AliasDb aliasDb(graph); + auto listUse = vmap["40"]->node(); + auto internalWrite = vmap["39"]->node(); + EXPECT_FALSE(aliasDb.moveBeforeTopologicallyValid(listUse, internalWrite)); +} + +TEST(WildcardsTest, Basic) { + RegisterOperators reg( + {Operator( + "prim::returns_wildcard(Tensor a) -> Tensor(*)", + [](Stack&) {}, + aliasAnalysisFromSchema()), + Operator( + "prim::writes(Tensor(z!) a) -> Tensor(a)", + [](Stack&) {}, + aliasAnalysisFromSchema())}); + const auto returns_wildcard = + Symbol::fromQualString("prim::returns_wildcard"); + const auto writes = Symbol::fromQualString("prim::writes"); + + auto graph = std::make_shared(); + const auto a = graph->addInput(); + + const auto constant = graph->insertConstant(1); + const auto fresh = graph->insert(aten::rand, {constant}); + const auto fresh2 = graph->insert(aten::rand, {constant}); + const auto wildcard = graph->insert(returns_wildcard, {fresh}); + + { + graph->lint(); + AliasDb aliasDb(graph); + + EXPECT_FALSE(aliasDb.mayAlias(a, fresh)); + EXPECT_FALSE(aliasDb.mayAlias(wildcard, fresh)); + EXPECT_TRUE(aliasDb.mayAlias(wildcard, a)); + EXPECT_FALSE(aliasDb.mayAlias(ValueSet{wildcard}, ValueSet{})); + EXPECT_FALSE(aliasDb.hasWriters(wildcard->node())); + } + + graph->insert(writes, {fresh2})->node(); + { + graph->lint(); + AliasDb aliasDb(graph); + EXPECT_FALSE(aliasDb.hasWriters(wildcard->node())); + } + + const auto wildcardWrite = graph->insert(writes, {wildcard})->node(); + { + graph->lint(); + AliasDb aliasDb(graph); + // Test writes to wildcards + EXPECT_FALSE(aliasDb.writesToAlias( + wildcardWrite, std::unordered_set{fresh})); + EXPECT_FALSE(aliasDb.writesToAlias( + wildcardWrite, std::unordered_set{fresh2})); + EXPECT_TRUE(aliasDb.writesToAlias( + wildcardWrite, std::unordered_set{a})); + EXPECT_TRUE(aliasDb.hasWriters(wildcard->node())); + } +} + +// test that wildcards are correctly divided by type +TEST(WildcardsTest, TypeIsolation) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( + graph(%ten_list : Tensor[], %int_list : int[], %opt_ten_list : Tensor[]?): + %ten : Tensor = prim::Constant() + %4 : Tensor[] = aten::append(%ten_list, %ten) + %ten_ten_list : Tensor[][] = prim::Constant() + %int_int_list : int[][] = prim::Constant() + return () + )IR", + &*graph, + vmap); + AliasDb aliasDb(graph); + auto opt_ten_list = vmap["opt_ten_list"]; + auto ten_list = vmap["ten_list"]; + auto int_list = vmap["int_list"]; + EXPECT_FALSE(aliasDb.hasWriters(int_list)); + EXPECT_TRUE(aliasDb.hasWriters(opt_ten_list)); + EXPECT_TRUE(aliasDb.hasWriters(ten_list)); + EXPECT_FALSE(aliasDb.mayContainAlias(int_list, opt_ten_list)); + EXPECT_TRUE(aliasDb.mayContainAlias(ten_list, opt_ten_list)); + EXPECT_TRUE(aliasDb.mayAlias(ten_list, opt_ten_list)); + + auto list_of_tensor_lists = vmap["ten_ten_list"]; + EXPECT_TRUE(aliasDb.mayContainAlias(ten_list, list_of_tensor_lists)); + EXPECT_TRUE(aliasDb.mayContainAlias(ten_list, vmap["ten"])); + + EXPECT_TRUE( + !aliasDb.mayContainAlias(vmap["int_int_list"], list_of_tensor_lists)); +} + +// test invariant container aliasing +// the containers of different type cannot alias each other, +// however they may contain elements which alias each other +TEST(WildcardsTest, InvariantContainerAliasing) { + { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( + graph(%ten_list : Tensor[], %ten_opt_list : Tensor?[]): + %ten : Tensor = prim::Constant() + %4 : Tensor[] = aten::append(%ten_list, %ten) + return () + )IR", + &*graph, + vmap); + AliasDb aliasDb(graph); + auto ten_opt_list = vmap["ten_opt_list"]; + auto ten_list = vmap["ten_list"]; + EXPECT_FALSE(aliasDb.hasWriters(ten_opt_list)); + EXPECT_TRUE(aliasDb.hasWriters(ten_list)); + EXPECT_TRUE(aliasDb.mayContainAlias(ten_list, ten_opt_list)); + EXPECT_FALSE(aliasDb.mayAlias(ten_list, ten_opt_list)); + } + { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( + graph(%float_3D : Float(*, *, *), %float_2D : Float(*, *)): + return () + )IR", + &*graph, + vmap); + AliasDb aliasDb(graph); + EXPECT_TRUE(aliasDb.mayAlias(vmap["float_3D"], vmap["float_2D"])); + } + + { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( + graph(%float_3D_list : Float(*, *, *)[], %float_2D_list : Float(*, *)[], %ten: Tensor): + return () + )IR", + &*graph, + vmap); + AliasDb aliasDb(graph); + EXPECT_TRUE(aliasDb.mayAlias(vmap["float_3D_list"], vmap["float_2D_list"])); + EXPECT_TRUE(aliasDb.mayContainAlias(vmap["float_3D_list"], vmap["ten"])); + EXPECT_TRUE(aliasDb.mayContainAlias(vmap["float_2D_list"], vmap["ten"])); + } +} + +TEST(AliasRegistrationTest, ConservativeWithInferredSchema) { + auto registry = torch::RegisterOperators().op( + "foo::rand1", + torch::RegisterOperators::options() + .catchAllKernel( + [](at::Tensor) -> at::Tensor { return at::rand({2, 2}); }) + .aliasAnalysis(AliasAnalysisKind::CONSERVATIVE)); + const auto rand_op = Symbol::fromQualString("foo::rand1"); + auto graph = std::make_shared(); + auto a = graph->addInput(); + auto b = graph->insert(rand_op, {a}); + AliasDb aliasDb(graph); + // Conservatively we assume there is a reference + EXPECT_TRUE(aliasDb.mayAlias(a, b)); +} + +TEST(AliasRegistrationTest, ConservativeWithSpecifiedSchema) { + auto registry = torch::RegisterOperators().op( + "foo::rand2(Tensor arg1) -> Tensor", + torch::RegisterOperators::options() + .catchAllKernel( + [](at::Tensor) -> at::Tensor { return at::rand({2, 2}); }) + .aliasAnalysis(AliasAnalysisKind::CONSERVATIVE)); + const auto rand_op = Symbol::fromQualString("foo::rand2"); + auto graph = std::make_shared(); + auto a = graph->addInput(); + auto b = graph->insert(rand_op, {a}); + AliasDb aliasDb(graph); + // Conservatively we assume there is a reference + EXPECT_TRUE(aliasDb.mayAlias(a, b)); +} + +TEST(AliasRegistrationTest, ConservativeWithAliasingAnnotationsShouldError) { + auto registry = torch::RegisterOperators().op( + "foo::rand3(Tensor(a) arg1) -> Tensor(b)", + torch::RegisterOperators::options() + .catchAllKernel( + [](at::Tensor) -> at::Tensor { return at::rand({2, 2}); }) + .aliasAnalysis(AliasAnalysisKind::CONSERVATIVE)); + + const auto rand_op = Symbol::fromQualString("foo::rand3"); + auto graph = std::make_shared(); + auto a = graph->addInput(); + graph->insert(rand_op, {a}); + + // Registration time is okay, but throw exception when fetch from + // registration. + expectThrows( + [&graph] { AliasDb aliasDb(graph); }, + "Tried to register operator foo::rand3(Tensor(a) arg1) -> Tensor(b) with aliasing information in the schema but without AliasAnalysisKind::FROM_SCHEMA"); +} + +TEST(AliasRegistrationTest, ConservativeWithAliasingAnnotationsShouldError2) { + auto registry = torch::RegisterOperators().op( + "foo::rand4(Tensor(a) arg1) -> Tensor(a)", + torch::RegisterOperators::options() + .catchAllKernel( + [](at::Tensor) -> at::Tensor { return at::rand({2, 2}); }) + .aliasAnalysis(AliasAnalysisKind::CONSERVATIVE)); + const auto rand_op = Symbol::fromQualString("foo::rand4"); + auto graph = std::make_shared(); + auto a = graph->addInput(); + graph->insert(rand_op, {a}); + + // Registration time is okay, but throw exception when fetch from + // registration. + expectThrows( + [&graph] { AliasDb aliasDb(graph); }, + "Tried to register operator foo::rand4(Tensor(a) arg1) -> Tensor(a) with aliasing information in the schema but without AliasAnalysisKind::FROM_SCHEMA"); +} + +TEST(AliasRegistrationTest, FromSchemaWithInferredSchemaShouldError) { + expectThrows( + [] { + torch::RegisterOperators().op( + "foo::rand5", + torch::RegisterOperators::options() + .catchAllKernel( + [](at::Tensor) -> at::Tensor { return at::rand({2, 2}); }) + .aliasAnalysis(AliasAnalysisKind::FROM_SCHEMA)); + }, + "Tried to register operator foo::rand5(Tensor _0) -> Tensor _0 with AliasAnalysisKind::FROM_SCHEMA, but the schema is inferred"); +} + +TEST(AliasRegistrationTest, FromSchemaInferredPure) { + auto registry = torch::RegisterOperators().op( + "foo::rand6(Tensor arg1) -> Tensor", + torch::RegisterOperators::options() + .catchAllKernel( + [](at::Tensor) -> at::Tensor { return at::rand({2, 2}); }) + .aliasAnalysis(AliasAnalysisKind::FROM_SCHEMA)); + const auto rand_op = Symbol::fromQualString("foo::rand6"); + auto graph = std::make_shared(); + auto a = graph->addInput(); + auto b = graph->insert(rand_op, {a}); + AliasDb aliasDb(graph); + // The schema doesn't contain alias information, which means it's pure + // (meh!) + EXPECT_FALSE(aliasDb.mayAlias(a, b)); +} + +TEST(AliasRegistrationTest, FromSchemaAliased) { + auto registry = torch::RegisterOperators().op( + "foo::rand7(Tensor(a) arg1) -> Tensor(a)", + torch::RegisterOperators::options() + .catchAllKernel([](at::Tensor t) -> at::Tensor { return t * 2; }) + .aliasAnalysis(AliasAnalysisKind::FROM_SCHEMA)); + const auto rand_op = Symbol::fromQualString("foo::rand7"); + + auto graph = std::make_shared(); + auto a = graph->addInput(); + auto b = graph->insert(rand_op, {a}); + AliasDb aliasDb(graph); + // The schema has an alias reference + EXPECT_TRUE(aliasDb.mayAlias(a, b)); +} + +TEST(AliasRegistrationTest, FromSchemaPure) { + auto registry = torch::RegisterOperators().op( + "foo::rand8(Tensor(a) arg1) -> Tensor(b)", + torch::RegisterOperators::options() + .catchAllKernel([](at::Tensor t) -> at::Tensor { return t * 2; }) + .aliasAnalysis(AliasAnalysisKind::FROM_SCHEMA)); + const auto rand_op = Symbol::fromQualString("foo::rand8"); + auto graph = std::make_shared(); + auto a = graph->addInput(); + auto b = graph->insert(rand_op, {a}); + AliasDb aliasDb(graph); + // The schema does not have an alias reference + EXPECT_FALSE(aliasDb.mayAlias(a, b)); +} + +TEST(AliasRegistrationTest, PureNoSchema) { + auto registry = torch::RegisterOperators().op( + "foo::rand9", + torch::RegisterOperators::options() + .catchAllKernel( + [](at::Tensor) -> at::Tensor { return at::rand({2, 2}); }) + .aliasAnalysis(AliasAnalysisKind::PURE_FUNCTION)); + const auto rand_op = Symbol::fromQualString("foo::rand9"); + auto graph = std::make_shared(); + auto a = graph->addInput(); + auto b = graph->insert(rand_op, {a}); + AliasDb aliasDb(graph); + // The schema is pure, there cannot be any alias + EXPECT_FALSE(aliasDb.mayAlias(a, b)); +} + +TEST(AliasRegistrationTest, PureWithSchema) { + auto registry = torch::RegisterOperators().op( + "foo::rand10(Tensor arg1) -> Tensor", + torch::RegisterOperators::options() + .catchAllKernel( + [](at::Tensor) -> at::Tensor { return at::rand({2, 2}); }) + .aliasAnalysis(AliasAnalysisKind::PURE_FUNCTION)); + const auto rand_op = Symbol::fromQualString("foo::rand10"); + auto graph = std::make_shared(); + auto a = graph->addInput(); + auto b = graph->insert(rand_op, {a}); + AliasDb aliasDb(graph); + // The schema is pure, there cannot be any alias + EXPECT_FALSE(aliasDb.mayAlias(a, b)); +} + +TEST(AliasRegistrationTest, PureWithAnnotationsShouldError) { + auto registry = torch::RegisterOperators().op( + "foo::rand11(Tensor(a) arg1) -> Tensor(a)", + torch::RegisterOperators::options() + .catchAllKernel([](at::Tensor t) -> at::Tensor { return t * 2; }) + .aliasAnalysis(AliasAnalysisKind::PURE_FUNCTION)); + const auto rand_op = Symbol::fromQualString("foo::rand11"); + auto graph = std::make_shared(); + auto a = graph->addInput(); + graph->insert(rand_op, {a}); + + // Registration time is okay, but throw exception when fetch from + // registration. + expectThrows( + [&graph] { AliasDb aliasDb(graph); }, + "Tried to register operator foo::rand11(Tensor(a) arg1) -> Tensor(a) with aliasing information in the schema but without AliasAnalysisKind::FROM_SCHEMA"); +} + +TEST(AliasRegistrationTest, AliasMoveAtenListOp) { + auto graph = std::make_shared(); + std::unordered_map vmap; + auto graph_string = R"IR( + graph(): + %x : Tensor = prim::MakeTestTensor() + %8 : int = prim::Constant[value=0]() + %5 : int = prim::Constant[value=1]() + %4 : int = prim::Constant[value=2]() + %y : Tensor[] = prim::ListConstruct(%x) + %6 : Tensor = aten::add_(%x, %4, %5) + %9 : Tensor = aten::cat(%y, %8) + return (%9))IR"; + + torch::jit::parseIR(graph_string, graph.get(), vmap); + AliasDb aliasDb(graph); + + // bc y.1 has a single used in a single non-aliasing aten op, + // x is added to y.1 contained elements instead of wildcard set + EXPECT_TRUE(!aliasDb.mayAlias(vmap["x"], vmap["9"])); + + // write to contained element should prevent move + EXPECT_TRUE(!aliasDb.moveBeforeTopologicallyValid( + vmap["y"]->node(), vmap["9"]->node())); +} + +TEST( + AliasRegistrationTest, + AliasMoveForTupleConstructWithSingleUseAsGraphOutput) { + auto graph = std::make_shared(); + std::unordered_map vmap; + auto graph_string = R"IR( + graph(): + %x : Tensor = prim::MakeTestTensor() + %y : Tensor = prim::MakeTestTensor() + %z : (Tensor) = prim::TupleConstruct(%x, %y) + return (%z))IR"; + + torch::jit::parseIR(graph_string, graph.get(), vmap); + AliasDb aliasDb(graph, /*isFrozen=*/false); + + EXPECT_TRUE(!aliasDb.mayAlias(vmap["x"], vmap["y"])); + EXPECT_TRUE(aliasDb.mayContainAlias(vmap["z"], vmap["x"])); + EXPECT_TRUE(aliasDb.mayContainAlias(vmap["z"], vmap["y"])); +} + +TEST(AliasRegistrationTest, RecursiveSubgraphTupleContainment) { + auto graph = std::make_shared(); + std::unordered_map vmap; + auto graph_string = R"IR( + graph(): + %x : Tensor = prim::MakeTestTensor() + %y : Tensor = prim::MakeTestTensor() + %z : (Tensor, Tensor) = prim::TupleConstruct(%x, %y) + return (%z))IR"; + + torch::jit::parseIR(graph_string, graph.get(), vmap); + auto node = vmap["z"]->node(); + auto subgraph = + SubgraphUtils::createSingletonSubgraph(node, prim::FunctionalGraph); + AliasDb aliasDb(graph); + + EXPECT_TRUE(aliasDb.mayContainAlias(subgraph->output(), vmap["x"])); + EXPECT_TRUE(aliasDb.mayContainAlias(subgraph->output(), vmap["y"])); + EXPECT_TRUE(aliasDb.mayAlias(vmap["x"], vmap["y"])); +} + +TEST(AliasRegistrationTest, WildcardAliasForTupleConstructWithUses) { + auto graph = std::make_shared(); + std::unordered_map vmap; + auto graph_string = R"IR( + graph(): + %x : Tensor = prim::MakeTestTensor() + %y : Tensor = prim::MakeTestTensor() + %z : Tensor = prim::MakeTestTensor() + %0 : int = prim::Constant[value=0]() + %a : (Tensor) = prim::TupleConstruct(%x, %y) + %b : (Tensor) = prim::TupleConstruct(%z) + %c : Tensor = prim::TupleIndex(%a, %0) + %d : Tensor = prim::TupleIndex(%b, %0) + return (%c, %d))IR"; + + torch::jit::parseIR(graph_string, graph.get(), vmap); + AliasDb aliasDb(graph, /*isFrozen=*/false); + + EXPECT_TRUE(aliasDb.mayAlias(vmap["x"], vmap["y"])); + EXPECT_TRUE(aliasDb.mayAlias(vmap["x"], vmap["z"])); + EXPECT_TRUE(aliasDb.mayAlias(vmap["y"], vmap["z"])); + EXPECT_TRUE(aliasDb.mayContainAlias(vmap["a"], vmap["x"])); + EXPECT_TRUE(aliasDb.mayContainAlias(vmap["a"], vmap["y"])); + EXPECT_TRUE(aliasDb.mayContainAlias(vmap["a"], vmap["z"])); + EXPECT_TRUE(aliasDb.mayContainAlias(vmap["b"], vmap["x"])); + EXPECT_TRUE(aliasDb.mayContainAlias(vmap["b"], vmap["y"])); + EXPECT_TRUE(aliasDb.mayContainAlias(vmap["b"], vmap["z"])); +} + +TEST(AliasRegistrationTest, ATenSplitIntListAliasCheck) { + auto graph = std::make_shared(); + std::unordered_map vmap; + auto graph_string = R"IR( + graph(): + %x : Tensor = prim::MakeTestTensor() + %0 : int = prim::Constant[value=0]() + %1 : int = prim::Constant[value=1]() + %2 : int = prim::Constant[value=2]() + %y : Tensor = aten::add(%x, %x, %0) + %lengths_list : int[] = prim::tolist(%1, %2) + %a : Tensor[] = aten::split(%y, %lengths_list, %0) + %b : Tensor, %c : Tensor = prim::ListUnpack(%a) + %b1 : Tensor = aten::flatten(%b, %0, %1) + %c1 : Tensor = aten::flatten(%c, %0, %1) + %d : Tensor = aten::add(%b1, %c1, %0) + return (%d))IR"; + + torch::jit::parseIR(graph_string, graph.get(), vmap); + AliasDb aliasDb(graph, /*isFrozen=*/false); + + EXPECT_TRUE(aliasDb.mayAlias(vmap["y"], vmap["b"])); + EXPECT_TRUE(aliasDb.mayAlias(vmap["y"], vmap["c"])); + EXPECT_TRUE(aliasDb.mayAlias(vmap["y"], vmap["b1"])); + EXPECT_TRUE(aliasDb.mayAlias(vmap["y"], vmap["c1"])); +} + +TEST(AliasRegistrationTest, ATenSplitIntAliasCheck) { + auto graph = std::make_shared(); + std::unordered_map vmap; + auto graph_string = R"IR( + graph(): + %x : Tensor = prim::MakeTestTensor() + %0 : int = prim::Constant[value=0]() + %1 : int = prim::Constant[value=1]() + %2 : int = prim::Constant[value=2]() + %y : Tensor = aten::add(%x, %x, %0) + %a : Tensor[] = aten::split(%y, %2, %0) + %b : Tensor, %c : Tensor = prim::ListUnpack(%a) + %b1 : Tensor = aten::flatten(%b, %0, %1) + %c1 : Tensor = aten::flatten(%c, %0, %1) + %d : Tensor = aten::add(%b1, %c1, %0) + return (%d))IR"; + + torch::jit::parseIR(graph_string, graph.get(), vmap); + AliasDb aliasDb(graph, /*isFrozen=*/false); + + EXPECT_TRUE(aliasDb.mayAlias(vmap["y"], vmap["b"])); + EXPECT_TRUE(aliasDb.mayAlias(vmap["y"], vmap["c"])); + EXPECT_TRUE(aliasDb.mayAlias(vmap["y"], vmap["b1"])); + EXPECT_TRUE(aliasDb.mayAlias(vmap["y"], vmap["c1"])); +} + +TEST(AliasRegistrationTest, PureWithAnnotationsShouldError2) { + auto registry = torch::RegisterOperators().op( + "foo::rand12(Tensor(a) arg1) -> Tensor(b)", + torch::RegisterOperators::options() + .catchAllKernel([](at::Tensor t) -> at::Tensor { return t * 2; }) + .aliasAnalysis(AliasAnalysisKind::PURE_FUNCTION)); + const auto rand_op = Symbol::fromQualString("foo::rand12"); + auto graph = std::make_shared(); + auto a = graph->addInput(); + graph->insert(rand_op, {a}); + + // Registration time is okay, but throw exception when fetch from + // registration. + expectThrows( + [&graph] { AliasDb aliasDb(graph); }, + "Tried to register operator foo::rand12(Tensor(a) arg1) -> Tensor(b) with aliasing information in the schema but without AliasAnalysisKind::FROM_SCHEMA"); +} + +TEST(IRNonDeterminismTest, Basic) { + auto graph = std::make_shared(); + auto graph_string = R"IR( + graph(): + %x : Tensor = prim::MakeTestTensor() + %0 : int = prim::Constant[value=0]() + %1 : NoneType = prim::Constant() + %2 : Tensor = aten::bernoulli(%x, %1) + %3 : Tensor = aten::add(%x, %2, %0) + return (%3))IR"; + parseIR(graph_string, graph.get()); + + for (Node* n : graph->nodes()) { + if (n->kind() == aten::bernoulli) { + ASSERT_TRUE(n->isNondeterministic()); + } else { + ASSERT_FALSE(n->isNondeterministic()); + } + } +} + +TEST(IRNonDeterminismTest, DropoutSpecialCase) { + auto graph = std::make_shared(); + auto graph_string = R"IR( + graph(): + %x : Tensor = prim::MakeTestTensor() + %0 : bool = prim::Constant[value=0]() + %1 : bool = prim::Constant[value=1]() + %3 : int = prim::Constant[value=1]() + %3 : float = prim::Constant[value=1.0]() + %4 : Tensor = aten::dropout(%x, %3, %0) + %5 : Tensor = aten::dropout(%x, %3, %1) + %6 : Tensor = aten::add(%4, %5, %3) + return (%6))IR"; + parseIR(graph_string, graph.get()); + + bool train = false; + for (Node* n : graph->nodes()) { + if (n->kind() == aten::dropout) { + if (!train) { + ASSERT_FALSE(n->isNondeterministic()); + train = true; + } else { + ASSERT_TRUE(n->isNondeterministic()); + } + } else { + ASSERT_FALSE(n->isNondeterministic()); + } + } +} + +TEST(NonDeterminismBackwardsCompatibility, BackwardsCompatibility) { + static const std::vector nondeterministic_ops = { + "aten::dropout(Tensor input, float p, bool train) -> Tensor", + "aten::_fused_dropout(Tensor self, float p, Generator? generator) -> (Tensor, Tensor)", + "aten::_standard_gamma(Tensor self, Generator? generator) -> Tensor", + "aten::bernoulli(Tensor self, *, Generator? generator) -> Tensor", + "aten::bernoulli(Tensor self, float p, *, Generator? generator) -> Tensor", + "aten::multinomial(Tensor self, int num_samples, bool replacement, *, Generator? generator) -> Tensor", + "aten::native_dropout(Tensor input, float p, bool? train) -> (Tensor, Tensor)", + "aten::normal.Tensor_Tensor(Tensor mean, Tensor std, *, Generator? generator) -> Tensor", + "aten::normal.float_Tensor(float mean, Tensor std, *, Generator? generator) -> Tensor", + "aten::normal.Tensor_float(Tensor mean, float std, *, Generator? generator) -> Tensor", + "aten::poisson(Tensor self, Generator? generator) -> Tensor", + "aten::binomial(Tensor count, Tensor prob, Generator? generator=None) -> Tensor", + "aten::rrelu(Tensor self, Scalar lower, Scalar upper, bool training, Generator? generator) -> Tensor", + "aten::rrelu_with_noise(Tensor self, Tensor noise, Scalar lower, Scalar upper, bool training, Generator? generator) -> Tensor", + "aten::rand(int[] size, *, int? dtype, int? layout, Device? device, bool? pin_memory) -> Tensor", + "aten::rand_like(Tensor self, *, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor", + "aten::rand_like.generator(Tensor self, *, Generator? generator, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor", + "aten::randint(int high, int[] size, *, int? dtype, int? layout, Device? device, bool? pin_memory) -> Tensor", + "aten::randint(int low, int high, int[] size, *, int? dtype, int? layout, Device? device, bool? pin_memory) -> Tensor", + "aten::randint_like(Tensor self, int high, *, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor", + "aten::randint_like.generator(Tensor self, int high, *, Generator? generator, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor", + "aten::randint_like.Tensor(Tensor self, Tensor high, *, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor", + "aten::randint_like.Tensor_generator(Tensor self, Tensor high, *, Generator? generator, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor", + "aten::randint_like.low_dtype(Tensor self, int low, int high, *, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor", + "aten::randint_like.low_generator_dtype(Tensor self, int low, int high, *, Generator? generator, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor", + "aten::randn(int[] size, *, int? dtype, int? layout, Device? device, bool? pin_memory) -> Tensor", + "aten::randn_like(Tensor self, *, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor", + "aten::randn_like.generator(Tensor self, *, Generator? generator, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor", + "aten::randperm(int n, *, int? dtype, int? layout, Device? device, bool? pin_memory) -> Tensor"}; + for (const std::string& op : nondeterministic_ops) { + const c10::FunctionSchema& schema = torch::jit::parseSchema(op); + const auto& op_handle = c10::Dispatcher::singleton().findOp( + c10::OperatorName(schema.name(), schema.overload_name())); + ASSERT_TRUE(op_handle->hasTag(at::Tag::nondeterministic_seeded)); + } +} + +TEST(TypeHashing, HashTypes) { + HashType hasher; + + const TypePtr int_type = IntType::get(); + const TypePtr float_type = FloatType::get(); + ASSERT_NE(hasher(int_type), hasher(float_type)); + + const TypePtr int2_type = TupleType::create({int_type, int_type}); + const TypePtr int3_type = TupleType::create({int_type, int_type, int_type}); + ASSERT_NE(hasher(int2_type), hasher(int3_type)); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_argument_spec.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_argument_spec.cpp new file mode 100644 index 0000000000000000000000000000000000000000..112b96ff5fc406df81d13803e7da7d4519f03c71 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_argument_spec.cpp @@ -0,0 +1,201 @@ +#include + +#include +#include +#include +#include + +#include "test/cpp/jit/test_utils.h" + +namespace torch { +namespace jit { + +namespace { + +at::Device device(const autograd::Variable& v) { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + return v.device(); +} + +bool isEqual(at::IntArrayRef lhs, at::IntArrayRef rhs) { + return lhs.size() == rhs.size() && + std::equal(lhs.begin(), lhs.end(), rhs.begin()); +} + +bool isEqual(const CompleteArgumentInfo& ti, const autograd::Variable& v) { + if (!ti.defined()) + return ti.defined() == v.defined(); + return ti.device() == device(v) && ti.requires_grad() == v.requires_grad() && + ti.type() == v.scalar_type() && isEqual(ti.sizes(), v.sizes()) && + isEqual(ti.strides(), v.strides()); +} + +bool isEqual(const ArgumentInfo& ti, const autograd::Variable& v) { + if (!ti.defined()) + return ti.defined() == v.defined(); + return ti.device() == device(v) && ti.requires_grad() == v.requires_grad() && + ti.type() == v.scalar_type() && ti.dim() == v.dim(); +} + +autograd::Variable var( + at::TensorOptions t, + at::IntArrayRef sizes, + bool requires_grad) { + return autograd::make_variable(at::rand(sizes, t), requires_grad); +} +autograd::Variable undef() { + return autograd::Variable(); +} +} // namespace + +TEST(ArgumentSpecTest, CompleteArgumentSpec_CUDA) { + auto const CF = at::CPU(at::kFloat); + auto const CD = at::CPU(at::kDouble); + auto const GF = at::CUDA(at::kFloat); + auto const GD = at::CUDA(at::kDouble); + + auto list = createStack( + {var(CF, {1}, true), + var(CD, {1, 2}, false), + var(GF, {}, true), + var(GD, {4, 5, 6}, false), + undef()}); + + // make sure we have some non-standard strides + list[1].toTensor().transpose_(0, 1); + + // same list but different backing values + auto list2 = createStack( + {var(CF, {1}, true), + var(CD, {1, 2}, false), + var(GF, {}, true), + var(GD, {4, 5, 6}, false), + undef()}); + list2[1].toTensor().transpose_(0, 1); + + CompleteArgumentSpec a(true, list); + CompleteArgumentSpec b(true, list); + ASSERT_EQ(a.hashCode(), b.hashCode()); + + ASSERT_EQ(a, b); + CompleteArgumentSpec d(true, list2); + ASSERT_EQ(d, a); + ASSERT_EQ(d.hashCode(), a.hashCode()); + + for (size_t i = 0; i < list.size(); ++i) { + ASSERT_TRUE(isEqual(a.at(i), list[i].toTensor())); + } + CompleteArgumentSpec no_grad(/*with_grad=*/false, list); + ASSERT_TRUE(no_grad != a); + + std::unordered_set spec; + spec.insert(a); // we use a below, so no move + ASSERT_TRUE(spec.count(b) > 0); + ASSERT_EQ(spec.count(no_grad), 0); + spec.insert(std::move(no_grad)); + ASSERT_EQ(spec.count(CompleteArgumentSpec(true, list)), 1); + + list2[1].toTensor().transpose_(0, 1); + CompleteArgumentSpec c(true, list2); // same as list, except for one stride + ASSERT_FALSE(c == a); + ASSERT_EQ(spec.count(c), 0); + + Stack stack = {var(CF, {1, 2}, true), 3, var(CF, {1, 2}, true)}; + CompleteArgumentSpec with_const(true, stack); + ASSERT_EQ(with_const.at(2).sizes().size(), 2); +} + +// TODO: this test was disabled for unknown reasons and doesn't run. +// static size_t hashCode(const TensorTypePtr& ptr) { +// return std::hash()(*ptr.get()); +// } + +// TEST(ArgumentSpecTest, VaryingShape) { +// c10::VaryingShape vs(std::optional{}); +// auto ptt_empty1 = TensorType::create({}, {}, vs, vs, false); +// auto ptt_empty2 = TensorType::create({}, {}, vs, vs, false); +// ASSERT_EQ(hashCode(ptt_empty1), hashCode(ptt_empty2)); + +// c10::VaryingShape vs22(std::vector{2, 2}); +// auto ptt_vs22_vs22_1 = TensorType::create({}, {}, vs22, vs22, false); +// auto ptt_vs22_vs22_2 = TensorType::create({}, {}, vs22, vs22, false); +// ASSERT_EQ(hashCode(ptt_vs22_vs22_1), hashCode(ptt_vs22_vs22_2)); + +// c10::VaryingShape vs23(std::vector{2, 3}); +// auto ptt_vs22_vs23_2 = TensorType::create({}, {}, vs22, vs23, false); +// ASSERT_NE(hashCode(ptt_vs22_vs22_1), hashCode(ptt_vs22_vs23_2)); + +// auto ptt_vs22_vs22_1_true = TensorType::create({}, {}, vs22, vs22, true); +// auto ptt_vs22_vs22_2_true = TensorType::create({}, {}, vs22, vs22, true); +// ASSERT_EQ(hashCode(ptt_vs22_vs22_1_true), hashCode(ptt_vs22_vs22_2_true)); + +// auto ptt_vs22_vs22_1_false = TensorType::create({}, {}, vs22, vs22, false); +// ASSERT_NE(hashCode(ptt_vs22_vs22_1_true), hashCode(ptt_vs22_vs22_1_false)); +// } + +TEST(ArgumentSpecTest, Basic_CUDA) { + auto& CF = at::CPU(at::kFloat); + auto& CD = at::CPU(at::kDouble); + auto& GF = at::CUDA(at::kFloat); + auto& GD = at::CUDA(at::kDouble); + + auto graph = toGraphFunction(jit::compile(R"JIT( + def fn(a, b, c, d, e): + return a, b, c, d, e + )JIT") + ->get_function("fn")) + .graph(); + + ArgumentSpecCreator arg_spec_creator(*graph); + + auto list = createStack( + {var(CF, {1}, true), + var(CD, {1, 2}, false), + var(GF, {}, true), + var(GD, {4, 5, 6}, false), + undef()}); + + // make sure we have some non-standard strides + list[1].toTensor().transpose_(0, 1); + + // same list but different backing values + auto list2 = createStack( + {var(CF, {1}, true), + var(CD, {1, 2}, false), + var(GF, {}, true), + var(GD, {4, 5, 6}, false), + undef()}); + list2[1].toTensor().transpose_(0, 1); + + ArgumentSpec a = arg_spec_creator.create(true, list); + ArgumentSpec b = arg_spec_creator.create(true, list); + ASSERT_EQ(a.hashCode(), b.hashCode()); + + ASSERT_EQ(a, b); + ArgumentSpec d = arg_spec_creator.create(true, list2); + ASSERT_EQ(d, a); + ASSERT_EQ(d.hashCode(), a.hashCode()); + + for (size_t i = 0; i < list.size(); ++i) { + ASSERT_TRUE(isEqual(a.tensorAt(i), list[i].toTensor())); + } + ArgumentSpec no_grad = arg_spec_creator.create(/*with_grad=*/false, list); + ASSERT_TRUE(no_grad != a); + + std::unordered_set spec; + spec.insert(a); // we still need a for the test below + ASSERT_TRUE(spec.count(b) > 0); + ASSERT_EQ(spec.count(no_grad), 0); + spec.insert(std::move(no_grad)); + ASSERT_EQ(spec.count(arg_spec_creator.create(true, list)), 1); + + list2[1].toTensor().transpose_(0, 1); + ArgumentSpec c = arg_spec_creator.create( + true, list2); // same as list, except for one stride, used to be + // different, now the same + ASSERT_TRUE(c == a); + ASSERT_EQ(spec.count(c), 1); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_autodiff.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_autodiff.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6266e707f1f4b7cbe9b196e9ca0df5aa3f263db4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_autodiff.cpp @@ -0,0 +1,343 @@ +#include + +#include "test/cpp/jit/test_utils.h" +#include "torch/csrc/jit/frontend/tracer.h" +#include "torch/csrc/jit/passes/common_subexpression_elimination.h" +#include "torch/csrc/jit/passes/constant_propagation.h" +#include "torch/csrc/jit/passes/create_autodiff_subgraphs.h" +#include "torch/csrc/jit/passes/dead_code_elimination.h" +#include "torch/csrc/jit/passes/graph_fuser.h" +#include "torch/csrc/jit/passes/lower_grad_of.h" +#include "torch/csrc/jit/passes/requires_grad_analysis.h" +#include "torch/csrc/jit/passes/shape_analysis.h" +#include "torch/csrc/jit/passes/utils/subgraph_utils.h" +#include "torch/csrc/jit/runtime/argument_spec.h" +#include "torch/csrc/jit/runtime/autodiff.h" +#include "torch/csrc/jit/runtime/graph_iterator.h" +#include "torch/csrc/jit/runtime/profiling_graph_executor_impl.h" +#include "torch/torch.h" + +#include +#include "torch/csrc/autograd/engine.h" +#include "torch/csrc/autograd/generated/variable_factories.h" +#include "torch/csrc/autograd/variable.h" + +namespace torch { +namespace jit { + +using namespace torch::autograd; + +using var_meta_type = std::vector; +using var_meta_list = std::vector; +using test_fn_type = std::function; + +struct ADTestSpec { + ADTestSpec( + const char* name, + // NOLINTNEXTLINE(modernize-pass-by-value) + var_meta_list input_meta, + // NOLINTNEXTLINE(modernize-pass-by-value) + test_fn_type test_fn, + float clampMax = -1.0f) + : name(name), + input_meta(input_meta), + test_fn(test_fn), + clampMax(clampMax) {} + + variable_list operator()(const variable_list& inputs) const { + return test_fn(inputs); + }; + + std::vector make_vars() const { + std::vector out; + for (const auto& m : input_meta) { + if (clampMax > 0.0f) { + out.push_back(torch::randn(m, at::requires_grad(true)) + .clamp(-clampMax, clampMax)); + continue; + } + out.push_back(torch::randn(m, at::requires_grad(true))); + } + return out; + } + + const char* name; + var_meta_list input_meta; + test_fn_type test_fn; + float clampMax; +}; + +variable_list get_grad_outputs(const variable_list& vars) { + return fmap(vars, [](const Variable& v) -> Variable { + return at::randn(v.sizes(), v.options()); + }); +} + +variable_list grad( + const variable_list& outputs, + const variable_list& inputs, + const variable_list& grad_outputs) { + const auto get_edge = [](const Variable& v) { + return torch::autograd::impl::gradient_edge(v); + }; + auto& engine = torch::autograd::Engine::get_default_engine(); + return engine.execute( + fmap(outputs, get_edge), + grad_outputs, + true, + false, + false, + fmap(inputs, get_edge)); +} + +TEST(AutodiffTest, ADFormulas) { + const auto cast = [](const Variable& v) { + return static_cast(v); + }; + + using VL = variable_list; + const var_meta_list binary_pointwise = {{2, 3, 4, 5}, {2, 3, 4, 5}}; + const var_meta_list unary_pointwise = {{2, 3, 4, 5}}; + const var_meta_list unary_pointwise_2d = {{2, 3}}; + const std::vector ad_tests = { + {"add", + binary_pointwise, + [](const VL& v) -> VL { return {v[0] + v[1]}; }}, + {"sub", + binary_pointwise, + [](const VL& v) -> VL { return {v[0] - v[1]}; }}, + {"mul", + binary_pointwise, + [](const VL& v) -> VL { return {v[0] * v[1]}; }}, + {"sigmoid", + unary_pointwise, + [](const VL& v) -> VL { return {v[0].sigmoid()}; }}, + // Clamp tanh input tensor values to [-3, 3] + // to set a minimum on gradient absolute values + {"tanh", + unary_pointwise, + [](const VL& v) -> VL { return {v[0].tanh()}; }, + 3.0f}, + {"t", unary_pointwise_2d, [](const VL& v) -> VL { return {v[0].t()}; }}, + {"view", + unary_pointwise_2d, + [](const VL& v) -> VL { return {v[0].view({3, 2})}; }}, + {"expand", + {{2, 1}}, + [](const VL& v) -> VL { return {v[0].expand({2, 3})}; }}, + {"mm", + {{10, 12}, {12, 15}}, + [](const VL& v) -> VL { return {v[0].mm(v[1])}; }}, + // TODO: enable once we'll be able to capture lists across + // forward-backward + //{"chunk", {{10, 12, 15}}, [](const VL& v) -> VL { return + // fmap(v[0].chunk(4, 1)); }}, + //{"chunk", {{10, 12, 15}}, [](const VL& v) -> VL { return + // fmap(v[0].chunk(3, 2)); }}, + //{"split", {{10, 12, 15}}, [](const VL& v) -> VL { return + // fmap(v[0].split(4, 1)); }}, + //{"split", {{10, 12, 15}}, [](const VL& v) -> VL { return + // fmap(v[0].split(3, 2)); }}, + }; + + for (const auto& test : ad_tests) { + // Get reference values form autograd + auto vars_in = test.make_vars(); + auto vars_out = test(vars_in); + auto var_grads_in = get_grad_outputs(vars_out); + auto var_grads_out = grad(vars_out, vars_in, var_grads_in); + + // Trace and differentiate the op + auto graph = tracer::trace( + fmap(vars_in), + [&test](Stack in) -> Stack { + auto ivalue_inps = fmap(in, [](const IValue& v) { + return Variable(v.toTensor()); + }); + return fmap(test(ivalue_inps)); + }, + [](const Variable& var) { return ""; }) + .first->graph; + EliminateDeadCode(graph); // Tracing of some ops depends on the DCE trick + ConstantPropagation(graph); + auto grad_spec = differentiate(graph); + LowerGradOf(*grad_spec.df); + // Get outputs from the interpreter + auto tensors_in = fmap(vars_in, cast); + auto tensor_grads_in = fmap(var_grads_in, cast); + auto [tensors_out, tensor_grads_out] = + runGradient(grad_spec, tensors_in, tensor_grads_in); + + // Compare results + auto expected_tensors_out = fmap(vars_out, cast); + auto expected_tensor_grads_out = fmap(var_grads_out, cast); + assertAllClose(tensors_out, expected_tensors_out); + assertAllClose(tensor_grads_out, expected_tensor_grads_out); + } +} + +TEST(AutodiffTest, Differentiate) { + // Note: can't use IRParser for this test due to issue #23989 + auto graph = std::make_shared(); + std::vector sizes{2, 3, 4}; + std::vector strides{12, 4, 1}; + const auto type = TensorType::create( + at::ScalarType::Float, + at::kCPU, + c10::VaryingShape{sizes}, + c10::VaryingShape{strides}, + true); + + // Builds graph a * b * a + b + auto* a = graph->addInput()->setType(type); + auto* b = graph->addInput()->setType(type); + auto* cOne = graph->insertConstant(1); + + auto* ab = graph->insertNode(graph->create(aten::mul, /*num_outputs =*/1)); + ab->addInput(a); + ab->addInput(b); + + auto* aba = graph->insertNode(graph->create(aten::mul, /*num_outputs =*/1)); + aba->addInput(ab->output()); + aba->addInput(a); + + auto* abaplusb = + graph->insertNode(graph->create(aten::add, /*num_outputs =*/1)); + abaplusb->addInput(aba->output()); + abaplusb->addInput(b); + abaplusb->addInput(cOne); + + graph->registerOutput(abaplusb->output()); + + auto grad_spec = differentiate(graph); + std::vector expected_captured_inputs = {0, 1}; + std::vector expected_captured_outputs = {1, 2, 3, 4, 5, 6, 7}; + std::vector expected_input_vjps = {0, 1}; + std::vector expected_output_vjps = {0, 1}; + ASSERT_EQ(grad_spec.f_real_outputs, 1); + ASSERT_EQ(grad_spec.df_input_captured_inputs, expected_captured_inputs); + ASSERT_EQ(grad_spec.df_input_captured_outputs, expected_captured_outputs); + ASSERT_EQ(grad_spec.df_input_vjps, expected_input_vjps); + ASSERT_EQ(grad_spec.df_output_vjps, expected_output_vjps); + testing::FileCheck() + .check_count("aten::mul", 2) + ->check("aten::size") + ->check("aten::add") + ->run(*grad_spec.f); + testing::FileCheck() + .check("prim::GradOf[name=\"aten::add\"]") + ->check_count("prim::GradOf[name=\"aten::mul\"]", 2) + ->check_count("AutogradAdd", 2) + ->run(*grad_spec.df); +} + +TEST(AutodiffTest, DifferentiateWithRequiresGrad) { + const auto graph_string = R"IR( + graph(%0 : Tensor, + %1 : Tensor): + %2 : int = prim::Constant[value=1]() + %3 : Tensor = aten::mul(%1, %1) + %4 : Tensor = aten::add(%3, %1, %2) + %5 : Tensor = aten::add(%4, %0, %2) + %6 : Tensor = aten::mul(%5, %0) + %7 : Tensor = aten::add(%6, %1, %2) + return (%4, %7))IR"; + auto g = std::make_shared(); + torch::jit::parseIR(graph_string, g.get()); + + auto a_var = autograd::make_variable( + at::empty_strided(2, 2, at::CPU(at::kFloat).options()), true); + auto b_var = autograd::make_variable( + at::empty_strided(2, 2, at::CPU(at::kFloat).options()), false); + + ArgumentSpecCreator asc(*g); + asc.specializeTypes(*g, asc.create(true, {a_var, b_var})); + + PropagateInputShapes(g); + PropagateRequiresGrad(g); + + auto grad_spec = differentiate(g); + std::vector expected_input_vjps = {1, 2}; // for e and %4 = (d + a) + std::vector expected_output_vjps = {0}; // only a requires grad + ASSERT_EQ(grad_spec.f_real_outputs, 2); + ASSERT_EQ(grad_spec.df_input_captured_inputs, std::vector({0})); + ASSERT_EQ( + grad_spec.df_input_captured_outputs, + std::vector({2, 3, 4, 5, 6})); + ASSERT_EQ(grad_spec.df_input_vjps, expected_input_vjps); + ASSERT_EQ(grad_spec.df_output_vjps, expected_output_vjps); + testing::FileCheck() + .check("aten::mul") + ->check_count("aten::add", 2) + ->check("aten::mul") + ->check("aten::size") + ->check("aten::add") + ->run(*grad_spec.f); + + testing::FileCheck() + .check_count("prim::GradOf[name=\"aten::mul\"]", 1, /*exactly*/ true) + ->run(*grad_spec.df); +} + +class AutodiffRemoveUnusedGradientsTest : public ::testing::Test { + protected: + void SetUp() override { + prev_exec = getExecutorMode(); + getExecutorMode() = true; + prev_inline_autodiff = getAutodiffSubgraphInlining(); + debugSetAutodiffSubgraphInlining(false); + } + void TearDown() override { + getExecutorMode() = prev_exec; + debugSetAutodiffSubgraphInlining(prev_inline_autodiff); + } + + bool prev_exec; + bool prev_profiling; + bool prev_inline_autodiff; +}; + +TEST_F(AutodiffRemoveUnusedGradientsTest, Linear) { + auto graph = std::make_shared(); + const std::string input = + R"IR( +graph(%inp.1 : Tensor, + %weight.1 : Tensor, + %bias.1 : Tensor): + %6 : Tensor = aten::linear(%inp.1, %weight.1, %bias.1) + return (%6))IR"; + parseIR(input, graph.get()); + + auto inp = torch::randn({10, 10}).requires_grad_(false); + auto weight = torch::randn({10, 10}).requires_grad_(true); + auto bias = torch::randn({1, 10}).requires_grad_(true); + auto stack = createStack({inp, weight, bias}); + + ProfilingGraphExecutorImpl executor(graph, "linear"); + + // initial run to profile requires_grad information + auto plan = executor.getPlanFor(stack, 20); + InterpreterState is{plan.code}; + is.run(stack); + + auto optimized_plan = executor.getPlanFor(stack, 20); + DepthFirstGraphNodeIterator it(optimized_plan.graph); + Node* diff_graph_node = nullptr; + + while ((diff_graph_node = it.next()) != nullptr) { + if (diff_graph_node->kind() == prim::DifferentiableGraph) { + break; + } + } + ASSERT_NE(nullptr, diff_graph_node); + + auto backward_graph = diff_graph_node->g(attr::ReverseSubgraph); + + // we expect to compute grad_weight (which requires a matmul) but we don't + // expect to compute grad_input. So, we expect exactly 1 matmul. + // Note: this could change, e.g. if mm is used instead + testing::FileCheck().check_count("matmul", 1, true)->run(*backward_graph); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_backend.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_backend.cpp new file mode 100644 index 0000000000000000000000000000000000000000..08c29489ad22073e3728f51945b717eead0f2618 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_backend.cpp @@ -0,0 +1,861 @@ +#include +#include +#include +#include +#include +#include +#include + +// Tests go in torch::jit +namespace torch { +namespace jit { +TEST(BackendTest, ToBackend) { + Module m("m"); + m.define(R"( + def forward(self, x, h): + return self.accum(x, h), self.sub_accum(x, h) + + def accum(self, x, h): + return x + h + + def sub_accum(self, x, h): + return x - h + )"); + + std::vector inputs; + inputs.emplace_back(2.0 * torch::ones({})); + inputs.emplace_back(1.0 * torch::ones({})); + auto ref = m.forward(inputs).toTupleRef().elements().vec(); + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + // lowered module + auto lm = torch::jit::detail::codegen_backend_module( + "test_backend", m, compile_spec, any_dict_ty); + // lowered module code: + /* + class test_backendLoweredModule(Module): + __parameters__ = [] + __buffers__ = [] + __processed_module : Any + __method_compile_spec : Dict[str, Any] + __backend : __torch__.torch.classes.__backends__.test_backend + __handles : Dict[str, Any] + def __create_backend(self: torch.jit.test_backendLoweredModule) -> None: + _0 = + __torch__.torch.classes.__backends__.test_backend.__new__(__torch__.torch.classes.__backends__.test_backend) + _1 = (_0).__init__() + self.__backend = _0 + return None + def __getstate__(self: torch.jit.test_backendLoweredModule) -> + Tuple[Dict[str, Any], Any]: _2 = (self.__method_compile_spec, + self.__processed_module) return _2 def __setstate__(self: + torch.jit.test_backendLoweredModule, state: Tuple[Dict[str, Any], Any]) -> + None: self.__method_compile_spec = (state)[0] self.__processed_module = + (state)[1] _3 = (self).__create_backend() _4 = + (self.__backend).compile(self.__processed_module, + self.__method_compile_spec, ) self.__handles = _4 return None def + forward(self: torch.jit.test_backendLoweredModule, x: Tensor, h: Tensor) -> + Tuple[Tensor, Tensor]: _5 = uninitialized(Tensor) typed_inputs = + annotate(List[Any], [x, h]) _6 = + (self.__backend).execute((self.__handles)["forward"], typed_inputs, ) _7, + _8, = _6 _9 = isinstance(_7, Tensor) if _9: _10 = unchecked_cast(Tensor, _7) + else: + ops.prim.RaiseException("AssertionError: ") + _10 = _5 + _11 = isinstance(_8, Tensor) + if _11: + _12 = unchecked_cast(Tensor, _8) + else: + ops.prim.RaiseException("AssertionError: ") + _12 = _5 + return (_10, _12) + + */ + auto res = lm.forward(inputs).toTupleRef().elements().vec(); + AT_ASSERT(res[0].toTensor().equal(ref[0].toTensor())); + AT_ASSERT(res[1].toTensor().equal(ref[1].toTensor())); +} + +TEST(BackendTest, ToBackendNotAvailable) { + Module m("m"); + m.define(R"( + def forward(self, x, h): + return self.accum(x, h), self.sub_accum(x, h) + + def accum(self, x, h): + return x + h + + def sub_accum(self, x, h): + return x - h + )"); + + std::vector inputs; + inputs.emplace_back(2.0 * torch::ones({})); + inputs.emplace_back(1.0 * torch::ones({})); + auto ref = m.forward(inputs).toTupleRef().elements().vec(); + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + // Produce lowered module (backend not available). + // Exception is not thrown at this point. + auto lm = torch::jit::detail::codegen_backend_module( + "test_backend_unavailable", m, compile_spec, any_dict_ty); + // Validate exception is thrown when trying to execute and + // the backend is not available. + ASSERT_THROWS_WITH_MESSAGE( + lm.forward(inputs).toTupleRef().elements(), "Backend is not available."); +} + +TEST(BackendTest, TestCompiler) { + Module m("m"); + m.define(R"( + def forward(self, x, h): + return x + h + )"); + + std::vector inputs; + inputs.emplace_back(2.0 * torch::ones({})); + inputs.emplace_back(1.0 * torch::ones({})); + auto ref = m.forward(inputs); + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + // lowered module + auto lm = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", m, compile_spec, any_dict_ty); + auto res = lm.forward(inputs); + AT_ASSERT(res.toTensor().equal(ref.toTensor())); + + std::stringstream ss; + lm._save_for_mobile(ss); + auto mlm = _load_for_mobile(ss); + auto mres = mlm.forward(inputs); + AT_ASSERT(mres.toTensor().equal(ref.toTensor())); +} + +TEST(BackendTest, TestCompilerWithStringTable) { + setShouldUseFormatWithStringTable(true); + Module m("m"); + m.define(R"( + def forward(self, x, h): + return x + h + )"); + + std::vector inputs; + inputs.emplace_back(2.0 * torch::ones({})); + inputs.emplace_back(1.0 * torch::ones({})); + auto ref = m.forward(inputs); + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + // lowered module + auto lm = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", m, compile_spec, any_dict_ty); + auto res = lm.forward(inputs); + AT_ASSERT(res.toTensor().equal(ref.toTensor())); + + std::stringstream ss; + lm._save_for_mobile(ss); + auto mlm = _load_for_mobile(ss); + auto mres = mlm.forward(inputs); + setShouldUseFormatWithStringTable(false); + AT_ASSERT(mres.toTensor().equal(ref.toTensor())); +} + +TEST(BackendTest, TestComposite) { + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + + Module m_add("m_add"); + m_add.define(R"( + def forward(self, x, y): + return x + y + )"); + auto lm_add = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", m_add, compile_spec, any_dict_ty); + + Module m_sub("m_sub"); + m_sub.define(R"( + def forward(self, x, y): + return x - y + )"); + auto lm_sub = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", m_sub, compile_spec, any_dict_ty); + + Module c("C"); + c.register_module("Add", lm_add); + c.register_module("Sub", lm_sub); + c.define(R"( + def forward(self, x, y): + return self.Add.forward(x, y) * self.Sub.forward(x, y) + )"); + + std::vector inputs; + inputs.emplace_back(3.0 * torch::ones({})); + inputs.emplace_back(1.0 * torch::ones({})); + auto res_jit = c.forward(inputs); + + std::stringstream ss; + c._save_for_mobile(ss); + auto mc = _load_for_mobile(ss); + auto res_mobile = mc.forward(inputs); + + AT_ASSERT(res_jit.toTensor().equal(res_mobile.toTensor())); +} + +TEST(BackendTest, TestPrimDtype) { + Module c("name"); + c.define(R"( + def forward(self, x, y): + c = y.dtype + return c + )"); + + std::vector inputs; + inputs.emplace_back(3.0 * torch::ones({})); + inputs.emplace_back(1.0 * torch::ones({})); + auto res_jit = c.forward(inputs); + + std::stringstream ss; + c._save_for_mobile(ss); + auto mc = _load_for_mobile(ss); + auto res_mobile = mc.forward(inputs); + + ASSERT_EQ(res_jit.toInt(), res_mobile.toInt()); +} + +Module getCompositeModuleWithSameNameSubModules() { + // Two submodules with same module name but different forward and other + // functions should be serialized and loaded correctly. + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + + Module sub1("m_add"); + sub1.define(R"( + def forward(self, x, y): + return x + y + )"); + auto lowered_sub1 = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", sub1, compile_spec, any_dict_ty); + + Module sub2("m_add"); + sub2.define(R"( + def forward(self, x, y): + return x - y + )"); + auto lowered_sub2 = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", sub2, compile_spec, any_dict_ty); + + Module c("C"); + c.register_module("Add", lowered_sub1); + c.register_module("Sub", lowered_sub2); + c.define(R"( + def forward(self, a, b, s:int): + c = self.Add.forward(a, b) + d = self.Sub.forward(a, b) + y = s * (c * d) + return y + )"); + + return c; +} + +TEST(BackendTest, TestCompositeWithSetStates) { + Module c = getCompositeModuleWithSameNameSubModules(); + + std::vector inputs; + inputs.emplace_back(torch::ones({})); + inputs.emplace_back(3.0 * torch::ones({})); + inputs.emplace_back(3); + auto res_jit = c.forward(inputs); + + std::stringstream ss; + c._save_for_mobile(ss); + auto mc = _load_for_mobile(ss); + auto res_mobile = mc.forward(inputs); + AT_ASSERT(res_jit.toTensor().equal(res_mobile.toTensor())); +} + +TEST(BackendTest, TestConsistencyOfCompositeWithSetStates) { + Module c = getCompositeModuleWithSameNameSubModules(); + + std::vector inputs; + inputs.emplace_back(torch::ones({})); + inputs.emplace_back(3.0 * torch::ones({})); + inputs.emplace_back(3); + + std::stringstream ss, ss_resave; + c._save_for_mobile(ss); + auto mc = _load_for_mobile(ss); + auto res_mobile = mc.forward(inputs); + ss.seekg(0, ss.beg); + + // check if the methods names are always the same + // by reloading the script module and saving it back as mobile + // The below checks ensure that the names of Methods + // and numerical outputs of mobile and reloaded mobile + // modules are same. + auto script_module_load = torch::jit::load(ss); + script_module_load._save_for_mobile(ss_resave); + auto mc_reload = _load_for_mobile(ss_resave); + auto res_mobile_reload = mc_reload.forward(inputs); + + AT_ASSERT(res_mobile_reload.toTensor().equal(res_mobile.toTensor())); + + auto mc_methods = mc.get_methods(); + auto mc_reload_methods = mc_reload.get_methods(); + + std::vector mc_method_qns, mc_reload_method_qns; + + auto get_qual_name = [](mobile::Method method) -> std::string { + return method.function().qualname().qualifiedName(); + }; + + std::transform( + mc_methods.begin(), + mc_methods.end(), + std::back_inserter(mc_method_qns), + get_qual_name); + + std::transform( + mc_reload_methods.begin(), + mc_reload_methods.end(), + std::back_inserter(mc_reload_method_qns), + get_qual_name); + + AT_ASSERT(std::equal( + mc_method_qns.begin(), + mc_method_qns.end(), + mc_reload_method_qns.begin())); +} + +TEST(BackendTest, TestCompilerNotSupport) { + Module m("m"); + m.define(R"( + def forward(self, x, h): + return x * h + )"); + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + // lowered module + ASSERT_THROWS_WITH_MESSAGE( + torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", m, compile_spec, any_dict_ty), + "The node of aten::mul is not supported in this compiler. Source code:"); +} + +TEST(BackendTestDebugInfo, TestCompiler) { + Module m("m"); + m.define(R"( + def forward(self, x, h): + return x + h + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({2, 4})); + inputs.emplace_back(torch::rand({13, 9})); + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + // lowered module + auto lm = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", m, compile_spec, any_dict_ty); + + std::stringstream ss; + lm._save_for_mobile(ss, ExtraFilesMap(), true); + auto mlm = _load_for_mobile(ss); + std::string error_pattern = R"( + Module hierarchy:top(m)::.__loweredModule__(m)::forward.aten::add +Traceback of TorchScript (most recent call last): + File "", line 3, in + + def forward(self, x: Tensor, h: Tensor): + return self.__loweredModule__.forward(x, h) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + + File "", line 5, in forward + typed_inputs: List[Any] = [x, h, ] + if self.__backend.is_available() : + _0, = self.__backend.execute(self.__handles["forward"], typed_inputs) + ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + assert isinstance(_0, Tensor) + return _0 + File "", line 3, in + + def forward(self, x, h): + return x + h + ~~~~~ <--- HERE + )"; + ASSERT_THROWS_WITH_MESSAGE(mlm.forward(inputs), error_pattern); +} + +TEST(BackendTestDebugInfo, TestCompilerWithStringTable) { + setShouldUseFormatWithStringTable(true); + Module m("m"); + m.define(R"( + def forward(self, x, h): + return x + h + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({2, 4})); + inputs.emplace_back(torch::rand({13, 9})); + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + // lowered module + auto lm = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", m, compile_spec, any_dict_ty); + + std::stringstream ss; + lm._save_for_mobile(ss, ExtraFilesMap(), true); + auto mlm = _load_for_mobile(ss); + std::string error_pattern = R"( + Module hierarchy:top(m)::.__loweredModule__(m)::forward.aten::add +Traceback of TorchScript (most recent call last): + File "", line 3, in + + def forward(self, x: Tensor, h: Tensor): + return self.__loweredModule__.forward(x, h) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + + File "", line 5, in forward + typed_inputs: List[Any] = [x, h, ] + if self.__backend.is_available() : + _0, = self.__backend.execute(self.__handles["forward"], typed_inputs) + ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + assert isinstance(_0, Tensor) + return _0 + File "", line 3, in + + def forward(self, x, h): + return x + h + ~~~~~ <--- HERE + )"; + setShouldUseFormatWithStringTable(false); + ASSERT_THROWS_WITH_MESSAGE(mlm.forward(inputs), error_pattern); +} + +TEST(BackendTestDebugInfo, TestExceptionStackForCompilerWithModuleHierarchy) { + Module a("A"); + a.define(R"( + def forward(self, x, y): + return x + y + )"); + Module b("B"); + b.define(R"( + def forward(self, x): + return x + 2 + )"); + Module c("C"); + c.register_module("A0", a); + c.register_module("B0", b); + c.define(R"( + def forward(self, x, y): + return self.A0.forward(x, y) + self.B0.forward(x) + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({2, 4})); + inputs.emplace_back(torch::rand({13, 9})); + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + // lowered module + auto lm = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", c, compile_spec, any_dict_ty); + + std::stringstream ss; + lm._save_for_mobile(ss, ExtraFilesMap(), true); + auto mlm = _load_for_mobile(ss); + std::string error_pattern = R"( + Module hierarchy:top(C)::.__loweredModule__(C)::forward.A0(A)::forward.aten::add +Traceback of TorchScript (most recent call last): + File "", line 3, in + + def forward(self, x: Tensor, y: Tensor): + return self.__loweredModule__.forward(x, y) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + + File "", line 5, in forward + typed_inputs: List[Any] = [x, y, ] + if self.__backend.is_available() : + _0, = self.__backend.execute(self.__handles["forward"], typed_inputs) + ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + assert isinstance(_0, Tensor) + return _0 + File "", line 3, in + + def forward(self, x, y): + return self.A0.forward(x, y) + self.B0.forward(x) + ~~~~~~~~~~~~~~~ <--- HERE + + File "", line 3, in forward + + def forward(self, x, y): + return x + y + ~~~~~ <--- HERE + )"; + ASSERT_THROWS_WITH_MESSAGE(mlm.forward(inputs), error_pattern); +} + +TEST( + BackendTestDebugInfo, + TestExceptionStackForCompilerWithTwoLevelModuleHierarchy) { + Module a("A"); + a.define(R"( + def forward(self, x, y): + return x + y + )"); + Module b("B"); + b.register_module("A0", a); + b.define(R"( + def forward(self, x, y): + return self.A0.forward(x, y) + 2 + )"); + Module c("C"); + c.register_module("B0", b); + c.define(R"( + def forward(self, x, y): + return self.B0.forward(x, y) + 3 + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({2, 4})); + inputs.emplace_back(torch::rand({13, 9})); + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + // lowered module + auto lm = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", c, compile_spec, any_dict_ty); + + std::stringstream ss; + lm._save_for_mobile(ss, ExtraFilesMap(), true); + auto mlm = _load_for_mobile(ss); + /* + * Error stack throw will look like this: + * Module hierarchy:top(backend_with_compiler_demoLoweredModule).B0(B).A0(A) + * Traceback of TorchScript (most recent call last): + * File "", line 5, in FunctionName_UNKNOWN + * typed_inputs: List[Any] = [x, y, ] + * if self.__backend.is_available() : + * _0, = self.__backend.execute(self.__handles["forward"], + * typed_inputs) + * ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + * assert isinstance(_0, Tensor) + * return _0 + * File "", line 3, in FunctionName_UNKNOWN + * + * def forward(self, x, y): + * return self.B0.forward(x, y) + 3 + * ~~~~~~~~~~~~~~~ <--- HERE + * + * File "", line 3, in FunctionName_UNKNOWN + * + * def forward(self, x, y): + * return self.A0.forward(x, y) + 2 + * ~~~~~~~~~~~~~~~ <--- HERE + * + * File "", line 3, in FunctionName_UNKNOWN + * + * def forward(self, x, y): + * return x + y + * ~~~~~ <--- HERE + * + */ + std::string error_pattern = R"( + Module hierarchy:top(C)::.__loweredModule__(C)::forward.B0(B)::forward.A0(A)::forward.aten::add +Traceback of TorchScript (most recent call last): + File "", line 3, in + + def forward(self, x: Tensor, y: Tensor): + return self.__loweredModule__.forward(x, y) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + + File "", line 5, in forward + typed_inputs: List[Any] = [x, y, ] + if self.__backend.is_available() : + _0, = self.__backend.execute(self.__handles["forward"], typed_inputs) + ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + assert isinstance(_0, Tensor) + return _0 + File "", line 3, in + + def forward(self, x, y): + return self.B0.forward(x, y) + 3 + ~~~~~~~~~~~~~~~ <--- HERE + + File "", line 3, in forward + + def forward(self, x, y): + return self.A0.forward(x, y) + 2 + ~~~~~~~~~~~~~~~ <--- HERE + + File "", line 3, in forward + + def forward(self, x, y): + return x + y + ~~~~~ <--- HERE + )"; + ASSERT_THROWS_WITH_MESSAGE(mlm.forward(inputs), error_pattern); +} + +TEST(BackendTestDebugInfo, TestExceptionStackForCompilerWithLoweredSubModule) { + std::shared_ptr cu = std::make_shared(); + Module a("A"); + a.define(R"( + def forward(self, x, y): + return x + y + )"); + Module b("B"); + b.define(R"( + def forward(self, x): + return x + 2 + )"); + Module c("C"); + c.register_module("A0", a); + c.register_module("B0", b); + c.define(R"( + def forward(self, x, y): + return self.A0.forward(x, y) + self.B0.forward(x) + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({2, 4})); + inputs.emplace_back(torch::rand({13, 9})); + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + IValue submodule = c.attr("A0"); + Module current_sm = submodule.toModule(); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + // lowered module + auto lowered_submodule = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", current_sm, compile_spec, any_dict_ty); + + c.type()->unsafeChangeAttributeType("A0", lowered_submodule.type()); + c.setattr("A0", lowered_submodule._ivalue()); + std::unordered_map type_remap; + type_remap[a.type()] = lowered_submodule.type(); + auto type_remap_fn = [&type_remap](TypePtr in) { + auto it = type_remap.find(in); + if (it == type_remap.end()) + return in; + return it->second; + }; + for (auto& fn : c.type()->methods()) { + auto method = c.get_method(fn->name()); + auto graph = method.graph(); + graph->remapTypes(type_remap_fn); + auto new_schema = fn->getSchema().cloneWithRemappedTypes(type_remap_fn); + fn->setSchema(new_schema); + } + + std::stringstream ss; + c._save_for_mobile(ss, ExtraFilesMap(), true); + auto c_loaded = _load_for_mobile(ss); + std::string error_pattern = R"( + Module hierarchy:top(C)::.A0(A)::forward.__loweredModule__(A)::forward.aten::add +Traceback of TorchScript (most recent call last): + File "", line 3, in + + def forward(self, x, y): + return self.A0.forward(x, y) + self.B0.forward(x) + ~~~~~~~~~~~~~~~ <--- HERE + + File "", line 3, in forward + + def forward(self, x: Tensor, y: Tensor): + return self.__loweredModule__.forward(x, y) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + + File "", line 5, in forward + typed_inputs: List[Any] = [x, y, ] + if self.__backend.is_available() : + _0, = self.__backend.execute(self.__handles["forward"], typed_inputs) + ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + assert isinstance(_0, Tensor) + return _0 + File "", line 3, in + + def forward(self, x, y): + return x + y + ~~~~~ <--- HERE + )"; + ASSERT_THROWS_WITH_MESSAGE(c_loaded.forward(inputs), error_pattern); +} + +TEST( + BackendTestDebugInfo, + TestExceptionStackForCompilerWithSelectiveLoweredSubModule) { + std::shared_ptr cu = std::make_shared(); + Module aa("AA"); + aa.define(R"( + def forward(self, x, y): + return x + y + )"); + Module a("A"); + a.register_module("AA0", aa); + a.define(R"( + def forward(self, x, y): + return self.AA0.forward(x, y) + 3 + )"); + Module b("B"); + b.define(R"( + def forward(self, x): + return x + 2 + )"); + Module c("C"); + c.register_module("A0", a); + c.register_module("B0", b); + c.define(R"( + def forward(self, x, y): + return self.A0.forward(x, y) + self.B0.forward(x) + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({2, 4})); + inputs.emplace_back(torch::rand({13, 9})); + + c10::Dict compile_spec(StringType::get(), AnyType::get()); + c10::Dict fake_dict(StringType::get(), AnyType::get()); + fake_dict.insert("", ""); + compile_spec.insert("forward", fake_dict); + IValue submodule = c.attr("A0"); + Module current_sm = submodule.toModule(); + auto any_dict_ty = DictType::create(StringType::get(), AnyType::get()); + // lowered module + auto lowered_submodule = torch::jit::detail::codegen_backend_module( + "backend_with_compiler_demo", current_sm, compile_spec, any_dict_ty); + + c.type()->unsafeChangeAttributeType("A0", lowered_submodule.type()); + c.setattr("A0", lowered_submodule._ivalue()); + std::unordered_map type_remap; + type_remap[a.type()] = lowered_submodule.type(); + auto type_remap_fn = [&type_remap](TypePtr in) { + auto it = type_remap.find(in); + if (it == type_remap.end()) + return in; + return it->second; + }; + for (auto& fn : c.type()->methods()) { + auto method = c.get_method(fn->name()); + auto graph = method.graph(); + graph->remapTypes(type_remap_fn); + auto new_schema = fn->getSchema().cloneWithRemappedTypes(type_remap_fn); + fn->setSchema(new_schema); + } + + std::stringstream ss; + c._save_for_mobile(ss, ExtraFilesMap(), true); + auto c_loaded = _load_for_mobile(ss); + /* + * Error stack trace will look like this: + * Module hierarchy:top(C).A0(backend_with_compiler_demoLoweredModule).AA0(AA) + * Traceback of TorchScript (most recent call last): + * File "", line 3, in FunctionName_UNKNOWN + * + * def forward(self, x, y): + * return self.A0.forward(x, y) + self.B0.forward(x) + * ~~~~~~~~~~~~~~~ <--- HERE + * + * File "", line 5, in FunctionName_UNKNOWN + * typed_inputs: List[Any] = [x, y, ] + * if self.__backend.is_available() : + * _0, = self.__backend.execute(self.__handles["forward"], + * typed_inputs) + * ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + * assert isinstance(_0, Tensor) + * return _0 + * File "", line 3, in FunctionName_UNKNOWN + * + * def forward(self, x, y): + * return self.AA0.forward(x, y) + 3 + * ~~~~~~~~~~~~~~~~ <--- HERE + * + * File "", line 3, in FunctionName_UNKNOWN + * + * def forward(self, x, y): + * return x + y + * ~~~~~ <--- HERE + * + * + * */ + std::string error_pattern = R"( + Module hierarchy:top(C)::.A0(A)::forward.__loweredModule__(A)::forward.AA0(AA)::forward.aten::add +Traceback of TorchScript (most recent call last): + File "", line 3, in + + def forward(self, x, y): + return self.A0.forward(x, y) + self.B0.forward(x) + ~~~~~~~~~~~~~~~ <--- HERE + + File "", line 3, in forward + + def forward(self, x: Tensor, y: Tensor): + return self.__loweredModule__.forward(x, y) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + + File "", line 5, in forward + typed_inputs: List[Any] = [x, y, ] + if self.__backend.is_available() : + _0, = self.__backend.execute(self.__handles["forward"], typed_inputs) + ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + assert isinstance(_0, Tensor) + return _0 + File "", line 3, in + + def forward(self, x, y): + return self.AA0.forward(x, y) + 3 + ~~~~~~~~~~~~~~~~ <--- HERE + + File "", line 3, in forward + + def forward(self, x, y): + return x + y + ~~~~~ <--- HERE + )"; + ASSERT_THROWS_WITH_MESSAGE(c_loaded.forward(inputs), error_pattern); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_backend_compiler_lib.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_backend_compiler_lib.cpp new file mode 100644 index 0000000000000000000000000000000000000000..45e1f25ef0b462ce5fe8444bd673c9cd6623fa9e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_backend_compiler_lib.cpp @@ -0,0 +1,201 @@ +#include +#include +#include +#include +#include + +#ifndef NO_PROFILING +#include +#endif + +namespace torch { +namespace jit { + +// Implementation of a PyTorch Backend that can process, compile and execute +// TorchScript Modules composed of 'add' and 'sub' operators. It just supports +// for modules that implement a sum or subtraction of 2 inputs (i.e. in1 + in2 +// or in1 - in2). Hence the methods of the models expect exactly 2 inputs of +// type Tensor. This backend is used to demonstrate the flow of compilation and +// execution with minimum amount of work. It's not intended to a practical +// backend that can be used for actual inference. + +// Implementation details: +// +// Compilation +// 1. A backend with minimum compilation features, "backend_with_compiler_demo" +// is added. +// 2. The compilation happens AOT in the preprocess function registered to this +// backend. +// 3. Compiled results are stored in a string blob for each method. They are +// serialized to the lowered module with __getstate__ function. +// 4. Error message with model source code is thrown, for features not handled +// by the backend compiler. +// +// Runtime +// 1. The compiled blob is loaded in __setstate__ method. +// 2. The compile function of the backend: parse the preprocessed blob to the +// format (a list of tokens) that the backend can understand. +// 3. The execute function of the backend executes the specified method +// (handle). + +namespace { +std::vector> parseMethodHandle( + const std::string& blob) { + std::vector> result; + std::stringstream s_stream(blob); + constexpr char debug_handle_token[] = ""; + while (s_stream.good()) { + std::string substr; + getline(s_stream, substr, ','); + auto debug_handle_pos = substr.find(debug_handle_token); + int64_t debug_handle{-1}; + auto instruction = substr.substr(0); + if (debug_handle_pos != std::string::npos) { + instruction = substr.substr(0, debug_handle_pos); + debug_handle = stoi(substr.substr(debug_handle_pos + 14)); + } + result.push_back(std::make_tuple(instruction, debug_handle)); + } + return result; +} + +float* float_data_ptr(const at::Tensor& t) { + return t.data_ptr(); +} +} // namespace + +class BackendWithCompiler : public PyTorchBackendInterface { + public: + // Constructor. + // NOLINTNEXTLINE(modernize-use-equals-default) + explicit BackendWithCompiler() {} + virtual ~BackendWithCompiler() override = default; + + bool is_available() override { + return true; + } + + // Since the actual compilation is done AOT for this backend, compile just + // forwards everything along. In a non toy setup this could grab information + // from that runtime that might be relevant to execute, such as build flags + // the resolution of the devices camera, or basically any runtime specific + // information that wouldn't be available server side where preprocess is + // called. + c10::impl::GenericDict compile( + c10::IValue processed, + c10::impl::GenericDict method_compile_spec) override { + auto dict = processed.toGenericDict(); + auto handles = + c10::Dict>>(); + for (const auto& kv : dict) { + auto tokens = parseMethodHandle(kv.value().toStringRef()); + handles.insert(kv.key().toStringRef(), tokens); + } + return c10::impl::toGenericDict(handles); + } + + // Function that actually executes the model in the backend. Here there is + // nothing to dispatch to, so the backend is implemented locally within + // execute and it only supports add, subtract, and constant. In a non toy + // backend you can imagine how this function could be used to actually + // dispatch the inputs to the relevant backend/device. + c10::impl::GenericList execute( + c10::IValue + handle, // example: [('prim::Constant#1', 14), ('aten::add', 15)] + c10::impl::GenericList inputs) override { + TORCH_INTERNAL_ASSERT(inputs.size() == 2); + c10::IValue val0 = inputs[0]; + at::Tensor x = val0.toTensor(); + c10::IValue val1 = inputs[1]; + at::Tensor h = val1.toTensor(); + std::vector> op_runtimes_us; + op_runtimes_us.reserve(handle.toList().size()); + + c10::List output_list; +#ifndef NO_PROFILING + auto start_us = c10::getTime() / 1000; +#endif + for (const auto& token : handle.toList()) { + IValue val = token; + auto instruction = val.toTupleRef().elements()[0].toStringRef(); + auto debug_handle = val.toTupleRef().elements()[1].toInt(); +#ifndef NO_PROFILING + auto start_time_us = c10::getTime() / 1000; +#endif + try { + if (instruction.rfind("prim::Constant", 0) == 0) { + // 15 is the length of 'prim::Constant#' the constant val comes after + TORCH_CHECK( + instruction.size() > 15, + "Constant value is expected in ", + instruction); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) + auto sub = instruction.substr(15); + } else if (instruction == "aten::add" || instruction == "aten::sub") { + TORCH_CHECK(x.sizes() == h.sizes()); + if (x.dim() > 1 || (x.dim() == 1 && x.size(0) > 1)) { + TORCH_WARN( + "Only the first elements of the tensors are added or subbed."); + } + TORCH_CHECK( + (x.scalar_type() == c10::ScalarType::Float && + h.scalar_type() == c10::ScalarType::Float), + "Only float tensors are compatible for add and sub."); + at::Tensor y = at::detail::empty_cpu(x.sizes(), at::kFloat); + auto x_ptr = float_data_ptr(x); + auto h_ptr = float_data_ptr(h); + auto y_ptr = float_data_ptr(y); +#ifndef NO_PROFILING + RECORD_BACKEND_MEMORY_EVENT_TO_EDGE_PROFILER( + x_ptr, + x.numel() * sizeof(float), + x.numel() * sizeof(float), + x.numel() * sizeof(float) + y.numel() * sizeof(float) + + h.numel() * sizeof(float), + c10::Device(c10::kCPU)); +#endif + if (instruction == "aten::add") { + y_ptr[0] = x_ptr[0] + h_ptr[0]; + } else { + y_ptr[0] = x_ptr[0] - h_ptr[0]; + } + output_list.emplace_back(y); + } else { + TORCH_CHECK( + false, + "Instruction, ", + instruction, + " is not supported. ", + "Contact the backend POC for details. "); + } + } catch (c10::Error& e) { + TORCH_DELEGATED_BACKEND_THROW(false, e.what(), debug_handle); + } +#ifndef NO_PROFILING + auto end_time_us = c10::getTime() / 1000; + auto duration = end_time_us - start_time_us; + op_runtimes_us.emplace_back(duration, debug_handle, instruction); +#endif + } +#ifndef NO_PROFILING + for (const auto& tup : op_runtimes_us) { + RECORD_BACKEND_EVENT_TO_EDGE_PROFILER( + start_us, + start_us + std::get<0>(tup), + std::get<1>(tup), + std::get<2>(tup), + "test_backend"); + start_us = start_us + std::get<0>(tup); + } +#endif + return c10::impl::toList(output_list); + } +}; + +namespace { +constexpr auto backend_name = "backend_with_compiler_demo"; +static auto cls = torch::jit::backend(backend_name); +} // namespace + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_backend_compiler_preprocess.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_backend_compiler_preprocess.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d82595eae692768e9b40deea54655eddf70e8cd6 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_backend_compiler_preprocess.cpp @@ -0,0 +1,76 @@ +#include +#include +#include +#include + +namespace torch { +namespace jit { +namespace { +// For this backend, the actual compilation happens in preprocess function AOT. +// Put here for demonstration of backend +// as a whole piece. It's used when compilation is required. A dummy function +// can be passed when there's no usage of compilation in runtime backend lib. +c10::IValue preprocess( + const Module& mod, + const c10::Dict& method_compile_spec, + const BackendDebugHandleGenerator& generate_debug_handles) { + // The output of this process would produce a dictionary + // Key: method name. + // Val: compiled blob (represented by a string). + c10::Dict compiled(StringType::get(), StringType::get()); + + for (const auto& method : mod.get_methods()) { + auto graph = toGraphFunction(method.function()).graph()->copy(); + // Must inline the graph for debug info map. + Inline(*graph); + // This is here because to test module hierarchy we will have + // getattr nodes which after inlining dont serve any purpose. + // Without removing them we will run into compilation errors. + // So eliminate deadcode just remove those getattr nodes. + EliminateDeadCode(graph); + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto key = method.name(); + auto node_debug_handles = generate_debug_handles(graph); + std::stringstream ss; + for (const auto& node : graph->nodes()) { + switch (node->kind()) { + case prim::Constant: + ss << node->kind().toDisplayString() << '#' + << toIValue(node->output()).value(); + ss << "" << node_debug_handles[node]; + break; + // NOLINTNEXTLINE(bugprone-branch-clone) + case aten::add: + ss << node->kind().toQualString(); + ss << "" << node_debug_handles[node]; + break; + case aten::sub: + ss << node->kind().toQualString(); + ss << "" << node_debug_handles[node]; + break; + default: + TORCH_CHECK( + false, + "The node of ", + node->kind().toQualString(), + " is not supported in this compiler. Source code: ", + node->sourceRange().str()); + break; + } + ss << ','; + } + std::string blob = std::move(ss).str(); + if (!blob.empty()) { + blob.pop_back(); + } + compiled.insert(method.name(), blob); + } + return compiled; +} + +constexpr auto backend_name = "backend_with_compiler_demo"; +static auto pre_reg = backend_preprocess_register(backend_name, preprocess); +} // namespace + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_backend_lib.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_backend_lib.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8acb57331a64f6d6873cefb4ea0e43320160bf03 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_backend_lib.cpp @@ -0,0 +1,93 @@ +#include +#include +#include + +namespace torch { +namespace jit { +// This test JIT backend is intended to do the minimal amount of work +// necessary to test that the JIT backend registration endpoints and +// code generation are working correctly. It is not intended to +// produce numerically correct results. +template +class TestBackend : public PyTorchBackendInterface { + public: + // Constructor. + // NOLINTNEXTLINE(modernize-use-equals-default) + explicit TestBackend() {} + virtual ~TestBackend() override = default; + + bool is_available() override { + return isAvailable; + } + + c10::impl::GenericDict compile( + c10::IValue processed, + c10::impl::GenericDict method_compile_spec) override { + auto spec = + c10::impl::toTypedDict(method_compile_spec); + + // Return the same string as a value for every key in method_compile_spec. + auto handles = c10::Dict(); + for (const auto& it : spec) { + handles.insert(it.key(), it.key()); + } + return c10::impl::toGenericDict(handles); + } + c10::impl::GenericList execute( + c10::IValue handle, + c10::impl::GenericList inputs) override { + TORCH_INTERNAL_ASSERT(handle.isString()); + TORCH_INTERNAL_ASSERT(inputs.size() > 0); + + c10::List output_list; + + // Implement simple accumulator and negative accumulator (?) ops. Return one + // or both of them depending on the handle to make sure multiple outputs are + // handled. + c10::IValue value = inputs[0]; + at::Tensor accum = value.toTensor(); + accum = accum.clone(); + at::Tensor sub_accum = value.toTensor(); + sub_accum = sub_accum.clone(); + + for (size_t i = 1, e = inputs.size(); i < e; ++i) { + value = inputs[i]; + accum.add_(value.toTensor(), 1.0); + sub_accum.sub_(value.toTensor(), 1.0); + } + + if (handle.toStringRef() == "accum") { + output_list.emplace_back(accum); + } else if (handle.toStringRef() == "sub_accum") { + output_list.emplace_back(sub_accum); + } else if (handle.toStringRef() == "forward") { + output_list.emplace_back(accum); + output_list.emplace_back(sub_accum); + } + + return c10::impl::toList(output_list); + } +}; + +namespace { +c10::IValue preprocess( + const Module& mod, + const c10::Dict& method_compile_spec, + const BackendDebugHandleGenerator& generate_debug_handles) { + return mod._ivalue(); +} + +constexpr auto backend_name = "test_backend"; +static auto cls_available = + torch::jit::backend>(backend_name); +static auto pre_reg = backend_preprocess_register(backend_name, preprocess); + +constexpr auto backend_unavailable_name = "test_backend_unavailable"; +static auto cls_unavailable = + torch::jit::backend>(backend_unavailable_name); +static auto pre_reg_unavailable = + backend_preprocess_register(backend_unavailable_name, preprocess); + +} // namespace +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_class_import.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_class_import.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c8ddc1ae17d02c65c7a7cf23298847c0d7ca9958 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_class_import.cpp @@ -0,0 +1,158 @@ +#include + +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +static constexpr std::string_view classSrcs1 = R"JIT( +class FooNestedTest: + def __init__(self, y): + self.y = y + +class FooNestedTest2: + def __init__(self, y): + self.y = y + self.nested = __torch__.FooNestedTest(y) + +class FooTest: + def __init__(self, x): + self.class_attr = __torch__.FooNestedTest(x) + self.class_attr2 = __torch__.FooNestedTest2(x) + self.x = self.class_attr.y + self.class_attr2.y +)JIT"; + +static constexpr std::string_view classSrcs2 = R"JIT( +class FooTest: + def __init__(self, x): + self.dx = x +)JIT"; + +static void import_libs( + std::shared_ptr cu, + const std::string& class_name, + const std::shared_ptr& src, + const std::vector& tensor_table) { + SourceImporter si( + cu, + &tensor_table, + [&](const std::string& name) -> std::shared_ptr { return src; }, + /*version=*/2); + si.loadType(QualifiedName(class_name)); +} + +TEST(ClassImportTest, Basic) { + auto cu1 = std::make_shared(); + auto cu2 = std::make_shared(); + std::vector constantTable; + // Import different versions of FooTest into two namespaces. + import_libs( + cu1, + "__torch__.FooTest", + std::make_shared(classSrcs1), + constantTable); + import_libs( + cu2, + "__torch__.FooTest", + std::make_shared(classSrcs2), + constantTable); + + // We should get the correct version of `FooTest` for whichever namespace we + // are referencing + c10::QualifiedName base("__torch__"); + auto classType1 = cu1->get_class(c10::QualifiedName(base, "FooTest")); + ASSERT_TRUE(classType1->hasAttribute("x")); + ASSERT_FALSE(classType1->hasAttribute("dx")); + + auto classType2 = cu2->get_class(c10::QualifiedName(base, "FooTest")); + ASSERT_TRUE(classType2->hasAttribute("dx")); + ASSERT_FALSE(classType2->hasAttribute("x")); + + // We should only see FooNestedTest in the first namespace + auto c = cu1->get_class(c10::QualifiedName(base, "FooNestedTest")); + ASSERT_TRUE(c); + + c = cu2->get_class(c10::QualifiedName(base, "FooNestedTest")); + ASSERT_FALSE(c); +} + +TEST(ClassImportTest, ScriptObject) { + Module m1("m1"); + Module m2("m2"); + std::vector constantTable; + import_libs( + m1._ivalue()->compilation_unit(), + "__torch__.FooTest", + std::make_shared(classSrcs1), + constantTable); + import_libs( + m2._ivalue()->compilation_unit(), + "__torch__.FooTest", + std::make_shared(classSrcs2), + constantTable); + + // Incorrect arguments for constructor should throw + c10::QualifiedName base("__torch__"); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_ANY_THROW(m1.create_class(c10::QualifiedName(base, "FooTest"), {1})); + auto x = torch::ones({2, 3}); + auto obj = m2.create_class(c10::QualifiedName(base, "FooTest"), x).toObject(); + auto dx = obj->getAttr("dx"); + ASSERT_TRUE(almostEqual(x, dx.toTensor())); + + auto new_x = torch::rand({2, 3}); + obj->setAttr("dx", new_x); + auto new_dx = obj->getAttr("dx"); + ASSERT_TRUE(almostEqual(new_x, new_dx.toTensor())); +} + +static const auto methodSrc = R"JIT( +def __init__(self, x): + return x +)JIT"; + +TEST(ClassImportTest, ClassDerive) { + auto cu = std::make_shared(); + auto cls = ClassType::create("foo.bar", cu); + const auto self = SimpleSelf(cls); + auto methods = cu->define("foo.bar", methodSrc, nativeResolver(), &self); + auto method = methods[0]; + cls->addAttribute("attr", TensorType::get()); + ASSERT_TRUE(cls->findMethod(method->name())); + + // Refining a new class should retain attributes and methods + auto newCls = cls->refine({TensorType::get()}); + ASSERT_TRUE(newCls->hasAttribute("attr")); + ASSERT_TRUE(newCls->findMethod(method->name())); + + auto newCls2 = cls->withContained({TensorType::get()})->expect(); + ASSERT_TRUE(newCls2->hasAttribute("attr")); + ASSERT_TRUE(newCls2->findMethod(method->name())); +} + +static constexpr std::string_view torchbindSrc = R"JIT( +class FooBar1234(Module): + __parameters__ = [] + f : __torch__.torch.classes._TorchScriptTesting._StackString + training : bool + def forward(self: __torch__.FooBar1234) -> str: + return (self.f).top() +)JIT"; + +TEST(ClassImportTest, CustomClass) { + auto cu1 = std::make_shared(); + std::vector constantTable; + // Import different versions of FooTest into two namespaces. + import_libs( + cu1, + "__torch__.FooBar1234", + std::make_shared(torchbindSrc), + constantTable); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_class_parser.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_class_parser.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d0257e4088eb75bb689ff7a181a0a40694518a94 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_class_parser.cpp @@ -0,0 +1,37 @@ +#include + +#include +#include + +namespace torch { +namespace jit { +constexpr std::string_view testSource = R"JIT( + class FooTest: + def __init__(self, x): + self.x = x + + def get_x(self): + return self.x + + an_attribute : Tensor +)JIT"; + +TEST(ClassParserTest, Basic) { + Parser p(std::make_shared(testSource)); + std::vector definitions; + std::vector resolvers; + + const auto classDef = ClassDef(p.parseClass()); + p.lexer().expect(TK_EOF); + + ASSERT_EQ(classDef.name().name(), "FooTest"); + ASSERT_EQ(classDef.body().size(), 3); + ASSERT_EQ(Def(classDef.body()[0]).name().name(), "__init__"); + ASSERT_EQ(Def(classDef.body()[1]).name().name(), "get_x"); + ASSERT_EQ( + Var(Assign(classDef.body()[2]).lhs()).name().name(), "an_attribute"); + ASSERT_FALSE(Assign(classDef.body()[2]).rhs().present()); + ASSERT_TRUE(Assign(classDef.body()[2]).type().present()); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_class_type.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_class_type.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f39646c166f0b4227f17384018d85c29f26ae087 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_class_type.cpp @@ -0,0 +1,111 @@ +#include + +#include +#include +#include + +namespace torch { +namespace jit { + +TEST(ClassTypeTest, AddRemoveAttr) { + auto cu = std::make_shared(); + auto cls = ClassType::create("foo.bar", cu, true); + cls->addAttribute("attr1", TensorType::get(), true); + cls->addAttribute("attr2", TensorType::get()); + cls->addAttribute("attr3", TensorType::get()); + ASSERT_TRUE(cls->hasAttribute("attr1")); + ASSERT_TRUE(cls->hasAttribute("attr2")); + ASSERT_TRUE(cls->hasAttribute("attr3")); + + // removing attribute attr2 + cls->unsafeRemoveAttribute("attr2"); + ASSERT_TRUE(cls->hasAttribute("attr1")); + ASSERT_FALSE(cls->hasAttribute("attr2")); + ASSERT_TRUE(cls->hasAttribute("attr3")); + + // removing parameter attr1 + cls->unsafeRemoveAttribute("attr1"); + ASSERT_FALSE(cls->hasAttribute("attr1")); + ASSERT_FALSE(cls->hasAttribute("attr2")); + ASSERT_TRUE(cls->hasAttribute("attr3")); + + // check that we can still add a non-parameter attr1 with + // different type + cls->addAttribute("attr1", IntType::get()); +} + +TEST(ClassTypeTest, AddRemoveConstant) { + auto cu = std::make_shared(); + auto cls = ClassType::create("foo.bar", cu); + cls->addConstant("const1", IValue(1)); + cls->addConstant("const2", IValue(2)); + cls->addConstant("const3", IValue(3)); + ASSERT_EQ(cls->numConstants(), 3); + ASSERT_TRUE(cls->hasConstant("const1")); + ASSERT_TRUE(cls->hasConstant("const2")); + ASSERT_TRUE(cls->hasConstant("const3")); + ASSERT_FALSE(cls->hasConstant("const4")); + + ASSERT_EQ(cls->getConstant("const1").toInt(), 1); + ASSERT_EQ(cls->getConstant("const2").toInt(), 2); + ASSERT_EQ(cls->getConstant("const3").toInt(), 3); + + cls->unsafeRemoveConstant("const2"); + ASSERT_TRUE(cls->hasConstant("const1")); + ASSERT_FALSE(cls->hasConstant("const2")); + ASSERT_TRUE(cls->hasConstant("const3")); +} + +TEST(ClassTypeTest, IdenticalTypesDifferentCus) { + auto cu1 = std::make_shared(); + auto cu2 = std::make_shared(); + + // Create two identically named ClassTypes and put them + // in separate compilation units. + auto cls1 = ClassType::create("foo", cu1); + auto cls2 = ClassType::create("foo", cu2); + + // Create a function that accepts "foo" (cls1) as input. + Argument arg("arg", cls1); + Argument ret("ret", IntType::get()); + + FunctionSchema schema("fn", "", {arg}, {ret}); + + jit::BuiltinOpFunction method( + "method", + std::move(schema), + [](jit::Stack& stack) mutable -> void { + pop(stack); + push(stack, 0); + }, + ""); + + // Create an object of type cls2. + Object obj(cu2, cls2); + + // Call method with the above object; this should + // throw an error because the types have identical + // names but are in different compilation units. + Stack stack; + push(stack, obj._ivalue()); + try { + method(stack, {}); + } catch (const std::exception& e) { + // Check that the exception contains the address of the compilation unit + // in addition to the ClassType's name. + testing::FileCheck() + .check("foo (of Python compilation unit at: 0x") + ->check_same(")") + ->check("foo (of Python compilation unit at: 0x") + ->check_same(")") + ->run(e.what()); + + return; + } + + // This should never execute. + ASSERT_TRUE(false); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_cleanup_passes.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_cleanup_passes.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e15523b0d6aca3ca07496bc06ffe7c8a0ceaadb4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_cleanup_passes.cpp @@ -0,0 +1,49 @@ +#include + +#include +#include +#include +#include + +namespace torch { +namespace jit { + +TEST(CleanupPassTest, Basic) { + // Tests stability of clean up passes when dealing with constant pooling + // and constant propagation. + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%cond.1 : Tensor, + %suffix.1 : str): + %3 : bool = aten::Bool(%cond.1) # o.py:6:7 + %25 : str = prim::If(%3) # o.py:6:4 + block0(): + %a.1 : str = prim::Constant[value="same string"]() + %b.1 : str = prim::Constant[value=" with a twist"]() + %7 : str = aten::add(%a.1, %b.1) + %11 : str = aten::add(%7, %suffix.1) # o.py:10:15 + -> (%11) + block1(): + %c.1 : str = prim::Constant[value="same string"]() + %d.1 : str = prim::Constant[value=" with a twist"]() + %12 : str = aten::add(%c.1, %d.1) + -> (%12) + return (%25) + )IR", + &*graph); + runCleanupPasses(graph); + testing::FileCheck() + .check_count( + "prim::Constant[value=\"same string with a twist\"]", + 1, + /*exactly=*/true) + ->run(*graph); + + auto graph_after_pass_once = graph->toString(); + runCleanupPasses(graph); + auto graph_after_pass_twice = graph->toString(); + ASSERT_EQ(graph_after_pass_once, graph_after_pass_twice); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_code_template.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_code_template.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8d5409581f00bcaf62e69ec8fc79ec086f3e8762 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_code_template.cpp @@ -0,0 +1,62 @@ +#include + +#include +#include + +namespace torch { +namespace jit { + +static const auto ct = at::jit::CodeTemplate(R"( + int foo($args) { + + $bar + $bar + $a+$b + } + int commatest(int a${,stuff}) + int notest(int a${,empty,}) + )"); +static const auto ct_expect = R"( + int foo(hi, 8) { + + what + on many + lines... + 7 + what + on many + lines... + 7 + 3+4 + } + int commatest(int a, things..., others) + int notest(int a) + )"; + +TEST(TestCodeTemplate, Copying) { + at::jit::TemplateEnv e; + e.s("hi", "foo"); + e.v("what", {"is", "this"}); + at::jit::TemplateEnv c(e); + c.s("hi", "foo2"); + ASSERT_EQ(e.s("hi"), "foo"); + ASSERT_EQ(c.s("hi"), "foo2"); + ASSERT_EQ(e.v("what")[0], "is"); +} + +TEST(TestCodeTemplate, Formatting) { + at::jit::TemplateEnv e; + e.v("args", {"hi", "8"}); + e.v("bar", {"what\non many\nlines...", "7"}); + e.s("a", "3"); + e.s("b", "4"); + e.v("stuff", {"things...", "others"}); + e.v("empty", {}); + auto s = ct.format(e); + // std::cout << "'" << s << "'\n"; + // std::cout << "'" << ct_expect << "'\n"; + ASSERT_EQ(s, ct_expect); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_concat_opt.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_concat_opt.cpp new file mode 100644 index 0000000000000000000000000000000000000000..71b72c2aa123221a1d13674742e3395eab5993a1 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_concat_opt.cpp @@ -0,0 +1,746 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +TEST(ConcatOptTest, SimpleCommonInputsEliminationPrefix) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %2: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %5 : int = prim::Constant[value=0]() + %concat.2 : Float(96, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = prim::VarConcat(%0, %1, %5) + %concat.3 : Float(128, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = prim::VarConcat(%0, %1, %2, %5) + %res : Tensor[] = prim::ListConstruct(%concat.2, %concat.3) + return (%res) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(EliminateConcatCommonInputs(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // Graph after EliminateConcatCommonInputs: + // graph(%0 : ..., + // %1 : ..., + // %2 : ...): + // %3 : int = prim::Constant[value=0]() + // %4 : Tensor = prim::VarConcat(%0, %1, %3) + // %7 : Tensor = prim::VarConcat(%4, %2, %3) // UPDATED + // %8 : Tensor[] = prim::ListConstruct(%4, %7) + // return (%8) + + testing::FileCheck() + .check_count("= prim::VarConcat(%0, %1, %3)", 1, /*exactly*/ true) + ->check_count("= prim::VarConcat(%4, %2, %3)", 1, /*exactly*/ true) + ->check_count("= prim::ListConstruct(%4, %7)", 1, /*exactly*/ true) + ->check_count("= aten::cat(", 0, /*exactly*/ true) + ->check_count("= prim::ListConstruct(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConcatOptTest, SimpleCommonInputsEliminationSuffix) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %2: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %5 : int = prim::Constant[value=0]() + %concat.2 : Float(96, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = prim::VarConcat(%1, %2, %5) + %concat.3 : Float(128, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = prim::VarConcat(%0, %1, %2, %5) + %res : Tensor[] = prim::ListConstruct(%concat.2, %concat.3) + return (%res) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(EliminateConcatCommonInputs(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // Graph after EliminateConcatCommonInputs: + // graph(%0 : ..., + // %1 : ..., + // %2 : ...): + // %3 : int = prim::Constant[value=0]() + // %4 : Tensor = prim::VarConcat(%1, %2, %3) + // %7 : Tensor = prim::VarConcat(%0, %4, %3) // UPDATED + // %8 : Tensor[] = prim::ListConstruct(%4, %7) + // return (%8) + + testing::FileCheck() + .check_count("= prim::VarConcat(%1, %2, %3)", 1, /*exactly*/ true) + ->check_count("= prim::VarConcat(%0, %4, %3)", 1, /*exactly*/ true) + ->check_count("= prim::ListConstruct(%4, %7)", 1, /*exactly*/ true) + ->check_count("= aten::cat(", 0, /*exactly*/ true) + ->check_count("= prim::ListConstruct(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConcatOptTest, CommonInputsEliminationWithDifferentOrderInputs) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %2: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %5 : int = prim::Constant[value=0]() + + #CHECK: prim::VarConcat + %concat.1 : Float(96, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = prim::VarConcat(%0, %1, %5) + + #CHECK: prim::VarConcat + %concat.2 : Float(96, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = prim::VarConcat(%1, %0, %2, %5) + + #CHECK: prim::ListConstruct + %res : Tensor[] = prim::ListConstruct(%concat.1, %concat.2) + return (%res) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_FALSE(EliminateConcatCommonInputs(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // No optimizations should have happened in this case since the inputs + // to the `cat` are in different order. + testing::FileCheck().run(input, *graph); +} + +TEST(ConcatOptTest, MoreCommonInputsElimination) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %2: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %3: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %4: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %5 : int = prim::Constant[value=0]() + %concat.1 : Float(96, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = prim::VarConcat(%0, %1, %5) + %concat.2 : Float(128, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = prim::VarConcat(%0, %1, %2, %5) + %concat.3 : Float(160, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = prim::VarConcat(%0, %1, %2, %3, %5) + %concat.4 : Float(192, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = prim::VarConcat(%0, %1, %2, %3, %4, %5) + %res : Tensor[] = prim::ListConstruct(%concat.1, %concat.2, %concat.3, %concat.4) + return (%res) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(EliminateConcatCommonInputs(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + testing::FileCheck() + .check_count("= prim::VarConcat(%0, %1, %5)", 1, /*exactly*/ true) + ->check_count("= prim::VarConcat(%6, %2, %5)", 1, /*exactly*/ true) + ->check_count("= prim::VarConcat(%11, %3, %5)", 1, /*exactly*/ true) + ->check_count("= prim::VarConcat(%12, %4, %5)", 1, /*exactly*/ true) + ->check_count("= aten::cat(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConcatOptTest, ExpandConcat) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %2 : int = prim::Constant[value=0]() + %3 : float = prim::Constant[value=0.5]() + %4 : Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::clamp_max(%0, %3) + %5 : Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::clamp_max(%1, %3) + %input : Tensor[] = prim::ListConstruct(%4, %5) + %concat : Float(96, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%input, %2) + return (%concat) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ExpandConcatAndEliminateRedundancy(graph); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // After full concat optimization we should have the following graph: + // + // graph(%0 : ..., + // %1 : ...): + // ... + // %4 : Tensor = aten::clamp_max(...) + // %5 : Tensor = aten::clamp_max(...) + // %13 : int[] = prim::ListConstruct(...) + // %14 : Tensor = aten::empty(%13, ...) // concat buffer + // %17 : Tensor = aten::slice(%14, ...) // slice for %4 + // %18 : Tensor = aten::copy_(%17, %4) + // %20 : Tensor = aten::slice(%14, ...) // slice for %5 + // %21 : Tensor = aten::copy_(%20, %5) + // return (%14) + testing::FileCheck() + .check_count("= aten::cat(", 0, /*exactly*/ true) + ->check_count("= aten::clamp_max(", 2, /*exactly*/ true) + ->check_count("= aten::empty(", 1, /*exactly*/ true) + ->check_count("= aten::slice(", 1, /*exactly*/ true) + ->check_count("= aten::copy_(", 1, /*exactly*/ true) + ->check_count("= aten::slice(", 1, /*exactly*/ true) + ->check_count("= aten::copy_(", 1, /*exactly*/ true) + ->check_count("= aten::cat(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConcatOptTest, ConcatWithoutResultShape) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %2 : int = prim::Constant[value=0]() + %3 : float = prim::Constant[value=0.5]() + # CHECK: clamp_max + %4 : Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::clamp_max(%0, %3) + # CHECK: clamp_max + %5 : Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::clamp_max(%1, %3) + # CHECK: prim::ListConstruct + %6 : Tensor[] = prim::ListConstruct(%4, %5) + # CHECK: aten::cat + %7 : Tensor = aten::cat(%6, %2) + return (%7) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ExpandConcatAndEliminateRedundancy(graph); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // No optimizations should have happened in this case since the output + // shape of `aten::cat` is not known. + testing::FileCheck().run(input, *graph); +} + +TEST(ConcatOptTest, ConcatWithoutInputShape) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %2 : int = prim::Constant[value=0]() + %3 : float = prim::Constant[value=0.5]() + # CHECK: clamp_max + %4 : Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::clamp_max(%0, %3) + # CHECK: clamp_max + %5 : Tensor = aten::clamp_max(%1, %3) + # CHECK: prim::ListConstruct + %6 : Tensor[] = prim::ListConstruct(%4, %5) + # CHECK: aten::cat + %7 : Float(96, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%6, %2) + return (%7) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ExpandConcatAndEliminateRedundancy(graph); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // No optimizations should have happened in this case since the shape of %5, + // which is an input to `aten::cat`, is not known. + testing::FileCheck().run(input, *graph); +} + +TEST(ConcatOptTest, UseVariadicCat) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %2: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %3: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %4: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %5: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %10 : int = prim::Constant[value=0]() + %input : Tensor[] = prim::ListConstruct(%0, %1, %2, %3, %4, %5) + %concat : Float(224, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%input, %10) + return (%concat) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(UseVariadicCat(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // After replacing `aten::cat` with `prim::VarConcat` we should have the + // following graph: + // + // graph(%0 : ..., + // %1 : ...): + // %zero : int = prim:Constant[value=0]() + // %varcat : Tensor = prim::VarConcat(%0, %1, %2, %3, %4, %5, %zero) + // return (%varcat) + testing::FileCheck() + .check_count("= prim::VarConcat(", 1, /*exactly*/ true) + ->check_count("= aten::cat(", 0, /*exactly*/ true) + ->check_count("= prim::ListConstruct(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(OptimizeConcatTest, UseVariadicCatReplaceMultiple) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %2: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %3: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %10 : int = prim::Constant[value=0]() + %input1 : Tensor[] = prim::ListConstruct(%0, %1) + %concat1 : Float(96, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%input1, %10) + %input2 : Tensor[] = prim::ListConstruct(%2, %3) + %concat2 : Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%input2, %10) + return (%concat1, %concat2) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(UseVariadicCat(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // After full concat optimization we should have the following graph: + // + // graph(%0 : ..., + // %1 : ..., + // %2 : ..., + // %3 : ....): + // %zero : int = prim:Constant[value=0]() + // %varcat1 : Tensor = prim::VarConcat(%0, %1, %zero) + // %varcat2 : Tensor = prim::VarConcat(%2, %3, %zero) + // return (%varcat1, %varcat2) + testing::FileCheck() + .check_count("= prim::VarConcat(", 2, /*exactly*/ true) + ->check_count("= aten::cat(", 0, /*exactly*/ true) + ->check_count("= prim::ListConstruct(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConcatOptTest, UseVariadicCatWithMultipleListUses) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %2 : int = prim::Constant[value=0]() + %input : Tensor[] = prim::ListConstruct(%0, %1) + %concat : Float(96, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%input, %2) + return (%concat, %input) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(UseVariadicCat(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // After replacing `aten::cat` with `prim::VarConcat` we should have the + // following graph: + // + // graph(%0 : ..., + // %1 : ...): + // %zero : int = prim:Constant[value=0]() + // %input : Tensor[] = prim::ListConstruct(%0, %1) + // %varcat : Tensor = prim::VarConcat(%0, %1, %zero) + // return (%varcat, %input) + testing::FileCheck() + .check_count("= prim::ListConstruct(", 1, /*exactly*/ true) + ->check_count("= prim::VarConcat(", 1, /*exactly*/ true) + ->check_count("= aten::cat(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConcatOptTest, UseVariadicCatWithListMutationAfterCat) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %2: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %10 : int = prim::Constant[value=0]() + %input : Tensor[] = prim::ListConstruct(%0, %1) + %concat : Float(96, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%input, %10) + %11 : Tensor = aten::append(%input, %2) + return (%concat, %input) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(UseVariadicCat(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // The input list to `aten::cat` is mutated only after `aten::cat` op. So, + // it should have been replaced with `prim::VarConcat`. The transformed graph + // should look like the following: + // + // graph(%0 : ..., + // %1 : ..., + // %2 : ...): + // %3 : int = prim:Constant[value=0]() + // %4 : Tensor[] = prim::ListConstruct(%0, %1) + // %7 : Tensor = prim::VarConcat(%0, %1, %3) + // %6 : Tensor = aten::append(%4, %2) + // return (%7, %4) + testing::FileCheck() + .check_count("= prim::ListConstruct(", 1, /*exactly*/ true) + ->check_count("= prim::VarConcat(", 1, /*exactly*/ true) + ->check_count("= aten::cat(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConcatOptTest, UseVariadicCatWithListMutationBeforeCat) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %2: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %10 : int = prim::Constant[value=0]() + %input : Tensor[] = prim::ListConstruct(%0, %1) + %11 : Tensor = aten::append(%input, %2) + %concat : Float(128, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%input, %10) + return (%concat) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + { + ASSERT_FALSE(UseVariadicCat(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // No transformation should have happened since the `prim::ListConstruct` is + // mutated before `aten::cat`. + testing::FileCheck() + .check_count("= prim::ListConstruct(", 1, /*exactly*/ true) + ->check_count("= aten::cat(", 1, /*exactly*/ true) + ->check_count("= prim::VarConcat(", 0, /*exactly*/ true) + ->run(*graph); + } + + { + ASSERT_TRUE(RemoveListMutationAndUseVariadicCat(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // The mutation of the list must be removed and the `aten::cat` op must + // be replaced with the `prim::VarConcat` op in the graph. The transformed + // graph should look like the following: + // + // graph(%0 : ..., + // %1 : ..., + // %2 : ...): + // %3 : int = prim:Constant[value=0]() + // %7 : Tensor = prim::VarConcat(%0, %1, %2, %3) + // return (%7) + testing::FileCheck() + .check_count("= prim::VarConcat(", 1, /*exactly*/ true) + ->check_count("= prim::ListConstruct(", 0, /*exactly*/ true) + ->check_count("= aten::cat(", 0, /*exactly*/ true) + ->run(*graph); + } +} + +TEST(ConcatOptTest, UseVariadicCatWithMultipleListMutations) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %2: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %3: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %4: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %10 : int = prim::Constant[value=0]() + %input : Tensor[] = prim::ListConstruct(%0, %1) + %concat.1 : Float(96, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%input, %10) + %11 : Tensor = aten::append(%input, %2) + %concat.2 : Float(128, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%input, %10) + %12 : Tensor = aten::append(%input, %3) + %concat.3 : Float(160, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%input, %10) + %13 : Tensor = aten::append(%input, %4) + %concat.4 : Float(192, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%input, %10) + return (%concat.1, %concat.2, %concat.3, %concat.4) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(RemoveListMutationAndUseVariadicCat(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // All the mutations of the list must be removed and the `aten::cat` ops must + // be replaced with `prim::VarConcat` ops in the graph. The transformed graph + // should look like the following: + // + // graph(%0 : ..., + // %1 : ..., + // %2 : ..., + // %3 : ..., + // %4 : ...): + // %10 : int = prim:Constant[value=0]() + // %5 : Tensor = prim::VarConcat(%0, %1, %10) + // %6 : Tensor = prim::VarConcat(%0, %1, %2, %10) + // %7 : Tensor = prim::VarConcat(%0, %1, %2, %3, %10) + // %8 : Tensor = prim::VarConcat(%0, %1, %2, %3, %4, %10) + // return (%5, %6, %7, %8) + testing::FileCheck() + .check_count("= prim::VarConcat(", 4, /*exactly*/ true) + ->check_count("= prim::ListConstruct(", 0, /*exactly*/ true) + ->check_count("= aten::cat(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST( + ConcatOptTest, + RemoveListMutationUseVariadicCatAndCommonInputsElimination) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(64, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %1: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu), + %2: Float(32, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu)): + %5 : int = prim::Constant[value=0]() + + %features.2 : Tensor[] = prim::ListConstruct(%0, %1) + %6 : Tensor [] = aten::append(%features.2, %2) + %concat.2 : Float(128, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%features.2, %5) + + %7 : Tensor [] = aten::append(%features.2, %0) + %concat.3 : Float(160, 56, 56, strides=[3136, 56, 1], requires_grad=0, device=cpu) = aten::cat(%features.2, %5) + + %res : Tensor[] = prim::ListConstruct(%concat.2, %concat.3) + return (%res) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({64, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU), + at::rand({32, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(RemoveListMutationAndUseVariadicCat(graph)); + ASSERT_TRUE(EliminateConcatCommonInputs(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // After performing: + // * Remove list mutation + // * Use variadic cat + // * Eliminate common inputs + // we should have the following graph: + // + // graph(%0 : ..., + // %1 : ..., + // %2 : ...): + // %3 : int = prim::Constant[value=0]() + // %10 : Tensor = prim::VarConcat(%0, %1, %2, %3) + // %12 : Tensor = prim::VarConcat(%10, %0, %3) // UPDATED + // %8 : Tensor[] = prim::ListConstruct(%10, %12) + // return (%8) + testing::FileCheck() + .check_count("= prim::VarConcat(%0, %1, %2, %3)", 1, /*exactly*/ true) + ->check_count("= prim::VarConcat(%10, %0, %3)", 1, /*exactly*/ true) + ->check_count("= prim::ListConstruct(%10, %12)", 1, /*exactly*/ true) + ->check_count("= aten::cat(", 0, /*exactly*/ true) + ->check_count("= prim::ListConstruct(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConcatOpt, CombineConcatsSimpleCase) { + auto graph = std::make_shared(); + const std::string input = + R"IR( + graph(%0: Tensor): + %dim : int = prim::Constant[value=0]() + %input.1 : Tensor[] = prim::ListConstruct(%0, %0) + %concat.1 : Tensor = aten::cat(%input.1, %dim) + %input.2 : Tensor[] = prim::ListConstruct(%concat.1, %0) + %concat.2 : Tensor = aten::cat(%input.2, %dim) + return (%concat.2) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = {at::rand({1})}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(CombineConcats(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // After performing CombineConcats: + // graph(%0 : Tensor): + // %dim : int = prim::Constant[value=0]() + // %input : Tensor[] = prim::ListConstruct(%0, %0, %0) + // %concat : Tensor = aten::cat(%input, %dim) + // return (%concat) + testing::FileCheck() + .check_count("prim::ListConstruct", 1, /*exactly*/ true) + ->check_count("aten::cat", 1, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConcatOpt, CombineConcatsLongChain) { + auto graph = std::make_shared(); + const std::string input = + R"IR( + graph(%0: Tensor, %1 : Tensor): + %dim : int = prim::Constant[value=0]() + %input.1 : Tensor[] = prim::ListConstruct(%0, %0) + %concat.1 : Tensor = aten::cat(%input.1, %dim) + %input.2 : Tensor[] = prim::ListConstruct(%1, %concat.1, %1) + %concat.2 : Tensor = aten::cat(%input.2, %dim) + %input.3 : Tensor[] = prim::ListConstruct(%0, %concat.2, %0) + %concat.3 : Tensor = aten::cat(%input.3, %dim) + return (%concat.3) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = {at::rand({1}), at::randn({1})}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(CombineConcats(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // After performing CombineConcats: + // graph(%0 : Tensor): + // %dim : int = prim::Constant[value=0]() + // %input : Tensor[] = prim::ListConstruct(%0, %1, %0, %0, %1, %0) + // %concat : Tensor = aten::cat(%input, %dim) + // return (%concat) + testing::FileCheck() + .check_count("prim::ListConstruct", 1, /*exactly*/ true) + ->check_count("aten::cat", 1, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConcatOpt, CombineConcatsMutation) { + auto graph = std::make_shared(); + const std::string input = + R"IR( + graph(%0: Tensor, %1 : Tensor): + %dim : int = prim::Constant[value=0]() + %input.1 : Tensor[] = prim::ListConstruct(%0, %0) + %concat.1 : Tensor = aten::cat(%input.1, %dim) + %input.2 : Tensor[] = prim::ListConstruct(%1, %concat.1, %1) + %input.3 : Tensor[] = aten::append(%input.2, %0) + %concat.2 : Tensor = aten::cat(%input.2, %dim) + return (%concat.2) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = {at::rand({1}), at::randn({1})}; + // No modifications due to aten::append + ASSERT_FALSE(CombineConcats(graph)); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_constant_pooling.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_constant_pooling.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2930b5704d61c46cb618a256ef6cd31229935b3c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_constant_pooling.cpp @@ -0,0 +1,114 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace torch { +namespace jit { + +TEST(ConstantPoolingTest, Int) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(): + %8 : int = prim::Constant[value=1]() + %10 : int = prim::Constant[value=1]() + return (%8, %10) + )IR", + &*graph); + ConstantPooling(graph); + testing::FileCheck() + .check_count("prim::Constant", 1, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConstantPoolingTest, PoolingAcrossBlocks) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%cond : Tensor): + %a : str = prim::Constant[value="bcd"]() + %3 : bool = aten::Bool(%cond) + %b : str = prim::If(%3) + block0(): + %b.1 : str = prim::Constant[value="abc"]() + -> (%b.1) + block1(): + %b.2 : str = prim::Constant[value="abc"]() + -> (%b.2) + %7 : (str, str) = prim::TupleConstruct(%a, %b) + return (%7) + )IR", + &*graph); + ConstantPooling(graph); + testing::FileCheck() + .check_count("prim::Constant[value=\"abc\"]", 1, /*exactly*/ true) + ->check_count("prim::Constant[value=\"bcd\"]", 1, /*exactly*/ true) + ->run(*graph); +} + +TEST(ConstantPoolingTest, PoolingDifferentDevices) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(): + %2 : int = prim::Constant[value=2]() + %1 : int = prim::Constant[value=1]() + %5 : int? = prim::Constant() + %7 : Device? = prim::Constant() + %15: bool = prim::Constant[value=0]() + %10 : int = prim::Constant[value=6]() + %3 : int[] = prim::ListConstruct(%1, %2) + %x : Tensor = aten::tensor(%3, %5, %7, %15) + %y : Tensor = aten::tensor(%3, %10, %7, %15) + %9 : int[] = prim::ListConstruct(%1, %2) + %z : Tensor = aten::tensor(%9, %10, %7, %15) + prim::Print(%x, %y, %z) + return (%1) + )IR", + &*graph); + // three tensors created - two different devices among the three + // don't have good support for parsing tensor constants + ConstantPropagation(graph); + ConstantPooling(graph); + testing::FileCheck() + .check_count( + "Float(2, strides=[1], requires_grad=0, device=cpu) = prim::Constant", + 1, + /*exactly*/ true) + ->check_count( + "Long(2, strides=[1], requires_grad=0, device=cpu) = prim::Constant", + 1, + /*exactly*/ true) + ->run(*graph); +} + +TEST(ConstantPoolingTest, DictConstantPooling) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(): + %0 : int = prim::Constant[value=1]() # test/elias.py:6:9 + %1 : int = prim::Constant[value=2]() # test/elias.py:6:12 + %a.1 : Dict(int, int) = prim::DictConstruct(%0, %1) + %b.1 : Dict(int, int) = prim::DictConstruct(%1, %1) + return (%a.1, %b.1) + )IR", + &*graph); + ConstantPropagation(graph); + ConstantPooling(graph); + testing::FileCheck() + .check_count( + "Dict(int, int) = prim::Constant", + 2, + /*exactly*/ true) + ->run(*graph); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_create_autodiff_subgraphs.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_create_autodiff_subgraphs.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0390938492aad8dfa1c3c90e074a2d9aad80b4c4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_create_autodiff_subgraphs.cpp @@ -0,0 +1,25 @@ +#include + +#include "test/cpp/jit/test_utils.h" + +#include "torch/csrc/jit/passes/create_autodiff_subgraphs.h" + +namespace torch { +namespace jit { + +TEST(CreateAutodiffSubgraphsTest, Basic) { + auto graph = build_lstm(); + CreateAutodiffSubgraphs(graph, /*threshold=*/2); + // all of the ops are within the DifferentiableGraph + testing::FileCheck() + .check_not("aten::mm") + ->check_not("aten::sigmoid") + ->check_not("aten::tanh") + ->check_not("aten::mul") + ->check("DifferentiableGraph") + ->check_next("return") + ->run(*graph); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_cs_debug_info_serialization.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_cs_debug_info_serialization.cpp new file mode 100644 index 0000000000000000000000000000000000000000..48681f7beb64373f2a2196b2694cbf33e2782f9e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_cs_debug_info_serialization.cpp @@ -0,0 +1,155 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +// Tests go in torch::jit +namespace torch { +namespace jit { + +namespace { +bool validate_debug_info( + const DebugInfoTuple& pre_serialize, + const DebugInfoTuple& post_serialize) { + auto sr1 = std::get(pre_serialize); + auto sr2 = std::get(post_serialize); + if (sr1 != sr2) { + return false; + } + auto csptr1 = std::get(pre_serialize); + auto csptr2 = std::get(post_serialize); + if (!csptr1.defined()) { + return !csptr2.defined(); + } + if (!csptr2.defined()) { + return false; + } + auto vec1 = csptr1->vec(); + auto vec2 = csptr2->vec(); + if (vec1.size() != vec2.size()) { + return false; + } + while (csptr1) { + auto rhs_sr = csptr1->source_range(); + auto lhs_sr = csptr2->source_range(); + auto rhs_module = csptr1->module_instance(); + auto lhs_module = csptr2->module_instance(); + std::string rhs_fn_name, lhs_fn_name; + if (csptr1->function()) { + rhs_fn_name = csptr1->function()->name(); + } else { + rhs_fn_name = csptr1->function_name(); + } + if (csptr2->function()) { + lhs_fn_name = csptr2->function()->name(); + } else { + lhs_fn_name = csptr2->function_name(); + } + if (!((rhs_module.has_value() == lhs_module.has_value()) && + (rhs_module.has_value() && + (rhs_module.value().class_type()->name().value() == + lhs_module.value().class_type()->name().value()) && + (rhs_module.value().instance_name() == + lhs_module.value().instance_name())) && + (rhs_fn_name == lhs_fn_name) && (rhs_sr == lhs_sr))) { + return false; + } + if (csptr1->callee()) { + csptr1 = csptr1->callee().value(); + csptr2 = csptr2->callee().value(); + } else { + csptr1 = c10::intrusive_ptr(); + } + } + return true; +} + +TEST(CSDebugInfoSerializaitionTest, TwoSubmodules) { + std::shared_ptr cu = std::make_shared(); + Module a("A", cu); + a.define(R"JIT( + def forward(self, x): + return x + 1 + )JIT"); + Module b("B", cu); + b.define(R"JIT( + def forward(self, x): + return x + 2 + )JIT"); + Module c("C", cu); + c.register_module("A0", a); + c.register_module("B0", b); + c.define(R"JIT( + def forward(self, x): + return self.A0.forward(x) + self.B0.forward(x) + )JIT"); + + BackendDebugInfoRecorder debug_info_recorder; + auto graph = c.get_method("forward").graph(); + Inline(*graph); + std::stack blocks_to_visit; + + // maps from source range to debug handle + SourceRangeTagMap source_range_tags; + // Maps from debug handle to source range + ska::flat_hash_map source_range_map; + int64_t source_range_tag{0}; + + blocks_to_visit.push(graph->block()); + while (!blocks_to_visit.empty()) { + Block* b = blocks_to_visit.top(); + blocks_to_visit.pop(); + for (Node* n : b->nodes()) { + source_range_tags[n->sourceRange()] = source_range_tag; + source_range_map[source_range_tag] = n->sourceRange(); + source_range_tag++; + debug_info_recorder.getNextDebugHandle(n); + if (n->callstack().has_value()) { + for (const auto& e : n->callstack().value()->vec()) { + auto sr = std::get<1>(e); + source_range_tags[sr] = source_range_tag; + source_range_map[source_range_tag] = sr; + source_range_tag++; + } + } + } + } + auto debug_handle_cs_ptr_map = debug_info_recorder.stopRecording(); + CallStackDebugInfoPickler cs_debug_info_pickler; + auto cs_data = + cs_debug_info_pickler.pickle(debug_handle_cs_ptr_map, source_range_tags); + at::DataPtr data_ptr(cs_data.data(), DeviceType::CPU); + CallStackDebugInfoUnpickler unpickler; + auto deserialized_cs_map = unpickler.unpickle( + std::move(data_ptr), cs_data.size(), source_range_map, cu); + for (const auto& it : debug_handle_cs_ptr_map) { + auto handle = it.first; + auto debug_info_one = it.second; + TORCH_CHECK( + deserialized_cs_map.count(handle), + "Serialized debug handle must be in deserialized map."); + auto debug_info_two = deserialized_cs_map[handle]; + ASSERT_TRUE(validate_debug_info(debug_info_one, debug_info_two)); + } +} + +} // namespace + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_custom_class.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_custom_class.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3665ffa8f18381e04c1dd25cea0825a416ce9dc2 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_custom_class.cpp @@ -0,0 +1,150 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace torch { +namespace jit { + +TEST(CustomClassTest, TorchbindIValueAPI) { + script::Module m("m"); + + // test make_custom_class API + auto custom_class_obj = make_custom_class>( + std::vector{"foo", "bar"}); + m.define(R"( + def forward(self, s : __torch__.torch.classes._TorchScriptTesting._StackString): + return s.pop(), s + )"); + + auto test_with_obj = [&m](IValue obj, std::string expected) { + auto res = m.run_method("forward", obj); + auto tup = res.toTuple(); + AT_ASSERT(tup->elements().size() == 2); + auto str = tup->elements()[0].toStringRef(); + auto other_obj = + tup->elements()[1].toCustomClass>(); + AT_ASSERT(str == expected); + auto ref_obj = obj.toCustomClass>(); + AT_ASSERT(other_obj.get() == ref_obj.get()); + }; + + test_with_obj(custom_class_obj, "bar"); + + // test IValue() API + auto my_new_stack = c10::make_intrusive>( + std::vector{"baz", "boo"}); + auto new_stack_ivalue = c10::IValue(my_new_stack); + + test_with_obj(new_stack_ivalue, "boo"); +} + +TEST(CustomClassTest, ScalarTypeClass) { + script::Module m("m"); + + // test make_custom_class API + auto cc = make_custom_class(at::kFloat); + m.register_attribute("s", cc.type(), cc, false); + + std::ostringstream oss; + m.save(oss); + std::istringstream iss(oss.str()); + caffe2::serialize::IStreamAdapter adapter{&iss}; + auto loaded_module = torch::jit::load(iss, torch::kCPU); +} + +class TorchBindTestClass : public torch::jit::CustomClassHolder { + public: + std::string get() { + return "Hello, I am your test custom class"; + } +}; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) +constexpr char class_doc_string[] = R"( + I am docstring for TorchBindTestClass + Args: + What is an argument? Oh never mind, I don't take any. + + Return: + How would I know? I am just a holder of some meaningless test methods. + )"; +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) +constexpr char method_doc_string[] = + "I am docstring for TorchBindTestClass get_with_docstring method"; + +namespace { +static auto reg = + torch::class_( + "_TorchBindTest", + "_TorchBindTestClass", + class_doc_string) + .def("get", &TorchBindTestClass::get) + .def("get_with_docstring", &TorchBindTestClass::get, method_doc_string); + +} // namespace + +// Tests DocString is properly propagated when defining CustomClasses. +TEST(CustomClassTest, TestDocString) { + auto class_type = getCustomClass( + "__torch__.torch.classes._TorchBindTest._TorchBindTestClass"); + AT_ASSERT(class_type); + AT_ASSERT(class_type->doc_string() == class_doc_string); + + AT_ASSERT(class_type->getMethod("get").doc_string().empty()); + AT_ASSERT( + class_type->getMethod("get_with_docstring").doc_string() == + method_doc_string); +} + +TEST(CustomClassTest, Serialization) { + script::Module m("m"); + + // test make_custom_class API + auto custom_class_obj = make_custom_class>( + std::vector{"foo", "bar"}); + m.register_attribute( + "s", + custom_class_obj.type(), + custom_class_obj, + // NOLINTNEXTLINE(bugprone-argument-comment) + /*is_parameter=*/false); + m.define(R"( + def forward(self): + return self.s.return_a_tuple() + )"); + + auto test_with_obj = [](script::Module& mod) { + auto res = mod.run_method("forward"); + auto tup = res.toTuple(); + AT_ASSERT(tup->elements().size() == 2); + auto i = tup->elements()[1].toInt(); + AT_ASSERT(i == 123); + }; + + auto frozen_m = torch::jit::freeze_module(m.clone()); + + test_with_obj(m); + test_with_obj(frozen_m); + + std::ostringstream oss; + m.save(oss); + std::istringstream iss(oss.str()); + caffe2::serialize::IStreamAdapter adapter{&iss}; + auto loaded_module = torch::jit::load(iss, torch::kCPU); + + std::ostringstream oss_frozen; + frozen_m.save(oss_frozen); + std::istringstream iss_frozen(oss_frozen.str()); + caffe2::serialize::IStreamAdapter adapter_frozen{&iss_frozen}; + auto loaded_frozen_module = torch::jit::load(iss_frozen, torch::kCPU); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_custom_class_registrations.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_custom_class_registrations.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6e4cfbc22ec0b8838adfb40a5b4848b75335ab53 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_custom_class_registrations.cpp @@ -0,0 +1,789 @@ +#include + +#include +#include + +#include +#include +#include + +using namespace torch::jit; + +namespace { + +struct DefaultArgs : torch::CustomClassHolder { + int x; + DefaultArgs(int64_t start = 3) : x(start) {} + int64_t increment(int64_t val = 1) { + x += val; + return x; + } + int64_t decrement(int64_t val = 1) { + x += val; + return x; + } + int64_t scale_add(int64_t add, int64_t scale = 1) { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + x = scale * x + add; + return x; + } + int64_t divide(std::optional factor) { + if (factor) { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + x = x / *factor; + } + return x; + } +}; + +struct Foo : torch::CustomClassHolder { + int x, y; + Foo() : x(0), y(0) {} + Foo(int x_, int y_) : x(x_), y(y_) {} + int64_t info() { + return this->x * this->y; + } + int64_t add(int64_t z) { + return (x + y) * z; + } + at::Tensor add_tensor(at::Tensor z) { + return (x + y) * z; + } + void increment(int64_t z) { + this->x += z; + this->y += z; + } + int64_t combine(c10::intrusive_ptr b) { + return this->info() + b->info(); + } + bool eq(c10::intrusive_ptr other) { + return this->x == other->x && this->y == other->y; + } + std::tuple, std::tuple> + __obj_flatten__() { + return std::tuple(std::tuple("x", this->x), std::tuple("y", this->y)); + } +}; + +struct _StaticMethod : torch::CustomClassHolder { + // NOLINTNEXTLINE(modernize-use-equals-default) + _StaticMethod() {} + static int64_t staticMethod(int64_t input) { + return 2 * input; + } + static int64_t staticMethodWithDefault( + int64_t input, + std::optional scale) { + return scale.value_or(2) * input; + } +}; + +struct FooGetterSetter : torch::CustomClassHolder { + FooGetterSetter() : x(0), y(0) {} + FooGetterSetter(int64_t x_, int64_t y_) : x(x_), y(y_) {} + + int64_t getX() { + // to make sure this is not just attribute lookup + return x + 2; + } + void setX(int64_t z) { + // to make sure this is not just attribute lookup + x = z + 2; + } + + int64_t getY() { + // to make sure this is not just attribute lookup + return y + 4; + } + + private: + int64_t x, y; +}; + +struct FooGetterSetterLambda : torch::CustomClassHolder { + int64_t x; + FooGetterSetterLambda() : x(0) {} + FooGetterSetterLambda(int64_t x_) : x(x_) {} +}; + +struct FooReadWrite : torch::CustomClassHolder { + int64_t x; + const int64_t y; + FooReadWrite() : x(0), y(0) {} + FooReadWrite(int64_t x_, int64_t y_) : x(x_), y(y_) {} +}; + +struct LambdaInit : torch::CustomClassHolder { + int x, y; + LambdaInit(int x_, int y_) : x(x_), y(y_) {} + int64_t diff() { + return this->x - this->y; + } +}; + +// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) +struct NoInit : torch::CustomClassHolder { + int64_t x; +}; + +struct PickleTester : torch::CustomClassHolder { + PickleTester(std::vector vals) : vals(std::move(vals)) {} + std::vector vals; +}; + +// Thread-safe Tensor Queue +struct TensorQueue : torch::CustomClassHolder { + explicit TensorQueue(at::Tensor t) : init_tensor_(t) {} + + explicit TensorQueue(c10::Dict dict) { + init_tensor_ = dict.at(std::string("init_tensor")); + const std::string key = "queue"; + at::Tensor size_tensor; + size_tensor = dict.at(std::string(key + "/size")).cpu(); + const auto* size_tensor_acc = size_tensor.const_data_ptr(); + int64_t queue_size = size_tensor_acc[0]; + + for (const auto index : c10::irange(queue_size)) { + at::Tensor val; + queue_[index] = dict.at(key + "/" + std::to_string(index)); + queue_.push_back(val); + } + } + + std::tuple< + std::tuple, + std::tuple>> + serialize() { + return std::tuple( + std::tuple("init_tensor", this->init_tensor_.clone()), + std::tuple("queue", this->clone_queue())); + } + + static c10::intrusive_ptr deserialize( + std::tuple< + std::tuple, + std::tuple>> flattened) { + TORCH_CHECK(std::tuple_size::value == 2); + + auto init_tensor_tuple = std::get<0>(flattened); + TORCH_CHECK(std::tuple_size::value == 2); + TORCH_CHECK(std::get<0>(init_tensor_tuple) == std::string("init_tensor")); + + c10::intrusive_ptr queue = + c10::make_intrusive(std::get<1>(init_tensor_tuple)); + + auto queue_tuple = std::get<1>(flattened); + TORCH_CHECK(std::tuple_size::value == 2); + TORCH_CHECK(std::get<0>(queue_tuple) == std::string("queue")); + + for (auto& value : std::get<1>(queue_tuple)) { + queue->push(value); + } + + return queue; + } + + // Push the element to the rear of queue. + // Lock is added for thread safe. + void push(at::Tensor x) { + std::lock_guard guard(mutex_); + queue_.push_back(x); + } + // Pop the front element of queue and return it. + // If empty, return init_tensor_. + // Lock is added for thread safe. + at::Tensor pop() { + std::lock_guard guard(mutex_); + if (!queue_.empty()) { + auto val = queue_.front(); + queue_.pop_front(); + return val; + } else { + return init_tensor_; + } + } + // Return front element of queue, read-only. + // We might further optimize with read-write lock. + at::Tensor top() { + std::lock_guard guard(mutex_); + if (!queue_.empty()) { + auto val = queue_.front(); + return val; + } else { + return init_tensor_; + } + } + int64_t size() { + return queue_.size(); + } + + bool is_empty() { + std::lock_guard guard(mutex_); + return queue_.empty(); + } + + double float_size() { + return 1. * queue_.size(); + } + + std::vector clone_queue() { + std::lock_guard guard(mutex_); + std::vector ret; + for (const auto& t : queue_) { + ret.push_back(t.clone()); + } + return ret; + } + std::vector get_raw_queue() { + std::vector raw_queue(queue_.begin(), queue_.end()); + return raw_queue; + } + + std::tuple>> __obj_flatten__() { + return std::tuple(std::tuple("queue", this->get_raw_queue())); + } + + private: + std::deque queue_; + std::mutex mutex_; + at::Tensor init_tensor_; +}; + +struct ConstantTensorContainer : torch::CustomClassHolder { + explicit ConstantTensorContainer(at::Tensor x) : x_(x) {} + + at::Tensor get() { + return x_; + } + + std::string tracing_mode() { + return "real"; + } + + private: + at::Tensor x_; +}; + +at::Tensor take_an_instance(const c10::intrusive_ptr& instance) { + return torch::zeros({instance->vals.back(), 4}); +} + +struct ElementwiseInterpreter : torch::CustomClassHolder { + using InstructionType = std::tuple< + std::string /*op*/, + std::vector /*inputs*/, + std::string /*output*/>; + + // NOLINTNEXTLINE(modernize-use-equals-default) + ElementwiseInterpreter() {} + + // Load a list of instructions into the interpreter. As specified above, + // instructions specify the operation (currently support "add" and "mul"), + // the names of the input values, and the name of the single output value + // from this instruction + void setInstructions(std::vector instructions) { + instructions_ = std::move(instructions); + } + + // Add a constant. The interpreter maintains a set of constants across + // calls. They are keyed by name, and constants can be referenced in + // Instructions by the name specified + void addConstant(const std::string& name, at::Tensor value) { + constants_.insert_or_assign(name, std::move(value)); + } + + // Set the string names for the positional inputs to the function this + // interpreter represents. When invoked, the interpreter will assign + // the positional inputs to the names in the corresponding position in + // input_names. + void setInputNames(std::vector input_names) { + input_names_ = std::move(input_names); + } + + // Specify the output name for the function this interpreter represents. This + // should match the "output" field of one of the instructions in the + // instruction list, typically the last instruction. + void setOutputName(std::string output_name) { + output_name_ = std::move(output_name); + } + + // Invoke this interpreter. This takes a list of positional inputs and returns + // a single output. Currently, inputs and outputs must all be Tensors. + at::Tensor __call__(std::vector inputs) { + // Environment to hold local variables + std::unordered_map environment; + + // Load inputs according to the specified names + if (inputs.size() != input_names_.size()) { + std::stringstream err; + err << "Expected " << input_names_.size() << " inputs, but got " + << inputs.size() << '!'; + throw std::runtime_error(std::move(err).str()); + } + for (size_t i = 0; i < inputs.size(); ++i) { + environment[input_names_[i]] = inputs[i]; + } + + for (InstructionType& instr : instructions_) { + // Retrieve all input values for this op + std::vector inputs; + for (const auto& input_name : std::get<1>(instr)) { + // Operator output values shadow constants. + // Imagine all constants are defined in statements at the beginning + // of a function (a la K&R C). Any definition of an output value must + // necessarily come after constant definition in textual order. Thus, + // We look up values in the environment first then the constant table + // second to implement this shadowing behavior + if (environment.find(input_name) != environment.end()) { + inputs.push_back(environment.at(input_name)); + } else if (constants_.find(input_name) != constants_.end()) { + inputs.push_back(constants_.at(input_name)); + } else { + std::stringstream err; + err << "Instruction referenced unknown value " << input_name << '!'; + throw std::runtime_error(std::move(err).str()); + } + } + + // Run the specified operation + at::Tensor result; + const auto& op = std::get<0>(instr); + if (op == "add") { + if (inputs.size() != 2) { + throw std::runtime_error("Unexpected number of inputs for add op!"); + } + result = inputs[0] + inputs[1]; + } else if (op == "mul") { + if (inputs.size() != 2) { + throw std::runtime_error("Unexpected number of inputs for mul op!"); + } + result = inputs[0] * inputs[1]; + } else { + std::stringstream err; + err << "Unknown operator " << op << '!'; + throw std::runtime_error(std::move(err).str()); + } + + // Write back result into environment + const auto& output_name = std::get<2>(instr); + environment[output_name] = std::move(result); + } + + if (!output_name_) { + throw std::runtime_error("Output name not specified!"); + } + + return environment.at(*output_name_); + } + + // Ser/De infrastructure. See + // https://pytorch.org/tutorials/advanced/torch_script_custom_classes.html#defining-serialization-deserialization-methods-for-custom-c-classes + // for more info. + + // This is the type we will use to marshall information on disk during + // Ser/De. It is a simple tuple composed of primitive types and simple + // collection types like vector, optional, and dict. + using SerializationType = std::tuple< + std::vector /*input_names_*/, + std::optional /*output_name_*/, + c10::Dict /*constants_*/, + std::vector /*instructions_*/ + >; + + // This function yields the SerializationType instance for `this`. + SerializationType __getstate__() const { + return SerializationType{ + input_names_, output_name_, constants_, instructions_}; + } + + // This function will create an instance of `ElementwiseInterpreter` given + // an instance of `SerializationType`. + static c10::intrusive_ptr __setstate__( + SerializationType state) { + auto instance = c10::make_intrusive(); + std::tie( + instance->input_names_, + instance->output_name_, + instance->constants_, + instance->instructions_) = std::move(state); + return instance; + } + + // Class members + std::vector input_names_; + std::optional output_name_; + c10::Dict constants_; + std::vector instructions_; +}; + +struct ReLUClass : public torch::CustomClassHolder { + at::Tensor run(const at::Tensor& t) { + return t.relu(); + } +}; + +struct FlattenWithTensorOp : public torch::CustomClassHolder { + explicit FlattenWithTensorOp(at::Tensor t) : t_(t) {} + + at::Tensor get() { + // Need to return a copy of the tensor, otherwise the tensor will be + // aliased with a tensor that may be modified by the user or backend. + return t_.clone(); + } + + std::tuple> __obj_flatten__() { + return std::tuple(std::tuple("t", this->t_.sin())); + } + + private: + at::Tensor t_; + ; +}; + +struct ContainsTensor : public torch::CustomClassHolder { + explicit ContainsTensor(at::Tensor t) : t_(t) {} + + at::Tensor get() { + // Need to return a copy of the tensor, otherwise the tensor will be + // aliased with a tensor that may be modified by the user or backend. + return t_.clone(); + } + + std::tuple> __obj_flatten__() { + return std::tuple(std::tuple("t", this->t_)); + } + + at::Tensor t_; +}; + +TORCH_LIBRARY(_TorchScriptTesting, m) { + m.impl_abstract_pystub("torch.testing._internal.torchbind_impls"); + m.class_("_ScalarTypeClass") + .def(torch::init()) + .def_pickle( + [](const c10::intrusive_ptr& self) { + return std::make_tuple(self->scalar_type_); + }, + [](std::tuple s) { + return c10::make_intrusive(std::get<0>(s)); + }); + + m.class_("_ReLUClass") + .def(torch::init<>()) + .def("run", &ReLUClass::run); + + m.class_<_StaticMethod>("_StaticMethod") + .def(torch::init<>()) + .def_static("staticMethod", &_StaticMethod::staticMethod) + .def_static( + "staticMethodWithDefault", + &_StaticMethod::staticMethodWithDefault, + "", + {torch::arg("input"), torch::arg("scale") = torch::arg::none()}); + + m.class_("_DefaultArgs") + .def(torch::init(), "", {torch::arg("start") = 3}) + .def("increment", &DefaultArgs::increment, "", {torch::arg("val") = 1}) + .def("decrement", &DefaultArgs::decrement, "", {torch::arg("val") = 1}) + .def( + "scale_add", + &DefaultArgs::scale_add, + "", + {torch::arg("add"), torch::arg("scale") = 1}) + .def( + "divide", + &DefaultArgs::divide, + "", + {torch::arg("factor") = torch::arg::none()}); + + m.class_("_Foo") + .def(torch::init()) + // .def(torch::init<>()) + .def("info", &Foo::info) + .def("increment", &Foo::increment) + .def("add", &Foo::add) + .def("add_tensor", &Foo::add_tensor) + .def("__eq__", &Foo::eq) + .def("combine", &Foo::combine) + .def("__obj_flatten__", &Foo::__obj_flatten__) + .def_pickle( + [](c10::intrusive_ptr self) { // __getstate__ + return std::vector{self->x, self->y}; + }, + [](std::vector state) { // __setstate__ + return c10::make_intrusive(state[0], state[1]); + }); + + m.class_("_FlattenWithTensorOp") + .def(torch::init()) + .def("get", &FlattenWithTensorOp::get) + .def("__obj_flatten__", &FlattenWithTensorOp::__obj_flatten__) + .def_pickle( + // __getstate__ + [](const c10::intrusive_ptr& self) + -> at::Tensor { return self->get(); }, + // __setstate__ + [](at::Tensor data) -> c10::intrusive_ptr { + return c10::make_intrusive(std::move(data)); + }); + + m.class_("_ConstantTensorContainer") + .def(torch::init()) + .def("get", &ConstantTensorContainer::get) + .def("tracing_mode", &ConstantTensorContainer::tracing_mode); + + m.def( + "takes_foo(__torch__.torch.classes._TorchScriptTesting._Foo foo, Tensor x) -> Tensor"); + m.def( + "takes_foo_python_meta(__torch__.torch.classes._TorchScriptTesting._Foo foo, Tensor x) -> Tensor"); + m.def( + "takes_foo_list_return(__torch__.torch.classes._TorchScriptTesting._Foo foo, Tensor x) -> Tensor[]"); + m.def( + "takes_foo_tuple_return(__torch__.torch.classes._TorchScriptTesting._Foo foo, Tensor x) -> (Tensor, Tensor)"); + m.def( + "takes_foo_tensor_return(__torch__.torch.classes._TorchScriptTesting._Foo foo, Tensor x) -> Tensor"); + + m.class_("_FooGetterSetter") + .def(torch::init()) + .def_property("x", &FooGetterSetter::getX, &FooGetterSetter::setX) + .def_property("y", &FooGetterSetter::getY); + + m.class_("_FooGetterSetterLambda") + .def(torch::init()) + .def_property( + "x", + [](const c10::intrusive_ptr& self) { + return self->x; + }, + [](const c10::intrusive_ptr& self, + int64_t val) { self->x = val; }); + + m.class_("_FooReadWrite") + .def(torch::init()) + .def_readwrite("x", &FooReadWrite::x) + .def_readonly("y", &FooReadWrite::y); + + m.class_("_LambdaInit") + .def(torch::init([](int64_t x, int64_t y, bool swap) { + if (swap) { + return c10::make_intrusive(y, x); + } else { + return c10::make_intrusive(x, y); + } + })) + .def("diff", &LambdaInit::diff); + + m.class_("_NoInit").def( + "get_x", [](const c10::intrusive_ptr& self) { return self->x; }); + + m.class_>("_StackString") + .def(torch::init>()) + .def("push", &MyStackClass::push) + .def("pop", &MyStackClass::pop) + .def("clone", &MyStackClass::clone) + .def("merge", &MyStackClass::merge) + .def_pickle( + [](const c10::intrusive_ptr>& self) { + return self->stack_; + }, + [](std::vector state) { // __setstate__ + return c10::make_intrusive>( + std::vector{"i", "was", "deserialized"}); + }) + .def("return_a_tuple", &MyStackClass::return_a_tuple) + .def( + "top", + [](const c10::intrusive_ptr>& self) + -> std::string { return self->stack_.back(); }) + .def( + "__str__", + [](const c10::intrusive_ptr>& self) { + std::stringstream ss; + ss << '['; + for (size_t i = 0; i < self->stack_.size(); ++i) { + ss << self->stack_[i]; + if (i != self->stack_.size() - 1) { + ss << ", "; + } + } + ss << ']'; + return std::move(ss).str(); + }); + // clang-format off + // The following will fail with a static assert telling you you have to + // take an intrusive_ptr as the first argument. + // .def("foo", [](int64_t a) -> int64_t{ return 3;}); + // clang-format on + + m.class_("_PickleTester") + .def(torch::init>()) + .def_pickle( + [](c10::intrusive_ptr self) { // __getstate__ + return std::vector{1, 3, 3, 7}; + }, + [](std::vector state) { // __setstate__ + return c10::make_intrusive(std::move(state)); + }) + .def( + "top", + [](const c10::intrusive_ptr& self) { + return self->vals.back(); + }) + .def("pop", [](const c10::intrusive_ptr& self) { + auto val = self->vals.back(); + self->vals.pop_back(); + return val; + }); + + m.def( + "take_an_instance(__torch__.torch.classes._TorchScriptTesting._PickleTester x) -> Tensor Y", + take_an_instance); + // test that schema inference is ok too + m.def("take_an_instance_inferred", take_an_instance); + + m.class_("_ElementwiseInterpreter") + .def(torch::init<>()) + .def("set_instructions", &ElementwiseInterpreter::setInstructions) + .def("add_constant", &ElementwiseInterpreter::addConstant) + .def("set_input_names", &ElementwiseInterpreter::setInputNames) + .def("set_output_name", &ElementwiseInterpreter::setOutputName) + .def("__call__", &ElementwiseInterpreter::__call__) + .def_pickle( + /* __getstate__ */ + [](const c10::intrusive_ptr& self) { + return self->__getstate__(); + }, + /* __setstate__ */ + [](ElementwiseInterpreter::SerializationType state) { + return ElementwiseInterpreter::__setstate__(std::move(state)); + }); + + m.class_("_ContainsTensor") + .def(torch::init()) + .def("get", &ContainsTensor::get) + .def("__obj_flatten__", &ContainsTensor::__obj_flatten__) + .def_pickle( + // __getstate__ + [](const c10::intrusive_ptr& self) -> at::Tensor { + return self->t_; + }, + // __setstate__ + [](at::Tensor data) -> c10::intrusive_ptr { + return c10::make_intrusive(std::move(data)); + }); + m.class_("_TensorQueue") + .def(torch::init()) + .def("push", &TensorQueue::push) + .def("pop", &TensorQueue::pop) + .def("top", &TensorQueue::top) + .def("is_empty", &TensorQueue::is_empty) + .def("float_size", &TensorQueue::float_size) + .def("size", &TensorQueue::size) + .def("clone_queue", &TensorQueue::clone_queue) + .def("get_raw_queue", &TensorQueue::get_raw_queue) + .def("__obj_flatten__", &TensorQueue::__obj_flatten__) + .def_pickle( + // __getstate__ + [](const c10::intrusive_ptr& self) + -> std::tuple< + std::tuple, + std::tuple>> { + return self->serialize(); + }, + // __setstate__ + [](std::tuple< + std::tuple, + std::tuple>> data) + -> c10::intrusive_ptr { + return TensorQueue::deserialize(data); + }); +} + +at::Tensor takes_foo(c10::intrusive_ptr foo, at::Tensor x) { + return foo->add_tensor(x); +} + +std::vector takes_foo_list_return( + c10::intrusive_ptr foo, + at::Tensor x) { + std::vector result; + result.reserve(3); + auto a = foo->add_tensor(x); + auto b = foo->add_tensor(a); + auto c = foo->add_tensor(b); + result.push_back(a); + result.push_back(b); + result.push_back(c); + return result; +} + +std::tuple takes_foo_tuple_return( + c10::intrusive_ptr foo, + at::Tensor x) { + auto a = foo->add_tensor(x); + auto b = foo->add_tensor(a); + return std::make_tuple(a, b); +} + +at::Tensor takes_foo_tensor_return(c10::intrusive_ptr foo, at::Tensor x) { + return at::ones({foo->x, foo->y}, at::device(at::kCPU).dtype(at::kInt)); +} + +void queue_push(c10::intrusive_ptr tq, at::Tensor x) { + // clone the tensor to avoid aliasing + tq->push(x.clone()); +} + +at::Tensor queue_pop(c10::intrusive_ptr tq) { + return tq->pop(); +} + +int64_t queue_size(c10::intrusive_ptr tq) { + return tq->size(); +} + +TORCH_LIBRARY_FRAGMENT(_TorchScriptTesting, m) { + m.impl_abstract_pystub("torch.testing._internal.torchbind_impls"); + m.def( + "takes_foo_cia(__torch__.torch.classes._TorchScriptTesting._Foo foo, Tensor x) -> Tensor"); + m.def( + "queue_pop(__torch__.torch.classes._TorchScriptTesting._TensorQueue foo) -> Tensor"); + m.def( + "queue_push(__torch__.torch.classes._TorchScriptTesting._TensorQueue foo, Tensor x) -> ()"); + m.def( + "queue_size(__torch__.torch.classes._TorchScriptTesting._TensorQueue foo) -> int"); +} + +TORCH_LIBRARY_IMPL(_TorchScriptTesting, CPU, m) { + m.impl("takes_foo", takes_foo); + m.impl("takes_foo_list_return", takes_foo_list_return); + m.impl("takes_foo_tuple_return", takes_foo_tuple_return); + m.impl("queue_push", queue_push); + m.impl("queue_pop", queue_pop); + m.impl("queue_size", queue_size); + m.impl("takes_foo_tensor_return", takes_foo_tensor_return); +} + +TORCH_LIBRARY_IMPL(_TorchScriptTesting, CUDA, m) { + m.impl("queue_push", queue_push); + m.impl("queue_pop", queue_pop); +} + +TORCH_LIBRARY_IMPL(_TorchScriptTesting, Meta, m) { + m.impl("takes_foo", &takes_foo); + m.impl("takes_foo_list_return", takes_foo_list_return); + m.impl("takes_foo_tuple_return", takes_foo_tuple_return); +} + +TORCH_LIBRARY_IMPL(_TorchScriptTesting, CompositeImplicitAutograd, m) { + m.impl("takes_foo_cia", takes_foo); +} + +// Need to implement BackendSelect because these two operators don't have tensor +// inputs. +TORCH_LIBRARY_IMPL(_TorchScriptTesting, BackendSelect, m) { + m.impl("queue_pop", queue_pop); + m.impl("queue_size", queue_size); +} + +} // namespace diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_custom_class_registrations.h b/western_ai/usa/pytorch/test/cpp/jit/test_custom_class_registrations.h new file mode 100644 index 0000000000000000000000000000000000000000..08b11d7f524bc2bfff3595bde5a23013bdb9a04b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_custom_class_registrations.h @@ -0,0 +1,41 @@ +#include +#include + +namespace torch { +namespace jit { + +struct ScalarTypeClass : public torch::CustomClassHolder { + ScalarTypeClass(at::ScalarType s) : scalar_type_(s) {} + at::ScalarType scalar_type_; +}; + +template +struct MyStackClass : torch::CustomClassHolder { + std::vector stack_; + MyStackClass(std::vector init) : stack_(init.begin(), init.end()) {} + + void push(T x) { + stack_.push_back(x); + } + T pop() { + auto val = stack_.back(); + stack_.pop_back(); + return val; + } + + c10::intrusive_ptr clone() const { + return c10::make_intrusive(stack_); + } + + void merge(const c10::intrusive_ptr& c) { + for (auto& elem : c->stack_) { + push(elem); + } + } + + std::tuple return_a_tuple() const { + return std::make_tuple(1337.0f, 123); + } +}; +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_custom_operators.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_custom_operators.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5dfae23ab64303b96d6a6a674688dcdf89f9c429 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_custom_operators.cpp @@ -0,0 +1,254 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +TEST(CustomOperatorTest, InferredSchema) { + torch::RegisterOperators reg( + "foo::bar", [](double a, at::Tensor b) { return a + b; }); + auto& ops = getAllOperatorsFor(Symbol::fromQualString("foo::bar")); + ASSERT_EQ(ops.size(), 1); + + auto& op = ops.front(); + ASSERT_EQ(op->schema().name(), "foo::bar"); + + ASSERT_EQ(op->schema().arguments().size(), 2); + ASSERT_EQ(op->schema().arguments()[0].name(), "_0"); + ASSERT_EQ(op->schema().arguments()[0].type()->kind(), TypeKind::FloatType); + ASSERT_EQ(op->schema().arguments()[1].name(), "_1"); + ASSERT_EQ(op->schema().arguments()[1].type()->kind(), TypeKind::TensorType); + + ASSERT_EQ(op->schema().returns()[0].type()->kind(), TypeKind::TensorType); + + Stack stack; + push(stack, 2.0f, at::ones(5)); + op->getOperation()(stack); + at::Tensor output; + pop(stack, output); + + ASSERT_TRUE(output.allclose(at::full(5, 3.0f))); +} + +TEST(CustomOperatorTest, ExplicitSchema) { + torch::RegisterOperators reg( + "foo::bar_with_schema(float a, Tensor b) -> Tensor", + [](double a, at::Tensor b) { return a + b; }); + + auto& ops = + getAllOperatorsFor(Symbol::fromQualString("foo::bar_with_schema")); + ASSERT_EQ(ops.size(), 1); + + auto& op = ops.front(); + ASSERT_EQ(op->schema().name(), "foo::bar_with_schema"); + + ASSERT_EQ(op->schema().arguments().size(), 2); + ASSERT_EQ(op->schema().arguments()[0].name(), "a"); + ASSERT_EQ(op->schema().arguments()[0].type()->kind(), TypeKind::FloatType); + ASSERT_EQ(op->schema().arguments()[1].name(), "b"); + ASSERT_EQ(op->schema().arguments()[1].type()->kind(), TypeKind::TensorType); + + ASSERT_EQ(op->schema().returns().size(), 1); + ASSERT_EQ(op->schema().returns()[0].type()->kind(), TypeKind::TensorType); + + Stack stack; + push(stack, 2.0f, at::ones(5)); + op->getOperation()(stack); + at::Tensor output; + pop(stack, output); + + ASSERT_TRUE(output.allclose(at::full(5, 3.0f))); +} + +TEST(CustomOperatorTest, ListParameters) { + // Check that lists work well. + torch::RegisterOperators reg( + "foo::lists(int[] ints, float[] floats, complex[] complexdoubles, Tensor[] tensors) -> float[]", + [](torch::List ints, + torch::List floats, + torch::List> complexdoubles, + torch::List tensors) { return floats; }); + + auto& ops = getAllOperatorsFor(Symbol::fromQualString("foo::lists")); + ASSERT_EQ(ops.size(), 1); + + auto& op = ops.front(); + ASSERT_EQ(op->schema().name(), "foo::lists"); + + ASSERT_EQ(op->schema().arguments().size(), 4); + ASSERT_EQ(op->schema().arguments()[0].name(), "ints"); + ASSERT_TRUE( + op->schema().arguments()[0].type()->isSubtypeOf(*ListType::ofInts())); + ASSERT_EQ(op->schema().arguments()[1].name(), "floats"); + ASSERT_TRUE( + op->schema().arguments()[1].type()->isSubtypeOf(*ListType::ofFloats())); + ASSERT_EQ(op->schema().arguments()[2].name(), "complexdoubles"); + ASSERT_TRUE(op->schema().arguments()[2].type()->isSubtypeOf( + *ListType::ofComplexDoubles())); + ASSERT_EQ(op->schema().arguments()[3].name(), "tensors"); + ASSERT_TRUE( + op->schema().arguments()[3].type()->isSubtypeOf(*ListType::ofTensors())); + + ASSERT_EQ(op->schema().returns().size(), 1); + ASSERT_TRUE( + op->schema().returns()[0].type()->isSubtypeOf(*ListType::ofFloats())); + + Stack stack; + push(stack, c10::List({1, 2})); + push(stack, c10::List({1.0, 2.0})); + push( + stack, + c10::List>( + {c10::complex(2.4, -5.5), c10::complex(-1.3, 2)})); + push(stack, c10::List({at::ones(5)})); + op->getOperation()(stack); + c10::List output; + pop(stack, output); + + ASSERT_EQ(output.size(), 2); + ASSERT_EQ(output.get(0), 1.0); + ASSERT_EQ(output.get(1), 2.0); +} + +TEST(CustomOperatorTest, ListParameters2) { + torch::RegisterOperators reg( + "foo::lists2(Tensor[] tensors) -> Tensor[]", + [](torch::List tensors) { return tensors; }); + + auto& ops = getAllOperatorsFor(Symbol::fromQualString("foo::lists2")); + ASSERT_EQ(ops.size(), 1); + + auto& op = ops.front(); + ASSERT_EQ(op->schema().name(), "foo::lists2"); + + ASSERT_EQ(op->schema().arguments().size(), 1); + ASSERT_EQ(op->schema().arguments()[0].name(), "tensors"); + ASSERT_TRUE( + op->schema().arguments()[0].type()->isSubtypeOf(*ListType::ofTensors())); + + ASSERT_EQ(op->schema().returns().size(), 1); + ASSERT_TRUE( + op->schema().returns()[0].type()->isSubtypeOf(*ListType::ofTensors())); + + Stack stack; + push(stack, c10::List({at::ones(5)})); + op->getOperation()(stack); + c10::List output; + pop(stack, output); + + ASSERT_EQ(output.size(), 1); + ASSERT_TRUE(output.get(0).allclose(at::ones(5))); +} + +TEST(CustomOperatorTest, Aliasing) { + torch::RegisterOperators reg( + "foo::aliasing", [](at::Tensor a, at::Tensor b) -> at::Tensor { + a.add_(b); + return a; + }); + getAllOperatorsFor(Symbol::fromQualString("foo::aliasing")); + + { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%x: Tensor, %y: Tensor): + %ret : Tensor = foo::aliasing(%x, %y) + return (%ret) + )IR", + graph.get()); + + auto opNode = *graph->block()->nodes().begin(); + + AliasDb aliasDb(graph); + for (const auto input : opNode->inputs()) { + // The custom op writes to all its inputs + ASSERT_TRUE(aliasDb.writesToAlias(opNode, {input})); + // The output should be a wildcard and thus alias all inputs + ASSERT_TRUE(aliasDb.mayAlias(opNode->output(), input)); + } + } + { + // DCE should not remove a custom op + auto graph = std::make_shared(); + const auto text = R"IR( +graph(%x: Tensor, %y: Tensor): + # CHECK: foo::aliasing + %ret : Tensor = foo::aliasing(%x, %y) + return (%x) + )IR"; + parseIR(text, graph.get()); + EliminateDeadCode(graph); + + testing::FileCheck().run(text, *graph); + } +} + +// NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) +static constexpr char op_list[] = "foofoo::bar.template;foo::another"; +#define TORCH_SELECTIVE_NAME_IN_SCHEMA(l, n) \ + torch::detail::SelectiveStr(n) + +TEST(TestCustomOperator, OperatorGeneratorUndeclared) { + // Try to register an op name that does not exist in op_list. + // Expected: the op name is not registered. + torch::jit::RegisterOperators reg({OperatorGenerator( + TORCH_SELECTIVE_NAME_IN_SCHEMA( + op_list, "foofoo::not_exist(float a, Tensor b) -> Tensor"), + [](Stack& stack) { + auto [a, b] = pop(stack); + push(stack, a + b); + }, + aliasAnalysisFromSchema())}); + + auto& ops = getAllOperatorsFor(Symbol::fromQualString("foofoo::not_exist")); + ASSERT_EQ(ops.size(), 0); +} + +TEST(TestCustomOperator, OperatorGeneratorBasic) { + // The operator should be successfully registered since its name is in the + // whitelist. + torch::jit::RegisterOperators reg({OperatorGenerator( + TORCH_SELECTIVE_NAME_IN_SCHEMA( + op_list, "foofoo::bar.template(float a, Tensor b) -> Tensor"), + [](Stack& stack) { + auto [a, b] = pop(stack); + push(stack, a + b); + }, + aliasAnalysisFromSchema())}); + + auto& ops = getAllOperatorsFor(Symbol::fromQualString("foofoo::bar")); + ASSERT_EQ(ops.size(), 1); + + auto& op = ops.front(); + ASSERT_EQ(op->schema().name(), "foofoo::bar"); + + ASSERT_EQ(op->schema().arguments().size(), 2); + ASSERT_EQ(op->schema().arguments()[0].name(), "a"); + ASSERT_EQ(op->schema().arguments()[0].type()->kind(), TypeKind::FloatType); + ASSERT_EQ(op->schema().arguments()[1].name(), "b"); + ASSERT_EQ(op->schema().arguments()[1].type()->kind(), TypeKind::TensorType); + + ASSERT_EQ(op->schema().returns().size(), 1); + ASSERT_EQ(op->schema().returns()[0].type()->kind(), TypeKind::TensorType); + + Stack stack; + push(stack, 2.0f, at::ones(5)); + op->getOperation()(stack); + at::Tensor output; + pop(stack, output); + + ASSERT_TRUE(output.allclose(at::full(5, 3.0f))); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_dce.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_dce.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fb81f0261780f8c72543934580d880c413909bbb --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_dce.cpp @@ -0,0 +1,52 @@ +#include + +#include +#include +#include + +namespace torch { +namespace jit { +TEST(EliminateDeadCodeTest, Basic) { + auto graph = std::make_shared(); + + // Consider the following loop: + // for i in range(3): + // tot += a[0][0] + // b = a[0] + // b[0] += 1 + // print(tot) + // We want to check that b[0] and b are properly marked as live and thus not + // DCE'd. + const std::string input = + R"IR( +graph(): + %48 : None = prim::Constant() + %50 : bool = prim::Constant[value=1]() + %0 : int = prim::Constant[value=2]() + %12 : int = prim::Constant[value=1]() + %24 : int = prim::Constant[value=3]() + %31 : int = prim::Constant[value=0]() + %2 : int[] = prim::ListConstruct(%0, %0) + %a.1 : Tensor = prim::MakeTestTensor() + %14 : int[] = prim::ListConstruct(%12) + %tot.1 : Tensor = prim::MakeTestTensor() + %tot : Tensor = prim::Loop(%24, %50, %tot.1) + block0(%i : int, %tot.6 : Tensor): + %33 : Tensor = aten::select(%a.1, %31, %31) + %35 : Tensor = aten::select(%33, %31, %31) + # CHECK: add_ + %tot.3 : Tensor = aten::add_(%tot.6, %35, %12) + %b.1 : Tensor = aten::select(%a.1, %31, %31) + %44 : Tensor = aten::select(%b.1, %31, %31) + # CHECK: add_ + %46 : Tensor = aten::add_(%44, %12, %12) + -> (%50, %tot.3) + return (%tot) +)IR"; + parseIR(input, graph.get()); + EliminateDeadCode(graph); + // Check that dead code elimin + testing::FileCheck().run(input, *graph); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_exception.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_exception.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b8ff61c1b7de663c769260f262231e5843010f11 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_exception.cpp @@ -0,0 +1,160 @@ +/* + * We have a python unit test for exceptions in test/jit/test_exception.py . + * Add a CPP version here to verify that excepted exception types thrown from + * C++. This is hard to test in python code since C++ exceptions will be + * translated to python exceptions. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +namespace py = pybind11; + +TEST(TestException, TestAssertion) { + std::string pythonCode = R"PY( + def foo(): + raise AssertionError("An assertion failed") + )PY"; + auto cu_ptr = torch::jit::compile(pythonCode); + torch::jit::GraphFunction* gf = + (torch::jit::GraphFunction*)&cu_ptr->get_function("foo"); + std::cerr << "Graph is\n" << *gf->graph() << std::endl; + + bool is_jit_exception = false; + std::string message; + std::optional exception_class; + try { + cu_ptr->run_method("foo"); + } catch (JITException& e) { + is_jit_exception = true; + message = e.what(); + exception_class = e.getPythonClassName(); + } + EXPECT_TRUE(is_jit_exception); + EXPECT_FALSE(exception_class); + EXPECT_TRUE( + message.find("RuntimeError: AssertionError: An assertion failed") != + std::string::npos); +} + +struct MyPythonExceptionValue : public torch::jit::SugaredValue { + explicit MyPythonExceptionValue(const py::object& exception_class) { + qualified_name_ = + (py::str(py::getattr(exception_class, "__module__", py::str(""))) + + py::str(".") + + py::str(py::getattr(exception_class, "__name__", py::str("")))) + .cast(); + } + + std::string kind() const override { + return "My Python exception"; + } + + // Simplified from PythonExceptionValue::call + std::shared_ptr call( + const torch::jit::SourceRange& loc, + torch::jit::GraphFunction& caller, + at::ArrayRef args, + at::ArrayRef kwargs, + size_t n_binders) override { + TORCH_CHECK(args.size() == 1); + Value* error_message = args.at(0).value(*caller.graph()); + Value* qualified_class_name = + insertConstant(*caller.graph(), qualified_name_, loc); + return std::make_shared( + error_message, qualified_class_name); + } + + private: + std::string qualified_name_; +}; + +class SimpleResolver : public torch::jit::Resolver { + public: + explicit SimpleResolver() {} + + std::shared_ptr resolveValue( + const std::string& name, + torch::jit::GraphFunction& m, + const torch::jit::SourceRange& loc) override { + // follows toSugaredValue (toSugaredValue is defined in caffe2:_C which is + // a python extension. We can not add that as a cpp_binary's dep) + if (name == "SimpleValueError") { + py::object obj = py::globals()["SimpleValueError"]; + return std::make_shared(obj); + } + TORCH_CHECK(false, "resolveValue: can not resolve '", name, "{}'"); + } + + torch::jit::TypePtr resolveType( + const std::string& name, + const torch::jit::SourceRange& loc) override { + return nullptr; + } +}; + +/* + * - The python source code parsing for TorchScript here is learned from + * torch::jit::compile. + * - The code only parses one Def. If there are multiple in the code, those + * except the first one are skipped. + */ +TEST(TestException, TestCustomException) { + py::scoped_interpreter guard{}; + py::exec(R"PY( + class SimpleValueError(ValueError): + def __init__(self, message): + super().__init__(message) + )PY"); + + std::string pythonCode = R"PY( + def foo(): + raise SimpleValueError("An assertion failed") + )PY"; + + torch::jit::Parser p( + std::make_shared(pythonCode, "", 1)); + auto def = torch::jit::Def(p.parseFunction(/*is_method=*/false)); + std::cerr << "Def is:\n" << def << std::endl; + auto cu = std::make_shared(); + (void)cu->define( + std::nullopt, + {}, + {}, + {def}, + // class PythonResolver is defined in + // torch/csrc/jit/python/script_init.cpp. It's not in a header file so I + // can not use it. Create a SimpleResolver instead + {std::make_shared()}, + nullptr); + torch::jit::GraphFunction* gf = + (torch::jit::GraphFunction*)&cu->get_function("foo"); + std::cerr << "Graph is\n" << *gf->graph() << std::endl; + bool is_jit_exception = false; + std::optional exception_class; + std::string message; + try { + cu->run_method("foo"); + } catch (JITException& e) { + is_jit_exception = true; + exception_class = e.getPythonClassName(); + message = e.what(); + } + EXPECT_TRUE(is_jit_exception); + EXPECT_EQ("__main__.SimpleValueError", *exception_class); + EXPECT_TRUE( + message.find("__main__.SimpleValueError: An assertion failed") != + std::string::npos); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_file_format.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_file_format.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ecc49259bc67c7f40af4fdf9e579b469aa291eec --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_file_format.cpp @@ -0,0 +1,124 @@ +#include + +#include + +#include + +// Tests go in torch::jit +namespace torch { +namespace jit { + +TEST(FileFormatTest, IdentifiesFlatbufferStream) { + // Create data whose initial bytes look like a Flatbuffer stream. + std::stringstream data; + data << "abcd" // First four bytes don't matter. + << "PTMF" // Magic string. + << "efgh"; // Trailing bytes don't matter. + + // The data should be identified as Flatbuffer. + EXPECT_EQ(getFileFormat(data), FileFormat::FlatbufferFileFormat); +} + +TEST(FileFormatTest, IdentifiesZipStream) { + // Create data whose initial bytes look like a ZIP stream. + std::stringstream data; + data << "PK\x03\x04" // Magic string. + << "abcd" // Trailing bytes don't matter. + << "efgh"; + + // The data should be identified as ZIP. + EXPECT_EQ(getFileFormat(data), FileFormat::ZipFileFormat); +} + +TEST(FileFormatTest, FlatbufferTakesPrecedence) { + // Since the Flatbuffer and ZIP magic bytes are at different offsets, + // the same data could be identified as both. Demonstrate that Flatbuffer + // takes precedence. (See details in file_format.h) + std::stringstream data; + data << "PK\x03\x04" // ZIP magic string. + << "PTMF" // Flatbuffer magic string. + << "abcd"; // Trailing bytes don't matter. + + // The data should be identified as Flatbuffer. + EXPECT_EQ(getFileFormat(data), FileFormat::FlatbufferFileFormat); +} + +TEST(FileFormatTest, HandlesUnknownStream) { + // Create data that doesn't look like any known format. + std::stringstream data; + data << "abcd" + << "efgh" + << "ijkl"; + + // The data should be classified as unknown. + EXPECT_EQ(getFileFormat(data), FileFormat::UnknownFileFormat); +} + +TEST(FileFormatTest, ShortStreamIsUnknown) { + // Create data with fewer than kFileFormatHeaderSize (8) bytes. + std::stringstream data; + data << "ABCD"; + + // The data should be classified as unknown. + EXPECT_EQ(getFileFormat(data), FileFormat::UnknownFileFormat); +} + +TEST(FileFormatTest, EmptyStreamIsUnknown) { + // Create an empty stream. + std::stringstream data; + + // The data should be classified as unknown. + EXPECT_EQ(getFileFormat(data), FileFormat::UnknownFileFormat); +} + +TEST(FileFormatTest, BadStreamIsUnknown) { + // Create a stream with valid Flatbuffer data. + std::stringstream data; + data << "abcd" + << "PTMF" // Flatbuffer magic string. + << "efgh"; + + // Demonstrate that the data would normally be identified as Flatbuffer. + EXPECT_EQ(getFileFormat(data), FileFormat::FlatbufferFileFormat); + + // Mark the stream as bad, and demonstrate that it is in an error state. + data.setstate(std::stringstream::badbit); + // Demonstrate that the stream is in an error state. + EXPECT_FALSE(data.good()); + + // The data should now be classified as unknown. + EXPECT_EQ(getFileFormat(data), FileFormat::UnknownFileFormat); +} + +TEST(FileFormatTest, StreamOffsetIsObservedAndRestored) { + // Create data with a Flatbuffer header at a non-zero offset into the stream. + std::stringstream data; + // Add initial padding. + data << "PADDING"; + size_t offset = data.str().size(); + // Add a valid Flatbuffer header. + data << "abcd" + << "PTMF" // Flatbuffer magic string. + << "efgh"; + // Seek just after the padding. + data.seekg(static_cast(offset), data.beg); + // Demonstrate that the stream points to the beginning of the Flatbuffer data, + // not to the padding. + EXPECT_EQ(data.peek(), 'a'); + + // The data should be identified as Flatbuffer. + EXPECT_EQ(getFileFormat(data), FileFormat::FlatbufferFileFormat); + + // The stream position should be where it was before identification. + EXPECT_EQ(offset, data.tellg()); +} + +TEST(FileFormatTest, HandlesMissingFile) { + // A missing file should be classified as unknown. + EXPECT_EQ( + getFileFormat("NON_EXISTENT_FILE_4965c363-44a7-443c-983a-8895eead0277"), + FileFormat::UnknownFileFormat); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_flatbuffer.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_flatbuffer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4a709b586f8c749bd1abaa6ead44226bf367f766 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_flatbuffer.cpp @@ -0,0 +1,1942 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#if defined(FB_XPLAT_BUILD) || defined(FBCODE_CAFFE2) +#include // NOLINT +namespace flatbuffers = flatbuffers_fbsource; +#define FLATBUFFERS_MAX_ALIGNMENT FLATBUFFERS_FBSOURCE_MAX_ALIGNMENT +#else +#include // NOLINT +#endif +// Tests go in torch::jit +namespace torch { +namespace jit { + +namespace { +mobile::Module parse_mobile_module( + void* data, + size_t size, + bool should_copy_tensor_memory = false) { + return parse_and_initialize_mobile_module( + static_cast(data), + size, + /*device=*/std::nullopt, + /*extra_files=*/nullptr, + should_copy_tensor_memory); +} +} // namespace + +TEST(FlatbufferTest, LoadMalformedModule) { + // Manually create some data with Flatbuffer header. + std::stringstream bad_data; + bad_data << "PK\x03\x04PTMF\x00\x00" + << "*}NV\xb3\xfa\xdf\x00pa"; + + // Loading module from it should throw an exception. + // Check guard at parse_and_initialize_mobile_module_for_jit. + ASSERT_THROWS_WITH_MESSAGE( + torch::jit::load(bad_data), "Malformed Flatbuffer module"); + + // Check guard at parse_and_initialize_mobile_module. + ASSERT_THROWS_WITH_MESSAGE( + parse_mobile_module(bad_data.str().data(), bad_data.str().size()), + "Malformed Flatbuffer module"); +} + +TEST(FlatbufferTest, UpsampleNearest2d) { + Module m("m"); + m.define(R"( + def forward(self, input: Tensor, scale:float): + return torch.upsample_nearest2d(input, [1, 1], float(scale), float(scale)) + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({1, 3, 128, 128})); + inputs.emplace_back(at::Scalar(2.0)); + auto ref = m.forward(inputs); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + res = bc.forward(inputs); + + auto resd = res.toTensor(); + auto refd = ref.toTensor(); + ASSERT_TRUE(resd.equal(refd)); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + auto res2 = bc2.forward(inputs); + auto resd2 = res2.toTensor(); + ASSERT_TRUE(resd2.equal(refd)); +} + +TEST(FlatbufferTest, UpsampleNearest2dWithCopyTensorMemory) { + Module m("m"); + m.define(R"( + def forward(self, input: Tensor, scale:float): + return torch.upsample_nearest2d(input, [1, 1], float(scale), float(scale)) + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({1, 3, 128, 128})); + inputs.emplace_back(at::Scalar(2.0)); + auto ref = m.forward(inputs); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + res = bc.forward(inputs); + + auto resd = res.toTensor(); + auto refd = ref.toTensor(); + ASSERT_TRUE(resd.equal(refd)); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size(), true); + + auto res2 = bc2.forward(inputs); + auto resd2 = res2.toTensor(); + ASSERT_TRUE(resd2.equal(refd)); +} + +TEST(FlatbufferTest, CheckAttrAccess) { + Module m("m"); + m.register_attribute("mobile_optimized", BoolType::get(), true); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + bool mobile_optimized = bc.attr("mobile_optimized", false).toBool(); + + AT_ASSERT(mobile_optimized); + m.setattr("mobile_optimized", false); + bc = jitModuleToMobile(m, options); + mobile_optimized = bc.attr("mobile_optimized", false).toBool(); + + AT_ASSERT(!mobile_optimized); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + auto mobile_optimized2 = bc2.attr("mobile_optimized", false).toBool(); + AT_ASSERT(!mobile_optimized2); +} + +TEST(FlatbufferTest, MethodInvocation) { // NOLINT (use =delete in gtest) + const std::vector test_programs{ + // test invoking a method with default parameter + R"( + def test_func(self, x, b : int = 4): + return self.foo + x + b + )", + // inner method call with default parameter (gets inlined) + R"( + def add_with_default_arg(self, x, b : int = 4): + return self.foo + x + b + def test_func(self, x): + return self.add_with_default_arg(x) # invoke method w/ default arg + )", + // simple method call + R"( + def test_func(self, x): + b = 4 + return self.foo + x + b + )", + }; + for (const auto& test_program : test_programs) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(test_program); + + const int fortyTwo = 42; // (keep linter happy) + auto minput = fortyTwo * torch::ones({}); + auto ref = m.run_method("test_func", minput); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + const auto& test_func = bc.get_method("test_func"); + IValue res; + for (int i = 0; i < 3; ++i) { + res = test_func({minput}); + } + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + const auto& test_func2 = bc2.get_method("test_func"); + IValue res2; + for (int i = 0; i < 3; ++i) { + res2 = test_func2({minput}); + } + auto resd2 = res2.toTensor().item(); + AT_ASSERT(resd2 == refd); + } +} + +#if !defined(FB_XPLAT_BUILD) +TEST(FlatbufferTest, FlatbufferBackPortTest) { + Module m("m"); + m.define(R"( + def forward(self, input: Tensor, scale:float): + return torch.upsample_nearest2d(input, [1, 1], float(scale), float(scale)) + )"); + std::stringstream ss; + m._save_for_mobile(ss, {}, false, true); + + std::stringstream oss; + bool backPortSuccess = _backport_for_mobile(ss, oss, 5); + ASSERT_TRUE(backPortSuccess); +} +#endif // !defined(FB_XPLAT_BUILD) + +TEST(FlatbufferTest, ExtraFiles) { + const auto script = R"JIT( + def forward(self): + x = torch.rand(5, 5) + x = x.mm(x) + return x + )JIT"; + + auto module = + std::make_shared("Module", std::make_shared()); + module->define(script); + std::ostringstream oss; + std::unordered_map extra_files; + extra_files["metadata.json"] = "abc"; + extra_files["mobile_info.json"] = "{\"key\": 23}"; + + std::unordered_map loaded_extra_files; + std::stringstream ss; + module->_save_for_mobile(ss, extra_files, true, /*use_flatbuffer=*/true); + + loaded_extra_files["metadata.json"] = ""; + auto mobile_module = _load_for_mobile(ss, std::nullopt, loaded_extra_files); + + ASSERT_EQ(loaded_extra_files["metadata.json"], "abc"); + ASSERT_EQ(loaded_extra_files["mobile_info.json"], "{\"key\": 23}"); + + // load it twice using the same stream + auto mobile_module2 = _load_for_mobile(ss, std::nullopt, loaded_extra_files); + + ASSERT_EQ(loaded_extra_files["metadata.json"], "abc"); + ASSERT_EQ(loaded_extra_files["mobile_info.json"], "{\"key\": 23}"); + + // Test if flatbuffer does not require any explicit key entries mapping in the + // extra file map. + std::unordered_map + loaded_extra_files_without_explicit_entries; + auto mobile_module3 = _load_for_mobile( + ss, + std::nullopt, + loaded_extra_files_without_explicit_entries, + MobileModuleLoadOptions::PARSE_ALL_EXTRA_FILE_MAPS); + + ASSERT_EQ( + loaded_extra_files_without_explicit_entries["metadata.json"], "abc"); + ASSERT_EQ( + loaded_extra_files_without_explicit_entries["mobile_info.json"], + "{\"key\": 23}"); +} + +TEST(FlatbufferTest, Conv) { + auto s = std::getenv("PYTORCH_TEST_WITH_TSAN"); + if (s && strcmp(s, "1") == 0) + return; + + std::vector inputs; + + Module m("m"); + m.register_parameter("weight", torch::ones({20, 1, 5, 5}), false); + m.register_parameter("bias", torch::ones({20}), false); + m.define(R"( + def forward(self, input): + return torch._convolution(input, self.weight, self.bias, [1, 1], [0, 0], [1, 1], False, [0, 0], 1, False, False, True, True) + )"); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,modernize-use-emplace) + inputs.push_back(torch::ones({1, 1, 28, 28})); + + auto outputref = m.forward(inputs).toTensor(); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + res = bc.get_method("forward")(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT( + outputref[0][0][0][0].item() == output[0][0][0][0].item()); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + for (int i = 0; i < 3; ++i) { + res = bc2.get_method("forward")(inputs); + } + output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT( + outputref[0][0][0][0].item() == output[0][0][0][0].item()); +} + +TEST(FlatbufferTest, ConvWithCopyTensorMemory) { + auto s = std::getenv("PYTORCH_TEST_WITH_TSAN"); + if (s && strcmp(s, "1") == 0) + return; + + std::vector inputs; + + Module m("m"); + m.register_parameter("weight", torch::ones({20, 1, 5, 5}), false); + m.register_parameter("bias", torch::ones({20}), false); + m.define(R"( + def forward(self, input): + return torch._convolution(input, self.weight, self.bias, [1, 1], [0, 0], [1, 1], False, [0, 0], 1, False, False, True, True) + )"); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,modernize-use-emplace) + inputs.push_back(torch::ones({1, 1, 28, 28})); + + auto outputref = m.forward(inputs).toTensor(); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + res = bc.get_method("forward")(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT( + outputref[0][0][0][0].item() == output[0][0][0][0].item()); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size(), true); + + for (int i = 0; i < 3; ++i) { + res = bc2.get_method("forward")(inputs); + } + output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT( + outputref[0][0][0][0].item() == output[0][0][0][0].item()); +} + +TEST(FlatbufferTest, Inline) { + Module m("m"); + m.define(R"JIT( + def foo1(self, x): + return x + 1 + + def foo2(self, x): + return self.foo1(x) + 2 + + def foo3(self, x): + return self.foo2(x) + 3 + )JIT"); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("foo3")(inputs); + AT_ASSERT(output.toTensor().item() == 7.0); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + std::vector inputs2({torch::ones({})}); + output = bc2.get_method("foo3")(inputs2); + AT_ASSERT(output.toTensor().item() == 7.0); +} + +TEST(FlatbufferTest, InlineWithCopyTensorMemory) { + Module m("m"); + m.define(R"JIT( + def foo1(self, x): + return x + 1 + + def foo2(self, x): + return self.foo1(x) + 2 + + def foo3(self, x): + return self.foo2(x) + 3 + )JIT"); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("foo3")(inputs); + AT_ASSERT(output.toTensor().item() == 7.0); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size(), true); + std::vector inputs2({torch::ones({})}); + output = bc2.get_method("foo3")(inputs2); + AT_ASSERT(output.toTensor().item() == 7.0); +} + +TEST(FlatbufferTest, Tuple) { + Module m("m"); + m.define(R"JIT( + def foo(self, x): + return (1, 2, x + 3) + + def forward(self, x): + tuple = self.foo(x) + return tuple + )JIT"); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("forward")(inputs); + AT_ASSERT(output.toTupleRef().elements()[1].toInt() == 2); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + output = bc2.get_method("forward")(inputs); + AT_ASSERT(output.toTuple()->elements()[1].toInt() == 2); +} + +TEST(FlatbufferTest, Dict) { + Module m("m"); + m.define(R"JIT( + def foo(self, x): + return {"result": x + 1} + + def forward(self, x): + d = self.foo(x) + return d + )JIT"); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("forward")(inputs); + AT_ASSERT(output.toGenericDict().at("result").toTensor().item().toInt() == 2); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + output = bc2.get_method("forward")(inputs); + AT_ASSERT(output.toGenericDict().at("result").toTensor().item().toInt() == 2); +} + +TEST(FlatbufferTest, Prim) { + Module m("m"); + m.define(R"JIT( + def forward(self, x): + return int(x) + )JIT"); + + std::vector inputs; + auto minput = 3.5 * torch::ones({}); + inputs.emplace_back(minput); + auto ref = m.run_method("forward", minput); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc.get_method("forward")(bcinputs); + } + + auto resi = res.toInt(); + auto refi = ref.toInt(); + AT_ASSERT(resi == refi); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc2.get_method("forward")(bcinputs); + } + auto resi2 = res.toInt(); + AT_ASSERT(resi2 == refi); +} + +TEST(FlatbufferTest, PrimScalar) { + Module m("m"); + m.define(R"JIT( + def forward(self, x): + return int(x.item()) + )JIT"); + + std::vector inputs; + auto minput = 3.5 * torch::ones({}); + inputs.emplace_back(minput); + auto ref = m.run_method("forward", minput); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc.get_method("forward")(bcinputs); + } + + auto resi = res.toInt(); + auto refi = ref.toInt(); + AT_ASSERT(resi == refi); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc2.get_method("forward")(bcinputs); + } + auto resi2 = res.toInt(); + AT_ASSERT(resi2 == refi); +} + +TEST(FlatbufferTest, WrongMethodName) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add(self, x): + b = 4 + return self.foo + x + b + )"); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + std::vector inputs; + auto minput = 5 * torch::ones({}); + inputs.emplace_back(minput); + ASSERT_THROWS_WITH_MESSAGE( + bc.get_method("forward")(inputs), "is not defined"); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + ASSERT_THROWS_WITH_MESSAGE( + bc2.get_method("forward")(inputs), "is not defined"); +} + +TEST(FlatbufferTest, SetState) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def __getstate__(self): + return self.foo + def __setstate__(self, a): + self.foo = a + def forward(self, x): + b = 4 + return self.foo + x + b + )"); + + std::vector inputs; + auto minput = 5 * torch::ones({}); + inputs.emplace_back(minput); + + std::stringstream ms; + m.save(ms); + auto loaded_m = load(ms); + auto ref = loaded_m.run_method("forward", minput); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc.get_method("forward")(bcinputs); + } + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc2.get_method("forward")(bcinputs); + } + + auto resd2 = res.toTensor().item(); + AT_ASSERT(resd2 == refd); +} + +class TorchBindFlatbufferTestStruct : public torch::jit::CustomClassHolder { + public: + std::string get(at::Tensor t) { + std::stringstream ss; + ss << "Hello! Your tensor has "; + ss << t.numel(); + ss << " elements!"; + return std::move(ss).str(); + } +}; + +namespace { +struct ClassNamespaceValue : public SugaredValue { + explicit ClassNamespaceValue(c10::QualifiedName name) + : basename_(std::move(name)) {} + + std::shared_ptr attr( + const SourceRange& loc, + GraphFunction& m, + const std::string& name) override { + const auto fullName = c10::QualifiedName(basename_, name); + + // Check to see if it is a custom class. + if (auto custom_class = getCustomClass(fullName.qualifiedName())) { + return std::make_shared(custom_class); + } + + // If it's not a custom class, assume it's another namespace + // NOLINTNEXTLINE(performance-move-const-arg) + return std::make_shared(std::move(fullName)); + } + + std::string kind() const override { + return "Class Namespace"; + } + + private: + c10::QualifiedName basename_; +}; + +struct TestModuleResolver : public Resolver { + std::shared_ptr resolveValue( + const std::string& name, + GraphFunction& m, + const SourceRange& loc) override { + if (name == "torch") { + return std::make_shared("aten"); + } else if (name == "__torch__") { + return std::make_shared(c10::QualifiedName(name)); + } + + return nullptr; + } + + TypePtr resolveType(const std::string& name, const SourceRange& loc) + override { + return nullptr; + } +}; +} // namespace + +TEST(FlatbufferTest, BuiltinClass) { + script::Module m("m"); + + auto cls = getCustomClass( + "__torch__.torch.classes._TorchScriptTesting._FlatbufferTest"); + TORCH_INTERNAL_ASSERT(cls); + c10::intrusive_ptr obj_holder; + m.register_attribute("my_obj", cls, IValue::make_capsule(obj_holder)); + + m.register_parameter("foo", torch::ones({}), false); + m.define( + R"( + def __getstate__(self): + return 1 + def __setstate__(self, a): + self.my_obj = __torch__.torch.classes._TorchScriptTesting._FlatbufferTest() + + def forward(self, x) -> str: + return self.my_obj.get(x) + )", + std::make_shared()); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + std::string expected = "Hello! Your tensor has 12 elements!"; + auto res = + bc2.get_method("forward")(std::vector{torch::zeros({3, 4})}); + const auto& str2 = res.toStringRef(); + AT_ASSERT(str2 == expected); +} + +TEST(FlatbufferTest, BuiltinFunction) { + script::Module m("m"); + auto custom_class_obj = make_custom_class(); + m.register_attribute("my_obj", custom_class_obj.type(), custom_class_obj); + m.define(R"( + def forward(self, x) -> str: + return self.my_obj.get(x) + )"); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + auto res = + bc.get_method("forward")(std::vector{torch::zeros({3, 4})}); + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto str = res.toStringRef(); + std::string expected = "Hello! Your tensor has 12 elements!"; + AT_ASSERT(str == expected); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + res = bc2.get_method("forward")(std::vector{torch::zeros({3, 4})}); + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + str = res.toStringRef(); + AT_ASSERT(str == expected); +} + +TEST(FlatbufferTest, Eval) { + std::vector inputs; + + Module m("m"); + m.define(R"( + def __init__(self, x): + self.training = True + + def forward(self, input): + return torch.dropout(input, 1.0, self.training) + )"); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,modernize-use-emplace) + inputs.push_back(torch::ones({1, 1, 28, 28})); + m.eval(); + auto outputref = m.forward(inputs).toTensor(); + + // save m in training mode to make sure that mobile eval() will correctly + // change back to eval mode + m.train(); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + bc.eval(); + IValue res; + for (int i = 0; i < 3; ++i) { + res = bc.get_method("forward")(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT( + outputref[0][0][0][0].item() == output[0][0][0][0].item()); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + bc2.eval(); + for (int i = 0; i < 3; ++i) { + res = bc2.get_method("forward")(inputs); + } + output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT( + outputref[0][0][0][0].item() == output[0][0][0][0].item()); +} + +TEST(FlatbufferTest, FindWrongMethodName) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add(self, x): + b = 4 + return self.foo + x + b + )"); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + ASSERT_TRUE(bc.find_method("forward") == std::nullopt); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + ASSERT_TRUE(bc2.find_method("forward") == std::nullopt); +} + +TEST(FlatbufferTest, FindAndRunMethod) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add_it(self, x): + b = 4 + return self.foo + x + b + )"); + + std::vector inputs; + auto minput = 5 * torch::ones({}); + inputs.emplace_back(minput); + auto ref = m.get_method("add_it")(inputs); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + auto bcinputs = inputs; + auto method = bc.find_method("add_it"); + AT_ASSERT(method != std::nullopt); + res = (*method)(std::move(bcinputs)); + } + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + + for (int i = 0; i < 3; ++i) { + auto bcinputs = inputs; + auto method = bc2.find_method("add_it"); + AT_ASSERT(method != std::nullopt); + res = (*method)(std::move(bcinputs)); + } + + resd = res.toTensor().item(); + AT_ASSERT(resd == refd); +} + +TEST(FlatbufferTest, RunMethodVariadic) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add_three(self, x, y): + return self.foo + x + y + )"); + + std::vector inputs; + auto inputx = 5 * torch::ones({}); + auto inputy = 4 * torch::ones({}); + auto ref = m.run_method("add_three", inputx, inputy); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res = bc.run_method("add_three", inputx, inputy); + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + res = bc.run_method("add_three", inputx, inputy); + resd = res.toTensor().item(); + AT_ASSERT(resd == refd); +} + +TEST(FlatbufferTest, DuplicateSetState) { + Module m("M"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def __getstate__(self): + return self.foo + self.foo + def __setstate__(self, a): + self.foo = a + def forward(self, x): + b = 4 + return self.foo + x + b + )"); + + Module b("B"); + b.register_module("M0", m); + b.register_module("M1", m); + b.define(R"( + def forward(self, x): + return self.M0.forward(x) + self.M1.forward(x) + )"); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + const auto methods = bc.get_methods(); + const size_t expected_n = 3; + ASSERT_EQ(methods.size(), expected_n); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + const auto methods2 = bc.get_methods(); + ASSERT_EQ(methods2.size(), expected_n); +} + +TEST(FlatbufferTest, OpNameExportFetchRootOperators) { + torch::jit::Module m("m"); + m.register_parameter("weight", torch::ones({20, 1, 5, 5}), false); + m.register_parameter("bias", torch::ones({20}), false); + m.define(R"( + def forward(self, input): + x1 = torch.zeros(2, 2) + x2 = torch.empty_like(torch.empty(2, 2)) + x3 = torch._convolution(input, self.weight, self.bias, [1, 1], [0, 0], [1, 1], False, [0, 0], 1, False, False, True, True) + return (x1, x2, x3) + )"); + m.eval(); + + CompilationOptions options; + mobile::Module ptl_model = jitModuleToMobile(m, options); + std::set operator_names = + torch::jit::mobile::_export_operator_list(ptl_model); + std::set expected_operator_names = { + "aten::_convolution", + "aten::empty.memory_format", + "aten::empty_like", + "aten::zeros", + }; + EXPECT_EQ(operator_names, expected_operator_names) + << "Expected the root operator lists to be the same"; + + auto buff = save_mobile_module_to_bytes(ptl_model); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + operator_names = torch::jit::mobile::_export_operator_list(bc2); + EXPECT_EQ(operator_names, expected_operator_names) + << "Expected the root operator lists to be the same"; +} + +TEST(FlatbufferTest, DefaultArgsConv) { + auto s = std::getenv("PYTORCH_TEST_WITH_TSAN"); + if (s && strcmp(s, "1") == 0) + return; + + std::vector inputs; + + Module m("m"); + m.register_parameter("weight", torch::ones({20, 1, 5, 5}), false); + m.register_parameter("bias", torch::ones({20}), false); + m.define(R"( + def forward(self, input): + return torch.conv2d(input, self.weight, self.bias, [1, 1], [0, 0], [1, 1], 1) + )"); + + inputs.emplace_back(torch::ones({1, 1, 28, 28})); + + auto outputref = m.forward(inputs).toTensor(); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 1; ++i) { + res = bc.get_method("forward")(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT(output.equal(outputref)); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + for (int i = 0; i < 1; ++i) { + res = bc2.get_method("forward")(inputs); + } + output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT(output.equal(outputref)); +} + +namespace { +void testLiteModuleCompareResultTensors( + Module& m, + const std::vector& inputs, + const std::string& method_name = "forward") { + auto outputref = m.get_method(method_name)(inputs).toTensor(); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + res = bc.get_method(method_name)(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT(output.equal(outputref)); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + for (int i = 0; i < 3; ++i) { + res = bc2.get_method(method_name)(inputs); + } + output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT(output.equal(outputref)); +} + +static void testDefaultArgsPinv(int num_args) { + Module m("m"); + if (num_args == 1) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input) + )"); + } else if (num_args == 2) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5) + )"); + } else if (num_args == 3) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5, True) + )"); + } + + std::vector inputs; + const int N = 28; + auto input = torch::range(1, N * N, 1); + input[0] = 1; // a more stable matrix + input = input.view({N, N}); + inputs.emplace_back(input); + testLiteModuleCompareResultTensors(m, inputs); +} +} // namespace + +#if !defined FB_XPLAT_BUILD +TEST(FlatbufferTest, DefaultArgsPinv) { + // Test with different number of specified arguments. + // Arguments not specified take default value. + for (int num_args = 1; num_args <= 3; ++num_args) { + testDefaultArgsPinv(num_args); + } + + // bytecode with one specified argument: + // (6, + // ('__torch__.m.forward', + // (('instructions', + // (('STOREN', 1, 2), + // ('DROPR', 1, 0), + // ('MOVE', 2, 0), + // ('OP', 0, 0), + // ('RET', 0, 0))), + // ('operators', (('aten::linalg_pinv', '', 1),)), + // ('constants', (False, 1e-15)), # default constants are not + // used + // ('types', ()), + // ('register_size', 2)), + // (('arguments', + // ((('name', 'self'), ('type', '__torch__.m'), ('default_value', + // None)), + // (('name', 'input'), ('type', 'Tensor'), ('default_value', + // None)))), + // ('returns', + // ((('name', ''), ('type', 'Tensor'), ('default_value', + // None)),))))) + + // bytecode with 2 specified argument: + // (6, + // ('__torch__.m.forward', + // (('instructions', + // (('STOREN', 1, 2), + // ('DROPR', 1, 0), + // ('MOVE', 2, 0), + // ('LOADC', 1, 0), # added LOADC for specified argument + // ('OP', 0, 0), + // ('RET', 0, 0))), + // ('operators', (('aten::linalg_pinv', '', 2),)), + // ('constants', (False, 1e-05)), # updated constant table + // ('types', ()), + // ('register_size', 2)), + // (('arguments', + // ((('name', 'self'), ('type', '__torch__.m'), ('default_value', + // None)), + // (('name', 'input'), ('type', 'Tensor'), ('default_value', + // None)))), + // ('returns', + // ((('name', ''), ('type', 'Tensor'), ('default_value', + // None)),))))) + + // bytecode with 3 specified arguments: + // (6, + // ('__torch__.m.forward', + // (('instructions', + // (('STOREN', 1, 2), + // ('DROPR', 1, 0), + // ('MOVE', 2, 0), + // ('LOADC', 1, 0), + // ('LOADC', 0, 0), + // ('OP', 0, 0), + // ('RET', 0, 0))), + // ('operators', (('aten::linalg_pinv', '', 3),)), + // ('constants', (True, 1e-05)), + // ('types', ()), + // ('register_size', 2)), + // (('arguments', + // ((('name', 'self'), ('type', '__torch__.m'), ('default_value', + // None)), + // (('name', 'input'), ('type', 'Tensor'), ('default_value', + // None)))), + // ('returns', + // ((('name', ''), ('type', 'Tensor'), ('default_value', + // None)),))))) +} + +TEST(FlatbufferTest, DefaultArgsTensorinvSpecifyDefault) { + // The second argument is specified, but the value is the same as the default + // value. It's treated as "not specified" since the value can be fetched from + // schema. + Module m("m"); + m.define(R"( + def forward(self, input): + return torch.linalg_tensorinv(input, 2) + )"); + torch::jit::MobileCode code(m.get_method("forward").graph(), "forward"); + auto arg_nums = code.op_to_num_specified_args(); + ASSERT_EQ(arg_nums.size(), 1); + ASSERT_EQ(arg_nums["aten::linalg_tensorinv"], 1); + std::vector inputs; + const int N = 4; + auto input = torch::rand({N, N, N, N}); + inputs.emplace_back(input); + testLiteModuleCompareResultTensors(m, inputs); +} + +static void testDefaultArgsPinvWithOutArg(int num_args) { + Module m("m"); + if (num_args == 1) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, out=input) + )"); + } else if (num_args == 2) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5, out=input) + )"); + } else if (num_args == 3) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5, True, out=input) + )"); + } + + const int N = 28; + auto input = torch::range(1, N * N, 1); + input[0] = 10000; // a more stable matrix + input = input.view({N, N}); + auto ref = m.run_method("forward", input); + TORCH_CHECK(!input.equal(torch::range(1, N * N, 1))); + TORCH_CHECK(input.equal(ref.toTensor())); +} + +TEST(FlatbufferTest, DefaultArgsPinvWithOutArg) { + // Test with different number of specified arguments + out arg. + // Arguments not specified take default value. + for (int num_args = 1; num_args <= 3; ++num_args) { + testDefaultArgsPinvWithOutArg(num_args); + } +} + +TEST(FlatbufferTest, DefaultArgsWithOutArg) { + Module m("m"); + m.define(R"( + def forward(self, x, h): + torch.add(x, h, out=x) + )"); + + std::vector inputs; + auto input_x = 2 * torch::ones({}); + auto input_h = torch::ones({}); + auto ref = m.run_method("forward", input_x, input_h); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + bc.run_method("forward", input_x, input_h); + AT_ASSERT(input_x.equal(4 * torch::ones({}))); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + auto input_x2 = 2 * torch::ones({}); + auto input_h2 = torch::ones({}); + m.run_method("forward", input_x2, input_h2); + bc2.run_method("forward", input_x2, input_h2); + AT_ASSERT(input_x2.equal(4 * torch::ones({}))); +} + +#endif // !defined(FB_XPLAT_BUILD) + +namespace { +static auto reg = + torch::class_( + "_TorchScriptTesting", + "_FlatbufferTest") + .def(torch::init<>()) + .def("get", &TorchBindFlatbufferTestStruct::get) + .def_pickle( + // __getattr__ + [](const c10::intrusive_ptr& self) + -> int64_t { return 0; }, + // __setattr__ + [](int64_t state) { + return c10::make_intrusive(); + }); + +} // namespace + +TEST(FlatbufferTest, OperatorCacheDifferentiatesDefaultArgs) { + // Create 3 methods: + // + // 1. forward() returns a tensor with dtype=torch.int64 (4) + // 2. forward2() returns a tensor with dtype=torch.float32 (6) + // 3. forward3() returns a tensor with dtype=torch.float32 but + // the dtype is inferred by the input tensor's dtype + // + // If caching works correctly, then the result from the full-jit + // module and the lite module will be the same. Otherwise, it + // will be different if we don't correctly ignore the cache + // entry for an operator that has a different number of + // arguments. + Module m("m"); + m.define(R"( + def forward(self): + ret1 = torch.new_empty(torch.zeros(10), [10], dtype=4) + return ret1.fill_(25) + )"); + m.define(R"( + def forward2(self): + ret1 = torch.new_empty(torch.zeros(10), [10], dtype=6) + return ret1.fill_(32.0) + )"); + m.define(R"( + def forward3(self): + ret1 = torch.new_empty(torch.zeros(10), [10]) + return ret1.fill_(12.0) + )"); + + std::vector inputs; + testLiteModuleCompareResultTensors(m, inputs, "forward"); + testLiteModuleCompareResultTensors(m, inputs, "forward2"); + testLiteModuleCompareResultTensors(m, inputs, "forward3"); +} + +TEST(FlatbufferTest, OperatorSize1) { + Module m("m"); + m.define(R"( + def forward(self, input: Tensor, scale:float): + return torch.upsample_nearest2d(input, [1, 1], float(scale), float(scale)) + )"); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + const auto& func = bc.get_method("forward").function(); + ASSERT_EQ( + func.get_code().operator_input_sizes_.size(), + func.get_code().operators_.size()); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + const auto& func2 = bc.get_method("forward").function(); + ASSERT_EQ( + func2.get_code().operator_input_sizes_.size(), + func2.get_code().operators_.size()); +} + +TEST(FlatbufferTest, BoolAndDoubleList) { + Module m("m"); + c10::List boollist; + boollist.push_back(false); + IValue boollist_ival = boollist; + IValue doublelist = std::vector{2.0}; + m.register_attribute("bool_list", boollist_ival.type(), boollist_ival); + m.register_attribute("double_list", doublelist.type(), doublelist); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + + // if the variables read are wrong type the conversion will raise exception + auto boolval = bc2.attr("bool_list", {}).toBoolList().get(0); + auto doubleval = bc2.attr("double_list", {}).toDoubleList().get(0); + + ASSERT_EQ(boolval, false); + ASSERT_EQ(doubleval, 2.0); +} + +TEST(FlatbufferTest, OperatorTest2) { // NOLINT (use =delete in gtest) + const std::vector test_programs{ + // test invoking a method with default parameter + R"( + def test_func(self, x, b : int = 4): + return self.foo + x + b + )", + // inner method call with default parameter (gets inlined) + R"( + def add_with_default_arg(self, x, b : int = 4): + return self.foo + x + b + def test_func(self, x): + return self.add_with_default_arg(x) # invoke method w/ default arg + )", + // simple method call + R"( + def test_func(self, x): + b = 4 + return self.foo + x + b + )", + }; + for (const auto& test_program : test_programs) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(test_program); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + const auto& func = bc.get_method("test_func").function(); + ASSERT_EQ( + func.get_code().operator_input_sizes_.size(), + func.get_code().operators_.size()); + + auto buff = save_mobile_module_to_bytes(bc); + mobile::Module bc2 = parse_mobile_module(buff->data(), buff->size()); + const auto& func2 = bc.get_method("test_func").function(); + ASSERT_EQ( + func2.get_code().operator_input_sizes_.size(), + func2.get_code().operators_.size()); + } +} + +Module jitModuleFromBuffer(void* data, size_t size) { + // Make a copy of the data so we can use the existing API, which takes + // ownership. The `data` param might point into the middle of a buffer, so we + // can't safely take ownership of it directly. + // @nolint CLANGTIDY cppcoreguidelines-no-malloc + std::shared_ptr copy(static_cast(malloc(size)), free); + memcpy(copy.get(), data, size); + + ExtraFilesMap extra_files; + return parse_and_initialize_jit_module(std::move(copy), size, extra_files); +} + +TEST(TestSourceFlatbuffer, UpsampleNearest2d) { + Module m("m"); + m.define(R"( + def forward(self, input: Tensor, scale:float): + return torch.upsample_nearest2d(input, [1, 1], float(scale), float(scale)) + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({1, 3, 128, 128})); + inputs.emplace_back(at::Scalar(2.0)); + auto ref = m.forward(inputs); + + std::stringstream ss; + m._save_for_mobile(ss, {}, false, /*use_fatbuffer=*/true); + auto mm = _load_for_mobile(ss); + auto m2 = load(ss); + + auto res = m2.forward(inputs); + auto resm = mm.forward(inputs); + + auto resd = res.toTensor(); + auto refd = ref.toTensor(); + auto resmd = resm.toTensor(); + ASSERT_TRUE(resd.equal(refd)); + ASSERT_TRUE(resmd.equal(refd)); +} + +TEST(TestSourceFlatbuffer, CheckAttrAccess) { + Module m("m"); + m.register_attribute("mobile_optimized", BoolType::get(), true); + auto data = save_jit_module_to_bytes(m); + Module m2 = jitModuleFromBuffer(data->data(), data->size()); + bool mobile_optimized = m2.attr("mobile_optimized", false).toBool(); + AT_ASSERT(mobile_optimized); + mobile::Module m3 = parse_mobile_module(data->data(), data->size()); + mobile_optimized = m3.attr("mobile_optimized", false).toBool(); + AT_ASSERT(mobile_optimized); +} + +TEST(TestSourceFlatbuffer, + MethodInvocation) { // NOLINT (use =delete in gtest) + const std::vector test_programs{ + // test invoking a method with default parameter + R"( + def test_func(self, x, b : int = 4): + return self.foo + x + b + )", + // inner method call with default parameter (gets inlined) + R"( + def add_with_default_arg(self, x, b : int = 4): + return self.foo + x + b + def test_func(self, x): + return self.add_with_default_arg(x) # invoke method w/ default arg + )", + // simple method call + R"( + def test_func(self, x): + b = 4 + return self.foo + x + b + )", + }; + for (const auto& test_program : test_programs) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(test_program); + + const int fortyTwo = 42; // (keep linter happy) + auto minput = fortyTwo * torch::ones({}); + auto ref = m.run_method("test_func", minput); + + auto data = save_jit_module_to_bytes(m); + Module m2 = jitModuleFromBuffer(data->data(), data->size()); + const auto& test_func = m2.get_method("test_func"); + IValue res; + for (int i = 0; i < 3; ++i) { + res = test_func({minput}); + } + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); + + mobile::Module m3 = parse_mobile_module(data->data(), data->size()); + const auto& test_func3 = m3.get_method("test_func"); + for (int i = 0; i < 3; ++i) { + res = test_func3({minput}); + } + resd = res.toTensor().item(); + refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); + } +} + +#if !defined FB_XPLAT_BUILD +// The following test run in fbcode only +TEST(FlatbufferUpgraderTest, DivTensorV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_tensor_v2.ptl.ff"); + /* + (('__torch__.MyModule.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('LOAD', 2, 0), + ('LOAD', 3, 0), + ('OP', 0, 0), + ('LOAD', 2, 0), + ('LOAD', 3, 0), + ('OP', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 2, 0), + ('TUPLE_CONSTRUCT', 3, 0), + ('RET', 0, 0))), + ('operators', + (('aten::div', 'Tensor'), + ('aten::div', 'Tensor'), + ('aten::div', 'Tensor'))), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + + */ + mobile::Module m_module = load_mobile_module_from_file(test_model_file); + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // 3 operators will use upgrader + ASSERT_EQ(number_of_call_instruction, 3); + + std::vector inputs = { + IValue(6 * torch::ones({1})), IValue(3 * torch::ones({1}))}; + auto actual_output = m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output_list = actual_output.toTuple()->elements(); + ASSERT_TRUE(actual_output_list[0].toTensor().equal(expect_output)); +} + +TEST(FlatbufferUpgraderTest, DivTensorOutV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_tensor_out_v2.ptl.ff"); + /* + (('__torch__.MyModule.forward', + (('instructions', + (('STOREN', 1, 4), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('MOVE', 4, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div', 'out'),)), + ('constants', ()), + ('types', ()), + ('register_size', 4))),) + */ + mobile::Module m_module = load_mobile_module_from_file(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{ + IValue(6 * torch::ones({1})), + IValue(3 * torch::ones({1})), + IValue(torch::empty({1}))}; + m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = inputs[2].toTensor(); + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(FlatbufferUpgraderTest, DivTensorInplaceV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_tensor_inplace_v2.ptl.ff"); + /* + (('__torch__.MyModule.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div_', 'Tensor'),)), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + mobile::Module m_module = load_mobile_module_from_file(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{ + IValue(6 * torch::ones({1})), IValue(3 * torch::ones({1}))}; + m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = inputs[0].toTensor(); + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(FlatbufferUpgraderTest, DivScalarFloatV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_scalar_float_v2.ptl.ff"); + /* + (('__torch__.MyModuleFloat.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div', 'Scalar'),)), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + + mobile::Module m_module = load_mobile_module_from_file(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3.0)}; + auto output = m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = output.toTensor(); + + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(FlatbufferUpgraderTest, DivScalarReciprocalFloatV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, + "upgrader_models/test_versioned_div_scalar_reciprocal_float_v2.ptl.ff"); + /* + (('__torch__.MyModuleFloat.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('OP', 0, 0), + ('MOVE', 3, 0), + ('OP', 1, 0), + ('RET', 0, 0))), + ('operators', (('aten::reciprocal', ''), ('aten::mul', 'Scalar'))), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + mobile::Module m_module = load_mobile_module_from_file(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // No operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 0); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3.0)}; + auto output = m_module.forward(inputs); + auto expect_output = 0.5 * torch::ones({1}); + auto actual_output = output.toTensor(); + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(FlatbufferUpgraderTest, DivScalarReciprocalIntV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, + "upgrader_models/test_versioned_div_scalar_reciprocal_int_v2.ptl.ff"); + /* + (('__torch__.MyModuleInt.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('OP', 0, 0), + ('MOVE', 3, 0), + ('OP', 1, 0), + ('RET', 0, 0))), + ('operators', (('aten::reciprocal', ''), ('aten::mul', 'Scalar'))), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + mobile::Module m_module = load_mobile_module_from_file(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // No operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 0); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3.0)}; + auto output = m_module.forward(inputs); + auto expect_output = 0.5 * torch::ones({1}); + auto actual_output = output.toTensor(); + + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(FlatbufferUpgraderTest, DivScalarScalarV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_scalar_scalar_v2.ptl.ff"); + /* + (('__torch__.MyModule.forward', + (('instructions', + (('STOREN', 1, 5), + ('DROPR', 1, 0), + ('LOAD', 2, 0), + ('LOAD', 3, 0), + ('OP', 0, 0), + ('MOVE', 2, 0), + ('LOAD', 4, 0), + ('OP', 1, 0), + ('LOAD', 3, 0), + ('MOVE', 4, 0), + ('OP', 2, 0), + ('MOVE', 3, 0), + ('MOVE', 5, 0), + ('OP', 3, 0), + ('TUPLE_CONSTRUCT', 4, 0), + ('RET', 0, 0))), + ('operators', + (('aten::div', ''), + ('aten::div', 'float'), + ('aten::div', ''), + ('aten::div', 'int'))), + ('constants', ()), + ('types', ()), + ('register_size', 5))),) + */ + mobile::Module m_module = load_mobile_module_from_file(test_model_file); + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // No operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 0); + + std::vector inputs{IValue(20.0), IValue(10), IValue(2.0), IValue(5)}; + auto output = m_module.forward(inputs); + auto output_list = output.toTupleRef().elements(); + auto expect_output = std::vector( + {IValue(2.0), IValue(10.0), IValue(5.0), IValue(2.0)}); + // auto actual_output = output.toTensor(); + for (size_t i = 0; i < expect_output.size(); i++) { + ASSERT_EQ(output_list[i], expect_output[i]); + } +} + +TEST(FlatbufferUpgraderTest, DivScalarIntV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_scalar_int_v2.ptl.ff"); + /* + (('__torch__.MyModuleInt.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div', 'Scalar'),)), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + mobile::Module m_module = load_mobile_module_from_file(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3)}; + auto output = m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = output.toTensor(); + + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(FlatbufferUpgraderTest, DivScalarInplaceFloatV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, + "upgrader_models/test_versioned_div_scalar_inplace_float_v2.ptl.ff"); + /* + (('__torch__.MyModuleFloat.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div_', 'Scalar'),)), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + + mobile::Module m_module = load_mobile_module_from_file(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3.0)}; + auto output = m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = output.toTensor(); + + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(FlatbufferUpgraderTest, DivScalarInplaceIntV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, + "upgrader_models/test_versioned_div_scalar_inplace_int_v2.ptl.ff"); + /* + (('__torch__.MyModuleInt.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div_', 'Scalar'),)), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + + mobile::Module m_module = load_mobile_module_from_file(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3)}; + auto output = m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = output.toTensor(); + + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +#endif // !defined(FB_XPLAT_BUILD) + +// +// Tests that need access to internal flatbuffers types/functions. +// Do not add any other tests after this section. +// + +} // namespace jit +} // namespace torch +namespace torch { +namespace jit { + +/** + * An Allocator that can only deallocate (using delete []), counting + * the number of times that it has been asked to deallocate. + */ +class TestAllocator : public flatbuffers::Allocator { + public: + /** + * *deallocate_call_count will be incremented whenever deallocate() is called. + */ + explicit TestAllocator(int* deallocate_call_count) + : deallocate_call_count_(deallocate_call_count) {} + + void deallocate(uint8_t* p, size_t /*size*/) override { + *deallocate_call_count_ += 1; + delete[] p; + } + + uint8_t* allocate(size_t) override { + TORCH_CHECK(false, "allocate() should not be called"); + } + uint8_t* reallocate_downward(uint8_t*, size_t, size_t, size_t, size_t) + override { + TORCH_CHECK(false, "reallocate_downward() should not be called"); + } + + private: + int* deallocate_call_count_; +}; + +/// Provides access to DetachedBuffer::destroy(). +struct DetachedBufferTestingFriend { + /// Returns a UniqueDetachedBuffer that wraps the provided DetachedBuffer. + /// A copy of similar code in flatbuffer_serializer.cpp. + static DetachedBuffer::UniqueDetachedBuffer make_unique_detached_buffer( + DetachedBuffer* buf) { + return DetachedBuffer::UniqueDetachedBuffer(buf, DetachedBuffer::destroy); + } +}; + +TEST(FlatbufferTest, DetachedBufferSmoke) { + // Use a custom Allocator to watch the lifecycle of a + // flatbuffers::DetachedBuffer. + int deallocate_call_count = 0; + TestAllocator alloc(&deallocate_call_count); + + // Data for the buffer. TestAllocator will free it with `delete []`. + constexpr size_t data_size = 4; + uint8_t* data = new uint8_t[data_size]; + + // An internal buffer on the stack that owns the data. + flatbuffers::DetachedBuffer fb_buf_local( + &alloc, /*own_allocator=*/false, data, data_size, data, data_size); + EXPECT_EQ(fb_buf_local.data(), data); + EXPECT_EQ(fb_buf_local.size(), data_size); + + // Mimic the code inside save_mobile_module_to_bytes by transferring ownership + // to a heap object. + auto fb_buf_ptr = new flatbuffers::DetachedBuffer(std::move(fb_buf_local)); + // The data should not have been deleted yet. + EXPECT_EQ(deallocate_call_count, 0); + // The new object points to the data. + EXPECT_EQ(fb_buf_ptr->data(), data); + EXPECT_EQ(fb_buf_ptr->size(), data_size); + // The old object points to nothing. + // @lint-ignore CLANGTIDY bugprone-use-after-move + EXPECT_EQ(fb_buf_local.data(), nullptr); + // @lint-ignore CLANGTIDY bugprone-use-after-move + EXPECT_EQ(fb_buf_local.size(), 0); + + // The top-level torch::jit::DetachedBuffer. + auto wrapped_buf = + new DetachedBuffer(fb_buf_ptr->data(), fb_buf_ptr->size(), fb_buf_ptr); + EXPECT_EQ(wrapped_buf->data(), data); + EXPECT_EQ(wrapped_buf->size(), data_size); + + // The unique_ptr that owns the torch::jit::DetachedBuffer and its contents. + { + DetachedBuffer::UniqueDetachedBuffer unique_buf = + DetachedBufferTestingFriend::make_unique_detached_buffer(wrapped_buf); + EXPECT_EQ(unique_buf->data(), data); + EXPECT_EQ(unique_buf->size(), data_size); + + // The data should not have been deleted yet. + EXPECT_EQ(deallocate_call_count, 0); + } + + // Now that the unique_ptr is out of scope, the data should have been deleted. + EXPECT_EQ(deallocate_call_count, 1); +} + +TEST(FlatbufferTest, DetachedBufferNullOwner) { + // a torch::jit::DetachedBuffer with a null internal owner. + std::vector data(4); + auto wrapped_buf = new DetachedBuffer(data.data(), data.size()); + + // A unique_ptr that owns the torch::jit::DetachedBuffer and its contents. + { + DetachedBuffer::UniqueDetachedBuffer unique_buf = + DetachedBufferTestingFriend::make_unique_detached_buffer(wrapped_buf); + EXPECT_EQ(unique_buf->data(), data.data()); + EXPECT_EQ(unique_buf->size(), data.size()); + } + + // The DetachedBuffer should have been destroyed when the UniqueDetachedBuffer + // went out of scope. If we didn't crash or get any ASAN warnings, we should + // be good. +} + +// +// Do not add tests here unless they require flatbuffers types. See comment at +// the beginning of this section. +// + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_fuser.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_fuser.cpp new file mode 100644 index 0000000000000000000000000000000000000000..233fcecc8494762856061a3cf10f816acaa3f67e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_fuser.cpp @@ -0,0 +1,280 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +class FuserTest : public ::testing::Test { + void SetUp() override { + old_nvfuser_value_ = fuser::cuda::setEnabled(false); + } + void TearDown() override { + fuser::cuda::setEnabled(old_nvfuser_value_); + } + + private: + bool old_nvfuser_value_; +}; + +TEST_F(FuserTest, TestSimple_CUDA) { +#if defined(FBCODE_CAFFE2) + return; +#endif + const auto graph_string = R"IR( + graph(%0 : Tensor, + %1 : Tensor): + %2 : Tensor = aten::mul(%0, %1) + return (%2))IR"; + Graph graph; + torch::jit::parseIR(graph_string, &graph); + + auto a = at::rand({3, 4}, at::kCUDA); + auto b = at::rand({4, 3}, at::kCUDA).transpose(0, 1); + auto o = at::zeros({3, 4}, at::kCUDA); + auto outputs = debugLaunchGraph(graph, {a, b}); + ASSERT_EQ(outputs.size(), 1); + auto o2 = a * b; + float max_diff = (o2 - outputs[0]).abs().max().item(); + // std::cout << "max diff: " << max_diff << "\n"; + ASSERT_EQ(max_diff, 0); +} + +TEST_F(FuserTest, TestOne_CUDA) { +#if defined(FBCODE_CAFFE2) + return; +#endif + auto testOne = [&](int ti, int tj) { + const auto graph_string = R"IR( + graph(%0 : Tensor, + %1 : Tensor, + %2 : Tensor, + %3 : Tensor, + %4 : Tensor): + %5 : Tensor = aten::sigmoid(%4) + %6 : Tensor = aten::sigmoid(%3) + %7 : Tensor = aten::tanh(%2) + %8 : Tensor = aten::sigmoid(%1) + %9 : Tensor = aten::mul(%6, %0) + %10 : Tensor = aten::mul(%5, %7) + %11 : int = prim::Constant[value=1]() + %12 : Tensor = aten::add(%9, %10, %11) + %13 : Tensor = aten::tanh(%12) + %14 : Tensor = aten::mul(%8, %13) + return (%14, %12))IR"; + Graph graph; + torch::jit::parseIR(graph_string, &graph); + + graph.lint(); + + std::vector inputs; + // We want to generate input/output tensors with dimension 128x128x32, but + // with different internal strides. To do this, we generate a tensor + // with the "wrong" dimensions, and then use transpose to get an + // appropriately sized view. + std::generate_n( + std::back_inserter(inputs), graph.inputs().size(), [ti, tj] { + std::array dims = {128, 128, 32}; + std::swap(dims[ti], dims[tj]); + return at::rand(dims, at::kCUDA).transpose(ti, tj); + }); + + auto t22 = inputs[4].sigmoid(); + auto t20 = inputs[3].sigmoid(); + auto t18 = inputs[2].tanh(); + auto t16 = inputs[1].sigmoid(); + auto t14 = t20 * inputs[0]; + auto t11 = t22 * t18; + auto out1 = t14 + t11; + auto t5 = out1.tanh(); + auto out0 = t16 * t5; + + auto outputs = debugLaunchGraph(graph, inputs); + ASSERT_EQ(outputs.size(), graph.outputs().size()); + ASSERT_TRUE(out0.is_same_size(outputs.front())); + float max_diff = (outputs.front() - out0).abs().max().item(); + ASSERT_TRUE(max_diff < 1e-6); + }; + testOne(0, 0); + testOne(0, 1); + testOne(1, 2); + testOne(0, 2); +} + +TEST_F(FuserTest, FusedConcat_CUDA) { +#if defined(FBCODE_CAFFE2) + return; +#endif + const auto graph_string0 = R"IR( + graph(%0 : Tensor, + %1 : Tensor): + %2 : Tensor = aten::mul(%0, %1) + %3 : Tensor = prim::FusedConcat[dim=0](%0, %2) + return (%2, %3))IR"; + const auto graph_string1 = R"IR( + graph(%0 : Tensor, + %1 : Tensor): + %2 : Tensor = aten::mul(%0, %1) + %3 : Tensor = prim::FusedConcat[dim=1](%0, %2) + return (%2, %3))IR"; + const auto graph_string2 = R"IR( + graph(%0 : Tensor, + %1 : Tensor): + %2 : Tensor = aten::mul(%0, %1) + %3 : Tensor = prim::FusedConcat[dim=2](%0, %2) + return (%2, %3))IR"; + + auto a = at::rand({3, 4, 5}, at::kCUDA); + auto b = at::rand({4, 3, 5}, at::kCUDA).transpose(0, 1); + const auto o_r = a * b; + + std::vector graph_strings{ + graph_string0, graph_string1, graph_string2}; + for (const auto i : c10::irange(graph_strings.size())) { + Graph g; + torch::jit::parseIR(graph_strings[i], &g); + + auto outputs = debugLaunchGraph(g, {a, b}); + ASSERT_EQ(outputs.size(), 2); + + float max_diff = (o_r - outputs[0]).abs().max().item(); + ASSERT_EQ(max_diff, 0); + + const auto o2_r = at::cat({a, o_r}, i); + float max_diff2 = (o2_r - outputs[1]).abs().max().item(); + ASSERT_EQ(max_diff2, 0); + }; +} + +TEST_F(FuserTest, FusionAliasing) { +#if defined(FBCODE_CAFFE2) + return; +#endif + const auto graph_string = R"IR( + graph(%0 : Tensor, + %1 : Tensor): + %12 : int = prim::Constant[value=1]() + %2.1 : Tensor = aten::mul(%0, %1) + %2 : Tensor = aten::mul(%2.1, %1) + %3 : Tensor = aten::add_(%2, %1, %12) + %4 : Tensor = aten::mul(%2, %1) + %5 : Tensor = aten::add(%2, %4, %12) + return (%5))IR"; + auto g = std::make_shared(); + torch::jit::parseIR(graph_string, g.get()); + + g->lint(); + FuseGraph(g); + + // We should not be able to fuse across the in-place operation here. + testing::FileCheck() + .check("prim::FusionGroup_0") + ->check("aten::add_") + ->check("prim::FusionGroup_1") + ->run(*g); +} + +TEST_F(FuserTest, KernelCaching) { +#if defined(FBCODE_CAFFE2) + return; +#endif + + // Constructs two functionally equivalent graphs + const auto graph0_string = R"IR( + graph(%0 : Float(2, 3, 4), + %1 : Float(2, 3, 4)): + %c0 : Float(2, 3, 4) = aten::mul(%0, %1) + %d0 : Float(2, 3, 4) = aten::mul(%c0, %0) + return (%d0))IR"; + auto g0 = std::make_shared(); + torch::jit::parseIR(graph0_string, g0.get()); + + const auto graph1_string = R"IR( + graph(%0 : Float(2, 3, 4), + %1 : Float(2, 3, 4)): + %c1 : Float(2, 3, 4) = aten::mul(%0, %1) + %d1 : Float(2, 3, 4) = aten::mul(%c1, %0) + return (%d1))IR"; + auto g1 = std::make_shared(); + torch::jit::parseIR(graph1_string, g1.get()); + + auto getFusionGroup = [](const std::shared_ptr& graph) { + const auto& nodes = graph->nodes(); + auto maybe_fusion_group = + std::find_if(nodes.begin(), nodes.end(), [](const Node* node) { + return node->kind() == prim::FusionGroup; + }); + TORCH_CHECK( + maybe_fusion_group != nodes.end(), + "testRegisterFusionCachesKernel: could not create FusionGroup"); + return *maybe_fusion_group; + }; + + // Creates two alpha-equivalent fusion groups + torch::jit::overrideCanFuseOnCPU(true); + FuseGraph(g0); + FuseGraph(g1); + torch::jit::overrideCanFuseOnCPU(false); + auto fg0 = getFusionGroup(g0); + auto fg1 = getFusionGroup(g1); + + // Registers both with the fusion compiler. + auto expected_key = registerFusion(fg0); + auto second_key = registerFusion(fg1); + + // Because the graphs are alpha-equivalent, they should return the same key + // and therefore share a KernelSpec to share kernels for specializations + ASSERT_EQ(second_key, expected_key); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_graph_executor.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_graph_executor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..67a8528feda6681b147acb9c63cfd76a9b54bc13 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_graph_executor.cpp @@ -0,0 +1,72 @@ +#include + +#include + +#include "test/cpp/jit/test_utils.h" +#include "torch/csrc/jit/runtime/graph_executor.h" +#include "torch/jit.h" +#include "torch/script.h" +#include "torch/torch.h" + +namespace torch { +namespace jit { + +TEST(GraphExecutorTest, Basic_CUDA) { + constexpr int batch_size = 4; + constexpr int input_size = 256; + + int hidden_size = 2 * input_size; + + auto input = at::randn({batch_size, input_size}, at::kCUDA); + auto hx = at::randn({batch_size, hidden_size}, at::kCUDA); + auto cx = at::randn({batch_size, hidden_size}, at::kCUDA); + auto w_ih = t_def(at::randn({4 * hidden_size, input_size}, at::kCUDA)); + auto w_hh = t_def(at::randn({4 * hidden_size, hidden_size}, at::kCUDA)); + + auto g = build_lstm(); + GraphExecutor executor(g, ""); + auto stack = createStack({input, hx, cx, w_ih, w_hh}); + executor.run(stack); + ASSERT_EQ(stack.size(), 2); + auto [r0, r1] = lstm(input, hx, cx, w_ih, w_hh); + ASSERT_TRUE(almostEqual(stack[0].toTensor(), r0)); + ASSERT_TRUE(almostEqual(stack[1].toTensor(), r1)); +} + +TEST(GraphExecutorTest, runAsync_executor) { + /* + TODO: there are some problem with C++ parsing script program involving + fork. Use the test module below for now. + issue about this: github.com/pytorch/pytorch/issues/46368 + The test module file is generated by following: + class DemoModule(torch.nn.Module): + def forward(self): + r1 = torch.jit.fork(torch.mm, torch.rand(100,100),torch.rand(100,100)) + r2 = torch.jit.fork(torch.mm, torch.rand(100,100),torch.rand(100,100)) + return r1.wait() + r2.wait() + demo = DemoModule() + torch.jit.save(torch.jit.script(demo), 'test_interpreter_async.pt') + */ + auto testModelFile = + resolveTestDataFile(__FILE__, "test_interpreter_async.pt"); + auto module = load(testModelFile); + auto graph = module.get_method("forward").graph(); + GraphExecutor graphExecutor(graph, ""); + auto asyncCounter = 0; + std::mutex mtx; + // a dummy executor which actually use at::launch, but add up a counter + auto launcher = [&](std::function f) { + mtx.lock(); + ++asyncCounter; + mtx.unlock(); + at::launch(std::move(f)); + }; + std::vector stack; + // NOLINTNEXTLINE(modernize-use-emplace) + stack.push_back(module._ivalue()); + graphExecutor.runAsync(stack, launcher)->wait(); + ASSERT_TRUE(asyncCounter > 0); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_graph_iterator.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_graph_iterator.cpp new file mode 100644 index 0000000000000000000000000000000000000000..faf656c0734a5a72fcf1926bf60d3ce26c3d3763 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_graph_iterator.cpp @@ -0,0 +1,203 @@ +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +/** + * Inverts an unordered map. + */ +template +std::unordered_map invert_map(std::unordered_map& map) { + std::unordered_map inverted; + std::for_each(map.begin(), map.end(), [&inverted](const std::pair& p) { + inverted.insert(std::make_pair(p.second, p.first)); + }); + return inverted; +} + +/** + * Traverses the graph using the DepthFirstGraphNodeIterator and + * returns an array containing the original names in the string + * graph. + */ +std::vector traverse_depth_first( + std::string graph_string, + int max_count = 100) { + auto graph = std::make_shared(); + std::unordered_map vmap; + torch::jit::parseIR(graph_string, graph.get(), vmap); + auto get_name = invert_map(vmap); + + std::vector result; + DepthFirstGraphNodeIterator graph_it(graph); + Node* node = graph_it.next(); + int count = 0; + while (node && count < max_count) { + std::stringstream buffer; + std::vector vec; + node->print(buffer, 0, &vec, false, true, true, false); + result.push_back(std::move(buffer).str()); + node = graph_it.next(); + ++count; + } + return result; +} + +/** Checks that the iteration order matches the expected/provided order. */ +void assert_ordering( + std::vector actual, + std::initializer_list expected_list) { + auto expected = std::vector(expected_list); + ASSERT_EQ(expected.size(), actual.size()) + << "Got " << actual.size() << " elements (" << actual << ')' + << " expected " << expected.size() << " elements (" << expected << ')'; + for (unsigned i = 0; i < expected.size(); i++) { + ASSERT_EQ(expected[i], actual[i]) + << "Difference at index " << i << " in " << actual << " (expected " + << actual << ')'; + } +} + +TEST(GraphIteratorTest, ConstantReturnGraph) { + const auto graph_string = R"IR( + graph(): + %1 : int = prim::Constant[value=0]() + return (%1))IR"; + auto graph = std::make_shared(); + torch::jit::parseIR(graph_string, graph.get()); + DepthFirstGraphNodeIterator graph_it(graph); + ASSERT_EQ(graph_it.next()->kind(), prim::Constant); + ASSERT_EQ(graph_it.next(), nullptr); +} + +TEST(GraphIteratorTest, GraphWithParameters) { + const auto graph_string = R"IR( + graph(%0 : Double(2)): + %1 : int = prim::Constant[value=0]() + return (%0))IR"; + auto ordering = traverse_depth_first(graph_string); + assert_ordering(ordering, {"%1 : int = prim::Constant[value=0]()"}); +} + +TEST(GraphIteratorTest, GraphWithIf) { + const auto graph_string = R"IR( +graph(%a : Tensor): + %a : int = prim::Constant[value=30]() + %b : int = prim::Constant[value=10]() + %c : bool = aten::Bool(%a) + %d : int = prim::If(%c) + block0(): + -> (%a) + block1(): + -> (%b) + %e : int = prim::Constant[value=20]() + return (%d) +)IR"; + auto ordering = traverse_depth_first(graph_string); + assert_ordering( + ordering, + {"%1 : int = prim::Constant[value=30]()", + "%2 : int = prim::Constant[value=10]()", + "%3 : bool = aten::Bool(%1)", + "%4 : int = prim::If(%3)", + "%5 : int = prim::Constant[value=20]()"}); +} + +TEST(GraphIteratorTest, GraphWithNestedIf) { + const auto graph_string = R"IR( +graph(%a.1 : Tensor, + %b.1 : Tensor): + %2 : int = prim::Constant[value=10]() + %3 : int = prim::Constant[value=20]() + %4 : int = prim::Constant[value=30]() + %5 : int = prim::Constant[value=40]() + %6 : bool = aten::Bool(%a.1) + %7 : int = prim::If(%6) + block0(): + %8 : bool = aten::Bool(%b.1) + %9 : int = prim::If(%8) + block0(): + -> (%2) + block1(): + -> (%3) + -> (%9) + block1(): + %10 : bool = aten::Bool(%b.1) + %11 : int = prim::If(%10) + block0(): + -> (%4) + block1(): + -> (%5) + -> (%11) + %8 : bool = aten::Bool(%b.1) + %9 : int = prim::If(%8) + block0(): + -> (%2) + block1(): + -> (%3) + %10 : bool = aten::Bool(%b.1) + %11 : int = prim::If(%10) + block0(): + -> (%4) + block1(): + -> (%5) + return (%7) +)IR"; + auto ordering = traverse_depth_first(graph_string); + assert_ordering( + ordering, + {"%2 : int = prim::Constant[value=10]()", + "%3 : int = prim::Constant[value=20]()", + "%4 : int = prim::Constant[value=30]()", + "%5 : int = prim::Constant[value=40]()", + "%6 : bool = aten::Bool(%a.1)", + "%7 : int = prim::If(%6)", + "%8 : bool = aten::Bool(%b.1)", + "%9 : int = prim::If(%8)", + "%10 : bool = aten::Bool(%b.1)", + "%11 : int = prim::If(%10)", + "%12 : bool = aten::Bool(%b.1)", + "%13 : int = prim::If(%12)", + "%14 : bool = aten::Bool(%b.1)", + "%15 : int = prim::If(%14)"}); +} + +TEST(GraphIteratorTest, GraphWithLoop) { + const auto graph_string = R"IR( +graph(%a.1 : Tensor): + %1 : bool = prim::Constant[value=1]() + %2 : int = prim::Constant[value=10]() + %3 : int = prim::Constant[value=1]() + %4 : Tensor = prim::Loop(%2, %1, %a.1) + block0(%i : int, %b.9 : Tensor): + %5 : Tensor = aten::add_(%b.9, %3, %3) + -> (%1, %5) + %6 : Tensor = prim::Loop(%2, %1, %a.1) + block0(%i : int, %b.9 : Tensor): + -> (%1, %4) + return (%6) +)IR"; + auto ordering = traverse_depth_first(graph_string); + assert_ordering( + ordering, + {"%1 : bool = prim::Constant[value=1]()", + "%2 : int = prim::Constant[value=10]()", + "%3 : int = prim::Constant[value=1]()", + "%4 : Tensor = prim::Loop(%2, %1, %a.1)", + "%7 : Tensor = aten::add_(%b.10, %3, %3)", + "%8 : Tensor = prim::Loop(%2, %1, %a.1)"}); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_inliner.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_inliner.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a1b17f368303039a968642074948f1391b126ae8 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_inliner.cpp @@ -0,0 +1,51 @@ +#include + +#include +#include +#include +#include + +const auto testSource = R"JIT( +def foo1(x): + print("one") + return x + +def foo2(x): + print("two") + return foo1(x) + +def foo3(x): + print("three") + return foo2(x) +)JIT"; + +namespace torch { +namespace jit { +using namespace testing; + +struct InlinerGuard { + explicit InlinerGuard(bool shouldInline) + : oldState_(getInlineEverythingMode()) { + getInlineEverythingMode() = shouldInline; + } + + ~InlinerGuard() { + getInlineEverythingMode() = oldState_; + } + + bool oldState_; +}; + +TEST(InlinerTest, Basic) { + // disable automatic inlining so we can test it manually + InlinerGuard guard(/*shouldInline=*/false); + + CompilationUnit cu(testSource); + auto& fn = cu.get_function("foo3"); + + auto g = toGraphFunction(fn).graph(); + Inline(*g); + FileCheck().check_count("prim::Print", 3)->run(*g); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_interface.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_interface.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6f7ded6de31e5dd8e9d6054e8e6786bc0fa922b4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_interface.cpp @@ -0,0 +1,80 @@ +#include + +#include + +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +static const std::vector subMethodSrcs = {R"JIT( +def one(self, x: Tensor, y: Tensor) -> Tensor: + return x + y + 1 + +def forward(self, x: Tensor) -> Tensor: + return x +)JIT"}; +static const std::string parentForward = R"JIT( +def forward(self, x: Tensor) -> Tensor: + return self.subMod.forward(x) +)JIT"; + +static constexpr std::string_view moduleInterfaceSrc = R"JIT( +class OneForward(ModuleInterface): + def one(self, x: Tensor, y: Tensor) -> Tensor: + pass + def forward(self, x: Tensor) -> Tensor: + pass +)JIT"; + +static void import_libs( + std::shared_ptr cu, + const std::string& class_name, + const std::shared_ptr& src, + const std::vector& tensor_table) { + SourceImporter si( + cu, + &tensor_table, + [&](const std::string& name) -> std::shared_ptr { return src; }, + /*version=*/2); + si.loadType(QualifiedName(class_name)); +} + +TEST(InterfaceTest, ModuleInterfaceSerialization) { + auto cu = std::make_shared(); + Module parentMod("parentMod", cu); + Module subMod("subMod", cu); + + std::vector constantTable; + import_libs( + cu, + "__torch__.OneForward", + std::make_shared(moduleInterfaceSrc), + constantTable); + + for (const std::string& method : subMethodSrcs) { + subMod.define(method, nativeResolver()); + } + parentMod.register_attribute( + "subMod", + cu->get_interface("__torch__.OneForward"), + subMod._ivalue(), + // NOLINTNEXTLINE(bugprone-argument-comment) + /*is_parameter=*/false); + parentMod.define(parentForward, nativeResolver()); + ASSERT_TRUE(parentMod.hasattr("subMod")); + std::stringstream ss; + parentMod.save(ss); + Module reloaded_mod = jit::load(ss); + ASSERT_TRUE(reloaded_mod.hasattr("subMod")); + InterfaceTypePtr submodType = + reloaded_mod.type()->getAttribute("subMod")->cast(); + ASSERT_TRUE(submodType->is_module()); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_interpreter.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_interpreter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..97f58f58506fbee6f8866b621f67ce6666706171 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_interpreter.cpp @@ -0,0 +1,297 @@ +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +class TypeCheckTest : public ::testing::Test { + protected: + TypeCheckTest() : interp(makeInterp()) {} + + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + InterpreterState interp; + + private: + static InterpreterState makeInterp() { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(%a.1 : Tensor, + %b.1 : Tensor): + %t0 : Float(2, 2, strides=[2, 1], device=cpu, requires_grad=1), %t1 : Float(3, 3, strides=[3, 1]), %type_matched : bool = prim::TypeCheck[types=[Float(2, 2, strides=[2, 1], device=cpu, requires_grad=1), Float(3, 3, strides=[3, 1])]](%a.1, %b.1) + return (%t0, %t1, %type_matched) + )IR", + &*graph, + vmap); + + Code function(graph, ""); + return InterpreterState(function); + } +}; + +TEST_F(TypeCheckTest, MatchingType) { + // TypeCheck yields to true! Shape, grad and device matches. + auto a = at::zeros({2, 2}, at::kFloat); + auto b = at::ones({3, 3}, at::kFloat); + a.set_requires_grad(true); + a = a.to(at::kCPU); + std::vector stack({a, b}); + interp.run(stack); + ASSERT_TRUE(exactlyEqual(stack[0].toTensor(), a)); + ASSERT_TRUE(exactlyEqual(stack[1].toTensor(), b)); + ASSERT_TRUE(stack[2].toBool()); +} + +TEST_F(TypeCheckTest, SizeMismatch) { + auto a = at::zeros({2, 2}, at::kFloat); + auto b = at::ones({2, 2}, at::kFloat); // Size mismatch + a.set_requires_grad(true); + a = a.to(at::kCPU); + std::vector stack({a, b}); + interp.run(stack); + ASSERT_FALSE(stack[2].toBool()); +} + +TEST_F(TypeCheckTest, GradientMismatch) { + auto a = at::zeros({2, 2}, at::kFloat); + auto b = at::ones({3, 3}, at::kFloat); + a = a.to(at::kCPU); + a.set_requires_grad(false); // Gradient mismatch + std::vector stack({a, b}); + interp.run(stack); + ASSERT_FALSE(stack[2].toBool()); +} + +TEST_F(TypeCheckTest, ScalarTypeMismatch) { + auto a = at::zeros({2, 2}, at::kFloat); + auto b = at::ones({3, 3}, at::kFloat); + a = a.to(at::kCPU); + a.set_requires_grad(true); + a = a.to(at::kInt); // Scalar type mismatch + std::vector stack({a, b}); + interp.run(stack); + ASSERT_FALSE(stack[2].toBool()); +} + +TEST_F(TypeCheckTest, DeviceMismatch_CUDA) { + auto a = at::zeros({2, 2}, at::kFloat); + auto b = at::ones({3, 3}, at::kFloat); + a.set_requires_grad(true); + a = a.to(at::kCUDA); // Device mismatch + std::vector stack({a, b}); + interp.run(stack); + ASSERT_FALSE(stack[2].toBool()); +} + +// TODO: These tests weren't doing anything. +// TEST(TypeCheckErrorTest, EmptyCheckRaises) { +// // Test empty Typecheck raises an internal assertion +// auto graph = std::make_shared(); +// std::unordered_map vmap; +// EXPECT_ANY_THROW(parseIR( +// R"IR( +// graph(%a.1 : Tensor, +// %b.1 : Tensor): +// %type_matched : bool = prim::TypeCheck() +// return (%type_matched) +// )IR", +// &*graph, +// vmap)); +// } + +// TODO: These tests weren't doing anything. +// TEST(TypeCheckErrorTest, WrongInputOutputCountRaises) { +// // Test for assertion if num_inputs + 1 != num_outputs +// auto graph = std::make_shared(); +// std::unordered_map vmap; +// EXPECT_ANY_THROW(parseIR( +// R"IR( +// graph(%a.1 : Tensor, +// %b.1 : Tensor): +// %type_matched : bool = prim::TypeCheck(%a.1) +// return (%type_matched) +// )IR", +// &*graph, +// vmap)); +// } + +TEST(InterpreterTest, Basic_CUDA) { + constexpr int batch_size = 4; + constexpr int input_size = 256; + constexpr int seq_len = 32; + + int hidden_size = 2 * input_size; + + auto input = at::randn({seq_len, batch_size, input_size}, at::kCUDA); + auto hx = at::randn({batch_size, hidden_size}, at::kCUDA); + auto cx = at::randn({batch_size, hidden_size}, at::kCUDA); + auto w_ih = t_def(at::randn({4 * hidden_size, input_size}, at::kCUDA)); + auto w_hh = t_def(at::randn({4 * hidden_size, hidden_size}, at::kCUDA)); + + auto lstm_g = build_lstm(); + Code lstm_function(lstm_g, ""); + InterpreterState lstm_interp(lstm_function); + auto outputs = run(lstm_interp, {input[0], hx, cx, w_ih, w_hh}); + std::tie(hx, cx) = lstm(input[0], hx, cx, w_ih, w_hh); + + ASSERT_TRUE(exactlyEqual(outputs[0], hx)); + ASSERT_TRUE(exactlyEqual(outputs[1], cx)); +} + +TEST(InterpreterTest, IgnorableArgsInSchema) { + auto graph = build_mobile_export_analysis_graph(); + MobileCode function(graph, ""); + auto op_to_specified_args = function.op_to_num_specified_args(); + ASSERT_TRUE(op_to_specified_args.size() == 2); + ASSERT_TRUE(op_to_specified_args["aten::slice.Tensor"] == 4); + ASSERT_TRUE(op_to_specified_args["aten::slice.str"] == 4); + auto graph_vararg = build_mobile_export_analysis_graph_with_vararg(); + MobileCode function_vararg(graph_vararg, ""); + auto op_to_specified_args_vararg = function_vararg.op_to_num_specified_args(); + // should never register it + ASSERT_TRUE( + op_to_specified_args_vararg.find("prim::tolist") == + op_to_specified_args_vararg.end()); + + auto graph_nested = build_mobile_export_analysis_graph_nested(); + MobileCode function_nested(graph_nested, ""); + auto op_to_specified_args_nested = function_nested.op_to_num_specified_args(); + ASSERT_TRUE(op_to_specified_args_nested["aten::slice.Tensor"] == 4); + ASSERT_TRUE(op_to_specified_args_nested["aten::slice.str"] == 4); + + auto graph_non_const = build_mobile_export_analysis_graph_non_const(); + MobileCode function_non_const(graph_non_const, ""); + auto op_to_specified_args_non_const = + function_non_const.op_to_num_specified_args(); + ASSERT_TRUE(op_to_specified_args_non_const["aten::conv2d"] == 6); +} + +TEST(InterpreterTest, IgnorableArgsInSchemaWithOut) { + auto graph = build_mobile_export_with_out(); + MobileCode function(graph, ""); + auto op_to_specified_args = function.op_to_num_specified_args(); + ASSERT_TRUE(op_to_specified_args.size() == 1); + // this should be 3 when the add_out flag is set to True + ASSERT_TRUE(op_to_specified_args["aten::add.out"] == 3); +} + +TEST(InterpreterTest, runAsyncBasicTest) { + /* + TODO: there are some problem with C++ parsing script program involving + fork. Use the test module below for now. + issue about this: github.com/pytorch/pytorch/issues/46368 + The test module file is generated by following: + class DemoModule(torch.nn.Module): + def forward(self): + r1 = torch.jit.fork(torch.mm, torch.rand(100,100),torch.rand(100,100)) + r2 = torch.jit.fork(torch.mm, torch.rand(100,100),torch.rand(100,100)) + return r1.wait() + r2.wait() + demo = DemoModule() + torch.jit.save(torch.jit.script(demo), 'test_interpreter_async.pt') + */ + auto testModelFile = + resolveTestDataFile(__FILE__, "test_interpreter_async.pt"); + auto model = load(testModelFile); + auto graph = model.get_method("forward").graph(); + Code function(graph, ""); + auto asyncCounter = 0; + std::mutex mtx; + // a dummy executor which actually use at::launch, but add up a counter + auto launcher = [&](std::function f) { + mtx.lock(); + ++asyncCounter; + mtx.unlock(); + at::launch(f); + }; + std::vector stack; + // NOLINTNEXTLINE(modernize-use-emplace) + stack.push_back(model._ivalue()); + InterpreterState interp(function, launcher); + interp.runAsync(stack)->wait(); + ASSERT_TRUE(asyncCounter > 0); +} + +TEST( + EnableRethrowCaughtExceptionTest, + EnableRethrowCaughtExceptionTestRethrowsCaughtException) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(%0 : Tensor, + %1 : Tensor): + %2 : int = prim::Constant[value=2]() + %3 : Tensor = aten::add(%0, %1, %2) + return (%3) + )IR", + &*graph, + vmap); + Code function(graph, ""); + InterpreterState interp = InterpreterState(function); + auto a = at::zeros({2, 2}, at::kFloat); + auto b = at::ones({2, 3}, at::kFloat); + a.set_requires_grad(true); + a = a.to(at::kCPU); + std::vector stack({a, b}); + + bool original_flag_value = FLAGS_torch_jit_enable_rethrow_caught_exception; + bool exception_handled = false; + try { + FLAGS_torch_jit_enable_rethrow_caught_exception = false; + interp.run(stack); + } catch (std::runtime_error& e) { + exception_handled = true; + std::string exception_msg = e.what(); + EXPECT_THAT( + exception_msg, + ::testing::HasSubstr("%3 : Tensor = aten::add(%0, %1, %2)")); + EXPECT_THAT( + exception_msg, + ::testing::HasSubstr( + "The size of tensor a (2) must match the size of tensor b (3) at non-singleton dimension 1")); + } + EXPECT_TRUE(exception_handled); + + exception_handled = false; + try { + FLAGS_torch_jit_enable_rethrow_caught_exception = true; + interp.run(stack); + } catch (c10::Error& e) { + exception_handled = true; + std::string exception_msg = e.what_without_backtrace(); + EXPECT_STREQ( + exception_msg.c_str(), + "The size of tensor a (2) must match the size of tensor b (3) at non-singleton dimension 1"); + } + EXPECT_TRUE(exception_handled); + + FLAGS_torch_jit_enable_rethrow_caught_exception = true; + c10::intrusive_ptr future = interp.runAsync(stack); + future->wait(); + ASSERT_TRUE(future->completed()); + ASSERT_TRUE(future->hasError()); + try { + std::rethrow_exception(future->exception_ptr()); + } catch (c10::Error& e) { + std::string exception_msg = e.what_without_backtrace(); + EXPECT_STREQ( + exception_msg.c_str(), + "The size of tensor a (2) must match the size of tensor b (3) at non-singleton dimension 1"); + } + + FLAGS_torch_jit_enable_rethrow_caught_exception = original_flag_value; +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_interpreter_async.pt b/western_ai/usa/pytorch/test/cpp/jit/test_interpreter_async.pt new file mode 100644 index 0000000000000000000000000000000000000000..65918f5774cd9995407c4cde1891a8c42c080f29 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_interpreter_async.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:549d04b98a683b79f1c5831a5ad922b563311a4216b5a2c28226182790368b21 +size 2840 diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_ir.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_ir.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6b48b2d23fa037d32026f6e30bd5f30d1b415614 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_ir.cpp @@ -0,0 +1,212 @@ +#include + +#include +#include + +namespace torch { +namespace jit { + +TEST(IRTest, Attributes) { + Graph g; + auto one = attr::alpha; + auto two = attr::device; + auto three = attr::end; + auto four = attr::perm; + Node* n = g.create(Symbol::fromQualString("foo::bar")); + Node& attr = *n; + attr.f_(one, 3.4)->i_(two, 5)->s_(three, "what"); + ASSERT_EQ(attr.f(one), 3.4); + ASSERT_EQ(attr.s(three), "what"); + ASSERT_EQ(attr.i(two), 5); + attr.s_(one, "no"); + ASSERT_EQ(attr.s(one), "no"); + ASSERT_TRUE(attr.hasAttribute(three)); + ASSERT_TRUE(!attr.hasAttribute(four)); + attr.ss_(two, {"hi", "now"}); + ASSERT_EQ(attr.ss(two).at(1), "now"); + + Node* n2 = g.create(Symbol::fromQualString("foo::baz")); + Node& attr2 = *n2; + attr2.copyAttributes(attr); + ASSERT_EQ(attr2.s(one), "no"); + attr2.f_(one, 5); + ASSERT_EQ(attr.s(one), "no"); + ASSERT_EQ(attr2.f(one), 5); +} + +TEST(IRTest, Blocks) { + auto g = std::make_shared(); + const auto graph_string = R"IR( + graph(%a : Tensor, + %b : Tensor, + %c : Tensor): + %2 : int = prim::Constant[value=1]() + %3 : Tensor = aten::add(%a, %b, %2) + %5 : Tensor = prim::If(%c) + block0(): + %6 : int = prim::Constant[value=1]() + %7 : Tensor = aten::add(%3, %3, %6) + -> (%7) + block1(): + %8 : int = prim::Constant[value=1]() + %9 : Tensor = aten::add(%b, %3, %8) + %10 : int = prim::Constant[value=1]() + %11 : Tensor = aten::add(%9, %3, %10) + -> (%11) + %12 : int = prim::Constant[value=1]() + %13 : Tensor = aten::add(%5, %3, %12) + return (%13))IR"; + torch::jit::parseIR(graph_string, g.get()); + + g->lint(); + testing::FileCheck() + .check("add") + ->check("prim::If") + ->check("block0") + ->check("aten::add") + ->check("block1") + ->check_count("aten::add", 3) + ->run(*g); + + // Removes block0 of the conditional + for (auto* node : g->block()->nodes()) { + if (node->kind() == prim::If) { + node->eraseBlock(0); + break; + } + } + + testing::FileCheck() + .check("add") + ->check("prim::If") + ->check("block0") + ->check_not("block") + ->run(*g); + g->lint(); + // test recursive copy of blocks works + auto g2 = g->copy(); + testing::FileCheck() + .check("add") + ->check("prim::If") + ->check("block0") + ->check_not("block") + ->run(*g2); +} + +TEST(IRTest, CommonAncestor) { + std::string input_str = R"( +graph(%x : Tensor, + %a.1 : bool, + %b.1 : bool, + %c.1 : bool): + %4 : int = prim::If(%a.1) + block0(): + %5 : int = prim::If(%b.1) + block0(): + %6 : int = prim::Constant[value=2]() + -> (%6) + block1(): + %7 : int = prim::Constant[value=3]() + -> (%7) + -> (%5) + block1(): + %8 : int = prim::If(%c.1) + block0(): + %9 : int = prim::Constant[value=4]() + -> (%9) + block1(): + %10 : int = prim::Constant[value=5]() + -> (%10) + -> (%8) + return (%4) +)"; + + torch::jit::Graph g; + std::unordered_map name_to_value; + torch::jit::parseIR(input_str, &g, name_to_value); + + std::vector value_names{"6", "7", "9", "10"}; + std::unordered_set value_names_set( + value_names.begin(), value_names.end()); + + /* clang-format off */ + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) + int ref_blocks_from_graph[4][4] = { + /* (6, 6), (6, 7), (6, 9), (6, 10) */ + { 2, 1, 0, 0 }, + /* (7, 6), (7, 7), (7, 9), (7, 10) */ + { 1, 2, 0, 0 }, + /* (9, 6), (9, 7), (9, 9), (9, 10) */ + { 0, 0, 2, 1, }, + /* (10, 6),(10, 7),(10, 9),(10, 10) */ + { 0, 0, 1, 2 } + }; + /* clang-format on */ + + for (size_t i = 0; i < value_names.size(); ++i) { + Value* i_val = name_to_value[value_names[i]]; + for (size_t j = 0; j < value_names.size(); ++j) { + Value* j_val = name_to_value[value_names[j]]; + Block* common_ancestor = + i_val->node()->findCommonAncestorBlockWith(j_val->node()); + int blocks_from_graph_block = + common_ancestor->param_node()->blocksFromGraphBlock(); + ASSERT_EQ(blocks_from_graph_block, ref_blocks_from_graph[i][j]); + } + } +} + +TEST(IRTest, OperatorMap) { + OperatorMap op_map; + const char* literal1 = + "aten::dropout(Tensor input, float p, bool train) -> Tensor"; + const char* literal2 = + "aten::bernoulli(Tensor self, *, Generator? generator) -> Tensor"; + const char* literal3 = + "aten::bernoulli(Tensor self, float p, *, Generator? generator) -> Tensor"; + const char* literal4 = + "aten::normal(Tensor mean, Tensor std, *, Generator? generator) -> Tensor"; + const char* literal5 = + "aten::normal(float mean, Tensor std, *, Generator? generator) -> Tensor"; + const char* literal6 = + "aten::normal(Tensor mean, float std, *, Generator? generator) -> Tensor"; + std::shared_ptr op1 = getOperatorForLiteral(literal1); + std::shared_ptr op2 = getOperatorForLiteral(literal2); + std::shared_ptr op3 = getOperatorForLiteral(literal3); + std::shared_ptr op4 = getOperatorForLiteral(literal4); + std::shared_ptr op5 = getOperatorForLiteral(literal5); + std::shared_ptr op6 = getOperatorForLiteral(literal6); + op_map.insert(op1, 1); + op_map.insert({{op2, 2}, {op3, 3}}); + op_map.insert({{op4, 4}, {op5, 5}}); + op_map.insert(op6, 6); + ASSERT_TRUE(op_map.contains(*op1)); + ASSERT_TRUE(op_map.contains(*op2)); + ASSERT_TRUE(op_map.contains(*op3)); + ASSERT_TRUE(op_map.contains(*op4)); + ASSERT_TRUE(op_map.contains(*op5)); + ASSERT_TRUE(op_map.contains(*op6)); + op_map.erase(op6); + op_map.erase(op3); + op_map.erase(op1); + ASSERT_FALSE(op_map.contains(*op1)); + ASSERT_FALSE(op_map.contains(*op3)); + ASSERT_FALSE(op_map.contains(*op6)); + op_map.insert(op1, 1); + ASSERT_TRUE(op_map.contains(*op1)); + std::optional o1 = op_map.find(*op1); + ASSERT_TRUE(o1.has_value()); + std::optional o2 = op_map.find(*op2); + ASSERT_TRUE(o2.has_value()); + std::optional o3 = op_map.find(*op3); + ASSERT_FALSE(o3.has_value()); + std::optional o4 = op_map.find(*op4); + ASSERT_TRUE(o4.has_value()); + std::optional o5 = op_map.find(*op5); + ASSERT_TRUE(o5.has_value()); + std::optional o6 = op_map.find(*op6); + ASSERT_FALSE(o6.has_value()); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_irparser.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_irparser.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e1c92618c2a7473e87a623f0e54096ed5351a93b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_irparser.cpp @@ -0,0 +1,381 @@ +#include + +#include +#include +#include + +#include +#include + +namespace torch { +namespace jit { + +/** \brief Parse IR from \p S, print the parsed graph and verify that the output + * string matches the original string. + * + * The function is sensitive to value naming and whitespace, so it should be + * used with care. Nevertheless, it helps to keep tests more compact. + */ +static void checkRoundtrip(const std::string& s) { + auto graph = std::make_shared(); + parseIR(s, &*graph); + std::ostringstream ss; + ss << *graph; + std::string parsed = ss.str(); + + // Skip whitespace in the beginning of the input string. + int i = 0; + for (char c : s) { + if (!isspace(c)) { + break; + } + i++; + } + std::string original = s.substr(i, s.size()); + if (original != parsed) { + std::cerr << "Input:" << std::endl << original << std::endl; + std::cerr << "Parsed:" << std::endl << parsed << std::endl; + } + AT_ASSERT(original == parsed); +} + +TEST(IRParserTest, Basic) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(%0 : Tensor, %1 : Tensor): + %2 : Tensor = foo::add(%0, %1) + %res, %3 = foo::mul(%0, %2) + %x, %y = foo::combine(%res, %2, %3) + return (%x, %y, %res))IR", + &*graph, + vmap); + + AT_ASSERT(graph->inputs().size() == 2); + AT_ASSERT(graph->outputs().size() == 3); + Value* x = graph->outputs()[0]; + Value* y = graph->outputs()[1]; + Value* res = graph->outputs()[2]; + Value* t0 = graph->inputs()[0]; + Value* t1 = graph->inputs()[1]; + AT_ASSERT(vmap["x"] == x); + AT_ASSERT(vmap["y"] == y); + AT_ASSERT(vmap["res"] == res); + AT_ASSERT(vmap["0"] == t0); + AT_ASSERT(vmap["1"] == t1); + AT_ASSERT(x->node() == y->node()); + Node* comb = x->node(); + Value* t2 = comb->inputs()[1]; + Value* t3 = comb->inputs()[2]; + AT_ASSERT(vmap["2"] == t2); + AT_ASSERT(vmap["3"] == t3); + AT_ASSERT(comb->kind().toQualString() == std::string("foo::combine")); + AT_ASSERT(comb->outputs() == std::vector({x, y})); + AT_ASSERT(comb->inputs() == std::vector({res, t2, t3})); + Node* mul = res->node(); + AT_ASSERT(mul->kind().toQualString() == std::string("foo::mul")); + AT_ASSERT(mul->inputs() == std::vector({t0, t2})); + AT_ASSERT(mul->outputs() == std::vector({res, t3})); + Node* add = t2->node(); + AT_ASSERT(add->kind().toQualString() == std::string("foo::add")); + AT_ASSERT(add->inputs() == std::vector({t0, t1})); + AT_ASSERT(add->outputs() == std::vector({t2})); +} + +TEST(IRParserTest, NestedBlock) { + checkRoundtrip(R"IR( +graph(): + %0 : Tensor = a::a() + block0(): + %1 : Tensor = b::b() + block0(): + %2 : Tensor = c::c() + -> () + -> () + %3 : Tensor = d::d() + return (%3) +)IR"); +} + +TEST(IRParserTest, If) { + checkRoundtrip(R"IR( +graph(%0 : Tensor, + %1 : Tensor, + %2 : Tensor): + %3 : int = prim::Constant[value=1]() + %4 : Tensor = aten::add(%0, %1, %3) + %5 : Tensor = prim::If(%2) + block0(): + %6 : int = prim::Constant[value=1]() + %7 : Tensor = aten::add(%1, %3, %6) + %8 : int = prim::Constant[value=1]() + %9 : Tensor = aten::add(%7, %3, %8) + -> (%9) + %10 : int = prim::Constant[value=1]() + %11 : Tensor = aten::add(%5, %3, %10) + return (%11) +)IR"); +} + +TEST(IRParserTest, If2) { + checkRoundtrip(R"IR( +graph(%0 : Tensor, + %1 : Tensor, + %2 : Tensor): + %3 : int = prim::Constant[value=-1]() + %4 : Tensor = aten::add(%0, %1, %3) + %5 : Tensor = prim::If(%2) + block0(): + %6 : int = prim::Constant[value=1]() + %7 : Tensor = aten::add(%1, %3, %6) + %8 : int = prim::Constant[value=1]() + %9 : Tensor = aten::add(%7, %3, %8) + -> (%9) + %10 : int = prim::Constant[value=-987]() + %11 : Tensor = aten::add(%5, %3, %10) + return (%11) +)IR"); +} + +TEST(IRParserTest, InferredTypeIsTensor) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%a): + return (%a))IR", + &*graph); + AT_ASSERT(graph->inputs()[0]->type()->isSubtypeOf(*TensorType::get())); +} + +TEST(IRParserTest, ValueReuse) { + // Check that parser correctly handles values reusing the same name. + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%x): + %x = a::a(%x) + %x = b::b(%x) + return (%x))IR", + &*graph); + Value* x0 = graph->inputs()[0]; + Value* x2 = graph->outputs()[0]; + Node* b = x2->node(); + Value* x1 = b->inputs()[0]; + Node* a = x1->node(); + AT_ASSERT(a->inputs() == std::vector({x0})); + AT_ASSERT(a->outputs() == std::vector({x1})); + AT_ASSERT(b->inputs() == std::vector({x1})); + AT_ASSERT(b->outputs() == std::vector({x2})); +} + +TEST(IRParserTest, Attributes) { + // Check that parser handles attributes and types. + checkRoundtrip( + R"IR( +graph(%0 : Tensor, + %1 : Tensor, + %2 : Tensor): + %3 : int, %4 : Tensor = qqq::qqq[i_asdf=2, f_asdf=3., s_asdf="hello", ss_asdf=["hello world", "bye bye"]](%0) + %5 : int, %6 : Tensor = ppp::ppp[i_asdf=2, f_asdf=3., s_asdf="\"\"\"\"\nhe\"llo", q=[3, 2, 4]](%0) + %7 : float = vvv::vvv[s_asdf="hello"](%0) + %8 : string = z::z() + return (%7) +)IR"); +} + +TEST(IRParserTest, OptionalTypes) { + checkRoundtrip( + R"IR( +graph(%0 : Tensor, + %1 : Tensor, + %2 : Tensor): + %3 : int? = prim::Constant() + return (%3) +)IR"); +} + +TEST(IRParserTest, StarTensor) { + checkRoundtrip( + R"IR( +graph(%0 : Tensor, + %1 : Tensor, + %2 : Tensor): + %3 : Float(*, *, *) = prim::Constant() + return (%3) +)IR"); +} + +TEST(IRParserTest, UnshapedTensor) { + checkRoundtrip( + R"IR( +graph(%0 : Tensor, + %1 : Tensor, + %2 : Tensor): + %3 : Long() = prim::Constant() + return (%3) +)IR"); +} + +TEST(IRParserTest, ShapedTensor) { + checkRoundtrip( + R"IR( +graph(%0 : Tensor, + %1 : Tensor, + %2 : Tensor): + %3 : Double(4, 4, 5) = prim::Constant() + return (%3) +)IR"); +} + +TEST(IRParserTest, NestedContrainer) { + checkRoundtrip( + R"IR( +graph(): + %0 : float[] = prim::Constant[value=[1., 2., 3.]]() + %1 : str[] = prim::Constant[value=["ab", "cd", "ef"]]() + %2 : (float[], str[]) = prim::TupleConstruct(%0, %1) + return (%2) +)IR"); +} + +TEST(IRParserTest, MalformedShapeAnnotation) { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + EXPECT_ANY_THROW(checkRoundtrip( + R"IR( +graph(%0 : Tensor, + %1 : Tensor, + %2 : Tensor): + %3 : Double(4!, 4, 5) = prim::Constant() + return (%3) +)IR")); +} + +TEST(IRParserTest, FileCheck) { + auto graph = std::make_shared(); + const std::string& text = + R"IR( + graph(%a): + # CHECK: return + return (%a))IR"; + + parseIR(text, &*graph); + AT_ASSERT(graph->inputs()[0]->type()->isSubtypeOf(*TensorType::get())); + torch::jit::testing::FileCheck().run(text, *graph); +} + +TEST(IRParserTest, Strides) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(%a : Float(4, 5), + %b : Float(4, 5, strides=[5, 1]), + %c : Double(*, *)): + return (%a) +)IR", + &*graph, + vmap); + Value* a = graph->inputs()[0]; + Value* b = graph->inputs()[1]; + Value* c = graph->inputs()[2]; + + auto a_type = a->type()->cast(); + auto a_sizes = *a_type->sizes().concrete_sizes(); + auto a_strides = a_type->strides().concrete_sizes(); + AT_ASSERT(a_sizes[0] == 4 && a_sizes[1] == 5); + AT_ASSERT(a_strides == std::nullopt); + + auto b_type = b->type()->cast(); + auto b_sizes = *b_type->sizes().concrete_sizes(); + auto b_strides = *(b_type->strides().sizes()); + AT_ASSERT(b_sizes[0] == 4 && b_sizes[1] == 5); + AT_ASSERT(*b_strides[0] == 5 && *b_strides[1] == 1); + + auto c_type = c->type()->cast(); + AT_ASSERT(*c_type->sizes().size() == 2); + AT_ASSERT(c_type->sizes().concrete_sizes() == std::nullopt); + AT_ASSERT(c_type->strides().concrete_sizes() == std::nullopt); +} + +TEST(IRParserTest, MalformedStrides) { + auto graph = std::make_shared(); + std::unordered_map vmap; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + EXPECT_ANY_THROW(parseIR( + R"IR( +graph(%a : Float(4, strides=[5], 5)): + return (%a) +)IR", + &*graph, + vmap)); +} + +TEST(IRParserTest, TensorShapes) { + checkRoundtrip( + R"IR( +graph(%a : Float(4, 5), + %b : Float(4, 5, strides=[5, 1]), + %c : Double(*, *)): + return (%a) +)IR"); +} + +TEST(IRParserTest, DeviceAndRequiresGradTensors) { + checkRoundtrip( + R"IR( +graph(%a : Float(*, *, device=cpu), + %b : Float(*, *, requires_grad=1), + %c : Long(5, 10, requires_grad=1, device=cpu), + %d : Float(5, requires_grad=0, device=cuda:2), + %e : Long(4, 3, 1, strides=[6, 2, 1], requires_grad=0, device=cuda:1), + %f : Float(), + %g : Float(device=cpu), + %h : Float(requires_grad=1), + %i : Float(requires_grad=0, device=cuda:1), + %j : Double(*, *, requires_grad=0)): + return (%a) +)IR"); +} + +TEST(IRParserTest, ListConstant) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(): + %d : int[] = prim::Constant[value=[1,2,3]]() + return (%d) +)IR", + &*graph); + Node* n = graph->outputs()[0]->node(); + AT_ASSERT(n->kind() == prim::Constant); + AT_ASSERT(n->kindOf(attr::value) == AttributeKind::ival); + const auto& genericList = n->ival(attr::value).toList(); + std::vector int_vals; + // NOLINTNEXTLINE(performance-implicit-conversion-in-loop) + for (const IValue& ival : genericList) { + int_vals.push_back(ival.toInt()); + } + AT_ASSERT(int_vals.size() == 3); + AT_ASSERT(int_vals[0] == 1 && int_vals[1] == 2 && int_vals[2] == 3); +} + +TEST(IRParserTest, PartialStarTensor) { + checkRoundtrip( + R"IR( +graph(%x : Float(10, *, 10)): + return (%x) +)IR"); +} + +TEST(IRParserTest, ComplexTensorAttributes) { + checkRoundtrip( + R"IR( +graph(%x : Double(*, 200, *, requires_grad=1, device=cuda:1), + %b : Float(5, *, requires_grad=1), + %c : Long(*, 10, device=cpu)): + return (%x) +)IR"); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_jit_logging_levels.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_jit_logging_levels.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c4cdfacb705870d0cd550b7bcb8c7351ea1bb7d8 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_jit_logging_levels.cpp @@ -0,0 +1,57 @@ +#include +#include + +#include +#include + +namespace torch { +namespace jit { + +TEST(JitLoggingTest, CheckSetLoggingLevel) { + ::torch::jit::set_jit_logging_levels("file_to_test"); + ASSERT_TRUE(::torch::jit::is_enabled( + "file_to_test.cpp", JitLoggingLevels::GRAPH_DUMP)); +} + +TEST(JitLoggingTest, CheckSetMultipleLogLevels) { + ::torch::jit::set_jit_logging_levels("f1:>f2:>>f3"); + ASSERT_TRUE(::torch::jit::is_enabled("f1.cpp", JitLoggingLevels::GRAPH_DUMP)); + ASSERT_TRUE( + ::torch::jit::is_enabled("f2.cpp", JitLoggingLevels::GRAPH_UPDATE)); + ASSERT_TRUE( + ::torch::jit::is_enabled("f3.cpp", JitLoggingLevels::GRAPH_DEBUG)); +} + +TEST(JitLoggingTest, CheckLoggingLevelAfterUnset) { + ::torch::jit::set_jit_logging_levels("f1"); + ASSERT_EQ("f1", ::torch::jit::get_jit_logging_levels()); + ::torch::jit::set_jit_logging_levels("invalid"); + ASSERT_FALSE( + ::torch::jit::is_enabled("f1.cpp", JitLoggingLevels::GRAPH_DUMP)); +} + +TEST(JitLoggingTest, CheckAfterChangingLevel) { + ::torch::jit::set_jit_logging_levels("f1"); + ::torch::jit::set_jit_logging_levels(">f1"); + ASSERT_TRUE( + ::torch::jit::is_enabled("f1.cpp", JitLoggingLevels::GRAPH_UPDATE)); +} + +TEST(JitLoggingTest, CheckOutputStreamSetting) { + ::torch::jit::set_jit_logging_levels("test_jit_logging_levels"); + std::ostringstream test_stream; + ::torch::jit::set_jit_logging_output_stream(test_stream); + /* Using JIT_LOG checks if this file has logging enabled with + is_enabled(__FILE__, level) making the test fail. since we are only testing + the OutputStreamSetting we can forcefully output to it directly. + */ + ::torch::jit::get_jit_logging_output_stream() << ::torch::jit::jit_log_prefix( + ::torch::jit::JitLoggingLevels::GRAPH_DUMP, + __FILE__, + __LINE__, + ::c10::str("Message")); + ASSERT_TRUE(test_stream.str().size() > 0); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_jit_type.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_jit_type.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1da8df8ad3d097c26a37d3c961a93004d52ab7d9 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_jit_type.cpp @@ -0,0 +1,64 @@ +#include + +#include +#include +#include "torch/csrc/jit/ir/ir.h" +#include "torch/csrc/jit/ir/irparser.h" + +namespace torch { +namespace jit { + +TEST(JitTypeTest, IsComplete) { + auto tt = c10::TensorType::create( + at::kFloat, + at::kCPU, + c10::SymbolicShape(std::vector>({1, 49})), + std::vector( + {c10::Stride{2, true, 1}, + c10::Stride{1, true, 1}, + c10::Stride{0, true, std::nullopt}}), + false); + TORCH_INTERNAL_ASSERT(!tt->isComplete()); + TORCH_INTERNAL_ASSERT(!tt->strides().isComplete()); +} + +TEST(JitTypeTest, UnifyTypes) { + auto bool_tensor = TensorType::get()->withScalarType(at::kBool); + auto opt_bool_tensor = OptionalType::create(bool_tensor); + auto unified_opt_bool = unifyTypes(bool_tensor, opt_bool_tensor); + TORCH_INTERNAL_ASSERT(opt_bool_tensor->isSubtypeOf(**unified_opt_bool)); + + auto tensor = TensorType::get(); + TORCH_INTERNAL_ASSERT(!tensor->isSubtypeOf(*opt_bool_tensor)); + auto unified = unifyTypes(opt_bool_tensor, tensor); + TORCH_INTERNAL_ASSERT(unified); + auto elem = (*unified)->expectRef().getElementType(); + TORCH_INTERNAL_ASSERT(elem->isSubtypeOf(*TensorType::get())); + + auto opt_tuple_none_int = OptionalType::create( + TupleType::create({NoneType::get(), IntType::get()})); + auto tuple_int_none = TupleType::create({IntType::get(), NoneType::get()}); + auto out = unifyTypes(opt_tuple_none_int, tuple_int_none); + TORCH_INTERNAL_ASSERT(out); + + std::stringstream ss; + ss << (*out)->annotation_str(); + testing::FileCheck() + .check("Optional[Tuple[Optional[int], Optional[int]]]") + ->run(std::move(ss).str()); + + auto fut_1 = FutureType::create(IntType::get()); + auto fut_2 = FutureType::create(NoneType::get()); + auto fut_out = unifyTypes(fut_1, fut_2); + TORCH_INTERNAL_ASSERT(fut_out); + TORCH_INTERNAL_ASSERT((*fut_out)->isSubtypeOf( + *FutureType::create(OptionalType::create(IntType::get())))); + + auto dict_1 = DictType::create(IntType::get(), NoneType::get()); + auto dict_2 = DictType::create(IntType::get(), IntType::get()); + auto dict_out = unifyTypes(dict_1, dict_2); + TORCH_INTERNAL_ASSERT(!dict_out); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_lexer.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_lexer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..07c443aba6903c30f8e5e395d32f79966ba360e5 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_lexer.cpp @@ -0,0 +1,88 @@ +#include + +#include + +namespace torch::jit { + +TEST(LexerTest, AllTokens) { + std::vector> tokens; + for (const char* ch = valid_single_char_tokens; *ch; ch++) { + tokens.emplace_back(*ch, std::string(1, *ch)); + } +#define ADD_TOKEN(tok, _, tokstring) \ + if (*tokstring) { \ + tokens.emplace_back(tok, tokstring); \ + } + TC_FORALL_TOKEN_KINDS(ADD_TOKEN); +#undef ADD_TOKEN + + for (const auto& [kind, token] : tokens) { + Lexer l(std::make_shared(token)); + const auto& tok = l.cur(); + EXPECT_EQ(kind, tok.kind) << tok.range.text().str(); + EXPECT_EQ(token, tok.range.text().str()) << tok.range.text().str(); + l.next(); + EXPECT_EQ(l.cur().kind, TK_EOF); + } +} + +TEST(LexerTest, SlightlyOffIsNot) { + std::vector suffixes = {"", " ", "**"}; + for (const auto& suffix : suffixes) { + std::vector extras = {"n", "no", "no3"}; + for (const auto& extra : extras) { + std::string s = "is " + extra + suffix; + Lexer l(std::make_shared(s)); + const auto& is_tok = l.next(); + EXPECT_EQ(is_tok.kind, TK_IS) << is_tok.range.text().str(); + const auto& no_tok = l.cur(); + EXPECT_EQ(no_tok.kind, TK_IDENT) << no_tok.range.text().str(); + EXPECT_EQ(no_tok.range.text().str(), extra) << no_tok.range.text().str(); + } + } +} + +TEST(LexerTest, SlightlyOffNotIn) { + std::vector suffixes = {"", " ", "**"}; + for (const auto& suffix : suffixes) { + std::vector extras = {"i", "i3"}; + for (const auto& extra : extras) { + std::string s = "not " + extra + suffix; + Lexer l(std::make_shared(s)); + const auto& not_tok = l.next(); + EXPECT_EQ(not_tok.kind, TK_NOT) << not_tok.range.text().str(); + const auto& in_tok = l.cur(); + EXPECT_EQ(in_tok.kind, TK_IDENT) << in_tok.range.text().str(); + EXPECT_EQ(in_tok.range.text().str(), extra) << in_tok.range.text().str(); + } + } +} + +TEST(LexerTest, IsNoteBug) { + // The code string `is note` is lexed as TK_ISNOT followed by a + // TK_IDENT that is an e. This is not how it works in Python, but + // presumably we need to maintain this behavior. + Lexer l(std::make_shared("is note")); + const auto is_not_tok = l.next(); + EXPECT_EQ(is_not_tok.kind, TK_ISNOT); + const auto e_tok = l.next(); + EXPECT_EQ(e_tok.kind, TK_IDENT); + EXPECT_EQ(e_tok.range.text(), "e"); + const auto eof_tok = l.next(); + EXPECT_EQ(eof_tok.kind, TK_EOF); +} + +TEST(LexerTest, NotInpBug) { + // Another manifestation of the above IsNoteBug; `not inp` is lexed + // as TK_NOT_IN followed by a TK_IDENT that is a p. Again, not how + // it works in Python. + Lexer l(std::make_shared("not inp")); + const auto not_in_tok = l.next(); + EXPECT_EQ(not_in_tok.kind, TK_NOTIN); + const auto p_tok = l.next(); + EXPECT_EQ(p_tok.kind, TK_IDENT); + EXPECT_EQ(p_tok.range.text(), "p"); + const auto eof_tok = l.next(); + EXPECT_EQ(eof_tok.kind, TK_EOF); +} +} // namespace torch::jit diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_lite_interpreter.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_lite_interpreter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a6fa210929e8f609eda0172fc70597858171e578 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_lite_interpreter.cpp @@ -0,0 +1,2221 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +// Tests go in torch::jit +namespace torch { +namespace jit { + +TEST(LiteInterpreterTest, UpsampleNearest2d) { + Module m("m"); + m.define(R"( + def forward(self, input: Tensor, scale:float): + return torch.upsample_nearest2d(input, [1, 1], float(scale), float(scale)) + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({1, 3, 128, 128})); + inputs.emplace_back(at::Scalar(2.0)); + auto ref = m.forward(inputs); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + IValue res; + res = bc.forward(inputs); + + auto resd = res.toTensor(); + auto refd = ref.toTensor(); + ASSERT_TRUE(resd.equal(refd)); +} + +TEST(LiteInterpreterTest, CheckAttrAccess) { + Module m("m"); + m.register_attribute("mobile_optimized", BoolType::get(), true); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + bool mobile_optimized = bc.attr("mobile_optimized", false).toBool(); + + AT_ASSERT(mobile_optimized); + m.setattr("mobile_optimized", false); + ss = std::stringstream(); + m._save_for_mobile(ss); + bc = _load_for_mobile(ss); + mobile_optimized = bc.attr("mobile_optimized", false).toBool(); + + AT_ASSERT(!mobile_optimized); +} + +TEST(LiteInterpreterTest, MethodInvocation) { // NOLINT (use =delete in gtest) + const std::vector test_programs{ + // test invoking a method with default parameter + R"( + def test_func(self, x, b : int = 4): + return self.foo + x + b + )", + // inner method call with default parameter (gets inlined) + R"( + def add_with_default_arg(self, x, b : int = 4): + return self.foo + x + b + def test_func(self, x): + return self.add_with_default_arg(x) # invoke method w/ default arg + )", + // simple method call + R"( + def test_func(self, x): + b = 4 + return self.foo + x + b + )", + }; + for (const auto& test_program : test_programs) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(test_program); + + const int fortyTwo = 42; // (keep linter happy) + auto minput = fortyTwo * torch::ones({}); + auto ref = m.run_method("test_func", minput); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + const auto& test_func = bc.get_method("test_func"); + IValue res; + for (int i = 0; i < 3; ++i) { + res = test_func({minput}); + } + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); + } +} + +TEST(LiteInterpreterTest, Conv) { + auto s = std::getenv("PYTORCH_TEST_WITH_TSAN"); + if (s && strcmp(s, "1") == 0) + return; + + std::vector inputs; + + Module m("m"); + m.register_parameter("weight", torch::ones({20, 1, 5, 5}), false); + m.register_parameter("bias", torch::ones({20}), false); + m.define(R"( + def forward(self, input): + return torch._convolution(input, self.weight, self.bias, [1, 1], [0, 0], [1, 1], False, [0, 0], 1, False, False, True, True) + )"); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,modernize-use-emplace) + inputs.push_back(torch::ones({1, 1, 28, 28})); + + auto outputref = m.forward(inputs).toTensor(); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + IValue res; + for (int i = 0; i < 3; ++i) { + res = bc.get_method("forward")(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT( + outputref[0][0][0][0].item() == output[0][0][0][0].item()); +} + +TEST(LiteInterpreterTest, Inline) { + Module m("m"); + m.define(R"JIT( + def foo1(self, x): + return x + 1 + + def foo2(self, x): + return self.foo1(x) + 2 + + def foo3(self, x): + return self.foo2(x) + 3 + )JIT"); + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("foo3")(inputs); + AT_ASSERT(output.toTensor().item() == 7.0); +} + +TEST(LiteInterpreterTest, Tuple) { + Module m("m"); + m.define(R"JIT( + def foo(self, x): + return (1, 2, x + 3) + + def forward(self, x): + tuple = self.foo(x) + return tuple + )JIT"); + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("forward")(inputs); + AT_ASSERT(output.toTupleRef().elements()[1].toInt() == 2); +} + +TEST(LiteInterpreterTest, AtenFormat) { + Module m("m"); + m.define(R"""( + def forward(self, fmt:str="first {} {}", num:str="abc"): + x = 2 + x = x * x + return fmt.format(num, x) + )"""); + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + std::vector inputs; + auto output_bc = bc.get_method("forward")(inputs); + auto output_m = m.get_method("forward")(inputs); + // std::cout << output_m.toStringRef() << "\n" + // << output_bc.toStringRef() << std::endl; + AT_ASSERT(output_m.toStringRef() == output_bc.toStringRef()); +} + +TEST(LiteInterpreterTest, PrimDevice) { + Module m("m"); + m.define(R"""( + def forward(self, x:torch.Tensor): + return x.device + )"""); + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + std::vector inputs; + auto minput = 3.5 * torch::ones({}); + inputs.emplace_back(minput); + auto output_bc = bc.get_method("forward")(inputs); + auto output_m = m.get_method("forward")(inputs); + AT_ASSERT(output_bc.toDevice().str() == output_m.toDevice().str()); +} + +TEST(LiteInterpreterTest, Dict) { + Module m("m"); + m.define(R"JIT( + def foo(self, x): + return {"result": x + 1} + + def forward(self, x): + d = self.foo(x) + return d + )JIT"); + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("forward")(inputs); + AT_ASSERT(output.toGenericDict().at("result").toTensor().item().toInt() == 2); +} + +TEST(LiteInterpreterTest, List) { + Module m("m"); + m.define(R"JIT( + def foo(self, x): + return [x + 2] + + def forward(self, x): + d = self.foo(x) + return d + )JIT"); + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("forward")(inputs); + auto server_output = m.forward(inputs); + EXPECT_EQ(output.toList().get(0).toTensor().item().toInt(), 3); + EXPECT_EQ(output, server_output); +} + +TEST(LiteInterpreterTest, PrimOverload) { + /* + // temporarily disabled + script::Module m("m"); + m.define(R"JIT( + def forward(self, x): + result = [1, 2] + result.append(3) + return result + )JIT"); + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("forward")(inputs); + AT_ASSERT(output.toIntList()[2] == 3); + */ +} + +TEST(LiteInterpreterTest, Prim) { + Module m("m"); + m.define(R"JIT( + def forward(self, x): + return int(x) + )JIT"); + + std::vector inputs; + auto minput = 3.5 * torch::ones({}); + inputs.emplace_back(minput); + auto ref = m.run_method("forward", minput); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + IValue res; + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc.get_method("forward")(bcinputs); + } + + auto resi = res.toInt(); + auto refi = ref.toInt(); + AT_ASSERT(resi == refi); +} + +TEST(LiteInterpreterTest, PrimScalar) { + Module m("m"); + m.define(R"JIT( + def forward(self, x): + return int(x.item()) + )JIT"); + + std::vector inputs; + auto minput = 3.5 * torch::ones({}); + inputs.emplace_back(minput); + auto ref = m.run_method("forward", minput); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + IValue res; + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc.get_method("forward")(bcinputs); + } + + auto resi = res.toInt(); + auto refi = ref.toInt(); + AT_ASSERT(resi == refi); +} + +TEST(LiteInterpreterTest, LoadOrigJit) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def forward(self, x): + b = 4 + return self.foo + x + b + )"); + std::stringstream ss; + m.save(ss); + ASSERT_THROWS_WITH_MESSAGE(_load_for_mobile(ss), "file not found"); +} + +TEST(LiteInterpreterTest, WrongMethodName) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add(self, x): + b = 4 + return self.foo + x + b + )"); + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + std::vector inputs; + auto minput = 5 * torch::ones({}); + inputs.emplace_back(minput); + ASSERT_THROWS_WITH_MESSAGE( + bc.get_method("forward")(inputs), "is not defined"); +} + +TEST(LiteInterpreterTest, SetState) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def __getstate__(self): + return self.foo + self.foo + def __setstate__(self, a): + self.foo = a + def forward(self, x): + b = 4 + return self.foo + x + b + )"); + + std::vector inputs; + auto minput = 5 * torch::ones({}); + inputs.emplace_back(minput); + + std::stringstream ms; + m.save(ms); + auto loaded_m = load(ms); + auto ref = loaded_m.run_method("forward", minput); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + IValue res; + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc.get_method("forward")(bcinputs); + } + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); +} + +class TorchBindLiteInterpreterTestStruct + : public torch::jit::CustomClassHolder { + public: + std::string get(at::Tensor t) { + std::stringstream ss; + ss << "Hello! Your tensor has "; + ss << t.numel(); + ss << " elements!"; + return std::move(ss).str(); + } +}; + +namespace { +struct ClassNamespaceValue : public SugaredValue { + explicit ClassNamespaceValue(c10::QualifiedName name) + : basename_(std::move(name)) {} + + std::shared_ptr attr( + const SourceRange& loc, + GraphFunction& m, + const std::string& name) override { + const auto fullName = c10::QualifiedName(basename_, name); + + // Check to see if it is a custom class. + if (auto custom_class = getCustomClass(fullName.qualifiedName())) { + return std::make_shared(custom_class); + } + + // If it's not a custom class, assume it's another namespace + // NOLINTNEXTLINE(performance-move-const-arg) + return std::make_shared(std::move(fullName)); + } + + std::string kind() const override { + return "Class Namespace"; + } + + private: + c10::QualifiedName basename_; +}; + +struct TestModuleResolver : public Resolver { + std::shared_ptr resolveValue( + const std::string& name, + GraphFunction& m, + const SourceRange& loc) override { + if (name == "torch") { + return std::make_shared("aten"); + } else if (name == "__torch__") { + return std::make_shared(c10::QualifiedName(name)); + } + + return nullptr; + } + + TypePtr resolveType(const std::string& name, const SourceRange& loc) + override { + return nullptr; + } +}; +} // namespace + +TEST(LiteInterpreterTest, BuiltinClass) { + script::Module m("m"); + + auto cls = getCustomClass( + "__torch__.torch.classes._TorchScriptTesting._LiteInterpreterTest"); + TORCH_INTERNAL_ASSERT(cls); + c10::intrusive_ptr obj_holder; + m.register_attribute("my_obj", cls, IValue::make_capsule(obj_holder)); + + m.register_parameter("foo", torch::ones({}), false); + m.define( + R"( + def __getstate__(self): + return 1 + def __setstate__(self, a): + self.my_obj = __torch__.torch.classes._TorchScriptTesting._LiteInterpreterTest() + + def forward(self, x) -> str: + return self.my_obj.get(x) + )", + std::make_shared()); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + auto res = + bc.get_method("forward")(std::vector{torch::zeros({3, 4})}); + const auto& str = res.toStringRef(); + std::string expected = "Hello! Your tensor has 12 elements!"; + AT_ASSERT(str == expected); +} + +TEST(LiteInterpreterTest, BuiltinFunction) { + script::Module m("m"); + auto custom_class_obj = + make_custom_class(); + m.register_attribute("my_obj", custom_class_obj.type(), custom_class_obj); + m.define(R"( + def forward(self, x) -> str: + return self.my_obj.get(x) + )"); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + auto res = + bc.get_method("forward")(std::vector{torch::zeros({3, 4})}); + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto str = res.toStringRef(); + std::string expected = "Hello! Your tensor has 12 elements!"; + AT_ASSERT(str == expected); +} + +#if !defined FB_XPLAT_BUILD +TEST(LiteInterpreterTest, GetRuntimeByteCodeVersion) { + auto runtime_bytecode_version = _get_runtime_bytecode_version(); + AT_ASSERT( + runtime_bytecode_version == + caffe2::serialize::kMaxSupportedBytecodeVersion); +} + +TEST(LiteInterpreterTest, GetRuntimeOperatorsVersion) { + auto runtime_operators_version = _get_runtime_operators_min_max_versions(); + AT_ASSERT( + runtime_operators_version.first == + caffe2::serialize::kMinSupportedFileFormatVersion && + runtime_operators_version.second == + caffe2::serialize::kMaxSupportedFileFormatVersion); +} + +/** + * The test below is disarmed for FB internal xplat builds since + * BUCK requires us to pass in the script_module_v4.ptl file in + * as a resource dependency of the build rule for this file, and + * we would need to access it via the C++ Resources API instead + * of directly reading from disk (which is what the open source + * build/run does). + */ +TEST(LiteInterpreterTest, GetByteCodeVersion) { + auto test_model_file_v4 = + resolveTestDataFile(__FILE__, "script_module_v4.ptl"); + + auto version_v4 = _get_model_bytecode_version(test_model_file_v4); + AT_ASSERT(version_v4 == 4); +} + +#endif // !defined(FB_XPLAT_BUILD) + +TEST(LiteInterpreterTest, GetContainTypes) { + Module m("m"); + m.define(R"( + def forward(self): + return 3 + )"); + + std::stringstream ss; + m._save_for_mobile(ss, {}, true); + + _get_mobile_model_contained_types(ss); +} + +namespace { + +void compareModelOutput( + c10::ArrayRef actual_result_list, + const std::vector& expect_result_list) { + AT_ASSERT(actual_result_list.size() == expect_result_list.size()); + AT_ASSERT( + actual_result_list[0].toTensor().equal(expect_result_list[0].toTensor())); + AT_ASSERT( + actual_result_list[1].toTensor().dim() == + expect_result_list[1].toTensor().dim()); + AT_ASSERT( + actual_result_list[2].toTensor().equal(expect_result_list[2].toTensor())); + AT_ASSERT( + actual_result_list[3].toTensor().equal(expect_result_list[3].toTensor())); + ASSERT_EQ( + actual_result_list[4].toStringRef(), expect_result_list[4].toStringRef()); + ASSERT_EQ(actual_result_list[5].toBool(), expect_result_list[5].toBool()); + ASSERT_EQ(actual_result_list[6].toBool(), expect_result_list[6].toBool()); + ASSERT_EQ(actual_result_list[7].toBool(), expect_result_list[7].toBool()); + AT_ASSERT( + actual_result_list[8].toTensor().equal(expect_result_list[8].toTensor())); + ASSERT_EQ( + actual_result_list[9].toStringRef(), expect_result_list[9].toStringRef()); + ASSERT_EQ(actual_result_list[10].toInt(), expect_result_list[10].toInt()); + ASSERT_EQ(actual_result_list[11].toBool(), expect_result_list[11].toBool()); +} + +void runAndCheckTorchScriptModel( + std::stringstream& input_model_stream, + const std::vector& input_data, + const std::vector& expect_result_list, + const uint64_t expect_version) { + auto actual_version = _get_model_bytecode_version(input_model_stream); + AT_ASSERT(actual_version == expect_version); + + // Load and run the backport model, then compare the result with expect + // result + Module m_mobile = load(input_model_stream); + + auto actual_result = m_mobile.forward(input_data); + const auto& actual_result_list = actual_result.toTupleRef().elements(); + compareModelOutput(actual_result_list, expect_result_list); +} + +void runAndCheckBytecodeModel( + std::stringstream& input_model_stream, + const std::vector& input_data, + const std::vector& expect_result_list, + const uint64_t expect_version) { + auto actual_version = _get_model_bytecode_version(input_model_stream); + AT_ASSERT(actual_version == expect_version); + + // Load and run the backport model, then compare the result with expect + // result + Module m_mobile = load(input_model_stream); + + auto actual_result = m_mobile.forward(input_data); + const auto& actual_result_list = actual_result.toTupleRef().elements(); + + compareModelOutput(actual_result_list, expect_result_list); +} + +void backportAllVersionCheck( + std::stringstream& test_model_file_stream, + std::vector& input_data, + std::vector& expect_result_list, + const uint64_t expect_from_version) { + auto from_version = _get_model_bytecode_version(test_model_file_stream); + EXPECT_EQ(from_version, expect_from_version); + AT_ASSERT(from_version > 0); + + // Backport script_module_v5.ptl to an older version + constexpr int64_t minimum_to_version = 4; + auto current_to_version = from_version - 1; + + // Verify all candidate to_version work as expected. All backport to version + // larger than minimum_to_version should success. + while (current_to_version >= minimum_to_version) { + // Do not declare std::stringstream oss outside of the while loop as + // oss.clear() doesn't reset the stream content, only clears out error state + // flag in stringstream causing a problematic stream. Instead, it's cleaner + // and safer to just declare a new std::stringstream one and swap them. + std::stringstream oss; + bool backPortSuccess = + _backport_for_mobile(test_model_file_stream, oss, current_to_version); + AT_ASSERT(backPortSuccess); + + // Check backport model version + auto backport_version = _get_model_bytecode_version(oss); + backport_version = _get_model_bytecode_version(oss); + AT_ASSERT(backport_version == current_to_version); + + // Load and run the backport model, then compare the result with expect + // result + runAndCheckBytecodeModel( + oss, input_data, expect_result_list, current_to_version); + oss.seekg(0, oss.beg); + runAndCheckTorchScriptModel( + oss, input_data, expect_result_list, current_to_version); + + current_to_version--; + } + // backport to minimum version - 1 should fail + std::stringstream oss; + bool backPortSuccess = + _backport_for_mobile(test_model_file_stream, oss, minimum_to_version - 1); + AT_ASSERT(!backPortSuccess); +} +} // namespace + +#if !defined FB_XPLAT_BUILD +TEST(LiteInterpreterTest, BackPortByteCodeModelAllVersions) { + torch::jit::Module module("m"); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) + module.register_parameter("weight", torch::ones({20, 1, 5, 5}), false); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) + module.register_parameter("bias", torch::ones({20}), false); + module.define(R"( + def fn(self, x:float=1.0): + return x + + def forward(self, input): + x1 = torch.zeros(2, 2) + x2 = torch.empty_like(torch.empty(2, 2)) + x3 = torch._convolution(input, self.weight, self.bias, [1, 1], [0, 0], [1, 1], False, [0, 0], 1, False, False, True, True) + # Add torch.add operator to cover bytecode version bump from 6 to 7 + # for bytecode version 7, the main change is to support defaults arguments with out arguments + x = 2 * torch.ones(1) + h = torch.ones(1) + torch.add(x, h, out=x) + device = torch.ones(1, 1).cpu().device.type + is_cuda = x1.is_cuda + bool_val = True + check_is = [] is None + check_is_not = [1] is not None + check_not = not bool_val + num_to_tensor = torch.tensor([self.fn()]) + d = {"a": "abc"} + check_dict_index = d["a"] + check_dim = x1.dim() + return ( + x1, x2, x3, x, device, is_cuda, check_is, + check_is_not, num_to_tensor, check_dict_index, + check_dim, check_not + ) + )"); + + torch::jit::Module module_freeze = freeze(module); + + std::stringstream input_model_stream; + module_freeze._save_for_mobile( + input_model_stream, + /*extra_files=*/{}, + /*save_mobile_debug_info=*/false, + /*use_flatbuffer=*/true); + std::vector input_data = + std::vector({torch::ones({1, 1, 28, 28})}); + std::vector expect_result_list; + expect_result_list.emplace_back(at::ones({2, 2}, ScalarType::Float) * 0); + expect_result_list.emplace_back(at::ones({2, 2}, ScalarType::Float)); + expect_result_list.emplace_back( + at::ones({1, 20, 24, 24}, ScalarType::Float) * 26); + expect_result_list.emplace_back(3 * at::ones({1})); + // "cpu" False, False, True, tensor(1), "abc", 2, False) + expect_result_list.emplace_back(c10::IValue("cpu")); + expect_result_list.emplace_back(c10::IValue(false)); + expect_result_list.emplace_back(c10::IValue(false)); + expect_result_list.emplace_back(c10::IValue(true)); + expect_result_list.emplace_back(c10::IValue(at::ones({1}))); + expect_result_list.emplace_back(c10::IValue("abc")); + expect_result_list.emplace_back(c10::IValue(2)); + expect_result_list.emplace_back(c10::IValue(false)); + + backportAllVersionCheck( + input_model_stream, + input_data, + expect_result_list, + 9); // flatbuffer starts at 9 +} +#endif // !defined(FB_XPLAT_BUILD) + +TEST(LiteInterpreterTest, GetRuntimeOpsAndInfo) { + auto runtime_ops = _get_runtime_ops_and_info(); + // Ballpark estimate of the minimal number of ops; just used to + // verify API returns a reasonably large number. + AT_ASSERT(runtime_ops.size() > 2900); +} + +TEST(LiteInterpreterTest, isCompatibleSuccess) { + // test trivial success case + auto runtime_info = RuntimeCompatibilityInfo::get(); + std::unordered_map model_ops; + model_ops["aten::add.Scalar"] = OperatorInfo{2}; + + std::unordered_set types = {"List", "int", "NamedTuple"}; + auto model_info = ModelCompatibilityInfo{ + caffe2::serialize::kMaxSupportedBytecodeVersion, + model_ops, + types, + _get_runtime_bytecode_min_max_versions().first}; + + AT_ASSERT( + is_compatible(runtime_info, model_info).status == + ModelCompatibilityStatus::OK); +} + +TEST(LiteInterpreterTest, isCompatibleFail) { + // test trivial failure due to ops + std::unordered_map model_ops; + model_ops["aten::add.Scalar"] = OperatorInfo{2}; + auto model_info = ModelCompatibilityInfo{ + caffe2::serialize::kMaxSupportedBytecodeVersion, model_ops}; + std::unordered_map runtime_ops; + runtime_ops["aten::add.Int"] = OperatorInfo{2}; + auto runtime_info = RuntimeCompatibilityInfo{ + std::pair( + caffe2::serialize::kMinSupportedBytecodeVersion, + caffe2::serialize::kMaxSupportedBytecodeVersion), + runtime_ops, + _get_mobile_supported_types()}; + + auto result = is_compatible(runtime_info, model_info); + AT_ASSERT(result.status = ModelCompatibilityStatus::ERROR); + AT_ASSERT( + result.errors[0] == + "Operator 'aten::add.Scalar' missing from runtime (not found)"); + + // test trivial failure due to bytecode greater than max supported bytecode + // version + runtime_ops["aten::add.Scalar"] = OperatorInfo{2}; + runtime_info = RuntimeCompatibilityInfo{ + std::pair( + caffe2::serialize::kMinSupportedBytecodeVersion, + caffe2::serialize::kMaxSupportedBytecodeVersion), + runtime_ops, + _get_mobile_supported_types()}; + model_info.bytecode_version = + caffe2::serialize::kMaxSupportedBytecodeVersion + 1; + + result = is_compatible(runtime_info, model_info); + AT_ASSERT(result.status = ModelCompatibilityStatus::ERROR); + + // test trivial failure due to bytecode less than min supported bytecode + // version + runtime_ops["aten::add.Scalar"] = OperatorInfo{2}; + runtime_info = RuntimeCompatibilityInfo{ + std::pair( + caffe2::serialize::kMinSupportedBytecodeVersion, + caffe2::serialize::kMaxSupportedBytecodeVersion), + runtime_ops, + _get_mobile_supported_types()}; + model_info.bytecode_version = + caffe2::serialize::kMinSupportedBytecodeVersion - 1; + + result = is_compatible(runtime_info, model_info); + AT_ASSERT(result.status = ModelCompatibilityStatus::ERROR); + + // test trivial failure due to type + runtime_info = RuntimeCompatibilityInfo::get(); + std::unordered_set types = {"List", "int", "Sequence"}; + + model_info = ModelCompatibilityInfo{ + caffe2::serialize::kMaxSupportedBytecodeVersion, + model_ops, + types, + _get_runtime_bytecode_min_max_versions().first}; + + AT_ASSERT( + is_compatible(runtime_info, model_info).status == + ModelCompatibilityStatus::ERROR); + + // test trivial failure due to operator version + runtime_info = RuntimeCompatibilityInfo::get(); + + model_info = ModelCompatibilityInfo{ + caffe2::serialize::kMaxSupportedBytecodeVersion, model_ops, {}, 0}; + + AT_ASSERT( + is_compatible(runtime_info, model_info).status == + ModelCompatibilityStatus::ERROR); +} + +TEST(LiteInterpreterTest, Eval) { + std::vector inputs; + + Module m("m"); + m.define(R"( + def __init__(self, x): + self.training = True + + def forward(self, input): + return torch.dropout(input, 1.0, self.training) + )"); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,modernize-use-emplace) + inputs.push_back(torch::ones({1, 1, 28, 28})); + m.eval(); + auto outputref = m.forward(inputs).toTensor(); + + // save m in training mode to make sure that mobile eval() will correctly + // change back to eval mode + m.train(); + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + bc.eval(); + IValue res; + for (int i = 0; i < 3; ++i) { + res = bc.get_method("forward")(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT( + outputref[0][0][0][0].item() == output[0][0][0][0].item()); +} + +TEST(LiteInterpreterTest, FindWrongMethodName) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add(self, x): + b = 4 + return self.foo + x + b + )"); + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + ASSERT_TRUE(bc.find_method("forward") == std::nullopt); +} + +TEST(LiteInterpreterTest, FindAndRunMethod) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add_it(self, x): + b = 4 + return self.foo + x + b + )"); + + std::vector inputs; + auto minput = 5 * torch::ones({}); + inputs.emplace_back(minput); + auto ref = m.get_method("add_it")(inputs); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + IValue res; + for (int i = 0; i < 3; ++i) { + auto bcinputs = inputs; + auto method = bc.find_method("add_it"); + AT_ASSERT(method != std::nullopt); + res = (*method)(std::move(bcinputs)); + } + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); +} + +TEST(LiteInterpreterTest, RunMethodVariadic) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add_three(self, x, y): + return self.foo + x + y + )"); + + std::vector inputs; + auto inputx = 5 * torch::ones({}); + auto inputy = 4 * torch::ones({}); + auto ref = m.run_method("add_three", inputx, inputy); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + IValue res = bc.run_method("add_three", inputx, inputy); + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); +} + +TEST(LiteInterpreterTest, DuplicateSetState) { + Module m("M"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def __getstate__(self): + return self.foo + self.foo + def __setstate__(self, a): + self.foo = a + def forward(self, x): + b = 4 + return self.foo + x + b + )"); + + Module b("B"); + b.register_module("M0", m); + b.register_module("M1", m); + b.define(R"( + def forward(self, x): + return self.M0.forward(x) + self.M1.forward(x) + )"); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + const auto methods = bc.get_methods(); + const size_t expected_n = 3; + ASSERT_EQ(methods.size(), expected_n); +} + +TEST(LiteInterpreterTest, ExtraFiles) { + const auto script = R"JIT( + def forward(self): + x = torch.rand(5, 5) + x = x.mm(x) + return x + )JIT"; + + auto module = + std::make_shared("Module", std::make_shared()); + module->define(script); + std::ostringstream oss; + std::unordered_map extra_files; + extra_files["metadata.json"] = "abc"; + extra_files["mobile_info.json"] = "{\"key\": 23}"; + module->_save_for_mobile(oss, extra_files); + + std::istringstream iss(oss.str()); + std::unordered_map loaded_extra_files; + loaded_extra_files["metadata.json"] = ""; + torch::jit::_load_for_mobile(iss, torch::kCPU, loaded_extra_files); + ASSERT_EQ(loaded_extra_files["metadata.json"], "abc"); + + loaded_extra_files.clear(); + std::vector all_files = + caffe2::serialize::PyTorchStreamReader(&iss).getAllRecords(); + + for (auto& file_name : all_files) { + if (file_name.find("extra/") == 0) { + loaded_extra_files[file_name.substr(6)] = ""; + } + } + iss.seekg(0, iss.beg); + torch::jit::_load_for_mobile(iss, torch::kCPU, loaded_extra_files); + ASSERT_EQ(loaded_extra_files["metadata.json"], "abc"); + ASSERT_EQ(loaded_extra_files["mobile_info.json"], "{\"key\": 23}"); + + std::unordered_map + loaded_extra_files_without_explicit_mapping; + iss.seekg(0, iss.beg); + torch::jit::_load_for_mobile( + iss, + torch::kCPU, + loaded_extra_files_without_explicit_mapping, + MobileModuleLoadOptions::PARSE_ALL_EXTRA_FILE_MAPS); + ASSERT_EQ( + loaded_extra_files_without_explicit_mapping["metadata.json"], "abc"); + ASSERT_EQ( + loaded_extra_files_without_explicit_mapping["mobile_info.json"], + "{\"key\": 23}"); +} + +TEST(LiteInterpreterTest, OpNameExportFetchRootOperators) { + torch::jit::Module m("m"); + m.register_parameter("weight", torch::ones({20, 1, 5, 5}), false); + m.register_parameter("bias", torch::ones({20}), false); + m.define(R"( + def forward(self, input): + x1 = torch.zeros(2, 2) + x2 = torch.empty_like(torch.empty(2, 2)) + x3 = torch._convolution(input, self.weight, self.bias, [1, 1], [0, 0], [1, 1], False, [0, 0], 1, False, False, True, True) + return (x1, x2, x3) + )"); + m.eval(); + + std::stringstream ss; + m._save_for_mobile(ss); + + torch::jit::mobile::Module ptl_model = torch::jit::_load_for_mobile(ss); + std::set operator_names = + torch::jit::mobile::_export_operator_list(ptl_model); + std::set expected_operator_names = { + "aten::_convolution", + "aten::empty.memory_format", + "aten::empty_like", + "aten::zeros", + }; + EXPECT_EQ(operator_names, expected_operator_names) + << "Expected the root operator lists to be the same"; +} + +TEST(LiteInterpreterTest, DefaultArgsConv) { + auto s = std::getenv("PYTORCH_TEST_WITH_TSAN"); + if (s && strcmp(s, "1") == 0) + return; + + std::vector inputs; + + Module m("m"); + m.register_parameter("weight", torch::ones({20, 1, 5, 5}), false); + m.register_parameter("bias", torch::ones({20}), false); + m.define(R"( + def forward(self, input): + return torch.conv2d(input, self.weight, self.bias, [1, 1], [0, 0], [1, 1], 1) + )"); + + inputs.push_back(torch::ones({1, 1, 28, 28})); + + auto outputref = m.forward(inputs).toTensor(); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + IValue res; + for (int i = 0; i < 1; ++i) { + res = bc.get_method("forward")(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT(output.equal(outputref)); +} + +TEST(RunTimeTest, ParseBytecode) { + // A simple example to show a simple bytecode that can be used independent of + // PyTorch TorchScript serialization (unpickler, etc) and operator library. + // It has basic control flow (if, else) and basic data orchestration (list + // construction). The original PyTorch program: + + // class Module(torch.nn.Module): + // + // def __init__(self) -> None: + // super().__init__() + // + // def forward(self, x: int, h: int, xfirst: bool): + // if xfirst: + // return [x, h] + // else: + // return [h, x] + + // 1. Prepare for the bytecode. In reality it can be from a customized + // deserializer. + std::vector instructions{ + to_tuple({"STOREN", 1, 4}), + to_tuple({"DROPR", 1, 0}), + to_tuple({"MOVE", 4, 0}), + to_tuple({"JF", 5, 0}), + to_tuple({"LOAD", 2, 0}), + to_tuple({"LOAD", 3, 0}), + to_tuple({"LIST_CONSTRUCT", 0, 2}), + to_tuple({"JMP", 4, 0}), + to_tuple({"LOAD", 3, 0}), + to_tuple({"LOAD", 2, 0}), + to_tuple({"LIST_CONSTRUCT", 1, 2}), + to_tuple({"STORE", 5, 0}), + to_tuple({"DROPR", 3, 0}), + to_tuple({"DROPR", 2, 0}), + to_tuple({"MOVE", 5, 0}), + to_tuple({"RET", 0, 0}), + }; + std::vector operators; // empty for this example + std::vector constants; // empty for this example + + std::vector types{"List[int]", "List[int]"}; + // 2. Parse the function + std::string function_name("test_function"); + auto function = std::unique_ptr( + new mobile::Function(c10::QualifiedName(function_name))); + c10::ivalue::TupleElements debug_handles_m_tuple; + parseInstructions( + function_name, + std::move(*c10::ivalue::Tuple::create(instructions)).elements(), + debug_handles_m_tuple, + function.get()); + parseTypes(c10::ivalue::Tuple::create(types)->elements(), function.get()); + const size_t rsize = 5; + parseRegisterSize(rsize, function.get()); + + // 3. Prepare for inputs and run the function + // Note that the first input is reserved for Module object. + // Since this is a function test and Module object is not required, + // a dummy IValue (0) is added here. + std::vector inputs{0, 1, 2, true}; + function->run(inputs); + auto output = inputs[0].toList(); + ASSERT_EQ(output[0], 1); + ASSERT_EQ(output[1], 2); + + std::vector inputs1{0, 1, 2, false}; + function->run(inputs1); + auto output1 = inputs1[0].toList(); + ASSERT_EQ(output1[0], 2); + ASSERT_EQ(output1[1], 1); +} + +TEST(RunTimeTest, ParseOperator) { + // A simple example to show a simple bytecode that can be used independent of + // PyTorch TorchScript serialization (unpickler, etc) and operator library. + // It has one operator and we should be able to register it. The original + // PyTorch program: + + // class Add(torch.nn.Module): + // def __init__(self) -> None: + // super().__init__() + + // def forward(self, a, b): + // return a + b + + // 1. Prepare for the bytecode. In reality it can be from a customized + // deserializer. + std::vector instructions{ + to_tuple({"STOREN", 1, 3}), + to_tuple({"DROPR", 1, 0}), + to_tuple({"MOVE", 2, 0}), + to_tuple({"MOVE", 3, 0}), + to_tuple({"OP", 0, 0}), + to_tuple({"RET", 0, 0}), + }; + std::vector operators{ + to_tuple({"aten::add", "Tensor", 2}), + }; + std::vector constants{ + to_tuple({1}), + }; + // 2. Parse the function + std::string function_name("test_function"); + auto function = std::unique_ptr( + new mobile::Function(c10::QualifiedName(function_name))); + c10::ivalue::TupleElements debug_handles_m_tuple; + parseInstructions( + function_name, + std::move(*c10::ivalue::Tuple::create(instructions)).elements(), + debug_handles_m_tuple, + function.get()); + parseOperators( + std::move(*c10::ivalue::Tuple::create(operators)).elements(), + 1, + function.get()); + const size_t rsize = 5; + parseRegisterSize(rsize, function.get()); + + // 3. Prepare for inputs and run the function + // Note that the first input is reserved for Module object. + // Since this is a function test and Module object is not required, + // a dummy IValue (0) is added here. + std::vector inputs{0, at::tensor(1), at::tensor(2)}; + function->run(inputs); + auto output = inputs[0]; + ASSERT_EQ(output, at::tensor(3)); +} + +namespace { +void testLiteModuleCompareResultTensors( + Module& m, + const std::vector& inputs, + const std::string& method_name = "forward") { + auto outputref = m.get_method(method_name)(inputs).toTensor(); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + IValue res; + for (int i = 0; i < 3; ++i) { + res = bc.get_method(method_name)(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT(output.equal(outputref)); +} + +void testDefaultArgsPinv(int num_args) { + Module m("m"); + if (num_args == 1) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input) + )"); + } else if (num_args == 2) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5) + )"); + } else if (num_args == 3) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5, True) + )"); + } + + std::vector inputs; + const int N = 28; + auto input = torch::range(1, N * N, 1); + input[0] = 1; // a more stable matrix + input = input.view({N, N}); + inputs.push_back(input); + testLiteModuleCompareResultTensors(m, inputs); +} +} // namespace + +#if !defined FB_XPLAT_BUILD +TEST(LiteInterpreterTest, DefaultArgsPinv) { + // Test with different number of specified arguments. + // Arguments not specified take default value. + for (int num_args = 1; num_args <= 3; ++num_args) { + testDefaultArgsPinv(num_args); + } + + // bytecode with one specified argument: + // (6, + // ('__torch__.m.forward', + // (('instructions', + // (('STOREN', 1, 2), + // ('DROPR', 1, 0), + // ('MOVE', 2, 0), + // ('OP', 0, 0), + // ('RET', 0, 0))), + // ('operators', (('aten::linalg_pinv', '', 1),)), + // ('constants', (False, 1e-15)), # default constants are not + // used + // ('types', ()), + // ('register_size', 2)), + // (('arguments', + // ((('name', 'self'), ('type', '__torch__.m'), ('default_value', + // None)), + // (('name', 'input'), ('type', 'Tensor'), ('default_value', + // None)))), + // ('returns', + // ((('name', ''), ('type', 'Tensor'), ('default_value', + // None)),))))) + + // bytecode with 2 specified argument: + // (6, + // ('__torch__.m.forward', + // (('instructions', + // (('STOREN', 1, 2), + // ('DROPR', 1, 0), + // ('MOVE', 2, 0), + // ('LOADC', 1, 0), # added LOADC for specified argument + // ('OP', 0, 0), + // ('RET', 0, 0))), + // ('operators', (('aten::linalg_pinv', '', 2),)), + // ('constants', (False, 1e-05)), # updated constant table + // ('types', ()), + // ('register_size', 2)), + // (('arguments', + // ((('name', 'self'), ('type', '__torch__.m'), ('default_value', + // None)), + // (('name', 'input'), ('type', 'Tensor'), ('default_value', + // None)))), + // ('returns', + // ((('name', ''), ('type', 'Tensor'), ('default_value', + // None)),))))) + + // bytecode with 3 specified arguments: + // (6, + // ('__torch__.m.forward', + // (('instructions', + // (('STOREN', 1, 2), + // ('DROPR', 1, 0), + // ('MOVE', 2, 0), + // ('LOADC', 1, 0), + // ('LOADC', 0, 0), + // ('OP', 0, 0), + // ('RET', 0, 0))), + // ('operators', (('aten::linalg_pinv', '', 3),)), + // ('constants', (True, 1e-05)), + // ('types', ()), + // ('register_size', 2)), + // (('arguments', + // ((('name', 'self'), ('type', '__torch__.m'), ('default_value', + // None)), + // (('name', 'input'), ('type', 'Tensor'), ('default_value', + // None)))), + // ('returns', + // ((('name', ''), ('type', 'Tensor'), ('default_value', + // None)),))))) +} + +TEST(LiteInterpreterTest, DefaultArgsTensorinvSpecifyDefault) { + // The second argument is specified, but the value is the same as the default + // value. It's treated as "not specified" since the value can be fetched from + // schema. + Module m("m"); + m.define(R"( + def forward(self, input): + return torch.linalg_tensorinv(input, 2) + )"); + torch::jit::MobileCode code(m.get_method("forward").graph(), "forward"); + auto arg_nums = code.op_to_num_specified_args(); + ASSERT_EQ(arg_nums.size(), 1); + ASSERT_EQ(arg_nums["aten::linalg_tensorinv"], 1); + std::vector inputs; + const int N = 4; + auto input = torch::rand({N, N, N, N}); + inputs.push_back(input); + testLiteModuleCompareResultTensors(m, inputs); +} + +void testDefaultArgsPinvWithOutArg(int num_args) { + Module m("m"); + if (num_args == 1) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, out=input) + )"); + } else if (num_args == 2) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5, out=input) + )"); + } else if (num_args == 3) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5, True, out=input) + )"); + } + + const int N = 28; + auto input = torch::range(1, N * N, 1); + input[0] = 10000; // a more stable matrix + input = input.view({N, N}); + auto ref = m.run_method("forward", input); + TORCH_CHECK(!input.equal(torch::range(1, N * N, 1))); + TORCH_CHECK(input.equal(ref.toTensor())); +} + +TEST(LiteInterpreterTest, DefaultArgsPinvWithOutArg) { + // Test with different number of specified arguments + out arg. + // Arguments not specified take default value. + for (int num_args = 1; num_args <= 3; ++num_args) { + testDefaultArgsPinvWithOutArg(num_args); + } +} + +TEST(LiteInterpreterTest, DefaultArgsWithOutArg) { + Module m("m"); + m.define(R"( + def forward(self, x, h): + torch.add(x, h, out=x) + )"); + + std::vector inputs; + auto input_x = 2 * torch::ones({}); + auto input_h = torch::ones({}); + auto ref = m.run_method("forward", input_x, input_h); + + std::stringstream ss; + + m._save_for_mobile(ss, {}, true); + mobile::Module bc = _load_for_mobile(ss); + bc.run_method("forward", input_x, input_h); + AT_ASSERT(input_x.equal(4 * torch::ones({}))); + + auto ops = _get_model_ops_and_info(ss); + auto op = ops.find("aten::add.out"); + TORCH_CHECK( + op != ops.end() && op->second.num_schema_args.has_value() && + op->second.num_schema_args.value() == 3); +} + +TEST(LiteInterpreterTest, TestExceptionStackWithTwoLevelModuleHierarchy) { + Module a("A"); + a.define(R"( + def bar(self, x, y): + return x + y + )"); + Module b("B"); + b.register_module("A0", a); + b.define(R"( + def foo(self, x, y): + return self.A0.bar(x, y) + 2 + )"); + Module c("C"); + c.register_module("B0", b); + c.define(R"( + def forward(self, x, y): + return self.B0.foo(x, y) + 3 + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({2, 4})); + inputs.emplace_back(torch::rand({13, 9})); + + std::stringstream ss; + c._save_for_mobile(ss, ExtraFilesMap(), true); + auto lite_m = _load_for_mobile(ss); + std::string error_pattern = R"( + Module hierarchy:top(C)::.B0(B)::foo.A0(A)::bar.aten::add +Traceback of TorchScript (most recent call last): + File "", line 3, in + + def forward(self, x, y): + return self.B0.foo(x, y) + 3 + ~~~~~~~~~~~ <--- HERE + + File "", line 3, in foo + + def foo(self, x, y): + return self.A0.bar(x, y) + 2 + ~~~~~~~~~~~ <--- HERE + + File "", line 3, in bar + + def bar(self, x, y): + return x + y + ~~~~~ <--- HERE + )"; + ASSERT_THROWS_WITH_MESSAGE(lite_m.forward(inputs), error_pattern); +} +#endif // !defined(FB_XPLAT_BUILD) + +namespace { +static auto reg = + torch::class_( + "_TorchScriptTesting", + "_LiteInterpreterTest") + .def(torch::init<>()) + .def("get", &TorchBindLiteInterpreterTestStruct::get) + .def_pickle( + // __getattr__ + [](const c10::intrusive_ptr& + self) -> int64_t { return 0; }, + // __setattr__ + [](int64_t state) { + return c10::make_intrusive(); + }); + +} // namespace + +TEST(LiteInterpreterTest, OperatorCacheDifferentiatesDefaultArgs) { + // Create 3 methods: + // + // 1. forward() returns a tensor with dtype=torch.int64 (4) + // 2. forward2() returns a tensor with dtype=torch.float32 (6) + // 3. forward3() returns a tensor with dtype=torch.float32 but + // the dtype is inferred by the input tensor's dtype + // + // If caching works correctly, then the result from the full-jit + // module and the lite module will be the same. Otherwise, it + // will be different if we don't correctly ignore the cache + // entry for an operator that has a different number of + // arguments. + Module m("m"); + m.define(R"( + def forward(self): + ret1 = torch.new_empty(torch.zeros(10), [10], dtype=4) + return ret1.fill_(25) + )"); + m.define(R"( + def forward2(self): + ret1 = torch.new_empty(torch.zeros(10), [10], dtype=6) + return ret1.fill_(32.0) + )"); + m.define(R"( + def forward3(self): + ret1 = torch.new_empty(torch.zeros(10), [10]) + return ret1.fill_(12.0) + )"); + + std::vector inputs; + testLiteModuleCompareResultTensors(m, inputs, "forward"); + testLiteModuleCompareResultTensors(m, inputs, "forward2"); + testLiteModuleCompareResultTensors(m, inputs, "forward3"); +} + +TEST(RunTimeTest, RuntimeCall) { + // def call(x): + // return x + x + // + // def forward(a): + // x = a + call(a) + // y = a + call(x) + // return y + + std::vector instructionsCall{ + to_tuple({"STORE", 1, 0}), + to_tuple({"LOAD", 1, 0}), + to_tuple({"MOVE", 1, 0}), + to_tuple({"LOADC", 0, 0}), + to_tuple({"OP", 0, 0}), + to_tuple({"RET", 0, 0}), + }; + std::vector instructionsFoo{ + to_tuple({"STORE", 1, 0}), + to_tuple({"LOAD", 1, 0}), + to_tuple({"LOAD", 1, 0}), + to_tuple({"MOVE", 1, 0}), + to_tuple({"CALL", 0, 0}), + to_tuple({"LOADC", 0, 0}), + to_tuple({"OP", 0, 0}), + to_tuple({"CALL", 0, 0}), + to_tuple({"LOADC", 0, 0}), + to_tuple({"OP", 0, 0}), + to_tuple({"RET", 0, 0}), + }; + std::vector operatorsFoo{ + to_tuple({"aten::add", "Tensor", 3}), + }; + std::vector constantsFoo{ + 1, + }; + std::vector operatorsCall{ + to_tuple({"aten::add", "Tensor", 3}), + }; + std::vector constantsCall{ + 1, + }; + + auto foo = std::make_unique(c10::QualifiedName("foo")); + c10::ivalue::TupleElements debug_handles_m_tuple; + parseInstructions( + "foo", + std::move(*c10::ivalue::Tuple::create(instructionsFoo)).elements(), + debug_handles_m_tuple, + foo.get()); + parseOperators( + std::move(*c10::ivalue::Tuple::create(operatorsFoo)).elements(), + 1, + foo.get()); + parseConstants( + std::move(*c10::ivalue::Tuple::create(constantsFoo)).elements(), + foo.get()); + const size_t rsize = 5; + parseRegisterSize(rsize, foo.get()); + + auto call = std::make_unique(c10::QualifiedName("call")); + parseInstructions( + "call", + std::move(*c10::ivalue::Tuple::create(instructionsCall)).elements(), + debug_handles_m_tuple, + call.get()); + parseOperators( + std::move(*c10::ivalue::Tuple::create(operatorsCall)).elements(), + 1, + call.get()); + parseConstants( + std::move(*c10::ivalue::Tuple::create(constantsCall)).elements(), + call.get()); + parseRegisterSize(rsize, call.get()); + + foo->append_function(*call); + + std::vector inputs{at::tensor(1)}; + foo->run(inputs); + auto output = inputs[0]; + ASSERT_EQ(output, at::tensor(7)); +} + +TEST(LiteInterpreterTest, OperatorSize1) { + Module m("m"); + m.define(R"( + def forward(self, input: Tensor, scale:float): + return torch.upsample_nearest2d(input, [1, 1], float(scale), float(scale)) + )"); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + const auto& func = bc.get_method("forward").function(); + ASSERT_EQ( + func.get_code().operator_input_sizes_.size(), + func.get_code().operators_.size()); +} + +TEST(LiteInterpreterTest, OperatorTest2) { // NOLINT (use =delete in gtest) + const std::vector test_programs{ + // test invoking a method with default parameter + R"( + def test_func(self, x, b : int = 4): + return self.foo + x + b + )", + // inner method call with default parameter (gets inlined) + R"( + def add_with_default_arg(self, x, b : int = 4): + return self.foo + x + b + def test_func(self, x): + return self.add_with_default_arg(x) # invoke method w/ default arg + )", + // simple method call + R"( + def test_func(self, x): + b = 4 + return self.foo + x + b + )", + }; + for (const auto& test_program : test_programs) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(test_program); + + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + const auto& func = bc.get_method("test_func").function(); + ASSERT_EQ( + func.get_code().operator_input_sizes_.size(), + func.get_code().operators_.size()); + } +} + +#if !defined FB_XPLAT_BUILD +// The following test run in fbcode only +TEST(LiteInterpreterUpgraderTest, DivTensorV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_tensor_v2.ptl"); + /* + (('__torch__.MyModule.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('LOAD', 2, 0), + ('LOAD', 3, 0), + ('OP', 0, 0), + ('LOAD', 2, 0), + ('LOAD', 3, 0), + ('OP', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 2, 0), + ('TUPLE_CONSTRUCT', 3, 0), + ('RET', 0, 0))), + ('operators', + (('aten::div', 'Tensor'), + ('aten::div', 'Tensor'), + ('aten::div', 'Tensor'))), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + + */ + mobile::Module m_module = _load_for_mobile(test_model_file); + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // 3 operators will use upgrader + ASSERT_EQ(number_of_call_instruction, 3); + + std::vector inputs = { + IValue(6 * torch::ones({1})), IValue(3 * torch::ones({1}))}; + auto actual_output = m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output_list = actual_output.toTuple()->elements(); + ASSERT_TRUE(actual_output_list[0].toTensor().equal(expect_output)); +} + +TEST(LiteInterpreterUpgraderTest, DivTensorOutV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_tensor_out_v2.ptl"); + /* + (('__torch__.MyModule.forward', + (('instructions', + (('STOREN', 1, 4), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('MOVE', 4, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div', 'out'),)), + ('constants', ()), + ('types', ()), + ('register_size', 4))),) + */ + mobile::Module m_module = _load_for_mobile(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{ + IValue(6 * torch::ones({1})), + IValue(3 * torch::ones({1})), + IValue(torch::empty({1}))}; + m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = inputs[2].toTensor(); + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(LiteInterpreterUpgraderTest, DivTensorInplaceV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_tensor_inplace_v2.ptl"); + /* + (('__torch__.MyModule.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div_', 'Tensor'),)), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + mobile::Module m_module = _load_for_mobile(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{ + IValue(6 * torch::ones({1})), IValue(3 * torch::ones({1}))}; + m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = inputs[0].toTensor(); + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(LiteInterpreterUpgraderTest, DivScalarFloatV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_scalar_float_v2.ptl"); + /* + (('__torch__.MyModuleFloat.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div', 'Scalar'),)), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + + mobile::Module m_module = _load_for_mobile(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3.0)}; + auto output = m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = output.toTensor(); + + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(LiteInterpreterUpgraderTest, DivScalarReciprocalFloatV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, + "upgrader_models/test_versioned_div_scalar_reciprocal_float_v2.ptl"); + /* + (('__torch__.MyModuleFloat.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('OP', 0, 0), + ('MOVE', 3, 0), + ('OP', 1, 0), + ('RET', 0, 0))), + ('operators', (('aten::reciprocal', ''), ('aten::mul', 'Scalar'))), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + mobile::Module m_module = _load_for_mobile(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // No operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 0); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3.0)}; + auto output = m_module.forward(inputs); + auto expect_output = 0.5 * torch::ones({1}); + auto actual_output = output.toTensor(); + std::cout << "expect output: " << expect_output; + std::cout << "actual output: " << actual_output; + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(LiteInterpreterUpgraderTest, DivScalarReciprocalIntV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, + "upgrader_models/test_versioned_div_scalar_reciprocal_int_v2.ptl"); + /* + (('__torch__.MyModuleInt.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('OP', 0, 0), + ('MOVE', 3, 0), + ('OP', 1, 0), + ('RET', 0, 0))), + ('operators', (('aten::reciprocal', ''), ('aten::mul', 'Scalar'))), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + mobile::Module m_module = _load_for_mobile(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // No operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 0); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3.0)}; + auto output = m_module.forward(inputs); + auto expect_output = 0.5 * torch::ones({1}); + auto actual_output = output.toTensor(); + + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(LiteInterpreterUpgraderTest, DivScalarScalarV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_scalar_scalar_v2.ptl"); + /* + (('__torch__.MyModule.forward', + (('instructions', + (('STOREN', 1, 5), + ('DROPR', 1, 0), + ('LOAD', 2, 0), + ('LOAD', 3, 0), + ('OP', 0, 0), + ('MOVE', 2, 0), + ('LOAD', 4, 0), + ('OP', 1, 0), + ('LOAD', 3, 0), + ('MOVE', 4, 0), + ('OP', 2, 0), + ('MOVE', 3, 0), + ('MOVE', 5, 0), + ('OP', 3, 0), + ('TUPLE_CONSTRUCT', 4, 0), + ('RET', 0, 0))), + ('operators', + (('aten::div', ''), + ('aten::div', 'float'), + ('aten::div', ''), + ('aten::div', 'int'))), + ('constants', ()), + ('types', ()), + ('register_size', 5))),) + */ + mobile::Module m_module = _load_for_mobile(test_model_file); + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // No operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 0); + + std::vector inputs{IValue(20.0), IValue(10), IValue(2.0), IValue(5)}; + auto output = m_module.forward(inputs); + auto output_list = output.toTupleRef().elements(); + auto expect_output = std::vector( + {IValue(2.0), IValue(10.0), IValue(5.0), IValue(2.0)}); + // auto actual_output = output.toTensor(); + for (size_t i = 0; i < expect_output.size(); i++) { + ASSERT_EQ(output_list[i], expect_output[i]); + } +} + +TEST(LiteInterpreterUpgraderTest, DivScalarIntV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_scalar_int_v2.ptl"); + /* + (('__torch__.MyModuleInt.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div', 'Scalar'),)), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + mobile::Module m_module = _load_for_mobile(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3)}; + auto output = m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = output.toTensor(); + + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(LiteInterpreterUpgraderTest, DivScalarInplaceFloatV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, + "upgrader_models/test_versioned_div_scalar_inplace_float_v2.ptl"); + /* + (('__torch__.MyModuleFloat.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div_', 'Scalar'),)), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + + mobile::Module m_module = _load_for_mobile(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3.0)}; + auto output = m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = output.toTensor(); + + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +TEST(LiteInterpreterUpgraderTest, DivScalarInplaceIntV2) { + auto test_model_file = resolveTestDataFile( + __FILE__, "upgrader_models/test_versioned_div_scalar_inplace_int_v2.ptl"); + /* + (('__torch__.MyModuleInt.forward', + (('instructions', + (('STOREN', 1, 3), + ('DROPR', 1, 0), + ('MOVE', 2, 0), + ('MOVE', 3, 0), + ('OP', 0, 0), + ('RET', 0, 0))), + ('operators', (('aten::div_', 'Scalar'),)), + ('constants', ()), + ('types', ()), + ('register_size', 3))),) + */ + + mobile::Module m_module = _load_for_mobile(test_model_file); + + auto intrsuction_list = + m_module.get_method("forward").function().get_code().instructions_; + uint64_t number_of_call_instruction = 0; + for (auto& instruction : intrsuction_list) { + number_of_call_instruction += (instruction.op == OpCode::CALL); + } + // One operator will use upgrader + ASSERT_EQ(number_of_call_instruction, 1); + + std::vector inputs{IValue(6 * torch::ones({1})), IValue(3)}; + auto output = m_module.forward(inputs); + auto expect_output = 2.0 * torch::ones({1}); + auto actual_output = output.toTensor(); + + // The out argument will be overwritten with the output + ASSERT_TRUE(actual_output.equal(expect_output)); +} + +#endif // !defined(FB_XPLAT_BUILD) + +TEST(LiteInterpreterUpgraderTest, Upgrader) { + std::vector upgrader_functions; + + for (auto& byteCodeFunctionWithOperator : getUpgraderBytecodeList()) { + byteCodeFunctionWithOperator.function.initialize_operators(true); + ASSERT_EQ( + byteCodeFunctionWithOperator.function.get_code().operators_.size(), + byteCodeFunctionWithOperator.function.get_code().op_names_.size()); + if (byteCodeFunctionWithOperator.function.get_code().operators_.empty()) { + for (const auto& op : byteCodeFunctionWithOperator.operators) { + byteCodeFunctionWithOperator.function.append_operator( + op.name, op.overload_name, op.num_specified_args); + } + } + upgrader_functions.push_back(byteCodeFunctionWithOperator.function); + } + + ASSERT_EQ(getUpgraderBytecodeList().size(), upgrader_functions.size()); +} + +void enumerateTupleType( + size_t depth, + std::vector& current, + const std::vector& candidates, + std::vector& out) { + static std::vector fieldNames; + if (depth > fieldNames.size()) { + fieldNames.reserve(depth); + for (size_t i = fieldNames.size(); i < depth; i++) { + fieldNames.push_back("field" + std::to_string(i)); + } + } + if (depth == 0) { + out.push_back(TupleType::create(current)); + while (fieldNames.size() > current.size()) { + fieldNames.pop_back(); + } + out.push_back(TupleType::createNamed("NamedTuple", fieldNames, current)); + return; + } + for (const auto& type : candidates) { + if (containsAnyType(type)) { + continue; + } + current.push_back(type); + enumerateTupleType(depth - 1, current, candidates, out); + current.pop_back(); + } +} + +class LiteInterpreterDynamicTypeTestFixture + : public ::testing::TestWithParam { + protected: + void SetUp() override { + cu = std::make_shared(); + std::vector keyTypes = { + AnyType::get(), + IntType::get(), + BoolType::get(), + FloatType::get(), + ComplexType::get(), + StringType::get(), + TensorType::get(), + DeviceObjType::get(), + }; + types = { + NoneType::get(), + NumberType::get(), + ClassType::create("__torch__.TestClass1", cu), + ClassType::create("__torch__.TestClass2", cu), + AnyListType::get(), + AnyTupleType::get(), + StreamObjType::get(), + CapsuleType::get(), + GeneratorType::get(), + StorageType::get(), + VarType::create("t"), + VarType::create("v"), + AnyClassType::get()}; + std::copy(keyTypes.begin(), keyTypes.end(), back_inserter(types)); + auto expandTypes = [&](size_t tupleSize) { + std::vector nested; + for (const auto& type : types) { + if (!(type == AnyType::get())) { + nested.emplace_back(ListType::create(type)); + if (!(type == NoneType::get() || + type->kind() == OptionalType::Kind)) { + nested.emplace_back(OptionalType::create(type)); + } + } + for (const auto& keyType : keyTypes) { + nested.emplace_back(DictType::create(keyType, type)); + } + } + std::vector tmp; + enumerateTupleType(tupleSize, tmp, types, nested); + std::move( + std::begin(nested), std::end(nested), std::back_inserter(types)); + }; + expandTypes(1); + expandTypes(1); + } + std::shared_ptr cu; + std::vector types; + + public: + static constexpr size_t kNumSplits = 10; +}; + +/** + * Enumerate all possible JIT types appearing in mobile runtime, and test + * whether subtyping relation is preserved after one of the JIT types is + * converted to DynamicType. + * + * We firstly enumerate all "base" types in a vector, and implement + * expandTypes() to enumerate container types one "level" up for a given set + * of types. We call expandTypes() twice to test types nested less or equal + * to two levels. e.g. List[Optional[Tensor]], Optional[Dict[Int, Bool]], etc. + */ +TEST_P(LiteInterpreterDynamicTypeTestFixture, Conformance) { + size_t num = types.size() / LiteInterpreterDynamicTypeTestFixture::kNumSplits; + size_t begin = num * GetParam(); + size_t end = std::min(types.size(), begin + num); + for (const auto& a : types) { + auto da = DynamicType::create(*a); + for (size_t i = begin; i < end; i++) { + const auto& b = types[i]; + bool result = a->isSubtypeOf(*b); + EXPECT_EQ(result, da->isSubtypeOf(*b)); + result = b->isSubtypeOf(*a); + EXPECT_EQ(result, b->isSubtypeOf(*da)); + } + } +} + +INSTANTIATE_TEST_SUITE_P( + PyTorch, + LiteInterpreterDynamicTypeTestFixture, + ::testing::Range( + static_cast(0), + LiteInterpreterDynamicTypeTestFixture::kNumSplits)); + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_lite_interpreter_direct.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_lite_interpreter_direct.cpp new file mode 100644 index 0000000000000000000000000000000000000000..30faa32b3e1bf600eda0ac824708995218004d45 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_lite_interpreter_direct.cpp @@ -0,0 +1,919 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +// Tests go in torch::jit +namespace torch { +namespace jit { + +TEST(LiteInterpreterDirectTest, UpsampleNearest2d) { + Module m("m"); + m.define(R"( + def forward(self, input: Tensor, scale:float): + return torch.upsample_nearest2d(input, [1, 1], float(scale), float(scale)) + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({1, 3, 128, 128})); + inputs.emplace_back(at::Scalar(2.0)); + auto ref = m.forward(inputs); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + res = bc.forward(inputs); + + auto resd = res.toTensor(); + auto refd = ref.toTensor(); + ASSERT_TRUE(resd.equal(refd)); +} + +TEST(LiteInterpreterDirectTest, CheckAttrAccess) { + Module m("m"); + m.register_attribute("mobile_optimized", BoolType::get(), true); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + bool mobile_optimized = bc.attr("mobile_optimized", false).toBool(); + + AT_ASSERT(mobile_optimized); + m.setattr("mobile_optimized", false); + bc = jitModuleToMobile(m, options); + mobile_optimized = bc.attr("mobile_optimized", false).toBool(); + AT_ASSERT(!mobile_optimized); +} + +TEST( + LiteInterpreterDirectTest, + MethodInvocation) { // NOLINT (use =delete in gtest) + const std::vector test_programs{ + // test invoking a method with default parameter + R"( + def test_func(self, x, b : int = 4): + return self.foo + x + b + )", + // inner method call with default parameter (gets inlined) + R"( + def add_with_default_arg(self, x, b : int = 4): + return self.foo + x + b + def test_func(self, x): + return self.add_with_default_arg(x) # invoke method w/ default arg + )", + // simple method call + R"( + def test_func(self, x): + b = 4 + return self.foo + x + b + )", + }; + for (const auto& test_program : test_programs) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(test_program); + + const int fortyTwo = 42; // (keep linter happy) + auto minput = fortyTwo * torch::ones({}); + auto ref = m.run_method("test_func", minput); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + const auto& test_func = bc.get_method("test_func"); + std::cerr << "hello " << std::endl; + IValue res; + for (int i = 0; i < 3; ++i) { + res = test_func({minput}); + } + std::cerr << "hello 3" << std::endl; + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); + } +} + +TEST(LiteInterpreterDirectTest, Conv) { + auto s = std::getenv("PYTORCH_TEST_WITH_TSAN"); + if (s && strcmp(s, "1") == 0) + return; + + std::vector inputs; + + Module m("m"); + m.register_parameter("weight", torch::ones({20, 1, 5, 5}), false); + m.register_parameter("bias", torch::ones({20}), false); + m.define(R"( + def forward(self, input): + return torch._convolution(input, self.weight, self.bias, [1, 1], [0, 0], [1, 1], False, [0, 0], 1, False, False, True, True) + )"); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,modernize-use-emplace) + inputs.push_back(torch::ones({1, 1, 28, 28})); + + auto outputref = m.forward(inputs).toTensor(); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + res = bc.get_method("forward")(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT( + outputref[0][0][0][0].item() == output[0][0][0][0].item()); +} + +TEST(LiteInterpreterDirectTest, Inline) { + Module m("m"); + m.define(R"JIT( + def foo1(self, x): + return x + 1 + + def foo2(self, x): + return self.foo1(x) + 2 + + def foo3(self, x): + return self.foo2(x) + 3 + )JIT"); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("foo3")(inputs); + AT_ASSERT(output.toTensor().item() == 7.0); +} + +TEST(LiteInterpreterDirectTest, Tuple) { + Module m("m"); + m.define(R"JIT( + def foo(self, x): + return (1, 2, x + 3) + + def forward(self, x): + tuple = self.foo(x) + return tuple + )JIT"); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("forward")(inputs); + AT_ASSERT(output.toTupleRef().elements()[1].toInt() == 2); +} + +TEST(LiteInterpreterDirectTest, Dict) { + Module m("m"); + m.define(R"JIT( + def foo(self, x): + return {"result": x + 1} + + def forward(self, x): + d = self.foo(x) + return d + )JIT"); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + std::vector inputs({torch::ones({})}); + auto output = bc.get_method("forward")(inputs); + AT_ASSERT(output.toGenericDict().at("result").toTensor().item().toInt() == 2); +} + +TEST(LiteInterpreterDirectTest, Prim) { + Module m("m"); + m.define(R"JIT( + def forward(self, x): + return int(x) + )JIT"); + + std::vector inputs; + auto minput = 3.5 * torch::ones({}); + inputs.emplace_back(minput); + auto ref = m.run_method("forward", minput); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + + IValue res; + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc.get_method("forward")(bcinputs); + } + + auto resi = res.toInt(); + auto refi = ref.toInt(); + AT_ASSERT(resi == refi); +} + +TEST(LiteInterpreterDirectTest, PrimScalar) { + Module m("m"); + m.define(R"JIT( + def forward(self, x): + return int(x.item()) + )JIT"); + + std::vector inputs; + auto minput = 3.5 * torch::ones({}); + inputs.emplace_back(minput); + auto ref = m.run_method("forward", minput); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc.get_method("forward")(bcinputs); + } + + auto resi = res.toInt(); + auto refi = ref.toInt(); + AT_ASSERT(resi == refi); +} + +TEST(LiteInterpreterDirectTest, WrongMethodName) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add(self, x): + b = 4 + return self.foo + x + b + )"); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + std::vector inputs; + auto minput = 5 * torch::ones({}); + inputs.emplace_back(minput); + ASSERT_THROWS_WITH_MESSAGE( + bc.get_method("forward")(inputs), "is not defined"); +} + +TEST(LiteInterpreterDirectTest, SetState) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def __getstate__(self): + return self.foo + def __setstate__(self, a): + self.foo = a + def forward(self, x): + b = 4 + return self.foo + x + b + )"); + + std::vector inputs; + auto minput = 5 * torch::ones({}); + inputs.emplace_back(minput); + + std::stringstream ms; + m.save(ms); + auto loaded_m = load(ms); + auto ref = loaded_m.run_method("forward", minput); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto bcinputs = inputs; + res = bc.get_method("forward")(bcinputs); + } + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); +} + +class TorchBindLiteInterpreterDirectTestStruct + : public torch::jit::CustomClassHolder { + public: + std::string get(at::Tensor t) { + std::stringstream ss; + ss << "Hello! Your tensor has "; + ss << t.numel(); + ss << " elements!"; + return std::move(ss).str(); + } +}; + +namespace { +struct ClassNamespaceValue : public SugaredValue { + explicit ClassNamespaceValue(c10::QualifiedName name) + : basename_(std::move(name)) {} + + std::shared_ptr attr( + const SourceRange&, + GraphFunction&, + const std::string& name) override { + const auto fullName = c10::QualifiedName(basename_, name); + + // Check to see if it is a custom class. + if (auto custom_class = getCustomClass(fullName.qualifiedName())) { + return std::make_shared(custom_class); + } + + // If it's not a custom class, assume it's another namespace + // NOLINTNEXTLINE(performance-move-const-arg) + return std::make_shared(fullName); + } + + std::string kind() const override { + return "Class Namespace"; + } + + private: + c10::QualifiedName basename_; +}; + +struct TestModuleResolver : public Resolver { + std::shared_ptr resolveValue( + const std::string& name, + GraphFunction&, + const SourceRange&) override { + if (name == "torch") { + return std::make_shared("aten"); + } else if (name == "__torch__") { + return std::make_shared(c10::QualifiedName(name)); + } + + return nullptr; + } + + TypePtr resolveType(const std::string&, const SourceRange&) override { + return nullptr; + } +}; +} // namespace + +TEST(LiteInterpreterDirectTest, BuiltinFunction) { + script::Module m("m"); + auto custom_class_obj = + make_custom_class(); + m.register_attribute("my_obj", custom_class_obj.type(), custom_class_obj); + m.define(R"( + def forward(self, x) -> str: + return self.my_obj.get(x) + )"); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + auto res = + bc.get_method("forward")(std::vector{torch::zeros({3, 4})}); + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + auto str = res.toStringRef(); + std::string expected = "Hello! Your tensor has 12 elements!"; + AT_ASSERT(str == expected); +} + +#if !defined FB_XPLAT_BUILD +TEST(LiteInterpreterDirectTest, GetRuntimeByteCodeVersion) { + auto runtime_bytecode_version = _get_runtime_bytecode_version(); + AT_ASSERT( + runtime_bytecode_version == + caffe2::serialize::kMaxSupportedBytecodeVersion); +} + +TEST(LiteInterpreterDirectTest, GetRuntimeOperatorsVersion) { + auto runtime_operators_version = _get_runtime_operators_min_max_versions(); + AT_ASSERT( + runtime_operators_version.first == + caffe2::serialize::kMinSupportedFileFormatVersion && + runtime_operators_version.second == + caffe2::serialize::kMaxSupportedFileFormatVersion); +} + +/** + * The test below is disarmed for FB internal xplat builds since + * BUCK requires us to pass in the script_module_v4.ptl file in + * as a resource dependency of the build rule for this file, and + * we would need to access it via the C++ Resources API instead + * of directly reading from disk (which is what the open source + * build/run does). + */ +TEST(LiteInterpreterDirectTest, GetByteCodeVersion) { + auto test_model_file_v4 = + resolveTestDataFile(__FILE__, "script_module_v4.ptl"); + + auto version_v4 = _get_model_bytecode_version(test_model_file_v4); + AT_ASSERT(version_v4 == 4); +} + +#endif // !defined(FB_XPLAT_BUILD) + +TEST(LiteInterpreterDirectTest, GetRuntimeOpsAndInfo) { + auto runtime_ops = _get_runtime_ops_and_info(); + // Ballpark estimate of the minimal number of ops; just used to + // verify API returns a reasonably large number. + AT_ASSERT(runtime_ops.size() > 2900); +} + +TEST(LiteInterpreterDirectTest, Eval) { + std::vector inputs; + + Module m("m"); + m.define(R"( + def __init__(self, x): + self.training = True + + def forward(self, input): + return torch.dropout(input, 1.0, self.training) + )"); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,modernize-use-emplace) + inputs.push_back(torch::ones({1, 1, 28, 28})); + m.eval(); + auto outputref = m.forward(inputs).toTensor(); + + // save m in training mode to make sure that mobile eval() will correctly + // change back to eval mode + m.train(); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + bc.eval(); + IValue res; + for (int i = 0; i < 3; ++i) { + res = bc.get_method("forward")(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT( + outputref[0][0][0][0].item() == output[0][0][0][0].item()); +} + +TEST(LiteInterpreterDirectTest, FindWrongMethodName) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add(self, x): + b = 4 + return self.foo + x + b + )"); + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + ASSERT_TRUE(bc.find_method("forward") == std::nullopt); +} + +TEST(LiteInterpreterDirectTest, FindAndRunMethod) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add_it(self, x): + b = 4 + return self.foo + x + b + )"); + + std::vector inputs; + auto minput = 5 * torch::ones({}); + inputs.emplace_back(minput); + auto ref = m.get_method("add_it")(inputs); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + auto bcinputs = inputs; + auto method = bc.find_method("add_it"); + AT_ASSERT(method != std::nullopt); + res = (*method)(std::move(bcinputs)); + } + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); +} + +TEST(LiteInterpreterDirectTest, RunMethodVariadic) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add_three(self, x, y): + return self.foo + x + y + )"); + + std::vector inputs; + auto inputx = 5 * torch::ones({}); + auto inputy = 4 * torch::ones({}); + auto ref = m.run_method("add_three", inputx, inputy); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res = bc.run_method("add_three", inputx, inputy); + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); +} + +TEST(LiteInterpreterDirectTest, DuplicateSetState) { + Module m("M"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def __getstate__(self): + return self.foo + self.foo + def __setstate__(self, a): + self.foo = a + def forward(self, x): + b = 4 + return self.foo + x + b + )"); + + Module b("B"); + b.register_module("M0", m); + b.register_module("M1", m); + b.define(R"( + def forward(self, x): + return self.M0.forward(x) + self.M1.forward(x) + )"); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + const auto methods = bc.get_methods(); + const size_t expected_n = 3; + ASSERT_EQ(methods.size(), expected_n); +} + +TEST(LiteInterpreterDirectTest, OpNameExportFetchRootOperators) { + torch::jit::Module m("m"); + m.register_parameter("weight", torch::ones({20, 1, 5, 5}), false); + m.register_parameter("bias", torch::ones({20}), false); + m.define(R"( + def forward(self, input): + x1 = torch.zeros(2, 2) + x2 = torch.empty_like(torch.empty(2, 2)) + x3 = torch._convolution(input, self.weight, self.bias, [1, 1], [0, 0], [1, 1], False, [0, 0], 1, False, False, True, True) + return (x1, x2, x3) + )"); + m.eval(); + + CompilationOptions options; + mobile::Module ptl_model = jitModuleToMobile(m, options); + std::set operator_names = + torch::jit::mobile::_export_operator_list(ptl_model); + std::set expected_operator_names = { + "aten::_convolution", + "aten::empty.memory_format", + "aten::empty_like", + "aten::zeros", + }; + EXPECT_EQ(operator_names, expected_operator_names) + << "Expected the root operator lists to be the same"; +} + +TEST(LiteInterpreterDirectTest, DefaultArgsConv) { + auto s = std::getenv("PYTORCH_TEST_WITH_TSAN"); + if (s && strcmp(s, "1") == 0) + return; + + std::vector inputs; + + Module m("m"); + m.register_parameter("weight", torch::ones({20, 1, 5, 5}), false); + m.register_parameter("bias", torch::ones({20}), false); + m.define(R"( + def forward(self, input): + return torch.conv2d(input, self.weight, self.bias, [1, 1], [0, 0], [1, 1], 1) + )"); + + inputs.emplace_back(torch::ones({1, 1, 28, 28})); + + auto outputref = m.forward(inputs).toTensor(); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 1; ++i) { + res = bc.get_method("forward")(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT(output.equal(outputref)); +} + +namespace { +void testLiteModuleCompareResultTensors( + Module& m, + const std::vector& inputs, + const std::string& method_name = "forward") { + auto outputref = m.get_method(method_name)(inputs).toTensor(); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + IValue res; + for (int i = 0; i < 3; ++i) { + res = bc.get_method(method_name)(inputs); + } + auto output = res.toTensor(); + AT_ASSERT(outputref.dim() == output.dim()); + AT_ASSERT(output.equal(outputref)); +} + +void testDefaultArgsPinv2(int num_args) { + Module m("m"); + if (num_args == 1) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input) + )"); + } else if (num_args == 2) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5) + )"); + } else if (num_args == 3) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5, True) + )"); + } + + std::vector inputs; + const int N = 28; + auto input = torch::range(1, N * N, 1); + input[0] = 1; // a more stable matrix + input = input.view({N, N}); + inputs.emplace_back(input); + testLiteModuleCompareResultTensors(m, inputs); +} +} // namespace + +#if !defined FB_XPLAT_BUILD +TEST(LiteInterpreterDirectTest, DefaultArgsPinv) { + // Test with different number of specified arguments. + // Arguments not specified take default value. + for (int num_args = 1; num_args <= 3; ++num_args) { + testDefaultArgsPinv2(num_args); + } + + // bytecode with one specified argument: + // (6, + // ('__torch__.m.forward', + // (('instructions', + // (('STOREN', 1, 2), + // ('DROPR', 1, 0), + // ('MOVE', 2, 0), + // ('OP', 0, 0), + // ('RET', 0, 0))), + // ('operators', (('aten::linalg_pinv', '', 1),)), + // ('constants', (False, 1e-15)), # default constants are not + // used + // ('types', ()), + // ('register_size', 2)), + // (('arguments', + // ((('name', 'self'), ('type', '__torch__.m'), ('default_value', + // None)), + // (('name', 'input'), ('type', 'Tensor'), ('default_value', + // None)))), + // ('returns', + // ((('name', ''), ('type', 'Tensor'), ('default_value', + // None)),))))) + + // bytecode with 2 specified argument: + // (6, + // ('__torch__.m.forward', + // (('instructions', + // (('STOREN', 1, 2), + // ('DROPR', 1, 0), + // ('MOVE', 2, 0), + // ('LOADC', 1, 0), # added LOADC for specified argument + // ('OP', 0, 0), + // ('RET', 0, 0))), + // ('operators', (('aten::linalg_pinv', '', 2),)), + // ('constants', (False, 1e-05)), # updated constant table + // ('types', ()), + // ('register_size', 2)), + // (('arguments', + // ((('name', 'self'), ('type', '__torch__.m'), ('default_value', + // None)), + // (('name', 'input'), ('type', 'Tensor'), ('default_value', + // None)))), + // ('returns', + // ((('name', ''), ('type', 'Tensor'), ('default_value', + // None)),))))) + + // bytecode with 3 specified arguments: + // (6, + // ('__torch__.m.forward', + // (('instructions', + // (('STOREN', 1, 2), + // ('DROPR', 1, 0), + // ('MOVE', 2, 0), + // ('LOADC', 1, 0), + // ('LOADC', 0, 0), + // ('OP', 0, 0), + // ('RET', 0, 0))), + // ('operators', (('aten::linalg_pinv', '', 3),)), + // ('constants', (True, 1e-05)), + // ('types', ()), + // ('register_size', 2)), + // (('arguments', + // ((('name', 'self'), ('type', '__torch__.m'), ('default_value', + // None)), + // (('name', 'input'), ('type', 'Tensor'), ('default_value', + // None)))), + // ('returns', + // ((('name', ''), ('type', 'Tensor'), ('default_value', + // None)),))))) +} + +TEST(LiteInterpreterDirectTest, DefaultArgsTensorinvSpecifyDefault) { + // The second argument is specified, but the value is the same as the default + // value. It's treated as "not specified" since the value can be fetched from + // schema. + Module m("m"); + m.define(R"( + def forward(self, input): + return torch.linalg_tensorinv(input, 2) + )"); + torch::jit::MobileCode code(m.get_method("forward").graph(), "forward"); + auto arg_nums = code.op_to_num_specified_args(); + ASSERT_EQ(arg_nums.size(), 1); + ASSERT_EQ(arg_nums["aten::linalg_tensorinv"], 1); + std::vector inputs; + const int N = 4; + auto input = torch::rand({N, N, N, N}); + inputs.emplace_back(input); + testLiteModuleCompareResultTensors(m, inputs); +} + +void testDefaultArgsPinvWithOutArg2(int num_args) { + Module m("m"); + if (num_args == 1) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, out=input) + )"); + } else if (num_args == 2) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5, out=input) + )"); + } else if (num_args == 3) { + m.define(R"( + def forward(self, input): + return torch.linalg_pinv(input, 1e-5, True, out=input) + )"); + } + + const int N = 28; + auto input = torch::range(1, N * N, 1); + input[0] = 10000; // a more stable matrix + input = input.view({N, N}); + auto ref = m.run_method("forward", input); + TORCH_CHECK(!input.equal(torch::range(1, N * N, 1))); + TORCH_CHECK(input.equal(ref.toTensor())); +} + +TEST(LiteInterpreterDirectTest, DefaultArgsPinvWithOutArg) { + // Test with different number of specified arguments + out arg. + // Arguments not specified take default value. + for (int num_args = 1; num_args <= 3; ++num_args) { + testDefaultArgsPinvWithOutArg2(num_args); + } +} + +TEST(LiteInterpreterDirectTest, DefaultArgsWithOutArg) { + Module m("m"); + m.define(R"( + def forward(self, x, h): + torch.add(x, h, out=x) + )"); + + std::vector inputs; + auto input_x = 2 * torch::ones({}); + auto input_h = torch::ones({}); + auto ref = m.run_method("forward", input_x, input_h); + + CompilationOptions options; + mobile::Module bc = jitModuleToMobile(m, options); + bc.run_method("forward", input_x, input_h); + AT_ASSERT(input_x.equal(4 * torch::ones({}))); +} + +TEST(LiteInterpreterDirectTest, TestExceptionStackWithTwoLevelModuleHierarchy) { + Module a("A"); + a.define(R"( + def bar(self, x, y): + return x + y + )"); + Module b("B"); + b.register_module("A0", a); + b.define(R"( + def foo(self, x, y): + return self.A0.bar(x, y) + 2 + )"); + Module c("C"); + c.register_module("B0", b); + c.define(R"( + def forward(self, x, y): + return self.B0.foo(x, y) + 3 + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({2, 4})); + inputs.emplace_back(torch::rand({13, 9})); + + CompilationOptions options; + auto lite_m = jitModuleToMobile(c, options); + std::string error_pattern = R"( + Module hierarchy:top(C)::.B0(B)::foo.A0(A)::bar.aten::add +Traceback of TorchScript (most recent call last): + File "", line 3, in + + def forward(self, x, y): + return self.B0.foo(x, y) + 3 + ~~~~~~~~~~~ <--- HERE + + File "", line 3, in foo + + def foo(self, x, y): + return self.A0.bar(x, y) + 2 + ~~~~~~~~~~~ <--- HERE + + File "", line 3, in bar + + def bar(self, x, y): + return x + y + ~~~~~ <--- HERE + )"; + ASSERT_THROWS_WITH_MESSAGE(lite_m.forward(inputs), error_pattern); +} +#endif // !defined(FB_XPLAT_BUILD) + +namespace { +static auto reg = + torch::class_( + "_TorchScriptTesting", + "_LiteInterpreterDirectTest") + .def(torch::init<>()) + .def("get", &TorchBindLiteInterpreterDirectTestStruct::get) + .def_pickle( + // __getattr__ + [](const c10::intrusive_ptr< + TorchBindLiteInterpreterDirectTestStruct>&) -> int64_t { + return 0; + }, + // __setattr__ + [](int64_t) { + return c10::make_intrusive< + TorchBindLiteInterpreterDirectTestStruct>(); + }); + +} // namespace + +TEST(LiteInterpreterDirectTest, OperatorCacheDifferentiatesDefaultArgs) { + // Create 3 methods: + // + // 1. forward() returns a tensor with dtype=torch.int64 (4) + // 2. forward2() returns a tensor with dtype=torch.float32 (6) + // 3. forward3() returns a tensor with dtype=torch.float32 but + // the dtype is inferred by the input tensor's dtype + // + // If caching works correctly, then the result from the full-jit + // module and the lite module will be the same. Otherwise, it + // will be different if we don't correctly ignore the cache + // entry for an operator that has a different number of + // arguments. + Module m("m"); + m.define(R"( + def forward(self): + ret1 = torch.new_empty(torch.zeros(10), [10], dtype=4) + return ret1.fill_(25) + )"); + m.define(R"( + def forward2(self): + ret1 = torch.new_empty(torch.zeros(10), [10], dtype=6) + return ret1.fill_(32.0) + )"); + m.define(R"( + def forward3(self): + ret1 = torch.new_empty(torch.zeros(10), [10]) + return ret1.fill_(12.0) + )"); + + std::vector inputs; + testLiteModuleCompareResultTensors(m, inputs, "forward"); + testLiteModuleCompareResultTensors(m, inputs, "forward2"); + testLiteModuleCompareResultTensors(m, inputs, "forward3"); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_lite_trainer.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_lite_trainer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ad71b2449e81ba1afb498c3ae431bc43115d9e5f --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_lite_trainer.cpp @@ -0,0 +1,392 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Tests go in torch::jit +namespace torch { +namespace jit { + +TEST(LiteTrainerTest, Params) { + Module m("m"); + m.register_parameter("foo", torch::ones({1}, at::requires_grad()), false); + m.define(R"( + def forward(self, x): + b = 1.0 + return self.foo * x + b + )"); + double learning_rate = 0.1, momentum = 0.1; + int n_epoc = 10; + // init: y = x + 1; + // target: y = 2 x + 1 + std::vector> trainData{ + {1 * torch::ones({1}), 3 * torch::ones({1})}, + }; + // Reference: Full jit + std::stringstream ms; + m.save(ms); + auto mm = load(ms); + // mm.train(); + std::vector<::at::Tensor> parameters; + for (auto parameter : mm.parameters()) { + parameters.emplace_back(parameter); + } + ::torch::optim::SGD optimizer( + parameters, ::torch::optim::SGDOptions(learning_rate).momentum(momentum)); + for (int epoc = 0; epoc < n_epoc; ++epoc) { + for (auto& data : trainData) { + auto source = data.first, targets = data.second; + optimizer.zero_grad(); + std::vector train_inputs{source}; + auto output = mm.forward(train_inputs).toTensor(); + auto loss = ::torch::l1_loss(output, targets); + loss.backward(); + optimizer.step(); + } + } + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + std::vector<::at::Tensor> bc_parameters = bc.parameters(); + ::torch::optim::SGD bc_optimizer( + bc_parameters, + ::torch::optim::SGDOptions(learning_rate).momentum(momentum)); + for (int epoc = 0; epoc < n_epoc; ++epoc) { + for (auto& data : trainData) { + auto source = data.first, targets = data.second; + bc_optimizer.zero_grad(); + std::vector train_inputs{source}; + auto output = bc.forward(train_inputs).toTensor(); + auto loss = ::torch::l1_loss(output, targets); + loss.backward(); + bc_optimizer.step(); + } + } + AT_ASSERT(parameters[0].item() == bc_parameters[0].item()); +} + +// TODO Re-enable these tests after parameters are correctly loaded on mobile +/* +TEST(MobileTest, NamedParameters) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add_it(self, x): + b = 4 + return self.foo + x + b + )"); + Module child("m2"); + child.register_parameter("foo", 4 * torch::ones({}), false); + child.register_parameter("bar", 4 * torch::ones({}), false); + m.register_module("child1", child); + m.register_module("child2", child.clone()); + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + + auto full_params = m.named_parameters(); + auto mobile_params = bc.named_parameters(); + AT_ASSERT(full_params.size() == mobile_params.size()); + for (const auto& e : full_params) { + AT_ASSERT(e.value.item().toInt() == + mobile_params[e.name].item().toInt()); + } +} + +TEST(MobileTest, SaveLoadParameters) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add_it(self, x): + b = 4 + return self.foo + x + b + )"); + Module child("m2"); + child.register_parameter("foo", 4 * torch::ones({}), false); + child.register_parameter("bar", 3 * torch::ones({}), false); + m.register_module("child1", child); + m.register_module("child2", child.clone()); + auto full_params = m.named_parameters(); + std::stringstream ss; + std::stringstream ss_data; + m._save_for_mobile(ss); + + // load mobile module, save mobile named parameters + mobile::Module bc = _load_for_mobile(ss); + _save_parameters(bc.named_parameters(), ss_data); + + // load back the named parameters, compare to full-jit Module's + auto mobile_params = _load_parameters(ss_data); + AT_ASSERT(full_params.size() == mobile_params.size()); + for (const auto& e : full_params) { + AT_ASSERT(e.value.item() == mobile_params[e.name].item()); + } +} +*/ + +TEST(MobileTest, SaveLoadParametersEmpty) { + Module m("m"); + m.define(R"( + def add_it(self, x): + b = 4 + return x + b + )"); + Module child("m2"); + m.register_module("child1", child); + m.register_module("child2", child.clone()); + std::stringstream ss; + std::stringstream ss_data; + m._save_for_mobile(ss); + + // load mobile module, save mobile named parameters + mobile::Module bc = _load_for_mobile(ss); + _save_parameters(bc.named_parameters(), ss_data); + + // load back the named parameters, test is empty + auto mobile_params = _load_parameters(ss_data); + AT_ASSERT(mobile_params.size() == 0); +} + +TEST(MobileTest, SaveParametersDefaultsToZip) { + // Save some empty parameters. + std::map empty_parameters; + std::stringstream ss_data; + _save_parameters(empty_parameters, ss_data); + + // Verify that parameters were serialized to a ZIP container. + EXPECT_GE(ss_data.str().size(), 4); + EXPECT_EQ(ss_data.str()[0], 'P'); + EXPECT_EQ(ss_data.str()[1], 'K'); + EXPECT_EQ(ss_data.str()[2], '\x03'); + EXPECT_EQ(std::move(ss_data).str()[3], '\x04'); +} + +TEST(MobileTest, SaveParametersCanUseFlatbuffer) { + // Save some empty parameters using flatbuffer. + std::map empty_parameters; + std::stringstream ss_data; + _save_parameters(empty_parameters, ss_data, /*use_flatbuffer=*/true); + + // Verify that parameters were serialized to a flatbuffer. The flatbuffer + // magic bytes should be at offsets 4..7. The first four bytes contain an + // offset to the actual flatbuffer data. + EXPECT_GE(ss_data.str().size(), 8); + EXPECT_EQ(ss_data.str()[4], 'P'); + EXPECT_EQ(ss_data.str()[5], 'T'); + EXPECT_EQ(ss_data.str()[6], 'M'); + EXPECT_EQ(std::move(ss_data).str()[7], 'F'); +} + +TEST(MobileTest, SaveLoadParametersUsingFlatbuffers) { + // Create some simple parameters to save. + std::map input_params; + input_params["four_by_ones"] = 4 * torch::ones({}); + input_params["three_by_ones"] = 3 * torch::ones({}); + + // Serialize them using flatbuffers. + std::stringstream data; + _save_parameters(input_params, data, /*use_flatbuffer=*/true); + + // The flatbuffer magic bytes should be at offsets 4..7. + EXPECT_EQ(data.str()[4], 'P'); + EXPECT_EQ(data.str()[5], 'T'); + EXPECT_EQ(data.str()[6], 'M'); + EXPECT_EQ(data.str()[7], 'F'); + + // Read them back and check that they survived the trip. + auto output_params = _load_parameters(data); + EXPECT_EQ(output_params.size(), 2); + { + auto four_by_ones = 4 * torch::ones({}); + EXPECT_EQ( + output_params["four_by_ones"].item(), four_by_ones.item()); + } + { + auto three_by_ones = 3 * torch::ones({}); + EXPECT_EQ( + output_params["three_by_ones"].item(), three_by_ones.item()); + } +} + +TEST(MobileTest, LoadParametersUnexpectedFormatShouldThrow) { + // Manually create some data that doesn't look like a ZIP or Flatbuffer file. + // Make sure it's longer than 8 bytes, since getFileFormat() needs that much + // data to detect the type. + std::stringstream bad_data; + bad_data << "abcd" + << "efgh" + << "ijkl"; + + // Loading parameters from it should throw an exception. + EXPECT_ANY_THROW(_load_parameters(bad_data)); +} + +TEST(MobileTest, LoadParametersEmptyDataShouldThrow) { + // Loading parameters from an empty data stream should throw an exception. + std::stringstream empty; + EXPECT_ANY_THROW(_load_parameters(empty)); +} + +TEST(MobileTest, LoadParametersMalformedFlatbuffer) { + // Manually create some data with Flatbuffer header. + std::stringstream bad_data; + bad_data << "PK\x03\x04PTMF\x00\x00" + << "*}NV\xb3\xfa\xdf\x00pa"; + + // Loading parameters from it should throw an exception. + ASSERT_THROWS_WITH_MESSAGE( + _load_parameters(bad_data), "Malformed Flatbuffer module"); +} + +TEST(LiteTrainerTest, SGD) { + Module m("m"); + m.register_parameter("foo", torch::ones({1}, at::requires_grad()), false); + m.define(R"( + def forward(self, x): + b = 1.0 + return self.foo * x + b + )"); + double learning_rate = 0.1, momentum = 0.1; + int n_epoc = 10; + // init: y = x + 1; + // target: y = 2 x + 1 + std::vector> trainData{ + {1 * torch::ones({1}), 3 * torch::ones({1})}, + }; + // Reference: Full jit and torch::optim::SGD + std::stringstream ms; + m.save(ms); + auto mm = load(ms); + std::vector<::at::Tensor> parameters; + for (auto parameter : mm.parameters()) { + parameters.emplace_back(parameter); + } + ::torch::optim::SGD optimizer( + parameters, ::torch::optim::SGDOptions(learning_rate).momentum(momentum)); + for (int epoc = 0; epoc < n_epoc; ++epoc) { + for (auto& data : trainData) { + auto source = data.first, targets = data.second; + optimizer.zero_grad(); + std::vector train_inputs{source}; + auto output = mm.forward(train_inputs).toTensor(); + auto loss = ::torch::l1_loss(output, targets); + loss.backward(); + optimizer.step(); + } + } + // Test: lite interpreter and torch::jit::mobile::SGD + std::stringstream ss; + m._save_for_mobile(ss); + mobile::Module bc = _load_for_mobile(ss); + std::vector<::at::Tensor> bc_parameters = bc.parameters(); + ::torch::jit::mobile::SGD bc_optimizer( + bc_parameters, + ::torch::jit::mobile::SGDOptions(learning_rate).momentum(momentum)); + for (int epoc = 0; epoc < n_epoc; ++epoc) { + for (auto& data : trainData) { + auto source = data.first, targets = data.second; + bc_optimizer.zero_grad(); + std::vector train_inputs{source}; + auto output = bc.forward(train_inputs).toTensor(); + auto loss = ::torch::l1_loss(output, targets); + loss.backward(); + bc_optimizer.step(); + } + } + AT_ASSERT(parameters[0].item() == bc_parameters[0].item()); +} + +namespace { +struct DummyDataset : torch::data::datasets::Dataset { + explicit DummyDataset(size_t size = 100) : size_(size) {} + + int get(size_t index) override { + // NOLINTNEXTLINE(bugprone-narrowing-conversions,cppcoreguidelines-narrowing-conversions) + return 1 + index; + } + std::optional size() const override { + return size_; + } + + size_t size_; +}; +} // namespace + +TEST(LiteTrainerTest, SequentialSampler) { + // test that sampler can be used with dataloader + const int kBatchSize = 10; + auto data_loader = torch::data::make_data_loader( + DummyDataset(25), kBatchSize); + int i = 1; + for (const auto& batch : *data_loader) { + for (const auto& example : batch) { + AT_ASSERT(i == example); + i++; + } + } +} + +TEST(LiteTrainerTest, RandomSamplerReturnsIndicesInCorrectRange) { + mobile::RandomSampler sampler(10); + + std::vector indices = sampler.next(3).value(); + for (auto i : indices) { + AT_ASSERT(i < 10); + } + + indices = sampler.next(5).value(); + for (auto i : indices) { + AT_ASSERT(i < 10); + } + + indices = sampler.next(2).value(); + for (auto i : indices) { + AT_ASSERT(i < 10); + } + + AT_ASSERT(sampler.next(10).has_value() == false); +} + +TEST(LiteTrainerTest, RandomSamplerReturnsLessValuesForLastBatch) { + mobile::RandomSampler sampler(5); + AT_ASSERT(sampler.next(3).value().size() == 3); + AT_ASSERT(sampler.next(100).value().size() == 2); + AT_ASSERT(sampler.next(2).has_value() == false); +} + +TEST(LiteTrainerTest, RandomSamplerResetsWell) { + mobile::RandomSampler sampler(5); + AT_ASSERT(sampler.next(5).value().size() == 5); + AT_ASSERT(sampler.next(2).has_value() == false); + sampler.reset(); + AT_ASSERT(sampler.next(5).value().size() == 5); + AT_ASSERT(sampler.next(2).has_value() == false); +} + +TEST(LiteTrainerTest, RandomSamplerResetsWithNewSizeWell) { + mobile::RandomSampler sampler(5); + AT_ASSERT(sampler.next(5).value().size() == 5); + AT_ASSERT(sampler.next(2).has_value() == false); + sampler.reset(7); + AT_ASSERT(sampler.next(7).value().size() == 7); + AT_ASSERT(sampler.next(2).has_value() == false); + sampler.reset(3); + AT_ASSERT(sampler.next(3).value().size() == 3); + AT_ASSERT(sampler.next(2).has_value() == false); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_load_upgraders.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_load_upgraders.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e40c1219c7bf20ccc44cda67d920fbfc975536dc --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_load_upgraders.cpp @@ -0,0 +1,43 @@ +#include +#include +#include +#include +#include +#include + +#include + +namespace torch { +namespace jit { + +// Basic tests to check if C++ torch::jit::load +// can load the upgraders fine +// TODO (tugsuu) add more tests +TEST(UpgraderLoad, CanPopulateUpgradersGraph) { + Module m("m"); + m.define(R"( + def forward(self, x: Tensor): + b = 5 + return torch.div(x, b) + )"); + std::stringstream ms; + m.save(ms); + auto loaded_m = torch::jit::load(ms); + auto version_map = get_operator_version_map(); + auto upgraders = dump_upgraders_map(); + + for (const auto& entry : version_map) { + auto list_of_upgraders_for_op = entry.second; + for (const auto& upgrader_entry : list_of_upgraders_for_op) { + EXPECT_TRUE( + upgraders.find(upgrader_entry.upgrader_name) != upgraders.end()); + } + } + + auto test_graph = loaded_m.get_method("forward").graph(); + // should have saved with version 4, so it is still up to date + testing::FileCheck().check_count("aten::div", 1, true)->run(*test_graph); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_memory_dag.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_memory_dag.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b7ef07e221f659619300591228250727fc5774e2 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_memory_dag.cpp @@ -0,0 +1,138 @@ +#include + +#include +#include + +namespace torch { +namespace jit { + +TEST(MemoryDAGTest, Basic) { + auto graph = std::make_shared(); + const Value* aValue = graph->addInput(); + const Value* bValue = graph->addInput(); + const Value* cValue = graph->addInput(); + const Value* dValue = graph->addInput(); + const Value* eValue = graph->addInput(); + const Value* fValue = graph->addInput(); + const Value* gValue = graph->addInput(); + + { + // a <- b <- c + // b <- d + // a <- e + // f <- e + // g is by itself + auto t = std::make_unique(); + auto a = t->makeFreshValue(aValue); + auto b = t->makeFreshValue(bValue); + auto c = t->makeFreshValue(cValue); + auto d = t->makeFreshValue(dValue); + auto e = t->makeFreshValue(eValue); + auto f = t->makeFreshValue(fValue); + auto g = t->makeFreshValue(gValue); + t->makePointerTo(b, a); + t->makePointerTo(c, b); + t->makePointerTo(d, b); + t->makePointerTo(e, a); + t->makePointerTo(e, f); + + auto dag = std::move(*t).createMemoryDAG(); + + /** + * Test mayAlias() + */ + // Values should alias themselves + EXPECT_TRUE(dag->mayAlias(a, a)); + EXPECT_TRUE(dag->mayAlias(g, g)); + + // Values that point to the same location should alias + EXPECT_TRUE(dag->mayAlias(a, b)); + EXPECT_TRUE(dag->mayAlias(a, c)); + EXPECT_TRUE(dag->mayAlias(c, d)); + + // e may point to a OR f + EXPECT_TRUE(dag->mayAlias(e, a)); + EXPECT_TRUE(dag->mayAlias(e, f)); + // But a and f don't alias + EXPECT_FALSE(dag->mayAlias(a, f)); + } + { + // x(y) -> x contains y + + // b(a) + // c(a) + auto t = std::make_unique(); + auto a = t->makeFreshValue(aValue); + auto b = t->makeFreshValue(bValue); + t->addToContainedElements(a, b); + + auto c = t->makeFreshValue(cValue); + t->addToContainedElements(a, c); + + auto dag = std::move(*t).createMemoryDAG(); + EXPECT_TRUE(dag->mayContainAlias(a, b)); + EXPECT_TRUE(dag->mayContainAlias(b, a)); + + EXPECT_TRUE(dag->mayContainAlias(a, c)); + EXPECT_TRUE(dag->mayContainAlias(c, a)); + + EXPECT_TRUE(dag->mayContainAlias(b, c)); + EXPECT_TRUE(dag->mayContainAlias(c, b)); + + // containers contain an element in themselves + EXPECT_TRUE(dag->mayContainAlias(b, b)); + EXPECT_TRUE(dag->mayContainAlias(c, c)); + EXPECT_TRUE(dag->mayContainAlias(a, a)); + } + { + // b(a) + // c(a) + // d(b(a)) + auto t = std::make_unique(); + auto a = t->makeFreshValue(aValue); + auto b = t->makeFreshValue(bValue); + t->addToContainedElements(a, b); + + auto c = t->makeFreshValue(cValue); + t->addToContainedElements(a, c); + + auto d = t->makeFreshValue(dValue); + t->addToContainedElements(b, d); + + auto dag = std::move(*t).createMemoryDAG(); + EXPECT_TRUE(dag->mayContainAlias(b, d)); + EXPECT_TRUE(dag->mayContainAlias(d, b)); + + EXPECT_TRUE(dag->mayContainAlias(c, d)); + EXPECT_TRUE(dag->mayContainAlias(d, c)); + + EXPECT_TRUE(dag->mayContainAlias(a, d)); + } + { + // f(e) + auto t = std::make_unique(); + auto a = t->makeFreshValue(aValue); + auto b = t->makeFreshValue(bValue); + t->addToContainedElements(a, b); + + auto c = t->makeFreshValue(cValue); + t->addToContainedElements(a, c); + + auto d = t->makeFreshValue(dValue); + t->addToContainedElements(b, d); + + auto f = t->makeFreshValue(aValue); + auto e = t->makeFreshValue(bValue); + + t->addToContainedElements(f, e); + + auto dag = std::move(*t).createMemoryDAG(); + for (auto elem : {a, b, c, d}) { + EXPECT_FALSE(dag->mayContainAlias(f, elem)); + EXPECT_FALSE(dag->mayContainAlias(e, elem)); + } + } +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_misc.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_misc.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a5fc0485f888c299da363262f3229df2f19884e3 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_misc.cpp @@ -0,0 +1,3190 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { +inline c10::AliasAnalysisKind aliasAnalysisFromSchema() { + return c10::AliasAnalysisKind::FROM_SCHEMA; +} + +template +std::ostream& operator<<(std::ostream& out, const std::vector& list) { + size_t i = 0; + out << '{'; + for (auto&& e : list) { + if (i++ > 0) + out << ", "; + out << e; + } + out << '}'; + return out; +} + +TEST(InternedStringsTest, Basic) { + ASSERT_EQ(prim::Param, Symbol::prim("Param")); + ASSERT_EQ(prim::Return, Symbol::prim("Return")); + ASSERT_EQ(prim::Return.toUnqualString(), std::string("Return")); + ASSERT_EQ(prim::Return.toQualString(), std::string("prim::Return")); + Symbol newsym = Symbol::aten("__NEW_SYMBOL"); + size_t symstart = newsym; + ASSERT_EQ(newsym.toQualString(), std::string("aten::__NEW_SYMBOL")); + // TODO: This test is a bit too close to the implementation details. + ASSERT_EQ(Symbol::aten("What"), symstart + 1); + ASSERT_EQ(Symbol::aten("What2"), symstart + 2); + ASSERT_EQ(Symbol::aten("What"), symstart + 1); + ASSERT_EQ(Symbol::aten("What2"), symstart + 2); + ASSERT_EQ(Symbol(symstart + 2).toUnqualString(), std::string("What2")); +} + +TEST(FromQualStringTest, Basic) { + ASSERT_EQ(Symbol::fromQualString("prim::Param"), Symbol::prim("Param")); + ASSERT_EQ(Symbol::fromQualString("aten::mm"), Symbol::aten("mm")); + ASSERT_EQ(Symbol::fromQualString("onnx::LSTM"), Symbol::onnx("LSTM")); + ASSERT_EQ(Symbol::fromQualString("attr::value"), Symbol::attr("value")); + ASSERT_EQ(Symbol::fromQualString("scope::"), Symbol::scope("")); + ASSERT_EQ(Symbol::fromQualString("::").toUnqualString(), std::string("")); + ASSERT_EQ( + Symbol::fromQualString("::").ns().toQualString(), + std::string("namespaces::")); + ASSERT_EQ( + Symbol::fromQualString("new_ns::param").toUnqualString(), + std::string("param")); + ASSERT_EQ( + Symbol::fromQualString("new_ns::param").ns().toUnqualString(), + std::string("new_ns")); + ASSERT_EQ( + Symbol::fromQualString("new_ns::param").ns(), + Symbol::fromQualString("namespaces::new_ns")); + + auto bad_inputs = {"scope", ":", ""}; + for (auto input : bad_inputs) { + try { + Symbol::fromQualString(input); + ASSERT_TRUE(0); + } catch (const std::exception&) { + } + } +} + +TEST(THNNConvTest, Basic) { + std::vector input_size = {4, 3, 15, 17}; // B x C x H x W + std::vector kernel_size = {3, 5}; + std::vector stride = {1, 2}; + std::vector padding = {2, 1}; + constexpr int out_channels = 5; + + // make inputs + at::Tensor input = torch::randn(input_size); + at::Tensor weight = torch::randn( + {out_channels, input_size[1], kernel_size[0], kernel_size[1]}); + at::Tensor bias = torch::randn({out_channels}); + + // run forward eagerly + at::Tensor output = at::_slow_conv2d_forward( + input, weight, kernel_size, bias, stride, padding); + + // make grad_outputs + at::Tensor grad_output = + torch::randn_like(output, at::MemoryFormat::Preserve); + + // run backward eagerly + auto [grad_input, grad_weight, grad_bias] = at::_slow_conv2d_backward( + grad_output, + input, + weight, + kernel_size, + stride, + padding, + {true, true, true}); + + // make JIT graph + auto graph = std::make_shared(); + auto ksz_val = graph->insertConstant(kernel_size); + auto kst_val = graph->insertConstant(stride); + auto pad_val = graph->insertConstant(padding); + + auto inputg = graph->addInput("self"); + auto weightg = graph->addInput("weight"); + auto biasg = graph->addInput("bias"); + + Value* conv = graph->insert( + aten::_slow_conv2d_forward, + {inputg, weightg, ksz_val, biasg, kst_val, pad_val}); + auto outputs = conv->node()->outputs(); + for (auto output : outputs) { + graph->registerOutput(output); + } + LowerAllTuples(graph); + graph->lint(); + + // differentiate JIT graph + EliminateDeadCode(graph); // Tracing of some ops depends on the DCE trick + ConstantPropagation(graph); + auto grad_spec = differentiate(graph); + LowerGradOf(*grad_spec.df); + + // prepare JIT inputs / gradients + tensor_list tensors_in; + tensors_in.push_back(input); + tensors_in.push_back(weight); + tensors_in.push_back(bias); + + tensor_list tensor_grads_in; + tensor_grads_in.push_back(grad_output); + + // Get outputs from the interpreter + auto [tensors_out, tensor_grads_out] = + runGradient(grad_spec, tensors_in, tensor_grads_in); + + // prepare expected structs + tensor_list expected_tensors_out, expected_tensor_grads_out; + expected_tensors_out.push_back(output); + expected_tensor_grads_out.push_back(grad_input); + expected_tensor_grads_out.push_back(grad_weight); + expected_tensor_grads_out.push_back(grad_bias); + + // Compare results + assertAllClose(tensors_out, expected_tensors_out); + assertAllClose(tensor_grads_out, expected_tensor_grads_out); +} + +TEST(ATenNativeBatchNormTest, Basic) { + // aten::native_batch_norm(Tensor input, Tensor weight, Tensor bias, Tensor + // running_mean, Tensor running_var, bool training, float momentum, float eps) + // -> (Tensor, Tensor, Tensor) + std::vector input_size = {4, 3, 15, 17}; // B x C x H x W + bool training = true; + float momentum = 0.9; + float eps = 1e-5; + + // make inputs + at::Tensor input = torch::randn(input_size); + at::Tensor weight = torch::randn({input_size[1]}); + at::Tensor bias = torch::randn({input_size[1]}); + at::Tensor running_mean = torch::randn({input_size[1]}); + at::Tensor running_var = torch::randn({input_size[1]}); + + // running_mean and running_var are changed in-place, so clone and send them + at::Tensor running_mean_eager = running_mean.clone(); + at::Tensor running_var_eager = running_var.clone(); + at::Tensor running_mean_jit = running_mean.clone(); + at::Tensor running_var_jit = running_var.clone(); + + // run forward eagerly + auto [output, savemean, saveinvstd] = at::native_batch_norm( + input, + weight, + bias, + running_mean_eager, + running_var_eager, + training, + momentum, + eps); + + // make grad_outputs + at::Tensor grad_output = + torch::randn_like(output, at::MemoryFormat::Preserve); + at::Tensor grad_savemean = + torch::zeros_like(savemean, at::MemoryFormat::Preserve); + at::Tensor grad_saveinvstd = + torch::zeros_like(saveinvstd, at::MemoryFormat::Preserve); + + // run backward eagerly + // aten::native_batch_norm_backward(Tensor grad_out, Tensor input, Tensor + // weight, Tensor running_mean, Tensor running_var, Tensor save_mean, Tensor + // save_invstd, bool train, float eps, bool[3] output_mask) -> (Tensor, + // Tensor, Tensor) + auto [grad_input, grad_weight, grad_bias] = at::native_batch_norm_backward( + grad_output, + input, + weight, + running_mean_eager, + running_var_eager, + savemean, + saveinvstd, + training, + eps, + {true, true, true}); + + // make JIT graph + auto graph = std::make_shared(); + auto training_val = graph->insertConstant(IValue(training)); + auto momentum_val = graph->insertConstant(IValue(momentum)); + auto eps_val = graph->insertConstant(IValue(eps)); + + auto inputg = graph->addInput("self"); + auto weightg = graph->addInput("weight"); + auto biasg = graph->addInput("bias"); + auto running_meang = graph->addInput("running_mean"); + auto running_varg = graph->addInput("running_var"); + + Value* bn = graph->insert( + aten::native_batch_norm, + {inputg, + weightg, + biasg, + running_meang, + running_varg, + training_val, + momentum_val, + eps_val}); + auto outputs = bn->node()->outputs(); + for (auto output : outputs) { + graph->registerOutput(output); + } + LowerAllTuples(graph); + graph->lint(); + + // differentiate JIT graph + EliminateDeadCode(graph); // Tracing of some ops depends on the DCE trick + ConstantPropagation(graph); + auto grad_spec = differentiate(graph); + LowerGradOf(*grad_spec.df); + + // prepare JIT inputs / gradients + tensor_list tensors_in; + tensors_in.push_back(input); + tensors_in.push_back(weight); + tensors_in.push_back(bias); + tensors_in.push_back(running_mean_jit); + tensors_in.push_back(running_var_jit); + + tensor_list tensor_grads_in; + tensor_grads_in.push_back(grad_output); + tensor_grads_in.push_back(grad_savemean); + tensor_grads_in.push_back(grad_saveinvstd); + + // Get outputs from the interpreter + auto [tensors_out, tensor_grads_out] = + runGradient(grad_spec, tensors_in, tensor_grads_in); + + // prepare expected structs + tensor_list expected_tensors_out, expected_tensor_grads_out; + expected_tensors_out.push_back(output); + expected_tensors_out.push_back(savemean); + expected_tensors_out.push_back(saveinvstd); + expected_tensors_out.push_back(running_mean_eager); + expected_tensors_out.push_back(running_var_eager); + expected_tensor_grads_out.push_back(grad_input); + expected_tensor_grads_out.push_back(grad_weight); + expected_tensor_grads_out.push_back(grad_bias); + + tensors_out.push_back(running_mean_jit); + tensors_out.push_back(running_var_jit); + + // Compare results + assertAllClose(tensors_out, expected_tensors_out); + assertAllClose(tensor_grads_out, expected_tensor_grads_out); +} + +TEST(CustomFusionTest, Basic) { +#if defined(FBCODE_CAFFE2) + return; +#endif + + auto graph_string = R"IR( + graph(%0 : Float(2, 3, 4), + %1 : Float(2, 3, 4)): + %2 : Tensor = aten::mul(%0, %1) + %3 : Tensor = aten::mul(%2, %0) + return (%3))IR"; + auto g = std::make_shared(); + torch::jit::parseIR(graph_string, g.get()); + + torch::jit::overrideCanFuseOnCPU(true); + CustomFuseGraph( + g, + [](Node* n) { return n->kind() != prim::Param; }, + Symbol::fromQualString("prim::FusionGroup")); + torch::jit::overrideCanFuseOnCPU(false); + + const auto& nodes = g->nodes(); + auto fusion_group = + std::find_if(nodes.begin(), nodes.end(), [](const Node* node) { + return node->kind() == Symbol::fromQualString("prim::FusionGroup"); + }); + AT_ASSERT(fusion_group != nodes.end()); + + auto subgraph = fusion_group->g(attr::Subgraph); + auto hits = 0; + // two multiplications + for (const auto& n : subgraph->nodes()) { + (void)n; + hits++; + } + AT_ASSERT(hits == 2); +} + +TEST(CustomFusionTest, NestedBlocks) { +#if defined(FBCODE_CAFFE2) + return; +#endif + + auto graph_string = R"IR( + graph(%0 : Float(2, 3, 4), + %1 : Float(2, 3, 4), + %2 : Float(2, 3, 4)): + %3 : int = prim::Constant[value=1]() + %4 : Tensor = prim::If(%2) + block0(): + %5 : Tensor = aten::mul(%0, %2) + %6 : Tensor = aten::mul(%5, %1) + -> (%6) + block1(): + %7 : Tensor = aten::add(%0, %2, %3) + %8 : Tensor = aten::add(%7, %1, %3) + -> (%8) + %9 : Tensor = aten::add(%4, %2, %3) + return (%4))IR"; + auto g = std::make_shared(); + torch::jit::parseIR(graph_string, g.get()); + + CustomFuseGraph( + g, + [](Node* n) { return n->kind() == aten::mul; }, + Symbol::fromQualString("prim::FusionGroup")); + + // Could be done in more efficient ways, but this is only a test. + std::function dfs = [&](const Block* b, + Symbol s) { + for (auto node : b->nodes()) { + if (node->kind() == s) + return true; + for (auto nested_b : node->blocks()) + if (dfs(nested_b, s)) + return true; + } + return false; + }; + + AT_ASSERT(dfs(g->block(), Symbol::fromQualString("prim::FusionGroup"))); +} + +static const auto cf_examples = R"JIT( + def if_test(a, b): + # FIXME: use 0 instead of a. + # c = 0 + c = a + if bool(a < b): + c = b + else: + c = a + return c + def if_one(a, b): + c = b + if bool(a < b): + c = a + return c + def while_test(a, i): + while bool(i < 3): + a *= a + i += 1 + return a +)JIT"; + +TEST(ControlFlowTest, Basic) { + auto cu = compile(cf_examples); + + auto run = [&](const std::string& name, std::vector stack) { + auto graph = toGraphFunction(cu->get_function(name)).graph(); + Code code(graph, ""); + InterpreterState interp(code); + interp.run(stack); + return stack; + }; + + auto L = [](int64_t l) { return IValue(scalar_to_tensor(at::Scalar(l))); }; + auto V = [](IValue t) { return std::move(t).toTensor().item(); }; + auto run_binary = [&](const std::string& name, int64_t a, int64_t b) { + return V(run(name, {L(a), L(b)})[0]); + }; + ASSERT_EQ(2, run_binary("if_test", 1, 2)); + ASSERT_EQ(3, run_binary("if_test", 3, 2)); + ASSERT_EQ(2, run_binary("if_one", 2, 3)); + ASSERT_EQ(2, run_binary("if_one", 3, 2)); + ASSERT_EQ(256, run_binary("while_test", 2, 0)); +} + +#if !(C10_ASAN_ENABLED || C10_UBSAN_ENABLED) +// This test fails vptr UBSAN checks + +TEST(ProtoTest, Basic) { + ::ONNX_NAMESPACE::ModelProto proto; + proto.set_producer_name("foo"); +} +#endif + +// test a few features that are not directly used in schemas yet +TEST(SchemaParserTest, NestedArrays) { + // nested arrays + auto s = parseSchema("at::what(int[][4] foo) -> ()"); + ASSERT_TRUE(s.arguments().at(0).N() == 4); + ASSERT_TRUE(IntType::get()->isSubtypeOf(*s.arguments() + .at(0) + .type() + ->expectRef() + .getElementType() + ->expectRef() + .getElementType())); + auto s2 = parseSchema("at::what(int[][] foo) -> ()"); + ASSERT_TRUE(IntType::get()->isSubtypeOf(*s2.arguments() + .at(0) + .type() + ->expectRef() + .getElementType() + ->expectRef() + .getElementType())); +} + +TEST(SchemaParserTest, OutVariant) { + auto schema_with_out = parseSchema( + "at::foo(Tensor self, *, Tensor(a!) f, Tensor(b!) l) -> (Tensor(a!) f, Tensor(b!) l)"); + ASSERT_TRUE(schema_with_out.arguments().at(1).is_out()); + ASSERT_TRUE(schema_with_out.arguments().at(2).is_out()); + + auto schema_without_out = + parseSchema("at::foo(Tensor self, *, int scalar) -> (int)"); + + for (const auto& arg : schema_without_out.arguments()) { + ASSERT_TRUE(!arg.is_out()); + } + + auto schema_with_is_write = parseSchema( + "aten::ne_.Scalar(Tensor(a!) self, Scalar other) -> (Tensor(a!))"); + + for (const auto& arg : schema_with_is_write.arguments()) { + ASSERT_TRUE(!arg.is_out()); + } +} + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +TEST(SchemaParserTest, NamedReturns) { + // named returns + parseSchema("at::what(Tensor! i_will_be_written_to) -> ()"); + auto s3 = + parseSchema("at::what() -> (Tensor the_return, Tensor the_return2)"); + ASSERT_TRUE(s3.returns().at(0).name() == "the_return"); + ASSERT_TRUE(s3.returns().at(1).name() == "the_return2"); +} + +TEST(SchemaParserTest, Futures) { + // futures + auto s4 = parseSchema("at::what(Future(int) foo) -> ()"); + ASSERT_TRUE(IntType::get()->isSubtypeOf( + *s4.arguments().at(0).type()->expectRef().getElementType())); +} + +TEST(SchemaParserTest, AnnotatedAliasSets) { + // test tensor with annotated alias sets + parseSchema("at::what(Tensor(a) foo) -> (Tensor(a))"); +} + +TEST(SchemaParserTest, TensorListAnnotatedAliasSets) { + const auto s = parseSchema( + "at::foo(Tensor(a!) self, Tensor(b!)[] out)" + " -> ()"); + const AliasInfo* selfAliasInfo = s.arguments().at(0).alias_info(); + const AliasInfo* outAliasInfo = s.arguments().at(1).alias_info(); + ASSERT_TRUE( + selfAliasInfo->beforeSets() == + std::unordered_set{Symbol::fromQualString("alias::a")}); + ASSERT_TRUE(selfAliasInfo->isWrite()); + + ASSERT_TRUE(outAliasInfo->isWrite()); + ASSERT_TRUE(outAliasInfo->beforeSets().empty()); + ASSERT_EQ(outAliasInfo->containedTypes().size(), 1); + + auto containedType = outAliasInfo->containedTypes()[0]; + + ASSERT_TRUE(containedType.isWrite()); + ASSERT_TRUE( + containedType.beforeSets() == + std::unordered_set{Symbol::fromQualString("alias::b")}); +} + +TEST(SchemaParserTest, AnnotatedAliasWithoutBeforeSet) { + EXPECT_THAT( + []() { parseSchema("at::foo(Tensor(!) self) -> Tensor"); }, + ::testing::Throws(::testing::Property( + &std::runtime_error::what, + ::testing::HasSubstr("expected ident but found '!' here")))); +} + +TEST(SchemaParserTest, BeforeAfterSets) { + const auto s = parseSchema( + "at::what(Tensor(b|c)[](a!) list, Tensor(c) element)" + " -> (Tensor(b|c)[](a!))"); + + // The list itself is annotated with `a` + const AliasInfo* aliasInfo = s.arguments().at(0).alias_info(); + ASSERT_NE(aliasInfo, nullptr); + ASSERT_TRUE( + aliasInfo->beforeSets() == + std::unordered_set{Symbol::fromQualString("alias::a")}); + ASSERT_TRUE(aliasInfo->isWrite()); + + // Check the contained types + ASSERT_TRUE(!aliasInfo->containedTypes().empty()); + const auto& containedAliasInfo = aliasInfo->containedTypes()[0]; + const auto expected = std::unordered_set{ + Symbol::fromQualString("alias::b"), + Symbol::fromQualString("alias::c"), + }; + ASSERT_TRUE(containedAliasInfo.beforeSets() == expected); + ASSERT_TRUE(containedAliasInfo.afterSets() == expected); + ASSERT_FALSE(containedAliasInfo.isWrite()); +} + +TEST(SchemaParserTest, BeforeAfterSets2) { + const auto s = parseSchema( + "at::what(Tensor(b -> b|c)[](a!) list, Tensor(c) element)" + " -> (Tensor(b|c)[](a!))"); + + // The list itself is annotated with `a` + const AliasInfo* aliasInfo = s.arguments().at(0).alias_info(); + ASSERT_NE(aliasInfo, nullptr); + ASSERT_EQ( + aliasInfo->beforeSets(), + std::unordered_set{Symbol::fromQualString("alias::a")}); + ASSERT_EQ( + aliasInfo->afterSets(), + std::unordered_set{Symbol::fromQualString("alias::a")}); + ASSERT_TRUE(aliasInfo->isWrite()); + ASSERT_EQ(aliasInfo->containedTypes().size(), 1); + + // Check the contained types + ASSERT_TRUE(!aliasInfo->containedTypes().empty()); + const auto& containedAliasInfo = aliasInfo->containedTypes()[0]; + const auto expectedBefore = std::unordered_set{ + Symbol::fromQualString("alias::b"), + }; + const auto expectedAfter = std::unordered_set{ + Symbol::fromQualString("alias::b"), Symbol::fromQualString("alias::c")}; + ASSERT_TRUE(containedAliasInfo.beforeSets() == expectedBefore); + ASSERT_TRUE(containedAliasInfo.afterSets() == expectedAfter); + ASSERT_FALSE(containedAliasInfo.isWrite()); +} + +TEST(TopologicalIndexTest, Basic) { + Graph graph; + auto node1 = graph.create(prim::AutogradZero); + auto node2 = graph.create(prim::AutogradZero); + auto node3 = graph.create(prim::AutogradZero); + auto node4 = graph.create(prim::AutogradZero); + + graph.appendNode(node4); + graph.prependNode(node1); + node2->insertAfter(node1); + node3->insertBefore(node4); + + // nodes should be in numerical order + ASSERT_TRUE(node1->isBefore(node2)); + ASSERT_TRUE(node1->isBefore(node3)); + ASSERT_TRUE(node1->isBefore(node4)); + ASSERT_TRUE(node2->isAfter(node1)); + ASSERT_TRUE(node2->isBefore(node3)); + ASSERT_TRUE(node2->isBefore(node4)); + ASSERT_FALSE(node3->isBefore(node1)); + ASSERT_FALSE(node3->isBefore(node2)); + ASSERT_FALSE(node3->isAfter(node4)); + + // Built up a block structure + // node3 + // /\ ... + // A B block1 + // \ ... + // C block2 + auto block1 = node3->addBlock(); + auto A = graph.create(prim::AutogradZero); + block1->appendNode(A); + auto B = graph.create(prim::AutogradZero); + block1->appendNode(B); + auto block2 = B->addBlock(); + auto C = graph.create(prim::AutogradZero); + block2->appendNode(C); + + // Check isAfter on different block levels + ASSERT_TRUE(node1->isBefore(A)); + ASSERT_TRUE(A->isBefore(B)); + ASSERT_TRUE(A->isBefore(C)); + + // make sure things don't blow up on deletions + node2->destroy(); + auto node2p = graph.create(prim::AutogradZero); + node2p->insertAfter(node1); + ASSERT_TRUE(node1->isBefore(node2p)); + ASSERT_TRUE(node2p->isBefore(node3)); +} + +TEST(TopologicalIndexTest, Reindex) { + // Induce reindexing to test that path + Graph graph; + std::map nodes; + + auto anchor = graph.create(prim::AutogradZero); + graph.appendNode(anchor); + // Inserting to the same place a lot will trigger reindexing + for (auto i = 0; i < 100; ++i) { + auto n = graph.create(prim::AutogradZero); + n->insertAfter(anchor); + nodes[i] = n; + } + + // Nodes should be in reverse order + for (auto i = 0; i < 100; ++i) { + for (auto j = i + 1; j < 100; ++j) { + ASSERT_TRUE(nodes[i]->isAfter(nodes[j])); + } + } +} + +at::Tensor invokeTestRecordFunction(at::Tensor& t) { + RECORD_FUNCTION("test", std::vector({t})); + + auto t2 = t.pow(2); + return t2; +} + +static const auto invokeTestRecordFunction_JIT = R"JIT( + def foo(self, t): + t2 = t.pow(2) + return t2 + + def forward(self, t): + return self.foo(t) +)JIT"; + +at::Tensor invokeTestRecordFunctionJIT(at::Tensor& t) { + RECORD_FUNCTION("test", std::vector({t})); + + auto module = std::make_shared( + "RecordFunctionTestModule", std::make_shared()); + module->define(invokeTestRecordFunction_JIT); + return module->forward({t}).toTensor(); +} + +using TracedTestValues = + std::vector>>>; + +void checkTracedInputs(const TracedTestValues& inputs) { + bool found_test = false; + bool found_pow = false; + bool found_mul = false; + for (const auto& input : inputs) { + const auto& fn = std::get<0>(input); + const auto& sizes = std::get<1>(input); + + if (fn == "test") { + found_test = true; + TORCH_CHECK(sizes.size() == 1); + TORCH_CHECK(sizes[0] == std::vector({1, 2, 3})); + } else if (fn == "aten::pow") { + found_pow = true; + TORCH_CHECK(sizes.size() == 2); + TORCH_CHECK(sizes[0] == std::vector({1, 2, 3})); + TORCH_CHECK(sizes[1].empty()); + } else if (fn == "aten::mul") { + found_mul = true; + TORCH_CHECK(sizes.size() > 1); + TORCH_CHECK(sizes[0] == std::vector({1, 2, 3})); + } + } + TORCH_CHECK(found_test); + TORCH_CHECK(found_pow); + TORCH_CHECK(found_mul); +} + +void checkTracedOutputs(const TracedTestValues& outputs) { + bool found_test = false; + bool found_pow = false; + bool found_mul = false; + for (const auto& output : outputs) { + const auto& fn = std::get<0>(output); + const auto& sizes = std::get<1>(output); + + if (fn == "test") { + found_test = true; + TORCH_CHECK(sizes.empty()); + } else if (fn == "aten::pow") { + found_pow = true; + TORCH_CHECK(sizes.size() == 1); + TORCH_CHECK(sizes[0] == std::vector({1, 2, 3})); + } else if (fn == "aten::mul") { + found_mul = true; + TORCH_CHECK(sizes.size() == 1); + TORCH_CHECK(sizes[0] == std::vector({1, 2, 3})); + } + } + TORCH_CHECK(found_test); + TORCH_CHECK(found_pow); + TORCH_CHECK(found_mul); +} + +static bool bad_scope = false; +template +std::unique_ptr checkScopeCallback( + const at::RecordFunction& fn) { + if (fn.scope() == scope) { + ++(*cnt); + } else { + bad_scope = true; + } + return nullptr; +} + +template +void pushScopedCallback() { + at::addGlobalCallback( + at::RecordFunctionCallback(checkScopeCallback) + .scopes({scope})); +} + +// These cannot be function-local because that would prohibit them +// from being used as template arguments prior to C++17. +static size_t fun_cnt; +static size_t ts_fun_cnt; +static size_t user_scope_cnt; + +void checkScopeCallbacks() { + static bool found_function_scope; + static bool found_method_scope; + static bool found_user_scope; + found_function_scope = false; + found_method_scope = false; + found_user_scope = false; + at::addGlobalCallback(at::RecordFunctionCallback( + [](const at::RecordFunction& fn) -> std::unique_ptr { + if (fn.scope() == at::RecordScope::FUNCTION && + std::string(fn.name()) == "test_function") { + found_function_scope = true; + } + if (fn.scope() == at::RecordScope::TORCHSCRIPT_FUNCTION && + std::string(fn.name()) == "test_method") { + found_method_scope = true; + } + if (fn.scope() == at::RecordScope::USER_SCOPE && + std::string(fn.name()) == "test_user_scope") { + found_user_scope = true; + } + return nullptr; + })); + + bad_scope = false; + fun_cnt = 0; + pushScopedCallback(); + ts_fun_cnt = 0; + pushScopedCallback(); + user_scope_cnt = 0; + pushScopedCallback(); + + TORCH_CHECK(at::hasCallbacks()); + + { + RECORD_TORCHSCRIPT_FUNCTION("test_method", {}); + { + RECORD_FUNCTION("test_function", {}); + } + { + RECORD_USER_SCOPE("test_user_scope"); + } + } + + TORCH_CHECK(!bad_scope); + TORCH_CHECK(fun_cnt == 1); + TORCH_CHECK(ts_fun_cnt == 1); + TORCH_CHECK(user_scope_cnt == 1); + + TORCH_CHECK(found_function_scope); + TORCH_CHECK(found_method_scope); + TORCH_CHECK(found_user_scope); +} + +static TracedTestValues traced_inputs; +static TracedTestValues traced_outputs; +static std::unordered_set ts_input_names; +static std::unordered_set ts_output_names; + +std::unique_ptr tracedInputsCallback( + const RecordFunction& fn) { + if (fn.scope() == RecordScope::FUNCTION) { + auto inputs = fn.inputs(); + std::vector> sizes; + for (const auto& input : inputs) { + if (input.isTensor()) { + sizes.push_back(input.toTensor().sizes().vec()); + } else if (input.isScalar()) { + // NOLINTNEXTLINE(modernize-use-emplace) + sizes.push_back(std::vector()); + } + } + traced_inputs.push_back(std::make_tuple(fn.name(), sizes)); + } else if (fn.scope() == RecordScope::TORCHSCRIPT_FUNCTION) { + ts_input_names.insert(fn.name()); + } + return nullptr; +} + +void tracedOutputsCallback(const RecordFunction& fn, ObserverContext* ctx_ptr) { + if (fn.scope() == RecordScope::FUNCTION) { + auto outputs = fn.outputs(); + std::vector> sizes; + for (const auto& output : outputs) { + if (output.isTensor()) { + sizes.push_back(output.toTensor().sizes().vec()); + } else if (output.isScalar()) { + sizes.emplace_back(); + } + } + traced_outputs.push_back(std::make_tuple(fn.name(), sizes)); + } else if (fn.scope() == RecordScope::TORCHSCRIPT_FUNCTION) { + ts_output_names.insert(fn.name()); + } +} + +TEST(RecordFunctionTest, TracedTestInputsOutputs) { + // disabling the inlining of method calls + GraphOptimizerEnabledGuard opt_guard(false); + + // [(fn, [[sizes], [sizes], ...]), ...] + addGlobalCallback( + RecordFunctionCallback(tracedInputsCallback, tracedOutputsCallback) + .needsInputs(true) + .needsOutputs(true)); + + TracedTestValues eager_inputs, eager_outputs, jit_inputs, jit_outputs; + { + auto t = torch::randn({1, 2, 3}, at::kCPU); + t.set_requires_grad(true); + auto t2 = invokeTestRecordFunction(t); + t2.backward(torch::ones_like(t2, at::MemoryFormat::Preserve)); + eager_inputs = traced_inputs; + eager_outputs = traced_outputs; + traced_inputs.clear(); + traced_outputs.clear(); + + TORCH_CHECK(ts_input_names.empty()); + TORCH_CHECK(ts_output_names.empty()); + + t = torch::randn({1, 2, 3}, at::kCPU); + t.set_requires_grad(true); + t2 = invokeTestRecordFunctionJIT(t); + t2.backward(torch::ones_like(t2, at::MemoryFormat::Preserve)); + jit_inputs = traced_inputs; + jit_outputs = traced_outputs; + traced_inputs.clear(); + traced_outputs.clear(); + } + + TORCH_CHECK(ts_input_names.find("forward") != ts_input_names.end()); + TORCH_CHECK(ts_input_names.find("foo") != ts_input_names.end()); + TORCH_CHECK(ts_output_names.find("forward") != ts_output_names.end()); + TORCH_CHECK(ts_output_names.find("foo") != ts_output_names.end()); + + checkTracedInputs(eager_inputs); + checkTracedOutputs(eager_outputs); + checkTracedInputs(jit_inputs); + checkTracedOutputs(jit_outputs); + at::clearCallbacks(); +} + +static int sampled_cb_ctr = 0; +std::unique_ptr sampledCallback(const RecordFunction& fn) { + if (std::string(fn.name()) == "test") { + ++sampled_cb_ctr; + } + return nullptr; +} + +static int non_sampled_cb_ctr = 0; +std::unique_ptr nonSampledCallback(const RecordFunction& fn) { + if (std::string(fn.name()) == "test") { + ++non_sampled_cb_ctr; + } + return nullptr; +} + +TEST(RecordFunctionTest, SampledCallbacks) { + // disabling the inlining of method calls + GraphOptimizerEnabledGuard opt_guard(false); + + // test sampled callbacks + sampled_cb_ctr = 0; + auto setup_sampled_callback = [](double sampling_prob) { + return addGlobalCallback( + RecordFunctionCallback(sampledCallback).samplingProb(sampling_prob)); + }; + + addGlobalCallback(RecordFunctionCallback(nonSampledCallback)); + + auto handle = setup_sampled_callback(0.5); + + auto run_test_function = []() { + auto t = torch::randn({1, 2, 3}, at::kCPU); + for (auto k = 0; k < 1000; k++) { + invokeTestRecordFunction(t); + } + }; + + run_test_function(); + TORCH_CHECK(non_sampled_cb_ctr == 1000); + TORCH_CHECK(sampled_cb_ctr > 0 && sampled_cb_ctr < 1000); + + sampled_cb_ctr = 0; + removeCallback(handle); + handle = setup_sampled_callback(0.0); + run_test_function(); + + TORCH_CHECK(non_sampled_cb_ctr == 2000); + TORCH_CHECK(sampled_cb_ctr == 0); + + sampled_cb_ctr = 0; + removeCallback(handle); + // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores) + handle = setup_sampled_callback(1.0); + run_test_function(); + + TORCH_CHECK(non_sampled_cb_ctr == 3000); + TORCH_CHECK(sampled_cb_ctr == 1000); + clearCallbacks(); + + // test the scope of the callbacks + checkScopeCallbacks(); + clearCallbacks(); +} + +TEST(RecordFunctionTest, RecordFunctionGuard) { + // disabling the inlining of method calls + GraphOptimizerEnabledGuard opt_guard(false); + + static std::vector fn_names; + static std::mutex guard_mtx; + + // check record function guard + addGlobalCallback(RecordFunctionCallback( + [](const RecordFunction& fn) -> std::unique_ptr { + std::lock_guard lock(guard_mtx); + // NOLINTNEXTLINE(modernize-use-emplace) + fn_names.push_back(fn.name()); + return nullptr; + })); + { + RecordFunctionGuard g1(false); + { + RECORD_USER_SCOPE("A"); + { + RecordFunctionGuard g2(true); + RECORD_USER_SCOPE("B"); + { + DisableRecordFunctionGuard g3; + RECORD_USER_SCOPE("C"); + } + } + { + RECORD_USER_SCOPE("D"); + } + } + } + TORCH_CHECK(fn_names.size() == 1); + TORCH_CHECK(fn_names[0] == "B"); + clearCallbacks(); +} + +static std::vector ids; + +template +auto add_remove_test_add_cb() { + return addGlobalCallback(RecordFunctionCallback( + [](const RecordFunction& fn) -> std::unique_ptr { + ids.push_back(id); + return nullptr; + })); +} + +TEST(RecordFunctionTest, Callbacks) { + // disabling the inlining of method calls + GraphOptimizerEnabledGuard opt_guard(false); + + auto h1 = add_remove_test_add_cb<1>(); + add_remove_test_add_cb<2>(); + auto h3 = add_remove_test_add_cb<3>(); + + { + RECORD_USER_SCOPE("test"); + } + + TORCH_CHECK(ids.size() == 3); + TORCH_CHECK(std::find(ids.begin(), ids.end(), 1) != ids.end()); + TORCH_CHECK(std::find(ids.begin(), ids.end(), 2) != ids.end()); + TORCH_CHECK(std::find(ids.begin(), ids.end(), 3) != ids.end()); + + ids.clear(); + removeCallback(h1); + + { + RECORD_USER_SCOPE("test"); + } + + TORCH_CHECK(ids.size() == 2); + TORCH_CHECK(std::find(ids.begin(), ids.end(), 2) != ids.end()); + TORCH_CHECK(std::find(ids.begin(), ids.end(), 3) != ids.end()); + + ids.clear(); + removeCallback(h3); + + { + RECORD_USER_SCOPE("test"); + } + + TORCH_CHECK(ids.size() == 1); + TORCH_CHECK(std::find(ids.begin(), ids.end(), 2) != ids.end()); + + clearCallbacks(); + + // thread local / global callbacks + + ids.clear(); + add_remove_test_add_cb<1>(); + + { + RECORD_USER_SCOPE("test"); + } + + TORCH_CHECK(ids.size() == 1); + TORCH_CHECK(ids[0] == 1); + ids.clear(); + + auto th = std::thread([]() { + addThreadLocalCallback(RecordFunctionCallback( + [](const RecordFunction& fn) -> std::unique_ptr { + ids.push_back(2); + return nullptr; + })); + + { + RECORD_USER_SCOPE("test_thread"); + } + }); + th.join(); + TORCH_CHECK(ids.size() == 2); + TORCH_CHECK(std::find(ids.begin(), ids.end(), 1) != ids.end()); + TORCH_CHECK(std::find(ids.begin(), ids.end(), 2) != ids.end()); + ids.clear(); + + { + RECORD_USER_SCOPE("test"); + } + + TORCH_CHECK(ids.size() == 1); + TORCH_CHECK(ids[0] == 1); + ids.clear(); + + clearCallbacks(); + + // START: thread local / global context check callbacks + struct TestContext : public ObserverContext { + int a{0}; + std::string b; + }; + ids.clear(); + { // START: global test + addGlobalCallback(RecordFunctionCallback( + [](const RecordFunction& + /* unused */) -> std::unique_ptr { + auto ctx = std::make_unique(); + ctx->a = 123; + ctx->b = "test_str"; + ids.push_back(1); + return ctx; + }, + [](const RecordFunction& /* unused */, ObserverContext* ctx_ptr) { + auto ctx = dynamic_cast(ctx_ptr); + TORCH_CHECK(ctx != nullptr); + TORCH_CHECK(ctx->a == 123); + TORCH_CHECK(ctx->b == "test_str"); + })); + + { + RECORD_USER_SCOPE("test"); + } + + TORCH_CHECK(ids.size() == 1); + TORCH_CHECK(ids[0] == 1); + ids.clear(); + } // END: global test + { // START: thread local test + auto ctx_th = std::thread([]() { + const std::string test_str = "test thread str"; + addThreadLocalCallback(RecordFunctionCallback( + [](const RecordFunction& + /* unused */) -> std::unique_ptr { + auto ctx = std::make_unique(); + ctx->a = 234; + ctx->b = "test_thread_str"; + ids.push_back(2); + return ctx; + }, + [](const RecordFunction& /* unused */, ObserverContext* ctx_ptr) { + auto ctx = dynamic_cast(ctx_ptr); + TORCH_CHECK(ctx_ptr != nullptr); + TORCH_CHECK(ctx->a == 234); + TORCH_CHECK(ctx->b == "test_thread_str"); + })); + + // Will call both global and thread local callbacks. + { + RECORD_USER_SCOPE("test_thread"); + } + }); + ctx_th.join(); + TORCH_CHECK(ids.size() == 2); + TORCH_CHECK(std::find(ids.begin(), ids.end(), 1) != ids.end()); + TORCH_CHECK(std::find(ids.begin(), ids.end(), 2) != ids.end()); + ids.clear(); + } // END: thread local test + + clearCallbacks(); +} + +TEST(RecordFunctionTest, ShouldRun) { + // disabling the inlining of method calls + GraphOptimizerEnabledGuard opt_guard(false); + + static bool ran = false; + auto handle = addGlobalCallback(RecordFunctionCallback( + [](const RecordFunction& fn) -> std::unique_ptr { + ran = true; + return nullptr; + })); + + { + RECORD_USER_SCOPE("test"); + } + + EXPECT_TRUE(ran) << "first run didn't happen"; + ran = false; + + disableCallback(handle); + + { + RECORD_USER_SCOPE("test"); + } + + EXPECT_FALSE(ran) << "second run happened but shouldn't have"; + ran = false; + + reenableCallback(handle); + + { + RECORD_USER_SCOPE("test"); + } + + EXPECT_TRUE(ran) << "run after re-enable didn't happen"; + ran = false; + + clearCallbacks(); +} + +TEST(RecordFunctionTest, Basic) { + // disabling the inlining of method calls + GraphOptimizerEnabledGuard opt_guard(false); + + static std::string recorded_op; + static bool has_ids = false; + + // test propagation of TLS callbacks + std::thread t([]() { + RecordFunctionGuard enable_rec_fn; + auto handle = addThreadLocalCallback(RecordFunctionCallback( + [](const RecordFunction& fn) -> std::unique_ptr { + recorded_op = fn.name(); + return nullptr; + })); + ThreadLocalState state; + std::thread t_child([state]() { + ThreadLocalStateGuard g_tls(state); + RECORD_USER_SCOPE("test_in_thread"); + }); + t_child.join(); + EXPECT_EQ(recorded_op, "test_in_thread"); + removeCallback(handle); + }); + t.join(); + clearCallbacks(); + + // test set ids + addGlobalCallback( + RecordFunctionCallback( + [](const RecordFunction& fn) -> std::unique_ptr { + has_ids = fn.handle() > 0; + return nullptr; + }) + .needsIds(true)); + { + RECORD_USER_SCOPE("test"); + } + TORCH_CHECK(has_ids); + clearCallbacks(); + has_ids = false; + addGlobalCallback(RecordFunctionCallback( + [](const RecordFunction& fn) -> std::unique_ptr { + has_ids = fn.handle() > 0; + return nullptr; + })); + { + RECORD_USER_SCOPE("test"); + } + TORCH_CHECK(!has_ids); + clearCallbacks(); +} + +TEST(RecordFunctionTest, OperatorNameOverload) { + static std::set operator_names; + at::addGlobalCallback(at::RecordFunctionCallback( + [](const at::RecordFunction& fn) + -> std::unique_ptr { + std::optional op_name = + fn.operator_name(); + if (op_name.has_value()) { + operator_names.insert(c10::toString(*op_name)); + } else { + operator_names.insert("No Operator Name"); + } + return nullptr; + }) + .scopes({at::RecordScope::FUNCTION})); + auto t = torch::randn({1, 2, 3}, at::kCPU); + t.set_requires_grad(false); + auto t2 = t.pow(2); + + at::clearCallbacks(); + EXPECT_TRUE(operator_names.count("No Operator Name") == 0) + << "Expected that all traced operators had an associated OperatorName object"; + EXPECT_TRUE(operator_names.count("aten::randn") == 1) + << "Expected aten::randn to have been called and recorded, but it was not"; + EXPECT_TRUE(operator_names.count("aten::pow.Tensor_Scalar") == 1) + << "Expected aten::pow.Tensor_Scalar to have been called and recorded, but it was not"; +} + +class TestThreadLocalDebugInfo : public c10::DebugInfoBase { + public: + int getModelId() const { + return model_id_; + } + + void setModelId(int model_id) { + model_id_ = model_id; + } + + // NOLINTNEXTLINE(modernize-use-equals-default) + virtual ~TestThreadLocalDebugInfo() override {} + + private: + int model_id_ = 0; +}; + +void checkDebugInfo(c10::DebugInfoKind kind, int model_id) { + auto* debug_info = c10::ThreadLocalDebugInfo::get(kind); + TORCH_CHECK(debug_info != nullptr); + auto* test_debug_info = dynamic_cast(debug_info); + TORCH_CHECK(test_debug_info != nullptr); + TORCH_CHECK(test_debug_info->getModelId() == model_id); +} + +TEST(ThreadLocalDebugInfoTest, Basic) { + static std::atomic done{false}; + + TORCH_CHECK( + c10::ThreadLocalDebugInfo::get(c10::DebugInfoKind::TEST_INFO) == nullptr); + auto debug_info = std::make_shared(); + debug_info->setModelId(42); + { + c10::DebugInfoGuard guard(c10::DebugInfoKind::TEST_INFO, debug_info); + checkDebugInfo(c10::DebugInfoKind::TEST_INFO, 42); + } + + // check that thread local debug info is propagated through fork calls + TORCH_CHECK( + c10::ThreadLocalDebugInfo::get(c10::DebugInfoKind::TEST_INFO) == nullptr); + { + c10::DebugInfoGuard guard(c10::DebugInfoKind::TEST_INFO, debug_info); + at::launch([]() { + checkDebugInfo(c10::DebugInfoKind::TEST_INFO, 42); + done = true; + }); + } + while (!done) { + } + + // check that thread local debug info is propagated through backward pass + TORCH_CHECK( + c10::ThreadLocalDebugInfo::get(c10::DebugInfoKind::TEST_INFO) == nullptr); + done = false; + auto handle = addGlobalCallback(RecordFunctionCallback( + [](const RecordFunction&) -> std::unique_ptr { + checkDebugInfo(c10::DebugInfoKind::TEST_INFO, 42); + done = true; + return nullptr; + })); + { + c10::DebugInfoGuard guard(c10::DebugInfoKind::TEST_INFO, debug_info); + auto t = torch::randn({1, 2, 3}, at::kCPU); + t.set_requires_grad(true); + auto t2 = t.pow(2); + t2.backward(torch::ones_like(t2, at::MemoryFormat::Preserve)); + } + removeCallback(handle); + TORCH_CHECK(done); + + // check nested debug info + TORCH_CHECK( + c10::ThreadLocalDebugInfo::get(c10::DebugInfoKind::TEST_INFO) == nullptr); + { + c10::DebugInfoGuard guard(c10::DebugInfoKind::TEST_INFO, debug_info); + { + checkDebugInfo(c10::DebugInfoKind::TEST_INFO, 42); + { + auto debug_info = std::make_shared(); + debug_info->setModelId(314); + c10::DebugInfoGuard guard(c10::DebugInfoKind::TEST_INFO_2, debug_info); + { + checkDebugInfo(c10::DebugInfoKind::TEST_INFO, 42); + checkDebugInfo(c10::DebugInfoKind::TEST_INFO_2, 314); + done = false; + at::launch([]() { + checkDebugInfo(c10::DebugInfoKind::TEST_INFO, 42); + checkDebugInfo(c10::DebugInfoKind::TEST_INFO_2, 314); + done = true; + }); + while (!done) { + } + } + } + } + } +} + +TEST(TestSymIntArrayRef, BasicConversion) { + const size_t X = 2, Y = 4, Z = 5; + std::vector tgt_size_v{2, 4, 5}; + std::vector tgt_size({SymInt(X), SymInt(Y), SymInt(Z)}); + auto a = at::randn({1, 4, 1}, at::kCPU); + auto b = a.expand_symint(tgt_size); + auto c = a.expand(tgt_size_v); + ASSERT_TRUE(torch::allclose(b, c)); +} + +TEST(TestSymInt, NarrowCopyWithSymbolicInt) { + static const size_t LENGTH = 5; + auto a = at::randn({10}, at::kCPU); + c10::SymInt si(LENGTH); + auto b = a.narrow_copy_symint(0, 0, si); + auto c = a.narrow(0, 0, LENGTH); + ASSERT_TRUE(torch::allclose(b, c)); +} + +TEST(TestSymInt, NarrowCopy) { + static const size_t LENGTH = 5; + auto a = at::randn({10}, at::kCPU); + auto b = a.narrow_copy(0, 0, LENGTH); + auto c = a.narrow(0, 0, LENGTH); + ASSERT_TRUE(torch::allclose(b, c)); +} + +TEST(TestSymInt, AddSymbolicInt) { + c10::SymInt a(5); + c10::SymInt b(3); + ASSERT_TRUE((a + b).expect_int() == 8); +} + +TEST(FallbackGraphsTest, Basic) { + auto x = at::randn({1}, at::kCPU); + auto y = at::randn({1}, at::kCPU); + auto stack = createStack({x.clone(), y.clone()}); + + auto graph_string = R"IR( + graph(%0 : Float(1), + %1 : Float(1)): + %2 : Tensor = aten::mul(%0, %1) + %3 : Tensor = aten::mul(%2, %0) + return (%3))IR"; + auto graph = std::make_shared(); + torch::jit::parseIR(graph_string, graph.get()); + + { + Code code(graph, ""); + InterpreterState interpreter{code}; + interpreter.run(stack); + } + at::Tensor et; + pop(stack, et); + float ef = et.item(); + { + EnableProfilingGuard epg; + GraphFunction f("fallbackGraphs", graph, nullptr); + for (size_t i = 0; i < getNumProfiledRuns() + 1; i++) { + stack.emplace_back(x.clone()); + stack.emplace_back(y.clone()); + if (i == getNumProfiledRuns()) { + // we will be modifying a profiled graph + // before ProfilingGraphExecutor + // will optimize it in the next iteration + auto opt_graph = lastExecutedOptimizedGraph(); + // this is safe to do since we are done profiling + ProfilingRecord::removeProfileCounter(opt_graph->block()); + replaceBlockWithFallbackGraph(opt_graph->block(), opt_graph->inputs()); + auto it = opt_graph->block()->nodes().begin(); + ASSERT_EQ(it->kind(), prim::FallbackGraph); + auto fallback = *it++; + ASSERT_EQ(it, opt_graph->block()->nodes().end()); + ASSERT_TRUE(fallback->hasAttribute(attr::Subgraph)); + testing::FileCheck() + .check("Tensor = aten::mul") + ->check("Tensor = aten::mul") + ->run(*fallback->g(attr::Subgraph)); + } + f.run(stack); + at::Tensor at; + pop(stack, at); + float af = at.item(); + ASSERT_EQ(af, ef); + } + + auto opt_graph = lastExecutedOptimizedGraph(); + testing::FileCheck() + .check("(Tensor) = prim::CallFunction") + ->run(*opt_graph); + } +} + +// TODO this test wasn't running and is broken. +// TEST(AutogradProfilerTest, Basic) { +// constexpr int batch_size = 4; +// constexpr int input_size = 256; +// constexpr int seq_len = 32; + +// int hidden_size = 2 * input_size; +// auto input = torch::randn({seq_len, batch_size, input_size}, at::kCPU); +// auto hx = torch::randn({batch_size, hidden_size}, at::kCPU); +// auto cx = torch::randn({batch_size, hidden_size}, at::kCPU); +// auto w_ih = t_def(torch::randn({4 * hidden_size, input_size}, at::kCPU)); +// auto w_hh = t_def(torch::randn({4 * hidden_size, hidden_size}, at::kCPU)); + +// std::stringstream ss; +// { +// RecordProfile guard(ss); +// for (size_t i = 0; i < 100; ++i) { +// std::tie(hx, cx) = lstm(input[0], hx, cx, w_ih, w_hh); +// } +// } + +// std::string result = ss.str(); +// size_t count = 0; +// for (size_t pos = 0; (pos = result.find("tanh", pos)) != std::string::npos; +// count++, pos++) { +// } +// ASSERT_EQ((count, 200); +// } + +TEST(NoneSchemaMatchTest, Basic) { + RegisterOperators reg({ + Operator( + "prim::test_none() -> int?", + [](Stack& stack) { push(stack, IValue()); }, + aliasAnalysisFromSchema()), + Operator( + "prim::is_none(int? a) -> bool", + [](Stack& stack) { + IValue a = pop(stack); + if (a.isNone()) { + push(stack, true); + } else { + push(stack, false); + } + }, + aliasAnalysisFromSchema()), + }); + + // Constant propagation will run test_none and produce a None, + // testing that its type is set appropriately and schema matching doesn't + // fail when running is_none + + auto r = std::make_shared(); + auto& g = *r; + auto opt_int = g.insert(Symbol::fromQualString("prim::test_none"), {}); + auto out_bool = g.insert(Symbol::fromQualString("prim::is_none"), {opt_int}); + g.registerOutput(out_bool); + ConstantPropagation(r); + + auto nodes = r->block()->nodes(); + // checking that constant propagation ran wo/failure + AT_ASSERT(std::distance(nodes.begin(), nodes.end()) == 1); +} + +static int testPassValue = 0; +void fakePass(std::shared_ptr& g) { + testPassValue++; + return; +} + +RegisterPass p(fakePass); + +TEST(PassManagementTest, Basic) { + std::shared_ptr graph = std::make_shared(); + parseIR( + R"IR( +graph(%a): + return (%a))IR", + &*graph); + + std::vector stack = {IValue(torch::randn({22}, at::kCPU))}; + auto run = [&](std::shared_ptr& graph, std::vector stack) { + GraphExecutor executor(graph, ""); + executor.run(stack); + return stack; + }; + run(graph, stack); + // we will not run fusion in simple mode + if (!getExecutorMode()) { + AT_ASSERT(testPassValue); + } +} + +static void checkShape(TypePtr typ, std::vector expected) { + auto ptp = typ->expect(); + ASSERT_EQ(ptp->sizes().concrete_sizes().value(), expected); +} + +static void checkShape( + Node* n, + std::vector expected, + bool prev = true) { + auto profile = (prev) ? n->inputs().at(0)->node() : n; + checkShape(profile->output()->type(), expected); +} + +void count_( + Block* block, + const std::function& pred, + size_t& count) { + for (Node* n : block->nodes()) { + if (pred(n)) { + count++; + } + + for (Block* ib : n->blocks()) { + count_(ib, pred, count); + } + } +} + +size_t countNodes( + const std::shared_ptr& graph, + const std::function& pred) { + size_t count = 0; + count_(graph->block(), pred, count); + return count; +} + +bool true_pred(Node* n) { + return true; +}; + +bool is_loop(Node* n) { + return n->kind() == prim::Loop; +}; + +TEST(LoopPeelerTest, NoInductionVariableUse) { + // do not use an induction variable explicitly + static const auto str_func_def = R"JIT( + def test_peel_n_times(): + sum = 0 + for i in range(10): + sum += 2 + return sum + )JIT"; + + auto cu = compile(str_func_def); + auto& f = toGraphFunction(cu->get_function("test_peel_n_times")); + auto stack = createStack({}); + // peeling loop once + { + LoopsPeeler peeler(true_pred, 1); + auto copy = f.graph()->copy(); + peeler.run(copy); + int num_loops = + std::count_if(copy->nodes().begin(), copy->nodes().end(), is_loop); + ASSERT_EQ(num_loops, 2); + Code code(copy, ""); + InterpreterState interpreter{code}; + interpreter.run(stack); + ASSERT_EQ(stack.back().toInt(), 20); + } + + // test peeling more than one iteration + { + LoopsPeeler peeler(true_pred, 3); + auto copy = f.graph()->copy(); + peeler.run(copy); + int num_loops = + std::count_if(copy->nodes().begin(), copy->nodes().end(), is_loop); + ASSERT_EQ(num_loops, 2); + Code code(copy, ""); + InterpreterState interpreter{code}; + interpreter.run(stack); + ASSERT_EQ(stack.back().toInt(), 20); + } +} + +TEST(LoopPeelerTest, YesInductionVariableUse) { + // uses the induction variable + static const auto str_func_def = R"JIT( + def test_peel_n_times(): + sum = 0 + for i in range(10): + sum += i + return sum + )JIT"; + + auto cu = compile(str_func_def); + auto& f = toGraphFunction(cu->get_function("test_peel_n_times")); + auto stack = createStack({}); + // peeling loop once + { + LoopsPeeler peeler(true_pred, 1); + auto copy = f.graph()->copy(); + peeler.run(copy); + int num_loops = + std::count_if(copy->nodes().begin(), copy->nodes().end(), is_loop); + ASSERT_EQ(num_loops, 2); + Code code(copy, ""); + InterpreterState interpreter{code}; + interpreter.run(stack); + ASSERT_EQ(stack.back().toInt(), 45); + } + + // test peeling more than one iteration + { + LoopsPeeler peeler(true_pred, 3); + auto copy = f.graph()->copy(); + peeler.run(copy); + int num_loops = + std::count_if(copy->nodes().begin(), copy->nodes().end(), is_loop); + ASSERT_EQ(num_loops, 2); + Code code(copy, ""); + InterpreterState interpreter{code}; + interpreter.run(stack); + ASSERT_EQ(stack.back().toInt(), 45); + } +} + +TEST(LoopPeelerTest, LoopWithTerminationCondition) { + // tests with explicit termination conditions + static const auto str_func_def = R"JIT( + def test_with_cond_times(): + sum = 0 + i = 0 + while (sum < 2): + sum += i + i += 1 + return sum + )JIT"; + + // the peel changes the termination condition to false + // so the original loop doesn't run + auto cu = compile(str_func_def); + auto& f = toGraphFunction(cu->get_function("test_with_cond_times")); + auto stack = createStack({}); + // peeling 5 iterations should update the termination + // condition to false + { + LoopsPeeler peeler(true_pred, 5); + auto copy = f.graph()->copy(); + peeler.run(copy); + int num_loops = + std::count_if(copy->nodes().begin(), copy->nodes().end(), is_loop); + ASSERT_EQ(num_loops, 2); + Code code(copy, ""); + InterpreterState interpreter{code}; + interpreter.run(stack); + ASSERT_EQ(stack.back().toInt(), 3); + } + + // the termination condition remains true + { + LoopsPeeler peeler(true_pred, 1); + auto copy = f.graph()->copy(); + peeler.run(copy); + int num_loops = + std::count_if(copy->nodes().begin(), copy->nodes().end(), is_loop); + ASSERT_EQ(num_loops, 2); + Code code(copy, ""); + InterpreterState interpreter{code}; + interpreter.run(stack); + ASSERT_EQ(stack.back().toInt(), 3); + } +} + +// tests simple nested loops +TEST(LoopPeelerTest, SimpleNestedLoops) { + static const auto str_func_def = R"JIT( + def test_nested_loops(): + sum = 0 + i = 0 + for i in range(10): + for j in range(10): + sum += i + j + return sum + )JIT"; + + auto cu = compile(str_func_def); + auto& f = toGraphFunction(cu->get_function("test_nested_loops")); + auto stack = createStack({}); + + { + LoopsPeeler peeler(true_pred, 1); + auto copy = f.graph()->copy(); + peeler.run(copy); + ASSERT_EQ(countNodes(copy, is_loop), 5); + Code code(copy, ""); + InterpreterState interpreter{code}; + interpreter.run(stack); + ASSERT_EQ(stack.back().toInt(), 900); + } + + { + LoopsPeeler peeler(true_pred, 5); + auto copy = f.graph()->copy(); + peeler.run(copy); + ASSERT_EQ(countNodes(copy, is_loop), 5); + Code code(copy, ""); + InterpreterState interpreter{code}; + interpreter.run(stack); + ASSERT_EQ(stack.back().toInt(), 900); + } +} + +TEST(LoopPeelerTest, SimpleNestedLoops2) { + static const auto str_func_def = R"JIT( + def test_nested_loops(): + sum = 0 + i = 0 + for i in range(10): + j = 0 + while sum < 2: + sum += i + j + j += 1 + return sum + )JIT"; + + auto cu = compile(str_func_def); + auto& f = toGraphFunction(cu->get_function("test_nested_loops")); + auto stack = createStack({}); + { + LoopsPeeler peeler(true_pred, 1); + auto copy = f.graph()->copy(); + peeler.run(copy); + ASSERT_EQ(countNodes(copy, is_loop), 5); + Code code(copy, ""); + InterpreterState interpreter{code}; + interpreter.run(stack); + ASSERT_EQ(stack.back().toInt(), 3); + } + + { + LoopsPeeler peeler(true_pred, 5); + auto copy = f.graph()->copy(); + peeler.run(copy); + ASSERT_EQ(countNodes(copy, is_loop), 5); + Code code(copy, ""); + InterpreterState interpreter{code}; + interpreter.run(stack); + ASSERT_EQ(stack.back().toInt(), 3); + } +} + +TEST(JitTracing, Basic) { + constexpr int batch_size = 4; + constexpr int input_size = 256; + + int hidden_size = 2 * input_size; + + auto input = at::randn({batch_size, input_size}, at::kCPU); + auto hx = at::randn({batch_size, hidden_size}, at::kCPU); + auto cx = at::randn({batch_size, hidden_size}, at::kCPU); + auto w_ih = t_def(at::randn({4 * hidden_size, input_size}, at::kCPU)); + auto w_hh = t_def(at::randn({4 * hidden_size, hidden_size}, at::kCPU)); + + auto graph = build_lstm(); + auto stack = createStack({input, hx, cx, w_ih, w_hh}); + auto traced = TraceGraph(graph, stack); + + // Check that the inputs of traced graph have the same type as the inputs + // specified here. + ASSERT_EQ(*traced->inputs().at(0)->type(), *TensorType::create(input)); + ASSERT_EQ(*traced->inputs().at(1)->type(), *TensorType::create(hx)); + ASSERT_EQ(*traced->inputs().at(2)->type(), *TensorType::create(cx)); + ASSERT_EQ(*traced->inputs().at(3)->type(), *TensorType::create(w_ih)); + ASSERT_EQ(*traced->inputs().at(4)->type(), *TensorType::create(w_hh)); + + Tensor prof_out; + pop(stack, prof_out); + + { + stack = createStack({input, hx, cx, w_ih, w_hh}); + Code cd(traced, "traced"); + InterpreterState is{cd}; + is.run(stack); + Tensor traced_out; + pop(stack, traced_out); + torch::allclose(prof_out, traced_out); + } + + { + stack = createStack({input, hx, cx, w_ih, w_hh}); + Code cd(graph, "graph"); + InterpreterState is{cd}; + is.run(stack); + Tensor scripted_out; + pop(stack, scripted_out); + torch::allclose(prof_out, scripted_out); + } +} + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +TEST(InsertAndEliminateRedundantGuardsTest, Basic) { + static const auto basic_example = R"JIT( + def basic(x, y): + a = x + y + b = x * y + c = x + 1 + d = a - c + e = b - c + return d + e + )JIT"; + + auto cu = compile(basic_example); + auto& fun = toGraphFunction(cu->get_function("basic")); + auto pr = ProfilingRecord::instrumentGraph(fun.graph()); + auto x = at::randn({2, 3}, at::kCPU); + auto y = at::randn({2, 3}, at::kCPU); + auto stack = createStack({x, y}); + // introduce some profiling information + Code cd(pr->profiled_graph_, ""); + InterpreterState is{cd}; + is.run(stack); + auto copy = pr->profiled_graph_->copy(); + ProfilingRecord::removeProfileCounter(copy->block()); + InsertGuards(copy); + auto nodes = copy->block()->nodes(); + auto guard = std::find_if(nodes.begin(), nodes.end(), [](Node* n) { + return n->kind() == prim::Guard; + }); + ASSERT_NE(guard, nodes.end()); + ASSERT_EQ( + guard->input()->type()->expectRef().sizes().size(), + std::nullopt); + checkShape(*guard, {2, 3}, false); + auto is_guard = [](Node* n) { return n->kind() == prim::Guard; }; + int num_guards = std::count_if(nodes.begin(), nodes.end(), is_guard); + ASSERT_EQ(num_guards, 12); + // now eliminate as many guards as possible + // we should be left with two guards on x and y's defs + EliminateRedundantGuards(copy); + num_guards = std::count_if(nodes.begin(), nodes.end(), is_guard); + ASSERT_EQ(num_guards, 2); +} + +TEST(InsertBailOutsTest, Basic) { + static const auto basic_example = R"JIT( + def basic_loop(x, y): + + a = x + 1 + b = y + 2 + c = x + y + 3 + + for i in range(10): + a = a + b + # invariant + d = b * c + # + a = a - d + + e = a + 4 + return e + )JIT"; + + auto cu = compile(basic_example); + auto& fun = toGraphFunction(cu->get_function("basic_loop")); + auto pr = ProfilingRecord::instrumentGraph(fun.graph()); + auto x = at::randn({2, 3}, at::kCPU); + auto y = at::randn({2, 3}, at::kCPU); + auto stack = createStack({x, y}); + // introduce some profiling information + Code cd(pr->profiled_graph_, ""); + InterpreterState is{cd}; + is.run(stack); + auto copy = pr->profiled_graph_->copy(); + ProfilingRecord::removeProfileCounter(copy->block()); + InsertGuards(copy); + EliminateRedundantGuards(copy); + auto nodes = copy->block()->nodes(); + auto is_guard = [](Node* n) { return n->kind() == prim::Guard; }; + auto num_guards = std::count_if(nodes.begin(), nodes.end(), is_guard); + ASSERT_EQ(num_guards, 3); + InsertBailOuts(copy); + auto is_bailout = [](Node* n) { return n->kind() == prim::BailOut; }; + auto num_bailouts = std::count_if(nodes.begin(), nodes.end(), is_bailout); + ASSERT_EQ(num_guards, num_bailouts); + std::vector bailouts(num_bailouts); + std::copy_if(nodes.begin(), nodes.end(), bailouts.begin(), is_bailout); + + for (auto blo : bailouts) { + ASSERT_EQ(blo->inputs().at(0)->node()->kind(), prim::BailoutTemplate); + } +} + +TEST(ProfilerTest, Basic) { + constexpr int batch_size = 4; + constexpr int input_size = 256; + + int hidden_size = 2 * input_size; + + auto input = at::randn({batch_size, input_size}, at::kCPU); + auto hx = at::randn({batch_size, hidden_size}, at::kCPU); + auto cx = at::randn({batch_size, hidden_size}, at::kCPU); + auto w_ih = t_def(at::randn({4 * hidden_size, input_size}, at::kCPU)); + auto w_hh = t_def(at::randn({4 * hidden_size, hidden_size}, at::kCPU)); + + auto g = build_lstm(); + auto stack = createStack({input, hx, cx, w_ih, w_hh}); + + auto& opt_graph = *g.get(); + ArgumentSpecCreator arg_spec_creator(opt_graph); + ArgumentSpec spec = + arg_spec_creator.create(autograd::GradMode::is_enabled(), stack); + arg_spec_creator.specializeTypes(opt_graph, spec); + auto pr = ProfilingRecord::instrumentGraph(g); + Code cd(pr->profiled_graph_, ""); + InterpreterState is{cd}; + is.run(stack); + + // profiled types are stored as attributes and show up in the dump, e.g. + // Tensor = prim::profile[profiled_type=Double(4, 256, strides=[256, 1], + // requires_grad=0, device=cpu) + testing::FileCheck() + .check("Tensor = prim::profile[profiled_type") + ->check_same("256") + ->run(*pr->profiled_graph_); + + auto begin = pr->profiled_graph_->block()->nodes().begin(); + auto end = pr->profiled_graph_->block()->nodes().end(); + auto mm = + std::find_if(begin, end, [](Node* n) { return n->kind() == aten::add; }); + ASSERT_NE(mm, end); + std::vector mm_expected{4, 2048}; + std::vector eltwise{4, 512}; + checkShape(mm->inputs().at(0)->node()->ty(attr::profiled_type), mm_expected); + auto mul_n = + std::find_if(begin, end, [](Node* n) { return n->kind() == aten::mul; }); + ASSERT_NE(mul_n, end); + checkShape(mul_n->inputs().at(0)->node()->ty(attr::profiled_type), eltwise); + auto tanh_n = + std::find_if(begin, end, [](Node* n) { return n->kind() == aten::tanh; }); + checkShape(tanh_n->inputs().at(0)->node()->ty(attr::profiled_type), eltwise); +} + +TEST(ProfilerTest, OptionalProfiling) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(%inp : Tensor, + %weight : Tensor, + %bias : Tensor?): + %1 : Tensor = aten::linear(%inp, %weight, %bias) + return (%1))IR", + &*graph, + vmap); + + auto pr = ProfilingRecord::instrumentGraph(graph); + pr->profiling_count_ = 2; + + auto input = torch::randn({1, 2}); + auto weight = torch::randn({2, 2}); + auto bias = torch::randn({1, 2}); + + auto stack = createStack({input, weight, bias}); + Code cd(pr->profiled_graph_, ""); + InterpreterState is{cd}; + is.run(stack); + + testing::FileCheck() + .check_count("Tensor? = prim::profile[profiled_type", 1, true) + ->run(*pr->profiled_graph_); + + // make sure we recorded the shape + auto begin = pr->profiled_graph_->block()->nodes().begin(); + auto end = pr->profiled_graph_->block()->nodes().end(); + auto linear = std::find_if( + begin, end, [](Node* n) { return n->kind() == aten::linear; }); + ASSERT_NE(linear, end); + std::vector bias_expected_shape = {1, 2}; + auto profiled_bias = linear->namedInput("bias")->node(); + checkShape(profiled_bias->ty(attr::profiled_type), bias_expected_shape); + ASSERT_EQ(0, profiled_bias->i(attr::seen_none)); + + auto none_bias = c10::IValue(); + + stack.clear(); + stack.emplace_back(input); + stack.emplace_back(weight); + stack.emplace_back(none_bias); + is = InterpreterState{cd}; + is.run(stack); + + // make sure we recorded that "None" was seen. + begin = pr->profiled_graph_->block()->nodes().begin(); + end = pr->profiled_graph_->block()->nodes().end(); + linear = std::find_if( + begin, end, [](Node* n) { return n->kind() == aten::linear; }); + ASSERT_NE(linear, end); + profiled_bias = linear->namedInput("bias")->node(); + checkShape(profiled_bias->ty(attr::profiled_type), bias_expected_shape); + ASSERT_EQ(1, profiled_bias->i(attr::seen_none)); +} + +TEST(CallStackTest, Basic) { + const auto text = R"( +def ham(x): + return x/7 + +def bar(x): + return x*3 + +def baz(x): + return ham(x)*x + +def foo(x): + return bar(x)*baz(x)*11 + )"; + auto cu = compile(text); + const auto& foo = toGraphFunction(cu->get_function("foo")); + for (Node* n : foo.optimized_graph()->nodes()) { + if (n->kind() == prim::Constant) { + if (!n->hasAttribute(attr::value) || + n->kindOf(attr::value) != AttributeKind::i) { + continue; + } + int v = n->i(attr::value); + switch (v) { + case 3: { + // Const 3 comes from function 'bar', which gets inlined to 'foo'. + // The callstack for the corresponding node should contain only the + // function 'bar'. + ASSERT_TRUE(n->callstack()); + auto callstack_vector = (*n->callstack())->vec(); + ASSERT_EQ(callstack_vector.size(), 1); + ASSERT_EQ(std::get<0>(callstack_vector[0]), &cu->get_function("bar")); + break; + } + case 7: { + // Const 7 comes from function 'ham', which gets inlined to 'baz', + // which is then inlined to 'foo'. The callstack for the corresponding + // node should contain these two functions. + ASSERT_TRUE(n->callstack()); + auto callstack_vector = (*n->callstack())->vec(); + ASSERT_EQ(callstack_vector.size(), 2); + ASSERT_EQ(std::get<0>(callstack_vector[0]), &cu->get_function("baz")); + ASSERT_EQ(std::get<0>(callstack_vector[1]), &cu->get_function("ham")); + break; + } + case 11: { + // Const 11 comes from function 'foo', which is not inlined anywhere + // and thus it should not have a callstack. + ASSERT_FALSE(n->callstack()); + break; + } + } + } + } + + // Check that inlining doesn't corrupt callstack of the callee's nodes. + const auto& baz = toGraphFunction(cu->get_function("baz")); + for (Node* n : baz.optimized_graph()->nodes()) { + if (n->kind() == prim::Constant) { + if (!n->hasAttribute(attr::value) || + n->kindOf(attr::value) != AttributeKind::i) { + continue; + } + int v = n->i(attr::value); + ASSERT_TRUE(v == 7); + // Const 7 comes from function 'ham', which gets inlined to 'baz'. 'baz' + // was also inlined into 'foo', but when looking at the graph of 'baz' we + // should only see a callstack of depth 1 (containing only 'ham'). + ASSERT_TRUE(n->callstack()); + auto callstack_vector = (*n->callstack())->vec(); + ASSERT_EQ(callstack_vector.size(), 1); + ASSERT_EQ(std::get<0>(callstack_vector[0]), &cu->get_function("ham")); + } + } +} + +TEST(CallStackTest, Caching) { + const auto text = R"( + +def a(x): + print("a1") + print("a2") + return x + +def b(x): + print("b1") + print("b2") + a(x) + return x + +def c(x): + print("c1") + print("c2") + b(x) + return x + )"; + auto cu = compile(text); + const auto& baz = toGraphFunction(cu->get_function("c")); + std::unordered_map callstack_objects; + for (Node* n : baz.optimized_graph()->nodes()) { + if (n->kind() == prim::Constant) { + if (!n->hasAttribute(attr::value) || + n->kindOf(attr::value) != AttributeKind::s) { + continue; + } + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + std::string v = n->s(attr::value); + if (n->callstack()) { + callstack_objects[v] = n->callstack()->get(); + } + } + } + // We expect to see nodes prim::Constant[value="a1"] and + // prim::Constant[value="a2"] inlined to function 'c'. Their callstacks are + // the same (a->b->c), so we want to make sure we're not creating different + // callstack entries for them. + ASSERT_TRUE(callstack_objects.count("a1") && callstack_objects.count("a2")); + ASSERT_TRUE(callstack_objects.at("a1") == callstack_objects.at("a2")); +} + +TEST(InlinedCallStackTest, BlockAnnotation) { + Module a("A"); + a.define(R"( + def forward(self, x, y, z: int): + if (z == 1): + return x + y + else: + return x * y + )"); + Module b("B"); + b.define(R"( + def forward(self, x): + return x + 2 + )"); + Module c("C"); + c.register_module("A0", a); + c.register_module("B0", b); + c.define(R"( + def forward(self, x, y, z: int): + return self.A0.forward(x, y, z) + self.B0.forward(x) + )"); + + auto graph = + toGraphFunction(c.get_method("forward").function()).optimized_graph(); + std::stringstream add_ss, mul_ss; + for (Node* n : graph->nodes()) { + if (n->kind() == prim::If) { + for (Block* block : n->blocks()) { + for (Node* if_node : block->nodes()) { + if (if_node->kind() == aten::add) { + for (const auto& e : if_node->callstack().value()->vec()) { + add_ss << std::get<1>(e); + } + add_ss << if_node->sourceRange(); + } + if (if_node->kind() == aten::mul) { + for (const auto& e : if_node->callstack().value()->vec()) { + mul_ss << std::get<1>(e); + } + mul_ss << if_node->sourceRange(); + } + } + } + } + } + ASSERT_NE(add_ss.str().find("line 3"), std::string::npos); + ASSERT_NE(add_ss.str().find("line 4"), std::string::npos); + ASSERT_NE( + add_ss.str().find("return self.A0.forward(x, y, z)"), std::string::npos); + ASSERT_NE(std::move(add_ss).str().find("return x + y"), std::string::npos); + ASSERT_NE(mul_ss.str().find("line 3"), std::string::npos); + ASSERT_NE(mul_ss.str().find("line 6"), std::string::npos); + ASSERT_NE( + mul_ss.str().find("return self.A0.forward(x, y, z)"), std::string::npos); + ASSERT_NE(std::move(mul_ss).str().find("return x * y"), std::string::npos); +} + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +TEST(InlinedCallStackTest, SelfCallMethods) { + Module a("A"); + a.define(R"( + def my_new_method(self, x): + return x * 3 + def forward_impl_(self, x, y): + return self.my_new_method(x) + y + def forward(self, x, y): + y = y + 2 + return self.forward_impl_(x, y) + )"); + Module b("B"); + b.define(R"( + def forward(self, x): + return x + 2 + )"); + Module c("C"); + c.register_module("A0", a); + c.register_module("B0", b); + c.define(R"( + def call_b(self, x): + return self.B0.forward(x) + def forward(self, x, y): + return self.A0.forward(x, y) + self.call_b(x) + )"); + + auto graph = + toGraphFunction(c.get_method("forward").function()).optimized_graph(); + std::unordered_map module_hierarchies; + for (Node* n : graph->nodes()) { + auto hierarchy = torch::jit::utils::getNodesModuleHierarchy(*n); + if (module_hierarchies.count(hierarchy) == 0) { + module_hierarchies[hierarchy] = 0; + } + module_hierarchies[hierarchy] += 1; + } + ASSERT_EQ(module_hierarchies["A0(A)"], 2); + ASSERT_EQ(module_hierarchies["A0(A).SELF(A).SELF(A)"], 2); + ASSERT_EQ(module_hierarchies["A0(A).SELF(A)"], 1); + ASSERT_EQ(module_hierarchies["SELF(C)"], 1); + ASSERT_EQ(module_hierarchies["SELF(C).B0(B)"], 1); +} + +TEST(AutogradSymbolsTest, Basic) { + Symbol sym = Symbol::fromQualString("aten::test_symbol"); + Graph graph; + auto node = graph.create(sym); + TORCH_CHECK(canRunWithAutograd(node)); + + sym = Symbol::fromQualString("prim::test_symbol"); + node = graph.create(sym); + TORCH_CHECK(canRunWithAutograd(node)); + + sym = Symbol::fromQualString("prim::FusionGroup"); + node = graph.create(sym); + TORCH_CHECK(!canRunWithAutograd(node)); + + sym = Symbol::fromQualString("custom::test_symbol"); + node = graph.create(sym); + TORCH_CHECK(!canRunWithAutograd(node)); +} + +TEST(DefaultArgTypeHintingTest, Basic) { + const auto text_non_hinted = R"( + +def a(x, y=1): + print("a1") + print("a2") + return x + )"; + + const auto text_hinted = R"( + +def a(x, y:int=1): + print("a1") + print("a2") + return x + )"; + + try { + compile(text_non_hinted); + ASSERT_TRUE(0); + } catch (const std::exception&) { + } + + auto cu = compile(text_hinted); +} + +// Basic set case. +TEST(FuturesTest, Basic) { + auto f1 = c10::make_intrusive(IntType::get()); + ASSERT_FALSE(f1->completed()); + ASSERT_FALSE(f1->hasValue()); + int32_t sat1 = 0; + int32_t sat2 = 0; + f1->addCallback([&](Future& /* unused */) { ++sat1; }); + f1->markCompleted(43); + ASSERT_TRUE(f1->completed()); + ASSERT_TRUE(f1->hasValue()); + ASSERT_FALSE(f1->hasError()); + ASSERT_EQ(sat1, 1); + ASSERT_EQ(f1->constValue().toInt(), 43); + ASSERT_EQ(f1->value().toInt(), 43); + f1->addCallback([&](Future& /* unused */) { ++sat2; }); + ASSERT_EQ(sat1, 1); + ASSERT_EQ(sat2, 1); +} + +// Sparse CUDA tensor test +TEST(FutureTest, SparseTensor) { + // Skip test if CUDA is not available. + bool has_cuda = at::globalContext().hasCUDA(); + if (!has_cuda) { + LOG(INFO) << "CUDA not available, skipping test"; + } + for (int i = 0; i < 2; ++i) { + auto f = c10::make_intrusive(TensorType::get()); + at::TensorOptions opts = at::TensorOptions().device(at::DeviceType::CUDA); + auto sparse_tensor = i == 0 ? at::ones(10).to_sparse() + : at::sparse_coo_tensor( + at::arange(10).unsqueeze(0).to(at::kLong), + at::ones({10, 10}), + opts); + // Runs storage extraction for sparse CUDA tensors + f->markCompleted(sparse_tensor); + ASSERT_TRUE(f->completed()); + ASSERT_FALSE(f->hasError()); + } +} + +// Basic error cases. +TEST(FuturesTest, Error) { + auto f1 = c10::make_intrusive(IntType::get()); + int sat1 = 0; + int sat2 = 0; + f1->addCallback([&](Future& /* unused */) { ++sat1; }); + f1->setError( + std::make_exception_ptr(c10::ivalue::Future::FutureError("Failed"))); + ASSERT_EQ(sat1, 1); + ASSERT_TRUE(f1->completed()); + ASSERT_TRUE(f1->hasError()); + ASSERT_FALSE(f1->hasValue()); + try { + (void)f1->value(); + ASSERT_TRUE(false); // Supposed to throw. + } catch (const std::exception& e) { + ASSERT_TRUE(strcmp(e.what(), "Failed") == 0); + } + f1->addCallback([&](Future& /* unused */) { ++sat2; }); + ASSERT_EQ(sat1, 1); + ASSERT_EQ(sat2, 1); + f1->setErrorIfNeeded( + std::make_exception_ptr(c10::ivalue::Future::FutureError("Dup"))); + ASSERT_TRUE(strcmp(f1->tryRetrieveErrorMessage().c_str(), "Failed") == 0); + ASSERT_EQ(sat1, 1); + ASSERT_EQ(sat2, 1); + try { + (void)f1->constValue(); + ASSERT_TRUE(false); // Supposed to throw. + } catch (const std::exception& e) { + // Original error should be logged. + ASSERT_TRUE(std::string(e.what()).find("Failed") != std::string::npos); + } +} + +// then +TEST(FuturesTest, Then) { + auto f1 = c10::make_intrusive(IntType::get()); + auto f2 = f1->then( + [](Future& f1) -> IValue { return f1.constValue().toInt() + 1; }, + IntType::get()); + auto f3 = f2->then( + [](Future& f2) -> IValue { return f2.constValue().toInt() * 3; }, + IntType::get()); + bool done = false; + f3->addCallback([&done](Future& f3) { + ASSERT_EQ(f3.constValue().toInt(), (42 + 1) * 3); + done = true; + }); + ASSERT_FALSE(done); + f1->markCompleted(42); + ASSERT_TRUE(done); +} + +// collectAll() +TEST(FuturesTest, CollectAll) { + auto s1 = c10::make_intrusive(IntType::get()); + auto s2 = c10::make_intrusive(IntType::get()); + auto s3 = c10::make_intrusive(IntType::get()); + + // Empty case + c10::List> futures( + FutureType::create(IntType::get())); + auto c1 = collectAll(futures); + ASSERT_TRUE(c1->completed()); + ASSERT_EQ(c1->value().toList().size(), 0); + ASSERT_TRUE( + *(c1->value().toList().elementType()) == + *FutureType::create(IntType::get())); + + // 1-element, initially not completed. + futures.push_back(s1); + auto c2 = collectAll(futures); + ASSERT_FALSE(c2->completed()); + s1->markCompleted(5); + ASSERT_TRUE(c2->completed()); + ASSERT_EQ(c2->value().toList().size(), 1); + ASSERT_TRUE( + *(c2->value().toList().elementType()) == + *FutureType::create(IntType::get())); + ASSERT_EQ(c2->value().toList().get(0).toFuture()->value().toInt(), 5); + + // 1-element, already completed + auto c3 = collectAll(futures); + ASSERT_TRUE(c3->completed()); + ASSERT_EQ(c3->value().toList().size(), 1); + ASSERT_EQ(c3->value().toList().get(0).toFuture()->value().toInt(), 5); + + // 3 elements. + futures.push_back(s2); + futures.push_back(s3); + auto c4 = collectAll(futures); + ASSERT_FALSE(c4->completed()); + s3->markCompleted(7); + ASSERT_FALSE(c4->completed()); + s2->markCompleted(6); + ASSERT_TRUE(c4->completed()); + ASSERT_EQ(c4->value().toList().size(), 3); + ASSERT_EQ(c4->value().toList().get(0).toFuture()->value().toInt(), 5); + ASSERT_EQ(c4->value().toList().get(1).toFuture()->value().toInt(), 6); + ASSERT_EQ(c4->value().toList().get(2).toFuture()->value().toInt(), 7); + ASSERT_TRUE( + *(c4->value().toList().elementType()) == + *FutureType::create(IntType::get())); + + // Handle exception in the list. + auto s4 = c10::make_intrusive(IntType::get()); + futures.push_back(s4); + auto c5 = collectAll(futures); + ASSERT_FALSE(c5->completed()); + s4->setError( + std::make_exception_ptr(c10::ivalue::Future::FutureError("Failed"))); + ASSERT_TRUE(c5->completed()); + try { + c5->value(); + ASSERT_TRUE(false); // supposed to throw + } catch (const std::exception& e) { + ASSERT_EQ(std::string(e.what()), "Failed"); + } +} + +// collectAny() +TEST(FuturesTest, CollectAny) { + auto s1 = c10::make_intrusive(IntType::get()); + + // Empty case + c10::List> futures( + FutureType::create(IntType::get())); + auto c1 = collectAny(futures); + ASSERT_TRUE(c1->completed()); + + // 1 element, not yet satisfied + futures.push_back(s1); + auto c2 = collectAny(futures); + ASSERT_FALSE(c2->completed()); + s1->markCompleted(5); + ASSERT_TRUE(c2->completed()); + ASSERT_TRUE(c2->value().isInt()); + ASSERT_EQ(c2->value().toInt(), 5); + + // 1 element already satisfied. + auto c3 = collectAny(futures); + ASSERT_TRUE(c3->completed()); + ASSERT_TRUE(c3->value().isInt()); + ASSERT_EQ(c3->value().toInt(), 5); + + // 2 elements + futures.clear(); + auto s2 = c10::make_intrusive(IntType::get()); + auto s3 = c10::make_intrusive(IntType::get()); + futures.push_back(s2); + futures.push_back(s3); + auto c4 = collectAny(futures); + ASSERT_FALSE(c4->completed()); + s3->markCompleted(7); + ASSERT_TRUE(c4->completed()); + ASSERT_EQ(c4->value().toInt(), 7); + s2->markCompleted(1); + ASSERT_EQ(c4->value().toInt(), 7); +} + +TEST(TLSFutureCallbacksTest, Basic) { + // cb that verifies the profiler is enabled + auto profilerEnabledCb = [](Future& /* unused */) { + ASSERT_TRUE(torch::autograd::profiler::profilerEnabled()); + }; + // test running callbacks with propagation of TLS state. + { + // Enable the profiler in this thread + torch::autograd::profiler::enableProfilerLegacy( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::CPU, false, false)); + auto s1 = c10::make_intrusive(IntType::get()); + s1->addCallback(wrapPropagateTLSState(profilerEnabledCb)); + std::thread t([s1 = std::move(s1)]() { s1->markCompleted(); }); + // Since we join here, we can ensure that all callbacks corresponding to + // markCompleted() have finished. + t.join(); + torch::autograd::profiler::disableProfilerLegacy(); + } + // then() with TLS State + { + // Enable the profiler in this thread + torch::autograd::profiler::enableProfilerLegacy( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::CPU, false, false)); + auto s1 = c10::make_intrusive(IntType::get()); + auto s2 = s1->then( + wrapPropagateTLSState([&profilerEnabledCb](Future& s1) { + profilerEnabledCb(s1); + return at::IValue(1); + }), + IntType::get()); + std::thread t([s1 = std::move(s1)]() { s1->markCompleted(); }); + t.join(); + s2->wait(); + torch::autograd::profiler::disableProfilerLegacy(); + } +} + +TEST(ProfilerDisableInCallbackTest, Basic) { + // cb that verifies the profiler is enabled + auto profilerEnabledCb = []() { + ASSERT_TRUE(torch::autograd::profiler::profilerEnabled()); + }; + torch::autograd::profiler::enableProfilerLegacy( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::CPU, false, false)); + auto s1 = c10::make_intrusive(IntType::get()); + auto verifyProfilerCb = + wrapPropagateTLSState([&profilerEnabledCb](Future& /* unused */) { + // Ensure the profiler is still enabled in this thread. + profilerEnabledCb(); + auto t1 = torch::ones({2, 2}); + auto t2 = torch::ones({2, 2}); + torch::add(t1, t2); + // Don't cleanup TLSState, and just consolidate. + auto opts = + torch::autograd::profiler::ProfilerDisableOptions(false, true); + auto thread_event_lists = + // NOLINTNEXTLINE(performance-move-const-arg) + torch::autograd::profiler::disableProfilerLegacy(std::move(opts)); + // Ensure that the events from this thread are still profiled and we + // obtain the expected in events in our consolidated list when calling + // disableProfilerLegacy(). + bool found_ones = false; + bool found_add = false; + for (const auto& li : thread_event_lists) { + for (const auto& evt : li) { + if (strcmp(evt.name(), "aten::add") == 0) { + found_add = true; + } else if (strcmp(evt.name(), "aten::ones") == 0) { + found_ones = true; + } + } + if (found_add && found_ones) { + break; + } + } + ASSERT_TRUE(found_ones); + ASSERT_TRUE(found_add); + }); + + s1->addCallback(verifyProfilerCb); + // Disable the profiler, but do not consolidate results in the main thread. + auto opts = torch::autograd::profiler::ProfilerDisableOptions(true, false); + // NOLINTNEXTLINE(performance-move-const-arg) + torch::autograd::profiler::disableProfilerLegacy(std::move(opts)); + std::thread t([s1 = std::move(s1)]() { s1->markCompleted(at::IValue(1)); }); + t.join(); + + // Similar to above test, but verifies correctness in the case where + // continuation runs on the main thread. + torch::autograd::profiler::enableProfilerLegacy( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::CPU, false, false)); + s1 = c10::make_intrusive(IntType::get()); + s1->addCallback(verifyProfilerCb); + // Runs callback inline + s1->markCompleted(at::IValue(1)); + opts = torch::autograd::profiler::ProfilerDisableOptions(true, false); + // NOLINTNEXTLINE(performance-move-const-arg) + torch::autograd::profiler::disableProfilerLegacy(std::move(opts)); +} + +TEST(RecordDebugHandles, Basic) { + GTEST_SKIP() << "Test is flaky and sometimes hangs on CI. "; + // Enable the profiler in this thread + const std::set activities( + {torch::autograd::profiler::ActivityType::CPU}); + torch::autograd::profiler::prepareProfiler( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::KINETO, false, false), + activities); + torch::autograd::profiler::enableProfiler( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::KINETO, false, false), + activities); + { + RECORD_EDGE_SCOPE_WITH_DEBUG_HANDLE_AND_INPUTS("my_function", 42, {}); + float x{5.9999}, y{2.1212}; + float z = x / y; + (void)z; + } + { + RECORD_USER_SCOPE_WITH_INPUTS("not_my_function", {}); + float x{5.9999}, y{2.1212}; + float z = x / y; + (void)z; + } + auto profiler_results_ptr = torch::autograd::profiler::disableProfiler(); + const auto& kineto_events = profiler_results_ptr->events(); + size_t my_events{0}; + for (const auto& e : kineto_events) { + if (e.name() == "my_function") { + ASSERT_EQ(e.debugHandle(), 42); + my_events++; + } else if (e.name() == "not_my_function") { + ASSERT_EQ(e.debugHandle(), -1); + my_events++; + } + } + ASSERT_EQ(my_events, 2); +} + +TEST(RecordDebugHandles, ScopedCallbacks) { + // Enable the profiler in this thread + torch::autograd::profiler::prepareProfiler( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::KINETO, false, false), + {torch::autograd::profiler::ActivityType::CPU}); + torch::autograd::profiler::enableProfiler( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::KINETO, false, false), + {torch::autograd::profiler::ActivityType::CPU}); + + { + auto a = torch::rand({128, 128}); + auto b = torch::rand({128, 128}); + auto c = a + b; + } + auto profiler_results_ptr = torch::autograd::profiler::disableProfiler(); + ASSERT_TRUE(profiler_results_ptr->events().size() > 0); + + // Enable the profiler in this thread + torch::autograd::profiler::prepareProfiler( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::KINETO, false, false), + {torch::autograd::profiler::ActivityType::CPU}); + torch::autograd::profiler::enableProfiler( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::KINETO, false, false), + {torch::autograd::profiler::ActivityType::CPU}, + {at::RecordScope::LITE_INTERPRETER}); + { + auto a = torch::rand({128, 128}); + auto b = torch::rand({128, 128}); + auto c = a + b; + } + profiler_results_ptr = torch::autograd::profiler::disableProfiler(); + ASSERT_TRUE(profiler_results_ptr->events().size() == 0); + + torch::autograd::profiler::prepareProfiler( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::KINETO, false, false), + {torch::autograd::profiler::ActivityType::CPU}); + torch::autograd::profiler::enableProfiler( + torch::autograd::profiler::ProfilerConfig( + torch::autograd::profiler::ProfilerState::KINETO, false, false), + {torch::autograd::profiler::ActivityType::CPU}, + {at::RecordScope::LITE_INTERPRETER}); + { + RECORD_EDGE_SCOPE_WITH_DEBUG_HANDLE_AND_INPUTS("my_function", 42, {}); + auto a = torch::rand({128, 128}); + auto b = torch::rand({128, 128}); + auto c = a + b; + } + { + RECORD_USER_SCOPE_WITH_INPUTS("not_my_function", {}); + auto a = torch::rand({128, 128}); + auto b = torch::rand({128, 128}); + auto c = a + b; + } + profiler_results_ptr = torch::autograd::profiler::disableProfiler(); + const auto& kineto_events = profiler_results_ptr->events(); + for (const auto& e : kineto_events) { + if (e.name() == "my_function") { + ASSERT_EQ(e.debugHandle(), 42); + } + } + ASSERT_TRUE(profiler_results_ptr->events().size() == 1); +} + +TEST(IValueKWargsTest, Basic) { + const auto text = R"( + def foo(a : int, b : int, c : int = 4): + return a + 2*b + 3*c + )"; + auto cu = compile(text); + auto result = cu->get_function("foo")({1}, {{"b", 3}}); + ASSERT_EQ(result.toInt(), 19); +} + +TEST(ComputeFlopsTest, Basic) { + uint64_t flops = 0; + + // Test unknown operator + std::unordered_map extra_args; + flops = torch::profiler::impl::computeFlops( + std::string("aten::unknown"), extra_args); + ASSERT_EQ(flops, 0); + + // Test aten::conv2d + extra_args.clear(); + std::vector input_size = {4, 5, 6, 7}; + std::vector weight_size = {3, 5, 2, 1}; + std::vector padding = {1, 0}; + std::vector stride = {1, 1}; + std::vector dilation = {0, 0}; + extra_args["input_size"] = at::IValue(at::IntArrayRef(input_size)); + extra_args["weight_size"] = at::IValue(at::IntArrayRef(weight_size)); + extra_args["groups"] = 1; + extra_args["padding"] = at::IValue(at::IntArrayRef(padding)); + extra_args["stride"] = at::IValue(at::IntArrayRef(stride)); + extra_args["dilation"] = at::IValue(at::IntArrayRef(dilation)); + flops = torch::profiler::impl::computeFlops( + std::string("aten::conv2d"), extra_args); + ASSERT_EQ(flops, 13440); + + // Test aten::conv2d fail + input_size = {4, 5, 6, 7}; + weight_size = {4, 5, 6}; + extra_args["input_size"] = at::IValue(at::IntArrayRef(input_size)); + extra_args["weight_size"] = at::IValue(at::IntArrayRef(weight_size)); + flops = torch::profiler::impl::computeFlops( + std::string("aten::conv2d"), extra_args); + ASSERT_EQ(flops, 0); + + // Test aten::conv2d fail 2 + weight_size = {3, 5, 2, 1}; + stride = {0, 0}; + extra_args["weight_size"] = at::IValue(at::IntArrayRef(input_size)); + extra_args["stride"] = at::IValue(at::IntArrayRef(stride)); + flops = torch::profiler::impl::computeFlops( + std::string("aten::conv2d"), extra_args); + ASSERT_EQ(flops, 0); + + // Test aten::conv2d fail 3 + extra_args.clear(); + input_size = {4, 5, 6, 7}; + extra_args["input_size"] = at::IValue(at::IntArrayRef(input_size)); + flops = torch::profiler::impl::computeFlops( + std::string("aten::conv2d"), extra_args); + ASSERT_EQ(flops, 0); + + // Test aten::mm + extra_args.clear(); + std::vector mat1_sizes = {3, 4, 5, 6}; + std::vector mat2_sizes = {6, 5, 4, 3}; + extra_args["mat1_size"] = at::IValue(at::IntArrayRef(mat1_sizes)); + extra_args["mat2_size"] = at::IValue(at::IntArrayRef(mat2_sizes)); + flops = + torch::profiler::impl::computeFlops(std::string("aten::mm"), extra_args); + ASSERT_EQ(flops, 43200); + + // Test aten::addmm + flops = torch::profiler::impl::computeFlops( + std::string("aten::addmm"), extra_args); + ASSERT_EQ(flops, 43200); + + // Test aten::bmm + extra_args.clear(); + mat1_sizes = {7, 5, 6}; + mat2_sizes = {7, 6, 3}; + extra_args["mat1_size"] = at::IValue(at::IntArrayRef(mat1_sizes)); + extra_args["mat2_size"] = at::IValue(at::IntArrayRef(mat2_sizes)); + flops = + torch::profiler::impl::computeFlops(std::string("aten::bmm"), extra_args); + ASSERT_EQ(flops, 1260); + + // Test aten::baddbmm + flops = torch::profiler::impl::computeFlops( + std::string("aten::baddbmm"), extra_args); + ASSERT_EQ(flops, 1260); + + // Test mm out of range + extra_args.clear(); + flops = + torch::profiler::impl::computeFlops(std::string("aten::mm"), extra_args); + ASSERT_EQ(flops, 0); + + // Test aten::add.Tensor + extra_args.clear(); + std::vector mat_sizes = {3, 4, 5, 6}; + extra_args["mat_size"] = at::IValue(at::IntArrayRef(mat_sizes)); + flops = + torch::profiler::impl::computeFlops(std::string("aten::add"), extra_args); + ASSERT_EQ(flops, 360); + + // Test aten::mul.Tensor + extra_args.clear(); + mat_sizes = {3, 4, 5, 6}; + extra_args["mat_size"] = at::IValue(at::IntArrayRef(mat_sizes)); + flops = + torch::profiler::impl::computeFlops(std::string("aten::mul"), extra_args); + ASSERT_EQ(flops, 360); +} + +TEST(TestConstant, TensorGrad) { + auto graph = std::make_shared(); + IValue ten = torch::randn({3, 5}).requires_grad_(true); + auto con = tryInsertConstant(*graph, ten); + ASSERT_TRUE(con == std::nullopt); +} + +TEST(TestMutation, Basic) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(%x.1 : Tensor): + %2 : int = prim::Constant[value=1]() + %9 : int = prim::Constant[value=4]() + %x.3 : Tensor = aten::add(%x.1, %2, %2) + %7 : Tensor = aten::add_(%x.3, %2, %2) + %y.1 : Tensor = aten::add(%x.3, %9, %2) + return (%y.1))IR", + &*graph, + vmap); + RemoveTensorMutation(graph, [](Node*) { return false; }); + testing::FileCheck().check("aten::add_")->run(*graph); + RemoveTensorMutation(graph, [](Node*) { return true; }); + testing::FileCheck().check_not("aten::add_")->run(*graph); +} + +TEST(TestInplaceToFunctionalActivation, Basic) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(%x.1 : Tensor): + %2 : int = prim::Constant[value=1]() + %x.3 : Tensor = aten::add(%x.1, %2, %2) + %y : Tensor = aten::relu_(%x.3) + return (%y))IR", + &*graph, + vmap); + InplaceToFunctionalActivation(graph); + testing::FileCheck().check("aten::relu")->run(*graph); + testing::FileCheck().check_not("aten::relu_")->run(*graph); +} + +TEST(TestRegisterShapeOp, Basic) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(): + %2 : int = prim::Constant[value=5]() + %3: int[] = prim::ListConstruct(%2, %2) + return (%3))IR", + &*graph, + vmap); + + auto g2 = std::make_shared(); + parseIR( + R"IR( +graph(): + %2 : Tensor = prim::MakeTestTensor() + return (%2))IR", + &*g2, + vmap); + + const FunctionSchema& schema = g2->nodes().begin()->schema(); + torch::jit::RegisterShapeComputeGraphForSchema(schema, graph); + PropagateShapesOnGraph(g2); + testing::FileCheck().check("5, 5")->run(*g2); +} + +TEST(TestFunctionalToInplaceActivation, Basic) { + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR( + R"IR( +graph(%x.1 : Tensor): + %2 : int = prim::Constant[value=1]() + %x.3 : Tensor = aten::add(%x.1, %2, %2) + %y : Tensor = aten::relu(%x.3) + return (%y))IR", + &*graph, + vmap); + FunctionalToInplaceActivation(graph); + testing::FileCheck().check("aten::relu_")->run(*graph); + testing::FileCheck().check_not("aten::relu(")->run(*graph); +} + +TEST(TestFunctionExecutor, SimpleExecutorTest) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%x.1 : Tensor): + %2 : int = prim::Constant[value=1]() + %x.3 : Tensor = aten::add(%x.1, %2, %2) + %y : Tensor = aten::relu(%x.3) + return (%y))IR", + &*graph); + { + auto func = std::make_unique( + "name", graph, [](GraphFunction&) {}, ExecutorExecutionMode::PROFILING); + auto a = at::rand({2, 2, 2}, TensorOptions(kCPU).dtype(at::kFloat)); + Stack stack = {a}; + func->run(stack); + auto g = lastExecutedOptimizedGraph(); + testing::FileCheck() + .check("prim::profile") + ->check("aten::add") + ->check("aten::relu") + ->run(*g); + } + { + auto func = std::make_unique( + "name", graph, [](GraphFunction&) {}, ExecutorExecutionMode::SIMPLE); + auto a = at::rand({2, 2, 2}, TensorOptions(kCPU).dtype(at::kFloat)); + Stack stack = {a}; + func->run(stack); + auto g = func->getDebugState().graph; + testing::FileCheck() + .check_not("prim::profile") + ->check("aten::add") + ->check("aten::relu") + ->run(*g); + } +} + +TEST(TestFunctionExecutor, RunDecompositionTest) { + static auto* func = torch::jit::GetDecompositionExecutor( + "aten::var(Tensor self, bool unbiased=True) -> Tensor"); + for (bool unbiased : {true, false}) { + auto input = at::rand({4, 4}); + Stack stack = {input, unbiased}; + func->run(stack); + at::Tensor out = pop(stack).toTensor(); + ASSERT_TRUE(at::allclose(out, input.var(unbiased))); + } +} + +TEST(TestShapeGraphLinting, Basic) { + auto schemas = RegisteredShapeComputeSchemas(); + for (const auto& schema : schemas) { + // arange does not actually support complex, leave as + // union[int, float] for now + if (schema->name() == "aten::arange") { + continue; + } + auto g = shapeComputeGraphForSchema(*schema); + TORCH_INTERNAL_ASSERT(g); + LintShapeComputeGraph(schema, *g); + } +} + +// TODO: move to test_kernel when global settings are explicit +// fusion parameters +class Composed : public ::testing::Test { + public: + void SetUp() override { + torch::jit::tensorexpr::getTEMustUseLLVMOnCPU() = false; + } +}; + +TEST_F(Composed, ComposedOp) { + struct WithCPUFuser { + WithCPUFuser(bool val = true) : cpuFuserEnabled(canFuseOnCPU()) { + overrideCanFuseOnCPU(val); + } + + ~WithCPUFuser() { + overrideCanFuseOnCPU(cpuFuserEnabled); + } + + bool cpuFuserEnabled; + }; + +#ifdef TORCH_ENABLE_LLVM + const auto graph_string = R"IR( + graph(%0 : Float(5, 3, strides=[3, 1], device=cpu), + %1 : Float(5, 3, strides=[1, 5], device=cpu)): + %2 : Float(5, 3, strides=[3, 1], device=cpu) = aten::mul(%0, %1) + %3 : Float(5, 3, strides=[3, 1], device=cpu) = aten::mul(%0, %2) + %4 : Float(5, 3, strides=[3, 1], device=cpu) = aten::mul(%0, %3) + return (%3, %4))IR"; + auto graph = std::make_shared(); + parseIR(graph_string, &*graph); + + // wrong input sizes so we hit the fallback path + auto a = at::rand({2, 2, 2}, TensorOptions(kCPU).dtype(at::kFloat)); + auto b = at::rand({2, 2, 2}, TensorOptions(kCPU).dtype(at::kFloat)) + .transpose(0, 1); + auto ref1 = a * (a * b); + auto ref2 = a * ref1; + WithCPUFuser g(true); + bool fusable_on_device = torch::jit::tensorexpr::getTEMustUseLLVMOnCPU(); + torch::jit::tensorexpr::getTEMustUseLLVMOnCPU() = false; + FuseTensorExprs( + graph, + /*min_group_size*/ 2, + /*add_composed_op*/ true, + /*fuse_to_dynamic_shapes*/ true); + Code code(graph, ""); + InterpreterState interpreter{code}; + std::vector stack = {a, b}; + interpreter.run(stack); + at::Tensor out2 = pop(stack).toTensor(); + at::Tensor out1 = pop(stack).toTensor(); + ASSERT_TRUE(at::allclose(ref1, out1)); + ASSERT_TRUE(at::allclose(ref2, out2)); + + auto inp_1 = at::ones({4, 4}, TensorOptions(kCPU).dtype(at::kFloat)); + auto inp_2 = at::ones({4, 4}, TensorOptions(kCPU).dtype(at::kFloat)); + stack = {inp_1, inp_2, a, b}; + InterpreterState interpreter2{code}; + interpreter2.run(stack); + out2 = pop(stack).toTensor(); + out1 = pop(stack).toTensor(); + ASSERT_TRUE(at::allclose(ref1, out1)); + ASSERT_TRUE(at::allclose(ref2, out2)); + // inp_1 is on the bottom of the stack, and corresponds + // to the second output. inp_2 is on the top corresponds to first output + ASSERT_TRUE(at::allclose(inp_1, ref2)); + ASSERT_TRUE(at::allclose(inp_2, ref1)); + torch::jit::tensorexpr::getTEMustUseLLVMOnCPU() = fusable_on_device; +#endif +} + +TEST(ConstantPropagation, CustomClassesCanBePropagated) { +#if defined(__aarch64__) || defined(_M_ARM64) + // See https://github.com/pytorch/pytorch/issues/178522. + GTEST_SKIP() + << "Skipping ConstantPropagation.CustomClassesCanBePropagated on AArch64."; +#endif +#ifdef USE_PYTORCH_QNNPACK + const auto src = R"IR( + graph(): + %none: NoneType = prim::Constant() + %dim: int = prim::Constant[value=3]() + %shape: int[] = prim::ListConstruct(%dim, %dim) + %weight: Tensor = aten::ones(%shape, %none, %none, %none, %none) + %scale: float = prim::Constant[value=1.]() + %zero_point: int = prim::Constant[value=0]() + %dtype: int = prim::Constant[value=12]() + %weight_q: Tensor = aten::quantize_per_tensor(%weight, %scale, %zero_point, %dtype) + %params: __torch__.torch.classes.quantized.LinearPackedParamsBase = quantized::linear_prepack(%weight_q, %none) + return (%params) + )IR"; + auto graph = std::make_shared(); + std::unordered_map vmap; + parseIR(src, graph.get(), vmap); + + ConstantPropagation(graph); + + testing::FileCheck().check_not("quantized::linear_prepack")->run(*graph); +#endif +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_mobile_type_parser.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_mobile_type_parser.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6145fc002ec11e455b07a98b5d4cacc563ebd5fb --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_mobile_type_parser.cpp @@ -0,0 +1,231 @@ +#include +#include + +#include +#include + +namespace torch { +namespace jit { + +// Parse Success cases +TEST(MobileTypeParserTest, Int) { + std::string int_ps("int"); + auto int_tp = c10::parseType(int_ps); + EXPECT_EQ(*int_tp, *IntType::get()); +} + +TEST(MobileTypeParserTest, NestedContainersAnnotationStr) { + std::string tuple_ps( + "Tuple[str, Optional[float], Dict[str, List[Tensor]], int]"); + auto tuple_tp = c10::parseType(tuple_ps); + std::vector args = { + c10::StringType::get(), + c10::OptionalType::create(c10::FloatType::get()), + c10::DictType::create( + StringType::get(), ListType::create(TensorType::get())), + IntType::get()}; + auto tp = TupleType::create(std::move(args)); + ASSERT_EQ(*tuple_tp, *tp); +} + +TEST(MobileTypeParserTest, TorchBindClass) { + std::string tuple_ps("__torch__.torch.classes.rnn.CellParamsBase"); + auto tuple_tp = c10::parseType(tuple_ps); + std::string tuple_tps = tuple_tp->annotation_str(); + ASSERT_EQ(tuple_ps, tuple_tps); +} + +TEST(MobileTypeParserTest, ListOfTorchBindClass) { + std::string tuple_ps("List[__torch__.torch.classes.rnn.CellParamsBase]"); + auto tuple_tp = c10::parseType(tuple_ps); + EXPECT_TRUE(tuple_tp->isSubtypeOf(AnyListType::get())); + EXPECT_EQ( + "__torch__.torch.classes.rnn.CellParamsBase", + tuple_tp->containedType(0)->annotation_str()); +} + +TEST(MobileTypeParserTest, NestedContainersAnnotationStrWithSpaces) { + std::string tuple_space_ps( + "Tuple[ str, Optional[float], Dict[str, List[Tensor ]] , int]"); + auto tuple_space_tp = c10::parseType(tuple_space_ps); + // tuple_space_tps should not have weird white spaces + std::string tuple_space_tps = tuple_space_tp->annotation_str(); + ASSERT_TRUE(tuple_space_tps.find("[ ") == std::string::npos); + ASSERT_TRUE(tuple_space_tps.find(" ]") == std::string::npos); + ASSERT_TRUE(tuple_space_tps.find(" ,") == std::string::npos); +} + +TEST(MobileTypeParserTest, NamedTuple) { + std::string named_tuple_ps( + "__torch__.base_models.preproc_types.PreprocOutputType[" + " NamedTuple, [" + " [float_features, Tensor]," + " [id_list_features, List[Tensor]]," + " [label, Tensor]," + " [weight, Tensor]," + " [prod_prediction, Tuple[Tensor, Tensor]]," + " [id_score_list_features, List[Tensor]]," + " [embedding_features, List[Tensor]]," + " [teacher_label, Tensor]" + " ]" + " ]"); + + c10::TypePtr named_tuple_tp = c10::parseType(named_tuple_ps); + std::string named_tuple_annotation_str = named_tuple_tp->annotation_str(); + ASSERT_EQ( + named_tuple_annotation_str, + "__torch__.base_models.preproc_types.PreprocOutputType"); +} + +TEST(MobileTypeParserTest, DictNestedNamedTupleTypeList) { + std::string type_str_1( + "__torch__.base_models.preproc_types.PreprocOutputType[" + " NamedTuple, [" + " [float_features, Tensor]," + " [id_list_features, List[Tensor]]," + " [label, Tensor]," + " [weight, Tensor]," + " [prod_prediction, Tuple[Tensor, Tensor]]," + " [id_score_list_features, List[Tensor]]," + " [embedding_features, List[Tensor]]," + " [teacher_label, Tensor]" + " ]"); + std::string type_str_2( + "Dict[str, __torch__.base_models.preproc_types.PreprocOutputType]"); + std::vector type_strs = {type_str_1, type_str_2}; + std::vector named_tuple_tps = c10::parseType(type_strs); + EXPECT_EQ(*named_tuple_tps[1]->containedType(0), *c10::StringType::get()); + EXPECT_EQ(*named_tuple_tps[0], *named_tuple_tps[1]->containedType(1)); +} + +TEST(MobileTypeParserTest, NamedTupleNestedNamedTupleTypeList) { + std::string type_str_1( + " __torch__.ccc.xxx [" + " NamedTuple, [" + " [field_name_c_1, Tensor]," + " [field_name_c_2, Tuple[Tensor, Tensor]]" + " ]" + "]"); + std::string type_str_2( + "__torch__.bbb.xxx [" + " NamedTuple,[" + " [field_name_b, __torch__.ccc.xxx]]" + " ]" + "]"); + + std::string type_str_3( + "__torch__.aaa.xxx[" + " NamedTuple, [" + " [field_name_a, __torch__.bbb.xxx]" + " ]" + "]"); + + std::vector type_strs = {type_str_1, type_str_2, type_str_3}; + std::vector named_tuple_tps = c10::parseType(type_strs); + std::string named_tuple_annotation_str = named_tuple_tps[2]->annotation_str(); + ASSERT_EQ(named_tuple_annotation_str, "__torch__.aaa.xxx"); +} + +TEST(MobileTypeParserTest, NamedTupleNestedNamedTuple) { + std::string named_tuple_ps( + "__torch__.aaa.xxx[" + " NamedTuple, [" + " [field_name_a, __torch__.bbb.xxx [" + " NamedTuple, [" + " [field_name_b, __torch__.ccc.xxx [" + " NamedTuple, [" + " [field_name_c_1, Tensor]," + " [field_name_c_2, Tuple[Tensor, Tensor]]" + " ]" + " ]" + " ]" + " ]" + " ]" + " ]" + " ] " + "]"); + + c10::TypePtr named_tuple_tp = c10::parseType(named_tuple_ps); + std::string named_tuple_annotation_str = named_tuple_tp->str(); + ASSERT_EQ(named_tuple_annotation_str, "__torch__.aaa.xxx"); +} + +// Parse throw cases +TEST(MobileTypeParserTest, Empty) { + std::string empty_ps(""); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_ANY_THROW(c10::parseType(empty_ps)); +} + +TEST(MobileTypeParserTest, TypoRaises) { + std::string typo_token("List[tensor]"); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_ANY_THROW(c10::parseType(typo_token)); +} + +TEST(MobileTypeParserTest, MismatchBracketRaises) { + std::string mismatch1("List[Tensor"); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_ANY_THROW(c10::parseType(mismatch1)); +} + +TEST(MobileTypeParserTest, MismatchBracketRaises2) { + std::string mismatch2("List[[Tensor]"); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_ANY_THROW(c10::parseType(mismatch2)); +} + +TEST(MobileTypeParserTest, DictWithoutValueRaises) { + std::string mismatch3("Dict[Tensor]"); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_ANY_THROW(c10::parseType(mismatch3)); +} + +TEST(MobileTypeParserTest, ListArgCountMismatchRaises) { + // arg count mismatch + std::string mismatch4("List[int, str]"); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_ANY_THROW(c10::parseType(mismatch4)); +} + +TEST(MobileTypeParserTest, DictArgCountMismatchRaises) { + std::string trailing_commm("Dict[str,]"); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_ANY_THROW(c10::parseType(trailing_commm)); +} + +TEST(MobileTypeParserTest, ValidTypeWithExtraStuffRaises) { + std::string extra_stuff("int int"); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_ANY_THROW(c10::parseType(extra_stuff)); +} + +TEST(MobileTypeParserTest, NonIdentifierRaises) { + std::string non_id("(int)"); + // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) + ASSERT_ANY_THROW(c10::parseType(non_id)); +} + +TEST(MobileTypeParserTest, DictNestedNamedTupleTypeListRaises) { + std::string type_str_1( + "Dict[str, __torch__.base_models.preproc_types.PreprocOutputType]"); + std::string type_str_2( + "__torch__.base_models.preproc_types.PreprocOutputType[" + " NamedTuple, [" + " [float_features, Tensor]," + " [id_list_features, List[Tensor]]," + " [label, Tensor]," + " [weight, Tensor]," + " [prod_prediction, Tuple[Tensor, Tensor]]," + " [id_score_list_features, List[Tensor]]," + " [embedding_features, List[Tensor]]," + " [teacher_label, Tensor]" + " ]"); + std::vector type_strs = {type_str_1, type_str_2}; + std::string error_message = + R"(Can't find definition for the type: __torch__.base_models.preproc_types.PreprocOutputType)"; + ASSERT_THROWS_WITH_MESSAGE(c10::parseType(type_strs), error_message); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_module_api.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_module_api.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fc76d3e0f2ecb86fccd095612f6b63f783e64287 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_module_api.cpp @@ -0,0 +1,471 @@ +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +static constexpr std::string_view moduleInterfaceSrc = R"JIT( +class OneInterface(ModuleInterface): + def one(self, x: Tensor, y: Tensor) -> Tensor: + pass +)JIT"; + +static const std::vector subModuleMethodsSrc = {R"JIT( +def one(self, x: Tensor, y: Tensor) -> Tensor: + return self.attr * x + y + 1 + +def forward(self, x: Tensor) -> Tensor: + return self.attr + x +)JIT"}; + +static const std::string parentForward = R"JIT( +def forward(self, x: Tensor) -> Tensor: + return self.subMod1.one(x, x) + self.subMod2.one(x, x) +)JIT"; + +static void import_libs( + std::shared_ptr cu, + const std::string& class_name, + const std::shared_ptr& src, + const std::vector& tensor_table) { + SourceImporter si( + cu, + &tensor_table, + [&](const std::string& name) -> std::shared_ptr { return src; }, + /*version=*/2); + si.loadType(QualifiedName(class_name)); +} + +TEST(ModuleAPITest, MethodRunAsync) { + // Module m("m"); + // m.define(R"( + // def forward(self): + // r1 = torch.jit.fork(torch.mm, torch.rand(100,100),torch.rand(100,100)) + // r2 = torch.jit.fork(torch.mm, torch.rand(100,100),torch.rand(100,100)) + // return r1.wait() + r2.wait() + // )"); + // borrow model file from TEST(GraphExecutorTest, runAsync_executor) + auto testModelFile = + resolveTestDataFile(__FILE__, "test_interpreter_async.pt"); + auto m = load(testModelFile); + + auto counter = 0; + std::mutex mtx; + + auto launcher = [&](std::function f) { + mtx.lock(); + ++counter; + mtx.unlock(); + at::launch(std::move(f)); + }; + + auto method = m.get_method("forward"); + + std::vector stack; + auto kwargs = std::unordered_map(); + auto future = method.run_async(stack, kwargs, launcher); + + future->wait(); + + // expect 2 forks and 2 wait callbacks being executed on provided taskLauncher + // but ivalue::Future would be marked completed and release wait before + // finishing all callbacks + ASSERT_GE(counter, 2); +} + +TEST(ModuleAPITest, Clone) { + auto cu = std::make_shared(); + // creating child module + auto child = ClassType::create("child", cu, true); + auto attr_name = "attr"; + child->addAttribute(attr_name, IntType::get()); + Module c1(cu, child); + auto v1 = IValue(2); + c1.register_attribute(attr_name, IntType::get(), v1, false); + Module c2(cu, child); + auto v2 = IValue(3); + c2.register_attribute(attr_name, IntType::get(), v2, false); + + // attach two child module instance to parent that shares + // ClassType + auto parent = ClassType::create("parent", cu, true); + Module p(cu, parent); + p.register_attribute("c1", c1.type(), c1._ivalue(), false); + p.register_attribute("c2", c2.type(), c2._ivalue(), false); + + // clone parent + Module p2 = p.clone(); + // check the two child module has the same ClassType + ASSERT_EQ(p2.attr("c1").type(), p2.attr("c2").type()); + // but different instances + ASSERT_EQ(Module(p2.attr("c1").toObject()).attr(attr_name).toInt(), 2); + ASSERT_EQ(Module(p2.attr("c2").toObject()).attr(attr_name).toInt(), 3); +} + +TEST(ModuleAPITest, CloneWithModuleInterface) { + auto cu = std::make_shared(); + + // define a initial module with two submods share same interface + Module parentMod("parentMod", cu); + Module subMod1("subMod1", cu); + Module subMod2("subMod2", cu); + + std::vector constantTable; + import_libs( + cu, + "__torch__.OneInterface", + std::make_shared(moduleInterfaceSrc), + constantTable); + + auto v1 = IValue(2); + subMod1.register_attribute("attr", IntType::get(), v1, false); + + auto v2 = IValue(4); + subMod2.register_attribute("attr", IntType::get(), v2, false); + + for (const std::string& method : subModuleMethodsSrc) { + subMod1.define(method, nativeResolver()); + subMod2.define(method, nativeResolver()); + } + + parentMod.register_attribute( + "subMod1", + cu->get_interface("__torch__.OneInterface"), + subMod1._ivalue()); + parentMod.register_attribute( + "subMod2", + cu->get_interface("__torch__.OneInterface"), + subMod2._ivalue()); + + parentMod.define(parentForward, nativeResolver()); + + Module clonedMod = parentMod.clone(); + + // clone will copy both type and data, therefore we'll have a + // different type + ASSERT_NE(clonedMod.type(), parentMod.type()); +} + +TEST(ModuleAPITest, Copy) { + auto cu = std::make_shared(); + auto cls = ClassType::create("foo.bar", cu, true); + auto attr_name = "attr"; + cls->addAttribute(attr_name, IntType::get()); + Module m(cu, cls); + auto v = IValue(2); + m.register_attribute(attr_name, IntType::get(), v, false); + + Module m2 = m.clone(); + Module m3 = m.copy(); + + // Make sure copy works + ASSERT_EQ(m2.attr(attr_name).toInt(), 2); + ASSERT_EQ(m3.attr(attr_name).toInt(), 2); + + // clone will copy both type and data, therefore we'll have a + // different type + ASSERT_NE(m.type(), m2.type()); + // copy only copies data, type is shared + ASSERT_EQ(m.type(), m3.type()); + + // change value of copied instance + m3.register_attribute(attr_name, IntType::get(), IValue(3), false); + // Verify value of original instance doesn't change + ASSERT_EQ(m2.attr(attr_name).toInt(), 2); + ASSERT_EQ(m3.attr(attr_name).toInt(), 3); +} + +TEST(ModuleAPITest, DeepCopy) { + auto cu = std::make_shared(); + auto cls = ClassType::create("foo.bar", cu, true); + auto str_attr = "str_attr"; + auto int_attr = "int_attr"; + auto tensor_attr = "tensor_attr"; + auto tensor_list_attr = "tensor_list_attr"; + cls->addAttribute(int_attr, IntType::get()); + cls->addAttribute(str_attr, StringType::get()); + cls->addAttribute(tensor_attr, TensorType::get()); + cls->addAttribute(tensor_list_attr, ListType::ofTensors()); + Module m(cu, cls); + c10::List list({at::rand(5), at::rand(5)}); + m.setattr(int_attr, IValue(2)); + m.setattr(str_attr, IValue("str")); + m.setattr(tensor_attr, at::randn(5)); + m.setattr(tensor_list_attr, list); + + Module m2 = m.deepcopy(); + Module m3 = m.copy(); + // Make sure copy works + ASSERT_EQ(m2.attr(int_attr).toInt(), 2); + ASSERT_EQ(m3.attr(int_attr).toInt(), 2); + + // Test overlaps + ASSERT_TRUE(!IValue(m2._ivalue()).overlaps(IValue(m._ivalue()))); + ASSERT_TRUE(IValue(m3._ivalue()).overlaps(IValue(m._ivalue()))); + + // Both deepcopy and copy will preserve the type + ASSERT_EQ(m.type(), m2.type()); + ASSERT_EQ(m.type(), m3.type()); + + // change int value of copied instances + m2.setattr(int_attr, IValue(3)); + m3.setattr(int_attr, IValue(4)); + + // Verify value of original instance doesn't change + ASSERT_EQ(m.attr(int_attr).toInt(), 2); + ASSERT_EQ(m2.attr(int_attr).toInt(), 3); + ASSERT_EQ(m3.attr(int_attr).toInt(), 4); + + // change Tensor value of copied instances + at::Tensor t1 = m.attr(tensor_attr).toTensor(); + at::Tensor t2 = + m2.attr(tensor_attr).toTensor(); // deepcopy will copy the Tensor + at::Tensor t3 = + m3.attr(tensor_attr).toTensor(); // copy will not copy the Tensor + // check copy works + ASSERT_TRUE(t1.equal(t2)); + ASSERT_TRUE(t1.equal(t3)); + + // zero out t1 + t1.zero_(); + // check that t2 is not affected because it is a deep copy + ASSERT_TRUE(!t1.equal(t2)); + // check that t3 is the same as t1 since it is a shallow copy + ASSERT_TRUE(t1.equal(t3)); +} + +TEST(ModuleAPITest, DeepCopyString) { + auto cu = std::make_shared(); + auto cls = ClassType::create("foo.bar", cu, true); + auto attr1 = "attr1"; + cls->addAttribute(attr1, StringType::get()); + std::string str = "str"; + Module m(cu, cls); + m.setattr(attr1, str); + auto copied = m.deepcopy(); + auto original_str = str; + ASSERT_EQ(copied.attr(attr1).toStringRef(), original_str); + // check string mutation is not reflected in the copied module + str += "str"; + ASSERT_EQ(copied.attr(attr1).toStringRef(), original_str); +} + +TEST(ModuleAPITest, DeepCopyEnum) { + auto cu = std::make_shared(); + auto cls = ClassType::create("foo.bar", cu, true); + auto enum_attr = "enum_attr"; + auto int_enum_type = EnumType::create( + "enum_class", + IntType::get(), + {{"enum_name_1", 1}, {"enum_name_2", 2}}, + cu); + cls->addAttribute(enum_attr, int_enum_type); + Module m(cu, cls); + m.setattr( + enum_attr, + IValue(c10::make_intrusive( + int_enum_type, "enum_name_1", 1))); + Module m2 = m.deepcopy(); + + // Make sure deepcopy works + c10::ivalue::EnumHolder* m2_holder = m2.attr(enum_attr).toEnumHolder().get(); + ASSERT_EQ(m2_holder->value().toInt(), 1); + ASSERT_EQ(m2_holder->name(), "enum_name_1"); + ASSERT_EQ(m2_holder->type(), int_enum_type); + + // Test overlaps + ASSERT_TRUE(!IValue(m2._ivalue()).overlaps(IValue(m._ivalue()))); + + // Deepcopy will preserve the type + ASSERT_EQ(m.type(), m2.type()); + + // Change original, should not affect deepcopy + m.setattr( + enum_attr, + IValue(c10::make_intrusive( + int_enum_type, "enum_name_2", 2))); + ASSERT_NE( + m.attr(enum_attr).toEnumHolder().get()->value().toInt(), + m2.attr(enum_attr).toEnumHolder().get()->value().toInt()); +} + +TEST(ModuleAPITest, DeepCopyPreservesAliasing) { + // check deepcopy preserves aliasing + auto cu = std::make_shared(); + auto cls = ClassType::create("foo.bar", cu, true); + auto attr1 = "attr1"; + auto attr2 = "attr2"; + auto attr3 = "attr3"; + auto attr4 = "attr4"; + cls->addAttribute(attr1, ListType::ofTensors()); + cls->addAttribute(attr2, ListType::ofTensors()); + cls->addAttribute(attr3, TensorType::get()); + cls->addAttribute(attr4, TensorType::get()); + Module m(cu, cls); + auto t1 = at::rand(5); + auto t2 = at::rand(5); + auto t3 = at::rand(5); + auto t4 = at::rand({5, 2}); + c10::List list1({t1, t2}); + c10::List list2({t1, t3}); + // first element of attr1 and attr2 are aliased + m.setattr(attr1, list1); + m.setattr(attr2, list2); + m.setattr(attr3, t4); + m.setattr(attr4, t4.view(-1)); + + auto copied = m.deepcopy(); + // test tensor aliasing + auto copied_attr1_t1 = copied.attr(attr1).toList().get(0); + auto copied_attr2_t1 = copied.attr(attr2).toList().get(0); + ASSERT_TRUE(copied_attr1_t1.isAliasOf(copied_attr2_t1)); + + // test aliasing from view + auto copied_attr3 = copied.attr(attr3); + auto copied_attr4 = copied.attr(attr3); + ASSERT_TRUE(copied_attr3.isAliasOf(copied_attr4)); +} + +TEST(ModuleAPITest, Constants) { + auto cu = std::make_shared(); + auto cls = ClassType::create("foo.bar", cu, true); + auto attr_name = "attr"; + auto const_name = "const"; + cls->addAttribute(attr_name, IntType::get()); + cls->addConstant(const_name, IValue(3)); + Module m(cu, cls); + auto v = IValue(2); + m.register_attribute(attr_name, IntType::get(), v, false); + ASSERT_TRUE(m.hasattr(attr_name)); + ASSERT_TRUE(m.hasattr(const_name)); + ASSERT_EQ(m.attr(attr_name).toInt(), 2); + ASSERT_EQ(m.attr(const_name).toInt(), 3); +} + +TEST(ModuleAPITest, Parameters) { + auto cu = std::make_shared(); + auto cls = ClassType::create("foo.bar", cu, true); + Module m(cu, cls); + // Tensor parameter + m.register_parameter( + "tensor_param", at::empty({3}, at::kFloat), /* is_buffer */ false); + // None parameter + m.register_attribute( + "none_param", NoneType::get(), IValue(), /* is_param */ true); + m.register_attribute( + "none_param2", NoneType::get(), IValue(), /* is_param */ true); + auto param_list = m.parameters(); + ASSERT_EQ(param_list.size(), 1); + ASSERT_TRUE(m.hasattr("tensor_param")); + ASSERT_TRUE(m.hasattr("none_param")); + ASSERT_TRUE(m.hasattr("none_param2")); +} + +TEST(ModuleAPITest, Define) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def add_it(self, x, b : int = 4): + return self.foo + x + b + )"); + auto result = m.run_method("add_it", torch::ones({})); + AT_ASSERT(result.toTensor().item() == 6); +} + +TEST(ModuleAPITest, Freezing) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def forward(self, x, b : int = 4): + return self.foo + x + b + )"); + m.eval(); + auto forward_g = m.get_method("forward").graph(); + testing::FileCheck().check("GetAttr")->run(*forward_g); + + // Removal of GetAttr is done by freezing + auto frozen_mod = torch::jit::freeze(m); + forward_g = frozen_mod.get_method("forward").graph(); + testing::FileCheck().check_not("GetAttr")->run(*forward_g); + + // If no training mode is set, the module is NOT frozen by OFI + auto frozen_mod2 = torch::jit::optimize_for_inference(m); + forward_g = frozen_mod2.get_method("forward").graph(); + testing::FileCheck().check("GetAttr")->run(*forward_g); +} + +TEST(ModuleAPITest, OfiFreezesTraining) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def forward(self, x, b : int = 4): + return self.foo + x + b + )"); + m.register_attribute("training", BoolType::get(), true); + m.eval(); + + // Before freezing, we have a GetAttr check + auto forward_g = m.get_method("forward").graph(); + testing::FileCheck().check("GetAttr")->run(*forward_g); + + // Demonstrate that freezing happens when OFI is called + // Removal of GetAttr is done by freezing, but only when training + // attribute is set + auto frozen_mod = torch::jit::optimize_for_inference(m); + forward_g = frozen_mod.get_method("forward").graph(); + testing::FileCheck().check_not("GetAttr")->run(*forward_g); +} + +TEST(ModuleAPITest, OfiFreezesNoForward) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(R"( + def bar(self, x, b : int = 4): + return self.foo + x + b + )"); + m.eval(); + + // OFI is called without the presence of forward methods + auto frozen_mod = + torch::jit::optimize_for_inference(m, std::vector{"bar"}); + ASSERT_EQ( + m.run_method("bar", torch::ones({})).toTensor().item(), + frozen_mod.run_method("bar", torch::ones({})).toTensor().item()); +} + +TEST(ModuleAPITest, To_CUDA) { + Module m("test"); + { + // test cuda to cpu for params and buffers + m.register_parameter("foo", torch::ones({}, at::kCUDA), false); + m.register_buffer("bar", torch::ones({}, at::kCUDA)); + + m.to(at::kCUDA); + m.to(at::kCPU); + AT_ASSERT(m.attr("foo").toTensor().device().is_cpu()); + AT_ASSERT(m.attr("bar").toTensor().device().is_cpu()); + } + { + // test cpu to cuda for params and buffers + m.register_parameter("foo", torch::ones({}), false); + m.register_buffer("bar", torch::ones({})); + + m.to(at::kCUDA); + AT_ASSERT(m.attr("foo").toTensor().device().is_cuda()); + AT_ASSERT(m.attr("bar").toTensor().device().is_cuda()); + } +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_op_replacement.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_op_replacement.cpp new file mode 100644 index 0000000000000000000000000000000000000000..49bbc85946f95e2a8f76d067aed92d814263d1e0 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_op_replacement.cpp @@ -0,0 +1,145 @@ +#include + +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +std::unordered_map test_upgraders( + {{"_test_serialization_subcmul_0_2", R"IR(graph(%self.1 : Tensor, + %other.1 : Tensor, + %alpha.1 : Union(float, int)): + %7 : int = prim::Constant[value=1]() + %6 : Tensor = aten::mul(%self.1, %alpha.1) # torch/jit/operator_upgraders.py:18:20 + %8 : Tensor = aten::sub(%other.1, %6, %7) # torch/jit/operator_upgraders.py:18:11 + return (%8))IR"}, + {"div_Tensor_0_3", R"IR(graph(%self.1 : Tensor, + %other.1 : Tensor): + %32 : str = prim::Constant[value="trunc"]() + %6 : bool = prim::Constant[value=1]() + %4 : bool = aten::is_floating_point(%self.1) + %11 : bool = prim::If(%4) + block0(): + -> (%6) + block1(): + %9 : bool = aten::is_floating_point(%other.1) + -> (%9) + %35 : Tensor = prim::If(%11) + block0(): + %36 : Tensor = aten::div(%self.1, %other.1) + -> (%36) + block1(): + %37 : Tensor = aten::div(%self.1, %other.1, %32) + -> (%37) + return (%35))IR"}}); + +TEST(OpReplacementTest, ReplaceDivInSimpleFunction) { + const auto graph_string = R"IR( + graph(%0 : Tensor, + %1 : Tensor): + %2 : Tensor = aten::add(%0, %1) + %3 : Tensor = aten::div(%2, %1) + return (%3))IR"; + auto g = std::make_shared(); + test_only_populate_upgraders(test_upgraders); + torch::jit::parseIR(graph_string, g.get()); + g->set_op_version(2); + ReplaceOldOperatorsWithUpgraders(g); + testing::FileCheck() + .check("prim::If") + ->check_count("aten::div(%2, %1)", 1, /*exactly=*/true) + ->check_count("aten::div(%2, %1, %4)", 1, /*exactly=*/true) + ->run(*g); +} + +TEST(OpReplacementTest, ReplaceTwoOpsInSimpleFunction) { + const auto graph_string = R"IR( + graph(%0 : Tensor, + %1 : Tensor): + %2 : Tensor = aten::add(%0, %1) + %3 : Tensor = aten::div(%2, %1) + %4 : int = prim::Constant[value=1]() + %5: Tensor = aten::_test_serialization_subcmul(%0, %1, %4) + return (%3, %5))IR"; + auto g = std::make_shared(); + test_only_populate_upgraders(test_upgraders); + UpgraderEntry test_entry{ + 3, + "_test_serialization_subcmul_0_2", + "aten::_test_serialization_subcmul(Tensor self, Tensor other, Scalar alpha=2) -> Tensor"}; + test_only_add_entry("aten::_test_serialization_subcmul", test_entry); + torch::jit::parseIR(graph_string, g.get()); + g->set_op_version(2); + ReplaceOldOperatorsWithUpgraders(g); + testing::FileCheck() + .check("prim::If") + ->check_count("aten::div", 2, /*exactly=*/true) + ->run(*g); + test_only_remove_entry("aten::_test_serialization_subcmul"); + test_only_remove_upgraders(test_upgraders); +} + +TEST(OpReplacementTest, ReplaceDivInNestedFunction) { + const auto graph_string = R"IR( + graph(%0 : Tensor, + %1 : Tensor, + %8 : bool): + %9 : bool = prim::Constant[value=1]() + %7 : bool = prim::If(%8) + block0(): + -> (%9) + block1(): + %2 : Tensor = aten::add(%0, %1) + %3 : Tensor = aten::div(%2, %1) + %4 : Tensor = aten::add(%3, %0) + %10 : bool = aten::is_floating_point(%4) + -> (%10) + return (%7))IR"; + auto g = std::make_shared(); + test_only_populate_upgraders(test_upgraders); + torch::jit::parseIR(graph_string, g.get()); + g->set_op_version(2); + ReplaceOldOperatorsWithUpgraders(g); + testing::FileCheck() + .check("prim::If") + ->check_count("aten::add", 2, false) + ->run(*g); + + testing::FileCheck() + .check("prim::If") + ->check_count("aten::div", 2, false) + ->run(*g); + test_only_remove_upgraders(test_upgraders); +} + +TEST(OpReplacementTest, ReplaceTestSubcmulInSimpleFunction) { + const auto graph_string = R"IR( + graph(%0 : Tensor, + %1 : Tensor): + %3 : int = prim::Constant[value=1]() + %2 : Tensor = aten::_test_serialization_subcmul(%0, %1, %3) + return (%2))IR"; + auto g = std::make_shared(); + test_only_populate_upgraders(test_upgraders); + UpgraderEntry test_entry{ + 3, + "_test_serialization_subcmul_0_2", + "aten::_test_serialization_subcmul(Tensor self, Tensor other, Scalar alpha=2) -> Tensor"}; + test_only_add_entry("aten::_test_serialization_subcmul", test_entry); + torch::jit::parseIR(graph_string, g.get()); + g->set_op_version(2); + ReplaceOldOperatorsWithUpgraders(g); + testing::FileCheck().check_count("aten::mul", 1, false)->run(*g); + + testing::FileCheck().check_count("aten::sub", 1, false)->run(*g); + + test_only_remove_upgraders(test_upgraders); + test_only_remove_entry("aten::_test_serialization_subcmul"); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_peephole_optimize.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_peephole_optimize.cpp new file mode 100644 index 0000000000000000000000000000000000000000..263aef3e9be4dae386c0a839be8626458ca284b9 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_peephole_optimize.cpp @@ -0,0 +1,119 @@ +#include + +#include + +#include +#include +#include + +namespace torch { +namespace jit { + +TEST(PeepholeOptimizeTest, IsAndIsNot) +// test is / is not none optimization +{ + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%0 : int): + %1 : None = prim::Constant() + %2 : bool = aten::__is__(%0, %1) + %3 : bool = aten::__isnot__(%0, %1) + return (%2, %3) + )IR", + graph.get()); + PeepholeOptimize(graph); + testing::FileCheck() + .check_not("aten::__is__") + ->check_not("aten::__isnot__") + ->run(*graph); +} + +TEST(PeepholeOptimizeTest, IsAndIsNot2) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%0: int?): + %1 : None = prim::Constant() + %2 : bool = aten::__is__(%0, %1) + %3 : bool = aten::__isnot__(%0, %1) + return (%2, %3) + )IR", + graph.get()); + PeepholeOptimize(graph); + testing::FileCheck() + .check("aten::__is__") + ->check("aten::__isnot__") + ->run(*graph); +} + +TEST(PeepholeOptimizeTest, IsAndIsNot3) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%0: int?): + %1 : Tensor = prim::AutogradZero() + %2 : None = prim::Constant() + %4 : bool = aten::__is__(%0, %1) + %5 : bool = aten::__isnot__(%1, %2) + return (%4, %5) + )IR", + graph.get()); + PeepholeOptimize(graph); + testing::FileCheck() + .check("aten::__is__") + ->check_not("aten::__isnot__") + ->run(*graph); +} + +TEST(PeepholeOptimizeTest, UnwrapOptional) +// test unwrap optional +{ + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(): + %1 : Float(*, *, *) = prim::Constant() + %2 : bool = aten::_unwrap_optional(%1) + %3 : bool = prim::unchecked_unwrap_optional(%1) + return (%2, %3) + )IR", + graph.get()); + PeepholeOptimize(graph); + testing::FileCheck().check_not("unwrap")->run(*graph); +} + +TEST(PeepholeOptimizeTest, UnwrapOptional2) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%1 : Float(*, *, *)?): + %2 : bool = aten::_unwrap_optional(%1) + %3 : bool = prim::unchecked_unwrap_optional(%1) + return (%2, %3) + )IR", + graph.get()); + PeepholeOptimize(graph); + testing::FileCheck().check_count("unwrap", 2)->run(*graph); +} + +TEST(PeepholeOptimizeTest, AddMMFusion) { + auto graph = std::make_shared(); + parseIR( + R"IR( + graph( + %0 : Float(2, 3, 4), + %1 : Float(2, 3, 4), + %2 : Float(1, 1, 1)): + %3 : int = prim::Constant[value=1]() + %4 : Tensor = aten::mm(%0, %1) + %5 : Tensor = aten::add(%4, %2, %3) + %6 : Tensor = aten::add(%5, %2, %3) + return (%6) + )IR", + graph.get()); + FuseAddMM(graph); + testing::FileCheck().check("addmm")->run(*graph); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_qualified_name.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_qualified_name.cpp new file mode 100644 index 0000000000000000000000000000000000000000..32cb8b6e73b9130a2ba6c8d3e7d511bdfb19db97 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_qualified_name.cpp @@ -0,0 +1,74 @@ +#include + +#include +#include + +using c10::QualifiedName; + +namespace torch { +namespace jit { +TEST(QualifiedNameTest, PrefixConstruction) { + // Test prefix construction + auto foo = QualifiedName("foo"); + auto bar = QualifiedName(foo, "bar"); + auto baz = QualifiedName(bar, "baz"); + ASSERT_EQ(baz.qualifiedName(), "foo.bar.baz"); + ASSERT_EQ(baz.prefix(), "foo.bar"); + ASSERT_EQ(baz.name(), "baz"); + auto nullstate = QualifiedName(); + ASSERT_EQ(nullstate.qualifiedName(), ""); + ASSERT_EQ(nullstate.prefix(), ""); + ASSERT_EQ(nullstate.name(), ""); +} + +TEST(QualifiedNameTest, DottedConstruction) { + // Test dotted construction + auto foo = QualifiedName("foo.bar.baz"); + ASSERT_EQ(foo.qualifiedName(), "foo.bar.baz"); + ASSERT_EQ(foo.prefix(), "foo.bar"); + ASSERT_EQ(foo.name(), "baz"); + + auto bar = QualifiedName("bar"); + ASSERT_EQ(bar.qualifiedName(), "bar"); + ASSERT_EQ(bar.prefix(), ""); + ASSERT_EQ(bar.name(), "bar"); +} + +TEST(QualifiedNameTest, BadInputRaises) { + // throw some bad inputs at it + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_ANY_THROW(QualifiedName("foo..bar")); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_ANY_THROW(QualifiedName(".foo.bar")); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_ANY_THROW(QualifiedName("foo.bar.")); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_ANY_THROW(QualifiedName("")); +} + +TEST(QualifiedNameTest, Equality) { + // test equality api + auto foo1 = QualifiedName("foo.bar.baz"); + auto foo2 = QualifiedName("foo.bar.baz"); + auto foo3 = QualifiedName("bar.bar.baz"); + ASSERT_EQ(foo1, foo2); + ASSERT_NE(foo1, foo3); + auto bar1 = QualifiedName("sup"); + auto bar2 = QualifiedName("sup"); + ASSERT_EQ(foo1, foo2); +} + +TEST(QualifiedNameTest, IsPrefixOf) { + // test prefix api + auto foo1 = QualifiedName("foo.bar.baz"); + auto foo2 = QualifiedName("foo.bar"); + auto foo3 = QualifiedName("bar.bar.baz"); + auto foo4 = QualifiedName("foo.bar"); + ASSERT_TRUE(foo2.isPrefixOf(foo1)); + ASSERT_TRUE(foo2.isPrefixOf(foo4)); + ASSERT_TRUE(foo4.isPrefixOf(foo2)); + ASSERT_FALSE(foo1.isPrefixOf(foo2)); + ASSERT_FALSE(foo2.isPrefixOf(foo3)); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_save_load.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_save_load.cpp new file mode 100644 index 0000000000000000000000000000000000000000..dc29bbb3fbdf3db79c374c675f40d6e6cb43e177 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_save_load.cpp @@ -0,0 +1,365 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "caffe2/serialize/istream_adapter.h" + +namespace torch { +namespace jit { + +namespace { + +Module roundtripThroughMobile(const Module& m) { + ExtraFilesMap files; + std::vector constants; + jitModuleToPythonCodeAndConstants(m, &files, &constants); + CompilationOptions options; + mobile::Module mobilem = jitModuleToMobile(m, options); + return jitModuleFromSourceAndConstants( + mobilem._ivalue(), files, constants, 8); +} + +template +inline void expectThrowsEq(Functor&& functor, const char* expectedMessage) { + try { + std::forward(functor)(); + } catch (const Error& e) { + EXPECT_STREQ(e.what_without_backtrace(), expectedMessage); + return; + } + ADD_FAILURE() << "Expected to throw exception with message \"" + << expectedMessage << "\" but didn't throw"; +} + +} // namespace + +TEST(SerializationTest, ExtraFilesHookPreference) { + // Tests that an extra file written explicitly has precedence over + // extra files written by a hook + // TODO: test for the warning, too + const auto script = R"JIT( + def forward(self): + x = torch.rand(5, 5) + x = x.mm(x) + return x + )JIT"; + + auto module = + std::make_shared("Module", std::make_shared()); + module->define(script); + std::ostringstream oss; + std::unordered_map extra_files; + extra_files["metadata.json"] = "abc"; + SetExportModuleExtraFilesHook([](const Module&) -> ExtraFilesMap { + return {{"metadata.json", "def"}}; + }); + module->save(oss, extra_files); + SetExportModuleExtraFilesHook(nullptr); + + std::istringstream iss(oss.str()); + caffe2::serialize::IStreamAdapter adapter{&iss}; + std::unordered_map loaded_extra_files; + loaded_extra_files["metadata.json"] = ""; + auto loaded_module = torch::jit::load(iss, torch::kCPU, loaded_extra_files); + ASSERT_EQ(loaded_extra_files["metadata.json"], "abc"); +} + +TEST(SerializationTest, ExtraFileHooksNoSecret) { + // no secrets + std::stringstream ss; + { + Module m("__torch__.m"); + ExtraFilesMap extra; + extra["metadata.json"] = "abc"; + m.save(ss, extra); + } + ss.seekg(0); + { + ExtraFilesMap extra; + extra["metadata.json"] = ""; + extra["secret.json"] = ""; + jit::load(ss, std::nullopt, extra); + ASSERT_EQ(extra["metadata.json"], "abc"); + ASSERT_EQ(extra["secret.json"], ""); + } +} + +TEST(SerializationTest, ExtraFileHooksWithSecret) { + std::stringstream ss; + { + SetExportModuleExtraFilesHook([](const Module&) -> ExtraFilesMap { + return {{"secret.json", "topsecret"}}; + }); + Module m("__torch__.m"); + ExtraFilesMap extra; + extra["metadata.json"] = "abc"; + m.save(ss, extra); + SetExportModuleExtraFilesHook(nullptr); + } + ss.seekg(0); + { + ExtraFilesMap extra; + extra["metadata.json"] = ""; + extra["secret.json"] = ""; + jit::load(ss, std::nullopt, extra); + ASSERT_EQ(extra["metadata.json"], "abc"); + ASSERT_EQ(extra["secret.json"], "topsecret"); + } +} + +TEST(SerializationTest, TypeTags) { + auto list = c10::List>(); + list.push_back(c10::List({1, 2, 3})); + list.push_back(c10::List({4, 5, 6})); + auto dict = c10::Dict(); + dict.insert("Hello", torch::ones({2, 2})); + auto dict_list = c10::List>(); + for (size_t i = 0; i < 5; i++) { + auto another_dict = c10::Dict(); + another_dict.insert("Hello" + std::to_string(i), torch::ones({2, 2})); + dict_list.push_back(another_dict); + } + auto tuple = std::tuple(2, "hi"); + struct TestItem { + IValue value; + TypePtr expected_type; + }; + std::vector items = { + {list, ListType::create(ListType::create(IntType::get()))}, + {2, IntType::get()}, + {dict, DictType::create(StringType::get(), TensorType::get())}, + {dict_list, + ListType::create( + DictType::create(StringType::get(), TensorType::get()))}, + {tuple, TupleType::create({IntType::get(), StringType::get()})}}; + // NOLINTNEXTLINE(performance-for-range-copy) + for (auto item : items) { + auto bytes = torch::pickle_save(item.value); + auto loaded = torch::pickle_load(bytes); + ASSERT_TRUE(loaded.type()->isSubtypeOf(*item.expected_type)); + ASSERT_TRUE(item.expected_type->isSubtypeOf(*loaded.type())); + } +} + +TEST(SerializationTest, TestJitStream_CUDA) { + torch::jit::Module model; + std::vector inputs; + // Deserialize the ScriptModule from a file using torch::jit::load(). + // Load the scripted model. This should have been generated by tests_setup.py + // Refer: TorchSaveJitStream_CUDA in test/cpp/jit/tests_setup.py + model = torch::jit::load("saved_stream_model.pt"); + + auto output = model.forward(inputs); + const auto& list_of_elements = output.toTupleRef().elements(); + auto is_stream_s = list_of_elements[0].toBool(); + + // a,b: These are the two input tensors + // c: This is output tensor generated by the operation torch.cat(a,b) + auto a = list_of_elements[1].toTensor(); + auto b = list_of_elements[2].toTensor(); + auto c = list_of_elements[3].toTensor(); + // op: this is used to verify if the cat operation produced the same results + // as that on the GPU with torch.cat + auto op = at::cat({a, b}, 0); + + // Check if the stream is set + ASSERT_TRUE(is_stream_s); + // Check if the sizes of the outputs (op and c) is same on the GPU and CPU + ASSERT_EQ(op.sizes(), c.sizes()); + // Check if both the output tensors are equal + ASSERT_TRUE(op.equal(c)); +} + +TEST(TestSourceRoundTrip, UpsampleNearest2d) { + Module m("m"); + m.define(R"( + def forward(self, input: Tensor, scale:float): + return torch.upsample_nearest2d(input, [1, 1], float(scale), float(scale)) + )"); + + std::vector inputs; + inputs.emplace_back(torch::rand({1, 3, 128, 128})); + inputs.emplace_back(at::Scalar(2.0)); + auto ref = m.forward(inputs); + + Module m2 = roundtripThroughMobile(m); + auto res = m2.forward(inputs); + + auto resd = res.toTensor(); + auto refd = ref.toTensor(); + ASSERT_TRUE(resd.equal(refd)); +} + +TEST(TestSourceRoundTrip, CheckAttrAccess) { + Module m("m"); + m.register_attribute("mobile_optimized", BoolType::get(), true); + Module m2 = roundtripThroughMobile(m); + bool mobile_optimized = m2.attr("mobile_optimized", false).toBool(); + AT_ASSERT(mobile_optimized); +} + +TEST(TestSourceRoundTrip, + MethodInvocation) { // NOLINT (use =delete in gtest) + const std::vector test_programs{ + // test invoking a method with default parameter + R"( + def test_func(self, x, b : int = 4): + return self.foo + x + b + )", + // inner method call with default parameter (gets inlined) + R"( + def add_with_default_arg(self, x, b : int = 4): + return self.foo + x + b + def test_func(self, x): + return self.add_with_default_arg(x) # invoke method w/ default arg + )", + // simple method call + R"( + def test_func(self, x): + b = 4 + return self.foo + x + b + )", + }; + for (const auto& test_program : test_programs) { + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define(test_program); + + const int fortyTwo = 42; // (keep linter happy) + auto minput = fortyTwo * torch::ones({}); + auto ref = m.run_method("test_func", minput); + + Module m2 = roundtripThroughMobile(m); + const auto& test_func = m2.get_method("test_func"); + IValue res; + for (int i = 0; i < 3; ++i) { + res = test_func({minput}); + } + + auto resd = res.toTensor().item(); + auto refd = ref.toTensor().item(); + AT_ASSERT(resd == refd); + } +} + +TEST(SerializationTest, ParentDirNotExist) { + expectThrowsEq( + []() { + auto t = torch::nn::Linear(5, 5); + torch::save(t, "./doesnotexist/file.pt"); + }, + "Parent directory ./doesnotexist does not exist."); +} + +#ifdef WIN32 +TEST(SerializationTest, WindowsDrivePathTest) { + // "ZZZ" is typically not a valid drive letter. + // We expect to see "ZZZ:\\" or "ZZZ:/" in the error message. + // Note: slash should be included for the drive letter parent in Windows. + expectThrowsEq( + []() { + auto t = torch::nn::Linear(5, 5); + torch::save(t, "ZZZ:\\file.pt"); + }, + "Parent directory ZZZ:\\ does not exist."); + expectThrowsEq( + []() { + auto t = torch::nn::Linear(5, 5); + torch::save(t, "ZZZ:/file.pt"); + }, + "Parent directory ZZZ:/ does not exist."); +} + +TEST(SerializationTest, WindowsTempPathTest) { + // Test for verifying file saving and loading in the temporary folder + std::string temp_dir = std::getenv("TEMP"); + std::string file_path = temp_dir + "/file.pt"; + auto t1 = torch::tensor(1.0); + torch::save(t1, file_path); + torch::Tensor t2; + torch::load(t2, file_path); + ASSERT_TRUE(t1.allclose(t2, 0.0, 0.0)); +} +#endif + +TEST(SerializationTest, CalculateNecessaryArgsTest) { + auto schema = torch::schema( + "sync_stream(int stream_id = -1) -> ()", + c10::AliasAnalysisKind::CONSERVATIVE); + + auto graph = std::make_shared(); + auto one_val = graph->insertConstant(-1); + auto necessary = CalculateNecessaryArgs(schema.arguments(), {one_val}, true); + EXPECT_EQ(0, necessary.first); + EXPECT_EQ(0, necessary.second); +} + +TEST(TestSaveLoad, LoadWithoutDebugInfo) { // NOLINT (use =delete in gtest) + Module m("m"); + m.register_parameter("foo", torch::ones({}), false); + m.define( + R"( + def test_func(self, x): + b = 4 + return self.foo + x + b + )"); + m.define( + R"( + def exception(self): + assert False, "message" + )"); + std::stringstream ss; + m.save(ss); + ss.seekg(0); + caffe2::serialize::PyTorchStreamReader reader(&ss); + reader.setShouldLoadDebugSymbol(true); + EXPECT_TRUE(reader.hasRecord("code/__torch__.py.debug_pkl")); + reader.setShouldLoadDebugSymbol(false); + EXPECT_FALSE(reader.hasRecord("code/__torch__.py.debug_pkl")); + ss.seekg(0); + Module m2 = torch::jit::load(ss); + std::string error_msg = R"( + def exception(self): + assert False, "message" + ~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE)"; + ASSERT_THROWS_WITH_MESSAGE(m2.run_method("exception"), error_msg); + + ss.seekg(0); + // NO DEBUG trace so error message points to torchscript generated + // source instead of original python source. + std::string error2 = R"( + def exception(self: __torch__.m) -> NoneType: + _0 = uninitialized(NoneType) + ops.prim.RaiseException("AssertionError: message") + ~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + return _0 + )"; + Module m3 = torch::jit::load(ss, std::nullopt, false); + ASSERT_THROWS_WITH_MESSAGE(m3.run_method("exception"), error2); +} + +TEST(SerializationTest, TestPickleAppend) { + auto data = std::vector({'\x80', char(2), ']', 'K', char(2), 'a', '.'}); + + torch::IValue actual = torch::jit::unpickle(data.data(), data.size()); + + torch::IValue expected = c10::impl::GenericList(at::AnyType::get()); + expected.toList().push_back(2); + ASSERT_EQ(expected, actual); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_schema_info.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_schema_info.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3783d58ff40d61b6ff67725cc309a01c4efe0fa2 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_schema_info.cpp @@ -0,0 +1,394 @@ +#include +#include +#include + +namespace torch { +namespace utils { +using c10::SchemaArgType; + +TEST(FunctionSchemaIsAliasingTest, Basic) { + c10::FunctionSchema schema = torch::jit::parseSchema( + "aten::test.Tensor(Tensor(a) self, Tensor(b!) other, Tensor more_other) -> (Tensor(a), Tensor(b!))"); + ASSERT_TRUE(schema.is_aliasing({SchemaArgType::output, 0})); + ASSERT_TRUE(schema.is_aliasing({SchemaArgType::output, 1})); + ASSERT_TRUE(schema.is_aliasing({SchemaArgType::input, 0})); + ASSERT_TRUE(schema.is_aliasing({SchemaArgType::input, 1})); + ASSERT_FALSE(schema.is_aliasing({SchemaArgType::input, 2})); +} + +TEST(FunctionSchemaIsAliasingTest, InvalidArgument) { + c10::FunctionSchema schema = torch::jit::parseSchema( + "aten::sub_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> (Tensor(a!))"); + ASSERT_THROW(schema.is_aliasing({SchemaArgType::input, 4}), c10::Error); + ASSERT_THROW(schema.is_aliasing({SchemaArgType::output, 4}), c10::Error); +} + +TEST(FunctionSchemaIsMutableTest, Basic) { + c10::FunctionSchema schema = torch::jit::parseSchema( + "aten::sub_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> (Tensor(a!))"); + ASSERT_TRUE(schema.is_mutable({SchemaArgType::output, 0})); + ASSERT_TRUE(schema.is_mutable({SchemaArgType::input, 0})); + ASSERT_TRUE(schema.is_mutable("self")); + ASSERT_FALSE(schema.is_mutable({SchemaArgType::input, 1})); + ASSERT_FALSE(schema.is_mutable("other")); + ASSERT_FALSE(schema.is_mutable({SchemaArgType::input, 2})); + ASSERT_FALSE(schema.is_mutable("alpha")); +} + +TEST(FunctionSchemaIsMutableTest, InvalidArgument) { + c10::FunctionSchema schema = torch::jit::parseSchema( + "aten::sub_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> (Tensor(a!))"); + ASSERT_THROW(schema.is_mutable({SchemaArgType::input, 4}), c10::Error); + ASSERT_THROW(schema.is_mutable({SchemaArgType::output, 4}), c10::Error); + ASSERT_THROW(schema.is_mutable("named_argument"), c10::Error); +} + +TEST(SchemaInfoIsMutableTest, Basic) { + SchemaInfo schema( + "aten::sub_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> (Tensor(a!))"); + ASSERT_TRUE(schema.is_mutable({SchemaArgType::input, 0})); + ASSERT_TRUE(schema.is_mutable("self")); + ASSERT_FALSE(schema.is_mutable({SchemaArgType::input, 1})); + ASSERT_FALSE(schema.is_mutable("other")); + ASSERT_FALSE(schema.is_mutable({SchemaArgType::input, 2})); + ASSERT_FALSE(schema.is_mutable("alpha")); +} + +TEST(SchemaInfoIsMutableTest, InvalidArgument) { + SchemaInfo schema( + "aten::sub_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> (Tensor(a!))"); + ASSERT_THROW(schema.is_mutable({SchemaArgType::input, 4}), c10::Error); + ASSERT_THROW(schema.is_mutable("named_argument"), c10::Error); +} + +TEST(SchemaInfoIsMutableTest, AliasingInputs) { + SchemaInfo schema( + "aten::test.Tensor(Tensor(a!) self, Tensor(b) other, *, Scalar alpha=1) -> (Tensor(a!), Tensor(b))"); + ASSERT_TRUE(schema.is_mutable({SchemaArgType::input, 0})); + ASSERT_TRUE(schema.is_mutable({SchemaArgType::output, 0})); + ASSERT_TRUE(schema.is_mutable("self")); + ASSERT_FALSE(schema.is_mutable({SchemaArgType::input, 1})); + ASSERT_FALSE(schema.is_mutable({SchemaArgType::output, 1})); + ASSERT_FALSE(schema.is_mutable("other")); + at::Tensor input = at::randn({3, 3}); + schema.addArgumentValue("self", input); + schema.addArgumentValue("other", input); + ASSERT_TRUE(schema.is_mutable({SchemaArgType::input, 1})); + ASSERT_TRUE(schema.is_mutable({SchemaArgType::output, 1})); + ASSERT_TRUE(schema.is_mutable("other")); +} + +TEST(SchemaInfoIsMutableTest, InstanceNorm) { + SchemaInfo schema_info( + "aten::instance_norm(Tensor input, Tensor? weight, Tensor? bias, Tensor? running_mean, Tensor? running_var, bool use_input_stats, float momentum, float eps, bool cudnn_enabled) -> Tensor"); + ASSERT_TRUE(schema_info.is_mutable("running_mean")); + ASSERT_TRUE(schema_info.is_mutable("running_var")); + schema_info.addArgumentValue("use_input_stats", false); + ASSERT_FALSE(schema_info.is_mutable("running_mean")); + ASSERT_FALSE(schema_info.is_mutable("running_var")); +} + +TEST(SchemaInfoIsMutableTest, BatchNorm) { + SchemaInfo schema_info( + "aten::batch_norm(Tensor input, Tensor? weight, Tensor? bias, Tensor? running_mean, Tensor? running_var, bool training, float momentum, float eps, bool cudnn_enabled) -> Tensor"); + ASSERT_TRUE(schema_info.is_mutable("running_mean")); + ASSERT_TRUE(schema_info.is_mutable("running_var")); + schema_info.addArgumentValue("training", false); + ASSERT_FALSE(schema_info.is_mutable("running_mean")); + ASSERT_FALSE(schema_info.is_mutable("running_var")); +} + +TEST(SchemaInfoIsNonDeterministicTest, Basic) { + SchemaInfo deterministic_schema_info( + "aten::sub_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> (Tensor(a!))"); + SchemaInfo nondeterministic_schema_info( + "aten::bernoulli(Tensor self, *, Generator? generator) -> Tensor"); + ASSERT_FALSE(deterministic_schema_info.is_nondeterministic()); + ASSERT_TRUE(nondeterministic_schema_info.is_nondeterministic()); +} + +TEST(SchemaInfoIsNonDeterministicTest, Dropout) { + SchemaInfo droupout_schema_info( + "aten::dropout(Tensor input, float p, bool train) -> Tensor"); + ASSERT_TRUE(droupout_schema_info.is_nondeterministic()); + droupout_schema_info.addArgumentValue("train", false); + ASSERT_FALSE(droupout_schema_info.is_nondeterministic()); +} + +TEST(FunctionSchemaMayAliasTest, Basic) { + c10::FunctionSchema schema = torch::jit::parseSchema( + "aten::sub_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> (Tensor(a!))"); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::output, 0})); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::input, 1}, {SchemaArgType::output, 0})); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::input, 1}, {SchemaArgType::input, 0})); +} + +TEST(FunctionSchemaMayAliasTest, InvalidArgument) { + c10::FunctionSchema schema = torch::jit::parseSchema( + "aten::sub_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> (Tensor(a!))"); + ASSERT_THROW( + schema.may_alias({SchemaArgType::input, 15}, {SchemaArgType::output, 0}), + c10::Error); + ASSERT_THROW( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::output, 15}), + c10::Error); +} + +TEST(FunctionSchemaMayAliasTest, Wildcard) { + c10::FunctionSchema schema = torch::jit::parseSchema( + "aten::test.Tensor(Tensor(*) self) -> (Tensor(*), Tensor)"); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::output, 0}, {SchemaArgType::input, 0})); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::output, 1}, {SchemaArgType::input, 0})); +} + +TEST(SchemaInfoMayAliasTest, AliasingInputs) { + SchemaInfo schema( + "aten::sub.Tensor(Tensor self, Tensor other, *, Scalar alpha=1) -> Tensor"); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + at::Tensor input = at::randn({3, 3}); + schema.addArgumentValue("self", input); + schema.addArgumentValue("other", input); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::input, 1})); +} + +TEST(SchemaInfoMayAliasTest, AliasingOutputs) { + SchemaInfo schema( + "aten::aminmax.out(Tensor self, *, int? dim=None, bool keepdim=False, Tensor(a!) min, Tensor(b!) max) -> (Tensor(a!) min, Tensor(b!) max)"); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::output, 0}, {SchemaArgType::output, 1})); + at::Tensor input = at::randn({3, 3}); + schema.addArgumentValue("min", input); + schema.addArgumentValue("max", input); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::output, 0}, {SchemaArgType::output, 1})); +} + +TEST(SchemaInfoMayAliasTest, AliasingInputOutput) { + SchemaInfo schema( + "aten::sub_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> (Tensor(a!))"); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::output, 0})); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::input, 1}, {SchemaArgType::output, 0})); + at::Tensor input = at::randn({3, 3}); + schema.addArgumentValue("self", input); + schema.addArgumentValue("other", input); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::output, 0})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 1}, {SchemaArgType::output, 0})); +} + +TEST(SchemaInfoMayAliasTest, MultipleWildcardInputs) { + SchemaInfo schema( + "aten::test.Tensor(Tensor(a) a, Tensor(*) b, Tensor(*) c) -> (Tensor(a), Tensor(*))"); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::output, 0})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 1}, {SchemaArgType::output, 1})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 2}, {SchemaArgType::output, 1})); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::input, 2})); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::output, 1})); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::input, 1}, {SchemaArgType::output, 0})); + at::Tensor input = at::randn({3, 3}); + schema.addArgumentValue("a", input); + schema.addArgumentValue("b", input); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::output, 0})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 1}, {SchemaArgType::output, 1})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 2}, {SchemaArgType::output, 1})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::input, 2})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::output, 1})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 1}, {SchemaArgType::output, 0})); +} + +TEST(SchemaInfoMayAliasTest, MultipleNonWildcardInputs) { + SchemaInfo schema( + "aten::test.Tensor(Tensor(a) a, Tensor(a) b, Tensor(*) c, Tensor(b) d) -> (Tensor(a), Tensor(*))"); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::input, 2})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 2}, {SchemaArgType::input, 1})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 2}, {SchemaArgType::output, 0})); +} + +TEST(SchemaInfoMayAliasTest, MultipleNonWildcardOutputs) { + SchemaInfo schema( + "aten::test.Tensor(Tensor(a) a, Tensor(*) b) -> (Tensor(a), Tensor(a))"); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::output, 0}, {SchemaArgType::output, 1})); + ASSERT_TRUE( + schema.may_alias({SchemaArgType::output, 0}, {SchemaArgType::input, 1})); +} + +TEST(SchemaInfoMayAliasTest, MismatchingTypes) { + SchemaInfo schema("aten::test.Tensor(Tensor(a) a) -> int(a)"); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::input, 0}, {SchemaArgType::output, 0})); +} + +TEST(FunctionSchemaMayContainAliasTest, Basic) { + c10::FunctionSchema schema = torch::jit::parseSchema( + "aten::sub_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> (Tensor(a!))"); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::output, 0})); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::input, 1}, {SchemaArgType::output, 0})); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::input, 1}, {SchemaArgType::input, 0})); +} + +TEST(FunctionSchemaMayContainAliasTest, Wildcard) { + c10::FunctionSchema schema = torch::jit::parseSchema( + "aten::test.Tensor(Tensor(*) self) -> (Tensor[], Tensor)"); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::output, 0}, {SchemaArgType::input, 0})); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::input, 0})); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::input, 0}, false)); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::output, 0}, false)); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::output, 1}, {SchemaArgType::input, 0})); +} + +TEST(FunctionSchemaMayContainAliasTest, InputAndOutputContainers) { + c10::FunctionSchema schema = + torch::jit::parseSchema("aten::test.Tensor(Tensor[] self) -> Tensor[]"); + ASSERT_FALSE( + schema.may_alias({SchemaArgType::output, 0}, {SchemaArgType::input, 0})); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::input, 0})); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::input, 0}, false)); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::output, 0}, false)); +} + +TEST(SchemaInfoMayContainAliasTest, ContainAliasInputsEqual) { + SchemaInfo schema( + "aten::sub.Tensor(Tensor self, Tensor other, *, Scalar alpha=1) -> Tensor"); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + at::Tensor input = at::randn({3, 3}); + schema.addArgumentValue("self", input); + schema.addArgumentValue("other", input); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 1}, false)); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 1}, {SchemaArgType::input, 0}, false)); +} + +TEST(SchemaInfoMayContainAliasTest, ContainAliasInputsContained) { + SchemaInfo schema( + "aten::test.Tensor(Tensor[] self, Tensor other, *, Scalar alpha=1) -> Tensor"); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + at::Tensor input = at::randn({3, 3}); + schema.addArgumentValue("self", c10::List({input})); + schema.addArgumentValue("other", input); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 1}, false)); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::input, 1}, {SchemaArgType::input, 0}, false)); +} + +TEST(SchemaInfoMayContainAliasTest, ContainAliasOutputs) { + SchemaInfo schema( + "aten::aminmax.out(Tensor self, *, int? dim=None, bool keepdim=False, Tensor(a!) min, Tensor(b!) max) -> (Tensor(a!) min, Tensor(b!) max)"); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::output, 1})); + at::Tensor input = at::randn({3, 3}); + schema.addArgumentValue("min", input); + schema.addArgumentValue("max", input); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::output, 1})); +} + +TEST(SchemaInfoMayContainAliasTest, ContainAliasInputOutput) { + SchemaInfo schema( + "aten::test.tensor(Tensor(a) self, Tensor[] other) -> Tensor(a)"); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::input, 1})); + at::Tensor input = at::randn({3, 3}); + schema.addArgumentValue("other", c10::List({input})); + schema.addArgumentValue("self", input); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::input, 1})); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::input, 1}, false)); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 1}, {SchemaArgType::output, 0}, false)); +} + +TEST(SchemaInfoMayContainAliasTest, InputAndOutputContainers) { + SchemaInfo schema( + "aten::test.tensor(Tensor self, Tensor[] other) -> Tensor[]"); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::input, 1})); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::input, 0})); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + at::Tensor input = at::randn({3, 3}); + schema.addArgumentValue("other", c10::List({input})); + schema.addArgumentValue("self", input); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::input, 1})); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::output, 0}, {SchemaArgType::input, 0})); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 1})); +} + +TEST(SchemaInfoMayContainAliasTest, Wildcard) { + SchemaInfo schema( + "aten::test.tensor(Tensor a, Tensor[] b, Tensor(*) c) -> Tensor[]"); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 2})); + ASSERT_FALSE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 2}, {SchemaArgType::input, 1})); + at::Tensor input = at::randn({3, 3}); + schema.addArgumentValue("b", c10::List({input})); + schema.addArgumentValue("a", input); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 2})); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 0}, {SchemaArgType::input, 1})); + ASSERT_TRUE(schema.may_contain_alias( + {SchemaArgType::input, 2}, {SchemaArgType::input, 1})); +} +} // namespace utils +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_schema_matching.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_schema_matching.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b1fb962c7a06091a28fcd642a5c37e8564bee628 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_schema_matching.cpp @@ -0,0 +1,85 @@ +#include + +#include +#include +#include +#include + +#include +#include + +namespace torch { +namespace jit { + +TEST(SchemaMatchingTest, VarType) { + RegisterOperators reg({ + Operator( + "aten::test_vartype(t[] a, t b) -> (t)", + [](Stack& stack) { + auto [list, a] = pop, double>(stack); + push(stack, a); + }, + c10::AliasAnalysisKind::FROM_SCHEMA), + }); + Module m("m"); + m.define(R"( + def test(self): + a = (1.0, 2.0) + return torch.test_vartype(a, 2.0) + )"); + auto result = m.run_method("test"); + TORCH_INTERNAL_ASSERT(result.toDouble() == 2.0); + + const std::string error_example = R"JIT( + def test_2(self): + a = (1.0, 2.0) + non_float = (1, 1) + return torch.test_vartype(a, non_float) + )JIT"; + + std::string err = ""; + try { + m.define(error_example); + } catch (const std::exception& e) { + err = e.what(); + } + TORCH_INTERNAL_ASSERT( + err.find("previously matched to type") != std::string::npos); +} + +TEST(SchemaMatchingTest, VarType2) { + RegisterOperators reg({ + Operator( + "aten::test_vartype2(t a, t[] b) -> (t[])", + [](Stack& stack) { + auto [a, list] = pop>(stack); + push(stack, a); + }, + AliasAnalysisKind::FROM_SCHEMA), + }); + Module m("m"); + m.define(R"JIT( + def test(self): + a = (1.0, 2.0) + return torch.test_vartype2(3.0, a) + )JIT"); + auto result = m.run_method("test"); + TORCH_INTERNAL_ASSERT(result.toDouble() == 3.0); + + static const auto error_exam2 = R"JIT( + def test_2(self): + a = (1, 2) + return torch.test_vartype2(3.0, a) + )JIT"; + + std::string err = ""; + try { + m.define(error_exam2); + } catch (const std::exception& e) { + err = e.what(); + } + TORCH_INTERNAL_ASSERT( + err.find("previously matched to type") != std::string::npos); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_script_profile.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_script_profile.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f228d0c323d3916bc8930644757db0a4aec1d6b2 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_script_profile.cpp @@ -0,0 +1,62 @@ +#include + +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +TEST(ScriptProfileTest, Basic) { + const std::string source_string = R"V0G0N( + def foo(a, b): + return a + b # + )V0G0N"; + auto begin = source_string.find("return"); + auto end = source_string.find(" #"); + + Graph g; + const auto graph_string = R"IR( + graph(%a : Tensor, + %b : Tensor): + %2 : int = prim::Constant[value=1]() + %3 : Tensor = aten::add(%a, %b, %2) + return (%3))IR"; + + torch::jit::parseIR(graph_string, &g); + auto source = std::make_shared(source_string, "", 0); + auto node = *g.nodes().begin(); + node->setSourceRange(SourceRange{source, begin, end}); + + ScriptProfile p; + p.enable(); + { + profiling::InstructionSpan g0(*node); + profiling::InstructionSpan g1(*node); + profiling::InstructionSpan g2(*node); + } + p.disable(); + + auto stats = p.dumpStats(); + EXPECT_EQ(stats.size(), 1); + auto it = stats.find(*source.get()); + EXPECT_NE(it, stats.end()); + auto& lines = it->second; + EXPECT_EQ(lines.size(), 1); + const auto& stat = lines.at(source->lineno_for_offset(begin)); + EXPECT_EQ(stat.count, 3); +} + +TEST(ScriptProfileTest, CallingOrder) { + ScriptProfile p; + p.enable(); + EXPECT_THROW(p.dumpStats(), c10::Error); + p.disable(); + auto dp = std::make_shared(SourceRange{}); + EXPECT_THROW(p.addDatapoint(std::move(dp)), c10::Error); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_shape_analysis.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_shape_analysis.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b75c53f19835522c465a8b50dfa5eb64d06832cf --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_shape_analysis.cpp @@ -0,0 +1,499 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +namespace { + +Node* findNode(std::shared_ptr& g, Symbol k) { + DepthFirstGraphNodeIterator graph_it(g); + for (auto node = graph_it.next(); node != nullptr; node = graph_it.next()) { + if (node->kind() == k) { + return node; + } + } + TORCH_INTERNAL_ASSERT(false, "Couldn't find node"); +} +} // namespace + +TEST(ShapeAnalysisTest, DynamicShapesFusion) { + // Test Generalizing shapes to symbolic dimensions, guarding those symbolic + // dimensions and passing in runtime computed symbolic dimensions via inlined + // shape functions + std::shared_ptr subgraph = std::make_shared(); + const auto graph_string = R"IR( + graph(%x.1 : Tensor, %y.1 : Tensor, %z: Tensor): + %11 : int = prim::Constant[value=0]() + %3 : Tensor = aten::tanh(%x.1) + %out1.1 : Tensor = aten::erf(%3) + %out2.1 : Tensor = aten::relu(%y.1) + %10 : Tensor[] = prim::ListConstruct(%out1.1, %out2.1) + %25 : Tensor = aten::cat(%10, %11) + %28 : Tensor = aten::hardswish(%25) + %29 : Tensor = aten::mul(%28, %z) + return (%28))IR"; + torch::jit::parseIR(graph_string, subgraph.get()); + + /* + set up fused TensorExprGroup + */ + + std::shared_ptr g = std::make_shared(); + auto x_inp = g->addInput("x_inp"); + auto y_inp = g->addInput("y_inp"); + auto z_inp = g->addInput("z_inp"); + auto x_type = TensorType::create(at::rand({10, 5})); + auto y_type = TensorType::create(at::rand({4, 5})); + auto z_type = TensorType::create(at::rand({1, 1})); + x_inp->setType(x_type); + y_inp->setType(y_type); + z_inp->setType(z_type); + subgraph->inputs().at(0)->setType(x_type); + subgraph->inputs().at(1)->setType(y_type); + subgraph->inputs().at(2)->setType(z_type); + subgraph->outputs().at(0)->setType(TensorType::create(at::rand({14, 5}))); + auto output = g->insertNode(g->create(prim::TensorExprGroup))->output(); + subgraph->outputs().at(0)->setType(TensorType::create(at::rand({14, 5}))); + output->node()->addInput(x_inp); + output->node()->addInput(y_inp); + output->node()->addInput(z_inp); + output->node()->g_(attr::Subgraph, subgraph); + + auto success = GenerateGuard(output->node()); + TORCH_INTERNAL_ASSERT(success); + testing::FileCheck() + .check("TensorExprDynamicGuard") + ->check_next("prim::If") + ->check("aten::add") + ->check("TensorExprGroup") + ->check_same("symbolic_shape_inputs") + ->check("block1") + ->check("aten::cat") + ->run(*g); + + // clang-format off + /* Graph Should Look Something like: (note: strides not yet handled) + graph(%x_inp : Float(10, 5, strides=[5, 1], requires_grad=0, device=cpu), + %y_inp : Float(4, 5, strides=[5, 1], requires_grad=0, device=cpu), + %z_inp : Float(1, 1, strides=[1, 1], requires_grad=0, device=cpu)): + %4 : bool = prim::TensorExprDynamicGuard[types=[Float(SS(-2), SS(-3), strides=[5, 1], requires_grad=0, device=cpu), Float(SS(-4), SS(-3), strides=[5, 1], requires_grad=0, device=cpu), Float(1, 1, strides=[1, 1], requires_grad=0, device=cpu)]](%x_inp, %y_inp, %z_inp) + %5 : Tensor = prim::If(%4) + block0(): + %15 : int[] = aten::size(%x_inp) + %16 : int[] = aten::size(%y_inp) + %17 : int = prim::Constant[value=1]() + %18 : int = prim::Constant[value=0]() + %elem.3 : int = aten::__getitem__(%15, %18) # :40:10 + %elem.5 : int = aten::__getitem__(%15, %17) # :40:10 + %elem.11 : int = aten::__getitem__(%16, %18) # :40:10 + %cat_dim_size.48 : int = aten::add(%elem.3, %elem.11) # :321:29 + %3 : Tensor = prim::TensorExprGroup_0[symbolic_shape_inputs=[-5, -4, -3, -2]](%x_inp, %y_inp, %z_inp, %cat_dim_size.48, %elem.11, %elem.5, %elem.3) + -> (%3) + block1(): + // FallbackGraph is inlined + %14 : Tensor = prim::FallbackGraph_1(%x_inp, %y_inp, %z_inp) + -> (%14) + return () + with prim::TensorExprGroup_0 = graph(%x.1 : Float(SS(-2), SS(-3), strides=[5, 1], requires_grad=0, device=cpu), + %y.1 : Float(SS(-4), SS(-3), strides=[5, 1], requires_grad=0, device=cpu), + %z : Float(1, 1, strides=[1, 1], requires_grad=0, device=cpu), + %SS_5 : int, + %SS_4 : int, + %SS_3 : int, + %SS_2 : int): + %3 : int = prim::Constant[value=0]() + %4 : Tensor(SS(-2), SS(-3)) = aten::tanh(%x.1) + %5 : Tensor(SS(-2), SS(-3)) = aten::erf(%4) + %6 : Tensor(SS(-4), SS(-3)) = aten::relu(%y.1) + %7 : Tensor[] = prim::ListConstruct(%5, %6) + %8 : Tensor(SS(-5), SS(-3)) = aten::cat(%7, %3) + %9 : Tensor(SS(-5), SS(-3)) = aten::hardswish(%8) + %10 : Tensor(SS(-5), SS(-3)) = aten::mul(%9, %z) + return (%9) + */ + // clang-format on + + DepthFirstGraphNodeIterator graph_it(g); + Node* te_group = findNode(g, prim::TensorExprGroup); + + /* + Test that input to the kernel - (10, 5), (4, 5), (1, 1) - are correctly + generalized to sym dimensions, and that the output - (10 + 4, 5) + correctly preserves non-catted dim as sym shape and catted dim as new sym + shape + */ + + auto tensorexpr_graph = te_group->g(attr::Subgraph); + auto inp1 = tensorexpr_graph->inputs().at(0)->type()->expect(); + auto inp2 = tensorexpr_graph->inputs().at(1)->type()->expect(); + auto inp3 = tensorexpr_graph->inputs().at(2)->type()->expect(); + auto out = tensorexpr_graph->outputs().at(0)->type()->expect(); + + // 1 dims are preserved + auto inp3_sizes = inp3->sizes().concrete_sizes(); + TORCH_INTERNAL_ASSERT(inp3_sizes); + TORCH_INTERNAL_ASSERT( + inp3_sizes->size() == 2 && inp3_sizes->at(0) == 1 && + inp3_sizes->at(1) == 1); + + // 5 made into sym shape + ASSERT_EQ( + inp1->symbolic_sizes()[1].value(), inp2->symbolic_sizes()[1].value()); + ASSERT_EQ( + out->symbolic_sizes()[1].value(), inp2->symbolic_sizes()[1].value()); + + // 4, 10, 14 are different sym shapes + ASSERT_NE( + inp1->symbolic_sizes()[0].value(), inp2->symbolic_sizes()[0].value()); + ASSERT_NE( + out->symbolic_sizes()[0].value(), inp1->symbolic_sizes()[0].value()); + ASSERT_NE( + out->symbolic_sizes()[0].value(), inp2->symbolic_sizes()[0].value()); + + /* + Test guard behaves correctly at runtime and symbolic shapes are computed + correctly. As we don't have TE Kernel support for dynamic shapes we're + going to return all of the computed runtime symbolic dimensions as outputs + of the graph on guard success, and return None on guard failure + */ + + // Setting up guard to return sym shapes on guard success and None on failure + Node* if_node = findNode(g, prim::If); + IfView if_v(if_node); + if_node->eraseOutput(0); + if_v.thenBlock()->eraseOutput(0); + if_v.elseBlock()->eraseOutput(0); + WithInsertPoint guard(if_node); + auto none_val = g->insertConstant(IValue()); + + auto sym_shapes = te_group->is(Symbol::attr("symbolic_shape_inputs")); + auto offset = te_group->inputs().size() - sym_shapes.size(); + for (size_t i = 0; i < sym_shapes.size(); ++i) { + if_v.thenBlock()->insertOutput(i, te_group->inputs().at(offset + i)); + if_v.elseBlock()->insertOutput(i, none_val); + if_node->insertOutput(i)->setType(OptionalType::create(IntType::get())); + } + + auto new_outputs = g->createTuple(if_node->outputs())->insertAfter(if_node); + + g->registerOutput(new_outputs->output()); + te_group->destroy(); + findNode(g, prim::FallbackGraph)->destroy(); + + // Testing bad inputs + + auto first_inp = at::rand({2, 5}); + std::vector> second_inps = { + {at::rand({3, 4}), at::rand({1, 1})}, // sym shape mismatch + {at::rand({5, 2}).transpose(0, 1), at::rand({1, 1})}, // discontiguous + {at::zeros({2, 5}).to(at::ScalarType::Int), + at::rand({1, 1})}, // wrong dtype + {at::rand({2, 5, 1}), at::rand({1, 1})}, // wrong # dims + {at::rand({2, 5}).requires_grad_(true), + at::rand({1, 1})}, // requires grad + {at::rand({2, 5}), at::rand({1, 12})}, // concrete dim mismatch (1) + }; + if (torch::cuda::is_available()) { + second_inps.push_back({at::rand({2, 5}).cuda(), at::rand({1, 1})}); + } + for (const auto& last_inps : second_inps) { + // todo - reusing interpreter across iters gave error + Code code(g, ""); + InterpreterState interp(code); + auto stack = createStack({at::rand({2, 5}), last_inps[0], last_inps[1]}); + interp.run(stack); + TORCH_INTERNAL_ASSERT(pop(stack).toTuple()->elements().at(0).isNone()); + } + + // Test good inputs + Code code(g, ""); + InterpreterState interp(code); + std::vector inps = { + at::rand({2, 5}), at::rand({4, 5}), at::rand({1, 1})}; + Stack stack(inps.begin(), inps.end()); + interp.run(stack); + auto tuple = pop(stack).toTuple(); + TORCH_INTERNAL_ASSERT(tuple->elements().at(0).isInt()); + + // Testing that the sym shape calculation was correct + for (size_t i = 0; i < sym_shapes.size(); ++i) { + auto sym_shape = sym_shapes[i]; + auto computed_value = tuple->elements().at(i).toInt(); + if (sym_shape == inp1->symbolic_sizes().at(0).value()) { + ASSERT_EQ(computed_value, 2); + } else if (sym_shape == inp1->symbolic_sizes().at(1).value()) { + ASSERT_EQ(computed_value, 5); + } else if (sym_shape == inp2->symbolic_sizes().at(0).value()) { + ASSERT_EQ(computed_value, 4); + } else if (sym_shape == out->symbolic_sizes().at(0).value()) { + ASSERT_EQ(computed_value, 6); + } else { + TORCH_INTERNAL_ASSERT(false); + } + } +} + +TEST(ShapeAnalysisTest, MovingConstantOutOfFusionGroups) { + std::shared_ptr subgraph = std::make_shared(); + const auto graph_string = R"IR( + graph(%x.1 : Tensor): + %none : NoneType = prim::Constant() + %size1 : int = prim::Constant[value=1]() + %size10 : int = prim::Constant[value=10]() + %sizes : int[] = prim::ListConstruct(%size10, %size1) + %device : Device = prim::Constant[value="cpu"]() + %10 : Tensor = aten::ones(%sizes, %none, %none, %device, %none) + %3 : Tensor = aten::tanh(%x.1) + %29 : Tensor = aten::mul(%3, %10) + return (%29))IR"; + torch::jit::parseIR(graph_string, subgraph.get()); + ConstantPropagation(subgraph); + + std::shared_ptr g = std::make_shared(); + auto x_inp = g->addInput("x_inp"); + auto x_type = TensorType::create(at::rand({10, 5})); + x_inp->setType(x_type); + subgraph->inputs().at(0)->setType(x_type); + subgraph->outputs().at(0)->setType(x_type); + auto output = g->insertNode(g->create(prim::TensorExprGroup))->output(); + output->node()->addInput(x_inp); + output->node()->g_(attr::Subgraph, subgraph); + + auto success = GenerateGuard(output->node()); + TORCH_INTERNAL_ASSERT(success); + + // Check that the constants have been moved out of the fused graph. + // This should result in not have any conditionals other than the one + // checking the result of TensorExprDynamicGuard. + testing::FileCheck() + .check("TensorExprDynamicGuard") + ->check_next("prim::If") + ->check_not("prim::If") // no other IFs due to constants. + ->check("TensorExprGroup") + ->check("block1") + ->check("FallbackGraph") + ->run(*g); +} + +namespace { + +std::optional sym_dim = std::nullopt; + +// NOLINTNEXTLINE(bugprone-easily-swappable-parameters) +void assertShapeEqual(c10::SymbolicShape& a, c10::SymbolicShape& e) { + auto a_canonical = CanonicalizedSymbolicShape(a); + auto e_canonical = CanonicalizedSymbolicShape(e); + EXPECT_EQ(a_canonical, e_canonical); +} + +void assertShapeEqual( + std::optional>& actual, + std::vector> expected) { + ASSERT_TRUE(actual.has_value()); + ASSERT_EQ(actual->size(), 1); + + auto symb_expected = c10::SymbolicShape(expected); + assertShapeEqual(actual->at(0), symb_expected); +} + +const FunctionSchema* getSchema(const char* name) { + return &(getOperatorForLiteral(name)->schema()); +} +} // namespace + +TEST(ShapeAnalysisTest, SymbolicShapeAPI) { + // Figure out how to fetch a function schema + + // Ask someone else how to create a function schema / operator in C++ + auto schema = getSchema( + "aten::sub.Tensor(Tensor self, Tensor other, *, Scalar alpha=1) -> Tensor"); + + c10::IValue const_size_1 = std::vector{64, 56, 56}; + c10::IValue const_size_2 = std::vector{1, 56, 56}; + + // Check vector initializer list syntax + c10::SymbolicShape ss_concrete = + std::vector>{1, 56, 56}; + c10::SymbolicShape ss1 = std::vector>{sym_dim, 56, 56}; + c10::SymbolicShape ss2 = + std::vector>{64, sym_dim, sym_dim}; + c10::SymbolicShape ss3 = + std::vector>{sym_dim, sym_dim, sym_dim, sym_dim}; + + auto res = calculateSymbolicShapesOnOp( + schema, std::vector{const_size_1, const_size_1}); + assertShapeEqual(res, {64, 56, 56}); + + res = calculateSymbolicShapesOnOp( + schema, std::vector{const_size_1, const_size_2}); + assertShapeEqual(res, {64, 56, 56}); + + res = calculateSymbolicShapesOnOp( + schema, std::vector{const_size_1, ss1}); + assertShapeEqual(res, {64, 56, 56}); + + res = calculateSymbolicShapesOnOp( + schema, std::vector{const_size_2, ss1}); + assertShapeEqual(res, {sym_dim, 56, 56}); + + res = calculateSymbolicShapesOnOp( + schema, std::vector{ss_concrete, ss2}); + assertShapeEqual(res, {64, 56, 56}); + + res = calculateSymbolicShapesOnOp(schema, std::vector{ss2, ss3}); + assertShapeEqual(res, {sym_dim, 64, sym_dim, sym_dim}); +} + +TEST(ShapeAnalysisTest, BoundedSymbolicShapes) { + auto schema = getSchema("aten::nonzero(Tensor self) -> (Tensor)"); + + // Test that we generate symbolic shapes for the output of a nonzero op + c10::IValue const_size_1 = std::vector{5, 10}; + auto res = + calculateSymbolicShapesOnOp(schema, std::vector{const_size_1}); + assertShapeEqual(res, {sym_dim, 2}); + + // Test that nonzero can also create concrete shapes + c10::IValue const_size_2 = std::vector({1, 0}); + res = + calculateSymbolicShapesOnOp(schema, std::vector{const_size_2}); + assertShapeEqual(res, {0, 2}); +} + +TEST(ShapeAnalysisTest, SymbolicShapeCaching) { + clear_shape_cache(); + auto schema = getSchema("aten::mm(Tensor self, Tensor mat2) -> Tensor"); + + c10::IValue const_size_1 = std::vector{64, 56}; + c10::IValue const_size_2 = std::vector{64, 56}; + c10::IValue const_size_3 = std::vector{64, 20}; + + c10::SymbolicShape ss1 = c10::SymbolicShape({sym_dim, 64}); + c10::SymbolicShape ss2 = c10::SymbolicShape({sym_dim, 64}); + c10::SymbolicShape ss3 = c10::SymbolicShape({sym_dim, sym_dim}); + + auto res = calculateSymbolicShapesOnOp(schema, {ss1, const_size_1}); + assertShapeEqual(res, {sym_dim, 56}); + auto res1_val = res->at(0); + + // The exact same arguments should return the exact same result + res = calculateSymbolicShapesOnOp(schema, {ss1, const_size_1}); + auto res2_val = res->at(0); + EXPECT_EQ(res1_val, res2_val); + EXPECT_EQ(get_shape_cache_size(), 1); + + // Same shape but different symbols should return same shape + // but different symbolic indices + res = calculateSymbolicShapesOnOp(schema, {ss2, const_size_2}); + auto res3_val = res->at(0); + + assertShapeEqual(res3_val, res2_val); + EXPECT_NE(res3_val, res2_val); + EXPECT_EQ(get_shape_cache_size(), 1); + + // Different concrete shape should be cached separately + res = calculateSymbolicShapesOnOp(schema, {ss1, const_size_3}); + assertShapeEqual(res, {sym_dim, 20}); + EXPECT_EQ(get_shape_cache_size(), 2); + + res = calculateSymbolicShapesOnOp(schema, {ss3, const_size_3}); + assertShapeEqual(res, {sym_dim, 20}); + EXPECT_EQ(get_shape_cache_size(), 3); + + res = calculateSymbolicShapesOnOp(schema, {ss3, ss3}); + assertShapeEqual(res, {sym_dim, sym_dim}); + EXPECT_EQ(get_shape_cache_size(), 4); +} + +TEST(ShapeAnalysisTest, ShapeCacheMultipleFns) { + clear_shape_cache(); + + auto squeeze_op = + getSchema("aten::squeeze.dim(Tensor(a) self, int dim) -> Tensor(a)"); + auto mul_tensor = + getSchema("aten::mul.Tensor(Tensor self, Tensor other) -> Tensor"); + auto mul_scalar = + getSchema("aten::mul.Scalar(Tensor self, Scalar other) -> Tensor"); + auto div_tensor = + getSchema("aten::div.Tensor(Tensor self, Tensor other) -> Tensor"); + auto matmul = getSchema("aten::mm(Tensor self, Tensor mat2) -> Tensor"); + + c10::IValue const_int = 1; + + c10::SymbolicShape ss1 = c10::SymbolicShape({sym_dim, 64}); + + auto res = calculateSymbolicShapesOnOp(squeeze_op, {ss1, const_int}); + assertShapeEqual(res, {sym_dim, 64}); + + // Show that cache can handle multiple functions + res = calculateSymbolicShapesOnOp(mul_scalar, {ss1, const_int}); + assertShapeEqual(res, {sym_dim, 64}); + EXPECT_EQ(get_shape_cache_size(), 2); + + res = calculateSymbolicShapesOnOp(mul_tensor, {ss1, ss1}); + assertShapeEqual(res, {sym_dim, 64}); + EXPECT_EQ(get_shape_cache_size(), 3); + + // Even when the expected outcome is the same, should not collide + res = calculateSymbolicShapesOnOp(div_tensor, {ss1, ss1}); + assertShapeEqual(res, {sym_dim, 64}); + EXPECT_EQ(get_shape_cache_size(), 4); + + // Don't lose cached objects + res = calculateSymbolicShapesOnOp(mul_scalar, {ss1, const_int}); + assertShapeEqual(res, {sym_dim, 64}); + EXPECT_EQ(get_shape_cache_size(), 4); + + res = calculateSymbolicShapesOnOp(matmul, {ss1, ss1}); + // SSA can infer that sym_dim is 64 as both tensors + // use the same sym_dim + assertShapeEqual(res, {64, 64}); + EXPECT_EQ(get_shape_cache_size(), 5); +} + +TEST(ShapeAnalysisTest, TestShapeMultipleReturns) { + clear_shape_cache(); + + auto max_dim_op = getSchema( + "aten::max.dim(Tensor self, int dim, bool keepdim=False) -> (Tensor values, Tensor indices)"); + c10::IValue const_int = 1; + c10::IValue false_ival = false; + + c10::SymbolicShape ss1 = c10::SymbolicShape({sym_dim, 64}); + c10::SymbolicShape ss2 = c10::SymbolicShape({sym_dim, 64}); + + auto res = + calculateSymbolicShapesOnOp(max_dim_op, {ss1, const_int, false_ival}); + c10::SymbolicShape expected_res = + c10::SymbolicShape(std::vector>{sym_dim}); + assertShapeEqual(res->at(0), expected_res); + // res0 and res1 should share the same symbolic symbol + EXPECT_EQ(res->at(0), res->at(1)); + + // Also test that the shape cache also returns consistent result shapes + res = calculateSymbolicShapesOnOp(max_dim_op, {ss2, const_int, false_ival}); + assertShapeEqual(res->at(0), expected_res); + EXPECT_EQ(res->at(0), res->at(1)); + EXPECT_EQ(get_shape_cache_size(), 1); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_stack_opt.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_stack_opt.cpp new file mode 100644 index 0000000000000000000000000000000000000000..33eb90208c76f52c4f70bc668cbaa768fc63e61f --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_stack_opt.cpp @@ -0,0 +1,309 @@ +#include +#include + +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +TEST(StackOptTest, UseVariadicStack) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(56, 56, 56), + %1: Float(56, 56, 56), + %2: Float(56, 56, 56), + %3: Float(56, 56, 56), + %4: Float(56, 56, 56), + %5: Float(56, 56, 56)): + %10 : int = prim::Constant[value=0]() + %input : Tensor[] = prim::ListConstruct(%0, %1, %2, %3, %4, %5) + %stack : Float(5, 56, 56, 56) = aten::stack(%input, %10) + return (%stack) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(UseVariadicStack(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // After replacing `aten::stack` with `prim::VarStack` we should have the + // following graph: + // + // graph(%0 : ..., + // %1 : ...): + // %zero : int = prim:Constant[value=0]() + // %varstack : Tensor = prim::VarStack(%0, %1, %2, %3, %4, %5, %zero) + // return (%varstack) + testing::FileCheck() + .check_count("= prim::VarStack(", 1, /*exactly*/ true) + ->check_count("= aten::stack(", 0, /*exactly*/ true) + ->check_count("= prim::ListConstruct(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(StackOptTest, UseVariadicStackReplaceMultiple) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(56, 56, 56), + %1: Float(56, 56, 56), + %2: Float(56, 56, 56), + %3: Float(56, 56, 56)): + %10 : int = prim::Constant[value=0]() + %input1 : Tensor[] = prim::ListConstruct(%0, %1) + %stack1 : Float(4, 56, 56, 56) = aten::stack(%input1, %10) + %input2 : Tensor[] = prim::ListConstruct(%2, %3) + %stack2 : Float(4, 56, 56, 56) = aten::stack(%input2, %10) + return (%stack1, %stack2) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(UseVariadicStack(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // After full stack optimization we should have the following graph: + // + // graph(%0 : ..., + // %1 : ..., + // %2 : ..., + // %3 : ....): + // %zero : int = prim:Constant[value=0]() + // %varcat1 : Tensor = prim::VarStack(%0, %1, %zero) + // %varcat2 : Tensor = prim::VarStack(%2, %3, %zero) + // return (%varcat1, %varcat2) + testing::FileCheck() + .check_count("= prim::VarStack(", 2, /*exactly*/ true) + ->check_count("= aten::stack(", 0, /*exactly*/ true) + ->check_count("= prim::ListConstruct(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(StackOptTest, UseVariadicStackWithMultipleListUses) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(56, 56, 56), + %1: Float(56, 56, 56)): + %2 : int = prim::Constant[value=0]() + %input : Tensor[] = prim::ListConstruct(%0, %1) + %stack : Float(2, 56, 56, 56) = aten::stack(%input, %2) + return (%stack, %input) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({56, 56, 56}, at::kCPU), at::rand({56, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(UseVariadicStack(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // After replacing `aten::stack` with `prim::VarStack` we should have the + // following graph: + // + // graph(%0 : ..., + // %1 : ...): + // %zero : int = prim:Constant[value=0]() + // %input : Tensor[] = prim::ListConstruct(%0, %1) + // %varcat : Tensor = prim::VarStack(%0, %1, %zero) + // return (%varcat, %input) + testing::FileCheck() + .check_count("= prim::ListConstruct(", 1, /*exactly*/ true) + ->check_count("= prim::VarStack(", 1, /*exactly*/ true) + ->check_count("= aten::stack(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(StackOptTest, UseVariadicStackWithListMutationAfterCat) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(56, 56, 56), + %1: Float(56, 56, 56), + %2: Float(56, 56, 56)): + %10 : int = prim::Constant[value=0]() + %input : Tensor[] = prim::ListConstruct(%0, %1) + %stack : Float(3, 56, 56, 56) = aten::stack(%input, %10) + %11 : Tensor = aten::append(%input, %2) + return (%stack, %input) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(UseVariadicStack(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // The input list to `aten::stack` is mutated only after `aten::stack` op. So, + // it should have been replaced with `prim::VarStack`. The transformed graph + // should look like the following: + // + // graph(%0 : ..., + // %1 : ..., + // %2 : ...): + // %3 : int = prim:Constant[value=0]() + // %4 : Tensor[] = prim::ListConstruct(%0, %1) + // %7 : Tensor = prim::VarStack(%0, %1, %3) + // %6 : Tensor = aten::append(%4, %2) + // return (%7, %4) + testing::FileCheck() + .check_count("= prim::ListConstruct(", 1, /*exactly*/ true) + ->check_count("= prim::VarStack(", 1, /*exactly*/ true) + ->check_count("= aten::stack(", 0, /*exactly*/ true) + ->run(*graph); +} + +TEST(StackOptTest, UseVariadicStackWithListMutationBeforeCat) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(56, 56, 56), + %1: Float(56, 56, 56), + %2: Float(56, 56, 56)): + %10 : int = prim::Constant[value=0]() + %input : Tensor[] = prim::ListConstruct(%0, %1) + %11 : Tensor = aten::append(%input, %2) + %stack : Float(3, 56, 56, 56) = aten::stack(%input, %10) + return (%stack) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + { + ASSERT_FALSE(UseVariadicStack(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // No transformation should have happened since the `prim::ListConstruct` is + // mutated before `aten::stack`. + testing::FileCheck() + .check_count("= prim::ListConstruct(", 1, /*exactly*/ true) + ->check_count("= aten::stack(", 1, /*exactly*/ true) + ->check_count("= prim::VarStack(", 0, /*exactly*/ true) + ->run(*graph); + } + + { + ASSERT_TRUE(RemoveListMutationAndUseVariadicStack(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // The mutation of the list must be removed and the `aten::stack` op must + // be replaced with the `prim::VarStack` op in the graph. The transformed + // graph should look like the following: + // + // graph(%0 : ..., + // %1 : ..., + // %2 : ...): + // %3 : int = prim:Constant[value=0]() + // %7 : Tensor = prim::VarStack(%0, %1, %2, %3) + // return (%7) + testing::FileCheck() + .check_count("= prim::VarStack(", 1, /*exactly*/ true) + ->check_count("= prim::ListConstruct(", 0, /*exactly*/ true) + ->check_count("= aten::stack(", 0, /*exactly*/ true) + ->run(*graph); + } +} + +TEST(StackOptTest, UseVariadicStackWithMultipleListMutations) { + auto graph = std::make_shared(); + + const std::string input = + R"IR( + graph(%0: Float(56, 56, 56), + %1: Float(56, 56, 56), + %2: Float(56, 56, 56), + %3: Float(56, 56, 56), + %4: Float(56, 56, 56)): + %10 : int = prim::Constant[value=0]() + %input : Tensor[] = prim::ListConstruct(%0, %1) + %stack.1 : Float(5, 56, 56, 56) = aten::stack(%input, %10) + %11 : Tensor = aten::append(%input, %2) + %stack.2 : Float(5, 56, 56, 56) = aten::stack(%input, %10) + %12 : Tensor = aten::append(%input, %3) + %stack.3 : Float(5, 56, 56, 56) = aten::stack(%input, %10) + %13 : Tensor = aten::append(%input, %4) + %stack.4 : Float(5, 56, 56, 56) = aten::stack(%input, %10) + return (%stack.1, %stack.2, %stack.3, %stack.4) + )IR"; + parseIR(input, graph.get()); + std::vector inputs = { + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU), + at::rand({56, 56, 56}, at::kCPU)}; + auto orig_outputs = runGraph(graph, inputs); + + ASSERT_TRUE(RemoveListMutationAndUseVariadicStack(graph)); + graph->lint(); + auto opt_outputs = runGraph(graph, inputs); + ASSERT_TRUE(exactlyEqual(orig_outputs, opt_outputs)); + + // All the mutations of the list must be removed and the `aten::stack` ops + // must be replaced with `prim::VarStack` ops in the graph. The transformed + // graph should look like the following: + // + // graph(%0 : ..., + // %1 : ..., + // %2 : ..., + // %3 : ..., + // %4 : ...): + // %10 : int = prim:Constant[value=0]() + // %5 : Tensor = prim::VarStack(%0, %1, %10) + // %6 : Tensor = prim::VarStack(%0, %1, %2, %10) + // %7 : Tensor = prim::VarStack(%0, %1, %2, %3, %10) + // %8 : Tensor = prim::VarStack(%0, %1, %2, %3, %4, %10) + // return (%5, %6, %7, %8) + testing::FileCheck() + .check_count("= prim::VarStack(", 4, /*exactly*/ true) + ->check_count("= prim::ListConstruct(", 0, /*exactly*/ true) + ->check_count("= aten::stack(", 0, /*exactly*/ true) + ->run(*graph); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_subgraph_matcher.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_subgraph_matcher.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2dafcb4b64023fddee12cc934b8e073406ac46ec --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_subgraph_matcher.cpp @@ -0,0 +1,569 @@ +#include + +#include "test/cpp/jit/test_utils.h" +#include "torch/csrc/jit/ir/subgraph_matcher.h" + +namespace torch { +namespace jit { + +TEST(SubgraphMatcherTest, Trivial1) { + Graph graph, pattern; + parseIR( + R"IR( +graph(%0): + %a = a::aaa(%0) + return (%a))IR", + &graph); + parseIR( + R"IR( +graph(%0): + %x = a::aaa(%0) + return (%x))IR", + &pattern); + AT_ASSERT(!findPatternMatches(pattern, graph).empty()); +} + +TEST(SubgraphMatcherTest, Trivial2) { + Graph graph; + auto* g_in = graph.addInput(); + auto* g_tanh = graph.insertNode(graph.create(aten::tanh, /*num_outputs =*/1)); + g_tanh->addInput(g_in); + graph.registerOutput(g_tanh->output()); + + Graph pattern; + auto* p_in = pattern.addInput(); + auto* p_tanh = + pattern.insertNode(pattern.create(aten::tanh, /*num_outputs =*/1)); + p_tanh->addInput(p_in); + pattern.registerOutput(p_tanh->output()); + + auto matches = findPatternMatches(pattern, graph); + AT_ASSERT(matches.size() == 1); + for (const Match& m : matches) { + AT_ASSERT(m.values_map.at(p_in) == g_in); + AT_ASSERT(m.values_map.at(p_tanh->output()) == g_tanh->output()); + AT_ASSERT(m.nodes_map.at(p_tanh) == g_tanh); + } +} + +TEST(SubgraphMatcherTest, Trivial3) { + Graph graph, pattern; + parseIR( + R"IR( +graph(%0): + %a = a::a(%0) + %b = a::b(%0) + %c = a::c(%a, %b) + return (%c))IR", + &graph); + parseIR( + R"IR( +graph(%a, %b): + %c = a::c(%a, %b) + return (%c))IR", + &pattern); + AT_ASSERT(!findPatternMatches(pattern, graph).empty()); +} + +TEST(SubgraphMatcherTest, Trivial4) { + Graph graph; + auto* g_in0 = graph.addInput(); + auto* g_in1 = graph.addInput(); + auto* g_mul = graph.insertNode(graph.create(aten::mul, /*num_outputs =*/1)); + g_mul->addInput(g_in0); + g_mul->addInput(g_in1); + graph.registerOutput(g_mul->output()); + + Graph pattern; + auto* p_in0 = pattern.addInput(); + auto* p_in1 = pattern.addInput(); + auto* p_mul = + pattern.insertNode(pattern.create(aten::mul, /*num_outputs =*/1)); + p_mul->addInput(p_in0); + p_mul->addInput(p_in1); + pattern.registerOutput(p_mul->output()); + + auto matches = findPatternMatches(pattern, graph); + AT_ASSERT(matches.size() == 1); + for (const Match& m : matches) { + AT_ASSERT(m.values_map.at(p_in0) == g_in0); + AT_ASSERT(m.values_map.at(p_in1) == g_in1); + AT_ASSERT(m.values_map.at(p_mul->output()) == g_mul->output()); + AT_ASSERT(m.nodes_map.at(p_mul) == g_mul); + } +} + +TEST(SubgraphMatcherTest, Linear1) { + Graph graph, pattern; + parseIR( + R"IR( +graph(%0): + %a = a::aaa(%0) + %b = b::bbb(%a) + %c = c::ccc(%b) + %d = d::ddd(%c) + %a = a::aaa(%0) + return (%d))IR", + &graph); + parseIR( + R"IR( +graph(%0): + %x = b::bbb(%0) + %y = c::ccc(%x) + return (%y))IR", + &pattern); + AT_ASSERT(!findPatternMatches(pattern, graph).empty()); +} + +TEST(SubgraphMatcherTest, Linear2) { + Graph graph; + auto* g_in = graph.addInput(); + + auto* g_tanh = graph.insertNode(graph.create(aten::tanh, /*num_outputs =*/1)); + g_tanh->addInput(g_in); + + auto* g_tanh2 = + graph.insertNode(graph.create(aten::tanh, /*num_outputs =*/1)); + g_tanh2->addInput(g_tanh->output()); + + graph.registerOutput(g_tanh2->output()); + + Graph pattern; + auto* p_in = pattern.addInput(); + + auto* p_tanh = + pattern.insertNode(pattern.create(aten::tanh, /*num_outputs =*/1)); + p_tanh->addInput(p_in); + + auto* p_tanh2 = + pattern.insertNode(pattern.create(aten::tanh, /*num_outputs =*/1)); + p_tanh2->addInput(p_tanh->output()); + + pattern.registerOutput(p_tanh2->output()); + + auto matches = findPatternMatches(pattern, graph); + AT_ASSERT(matches.size() == 1); + for (const Match& m : matches) { + AT_ASSERT(m.values_map.at(p_in) == g_in); + AT_ASSERT(m.values_map.at(p_tanh->output()) == g_tanh->output()); + AT_ASSERT(m.values_map.at(p_tanh2->output()) == g_tanh2->output()); + AT_ASSERT(m.nodes_map.at(p_tanh) == g_tanh); + AT_ASSERT(m.nodes_map.at(p_tanh2) == g_tanh2); + } +} + +/** + * Test diamond pattern: + * + * ooo + * | + * aaa + * / \ + * bbb ccc + * \ / + * ddd + * | + * eee + */ +TEST(SubgraphMatcherTest, Diamond1) { + Graph graph, pattern1, pattern2; + parseIR( + R"IR( +graph(%0): + %o = o::ooo(%0) + %a = a::aaa(%o) + %b = b::bbb(%a) + %c = c::ccc(%a) + %d = d::ddd(%b, %c) + %e = e::eee(%d) + return (%e))IR", + &graph); + + parseIR( + R"IR( +graph(%0): + %a = a::aaa(%0) + %b = b::bbb(%a) + %c = c::ccc(%a) + %d = d::ddd(%b, %c) + return (%d))IR", + &pattern1); + AT_ASSERT(!findPatternMatches(pattern1, graph).empty()); + + // Check that order of nodes inside the diamond does not affect the result + parseIR( + R"IR( +graph(%0): + %a = a::aaa(%0) + %c = c::ccc(%a) + %b = b::bbb(%a) + %d = d::ddd(%b, %c) + return (%d))IR", + &pattern2); + AT_ASSERT(!findPatternMatches(pattern2, graph).empty()); +} + +/** + * Test diamond pattern: + * + * i0 + * | + * chunk + * / \ + * os[0] os[1] + * \ / + * * + * | + * o1 + */ +TEST(SubgraphMatcherTest, Diamond2) { + Graph graph; + auto* g_in = graph.addInput(); + + auto* g_chunk = + graph.insertNode(graph.create(prim::ConstantChunk, /*num_outputs =*/2)); + g_chunk->i_(attr::chunks, 2)->i_(attr::dim, 0); + g_chunk->addInput(g_in); + + auto* g_mul = graph.insertNode(graph.create(aten::mul, /*num_outputs =*/1)); + g_mul->addInput(g_chunk->outputs()[0]); + g_mul->addInput(g_chunk->outputs()[1]); + graph.registerOutput(g_mul->output()); + + Graph pattern; + auto* p_in = pattern.addInput(); + auto* p_chunk = pattern.insertNode( + pattern.create(prim::ConstantChunk, /*num_outputs =*/2)); + p_chunk->i_(attr::chunks, 2)->i_(attr::dim, 0); + p_chunk->addInput(p_in); + + auto* p_mul = + pattern.insertNode(pattern.create(aten::mul, /*num_outputs =*/1)); + p_mul->addInput(p_chunk->outputs()[0]); + p_mul->addInput(p_chunk->outputs()[1]); + pattern.registerOutput(p_mul->output()); + + auto matches = findPatternMatches(pattern, graph); + AT_ASSERT(matches.size() == 1); + for (const Match& m : matches) { + AT_ASSERT(m.values_map.at(p_in) == g_in); + AT_ASSERT(m.values_map.at(p_chunk->outputs()[0]) == g_chunk->outputs()[0]); + AT_ASSERT(m.values_map.at(p_chunk->outputs()[1]) == g_chunk->outputs()[1]); + AT_ASSERT(m.values_map.at(p_mul->output()) == g_mul->output()); + AT_ASSERT(m.nodes_map.at(p_mul) == g_mul); + } +} + +TEST(SubgraphMatcherTest, XPattern) { + Graph graph, pattern; + parseIR( + R"IR( +graph(%0, %1): + %b = b::bbb(%0) + %c = c::ccc(%1) + %x = x::xxx(%b, %c) + %e = e::eee(%x) + %f = f::fff(%x) + %g = g::ggg(%e, %f) + return (%g))IR", + &graph); + parseIR( + R"IR( +graph(%0, %1): + %b = b::bbb(%0) + %c = c::ccc(%1) + %x = x::xxx(%b, %c) + %e = e::eee(%x) + %f = f::fff(%x) + %g = g::ggg(%e, %f) + return (%g))IR", + &pattern); + AT_ASSERT(!findPatternMatches(pattern, graph).empty()); +} + +TEST(SubgraphMatcherTest, MultipleMatches) { + Graph graph, pattern; + parseIR( + R"IR( +graph(%t0): + %t1 = a::aaa(%t0) + %t2 = a::aaa(%t1) + %t3 = a::aaa(%t2) + %t4 = a::aaa(%t3) + return (%t4))IR", + &graph); + parseIR( + R"IR( +graph(%t0): + %t1 = a::aaa(%t0) + return (%t1))IR", + &pattern); + auto matches = findPatternMatches(pattern, graph); + AT_ASSERT(matches.size() == 4); +} + +TEST(SubgraphMatcherTest, OverlappingMatches) { + Graph graph, pattern; + parseIR( + R"IR( +graph(%t0): + %t1 = a::aaa(%t0) + %t2 = a::aaa(%t1) + %t3 = a::aaa(%t2) + %t4 = a::aaa(%t3) + return (%t4))IR", + &graph); + parseIR( + R"IR( +graph(%t0): + %t1 = a::aaa(%t0) + %t2 = a::aaa(%t1) + return (%t2))IR", + &pattern); + auto matches = findPatternMatches(pattern, graph); + AT_ASSERT(matches.size() == 3); +} + +TEST(SubgraphMatcherTest, MatchInBasicBlocks1) { + Graph graph; + parseIR( + R"IR( +graph(%a, %b, %c): + %d = aten::mul(%a, %b) + %x = prim::If(%c) + block0(): + %x1 = aten::mul(%a, %d) + -> (%x1) + block1(): + %x2 = aten::mul(%b, %d) + -> (%x2) + return (%x))IR", + &graph); + + // Ensure the matches don't cross basic block boundaries + Graph pattern0; + parseIR( + R"IR( +graph(%x, %y): + %z = aten::mul(%x, %y) + return (%z))IR", + &pattern0); + AT_ASSERT(findPatternMatches(pattern0, graph).size() == 3); + + Graph pattern1; + parseIR( + R"IR( +graph(%x, %y): + %z1 = aten::mul(%x, %y) + %z2 = aten::mul(%y, %z1) + return (%z2))IR", + &pattern1); + AT_ASSERT(findPatternMatches(pattern1, graph).size() == 0); +} + +TEST(SubgraphMatcherTest, MatchInBasicBlocks2) { + Graph graph; + parseIR( + R"IR( +graph(%a, %b): + %x = my::mul(%a, %b) + %y = my::node_with_subblock() + block0(): + %z = my::mul(%b, %x) + -> (%z) + return (%y))IR", + &graph); + + // Check that we can match both mul ops + Graph pattern0; + parseIR( + R"IR( +graph(%x, %y): + %z = my::mul(%x, %y) + return (%z))IR", + &pattern0); + AT_ASSERT(findPatternMatches(pattern0, graph).size() == 2); + + // Ensure the matches don't cross basic block boundaries + Graph pattern1; + parseIR( + R"IR( +graph(%x, %y): + %u = my::mul(%x, %y) + %v = my::mul(%y, %u) + return (%v))IR", + &pattern1); + AT_ASSERT(findPatternMatches(pattern1, graph).size() == 0); +} + +TEST(SubgraphMatcherTest, MatchesAttributes) { + Graph graph; + parseIR( + R"IR( +graph(%0): + %a = a::a[isattr=[1,2]](%0) + %b = a::b[intattr=10, floatattr=3.14, complexattr=-3.14j](%0) + %c = a::c[myattr="qqq"](%a, %b) + return (%c))IR", + &graph); + + { + Graph pattern; + parseIR( + R"IR( +graph(%a, %b): + %c = a::c[myattr="qqq"](%a, %b) + return (%c))IR", + &pattern); + AT_ASSERT(!findPatternMatches(pattern, graph).empty()); + } + { + Graph pattern; + parseIR( + R"IR( +graph(%a, %b): + %c = a::c[myattr="zzz"](%a, %b) + return (%c))IR", + &pattern); + AT_ASSERT(findPatternMatches(pattern, graph).empty()); + } + { + Graph pattern; + parseIR( + R"IR( +graph(%0): + %b = a::b[extraattr=10](%0) + return (%b))IR", + &pattern); + AT_ASSERT(findPatternMatches(pattern, graph).empty()); + } + { + Graph pattern; + parseIR( + R"IR( +graph(%0): + %b = a::b[intattr=10, floatattr=3.14, complexattr=-3.14j](%0) + return (%b))IR", + &pattern); + AT_ASSERT(!findPatternMatches(pattern, graph).empty()); + } + { + Graph pattern; + parseIR( + R"IR( +graph(%0): + %b = a::b[intattr=10, floatattr=3.14, complexattr=-3.14j, strattr="rrr"](%0) + return (%b))IR", + &pattern); + AT_ASSERT(findPatternMatches(pattern, graph).empty()); + } + { + Graph pattern; + parseIR( + R"IR( +graph(%0): + %a = a::a[isattr=[1,2]](%0) + return (%a))IR", + &pattern); + // Lists are not supported yet, thus we shouldn't match for now. + AT_ASSERT(findPatternMatches(pattern, graph).empty()); + } + { + Graph pattern; + parseIR( + R"IR( +graph(%a, %b): + %c = a::c[myattr="q.*"](%a, %b) + return (%c))IR", + &pattern); + AT_ASSERT(!findPatternMatches(pattern, graph).empty()); + } +} + +TEST(SubgraphMatcherTest, BadPattern) { + Graph graph, pattern1, pattern2; + parseIR( + R"IR( +graph(%x): + %y = my::op1(%x) + %z = my::op2(%x) + return (%y, %z))IR", + &graph); + + parseIR( + R"IR( +graph(%x): + %y = my::node_with_subblock() + block0(): + %z = my::op(%x) + -> (%z) + return (%y))IR", + &pattern1); + // No support for patterns with subblocks + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_ANY_THROW(findPatternMatches(pattern1, graph)); + + parseIR( + R"IR( +graph(%x): + %y = my::op1(%x) + %z = my::op2(%x) + return (%y, %z))IR", + &pattern2); + // Not supported multi-output pattern, because not the whole pattern is + // covered by a traversal up from the first output (`%z = ...` is not + // visited). See the note "Multi-output Patterns" in subgraph_matcher.h. + // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto,hicpp-avoid-goto) + ASSERT_ANY_THROW(findPatternMatches(pattern2, graph)); +} + +TEST(SubgraphMatcherTest, MultiOutput) { + { + Graph graph, pattern; + parseIR( + R"IR( +graph(%0): + %a = a::aaa(%0) + %b = b::bbb(%a) + %c = c::ccc(%a, %b) + %x = a::aaa(%c) + %y = b::bbb(%x) + %z = d::ddd(%x, %y) + return (%y))IR", + &graph); + parseIR( + R"IR( +graph(%0): + %a = a::aaa(%0) + %b = b::bbb(%a) + return (%b, %a))IR", + &pattern); + AT_ASSERT(findPatternMatches(pattern, graph).size() == 2); + } + { + Graph graph, pattern; + parseIR( + R"IR( +graph(%0, %1): + %a1, %a2 = a::aaa(%0, %1) + %b = b::bbb(%a1) + %c = c::ccc(%b) + + %x1, %x2 = a::aaa(%c, %a2) + %y = b::bbb(%x1) + %z = d::ddd(%y) + return (%z))IR", + &graph); + parseIR( + R"IR( +graph(%0, %1): + %a1, %a2 = a::aaa(%0, %1) + %b = b::bbb(%a1) + return (%b, %a2))IR", + &pattern); + AT_ASSERT(findPatternMatches(pattern, graph).size() == 2); + } +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_subgraph_rewriter.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_subgraph_rewriter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..24d901ea7f0eae817b94a18144dddd14e7bc0426 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_subgraph_rewriter.cpp @@ -0,0 +1,310 @@ +#include + +#include +#include +#include +#include + +namespace torch { +namespace jit { +using namespace testing; + +TEST(SubgraphRewriterTest, FilterMatch) { + auto graph = std::make_shared(); + + parseIR( + R"IR( +graph(%0): + %a = a::aaa(%0) + %b : int = prim::Constant[value=1]() + %c = c::ccc(%a, %b) + return (%c))IR", + graph.get()); + + std::string pattern = R"IR( +graph(%a, %b): + %c = c::ccc(%a, %b) + return (%c))IR"; + Graph pattern_graph; + std::unordered_map vmap; + + parseIR(pattern, &pattern_graph, vmap); + + auto b_is_constant = [](const Match& match, + const std::unordered_map& vmap) { + const auto& match_vmap = match.values_map; + auto b_node = match_vmap.at(vmap.at("b"))->node(); + return b_node->kind() == prim::Constant; + }; + + auto b_is_one = [](const Match& match, + const std::unordered_map& vmap) { + const auto& match_vmap = match.values_map; + auto b_val = toIValue(match_vmap.at(vmap.at("b"))); + return b_val && b_val->isInt() && b_val->toInt() == 1; + }; + + auto b_is_two = [](const Match& match, + const std::unordered_map& vmap) { + const auto& match_vmap = match.values_map; + auto b_val = toIValue(match_vmap.at(vmap.at("b"))); + return b_val && b_val->isInt() && b_val->toInt() == 2; + }; + + std::string replacement = R"IR( +graph(%a, %b): + %d = d::ddd(%a, %b) + return (%d))IR"; + + SubgraphRewriter rewriter; + rewriter.RegisterRewritePattern(pattern, replacement); + + // b is constant, so the match will succeed + { + auto g = graph->copy(); + rewriter.runOnGraph(g, b_is_constant); + FileCheck().check("d::ddd")->check_not("c::ccc")->run(*g); + } + + // b is constant and the value is one, the match will succeed + { + auto g = graph->copy(); + rewriter.runOnGraph(g, {b_is_constant, b_is_one}); + FileCheck().check("d::ddd")->check_not("c::ccc")->run(*g); + } + + // b is constant but the value is not two, the match will fail + { + auto g = graph->copy(); + rewriter.runOnGraph(g, {b_is_constant, b_is_two}); + FileCheck().check("c::ccc")->check_not("d::ddd")->run(*g); + } +} + +TEST(SubgraphRewriterTest, FilterNoMatch) { + auto graph = std::make_shared(); + parseIR( + R"IR( +graph(%0): + %a = a::aaa(%0) + %b = prim::Constant[value=1]() + %c = c::ccc(%a, %b) + return (%c))IR", + graph.get()); + + std::string pattern = R"IR( +graph(%a, %b): + %c = c::ccc(%a, %b) + return (%c))IR"; + Graph pattern_graph; + std::unordered_map vmap; + + parseIR(pattern, &pattern_graph, vmap); + + auto filter = [](const Match& match, + const std::unordered_map& vmap) { + const auto& match_vmap = match.values_map; + auto b_node = match_vmap.at(vmap.at("b"))->node(); + // b_node is not prim::Assign, so this won't match and we'll skip the + // rewrite + return b_node->kind() == prim::Assign; + }; + + std::string replacement = R"IR( +graph(%a, %b): + %d = d::ddd(%a, %b) + return (%d))IR"; + + SubgraphRewriter rewriter; + rewriter.RegisterRewritePattern(pattern, replacement); + rewriter.runOnGraph(graph, filter); + + FileCheck().check("c::ccc")->check_not("d::ddd")->run(*graph); +} + +TEST(SubgraphRewriterTest, MultiOutput) { + { + auto graph = std::make_shared(); + + // Basic multi-output pattern rewriting + parseIR( + R"IR( +graph(%0, %1): + %a1, %a2 = a::aaa(%0, %1) + %b = b::bbb(%a1) + %c = c::ccc(%b) + + %x1, %x2 = a::aaa(%c, %a2) + %y = b::bbb(%x1) + %z = d::ddd(%y) + return (%z))IR", + graph.get()); + + std::string pattern = R"IR( +graph(%0, %1): + %a1, %a2 = a::aaa(%0, %1) + %b = b::bbb(%a1) + return (%b, %a2))IR"; + + std::string replacement = R"IR( +graph(%a, %b): + %x, %y = ab::ababab(%a, %b) + return (%x, %y))IR"; + + SubgraphRewriter rewriter; + rewriter.RegisterRewritePattern(pattern, replacement); + + auto g = graph->copy(); + rewriter.runOnGraph(g); + FileCheck().check("ab::ababab")->check("ab::ababab")->run(*g); + } + { + auto graph = std::make_shared(); + + // Mimic a real model case + parseIR( + R"IR( + graph(%k, %m, %x1, %x2, %x3, %x4, %y1, %y2, %y3, %y4): + %a1 = aa::aaa(%x1, %k) + %b1_1, %b1_2 = bb::bbb(%y1, %a1) + %a2 = aa::aaa(%x2, %k) + %b2_1, %b2_2 = bb::bbb(%y2, %a2) + %a3 = aa::aaa(%x3, %k) + %b3_1, %b3_2 = bb::bbb(%y3, %a3) + %a4 = aa::aaa(%x4, %k) + %b4_1, %b4_2 = bb::bbb(%y4, %a4) + %c = cc::ccc(%b4_1) + %d1 = dd::ddd(%b1_2, %m) + %e1 = ee::eee(%b1_1, %d1) + %d2 = dd::ddd(%b2_2, %m) + %e2 = ee::eee(%b2_1, %d2) + %d3 = dd::ddd(%b3_2, %m) + %e3 = ee::eee(%b3_1, %d3) + %d4 = dd::ddd(%b4_2, %m) + %e4 = ee::eee(%b4_1, %d4) + return (%d1, %d2, %d3, %d4, %e1, %e2, %e3, %e4) + )IR", + graph.get()); + + std::string pattern = R"IR( + graph(%a, %b, %c, %d): + %y0 = aa::aaa(%b, %c) + %y1, %y2 = bb::bbb(%a, %y0) + %y3 = dd::ddd(%y2, %d) + return (%y3, %y1))IR"; + + std::string replacement = R"IR( + graph(%a, %b, %c, %d): + %x, %y = ab::ababab(%a, %b, %c, %d) + return (%x, %y))IR"; + + SubgraphRewriter rewriter; + rewriter.RegisterRewritePattern(pattern, replacement); + + auto g = graph->copy(); + rewriter.runOnGraph(g); + FileCheck().check("ab::ababab")->check("ab::ababab")->run(*g); + } + { + auto graph = std::make_shared(); + + // A case where no rewriting should occur due to data dependencies + parseIR( + R"IR( + graph(%x, %y): + %a = aa::aaa(%x) + %b = bb::bbb(%a) + %e = ee::eee(%b) + %c = cc::ccc(%y) + %d = dd::ddd(%b, %c) + %f = ff::fff(%b, %d) + return (%f) + )IR", + graph.get()); + + std::string pattern = R"IR( + graph(%a, %c): + %b = bb::bbb(%a) + %d = dd::ddd(%b, %c) + return (%d, %b))IR"; + + std::string replacement = R"IR( + graph(%a, %c): + %d, %b = db::fused(%a, %c) + return (%d, %b))IR"; + + SubgraphRewriter rewriter; + rewriter.RegisterRewritePattern(pattern, replacement); + + auto g = graph->copy(); + rewriter.runOnGraph(g); + // We should not perform the replacement on the given graph due to data + // dependency constraints: the output %b is used in %e, which precedes one + // def of the input %c. + FileCheck().check_not("db::fused")->run(*g); + } +} + +TEST(SubgraphRewriterTest, OutputType) { + std::string pattern = R"IR( +graph(%a, %b): + %c = c::ccc(%a, %b) + return (%c))IR"; + Graph pattern_graph; + std::unordered_map vmap; + + parseIR(pattern, &pattern_graph, vmap); + + auto b_is_constant = [](const Match& match, + const std::unordered_map& vmap) { + const auto& match_vmap = match.values_map; + auto b_node = match_vmap.at(vmap.at("b"))->node(); + return b_node->kind() == prim::Constant; + }; + + std::string replacement = R"IR( +graph(%a, %b): + %d = d::ddd(%a, %b) + return (%d))IR"; + + SubgraphRewriter rewriter; + rewriter.RegisterRewritePattern(pattern, replacement); + { + auto graph = std::make_shared(); + + parseIR( + R"IR( + graph(%0): + %a : Float(10, 20) = a::aaa(%0) + %b : int = prim::Constant[value=1]() + %c : Float(10, 20) = c::ccc(%a, %b) + return (%c))IR", + graph.get()); + + // output has shape info. + rewriter.runOnGraph(graph, b_is_constant); + FileCheck() + .check("Float(10, 20) = d::ddd") + ->check_not("c::ccc") + ->run(*graph); + } + { + auto graph = std::make_shared(); + + parseIR( + R"IR( + graph(%0): + %a = a::aaa(%0) + %b : int = prim::Constant[value=1]() + %c = c::ccc(%a, %b) + return (%c))IR", + graph.get()); + + // output has not shape info. + rewriter.runOnGraph(graph, b_is_constant); + FileCheck().check("Tensor = d::ddd")->check_not("c::ccc")->run(*graph); + } +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_subgraph_utils.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_subgraph_utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..606f658ee513c630efbd3ac89d66ca1ad66193f8 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_subgraph_utils.cpp @@ -0,0 +1,147 @@ +#include + +#include "test/cpp/jit/test_utils.h" + +#include +#include "torch/csrc/jit/passes/common_subexpression_elimination.h" +#include "torch/csrc/jit/passes/utils/subgraph_utils.h" + +namespace torch { +namespace jit { + +TEST(SubgraphUtilsTest, Basic) { + auto graph = build_lstm(); + EliminateCommonSubexpression(graph); + + std::vector originalNodes( + graph->nodes().begin(), graph->nodes().end()); + + for (bool reverse_iterate : {true, false}) { + // Merge everything into a single subgraph + bool first = true; + Node* subgraph = nullptr; + auto it = + reverse_iterate ? graph->nodes().rbegin() : graph->nodes().begin(); + auto end = reverse_iterate ? graph->nodes().rend() : graph->nodes().end(); + for (; it != end;) { + if (first) { + subgraph = SubgraphUtils::createSingletonSubgraph( + *it, prim::DifferentiableGraph); + it = reverse_iterate ? ++subgraph->reverseIterator() + : ++subgraph->iterator(); + first = false; + } + + SubgraphUtils::mergeNodeIntoSubgraph(*it, subgraph); + it = reverse_iterate ? ++subgraph->reverseIterator() + : ++subgraph->iterator(); + } + + // Unmerge and compare with original node listing + // NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) + SubgraphUtils::unmergeSubgraph(subgraph); + EliminateCommonSubexpression(graph); + + std::vector newNodes(graph->nodes().begin(), graph->nodes().end()); + ASSERT_EQ(originalNodes.size(), newNodes.size()); + } +} + +TEST(SubgraphUtilsTest, MergeSubgraphs) { + auto graph = std::make_shared(); + std::unordered_map parse_map; + parseIR( + R"IR( +graph(%a : Tensor, %b : Tensor, %c : Tensor): + %x : Tensor = aten::sigmoid(%a) + %y : Tensor = aten::mul(%a, %b) + %p : Tensor = aten::div(%c, %b) + %q1 : Tensor = aten::mul(%p, %a) + %q2 : Tensor = aten::tanh(%q1) + %q3 : Tensor = aten::tanh(%q2) + %q4 : Tensor = aten::tanh(%q3) + %q5 : Tensor = aten::hardsigmoid(%q4) + return (%x, %y, %q5))IR", + &*graph, + parse_map); + + std::vector originalNodes( + graph->nodes().begin(), graph->nodes().end()); + for (bool reverse_merge : {true, false}) { + // Merge everything into two adjacent subgraphs + Node* graph1 = SubgraphUtils::createSingletonSubgraph( + *graph->nodes().begin(), prim::DifferentiableGraph); + while (true) { + Node* next = graph1->next(); + if (next->kind() == aten::tanh) { + break; + } + SubgraphUtils::mergeNodeIntoSubgraph(next, graph1); + } + Node* graph2 = SubgraphUtils::createSingletonSubgraph( + graph1->next(), prim::DifferentiableGraph); + while (graph2->next() != *graph->nodes().end()) { + SubgraphUtils::mergeNodeIntoSubgraph(graph2->next(), graph2); + } + Node* subgraph = nullptr; + if (reverse_merge) { + SubgraphUtils::mergeNodeIntoSubgraph(graph2, graph1); + subgraph = graph1; + } else { + SubgraphUtils::mergeNodeIntoSubgraph(graph1, graph2); + subgraph = graph2; + } + auto run_file_check = [](std::shared_ptr graph) { + graph->lint(); + testing::FileCheck() + .check("aten::sigmoid") + ->check("aten::mul") + ->check("aten::div") + ->check("aten::mul") + ->check_count("aten::tanh", 3) + ->check("aten::hardsigmoid") + ->run(*graph); + }; + run_file_check(subgraph->g(attr::Subgraph)); + + // Unmerge and compare with original node listing + SubgraphUtils::unmergeSubgraph(subgraph); + EliminateCommonSubexpression(graph); + run_file_check(graph); + + std::vector newNodes(graph->nodes().begin(), graph->nodes().end()); + ASSERT_EQ(originalNodes.size(), newNodes.size()); + } +} + +TEST(SubgraphUtilsTest, GraphName) { + auto graph = std::make_shared(); + + std::unordered_map parse_map; + parseIR( + R"IR( +graph(%a : Tensor, %b : Tensor, %c : Tensor): + %x : Tensor = aten::tanh(%a) + %y : Tensor = aten::mul(%a, %b) + %p : Tensor = aten::div(%c, %b) + %q1 : Tensor = aten::mul(%p, %a) + %q2 : Tensor = aten::tanh(%q1) + %q3 : Tensor = aten::tanh(%q2) + %q4 : Tensor = aten::tanh(%q3) + %q5 : Tensor = aten::tanh(%q4) + return (%x, %y, %q5))IR", + &*graph, + parse_map); + std::string ref_full_name = "graph_tanh_mul_div_mul_tanh_tanh_tanh_tanh"; + std::string full_name = + SubgraphUtils::generateNameForGraph(graph, 80, "graph"); + ASSERT_EQ(full_name, ref_full_name); + + std::string truncated_name = + SubgraphUtils::generateNameForGraph(graph, 10, "graph"); + + ASSERT_LE(truncated_name.size(), ref_full_name.size()); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_te.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_te.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e7dba10a7c8a9e4f1cc488474833504570d4a551 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_te.cpp @@ -0,0 +1,41 @@ +#include + +#include +#include +#include + +#include + +namespace torch { +namespace jit { + +TEST(TETest, RemoveProfiling) { + auto g = std::make_shared(); + const auto graph_string = R"IR( + graph(%a : Tensor, + %b : bool): + %1 : None = prim::Constant() + %2 : Tensor? = prim::If(%b) + block0(): + %3 : Tensor? = prim::profile[profiled_type=Tensor, seen_none=0](%1) + -> (%3) + block1(): + %4 : Tensor = prim::profile[profiled_type=Tensor, seen_none=0](%a) + -> (%4) + return (%2))IR"; + torch::jit::parseIR(graph_string, g.get()); + + g->lint(); + RemoveProfileNodesAndSpecializeTypes(g); + g->lint(); + + testing::FileCheck() + .check("prim::Constant") + ->check("prim::If") + ->check("block") + ->check("block") + ->check("return") + ->run(*g); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_union.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_union.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1a80cbefe78ba4ab131544716edeb5aaef31526b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_union.cpp @@ -0,0 +1,149 @@ +#include + +#include +#include +#include + +namespace torch { +namespace jit { + +class UnionTypeTest : public ::testing::Test { + public: + // None + const TypePtr none = NoneType::get(); + + // List[str] + const TypePtr l1 = ListType::ofStrings(); + + // Optional[int] + const TypePtr opt1 = OptionalType::create(IntType::get()); + + // Optional[float] + const TypePtr opt2 = OptionalType::create(FloatType::get()); + + // Optional[List[str]] + const TypePtr opt3 = OptionalType::create(ListType::ofStrings()); + + // Tuple[Optional[int], int] + const TypePtr tup1 = + TupleType::create({OptionalType::create(IntType::get()), IntType::get()}); + + // Tuple[int, int] + const TypePtr tup2 = TupleType::create({IntType::get(), IntType::get()}); + + bool hasType(UnionTypePtr u, TypePtr t) { + auto res = std::find(u->getTypes().begin(), u->getTypes().end(), t); + return res != u->getTypes().end(); + } +}; + +TEST_F(UnionTypeTest, UnionOperatorEquals) { + const UnionTypePtr u1 = UnionType::create({l1, tup2, StringType::get()}); + + // Same thing, but using different TypePtrs + const TypePtr l1_ = ListType::ofStrings(); + const TypePtr tup2_ = TupleType::create({IntType::get(), IntType::get()}); + const UnionTypePtr u2 = UnionType::create({l1_, tup2_, StringType::get()}); + + ASSERT_TRUE(*u1 == *u2); +} + +TEST_F(UnionTypeTest, UnionCreate_OptionalT1AndOptionalT2) { + // Goal: Union[int, float, None] + const UnionTypePtr u = UnionType::create({opt1, opt2}); + + ASSERT_EQ(u->getTypes().size(), 3); + ASSERT_TRUE(UnionTypeTest::hasType(u, IntType::get())); + ASSERT_TRUE(UnionTypeTest::hasType(u, FloatType::get())); + ASSERT_TRUE(UnionTypeTest::hasType(u, NoneType::get())); +} + +TEST_F(UnionTypeTest, UnionCreate_OptionalTAndT) { + // Goal: Union[int, None] + const UnionTypePtr u = UnionType::create({opt1, IntType::get()}); + + ASSERT_EQ(u->getTypes().size(), 2); + ASSERT_TRUE(UnionTypeTest::hasType(u, IntType::get())); + ASSERT_TRUE(UnionTypeTest::hasType(u, NoneType::get())); +} + +TEST_F(UnionTypeTest, UnionCreate_TupleWithSubtypingRelationship) { + // Goal: Union[Tuple[Optional[int], int], str] + const UnionTypePtr u = UnionType::create({StringType::get(), tup1, tup2}); + + ASSERT_EQ(u->getTypes().size(), 2); + ASSERT_TRUE(UnionTypeTest::hasType(u, StringType::get())); + ASSERT_TRUE(UnionTypeTest::hasType(u, tup1)); +} + +TEST_F(UnionTypeTest, UnionCreate_ContainerTAndT) { + // Goal: Union[List[str], str] + const UnionTypePtr u = UnionType::create({l1, StringType::get()}); + + ASSERT_EQ(u->getTypes().size(), 2); + ASSERT_TRUE(UnionTypeTest::hasType(u, StringType::get())); + ASSERT_TRUE(UnionTypeTest::hasType(u, ListType::ofStrings())); +} + +TEST_F(UnionTypeTest, UnionCreate_OptionalContainerTAndContainerTAndT) { + // Goal: Union[List[str], None, str] + const UnionTypePtr u = UnionType::create({l1, opt3, StringType::get()}); + + ASSERT_EQ(u->getTypes().size(), 3); + ASSERT_TRUE(UnionTypeTest::hasType(u, StringType::get())); + ASSERT_TRUE(UnionTypeTest::hasType(u, ListType::ofStrings())); +} + +TEST_F(UnionTypeTest, Subtyping_NumberType) { + // Union[int, float, Complex] + const UnionTypePtr union1 = + UnionType::create({IntType::get(), FloatType::get(), ComplexType::get()}); + + // Union[int, float, Complex, None] + const UnionTypePtr union2 = UnionType::create( + {IntType::get(), FloatType::get(), ComplexType::get(), NoneType::get()}); + + const NumberTypePtr num = NumberType::get(); + + ASSERT_TRUE(num->isSubtypeOf(*union1)); + ASSERT_TRUE(union1->isSubtypeOf(*num)); + ASSERT_TRUE(*num == *union1); + + ASSERT_TRUE(num->isSubtypeOf(*union2)); + ASSERT_FALSE(union2->isSubtypeOf(*num)); + ASSERT_FALSE(*num == *union2); +} + +TEST_F(UnionTypeTest, Subtyping_OptionalType) { + // Union[int, None] + const UnionTypePtr union1 = + UnionType::create({IntType::get(), NoneType::get()}); + + // Union[int, str, None] + const UnionTypePtr union2 = + UnionType::create({IntType::get(), StringType::get(), NoneType::get()}); + + // Union[int, str, List[str]] + const UnionTypePtr union3 = UnionType::create( + {IntType::get(), StringType::get(), ListType::ofStrings()}); + + ASSERT_TRUE(none->isSubtypeOf(opt1)); + ASSERT_TRUE(none->isSubtypeOf(union1)); + ASSERT_TRUE(none->isSubtypeOf(union2)); + ASSERT_FALSE(none->isSubtypeOf(union3)); + + ASSERT_FALSE(opt1->isSubtypeOf(none)); + ASSERT_TRUE(opt1->isSubtypeOf(union1)); + ASSERT_TRUE(opt1->isSubtypeOf(union2)); + ASSERT_FALSE(opt1->isSubtypeOf(union3)); + + ASSERT_FALSE(union1->isSubtypeOf(none)); + ASSERT_TRUE(union1->isSubtypeOf(opt1)); + ASSERT_TRUE(union1->isSubtypeOf(union2)); + ASSERT_FALSE(union1->isSubtypeOf(union3)); + + ASSERT_FALSE(union2->isSubtypeOf(union1)); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_upgrader_utils.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_upgrader_utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b093616d9673e6f83eae91a8daf01c283a3fbaeb --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_upgrader_utils.cpp @@ -0,0 +1,99 @@ +#include +#include +#include + +#include + +#include + +namespace torch { +namespace jit { + +TEST(UpgraderUtils, FindCorrectUpgrader) { + std::vector dummy_entry = { + {4, "foo__0_3", "foo.bar()"}, + {8, "foo__4_7", "foo.bar()"}, + }; + + auto upgrader_at_6 = findUpgrader(dummy_entry, 6); + EXPECT_TRUE(upgrader_at_6.has_value()); + EXPECT_EQ(upgrader_at_6.value().upgrader_name, "foo__4_7"); + + auto upgrader_at_1 = findUpgrader(dummy_entry, 1); + EXPECT_TRUE(upgrader_at_1.has_value()); + EXPECT_EQ(upgrader_at_1.value().upgrader_name, "foo__0_3"); + + auto upgrader_at_10 = findUpgrader(dummy_entry, 10); + EXPECT_TRUE(upgrader_at_1.has_value()); + EXPECT_EQ(upgrader_at_1.value().upgrader_name, "foo__0_3"); +} + +TEST(UpgraderUtils, IsVersionMapSorted) { + auto map = get_operator_version_map(); + // tests if the each list of UpgraderEntry in the map is sorted by + // their bumped_at_version field. + for (const auto& entry : map) { + std::vector versions; + for (const auto& el : entry.second) { + versions.push_back(el.bumped_at_version); + } + EXPECT_TRUE(std::is_sorted(versions.begin(), versions.end())); + } +} + +TEST(UpgraderUtils, FindIfOpIsCurrent) { + std::vector dummy_entry = { + {4, "foo__0_3", "foo.bar()"}, + {8, "foo__4_7", "foo.bar()"}, + }; + + auto isCurrent = isOpCurrentBasedOnUpgraderEntries(dummy_entry, 6); + auto isCurrentV2 = isOpCurrentBasedOnUpgraderEntries(dummy_entry, 8); + EXPECT_FALSE(isCurrent); + EXPECT_TRUE(isCurrentV2); + + // symbol based look up + test_only_add_entry("foo", dummy_entry[0]); + test_only_add_entry("foo", dummy_entry[1]); + EXPECT_FALSE(isOpSymbolCurrent("foo", 6)); + EXPECT_TRUE(isOpSymbolCurrent("foo", 8)); + test_only_remove_entry("foo"); +} + +TEST(UpgraderUtils, CanLoadHistoricOp) { + std::vector dummy_entry = { + {4, "foo__0_3", "foo.bar()"}, + {8, "foo__4_7", "foo.foo()"}, + }; + + std::vector schemas = {"foo.bar()", "foo.foo()"}; + + // symbol based look up + test_only_add_entry("old_op_not_exist.first", dummy_entry[0]); + test_only_add_entry("old_op_not_exist.second", dummy_entry[1]); + + auto oldSchemas = loadPossibleHistoricOps("old_op_not_exist", 2); + EXPECT_EQ(oldSchemas.size(), 2); + for (const auto& entry : oldSchemas) { + EXPECT_TRUE( + std::find(schemas.begin(), schemas.end(), entry) != schemas.end()); + } + + auto oldSchemasWithCurrentVersion = + loadPossibleHistoricOps("old_op_not_exist", 9); + EXPECT_EQ(oldSchemasWithCurrentVersion.size(), 0); + + test_only_remove_entry("old_op_not_exist.first"); + test_only_remove_entry("old_op_not_exist.first"); + + // it is ok to have old schemas without overload + test_only_add_entry("old_op_not_exist_no_overload", dummy_entry[0]); + auto oldSchemasNoOverload = + loadPossibleHistoricOps("old_op_not_exist_no_overload", 2); + EXPECT_EQ(oldSchemasNoOverload.size(), 1); + EXPECT_EQ(oldSchemasNoOverload[0], "foo.bar()"); + test_only_remove_entry("old_op_not_exist_no_overload"); +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_utils.cpp b/western_ai/usa/pytorch/test/cpp/jit/test_utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d3101208acf655af5912e3c9ed30d7b4485a6400 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_utils.cpp @@ -0,0 +1,299 @@ +#include + +#include +#include +#include +#include + +namespace torch { +namespace jit { + +Stack createStack(std::vector&& list) { + return Stack( + std::make_move_iterator(list.begin()), + std::make_move_iterator(list.end())); +} + +void assertAllClose(const tensor_list& a, const tensor_list& b) { + ASSERT_EQ(a.size(), b.size()); + for (size_t i = 0; i < a.size(); ++i) { + ASSERT_TRUE(a[i].is_same_size(b[i])); + ASSERT_TRUE(a[i].allclose(b[i])); + } +} + +std::vector run( + InterpreterState& interp, + const std::vector& inputs) { + std::vector stack(inputs.begin(), inputs.end()); + interp.run(stack); + return fmap(stack, [](const IValue& i) { return i.toTensor(); }); +} + +static void unpackReturnTuple(Stack& stack) { + auto tuple = pop(stack).toTuple(); + stack.insert(stack.end(), tuple->elements().begin(), tuple->elements().end()); +} + +std::pair runGradient( + Gradient& grad_spec, + tensor_list& tensors_in, + tensor_list& tensor_grads_in) { + static const auto as_tensorlist = [](const Stack& stack) { + return fmap(stack, [](const IValue& i) { return i.toTensor(); }); + }; + ClearUndefinedness(grad_spec.df); + Code f_code{grad_spec.f, ""}, df_code{grad_spec.df, ""}; + InterpreterState f_interpreter{f_code}, df_interpreter{df_code}; + + auto f_stack = fmap(tensors_in); + f_interpreter.run(f_stack); + + Stack df_stack; + df_stack.insert( + df_stack.end(), tensor_grads_in.begin(), tensor_grads_in.end()); + for (auto offset : grad_spec.df_input_captured_inputs) + df_stack.push_back(tensors_in[offset]); + for (auto offset : grad_spec.df_input_captured_outputs) + df_stack.push_back(f_stack[offset]); + df_interpreter.run(df_stack); + unpackReturnTuple(df_stack); + // Outputs of f needs to be sliced + f_stack.erase(f_stack.begin() + grad_spec.f_real_outputs, f_stack.end()); + return std::make_pair(as_tensorlist(f_stack), as_tensorlist(df_stack)); +} + +std::shared_ptr build_lstm() { + const auto graph_string = R"IR( + graph(%0 : Tensor, + %1 : Tensor, + %2 : Tensor, + %3 : Tensor, + %4 : Tensor): + %5 : Tensor = aten::mm(%0, %3) + %6 : Tensor = aten::mm(%1, %4) + %7 : int = prim::Constant[value=1]() + %8 : Tensor = aten::add(%5, %6, %7) + %9 : Tensor, %10 : Tensor, %11 : Tensor, %12 : Tensor = prim::ConstantChunk[chunks=4, dim=1](%8) + %13 : Tensor = aten::sigmoid(%9) + %14 : Tensor = aten::sigmoid(%12) + %15 : Tensor = aten::tanh(%11) + %16 : Tensor = aten::sigmoid(%10) + %17 : Tensor = aten::mul(%16, %2) + %18 : Tensor = aten::mul(%13, %15) + %19 : int = prim::Constant[value=1]() + %20 : Tensor = aten::add(%17, %18, %19) + %21 : Tensor = aten::tanh(%20) + %22 : Tensor = aten::mul(%14, %21) + return (%22, %20))IR"; + auto g = std::make_shared(); + torch::jit::parseIR(graph_string, g.get()); + g->lint(); + + return g; +} + +std::shared_ptr build_mobile_export_analysis_graph() { + // We use following two schemas for this graph: + // 1. slice.Tensor(Tensor(a) self, int dim=0, int? start=None, + // int? end=None, int step=1) -> Tensor(a) + // 2. slice.str(str string, int? start=None, int? end=None, + // int step=1) -> str + // %3 and %4 use slice.Tensor while %5 use slice.str. + // Since we can see %3 and %4 have the same last argument that is never used + // (same as default value of schema), we know we can ignore that last arg. For + // %5, we see that last three args are same as schema default, hence + // unnecessary. + + const auto graph_string = R"IR( + graph(%0 : Tensor): + %1 : int = prim::Constant[value=1]() + %2 : int = prim::Constant[value=2]() + %20 : int = prim::Constant[value=0]() + %21 : int = prim::Constant[value=9223372036854775807]() + %22 : str = prim::Constant[value="value"]() + %3 : Tensor = aten::slice(%0, %1, %20, %2, %1) + %4 : Tensor = aten::slice(%0, %2, %20, %21, %1) + %5 : str = aten::slice(%22, %20, %21, %2) + return (%3, %4, %5))IR"; + + auto g = std::make_shared(); + torch::jit::parseIR(graph_string, g.get()); + g->lint(); + return g; +} + +std::shared_ptr build_mobile_export_with_out() { + const auto graph_string = R"IR( + graph(%x.1 : Tensor, + %y.1 : Tensor): + %8 : NoneType = prim::Constant() + %6 : int = prim::Constant[value=1]() + %7 : Tensor = aten::add(%x.1, %y.1, %6, %y.1) + return (%8))IR"; + + auto g = std::make_shared(); + torch::jit::parseIR(graph_string, g.get()); + g->lint(); + return g; +} + +std::shared_ptr build_mobile_export_analysis_graph_nested() { + // this is pretty much same test as build_mobile_export_analysis_graph(), + // but some aten::slice operators are hidden under block statement to check + // if we are correctly recursing all the nodes in graph. + const auto graph_string = R"IR( + graph(%0 : Tensor): + %1 : int = prim::Constant[value=1]() + %2 : int = prim::Constant[value=2]() + %20 : int = prim::Constant[value=0]() + %21 : int = prim::Constant[value=9223372036854775807]() + %22 : str = prim::Constant[value="value"]() + %3 : Tensor = aten::slice(%0, %1, %20, %2, %1) + %23 : bool = aten::Bool(%3) + %c : Tensor = prim::If(%23) + block0(): + %4 : Tensor = aten::slice(%0, %2, %20, %21, %1) + %5 : str = aten::slice(%22, %20, %21, %2) + %c.1 : Tensor = aten::slice(%0, %1, %20, %2, %1) + -> (%c.1) + block1(): + -> (%3) + return (%3, %3))IR"; + auto g = std::make_shared(); + torch::jit::parseIR(graph_string, g.get()); + g->lint(); + return g; +} + +std::shared_ptr build_mobile_export_analysis_graph_with_vararg() { + const auto graph_string = R"IR( + graph(%0 : Tensor): + %1 : int = prim::Constant[value=1]() + %2 : int = prim::Constant[value=2]() + %3 : int = prim::Constant[value=3]() + %4 : int[] = prim::tolist(%1, %2) + %5 : int[] = prim::tolist(%1, %2, %3) + return (%4, %5))IR"; + + auto g = std::make_shared(); + torch::jit::parseIR(graph_string, g.get()); + g->lint(); + return g; +} + +std::shared_ptr build_mobile_export_analysis_graph_non_const() { + const auto graph_string = R"IR( + graph(%input.1 : Tensor): + %7 : int = prim::Constant[value=1]() # :3:58 + %9 : int = prim::Constant[value=0]() # :3:66 + %8 : int[] = prim::ListConstruct(%7, %7) + %10 : int[] = prim::ListConstruct(%9, %9) + %11 : int[] = prim::ListConstruct(%7, %7) + %12 : Tensor = aten::conv2d(%input.1, %input.1, %input.1, %8, %10, %11, %7) + return (%12))IR"; + + auto g = std::make_shared(); + torch::jit::parseIR(graph_string, g.get()); + g->lint(); + return g; +} + +at::Tensor t_use(at::Tensor x) { + return x; +} +at::Tensor t_def(at::Tensor x) { + return x.t(); +} + +bool checkRtol(const at::Tensor& diff, const std::vector inputs) { + double maxValue = 0.0; + for (auto& tensor : inputs) { + maxValue = fmax(tensor.abs().max().item(), maxValue); + } + return diff.abs().max().item() < 2e-6 * maxValue; +} + +bool almostEqual(const at::Tensor& a, const at::Tensor& b) { + return checkRtol(a - b, {a, b}); +} + +bool exactlyEqual(const at::Tensor& a, const at::Tensor& b) { + return (a - b).abs().max().item() == 0.f; +} + +bool exactlyEqual( + const std::vector& a, + const std::vector& b) { + if (a.size() != b.size()) { + return false; + } + for (size_t i = 0; i < a.size(); ++i) { + if (!exactlyEqual(a[i], b[i])) { + return false; + } + } + return true; +} + +std::pair lstm( + at::Tensor input, + at::Tensor hx, + at::Tensor cx, + at::Tensor w_ih, + at::Tensor w_hh) { + auto gates = input.mm(t_use(w_ih)) + hx.mm(t_use(w_hh)); + + auto chunked_gates = gates.chunk(4, 1); + auto ingate = chunked_gates[0]; + auto forgetgate = chunked_gates[1]; + auto cellgate = chunked_gates[2]; + auto outgate = chunked_gates[3]; + + ingate = ingate.sigmoid(); + outgate = outgate.sigmoid(); + cellgate = cellgate.tanh(); + forgetgate = forgetgate.sigmoid(); + + auto cy = (forgetgate * cx) + (ingate * cellgate); + auto hy = outgate * cy.tanh(); + + return {hy, cy}; +} + +inline c10::AliasAnalysisKind aliasAnalysisFromSchema() { + return c10::AliasAnalysisKind::FROM_SCHEMA; +} + +namespace { +RegisterOperators reg({ + // This operator is intended to be used in JIT analysis and transformation + // pass unit tests in which Values with type Tensor are often required. It + // should not be used in situations in which the graph is actually executed + // because it always produces empty Tensors. + Operator( + "prim::MakeTestTensor() -> Tensor", + [](Stack& stack) { push(stack, at::Tensor()); }, + aliasAnalysisFromSchema()), +}); +} // namespace + +std::vector runGraph( + std::shared_ptr graph, + const std::vector& inputs) { + std::vector stack = fmap(inputs); + Code code(graph, "test"); + InterpreterState(code).run(stack); + TORCH_INTERNAL_ASSERT(!stack.empty()); + // Graph outputs that are handled below: + // * A list of Tensors. + // * 1 Tensor. + if (stack.front().isTensorList()) { + return stack.front().toTensorVector(); + } + TORCH_INTERNAL_ASSERT(stack.front().isTensor()); + return {stack.front().toTensor()}; +} + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/test_utils.h b/western_ai/usa/pytorch/test/cpp/jit/test_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..90b4b4a81bda2881a934e8827dd177698f809f55 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/test_utils.h @@ -0,0 +1,121 @@ +#pragma once + +#include + +#include +#include +#include +#include + +// Resolve a test data file relative to the test source directory. +// Tries the source-relative path first (for development builds), then +// falls back to the executable-relative path (for install trees where +// __FILE__ may contain a stale build-time path). +inline std::string resolveTestDataFile( + const char* sourceFile, + const std::string& relativePath) { + auto candidate = + std::filesystem::path(sourceFile).parent_path() / relativePath; + if (std::filesystem::exists(candidate)) + return candidate.string(); + auto exeDir = std::filesystem::read_symlink("/proc/self/exe").parent_path(); + return (exeDir / relativePath).string(); +} + +namespace { +static inline void trim(std::string& s) { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) { + return !std::isspace(ch); + })); + s.erase( + std::find_if( + s.rbegin(), + s.rend(), + [](unsigned char ch) { return !std::isspace(ch); }) + .base(), + s.end()); + for (size_t i = 0; i < s.size(); ++i) { + while (i < s.size() && s[i] == '\n') { + s.erase(i, 1); + } + } + for (size_t i = 0; i < s.size(); ++i) { + if (s[i] == ' ') { + while (i + 1 < s.size() && s[i + 1] == ' ') { + s.erase(i + 1, 1); + } + } + } +} +} // namespace + +#define ASSERT_THROWS_WITH_MESSAGE(statement, substring) \ + try { \ + (void)statement; \ + FAIL(); \ + } catch (const std::exception& e) { \ + std::string substring_s(substring); \ + trim(substring_s); \ + auto exception_string = std::string(e.what()); \ + trim(exception_string); \ + ASSERT_NE(exception_string.find(substring_s), std::string::npos) \ + << " Error was: \n" \ + << exception_string; \ + } + +namespace torch { +namespace jit { + +using tensor_list = std::vector; +using namespace torch::autograd; + +// work around the fact that variable_tensor_list doesn't duplicate all +// of std::vector's constructors. +// most constructors are never used in the implementation, just in our tests. +Stack createStack(std::vector&& list); + +void assertAllClose(const tensor_list& a, const tensor_list& b); + +std::vector run( + InterpreterState& interp, + const std::vector& inputs); + +std::pair runGradient( + Gradient& grad_spec, + tensor_list& tensors_in, + tensor_list& tensor_grads_in); + +std::shared_ptr build_lstm(); +std::shared_ptr build_mobile_export_analysis_graph(); +std::shared_ptr build_mobile_export_with_out(); +std::shared_ptr build_mobile_export_analysis_graph_with_vararg(); +std::shared_ptr build_mobile_export_analysis_graph_nested(); +std::shared_ptr build_mobile_export_analysis_graph_non_const(); + +at::Tensor t_use(at::Tensor x); +at::Tensor t_def(at::Tensor x); + +// given the difference of output vs expected tensor, check whether the +// difference is within a relative tolerance range. This is a standard way of +// matching tensor values up to certain precision +bool checkRtol(const at::Tensor& diff, const std::vector inputs); +bool almostEqual(const at::Tensor& a, const at::Tensor& b); + +bool exactlyEqual(const at::Tensor& a, const at::Tensor& b); +bool exactlyEqual( + const std::vector& a, + const std::vector& b); + +std::vector runGraph( + std::shared_ptr graph, + const std::vector& inputs); + +std::pair lstm( + at::Tensor input, + at::Tensor hx, + at::Tensor cx, + at::Tensor w_ih, + at::Tensor w_hh); + +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/tests_setup.py b/western_ai/usa/pytorch/test/cpp/jit/tests_setup.py new file mode 100644 index 0000000000000000000000000000000000000000..c8d2465197c63f485f4437dd21a0d51ae1fa2b99 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/tests_setup.py @@ -0,0 +1,118 @@ +import os +import sys + +import torch + + +class Setup: + def setup(self): + raise NotImplementedError + + def shutdown(self): + raise NotImplementedError + + +class FileSetup: + path = None + + def shutdown(self): + if os.path.exists(self.path): + os.remove(self.path) + + +class EvalModeForLoadedModule(FileSetup): + path = "dropout_model.pt" + + def setup(self): + class Model(torch.jit.ScriptModule): + def __init__(self) -> None: + super().__init__() + self.dropout = torch.nn.Dropout(0.1) + + @torch.jit.script_method + def forward(self, x): + x = self.dropout(x) + return x + + model = Model() + model = model.train() + model.save(self.path) + + +class SerializationInterop(FileSetup): + path = "ivalue.pt" + + def setup(self): + ones = torch.ones(2, 2) + twos = torch.ones(3, 5) * 2 + + value = (ones, twos) + + torch.save(value, self.path, _use_new_zipfile_serialization=True) + + +# See testTorchSaveError in test/cpp/jit/tests.h for usage +class TorchSaveError(FileSetup): + path = "eager_value.pt" + + def setup(self): + ones = torch.ones(2, 2) + twos = torch.ones(3, 5) * 2 + + value = (ones, twos) + + torch.save(value, self.path, _use_new_zipfile_serialization=False) + + +class TorchSaveJitStream_CUDA(FileSetup): + path = "saved_stream_model.pt" + + def setup(self): + if not torch.cuda.is_available(): + return + + class Model(torch.nn.Module): + def forward(self): + s = torch.cuda.Stream() + a = torch.rand(3, 4, device="cuda") + b = torch.rand(3, 4, device="cuda") + + with torch.cuda.stream(s): + is_stream_s = ( + torch.cuda.current_stream(s.device_index()).id() == s.id() + ) + c = torch.cat((a, b), 0).to("cuda") + s.synchronize() + return is_stream_s, a, b, c + + model = Model() + + # Script the model and save + script_model = torch.jit.script(model) + torch.jit.save(script_model, self.path) + + +tests = [ + EvalModeForLoadedModule(), + SerializationInterop(), + TorchSaveError(), + TorchSaveJitStream_CUDA(), +] + + +def setup(): + for test in tests: + test.setup() + + +def shutdown(): + for test in tests: + test.shutdown() + + +if __name__ == "__main__": + command = sys.argv[1] + if command == "setup": + setup() + elif command == "shutdown": + shutdown() diff --git a/western_ai/usa/pytorch/test/cpp/jit/torch_python_test.cpp b/western_ai/usa/pytorch/test/cpp/jit/torch_python_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bc0d8c9798f90633065afdcb1cd3724f23f686a4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/jit/torch_python_test.cpp @@ -0,0 +1,87 @@ +#include +#include +#include +#include +#include + +namespace torch { +namespace jit { + +#ifdef _MSC_VER +#define JIT_TEST_API +#else +#define JIT_TEST_API TORCH_API +#endif + +namespace { + +bool isSandcastle() { + return ( + (std::getenv("SANDCASTLE")) || + (std::getenv("TW_JOB_USER") && + std::string(std::getenv("TW_JOB_USER")) == "sandcastle")); +} + +void testEvalModeForLoadedModule() { + if (isSandcastle()) + return; // The module file to load is not generated in Sandcastle + std::string module_path = "dropout_model.pt"; + torch::jit::Module module = torch::jit::load(module_path); + AT_ASSERT(module.attr("dropout").toModule().is_training()); + module.eval(); + AT_ASSERT(!module.attr("dropout").toModule().is_training()); + module.train(); + AT_ASSERT(module.attr("dropout").toModule().is_training()); +} + +// TODO: this test never ran before and is broken. +// void testSerializationInterop() { +// if (isSandcastle()) { +// // The module file to load is not generated in Sandcastle +// return; +// } + +// // This should be generated by `test/cpp/jit/tests_setup.py` +// std::ifstream input_stream("ivalue.pt"); +// std::vector input; +// input.insert( +// input.begin(), +// std::istream_iterator(input_stream), +// std::istream_iterator()); +// IValue ivalue = pickle_load(input); + +// auto elements = ivalue.toTupleRef().elements(); +// auto ones = torch::ones({2, 2}); +// AT_ASSERT(ones.equal(elements.at(0).toTensor())); + +// // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) +// auto twos = torch::ones({3, 5}) * 2; +// AT_ASSERT(twos.equal(elements.at(1).toTensor())); +// } + +void testTorchSaveError() { + if (isSandcastle()) { + // The file to load is not generated in Sandcastle + return; + } + + // This should be generated by `test/cpp/jit/tests_setup.py` + bool passed = true; + try { + torch::jit::load("eager_value.pt"); + passed = false; + } catch (const std::exception& c) { + } + // Ensure torch::jit::load did not run + AT_ASSERT(passed); +} +} // namespace + +JIT_TEST_API void runJITCPPTests() { + // TODO: this test never ran before and is broken. + // testSerializationInterop(); + testEvalModeForLoadedModule(); + testTorchSaveError(); +} +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_float_v2.ptl b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_float_v2.ptl new file mode 100644 index 0000000000000000000000000000000000000000..ddee6be4c35afb9352e3c51df3b023c0b43da789 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_float_v2.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_float_v2.ptl.ff b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_float_v2.ptl.ff new file mode 100644 index 0000000000000000000000000000000000000000..8860b3f1eeb35a14f5ec37669fa503c8d01fe0dd Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_float_v2.ptl.ff differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_float_v2.ptl b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_float_v2.ptl new file mode 100644 index 0000000000000000000000000000000000000000..cb36f9aeba8bc66d8dfdd07395ce02e9569a439b Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_float_v2.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_float_v2.ptl.ff b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_float_v2.ptl.ff new file mode 100644 index 0000000000000000000000000000000000000000..ce612850374e367e90d48e12b3aa9c187e64d2b6 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_float_v2.ptl.ff differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_int_v2.ptl b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_int_v2.ptl new file mode 100644 index 0000000000000000000000000000000000000000..443074fe7130cdb2270c289ff48e138e55061ff0 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_int_v2.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_int_v2.ptl.ff b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_int_v2.ptl.ff new file mode 100644 index 0000000000000000000000000000000000000000..d895b209afb3986eb70a2bb8b30a19e627a72481 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_inplace_int_v2.ptl.ff differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_int_v2.ptl b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_int_v2.ptl new file mode 100644 index 0000000000000000000000000000000000000000..ac8b1b918de7c5d30ee03f839d078d080485a8bb Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_int_v2.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_int_v2.ptl.ff b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_int_v2.ptl.ff new file mode 100644 index 0000000000000000000000000000000000000000..ba723415ecf23f2651745402b6c8e4e8d4619f1c Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_int_v2.ptl.ff differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_float_v2.ptl b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_float_v2.ptl new file mode 100644 index 0000000000000000000000000000000000000000..323aa42dde4ec21f327ab731777e2098685153a6 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_float_v2.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_float_v2.ptl.ff b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_float_v2.ptl.ff new file mode 100644 index 0000000000000000000000000000000000000000..61b96da1f6fb0aad7f236c6dfd94b2b071dd289f Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_float_v2.ptl.ff differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_int_v2.ptl b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_int_v2.ptl new file mode 100644 index 0000000000000000000000000000000000000000..6d06dea6b5896d84b9d7a86e00067935d87144c6 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_int_v2.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_int_v2.ptl.ff b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_int_v2.ptl.ff new file mode 100644 index 0000000000000000000000000000000000000000..3085b13a71be1d0c7875120d52521f13a49c0e0b Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_reciprocal_int_v2.ptl.ff differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_scalar_v2.ptl b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_scalar_v2.ptl new file mode 100644 index 0000000000000000000000000000000000000000..4fd551d073aebfc52180a7efd63520704e5c05ef Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_scalar_v2.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_scalar_v2.ptl.ff b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_scalar_v2.ptl.ff new file mode 100644 index 0000000000000000000000000000000000000000..2f8cfb95856c0ae9c76895ed2e31cef645af4246 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_scalar_scalar_v2.ptl.ff differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_inplace_v2.ptl b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_inplace_v2.ptl new file mode 100644 index 0000000000000000000000000000000000000000..9680713a83e280711e6430876f630d857eae37f3 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_inplace_v2.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_inplace_v2.ptl.ff b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_inplace_v2.ptl.ff new file mode 100644 index 0000000000000000000000000000000000000000..20524c64f7c2d8119ed88fba29eb87a6b1cfc69f Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_inplace_v2.ptl.ff differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_out_v2.ptl b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_out_v2.ptl new file mode 100644 index 0000000000000000000000000000000000000000..0381636677b52b82d8de1592e04adabd4efc961c Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_out_v2.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_out_v2.ptl.ff b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_out_v2.ptl.ff new file mode 100644 index 0000000000000000000000000000000000000000..fc5e8f3bd2a029acf06f69c6b332e6065a5bc4b7 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_out_v2.ptl.ff differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_v2.ptl b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_v2.ptl new file mode 100644 index 0000000000000000000000000000000000000000..21792d35b8924f52d20bc6f00dc9ff90b5597675 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_v2.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_v2.ptl.ff b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_v2.ptl.ff new file mode 100644 index 0000000000000000000000000000000000000000..bdababe90d9161966ac4c9b96492b728f6735fa8 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/jit/upgrader_models/test_versioned_div_tensor_v2.ptl.ff differ diff --git a/western_ai/usa/pytorch/test/cpp/lazy/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/lazy/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a07581594d895aa7bb59bc07bce9c95a301c532c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/CMakeLists.txt @@ -0,0 +1,54 @@ +set(LAZY_TEST_ROOT ${TORCH_ROOT}/test/cpp/lazy) + +# Build the cpp gtest binary containing the cpp-only tests. +set(LAZY_TEST_SRCS + ${LAZY_TEST_ROOT}/test_backend_device.cpp + ${LAZY_TEST_ROOT}/test_cache.cpp + ${LAZY_TEST_ROOT}/test_ir.cpp + ${LAZY_TEST_ROOT}/test_ir_util.cpp + ${LAZY_TEST_ROOT}/test_misc.cpp + ${LAZY_TEST_ROOT}/test_permutation_util.cpp + ${LAZY_TEST_ROOT}/test_shape.cpp + ${LAZY_TEST_ROOT}/test_trie_cache.cpp + ${LAZY_TEST_ROOT}/test_util.cpp + ${LAZY_TEST_ROOT}/test_lazy_graph_executor.cpp +) +if(BUILD_LAZY_TS_BACKEND) + list(APPEND LAZY_TEST_SRCS + ${LAZY_TEST_ROOT}/test_lazy_ops.cpp + ${LAZY_TEST_ROOT}/test_lazy_ops_util.cpp + ) +endif() + +add_executable(test_lazy + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${LAZY_TEST_SRCS} +) + +# TODO temporary until we can delete the old gtest polyfills. +target_compile_definitions(test_lazy PRIVATE USE_GTEST) + +set(LAZY_TEST_DEPENDENCIES torch gtest_main) + +target_link_libraries(test_lazy PRIVATE ${LAZY_TEST_DEPENDENCIES}) +target_include_directories(test_lazy PRIVATE ${ATen_CPU_INCLUDE}) + +if(USE_CUDA) + target_compile_definitions(test_lazy PRIVATE USE_CUDA) +elseif(USE_ROCM) + target_link_libraries(test_lazy PRIVATE + hiprtc::hiprtc + hip::amdhip64 + ${TORCH_CUDA_LIBRARIES}) + + target_compile_definitions(test_lazy PRIVATE USE_ROCM) +endif() + +if(INSTALL_TEST) + set_target_properties(test_lazy PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS test_lazy DESTINATION bin) + # Install PDB files for MSVC builds + if(MSVC AND BUILD_SHARED_LIBS) + install(FILES $ DESTINATION bin OPTIONAL) + endif() +endif() diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_backend_device.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_backend_device.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6bffcadca6765b4f591f398104a47386c27a5592 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_backend_device.cpp @@ -0,0 +1,138 @@ +#include + +#include + +#include +#include +#include +#include + +namespace torch { +namespace lazy { + +TEST(BackendDeviceTest, BackendDeviceType) { + auto type = BackendDeviceType(); + + EXPECT_EQ(type.type, 0); + EXPECT_STREQ(type.toString().c_str(), "Unknown"); +} + +TEST(BackendDeviceTest, Basic1) { + auto device = BackendDevice(); + + EXPECT_EQ(device.ordinal(), 0); + if (std::getenv("LTC_TS_CUDA") != nullptr) { + EXPECT_EQ(device.type(), 1); + EXPECT_STREQ(device.toString().c_str(), "CUDA0"); + } else { + EXPECT_EQ(device.type(), 0); + EXPECT_STREQ(device.toString().c_str(), "CPU0"); + } +} + +TEST(BackendDeviceTest, Basic2) { + auto type = std::make_shared(); + type->type = 1; + auto device = BackendDevice(std::move(type), 1); + + EXPECT_EQ(device.type(), 1); + EXPECT_EQ(device.ordinal(), 1); + EXPECT_STREQ(device.toString().c_str(), "Unknown1"); +} + +TEST(BackendDeviceTest, Basic3) { + struct TestType : public BackendDeviceType { + std::string toString() const override { + return "Test"; + } + }; + + auto device = BackendDevice(std::make_shared(), 1); + + EXPECT_EQ(device.type(), 0); + EXPECT_EQ(device.ordinal(), 1); + EXPECT_STREQ(device.toString().c_str(), "Test1"); +} + +TEST(BackendDeviceTest, Basic4) { + // Seems weird to have setters in BackendImplInterface given getBackend() + // returns a const pointer. + auto default_type = getBackend()->GetDefaultDeviceType(); + auto default_ordinal = getBackend()->GetDefaultDeviceOrdinal(); + const_cast(getBackend()) + ->SetDefaultDeviceType(static_cast(c10::kCUDA)); + const_cast(getBackend())->SetDefaultDeviceOrdinal(1); + + auto device = BackendDevice(); + + EXPECT_EQ(device.type(), 1); + EXPECT_EQ(device.ordinal(), 1); + EXPECT_STREQ(device.toString().c_str(), "CUDA1"); + + const_cast(getBackend()) + ->SetDefaultDeviceType(default_type->type); + const_cast(getBackend()) + ->SetDefaultDeviceOrdinal(default_ordinal); +} + +TEST(BackendDeviceTest, Compare) { + auto type = std::make_shared(); + type->type = 1; + + auto device1 = BackendDevice(std::make_shared(), 1); + auto device2 = BackendDevice(std::move(type), 0); + auto device3 = BackendDevice(std::make_shared(), 2); + auto device4 = BackendDevice(std::make_shared(), 1); + + EXPECT_NE(device1, device2); + EXPECT_NE(device1, device3); + EXPECT_EQ(device1, device4); + EXPECT_LT(device1, device2); + EXPECT_LT(device1, device3); +} + +TEST(BackendDeviceTest, Ostream) { + auto device = BackendDevice(); + std::stringstream ss; + ss << device; + + EXPECT_EQ(device.toString(), std::move(ss).str()); +} + +TEST(BackendDeviceTest, FromAten) { + auto device = c10::Device(c10::kCPU); + EXPECT_THROW(atenDeviceToBackendDevice(device), c10::Error); + + device = c10::Device(c10::kLazy); +#ifndef FBCODE_CAFFE2 + auto backend_device = atenDeviceToBackendDevice(device); +#else + // Lazy Tensor is disabled in FBCODE until addressing non-virtual methods + // (e.g. sizes) in TensorImpl + EXPECT_THROW(atenDeviceToBackendDevice(device), c10::Error); +#endif // FBCODE_CAFFE2 +} + +TEST(BackendDeviceTest, ToAten) { + auto device = backendDeviceToAtenDevice(BackendDevice()); + EXPECT_EQ(device.type(), c10::kLazy); + EXPECT_TRUE(device.has_index()); + EXPECT_EQ(device.index(), 0); +} + +// TODO(alanwaketan): Update the following test once we have TorchScript backend +// upstreamed. +TEST(BackendDeviceTest, GetBackendDevice1) { + auto tensor = torch::rand({0, 1, 3, 0}); + EXPECT_FALSE(GetBackendDevice(tensor)); +} + +TEST(BackendDeviceTest, GetBackendDevice2) { + auto tensor1 = torch::rand({0, 1, 3, 0}); + auto tensor2 = torch::rand({0, 1, 3, 0}); + // TODO(alanwaketan): Cover the test case for GetBackendDevice(). + EXPECT_FALSE(GetBackendDevice(tensor1, tensor2)); +} + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_cache.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_cache.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2d63601bdca007aacdb07590b475a5d25fdd9450 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_cache.cpp @@ -0,0 +1,97 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace lazy { + +class CacheNode : public Node { + public: + explicit CacheNode(const std::string& str) + : Node(OpKind(), /* num_outputs */ 1), hash_(Hash(str)), str_(str) {} + ~CacheNode() override = default; + + const std::vector& operands() const override { + TORCH_INTERNAL_ASSERT(false, "Can't access operands of test node"); + } + + const Output& operand(size_t i) const override { + TORCH_INTERNAL_ASSERT(false, "Can't access operand[i] of test node"); + } + + hash_t hash() const override { + return hash_; + } + hash_t shapeHash() const override { + return hash_; + } + + private: + hash_t hash_; + std::string str_; +}; + +TEST(CacheTest, BasicTest) { + std::shared_ptr a = std::make_shared("a"); + std::shared_ptr b = std::make_shared("b"); + std::shared_ptr c = std::make_shared("c"); + Cache cache(2); + + cache.Add(a->hash(), a); + EXPECT_EQ(cache.Get(a->hash()), a); + EXPECT_EQ(cache.Get(b->hash()), nullptr); + EXPECT_EQ(cache.Get(c->hash()), nullptr); + + cache.Add(b->hash(), b); + EXPECT_EQ(cache.Get(a->hash()), a); + EXPECT_EQ(cache.Get(b->hash()), b); + EXPECT_EQ(cache.Get(c->hash()), nullptr); + + cache.Add(c->hash(), c); + EXPECT_EQ(cache.Get(a->hash()), nullptr); // a has been evicted + EXPECT_EQ(cache.Get(b->hash()), b); + EXPECT_EQ(cache.Get(c->hash()), c); + + cache.Erase(c->hash()); + EXPECT_EQ(cache.Get(a->hash()), nullptr); + EXPECT_EQ(cache.Get(b->hash()), b); + EXPECT_EQ(cache.Get(c->hash()), nullptr); // c has been removed + + cache.Clear(); + EXPECT_EQ(cache.Get(a->hash()), nullptr); + EXPECT_EQ(cache.Get(b->hash()), nullptr); + EXPECT_EQ(cache.Get(c->hash()), nullptr); +} + +class CacheNodeWithShape : public TsNode { + public: + explicit CacheNodeWithShape(const Shape& shape) + : TsNode(OpKind(), shape, /* num_outputs */ 1, /* seed */ 0) {} +}; + +TEST(CacheTest, ShapeCacheTestForDynamicShape) { + // enable dynamic shape + FLAGS_ltc_enable_dynamic_shapes = true; + + CacheNodeWithShape nodes[] = { + CacheNodeWithShape(Shape(c10::kFloat, {2, 4})), + CacheNodeWithShape(Shape(c10::kFloat, {4, 2}))}; + + /* + * Make sure the cached shape for node (2, 4) is not used for node (4, 2) + */ + for (auto& node : nodes) { + EXPECT_EQ(node.shape(), node.computeShape([&]() { return node.shape(); })); + } + + // reset the flag + FLAGS_ltc_enable_dynamic_shapes = false; +} + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_ir.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_ir.cpp new file mode 100644 index 0000000000000000000000000000000000000000..048b20f8554744768aac76668449b7be69fd968e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_ir.cpp @@ -0,0 +1,179 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace lazy { + +class TestLeafNode : public Node { + public: + static OpKind ClassOpKind() { + return OpKind(); + } + + explicit TestLeafNode(size_t param) + : Node(ClassOpKind(), /* num_outputs */ 1), hash_(Hash(param)) {} + ~TestLeafNode() override = default; + + const std::vector& operands() const override { + TORCH_INTERNAL_ASSERT(false, "Can't access operands of leaf node"); + } + + const Output& operand(size_t i) const override { + TORCH_INTERNAL_ASSERT(false, "Can't access operand[i] of leaf node"); + } + + hash_t hash() const override { + return hash_; + } + hash_t shapeHash() const override { + return hash_; + } + + private: + hash_t hash_; +}; + +TEST(IrTest, BasicTest) { + NodePtr node1 = MakeNode(1); + NodePtr node2 = MakeNode(2); + EXPECT_NE(node1->hash(), node2->hash()); + + EXPECT_EQ(node1->num_outputs(), 1); + + const TestLeafNode* leafptr = NodeCast(node1.get()); + EXPECT_TRUE(leafptr != nullptr); +} + +TEST(IrTest, MetaDataTest) { + bool restore_FLAGS_torch_lazy_ir_debug = FLAGS_torch_lazy_ir_debug; + FLAGS_torch_lazy_ir_debug = false; + NodePtr node = MakeNode(1); + auto metaWithoutDebug = node->metadata(); + EXPECT_EQ(metaWithoutDebug.scope.size(), 0); + EXPECT_EQ(metaWithoutDebug.frame_info.size(), 0); + + FLAGS_torch_lazy_ir_debug = true; + node = MakeNode(1); + auto metaWithEmptyDebug = node->metadata(); + EXPECT_EQ(metaWithEmptyDebug.scope.size(), 0); + EXPECT_EQ(metaWithEmptyDebug.frame_info.size(), 1); + + { + ScopePusher scope("TestScope"); + node = MakeNode(1); + auto metaWithScope = node->metadata(); + EXPECT_EQ(metaWithScope.scope, "TestScope.1"); + EXPECT_EQ(metaWithScope.frame_info.size(), 1); + } + + SourceLocation dummySourceLocation; + dummySourceLocation.file = "file"; + dummySourceLocation.function = "function"; + dummySourceLocation.line = 10; + GetPythonFramesFunction() = [&]() -> std::vector { + return {dummySourceLocation}; + }; + node = MakeNode(1); + auto metaWithSourceLoc = node->metadata(); + EXPECT_EQ(metaWithSourceLoc.scope.size(), 0); + EXPECT_EQ(metaWithSourceLoc.frame_info.size(), 1); + EXPECT_EQ(metaWithSourceLoc.frame_info[0].file, "file"); + EXPECT_EQ(metaWithSourceLoc.frame_info[0].function, "function"); + EXPECT_EQ(metaWithSourceLoc.frame_info[0].line, 10); + FLAGS_torch_lazy_ir_debug = restore_FLAGS_torch_lazy_ir_debug; +} + +TEST(IrTest, TsNodeTest) { + NodePtr node1 = MakeNode( + OpKind(at::aten::view), + Shape(), + /*num_outputs*/ 1, + /*hash_seed*/ kHashSeed); + NodePtr node2 = MakeNode( + OpKind(at::aten::view), + Shape(), + /*num_outputs*/ 1, + /*hash_seed*/ kHashSeed); + EXPECT_EQ(node1->hash(), node2->hash()); + + EXPECT_EQ(node1->num_outputs(), 1); + + const TsNode* leafptr = dynamic_cast(node1.get()); + EXPECT_TRUE(leafptr != nullptr); +} + +TEST(IrTest, DimensionNodeTest) { + const size_t DIM0 = 5; + const size_t DIM1 = 8; + NodePtr node1 = MakeNode( + OpKind(at::aten::view), + Shape(c10::kFloat, {DIM0, DIM1}), + /*num_outputs*/ 1, + /*hash_seed*/ kHashSeed); + + auto size0 = + std::dynamic_pointer_cast(MakeNode(Value{node1}, 0)); + auto size1 = + std::dynamic_pointer_cast(MakeNode(Value{node1}, 1)); + + ASSERT_EQ(DIM0, size0->getStaticValue()); + ASSERT_EQ(DIM1, size1->getStaticValue()); + + NodePtr size0_np = size0; + auto size0_dn = std::dynamic_pointer_cast(size0_np); + ASSERT_EQ(DIM0, size0_dn->getStaticValue()); + + auto add_dim = std::dynamic_pointer_cast( + MakeNode(Value{size0}, Value{size1})); + ASSERT_EQ(DIM0 + DIM1, add_dim->getStaticValue()); + + auto mul_dim = std::dynamic_pointer_cast( + MakeNode(Value{size0}, Value{size1})); + ASSERT_EQ(DIM0 * DIM1, mul_dim->getStaticValue()); +} + +TEST(IrTest, DimensionIsDynamicTest) { + const size_t DIM0 = 5; + const size_t DIM1 = 8; + const auto shape = Shape(c10::kFloat, {DIM0, DIM1}); + NodePtr node1 = MakeNode( + OpKind(at::aten::view), + shape.with_symbolic_dims(std::vector{true, false}), + /*num_outputs*/ 1, + /*hash_seed*/ kHashSeed); + + auto size0 = + std::dynamic_pointer_cast(MakeNode(Value{node1}, 0)); + auto size1 = + std::dynamic_pointer_cast(MakeNode(Value{node1}, 1)); + + ASSERT_EQ(true, size0->isSymbolic()); + ASSERT_EQ(false, size1->isSymbolic()); + + auto add_dim = std::dynamic_pointer_cast( + MakeNode(Value{size0}, Value{size1})); + ASSERT_EQ(true, add_dim->isSymbolic()); + + add_dim = std::dynamic_pointer_cast( + MakeNode(Value{size1}, Value{size1})); + ASSERT_EQ(false, add_dim->isSymbolic()); + + auto mul_dim = std::dynamic_pointer_cast( + MakeNode(Value{size0}, Value{size0})); + ASSERT_EQ(true, mul_dim->isSymbolic()); +} + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_ir_util.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_ir_util.cpp new file mode 100644 index 0000000000000000000000000000000000000000..56e44514180584c781cca10a7b52d6044975c6e3 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_ir_util.cpp @@ -0,0 +1,81 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace lazy { + +class IrUtilNode : public Node { + public: + explicit IrUtilNode() : Node(OpKind(), /* num_outputs */ 1), hash_(Hash(0)) {} + ~IrUtilNode() override = default; + + void AddOperand(Value v) { + if (!v.node) { + return; + } + operands_as_outputs_.emplace_back(v.node.get(), v.index); + operands_.push_back(std::move(v.node)); + } + + hash_t hash() const override { + return hash_; + } + hash_t shapeHash() const override { + return hash_; + } + + private: + hash_t hash_; +}; + +/* a + * / \ + *b c + * \ / + * d + * Post-order: d c b a + */ +TEST(IrUtilTest, BasicTest) { + NodePtr a = MakeNode(); + NodePtr b = MakeNode(); + NodePtr c = MakeNode(); + NodePtr d = MakeNode(); + + dynamic_cast(a.get())->AddOperand(Value(b, 0)); + dynamic_cast(a.get())->AddOperand(Value(c, 1)); + dynamic_cast(b.get())->AddOperand(Value(d, 0)); + dynamic_cast(c.get())->AddOperand(Value(d, 0)); + + auto postorder = Util::ComputePostOrder({a.get()}); + EXPECT_EQ(postorder.size(), 4); + EXPECT_EQ(postorder.at(0), d.get()); + EXPECT_EQ(postorder.at(1), c.get()); + EXPECT_EQ(postorder.at(2), b.get()); + EXPECT_EQ(postorder.at(3), a.get()); +} + +/* a + * / \ + *b---c + * Post-order: not valid + */ +TEST(IrUtilTest, TestCircle) { + NodePtr a = MakeNode(); + NodePtr b = MakeNode(); + NodePtr c = MakeNode(); + + dynamic_cast(a.get())->AddOperand(Value(b, 0)); + dynamic_cast(b.get())->AddOperand(Value(c, 0)); + dynamic_cast(c.get())->AddOperand(Value(a, 0)); + + EXPECT_THROW(Util::ComputePostOrder({a.get()}), c10::Error); +} + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_graph_executor.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_graph_executor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2208ea4de41ca01aed6d9a2d30c7543417c2abb8 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_graph_executor.cpp @@ -0,0 +1,98 @@ +#include + +#include +#include + +#include + +namespace torch { +namespace lazy { +namespace { + +class LazyGraphExecutorTest : public ::testing::Test { + protected: + void SetUp() override { + executor_ = LazyGraphExecutor::Get(); + executor_->ClearComputationCache(); + } + + using CachedComputationType = LazyGraphExecutor::CachedComputation; + + std::shared_ptr GetCachedComputation(hash_t hash) { + return executor_->GetComputationCache()->Get(hash); + } + + void EnsureComputationIsCached( + std::vector& tensors, + hash_t hash) { + // Force computation to be cached by syncing the tensors. + executor_->SyncTensorsGraph( + &tensors, /* devices */ {}, /* wait */ true, /* sync_ltc_data */ true); + + // Ensure that the computation cache entry exists. + auto cached_computation = GetCachedComputation(hash); + EXPECT_NE(cached_computation, nullptr) + << "Computation should be cached after sync"; + } + + LazyGraphExecutor* executor_; +}; + +TEST_F(LazyGraphExecutorTest, TestClearComputationCache) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor tensor_a = + torch::rand({2, 2}, at::TensorOptions(torch::kFloat)); + torch::Tensor tensor_b = + torch::rand({2, 2}, at::TensorOptions(torch::kFloat)); + + torch::Tensor xla_tensor_a = CopyToDevice(tensor_a, device); + torch::Tensor xla_tensor_b = CopyToDevice(tensor_b, device); + torch::Tensor result = xla_tensor_a + xla_tensor_b; + + std::vector tensors{TryGetLtcTensor(result)}; + hash_t hash = executor_->GetGraphHash(tensors); + EnsureComputationIsCached(tensors, hash); + EXPECT_EQ(executor_->GetComputationCache()->Numel(), 1); + + // Clear the entire computation cache. + executor_->ClearComputationCache(); + + // Ensure that there are no cache entries. + EXPECT_EQ(executor_->GetComputationCache()->Numel(), 0); + auto cached_computation = GetCachedComputation(hash); + EXPECT_EQ(cached_computation, nullptr) + << "Cache entry should be null after clearing"; + }); +} + +TEST_F(LazyGraphExecutorTest, TestRemoveSpecificCacheEntry) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor tensor_a = + torch::rand({2, 2}, at::TensorOptions(torch::kFloat)); + torch::Tensor tensor_b = + torch::rand({2, 2}, at::TensorOptions(torch::kFloat)); + + torch::Tensor xla_tensor_a = CopyToDevice(tensor_a, device); + torch::Tensor xla_tensor_b = CopyToDevice(tensor_b, device); + torch::Tensor result = xla_tensor_a + xla_tensor_b; + + std::vector tensors{TryGetLtcTensor(result)}; + hash_t hash = executor_->GetGraphHash(tensors); + EnsureComputationIsCached(tensors, hash); + + // Remove a specific cache entry. + executor_->RemoveFromComputationCache(hash); + + // Ensure that the cache entry has been removed. + auto cached_computation = GetCachedComputation(hash); + EXPECT_EQ(cached_computation, nullptr) + << "Cache entry should be null after removal"; + + // Attempting to remove again should not do anything. + executor_->RemoveFromComputationCache(hash); + }); +} + +} // namespace +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_ops.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_ops.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e736acabc3761925e1f22a74816ac0245bb1cedf --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_ops.cpp @@ -0,0 +1,11566 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace lazy { + +// Lazy Tensor is disabled in FBCODE until addressing non-virtual methods (e.g. +// sizes) in TensorImpl +#ifndef FBCODE_CAFFE2 + +namespace { +// This registers the torchscript backend, without which lazy device won't work. +// FIXME: This registers the backend for the whole test binary. We should +// probably do it and undo it in the test fixture below. +static bool inline init_backend() { + torch::lazy::InitTorchScriptBackend(); + return true; +} +static const bool backend_initialized = init_backend(); + +} // namespace + +class LazyTsTest : public ::testing::Test { + protected: + void SetUp() override; + + void TearDown() override; + + static void CommonSetup() {} + + void ExpectCounterNotChanged( + const std::string& counter_regex, + const std::unordered_set* ignore_set) {} + + void ExpectCounterChanged( + const std::string& counter_regex, + const std::unordered_set* ignore_set) {} + + void ResetCounters() {} + + private: + void MakeEndSnapshot() {} +}; + +class LazyOpsTestBase : public LazyTsTest { + protected: + static void SetUpTestCase() {} +}; + +void LazyTsTest::SetUp() { + (void)backend_initialized; // avoid unused parameter warning + at::manual_seed(42); + torch::lazy::LazyGraphExecutor::Get()->SetRngSeed( + torch::lazy::BackendDevice(), 42); +} + +void LazyTsTest::TearDown() {} + +namespace { +using torch::lazy::DebugUtil; + +class LazyOpsTest : public LazyOpsTestBase {}; + +static inline bool IsCuda() { + return torch::lazy::getBackend()->EagerFallbackDeviceType() == at::kCUDA; +} + +static inline at::DeviceType DefaultDevice() { + return torch::lazy::getBackend()->EagerFallbackDeviceType(); +} + +} // namespace + +TEST_F(LazyOpsTest, TestScalarTensor) { + torch::Tensor scalar_tensor = torch::scalar_tensor( + 1., torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_scalar_tensor = torch::scalar_tensor( + 1., torch::TensorOptions(torch::kFloat).device(torch::kLazy)); + AllClose(scalar_tensor, lazy_scalar_tensor); + }); +} + +TEST_F(LazyOpsTest, TestClone) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = lazy_a.clone(); + AllClose(a, lazy_b); + lazy_a.add_(1.0); + AllClose(a, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestTo) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_a = CopyToDevice(a, device); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestIsFloatingPoint) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_a = CopyToDevice(a, device); + bool is_float = torch::is_floating_point(a); + bool lazy_is_float = torch::is_floating_point(lazy_a); + EXPECT_EQ(is_float, lazy_is_float); + }); +} + +TEST_F(LazyOpsTest, TestIsSigned) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_a = CopyToDevice(a, device); + bool is_signed = torch::is_signed(a); + bool lazy_is_signed = torch::is_signed(lazy_a); + EXPECT_EQ(is_signed, lazy_is_signed); + }); +} + +TEST_F(LazyOpsTest, TestCastByte) { + torch::Tensor a = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = torch::_cast_Byte(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::_cast_Byte(lazy_a); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestCastChar) { + torch::Tensor a = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = torch::_cast_Char(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::_cast_Char(lazy_a); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestCastShort) { + torch::Tensor a = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = torch::_cast_Short(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::_cast_Short(lazy_a); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestCastInt) { + torch::Tensor a = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = torch::_cast_Int(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::_cast_Int(lazy_a); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestCastLong) { + torch::Tensor a = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = torch::_cast_Long(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::_cast_Long(lazy_a); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestCastFloat) { + torch::Tensor a = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = torch::_cast_Float(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::_cast_Float(lazy_a); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestRetainType) { + torch::Tensor lazy_a = torch::zeros( + {2, 2}, torch::TensorOptions(torch::kByte).device(torch::kLazy)); + torch::Tensor lazy_b = torch::ones( + {2, 2}, torch::TensorOptions(torch::kByte).device(torch::kLazy)); + torch::Tensor lazy_c = lazy_a + lazy_b; + EXPECT_EQ(lazy_c.scalar_type(), torch::ScalarType::Byte); +} + +TEST_F(LazyOpsTest, TestLogicalTypeWithInterop) { + torch::Tensor query = torch::rand( + {2, 12, 20, 64}, + torch::TensorOptions(torch::kFloat).device(torch::kLazy)); + torch::Tensor key = torch::rand( + {2, 12, 64, 20}, + torch::TensorOptions(torch::kFloat).device(torch::kLazy)); + torch::Tensor scores = + torch::matmul(query, key) / + torch::scalar_tensor( + 8, torch::TensorOptions(torch::kDouble).device(torch::kLazy)); + torch::Tensor p_attn = torch::softmax(scores, /*dim=*/-1); + EXPECT_EQ(p_attn.scalar_type(), torch::ScalarType::Float); +} + +TEST_F(LazyOpsTest, TestAdd) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::add(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::add(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestAddHalf) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kHalf).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kHalf).device(DefaultDevice())); + torch::Tensor c = torch::add(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::add(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestAddMixedPrecision) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kHalf).device(DefaultDevice())); + torch::Tensor c = torch::add(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::add(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestAddInPlace) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor c = a.add_(b); + torch::Tensor lazy_c = lazy_a.add_(lazy_b); + AllClose(a, lazy_a); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestAddScalar) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar b(1); + torch::Tensor c = torch::add(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_c = torch::add(lazy_a, b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestAddScalarInPlace) { + torch::Scalar b(1); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor c = a.add_(b); + torch::Tensor lazy_c = lazy_a.add_(b); + AllClose(a, lazy_a); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestAddZeroSizeDim) { + torch::Tensor a = torch::rand( + {0, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {1, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::add(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::add(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestSub) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::sub(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::sub(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestSubInPlace) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor c = a.sub_(b); + torch::Tensor lazy_c = lazy_a.sub_(lazy_b); + AllClose(a, lazy_a); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestSubScalar) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar b(1); + torch::Tensor c = torch::sub(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_c = torch::sub(lazy_a, b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestSubScalarInPlace) { + torch::Scalar b(1); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor c = a.sub_(b); + torch::Tensor lazy_c = lazy_a.sub_(b); + AllClose(a, lazy_a); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestMul) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::mul(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::mul(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestMulInPlace) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor c = a.mul_(b); + torch::Tensor lazy_c = lazy_a.mul_(lazy_b); + AllClose(a, lazy_a); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestMulScalar) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar b(3); + torch::Tensor c = torch::mul(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_c = torch::mul(lazy_a, b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestMulScalarInPlace) { + torch::Scalar b(3); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor c = a.mul_(b); + torch::Tensor lazy_c = lazy_a.mul_(b); + AllClose(a, lazy_a); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestDiv) { + for (torch::ScalarType scalar_type1 : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor a = isFloatingType(scalar_type1) + ? torch::rand({3, 4}, torch::TensorOptions(scalar_type1)) + : torch::randint(0, 100, {3, 4}, torch::TensorOptions(scalar_type1)); + for (torch::ScalarType scalar_type2 : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor b = isFloatingType(scalar_type2) + ? torch::rand({3, 4}, torch::TensorOptions(scalar_type2)) + : torch::randint(1, 100, {3, 4}, torch::TensorOptions(scalar_type2)); + torch::Tensor c = torch::div(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::div(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); + } + } +} + +TEST_F(LazyOpsTest, TestDivWithRoundingMode) { + std::optional rounding_modes[] = { + "trunc", "floor", std::nullopt}; + for (const auto& rounding_mode : rounding_modes) { + for (torch::ScalarType scalar_type1 : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + int lower_bound = (scalar_type1 == torch::kByte) ? 0 : -100; + torch::Tensor a = isFloatingType(scalar_type1) + ? torch::rand({3, 4}, torch::TensorOptions(scalar_type1)) + : torch::randint( + lower_bound, 50, {3, 4}, torch::TensorOptions(scalar_type1)); + for (torch::ScalarType scalar_type2 : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor b = isFloatingType(scalar_type2) + ? torch::rand({3, 4}, torch::TensorOptions(scalar_type2)) + : torch::randint( + 51, 100, {3, 4}, torch::TensorOptions(scalar_type2)); + torch::Tensor c = torch::div(a, b, rounding_mode); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::div(lazy_a, lazy_b, rounding_mode); + AllClose(c, lazy_c); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestDivInPlace) { + for (torch::ScalarType scalar_type1 : {torch::kFloat}) { + torch::Tensor a = isFloatingType(scalar_type1) + ? torch::rand({3, 4}, torch::TensorOptions(scalar_type1)) + : torch::randint(0, 100, {3, 4}, torch::TensorOptions(scalar_type1)); + for (torch::ScalarType scalar_type2 : {torch::kFloat}) { + torch::Tensor b = isFloatingType(scalar_type2) + ? torch::rand({3, 4}, torch::TensorOptions(scalar_type2)) + : torch::randint(1, 100, {3, 4}, torch::TensorOptions(scalar_type2)); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor c = a.div_(b); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = lazy_a.div_(lazy_b); + ; + AllClose(c, lazy_c); + }); + } + } +} + +TEST_F(LazyOpsTest, TestDivInPlaceWithRoundingMode) { + std::optional rounding_modes[] = { + "trunc", "floor", std::nullopt}; + for (const auto& rounding_mode : rounding_modes) { + for (torch::ScalarType scalar_type1 : {torch::kFloat}) { + torch::Tensor a = isFloatingType(scalar_type1) + ? torch::rand({3, 4}, torch::TensorOptions(scalar_type1)) + : torch::randint( + -100, 100, {3, 4}, torch::TensorOptions(scalar_type1)); + for (torch::ScalarType scalar_type2 : {torch::kFloat}) { + torch::Tensor b = isFloatingType(scalar_type2) + ? torch::rand({3, 4}, torch::TensorOptions(scalar_type2)) + : torch::randint( + 1, 100, {3, 4}, torch::TensorOptions(scalar_type2)); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor c = a.div_(b, rounding_mode); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = lazy_a.div_(lazy_b, rounding_mode); + AllClose(c, lazy_c); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestDivScalar) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor a = isFloatingType(scalar_type) + ? torch::rand( + {3, 4}, torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 1, + 100, + {3, 4}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool is_float : {true, false}) { + torch::Scalar b = is_float ? torch::Scalar(3.0) : torch::Scalar(3); + torch::Tensor c = torch::div(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_c = torch::div(lazy_a, b); + AllClose(c, lazy_c); + }); + } + } +} + +TEST_F(LazyOpsTest, TestDivScalarInPlace) { + for (torch::ScalarType scalar_type : {torch::kFloat}) { + torch::Tensor a = isFloatingType(scalar_type) + ? torch::rand( + {3, 4}, torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 1, + 100, + {3, 4}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool is_float : {true, false}) { + torch::Scalar b = is_float ? torch::Scalar(3.0) : torch::Scalar(3); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor c = a.div_(b); + torch::Tensor lazy_c = lazy_a.div_(b); + AllClose(c, lazy_c); + }); + } + } +} + +TEST_F(LazyOpsTest, TestDivOut) { + for (torch::ScalarType scalar_type : {torch::kFloat, torch::kDouble}) { + torch::Tensor a = torch::rand( + {3, 4}, torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3, 4}, torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor c = torch::empty( + {3, 4}, torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::div_out(c, a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::empty({3, 4}, lazy_b.options()); + torch::div_out(lazy_c, lazy_a, lazy_b); + AllClose(c, lazy_c); + }); + } +} + +TEST_F(LazyOpsTest, TestRsubScalar) { + torch::Tensor input = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar other(1.5); + torch::Scalar alpha(2.5); + torch::Tensor result = torch::rsub(input, other, alpha); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::rsub(lazy_input, other, alpha); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestNe) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::ne(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::ne(lazy_a, lazy_b); + AllEqual(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestNeInplace) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor a_copy = a.clone(); + torch::Tensor b = a.clone(); + b[0] += 1; + a.ne_(b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a_copy, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + lazy_a.ne_(lazy_b); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestEq) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = a.clone(); + torch::Tensor c = torch::eq(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::eq(lazy_a, lazy_b); + AllEqual(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestEqInplace) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = a.clone(); + b[0] += 1; + torch::Tensor a_copy = a.clone(); + a.eq_(b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a_copy, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + lazy_a.eq_(lazy_b); + AllClose(lazy_a, a); + }); +} + +TEST_F(LazyOpsTest, TestGe) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = a.clone(); + torch::Tensor c = torch::ge(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::ge(lazy_a, lazy_b); + AllEqual(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestGeInplace) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = a.clone(); + b[0] += 1; + b[1] -= 1; + torch::Tensor a_copy = a.clone(); + a.ge_(b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a_copy, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + lazy_a.ge_(lazy_b); + AllClose(lazy_a, a); + }); +} + +TEST_F(LazyOpsTest, TestLe) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = a.clone(); + torch::Tensor c = torch::le(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::le(lazy_a, lazy_b); + AllEqual(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestLeInplace) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = a.clone(); + b[0] += 1; + b[1] -= 1; + torch::Tensor a_copy = a.clone(); + a.le_(b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a_copy, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + lazy_a.le_(lazy_b); + AllClose(lazy_a, a); + }); +} + +TEST_F(LazyOpsTest, TestGt) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::add(a.clone(), torch::ones_like(a)); + torch::Tensor c = torch::gt(b, a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::gt(lazy_b, lazy_a); + AllEqual(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestGtInplace) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = a.clone(); + b[0] += 1; + b[1] -= 1; + torch::Tensor a_copy = a.clone(); + a.gt_(b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a_copy, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + lazy_a.gt_(lazy_b); + AllClose(lazy_a, a); + }); +} + +TEST_F(LazyOpsTest, TestLt) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::add(a.clone(), torch::ones_like(a)); + torch::Tensor c = torch::lt(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::lt(lazy_a, lazy_b); + AllEqual(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestLtInplace) { + torch::Tensor a = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = a.clone(); + b[0] += 1; + b[1] -= 1; + torch::Tensor a_copy = a.clone(); + a.lt_(b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a_copy, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + lazy_a.lt_(lazy_b); + AllClose(lazy_a, a); + }); +} + +TEST_F(LazyOpsTest, TestNeScalar) { + torch::Tensor input = torch::ones({2, 3}); + torch::Scalar other(float(0)); + torch::Tensor result = torch::ne(input, other); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::ne(lazy_input, other); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestEqScalar) { + torch::Tensor input = torch::ones({2, 3}); + torch::Scalar other(float(1)); + torch::Tensor result = torch::eq(input, other); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::eq(lazy_input, other); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestGeScalar) { + torch::Tensor input = torch::ones({2, 3}); + torch::Scalar other(float(1)); + torch::Tensor result = torch::ge(input, other); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::ge(lazy_input, other); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestGeScalarInplace) { + torch::Tensor input = torch::arange( + -1., + 1.5, + 0.5, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar other(float(0)); + torch::Tensor input_copy = input.clone(); + input.ge_(other); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input_copy, device); + lazy_input.ge_(other); + AllClose(lazy_input, input); + }); +} + +TEST_F(LazyOpsTest, TestLeScalar) { + torch::Tensor input = torch::ones({2, 3}); + torch::Scalar other(float(1)); + torch::Tensor result = torch::le(input, other); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::le(lazy_input, other); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestLeScalarInplace) { + torch::Tensor input = torch::arange( + -1., + 1.5, + 0.5, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar other(float(0)); + torch::Tensor input_copy = input.clone(); + input.le_(other); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input_copy, device); + lazy_input.le_(other); + AllClose(lazy_input, input); + }); +} + +TEST_F(LazyOpsTest, TestGtScalar) { + torch::Tensor input = torch::ones({2, 3}); + torch::Scalar other(float(0.5)); + torch::Tensor result = torch::gt(input, other); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::gt(lazy_input, other); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestGtScalarInplace) { + torch::Tensor input = torch::arange( + -1., + 1.5, + 0.5, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar other(float(0)); + torch::Tensor input_copy = input.clone(); + input.gt_(other); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input_copy, device); + lazy_input.gt_(other); + AllClose(lazy_input, input); + }); +} + +TEST_F(LazyOpsTest, TestLtScalar) { + torch::Tensor input = torch::ones({2, 3}); + torch::Scalar other(float(1.5)); + torch::Tensor result = torch::lt(input, other); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::lt(lazy_input, other); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestLtScalarInplace) { + torch::Tensor input = torch::arange( + -1., + 1.5, + 0.5, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar other(float(0)); + torch::Tensor input_copy = input.clone(); + input.lt_(other); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input_copy, device); + lazy_input.lt_(other); + AllClose(lazy_input, input); + }); +} + +TEST_F(LazyOpsTest, TestIntegerAdd) { + std::vector types( + {torch::kByte, torch::kChar, torch::kShort, torch::kInt, torch::kLong}); + + ForEachDevice([&](const torch::Device& device) { + for (auto type : types) { + torch::Tensor a = + torch::randint(0, 63, {2, 2}, torch::TensorOptions(type)); + torch::Tensor b = + torch::randint(0, 63, {2, 2}, torch::TensorOptions(type)); + torch::Scalar one = + isIntegralType(type, false) ? torch::Scalar(1) : torch::Scalar(1.0); + torch::Tensor c = torch::add(b, one); + + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::add(lazy_b, one); + + AllEqual(c, lazy_c); + } + }); +} + +TEST_F(LazyOpsTest, TestSVD) { + static const int dims[] = {4, 7}; + for (auto m : dims) { + for (auto n : dims) { + torch::Tensor a = torch::rand( + {m, n}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + auto b = torch::svd(a, /*some=*/true, /*compute_uv=*/true); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + auto lazy_b = torch::svd(lazy_a, /*some=*/true, /*compute_uv=*/true); + // The U and V matrices might have different sign for column vectors, so + // cannot be compared if not by absolute value. + AllClose( + std::get<0>(b).abs(), + std::get<0>(lazy_b).abs(), + /*rtol=*/1e-3, + /*atol=*/1e-4); + torch::Tensor diag = std::get<1>(b); + torch::Tensor lazy_diag = std::get<1>(lazy_b); + ASSERT_EQ(diag.sizes(), lazy_diag.sizes()); + AllClose( + diag, + lazy_diag, + /*rtol=*/1e-3, + /*atol=*/1e-4); + AllClose( + std::get<2>(b).abs(), + std::get<2>(lazy_b).abs(), + /*rtol=*/1e-3, + /*atol=*/1e-4); + }); + } + } +} + +TEST_F(LazyOpsTest, TestQR) { + static const int dims[] = {4, 7}; + for (auto m : dims) { + for (auto n : dims) { + torch::Tensor a = torch::rand( + {m, n}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + auto b = torch::qr(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + auto lazy_b = torch::qr(lazy_a); + AllClose( + std::get<0>(b).abs(), + std::get<0>(lazy_b).abs(), + /*rtol=*/1e-3, + /*atol=*/1e-4); + AllClose( + std::get<1>(b).abs(), + std::get<1>(lazy_b).abs(), + /*rtol=*/1e-3, + /*atol=*/1e-4); + }); + } + } +} + +TEST_F(LazyOpsTest, TestLogDet) { + static const int dims[] = {4, 7}; + for (auto m : dims) { + torch::Tensor a = torch::rand( + {3, m, m}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor pd_a = torch::matmul(a, torch::transpose(a, 1, 2)) + + torch::eye(m, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::logdet(pd_a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(pd_a, device); + torch::Tensor lazy_b = torch::logdet(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-4); + }); + } +} + +TEST_F(LazyOpsTest, TestTriangularSolve) { + static const int dims[] = {4, 7}; + for (bool batched_a : {true, false}) { + for (bool batched_b : {true, false}) { + for (auto m : dims) { + for (auto n : dims) { + for (bool upper : {true, false}) { + for (bool transpose : {true, false}) { + for (bool unitriangular : {true, false}) { + torch::Tensor a = torch::randn( + {m, m}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice())); + torch::Tensor b = torch::randn( + {m, n}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice())); + a = batched_a ? a.expand({3, m, m}).clone() : a; + b = batched_b ? b.expand({3, m, n}).clone() : b; + auto result = torch::triangular_solve( + b, + a, + /*upper=*/upper, + /*transpose=*/transpose, + /*unitriangular=*/unitriangular); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + auto lazy_result = torch::triangular_solve( + lazy_b, + lazy_a, + /*upper=*/upper, + /*transpose=*/transpose, + /*unitriangular=*/unitriangular); + AllClose( + std::get<0>(result), + std::get<0>(lazy_result), + /*rtol=*/1e-3, + /*atol=*/1e-4); + AllClose( + std::get<1>(result), + std::get<1>(lazy_result), + /*rtol=*/1e-3, + /*atol=*/1e-4); + }); + } + } + } + } + } + } + } +} + +TEST_F(LazyOpsTest, TestKthValue) { + torch::Tensor a = torch::rand( + {4, 5, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int k = 1; k <= 3; ++k) { + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + for (bool keepdim : {false, true}) { + auto b = torch::kthvalue(a, k, dim, keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + auto lazy_b = torch::kthvalue(lazy_a, k, dim, keepdim); + AllClose(std::get<0>(b), std::get<0>(lazy_b)); + AllEqual(std::get<1>(b), std::get<1>(lazy_b)); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestTopK) { + torch::Tensor a = torch::rand( + {4, 5, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int k = 1; k <= 3; ++k) { + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + for (bool largest : {false, true}) { + auto b = torch::topk(a, k, dim, largest, /*sorted=*/true); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + auto lazy_b = torch::topk(lazy_a, k, dim, largest, /*sorted=*/true); + AllClose(std::get<0>(b), std::get<0>(lazy_b)); + AllEqual(std::get<1>(b), std::get<1>(lazy_b)); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestSort) { + torch::Tensor a = torch::rand( + {4, 5, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int k = 1; k <= 3; ++k) { + for (int dim = 0; dim < 3; ++dim) { + for (bool descending : {false, true}) { + auto b = torch::sort(a, dim, descending); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + auto lazy_b = torch::sort(lazy_a, dim, descending); + AllClose(std::get<0>(b), std::get<0>(lazy_b)); + AllEqual(std::get<1>(b), std::get<1>(lazy_b)); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestSortDescWithMinValue) { + std::vector values{-128, 100}; + torch::Tensor input = + torch::tensor(values, torch::TensorOptions(torch::kChar)); + auto output = torch::sort(input, /*dim=*/0, /*descending=*/true); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + auto lazy_output = torch::sort(lazy_input, /*dim=*/0, /*descending=*/true); + AllEqual(std::get<0>(output), std::get<0>(lazy_output)); + AllEqual(std::get<1>(output), std::get<1>(lazy_output)); + }); +} + +TEST_F(LazyOpsTest, TestArgSort) { + torch::Tensor a = torch::rand( + {4, 5, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int k = 1; k <= 3; ++k) { + for (int dim = 0; dim < 3; ++dim) { + for (bool descending : {false, true}) { + torch::Tensor b = torch::argsort(a, dim, descending); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::argsort(lazy_a, dim, descending); + AllEqual(b, lazy_b); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestMin) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::min(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::min(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestMax) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::max(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::max(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestUnaryMin) { + torch::Tensor input = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::min(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::min(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestUnaryMax) { + torch::Tensor input = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::max(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::max(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestAll) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor a = isFloatingType(scalar_type) + ? torch::rand( + {3, 4}, torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {3, 4}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor b = torch::all(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::all(lazy_a); + EqualValues(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestAllDim) { + torch::Tensor a = torch::randint( + 0, + 5, + {2, 3, 4}, + torch::TensorOptions(torch::kByte).device(DefaultDevice())); + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor b = torch::all(a, dim, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::all(lazy_a, dim, /*keepdim=*/false); + EqualValues(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestAllDimKeep) { + torch::Tensor a = torch::randint( + 0, + 5, + {2, 3, 4}, + torch::TensorOptions(torch::kByte).device(DefaultDevice())); + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor b = torch::all(a, dim, /*keepdim=*/true); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::all(lazy_a, dim, /*keepdim=*/true); + EqualValues(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestAmax) { + torch::Tensor input = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (bool keepdim : {false, true}) { + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor values = torch::amax(input, {dim}, /*keepdim=*/keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_values = + torch::amax(lazy_input, {dim}, /*keepdim=*/keepdim); + AllClose(values, lazy_values); + }); + } + for (int dim1 = -rank; dim1 < rank; ++dim1) { + for (int dim2 = -rank; dim2 < rank; ++dim2) { + if ((dim1 == dim2) || (dim1 == rank + dim2) || (dim2 == rank + dim1)) + continue; + torch::Tensor values = + torch::amax(input, {dim1, dim2}, /*keepdim=*/keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_values = + torch::amax(lazy_input, {dim1, dim2}, /*keepdim=*/keepdim); + AllClose(values, lazy_values); + }); + } + } + } + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("xla::amax", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestAmin) { + torch::Tensor input = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (bool keepdim : {false, true}) { + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor values = torch::amin(input, {dim}, /*keepdim=*/keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_values = + torch::amin(lazy_input, {dim}, /*keepdim=*/keepdim); + AllClose(values, lazy_values); + }); + } + for (int dim1 = -rank; dim1 < rank; ++dim1) { + for (int dim2 = -rank; dim2 < rank; ++dim2) { + if ((dim1 == dim2) || (dim1 == rank + dim2) || (dim2 == rank + dim1)) + continue; + torch::Tensor values = + torch::amin(input, {dim1, dim2}, /*keepdim=*/keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_values = + torch::amin(lazy_input, {dim1, dim2}, /*keepdim=*/keepdim); + AllClose(values, lazy_values); + }); + } + } + } + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("xla::amin", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestAny) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor a = isFloatingType(scalar_type) + ? torch::rand( + {3, 4}, torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {3, 4}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor b = torch::any(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::any(lazy_a); + EqualValues(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestAnyDim) { + torch::Tensor a = torch::randint( + 0, + 5, + {2, 3, 4}, + torch::TensorOptions(torch::kByte).device(DefaultDevice())); + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor b = torch::any(a, dim, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::any(lazy_a, dim, /*keepdim=*/false); + EqualValues(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestAnyDimKeep) { + torch::Tensor a = torch::randint( + 0, + 5, + {2, 3, 4}, + torch::TensorOptions(torch::kByte).device(DefaultDevice())); + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor b = torch::any(a, dim, /*keepdim=*/true); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::any(lazy_a, dim, /*keepdim=*/true); + EqualValues(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestMean) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::mean(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::mean(lazy_a); + ASSERT_EQ(b.sizes(), lazy_b.sizes()); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestMeanCast) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::mean(a, torch::kDouble); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::mean(lazy_a, torch::kDouble); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestMeanInDim) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor b = torch::mean(a, {dim}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::mean(lazy_a, {dim}); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestMeanInDims) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto dims : std::vector>{{0, 1}, {-3, -2}}) { + torch::Tensor b = torch::mean(a, dims); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::mean(lazy_a, dims); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestMeanInDimsKeepCast) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto dims : std::vector>{{0, 1}, {-3, -2}}) { + torch::Tensor b = torch::mean(a, dims, true, torch::kDouble); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::mean(lazy_a, dims, true, torch::kDouble); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestMeanInDimOut) { + torch::Tensor a = torch::rand( + {4, 4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor b = torch::empty( + {4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::mean_out(b, a, {dim}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::empty({4, 4}, lazy_a.options()); + torch::mean_out(lazy_b, lazy_a, {dim}); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestStd) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto unbiased : {true, false}) { + torch::Tensor b = torch::std(a, unbiased); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::std(lazy_a, unbiased); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestStdInDim) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = a.dim(); + for (auto unbiased : {true, false}) { + for (auto keepdim : {true, false}) { + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor b = torch::std(a, {dim}, unbiased, keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::std(lazy_a, {dim}, unbiased, keepdim); + AllClose(b, lazy_b); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestStdWithCorrection) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // int rank = a.dim(); + std::optional corrections[] = {1, 2, std::nullopt}; + for (const auto& correction : corrections) { + for (auto keepdim : {true, false}) { + for (const auto& dim : + std::vector>{{0, 1}, {-3, -2}}) { + torch::Tensor b = torch::std(a, dim, correction, keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::std(lazy_a, dim, correction, keepdim); + AllClose(b, lazy_b); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestStdMeanWithCorrection) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // int rank = a.dim(); + std::optional corrections[] = {1, 2, std::nullopt}; + for (const auto& correction : corrections) { + for (auto keepdim : {true, false}) { + for (const auto& dim : + std::vector>{{0, 1}, {-3, -2}}) { + auto b = torch::std_mean(a, dim, correction, keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + auto lazy_b = torch::std_mean(lazy_a, dim, correction, keepdim); + AllClose(std::get<0>(b), std::get<0>(lazy_b)); + AllClose(std::get<1>(b), std::get<1>(lazy_b)); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestSum) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::sum(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sum(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestSumCast) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::sum(a, torch::kDouble); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sum(lazy_a, torch::kDouble); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestSumU8) { + torch::Tensor a = torch::ones( + {256}, torch::TensorOptions(torch::kByte).device(DefaultDevice())); + torch::Tensor b = torch::sum(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sum(lazy_a); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestSumInDim) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor b = torch::sum(a, {dim}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sum(lazy_a, {dim}); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestSumInDims) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto dims : std::vector>{{0, 1}, {-3, -2}}) { + torch::Tensor b = torch::sum(a, dims); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sum(lazy_a, dims); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestSumInDimsKeep) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto dims : std::vector>{{0, 1}, {-3, -2}}) { + torch::Tensor b = torch::sum(a, dims, /*keepdim=*/true); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sum(lazy_a, dims, /*keepdim=*/true); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestSumInDimsKeepCast) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto dims : std::vector>{{0, 1}, {-3, -2}}) { + torch::Tensor b = torch::sum(a, dims, /*keepdim=*/true, torch::kDouble); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = + torch::sum(lazy_a, dims, /*keepdim=*/true, torch::kDouble); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestVar) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (bool unbiased : {true, false}) { + torch::Tensor b = torch::var(a, unbiased); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::var(lazy_a, unbiased); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestVarWithDim) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto dims : std::vector>{{0, 1}, {-3, -2}}) { + for (bool keepDim : {true, false}) { + for (bool unbiased : {true, false}) { + torch::Tensor b = torch::var(a, dims, unbiased, keepDim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::var(lazy_a, dims, unbiased, keepDim); + AllClose(b, lazy_b); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestVarWithCorrection) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::optional corrections[] = {1, 2, std::nullopt}; + for (const auto& dim : std::vector>{{0, 1}, {-3, -2}}) { + for (bool keepDim : {true, false}) { + for (const auto& correction : corrections) { + torch::Tensor b = torch::var(a, dim, correction, keepDim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::var(lazy_a, dim, correction, keepDim); + AllClose(b, lazy_b); + }); + } + } + } + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("lazy::var", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestVarMeanWithCorrection) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::optional corrections[] = {1, 2, std::nullopt}; + for (const auto& dim : std::vector>{{0, 1}, {-3, -2}}) { + for (const auto& correction : corrections) { + for (auto keepdim : {true, false}) { + auto b = torch::var_mean(a, dim, correction, keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + auto lazy_b = torch::var_mean(lazy_a, dim, correction, keepdim); + AllClose(std::get<0>(b), std::get<0>(lazy_b)); + AllClose(std::get<1>(b), std::get<1>(lazy_b)); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxInDim) { + torch::Tensor input = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + for (bool keepdim : {false, true}) { + auto values_indices = torch::max(input, dim, /*keepdim=*/keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + auto lazy_values_indices = + torch::max(lazy_input, dim, /*keepdim=*/keepdim); + AllClose(std::get<0>(values_indices), std::get<0>(lazy_values_indices)); + AllEqual(std::get<1>(values_indices), std::get<1>(lazy_values_indices)); + }); + } + } +} + +TEST_F(LazyOpsTest, TestMinInDim) { + torch::Tensor input = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + for (bool keepdim : {false, true}) { + auto values_indices = torch::min(input, dim, /*keepdim=*/keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + auto lazy_values_indices = + torch::min(lazy_input, dim, /*keepdim=*/keepdim); + AllClose(std::get<0>(values_indices), std::get<0>(lazy_values_indices)); + AllEqual(std::get<1>(values_indices), std::get<1>(lazy_values_indices)); + }); + } + } +} + +TEST_F(LazyOpsTest, TestNorm) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::norm(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::norm(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestNormInDim) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int dim : {1, -2}) { + torch::Tensor b = torch::norm(a, 2, {dim}, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::norm(lazy_a, 2, {dim}, /*keepdim=*/false); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestNormInDims) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto dims : std::vector>{{1, 2}, {-2, -1}}) { + torch::Tensor b = torch::norm(a, 2, dims, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::norm(lazy_a, 2, dims, /*keepdim=*/false); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestNormInDimsKeep) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto dims : std::vector>{{1, 2}, {-2, -1}}) { + torch::Tensor b = torch::norm(a, 2, dims, /*keepdim=*/true); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::norm(lazy_a, 2, dims, /*keepdim=*/true); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestNormalTwoTensor) { + at::Tensor mean = at::zeros({10, 10, 10}, at::dtype(at::kFloat)); + at::Tensor std = at::ones({10, 10, 10}, at::dtype(at::kFloat)); + ForEachDevice([&](const torch::Device& device) { + at::Tensor lazy_mean = CopyToDevice(mean, device); + at::Tensor lazy_std = CopyToDevice(std, device); + at::Tensor lazy_normal = at::normal(lazy_mean, lazy_std); + double res_mean = lazy_normal.mean().item().toDouble(); + double res_std = lazy_normal.std().item().toDouble(); + EXPECT_GT(res_mean, -0.06); + EXPECT_LT(res_mean, 0.06); + EXPECT_GT(res_std, 0.94); + EXPECT_LT(res_std, 1.06); + }); +} + +TEST_F(LazyOpsTest, TestNormalDoubleMean) { + at::Tensor std = at::ones({10, 10, 10}, at::dtype(at::kFloat)); + ForEachDevice([&](const torch::Device& device) { + at::Tensor lazy_std = CopyToDevice(std, device); + at::Tensor lazy_normal = at::normal(0, lazy_std); + double res_mean = lazy_normal.mean().item().toDouble(); + double res_std = lazy_normal.std().item().toDouble(); + EXPECT_GT(res_mean, -0.06); + EXPECT_LT(res_mean, 0.06); + EXPECT_GT(res_std, 0.94); + EXPECT_LT(res_std, 1.06); + }); +} + +TEST_F(LazyOpsTest, TestNormalDoubleStd) { + at::Tensor mean = at::zeros({10, 10, 10}, at::dtype(at::kFloat)); + ForEachDevice([&](const torch::Device& device) { + at::Tensor lazy_mean = CopyToDevice(mean, device); + at::Tensor lazy_normal = at::normal(lazy_mean, 1); + double res_mean = lazy_normal.mean().item().toDouble(); + double res_std = lazy_normal.std().item().toDouble(); + EXPECT_GT(res_mean, -0.06); + EXPECT_LT(res_mean, 0.06); + EXPECT_GT(res_std, 0.94); + EXPECT_LT(res_std, 1.06); + }); +} + +TEST_F(LazyOpsTest, TestNormalInPlace) { + at::Tensor a = at::zeros({10, 10, 10}, at::dtype(at::kFloat)); + ForEachDevice([&](const torch::Device& device) { + at::Tensor lazy_a = CopyToDevice(a, device); + lazy_a.normal_(/*mean=*/0, /*std=*/1); + double res_mean = lazy_a.mean().item().toDouble(); + double res_std = lazy_a.std().item().toDouble(); + EXPECT_GT(res_mean, -0.06); + EXPECT_LT(res_mean, 0.06); + EXPECT_GT(res_std, 0.94); + EXPECT_LT(res_std, 1.06); + }); +} + +TEST_F(LazyOpsTest, TestUniformInPlace) { + const double eps = 1e-3; + at::Tensor a = at::zeros({10, 10, 10}, at::dtype(at::kFloat)); + ForEachDevice([&](const torch::Device& device) { + at::Tensor lazy_a = CopyToDevice(a, device); + lazy_a.uniform_(/*from=*/0, /*to=*/1); + at::Tensor cpu_a = ToCpuTensor(lazy_a); + double res_min = cpu_a.min().item().toDouble(); + double res_max = cpu_a.max().item().toDouble(); + EXPECT_GT(res_min, 0.0 - eps); + EXPECT_LT(res_max, 1.0 + eps); + }); +} + +TEST_F(LazyOpsTest, TestRandomInPlace) { + for (auto dtype : + {torch::kFloat, + torch::kDouble, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + const double eps = 0.2; + torch::Tensor a = torch::zeros({10, 10, 10}, torch::TensorOptions(dtype)); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + lazy_a.random_(/*from=*/0, /*to=*/10); + double res_mean = lazy_a.sum().item().toDouble() / a.numel(); + double res_min = lazy_a.min().item().toDouble(); + double res_max = lazy_a.max().item().toDouble(); + EXPECT_GT(res_mean, 4.5 - eps); + EXPECT_LT(res_mean, 4.5 + eps); + EXPECT_EQ(res_min, 0.0); + EXPECT_EQ(res_max, 9.0); + }); + } +} + +TEST_F(LazyOpsTest, TestRandomInPlaceDefaultFrom) { + for (auto dtype : + {torch::kFloat, + torch::kDouble, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + const double eps = 0.2; + torch::Tensor a = torch::zeros({10, 10, 10}, torch::TensorOptions(dtype)); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + lazy_a.random_(/*to=*/10); + double res_mean = lazy_a.sum().item().toDouble() / a.numel(); + double res_min = lazy_a.min().item().toDouble(); + double res_max = lazy_a.max().item().toDouble(); + EXPECT_GT(res_mean, 4.5 - eps); + EXPECT_LT(res_mean, 4.5 + eps); + EXPECT_EQ(res_min, 0.0); + EXPECT_EQ(res_max, 9.0); + }); + } +} + +TEST_F(LazyOpsTest, TestRandomInPlaceDefault) { + for (auto dtype : + {torch::kFloat, + torch::kDouble, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + auto input = torch::zeros({10}, torch::TensorOptions(dtype)); + ForEachDevice([&](const torch::Device& device) { + auto lazyInput = CopyToDevice(input, device); + lazyInput.random_(); + auto output = ToCpuTensor(lazyInput); + EXPECT_TRUE(torch::all(output.ne(input)).item()); + }); + } +} + +TEST_F(LazyOpsTest, TestNormGeneral) { + torch::Tensor a = torch::randn( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::norm(a, 3.5); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::norm(lazy_a, 3.5); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestNormNuclear) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::norm(a, 1); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::norm(lazy_a, 1); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestFrobeniusNormInDim) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int dim : {1, -2}) { + torch::Tensor b = torch::frobenius_norm(a, {dim}, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = + torch::frobenius_norm(lazy_a, {dim}, /*keepdim=*/false); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestFrobeniusNormInDims) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto dims : std::vector>{{1, 2}, {-2, -1}}) { + torch::Tensor b = torch::frobenius_norm(a, dims, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = + torch::frobenius_norm(lazy_a, dims, /*keepdim=*/false); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestGroupNorm) { + int num_channels = 6; + torch::Tensor input = torch::rand( + {20, num_channels, 10, 10}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {num_channels}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor bias = torch::rand( + {num_channels}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + double eps = 1e-05; + for (int num_groups : {3, 6, 1}) { + torch::Tensor output = torch::group_norm( + input, + num_groups, + weight, + bias, + eps, + /*cudnn_enabled=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_weight = CopyToDevice(weight, device); + torch::Tensor lazy_bias = CopyToDevice(bias, device); + torch::Tensor lazy_output = torch::group_norm( + lazy_input, + num_groups, + lazy_weight, + lazy_bias, + eps, + /*cudnn_enabled=*/false); + AllClose(output, lazy_output, /*rtol=*/1e-3, /*atol=*/1e-5); + }); + } +} + +TEST_F(LazyOpsTest, TestGroupNormBackward) { + int num_channels = 6; + torch::Tensor input = torch::rand( + {2, num_channels, 5, 5}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor weight = torch::rand( + {num_channels}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor bias = torch::rand( + {num_channels}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + double eps = 1e-05; + for (bool undef_weight : {true, false}) { + for (int num_groups : {3, 6, 1}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::group_norm( + /*input=*/inputs[0], + num_groups, + inputs[1], + inputs[2], + /*eps=*/eps, + /*cudnn_enabled=*/false); + }; + torch::Tensor undef; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {input, undef_weight ? undef : weight, undef_weight ? undef : bias}, + device, + testfn, + /*rtol=*/1e-3, + /*atol=*/1e-3, + /*derivative_level=*/2); + }); + } + } +} + +TEST_F(LazyOpsTest, TestInstanceNorm) { + int batch = 5; + int num_channels = 20; + torch::Tensor input = torch::rand( + {batch, num_channels, 10, 10}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {num_channels}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor bias = torch::rand( + {num_channels}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor running_mean = torch::zeros( + {num_channels}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor running_var = torch::ones( + {num_channels}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + double momentum = 0.1; + double eps = 1e-05; + torch::Tensor output = torch::instance_norm( + input, + weight, + bias, + running_mean, + running_var, + /*use_input_stats=*/true, + momentum, + eps, + /*cudnn_enabled=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_weight = CopyToDevice(weight, device); + torch::Tensor lazy_bias = CopyToDevice(bias, device); + torch::Tensor lazy_running_mean = CopyToDevice(running_mean, device); + torch::Tensor lazy_running_var = CopyToDevice(running_var, device); + torch::Tensor lazy_output = torch::instance_norm( + lazy_input, + lazy_weight, + lazy_bias, + lazy_running_mean, + lazy_running_var, + /*use_input_stats=*/true, + momentum, + eps, + /*cudnn_enabled=*/false); + AllClose(output, lazy_output, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestLayerNorm) { + torch::Tensor input = torch::rand( + {20, 10, 10, 10}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + double eps = 1e-05; + torch::Tensor undef; + for (bool undef_weight : {true, false}) { + for (int64_t normalized_size : {2, 3}) { + std::vector normalized_shape(normalized_size, 10); + torch::Tensor weight = torch::rand( + normalized_shape, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor bias = torch::rand( + normalized_shape, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::layer_norm( + input, + normalized_shape, + undef_weight ? undef : weight, + undef_weight ? undef : bias, + eps, + /*cudnn_enabled=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_weight = + undef_weight ? undef : CopyToDevice(weight, device); + torch::Tensor lazy_bias = + undef_weight ? undef : CopyToDevice(bias, device); + torch::Tensor lazy_output = torch::layer_norm( + lazy_input, + normalized_shape, + lazy_weight, + lazy_bias, + eps, + /*cudnn_enabled=*/false); + AllClose(output, lazy_output, /*rtol=*/1e-3, /*atol=*/1e-5); + }); + } + } +} + +TEST_F(LazyOpsTest, TestLayerNormBackward) { + torch::Tensor input = torch::rand( + {2, 3, 3, 3}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + double eps = 1e-05; + for (bool undef_weight : {true, false}) { + for (int64_t normalized_size : {2, 3}) { + std::vector normalized_shape(normalized_size, 3); + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::layer_norm( + /*input=*/inputs[0], + normalized_shape, + inputs[1], + inputs[2], + /*eps=*/eps, + /*cudnn_enabled=*/false); + }; + torch::Tensor weight = torch::rand( + normalized_shape, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor bias = torch::rand( + normalized_shape, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor undef; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {input, undef_weight ? undef : weight, undef_weight ? undef : bias}, + device, + testfn, + /*rtol=*/1e-3, + /*atol=*/1e-4, + /*derivative_level=*/2); + }); + } + } +} + +TEST_F(LazyOpsTest, TestNuclearNorm) { + torch::Tensor a = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::nuclear_norm(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::nuclear_norm(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestPairwiseDistance) { + torch::Tensor x1 = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor x2 = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + double eps = 1e-6; + for (bool keepdim : {false, true}) { + for (double p : {1, 2, 3, 4}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = + torch::pairwise_distance(x1, x2, p, eps, keepdim); + torch::Tensor lazy_x1 = CopyToDevice(x1, device); + torch::Tensor lazy_x2 = CopyToDevice(x2, device); + torch::Tensor lazy_output = + torch::pairwise_distance(lazy_x1, lazy_x2, p, eps, keepdim); + AllClose(output, lazy_output, /*rtol=*/1e-5, /*atol=*/1e-5); + }); + } + } +} + +TEST_F(LazyOpsTest, TestCosineSimilarity) { + torch::Tensor x1 = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor x2 = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + double eps = 1e-8; + int rank = x1.dim(); + for (int dim = -rank; dim < rank; ++dim) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = torch::cosine_similarity(x1, x2, dim, eps); + torch::Tensor lazy_x1 = CopyToDevice(x1, device); + torch::Tensor lazy_x2 = CopyToDevice(x2, device); + torch::Tensor lazy_output = + torch::cosine_similarity(lazy_x1, lazy_x2, dim, eps); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestCosineEmbeddingLoss) { + torch::Tensor input1 = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor input2 = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor target = torch::rand( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, torch::Reduction::Sum}) { + for (double margin : {0., 0.2}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = torch::cosine_embedding_loss( + input1, input2, target, margin, reduction); + torch::Tensor lazy_input1 = CopyToDevice(input1, device); + torch::Tensor lazy_input2 = CopyToDevice(input2, device); + torch::Tensor lazy_target = CopyToDevice(target, device); + torch::Tensor lazy_output = torch::cosine_embedding_loss( + lazy_input1, lazy_input2, lazy_target, margin, reduction); + AllClose(output, lazy_output); + }); + } + } +} + +TEST_F(LazyOpsTest, TestHingeEmbeddingLoss) { + torch::Tensor input = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor target = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, torch::Reduction::Sum}) { + for (double margin : {0., 0.2}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = + torch::hinge_embedding_loss(input, target, margin, reduction); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_target = CopyToDevice(target, device); + torch::Tensor lazy_output = torch::hinge_embedding_loss( + lazy_input, lazy_target, margin, reduction); + AllClose(output, lazy_output); + }); + } + } +} + +TEST_F(LazyOpsTest, TestTripletMarginLoss) { + torch::Tensor anchor = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor positive = torch::abs(torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice()))); + torch::Tensor negative = torch::neg(torch::abs(torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())))); + double eps = 1e-6; + for (double margin : {0., 0.2}) { + for (double p : {1, 2, 3, 4}) { + for (bool swap : {false, true}) { + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, torch::Reduction::Sum}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = torch::triplet_margin_loss( + anchor, positive, negative, margin, p, eps, swap, reduction); + torch::Tensor lazy_anchor = CopyToDevice(anchor, device); + torch::Tensor lazy_positive = CopyToDevice(positive, device); + torch::Tensor lazy_negative = CopyToDevice(negative, device); + torch::Tensor lazy_output = torch::triplet_margin_loss( + lazy_anchor, + lazy_positive, + lazy_negative, + margin, + p, + eps, + swap, + reduction); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestBinaryCrossEntropy) { + int batch = 10; + int classes = 5; + torch::Tensor input = torch::rand( + {batch, classes}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor target = torch::rand( + {batch, classes}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {batch, classes}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor undef; + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, + torch::Reduction::Sum, + torch::Reduction::None}) { + for (bool undef_weight : {false, true}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = torch::binary_cross_entropy( + input, target, undef_weight ? undef : weight, reduction); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_target = CopyToDevice(target, device); + torch::Tensor lazy_weight = + undef_weight ? undef : CopyToDevice(weight, device); + torch::Tensor lazy_output = torch::binary_cross_entropy( + lazy_input, lazy_target, lazy_weight, reduction); + AllClose(output, lazy_output, /*rtol=*/1e-4, /*atol=*/1e-5); + }); + } + } +} + +TEST_F(LazyOpsTest, TestMarginRankingLoss) { + torch::Tensor input1 = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor input2 = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor target = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, torch::Reduction::Sum}) { + for (double margin : {0., 0.2}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = torch::margin_ranking_loss( + input1, input2, target, margin, reduction); + torch::Tensor lazy_input1 = CopyToDevice(input1, device); + torch::Tensor lazy_input2 = CopyToDevice(input2, device); + torch::Tensor lazy_target = CopyToDevice(target, device); + torch::Tensor lazy_output = torch::margin_ranking_loss( + lazy_input1, lazy_input2, lazy_target, margin, reduction); + AllClose(output, lazy_output); + }); + } + } +} + +TEST_F(LazyOpsTest, TestBCEWithLogits) { + int batch = 10; + int classes = 5; + torch::Tensor input = torch::rand( + {batch, classes}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor target = torch::rand( + {batch, classes}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {classes}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor pos_weight = torch::rand( + {classes}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor undef; + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, torch::Reduction::Sum}) { + for (bool undef_weight : {false, true}) { + for (bool undef_pos_weight : {false, true}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = torch::binary_cross_entropy_with_logits( + input, + target, + undef_weight ? undef : weight, + undef_pos_weight ? undef : pos_weight, + reduction); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_target = CopyToDevice(target, device); + torch::Tensor lazy_weight = + undef_weight ? undef : CopyToDevice(weight, device); + torch::Tensor lazy_pos_weight = + undef_pos_weight ? undef : CopyToDevice(pos_weight, device); + torch::Tensor lazy_output = torch::binary_cross_entropy_with_logits( + lazy_input, lazy_target, lazy_weight, lazy_pos_weight, reduction); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestKlDiv) { + torch::Tensor input = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor target = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (bool log_target : {true, false}) { + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, torch::Reduction::Sum}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = + torch::kl_div(input, target, reduction, log_target); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_target = CopyToDevice(target, device); + torch::Tensor lazy_output = + torch::kl_div(lazy_input, lazy_target, reduction, log_target); + AllClose(output, lazy_output); + }); + } + } +} + +TEST_F(LazyOpsTest, TestProd) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::prod(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::prod(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestProdCast) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::prod(a, torch::kDouble); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::prod(lazy_a, torch::kDouble); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestProdInDim) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor b = torch::prod(a, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::prod(lazy_a, dim); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestProdInDimKeepCast) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor b = torch::prod(a, dim, /*keepdim=*/true, torch::kDouble); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = + torch::prod(lazy_a, dim, /*keepdim=*/true, torch::kDouble); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestProdInDimKeep) { + torch::Tensor a = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = a.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor b = torch::prod(a, dim, /*keepdim=*/true); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::prod(lazy_a, dim, /*keepdim=*/true); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestCumSum) { + torch::Tensor input = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::cumsum(input, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::cumsum(lazy_input, dim); + AllClose(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestCumSumCast) { + torch::Tensor input = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::cumsum(input, dim, torch::kDouble); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = + torch::cumsum(lazy_input, dim, torch::kDouble); + AllClose(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestCumSumLong) { + torch::Tensor input = torch::randint( + 1000, + {4, 3, 4}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::cumsum(input, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::cumsum(lazy_input, dim); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestCumSumCastLong) { + torch::Tensor input = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::cumsum(input, dim, torch::kLong); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::cumsum(lazy_input, dim, torch::kLong); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestCumProd) { + torch::Tensor input = torch::rand( + {4, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::cumprod(input, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::cumprod(lazy_input, dim); + AllClose(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestCumProdCast) { + torch::Tensor input = torch::mul( + torch::rand( + {4, 3, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())), + 10); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::cumprod(input, dim, torch::kDouble); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = + torch::cumprod(lazy_input, dim, torch::kDouble); + AllClose(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestCumProdLong) { + torch::Tensor input = torch::randint( + 7, {2, 3}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::cumsum(input, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::cumsum(lazy_input, dim); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestCumProdCastLong) { + torch::Tensor input = + torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 7; + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::cumsum(input, dim, torch::kLong); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::cumsum(lazy_input, dim, torch::kLong); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestArgMin) { + torch::Tensor a = torch::rand( + {4, 4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::argmin(a, std::nullopt, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = + torch::argmin(lazy_a, std::nullopt, /*keepdim=*/false); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestArgMinDim) { + torch::Tensor a = torch::rand( + {4, 4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int dim : {1, -2}) { + torch::Tensor b = torch::argmin(a, dim, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::argmin(lazy_a, dim, /*keepdim=*/false); + AllEqual(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestArgMinDimKeep) { + torch::Tensor a = torch::rand( + {4, 4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int dim : {1, -2}) { + torch::Tensor b = torch::argmin(a, dim, /*keepdim=*/true); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::argmin(lazy_a, dim, /*keepdim=*/true); + AllEqual(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestArgMinSameValue) { + torch::Tensor a = torch::ones( + {4, 4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::argmin(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::argmin(lazy_a); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestArgMinWrapper) { + torch::Tensor a = torch::rand( + {4, 4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int dim : {1, -2}) { + torch::Tensor b = torch::argmin(a, dim, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::argmin(lazy_a, dim, /*keepdim=*/false); + AllEqual(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestArgMax) { + torch::Tensor a = torch::rand( + {4, 4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::argmax(a, std::nullopt, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = + torch::argmax(lazy_a, std::nullopt, /*keepdim=*/false); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestArgMaxDim) { + torch::Tensor a = torch::rand( + {4, 4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int dim : {1, -2}) { + torch::Tensor b = torch::argmax(a, dim, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::argmax(lazy_a, dim, /*keepdim=*/false); + AllEqual(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestArgMaxDimKeep) { + torch::Tensor a = torch::rand( + {4, 4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int dim : {1, -2}) { + torch::Tensor b = torch::argmax(a, dim, /*keepdim=*/true); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::argmax(lazy_a, dim, /*keepdim=*/true); + AllEqual(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestArgMaxSameValue) { + torch::Tensor a = torch::ones( + {4, 4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::argmax(a, std::nullopt, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = + torch::argmax(lazy_a, std::nullopt, /*keepdim=*/false); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestArgMaxWrapper) { + torch::Tensor a = torch::rand( + {4, 4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int dim : {1, -2}) { + torch::Tensor b = torch::argmax(a, dim, /*keepdim=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::argmax(lazy_a, dim, /*keepdim=*/false); + AllEqual(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestAsin) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::asin(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::asin(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestAsinh) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::asinh(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::asinh(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestAsinhInPlace) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor b = torch::asinh_(a); + torch::Tensor lazy_b = torch::asinh_(lazy_a); + AllClose(a, lazy_a, /*rtol=*/1e-3, /*atol=*/1e-5); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestSin) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::sin(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sin(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestSinh) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::sinh(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sinh(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestAcos) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::acos(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::acos(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestAcosh) { + torch::Tensor a = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100; + torch::Tensor b = torch::acosh(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::acosh(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestAcoshInPlace) { + torch::Tensor a = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor b = torch::acosh_(a); + torch::Tensor lazy_b = torch::acosh_(lazy_a); + AllClose(a, lazy_a, /*rtol=*/1e-3, /*atol=*/1e-5); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestCos) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::cos(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::cos(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestCosh) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::cosh(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::cosh(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestAtan) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::atan(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::atan(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestAtanh) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::atanh(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::atanh(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestAtanhInPlace) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor b = torch::atanh_(a); + torch::Tensor lazy_b = torch::atanh_(lazy_a); + AllClose(a, lazy_a, /*rtol=*/1e-3, /*atol=*/1e-5); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestAtan2) { + torch::Tensor a = torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::atan2(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::atan2(lazy_a, lazy_b); + AllClose(c, lazy_c, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestTan) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::tan(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::tan(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestTanh) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::tanh(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::tanh(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestClampMinMax) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar min_val(0.311); + torch::Scalar max_val(0.409); + torch::Tensor b = torch::clamp(a, min_val, max_val); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::clamp(lazy_a, min_val, max_val); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestClampMin) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar min_val(0.311); + torch::Tensor b = torch::clamp(a, min_val, std::nullopt); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::clamp(lazy_a, min_val, std::nullopt); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestClampMax) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar max_val(0.409); + torch::Tensor b = torch::clamp(a, std::nullopt, max_val); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::clamp(lazy_a, std::nullopt, max_val); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestClampMinExplicit) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar min_val(0.311); + torch::Tensor b = torch::clamp_min(a, min_val); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::clamp_min(lazy_a, min_val); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestClampMaxExplicit) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar max_val(0.409); + torch::Tensor b = torch::clamp_max(a, max_val); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::clamp_max(lazy_a, max_val); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestClampMinExplicitInPlace) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar min_val(0.311); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor b = torch::clamp_min_(a, min_val); + torch::Tensor lazy_b = torch::clamp_min_(lazy_a, min_val); + AllClose(a, lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestClampMaxExplicitInPlace) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar max_val(0.409); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor b = torch::clamp_max_(a, max_val); + torch::Tensor lazy_b = torch::clamp_max_(lazy_a, max_val); + AllClose(a, lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestCeil) { + torch::Tensor a = + torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = torch::ceil(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::ceil(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestFloor) { + torch::Tensor a = + torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = torch::floor(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::floor(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestRound) { + torch::Tensor a = torch::cat( + {torch::randn( + {8}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0, + // Special case: 0.5, -0.5. lazy::Round impl rounds to -1/1 whereas + // lazy::RoundToEven properly implements bankers rounding. + torch::tensor( + {-0.5, 0.5}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice()))}, + 0); + torch::Tensor b = torch::round(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::round(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestTrunc) { + torch::Tensor a = + torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = torch::trunc(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::trunc(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestFrac) { + torch::Tensor a = + torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = torch::frac(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::frac(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestNeg) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::neg(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::neg(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseNot) { + std::vector types( + {torch::kByte, torch::kChar, torch::kShort, torch::kInt, torch::kLong}); + + ForEachDevice([&](const torch::Device& device) { + for (auto type : types) { + torch::Tensor a = + torch::randint(0, 63, {2, 2}, torch::TensorOptions(type)); + torch::Tensor b = torch::bitwise_not(a); + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::bitwise_not(lazy_a); + AllEqual(b, lazy_b); + } + }); +} + +TEST_F(LazyOpsTest, TestBitwiseNotInPlace) { + std::vector types( + {torch::kByte, torch::kChar, torch::kShort, torch::kInt, torch::kLong}); + + ForEachDevice([&](const torch::Device& device) { + for (auto type : types) { + torch::Tensor a = + torch::randint(0, 63, {2, 2}, torch::TensorOptions(type)); + torch::Tensor lazy_a = CopyToDevice(a, device); + a.bitwise_not_(); + lazy_a.bitwise_not_(); + AllEqual(a, lazy_a); + } + }); +} + +TEST_F(LazyOpsTest, TestSign) { + torch::Tensor a = + torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = torch::sign(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sign(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestSignByte) { + torch::Tensor a = torch::randint( + 256, {2, 2}, torch::TensorOptions(torch::kByte).device(DefaultDevice())); + torch::Tensor b = torch::sign(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sign(lazy_a); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestAbs) { + torch::Tensor a = torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::abs(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::abs(lazy_a); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestAbsByte) { + torch::Tensor a = torch::randint( + 256, {2, 2}, torch::TensorOptions(torch::kByte).device(DefaultDevice())); + torch::Tensor b = torch::abs(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::abs(lazy_a); + AllEqual(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestEmptyLike) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::empty_like(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::empty_like(lazy_a); + EXPECT_EQ(b.sizes(), lazy_b.sizes()); + }); +} + +TEST_F(LazyOpsTest, TestEmptyLikeOptions) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::empty_like( + a, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::empty_like( + lazy_a, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + EXPECT_EQ(b.sizes(), lazy_b.sizes()); + }); +} + +TEST_F(LazyOpsTest, TestEmpty) { + torch::Tensor a = torch::zeros( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = torch::empty( + {2, 2}, torch::TensorOptions(torch::kFloat).device(device)); + EXPECT_EQ(a.sizes(), lazy_a.sizes()); + }); +} + +TEST_F(LazyOpsTest, TestZeroInPlace) { + torch::Tensor input = torch::ones( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazyInput = CopyToDevice(input, device); + auto& output = torch::zero_(input); + auto& lazyOutput = torch::zero_(lazyInput); + AllClose(output, lazyOutput); + }); +} + +TEST_F(LazyOpsTest, TestZerosLike) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::zeros_like(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::zeros_like(lazy_a); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestZerosLikeOptions) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::zeros_like( + a, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::zeros_like( + lazy_a, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestZeros) { + torch::Tensor a = torch::zeros( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = torch::zeros( + {2, 2}, torch::TensorOptions(torch::kFloat).device(device)); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestOnes) { + torch::Tensor a = torch::ones( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = + torch::ones({2, 2}, torch::TensorOptions(torch::kFloat).device(device)); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestOnesLike) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::ones_like(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::ones_like(lazy_a); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestOnesLikeOptions) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::ones_like( + a, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::ones_like( + lazy_a, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestFull) { + torch::Tensor a = torch::full( + {2, 2}, + 3.1165, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = torch::full( + {2, 2}, 3.1165, torch::TensorOptions(torch::kFloat).device(device)); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestFullLike) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::full_like(a, 3.1165); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::full_like(lazy_a, 3.1165); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestFullLikeOptions) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::full_like( + a, 3.1165, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::full_like( + lazy_a, + 3.1165, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestARange) { + for (auto& ranges : std::vector>{ + {0.0, 100.0, 0.5}, {0.0, -100.0, -0.5}}) { + torch::Tensor a = torch::arange( + ranges[0], + ranges[1], + ranges[2], + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = torch::arange( + ranges[0], + ranges[1], + ranges[2], + torch::TensorOptions(torch::kFloat).device(device)); + AllClose(a, lazy_a); + }); + } +} + +TEST_F(LazyOpsTest, TestARangeOut) { + torch::Tensor a = torch::randn( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto& ranges : std::vector>{ + {0.0, 100.0, 0.5}, {0.0, -100.0, -0.5}}) { + torch::Tensor b = torch::arange_out(a, ranges[0], ranges[1], ranges[2]); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = + torch::arange_out(lazy_a, ranges[0], ranges[1], ranges[2]); + AllClose(b, lazy_b); + }); + } +} + +TEST_F(LazyOpsTest, TestDimARange) { + torch::Tensor like = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor a = torch::_dim_arange(like, 1); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_like = CopyToDevice(like, device); + torch::Tensor lazy_a = torch::_dim_arange(lazy_like, 1); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestBartlettWindow) { + int window_length = 10; + for (bool periodic : {false, true}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = torch::bartlett_window( + window_length, + periodic, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + + torch::Tensor lazy_output = torch::bartlett_window( + window_length, + periodic, + torch::TensorOptions(torch::kFloat).device(device)); + AllClose(output, lazy_output, /*rtol=*/1e-5, /*atol=*/1e-7); + }); + } +} + +TEST_F(LazyOpsTest, TestBlackmanWindow) { + int window_length = 10; + for (bool periodic : {false, true}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = torch::blackman_window( + window_length, + periodic, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_output = torch::blackman_window( + window_length, + periodic, + torch::TensorOptions(torch::kFloat).device(device)); + AllClose(output, lazy_output, /*rtol=*/1e-5, /*atol=*/1e-7); + }); + } +} + +TEST_F(LazyOpsTest, TestHammingWindow) { + double alpha = 0.54; + double beta = 0.46; + int window_length = 10; + for (bool periodic : {false, true}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = torch::hamming_window( + window_length, + periodic, + alpha, + beta, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_output = torch::hamming_window( + window_length, + periodic, + alpha, + beta, + torch::TensorOptions(torch::kFloat).device(device)); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestHannWindow) { + int window_length = 10; + for (bool periodic : {false, true}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor output = torch::hann_window( + window_length, + periodic, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_output = torch::hann_window( + window_length, + periodic, + torch::TensorOptions(torch::kFloat).device(device)); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestLogSigmoid) { + torch::Tensor a = torch::empty( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + a.uniform_(-1.0, 1.0); + torch::Tensor b = torch::log_sigmoid(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::log_sigmoid(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestLogSigmoidForward) { + torch::Tensor a = torch::empty( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + a.uniform_(-1.0, 1.0); + auto tuple = torch::log_sigmoid_forward(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + auto lazy_tuple = torch::log_sigmoid_forward(lazy_a); + AllClose( + std::get<0>(tuple), + std::get<0>(lazy_tuple), + /*rtol=*/1e-3, + /*atol=*/1e-5); + AllClose( + std::get<1>(tuple), + std::get<1>(lazy_tuple), + /*rtol=*/1e-3, + /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestLogsumexp) { + torch::Tensor a = torch::rand( + {3, 4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (auto dims : std::vector>{{0, 1}, {-3, -2}}) { + for (bool keepdim : {false, true}) { + torch::Tensor b = torch::logsumexp(a, dims, keepdim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::logsumexp(lazy_a, dims, keepdim); + AllClose(b, lazy_b); + }); + } + } +} + +TEST_F(LazyOpsTest, TestSiLU) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::silu(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::silu(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); + ExpectCounterChanged("lazy::silu_out", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestSigmoid) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::sigmoid(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sigmoid(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestMatmul_1x1) { + torch::Tensor a = torch::rand( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::matmul(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::matmul(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestMatmul_2x1) { + torch::Tensor a = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::matmul(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::matmul(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestMatmul_1x2) { + torch::Tensor a = torch::rand( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::matmul(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::matmul(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestMatmul_2x2) { + torch::Tensor a = torch::rand( + {2, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::matmul(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::matmul(lazy_a, lazy_b); + AllClose(c, lazy_c, /*rtol=*/1e-3, /*atol=*/1e-4); + }); +} + +TEST_F(LazyOpsTest, TestMatmulBcast) { + torch::Tensor a = torch::rand( + {4, 2, 3, 2, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 1, 4, 3}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::matmul(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::matmul(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestDot) { + torch::Tensor a = torch::rand( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::dot(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::dot(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestTensorDot) { + torch::Tensor a = torch::rand( + {6, 4, 8}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {4, 7, 8}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector dims_a = {1, 2}; + std::vector dims_b = {0, 2}; + torch::Tensor c = torch::tensordot(a, b, dims_a, dims_b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::tensordot(lazy_a, lazy_b, dims_a, dims_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestGer) { + torch::Tensor a = torch::rand( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::ger(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::ger(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestMv) { + torch::Tensor a = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::mv(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::mv(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestMvOut) { + torch::Tensor a = torch::rand( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::empty( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::mv_out(c, a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::empty({4}, lazy_b.options()); + torch::mv_out(lazy_c, lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestBatchAddBatchMatMul) { + torch::Tensor a = torch::rand( + {3, 6, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3, 6, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::rand( + {3, 4, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar alpha = 0.5; + torch::Scalar beta = 1.5; + torch::Tensor d = torch::baddbmm(a, b, c, beta, alpha); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::baddbmm(lazy_a, lazy_b, lazy_c, beta, alpha); + AllClose(d, lazy_d, /*rtol=*/1e-3, /*atol=*/1e-4); + }); +} + +TEST_F(LazyOpsTest, TestBatchAddBatchMatMulInPlace) { + torch::Tensor a = torch::rand( + {3, 6, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3, 6, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::rand( + {3, 4, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar alpha = 0.5; + torch::Scalar beta = 1.5; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor d = a.baddbmm_(b, c, beta, alpha); + torch::Tensor lazy_d = lazy_a.baddbmm_(lazy_b, lazy_c, beta, alpha); + AllClose(d, lazy_d, /*rtol=*/1e-3, /*atol=*/1e-4); + AllClose(a, lazy_a, /*rtol=*/1e-3, /*atol=*/1e-4); + }); +} + +TEST_F(LazyOpsTest, TestBatchMatMul) { + torch::Tensor a = torch::rand( + {3, 6, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3, 4, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::bmm(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::bmm(lazy_a, lazy_b); + AllClose(c, lazy_c, /*rtol=*/1e-3, /*atol=*/1e-4); + }); +} + +TEST_F(LazyOpsTest, TestChainMatMul) { + torch::Tensor a = torch::rand( + {5, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {4, 6}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::rand( + {6, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor d = torch::rand( + {2, 7}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor result = torch::chain_matmul({a, b, c, d}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = CopyToDevice(d, device); + torch::Tensor lazy_result = + torch::chain_matmul({lazy_a, lazy_b, lazy_c, lazy_d}); + AllClose(result, lazy_result, /*rtol=*/1e-3, /*atol=*/1e-4); + }); +} + +TEST_F(LazyOpsTest, TestLinear) { + torch::Tensor input = torch::rand( + {2, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor bias = torch::rand( + {3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor result = torch::linear(input, weight); + torch::Tensor result_with_bias = torch::linear(input, weight, bias); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_weight = CopyToDevice(weight, device); + torch::Tensor lazy_bias = CopyToDevice(bias, device); + torch::Tensor lazy_result = torch::linear(lazy_input, lazy_weight); + torch::Tensor lazy_result_with_bias = + torch::linear(lazy_input, lazy_weight, lazy_bias); + AllClose(result, lazy_result, /*rtol=*/1e-2, /*atol=*/1e-4); + AllClose( + result_with_bias, + lazy_result_with_bias, + /*rtol=*/1e-2, + /*atol=*/1e-4); + }); +} + +TEST_F(LazyOpsTest, TestPinverse) { + torch::Tensor input = torch::rand( + {4, 6}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor result = torch::pinverse(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::pinverse(lazy_input); + AllClose(result, lazy_result, /*rtol=*/1e-4); + }); +} + +TEST_F(LazyOpsTest, TestEinsumOuter) { + torch::Tensor a = torch::rand( + {5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::string equation = "i,j->ij"; + torch::Tensor c = torch::einsum(equation, {a, b}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::einsum(equation, {lazy_a, lazy_b}); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestEinsumOuterBackward) { + torch::Tensor a = torch::rand( + {5}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor b = torch::rand( + {5}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + std::string equation = "i,j->ij"; + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::einsum(equation, inputs); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward({a, b}, device, testfn, /*rtol=*/1e-3, /*atol=*/1e-4); + }); +} + +TEST_F(LazyOpsTest, TestEinsumBatchMatMul) { + torch::Tensor a = torch::rand( + {3, 2, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3, 5, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::string equation = "bij,bjk->bik"; + torch::Tensor c = torch::einsum(equation, {a, b}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::einsum(equation, {lazy_a, lazy_b}); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestEinsumPyTorchLowerBilinear) { + torch::Tensor a = torch::rand( + {3, 5, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor l = torch::rand( + {2, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor r = torch::rand( + {2, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::string equation = "bn,anm,bm->ba"; + torch::Tensor c = torch::einsum(equation, {l, a, r}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_l = CopyToDevice(l, device); + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_r = CopyToDevice(r, device); + torch::Tensor lazy_c = torch::einsum(equation, {lazy_l, lazy_a, lazy_r}); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestEinsumPyTorchLowerDiagonal) { + torch::Tensor input = torch::rand( + {3, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::string equation = "ii->i"; + torch::Tensor result = torch::einsum(equation, {input}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::einsum(equation, {lazy_input}); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestEinsumPyTorchLowerBatchDiagonal) { + torch::Tensor input = torch::rand( + {4, 3, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::string equation = "...ii->...i"; + torch::Tensor result = torch::einsum(equation, {input}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::einsum(equation, {lazy_input}); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestEinsumPyTorchLowerBatchPermute) { + torch::Tensor input = torch::rand( + {2, 3, 4, 5}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::string equation = "...ij->...ji"; + torch::Tensor result = torch::einsum(equation, {input}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::einsum(equation, {lazy_input}); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestEinsumPyTorchLowerRepeatedAxis) { + torch::Tensor x = torch::rand( + {2, 3, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor y = torch::rand( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::string equation = "ijj,k->ik"; + torch::Tensor result = torch::einsum(equation, {x, y}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_x = CopyToDevice(x, device); + torch::Tensor lazy_y = CopyToDevice(y, device); + torch::Tensor lazy_result = torch::einsum(equation, {lazy_x, lazy_y}); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestBilinear) { + int batch_size = 16; + int in1_features = 4; + int in2_features = 6; + int out_features = 8; + torch::Tensor input1 = torch::rand( + {batch_size, in1_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor input2 = torch::rand( + {batch_size, in2_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {out_features, in1_features, in2_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor bias = torch::rand( + {out_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input1 = CopyToDevice(input1, device); + torch::Tensor lazy_input2 = CopyToDevice(input2, device); + torch::Tensor lazy_weight = CopyToDevice(weight, device); + torch::Tensor lazy_bias = CopyToDevice(bias, device); + torch::Tensor result = torch::bilinear(input1, input2, weight, bias); + torch::Tensor lazy_result = + torch::bilinear(lazy_input1, lazy_input2, lazy_weight, lazy_bias); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestUpsampleNearest2D) { + int batch_size = 2; + int h = 5; + int w = 5; + int uh = 8; + int uw = 8; + int chans = 2; + torch::Tensor input = torch::rand( + {batch_size, chans, h, w}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor result = torch::upsample_nearest2d(input, {uh, uw}); + torch::Tensor lazy_result = torch::upsample_nearest2d(lazy_input, {uh, uw}); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestUpsampleNearest2DBackward) { + int batch_size = 2; + int h = 5; + int w = 5; + int uh = 8; + int uw = 8; + int chans = 2; + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::upsample_nearest2d(inputs[0], {uh, uw}); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {batch_size, chans, h, w}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestUpsampleNearest2DWithScale) { + int batch_size = 2; + int h = 5; + int w = 5; + int chans = 2; + double scale_h = 2.5; + double scale_w = 3.4; + torch::Tensor input = torch::rand( + {batch_size, chans, h, w}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor result = torch::upsample_nearest2d( + input, std::nullopt, at::ArrayRef{scale_h, scale_w}); + torch::Tensor lazy_result = torch::upsample_nearest2d( + lazy_input, std::nullopt, at::ArrayRef{scale_h, scale_w}); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestUpsampleNearest2DBackwardWithScale) { + int batch_size = 2; + int h = 5; + int w = 5; + int chans = 2; + double scale_h = 2.5; + double scale_w = 3.4; + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::upsample_nearest2d( + inputs[0], std::nullopt, at::ArrayRef{scale_h, scale_w}); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {batch_size, chans, h, w}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestUpsampleBilinear2D) { + int batch_size = 2; + int h = 5; + int w = 5; + int uh = 8; + int uw = 8; + int chans = 2; + for (bool align_corners : {true, false}) { + torch::Tensor input = torch::rand( + {batch_size, chans, h, w}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor result = + torch::upsample_bilinear2d(input, {uh, uw}, align_corners); + torch::Tensor lazy_result = + torch::upsample_bilinear2d(lazy_input, {uh, uw}, align_corners); + AllClose(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestUpsampleBilinear2DBackward) { + int batch_size = 2; + int h = 5; + int w = 5; + int uh = 8; + int uw = 8; + int chans = 2; + for (bool align_corners : {true, false}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::upsample_bilinear2d(inputs[0], {uh, uw}, align_corners); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {batch_size, chans, h, w}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } +} + +TEST_F(LazyOpsTest, TestAddCMul) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor d = torch::addcmul(a, b, c, 3.1165); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::addcmul(lazy_a, lazy_b, lazy_c, 3.1165); + AllClose(d, lazy_d); + }); +} + +TEST_F(LazyOpsTest, TestAddCDiv) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = + torch::abs(torch::rand( + {2, 2}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice()))) + + 1.0; + torch::Tensor d = torch::addcdiv(a, b, c, 3.1165); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::addcdiv(lazy_a, lazy_b, lazy_c, 3.1165); + AllClose(d, lazy_d); + }); +} + +TEST_F(LazyOpsTest, TestAddCDivWithBroadcast) { + torch::Tensor a = torch::rand( + {1, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3, 1}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = + torch::abs(torch::rand( + {1, 3}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice()))) + + 1.0; + torch::Tensor d = torch::addcdiv(a, b, c, 3.1165); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::addcdiv(lazy_a, lazy_b, lazy_c, 3.1165); + AllClose(d, lazy_d); + }); +} + +TEST_F(LazyOpsTest, TestSize) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + for (int dim = -rank; dim < rank; ++dim) { + EXPECT_EQ(torch::size(input, dim), torch::size(lazy_input, dim)); + } + }); +} + +TEST_F(LazyOpsTest, TestSelect) { + std::vector input_sizes = {14, 24, 8}; + int rank = input_sizes.size(); + for (int dim = -rank; dim < rank; ++dim) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::select(inputs[0], dim, 0); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + input_sizes, + torch::TensorOptions(torch::kFloat).requires_grad(true))}, + device, + testfn); + }); + }; +} + +TEST_F(LazyOpsTest, TestBernoulliScalarProb) { + torch::Tensor input = torch::zeros( + 1000, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::bernoulli(lazy_input, 0.1); + double frac = lazy_output.sum().item().toDouble() / input.numel(); + EXPECT_GT(frac, 0.06); + EXPECT_LT(frac, 0.14); + }); +} + +TEST_F(LazyOpsTest, TestBernoulliTensorProb) { + std::vector prob_values(1000, 0.1); + torch::Tensor input = torch::tensor( + prob_values, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::bernoulli(lazy_input); + double frac = lazy_output.sum().item().toDouble() / input.numel(); + EXPECT_GT(frac, 0.06); + EXPECT_LT(frac, 0.14); + }); +} + +TEST_F(LazyOpsTest, TestBernoulliScalarProbInPlace) { + torch::Tensor input = torch::zeros( + 1000, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + lazy_input.bernoulli_(0.1); + double frac = lazy_input.sum().item().toDouble() / input.numel(); + EXPECT_GT(frac, 0.06); + EXPECT_LT(frac, 0.14); + }); +} + +TEST_F(LazyOpsTest, TestBernoulliTensorProbInPlace) { + torch::Tensor input = torch::zeros( + 1000, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor prob = torch::scalar_tensor( + 0.1, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_prob = CopyToDevice(prob, device); + lazy_input.bernoulli_(lazy_prob); + double frac = lazy_input.sum().item().toDouble() / input.numel(); + EXPECT_GT(frac, 0.06); + EXPECT_LT(frac, 0.14); + }); +} + +TEST_F(LazyOpsTest, TestDropout) { + torch::Tensor a = torch::rand( + {17, 21}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::dropout(lazy_a, 0.1, /*train=*/true); + double prob = + static_cast(lazy_b.cpu().ne(0.0f).sum().item().toDouble()) / + a.numel(); + EXPECT_GT(prob, 0.86); + EXPECT_LT(prob, 0.94); + }); +} + +TEST_F(LazyOpsTest, TestDropoutInPlace) { + torch::Tensor a = torch::rand( + {17, 21}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::dropout_(lazy_a, 0.1, /*train=*/true); + double prob = + static_cast(lazy_a.cpu().ne(0.0f).sum().item().toDouble()) / + a.numel(); + EXPECT_GT(prob, 0.85); + EXPECT_LT(prob, 0.94); + }); +} + +TEST_F(LazyOpsTest, TestRandperm) { + unsigned n = 5; + torch::Tensor shuffle = torch::randperm( + n, torch::TensorOptions(torch::kLong).device(torch::kLazy)); + torch::Tensor shuffle_cpu = CopyToDevice(shuffle, torch::kCPU); + std::vector shuffle_data( + shuffle_cpu.data_ptr(), shuffle_cpu.data_ptr() + n); + EXPECT_TRUE( + shuffle_data.size() == n && torch::lazy::IsPermutation(shuffle_data)); +} + +TEST_F(LazyOpsTest, TestSlice) { + torch::Tensor a = torch::rand( + {32, 24, 16}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::slice(a, 1, 0, 16, 1); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::slice(lazy_a, 1, 0, 16, 1); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestTake) { + torch::Tensor a = torch::rand( + {4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::randint( + 16, {5}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor c = torch::take(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::take(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestTakeBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::take(inputs[0], inputs[1]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {4, 4}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)), + torch::randint( + 16, + {5}, + torch::TensorOptions(torch::kLong).device(DefaultDevice()))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestStack) { + torch::Tensor a = torch::rand( + {2, 4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::rand( + {2, 4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = a.dim() + 1; + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor d = torch::stack({a, b, c}, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::stack({lazy_a, lazy_b, lazy_c}, dim); + AllClose(d, lazy_d); + }); + } +} + +TEST_F(LazyOpsTest, TestCat) { + torch::Tensor a = torch::rand( + {2, 1, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::rand( + {2, 3, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int dim : {1, -2}) { + torch::Tensor d = torch::cat({a, b, c}, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::cat({lazy_a, lazy_b, lazy_c}, dim); + EXPECT_TRUE(d.sizes() == lazy_d.sizes() && d.dtype() == lazy_d.dtype()); + AllClose(d, lazy_d); + }); + } +} + +TEST_F(LazyOpsTest, TestUnbind) { + torch::Tensor input = torch::rand( + {4, 3, 7}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + std::vector output = torch::unbind(input, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + std::vector lazy_output = torch::unbind(lazy_input, dim); + ASSERT_EQ(output.size(), lazy_output.size()); + for (size_t i = 0; i < output.size(); ++i) { + AllClose(output[i], lazy_output[i]); + } + }); + } +} + +TEST_F(LazyOpsTest, TestRepeat) { + std::vector> repeats_list = {{4, 2}, {4, 2, 3}}; + std::vector> input_size_list = {{3}, {2, 4}}; + for (const auto& repeats : repeats_list) { + for (const auto& input_size : input_size_list) { + torch::Tensor input = torch::rand( + input_size, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = input.repeat(repeats); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = lazy_input.repeat(repeats); + AllClose(output, lazy_output); + }); + } + } +} + +TEST_F(LazyOpsTest, TestGather) { + torch::Tensor a = torch::rand( + {3, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::empty( + {3, 3}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + b[i][j] = (i + j) % 3; + } + } + for (bool sparse_grad : {false, true}) { + torch::Tensor c = torch::gather(a, 1, b, sparse_grad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::gather(lazy_a, 1, lazy_b, sparse_grad); + AllClose(c, lazy_c); + }); + } +} + +TEST_F(LazyOpsTest, TestScatter) { + torch::Tensor a = torch::rand( + {3, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::empty( + {3, 5}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (int dim = 0; dim < 2; ++dim) { + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 5; j++) { + c[i][j] = (i + j) % c.sizes()[dim]; + } + } + torch::Tensor d = torch::scatter(a, dim, c, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::scatter(lazy_a, dim, lazy_c, lazy_b); + AllClose(d, lazy_d); + }); + } +} + +TEST_F(LazyOpsTest, TestScatterR1) { + torch::Tensor a = torch::rand( + {5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::empty( + {2}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + c[0] = 1; + c[1] = 3; + torch::Tensor d = torch::scatter(a, 0, c, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::scatter(lazy_a, 0, lazy_c, lazy_b); + AllClose(d, lazy_d); + }); +} + +TEST_F(LazyOpsTest, TestScatterR3) { + torch::Tensor a = torch::rand( + {3, 5, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3, 4, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::empty( + {3, 4, 2}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 4; j++) { + for (int k = 0; k < 2; k++) { + c[i][j][k] = (i + j + k) % 4; + } + } + } + torch::Tensor d = torch::scatter(a, 1, c, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::scatter(lazy_a, 1, lazy_c, lazy_b); + AllClose(d, lazy_d); + }); +} + +TEST_F(LazyOpsTest, TestScatterBiggerSource) { + torch::Tensor a = torch::rand( + {4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {8, 8}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::empty( + {4, 4}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + c[i][j] = (i + j) % 4; + } + } + for (int dim = 0; dim < 2; ++dim) { + torch::Tensor d = torch::scatter(a, dim, c, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::scatter(lazy_a, dim, lazy_c, lazy_b); + AllClose(d, lazy_d); + }); + } +} + +TEST_F(LazyOpsTest, TestScatterScalar) { + torch::Tensor a = torch::rand( + {4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar b = 1.0f; + torch::Tensor c = torch::empty( + {4, 4}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + c[i][j] = (i + j) % 4; + } + } + for (int dim = 0; dim < 2; ++dim) { + torch::Tensor d = torch::scatter(a, dim, c, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::scatter(lazy_a, dim, lazy_c, b); + AllClose(d, lazy_d); + }); + } +} + +TEST_F(LazyOpsTest, TestScatterReduceAdd) { + torch::Tensor a = torch::rand( + {3, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::empty( + {3, 5}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (int dim = 0; dim < 2; ++dim) { + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 5; j++) { + c[i][j] = (i + j) % c.sizes()[dim]; + } + } + torch::Tensor d = torch::scatter(a, dim, c, b, "add"); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::scatter(lazy_a, dim, lazy_c, lazy_b, "add"); + AllClose(d, lazy_d); + }); + } + + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("lazy::scatter_out", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestScatterAdd) { + torch::Tensor a = torch::rand( + {3, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::empty( + {3, 5}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (int dim = 0; dim < 2; ++dim) { + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 5; j++) { + c[i][j] = (i + j) % c.sizes()[dim]; + } + } + torch::Tensor d = torch::scatter_add(a, dim, c, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::scatter_add(lazy_a, dim, lazy_c, lazy_b); + AllClose(d, lazy_d); + }); + } +} + +TEST_F(LazyOpsTest, TestScatterAddInPlace) { + torch::Tensor b = torch::rand( + {4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::empty( + {4, 4}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + c[i][j] = (i + j) % 4; + } + } + for (int dim = 0; dim < 2; ++dim) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = torch::rand( + {4, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor d = a.scatter_add_(dim, c, b); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = lazy_a.scatter_add_(dim, lazy_c, lazy_b); + AllClose(d, lazy_d); + AllClose(a, lazy_a); + }); + } +} + +TEST_F(LazyOpsTest, TestIndexSelect) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor a = isFloatingType(scalar_type) + ? torch::rand( + {3, 4}, torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {3, 4}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (torch::ScalarType index_scalar_type : {torch::kInt, torch::kLong}) { + torch::Tensor b = torch::empty( + {2}, torch::TensorOptions(index_scalar_type).device(DefaultDevice())); + b[0] = 0; + b[1] = 2; + for (auto offset : {-2, 0}) { + torch::Tensor c0 = torch::index_select(a, 0 + offset, b); + torch::Tensor c1 = torch::index_select(a, 1 + offset, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c0 = + torch::index_select(lazy_a, 0 + offset, lazy_b); + torch::Tensor lazy_c1 = + torch::index_select(lazy_a, 1 + offset, lazy_b); + AllEqual(c0, lazy_c0); + AllEqual(c1, lazy_c1); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestIndexSelectRank0) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor a = isFloatingType(scalar_type) + ? torch::rand( + {3, 4}, torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {3, 4}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor b = torch::scalar_tensor( + 2, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor c0 = torch::index_select(a, 0, b); + torch::Tensor c1 = torch::index_select(a, 1, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c0 = torch::index_select(lazy_a, 0, lazy_b); + torch::Tensor lazy_c1 = torch::index_select(lazy_a, 1, lazy_b); + AllEqual(c0, lazy_c0); + AllEqual(c1, lazy_c1); + }); + } +} + +TEST_F(LazyOpsTest, TestInverse) { + if (IsCuda()) { + // TODO(whc) debug failure on cuda, lazy_b comes back transposed + GTEST_SKIP(); + } + torch::Tensor a = torch::randn( + {5, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::inverse(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::inverse(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-4); + }); +} + +TEST_F(LazyOpsTest, TestIsnan) { + torch::Tensor a = torch::tensor( + {1.0, 2.0, std::nan("1"), 4.0}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::isnan(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::isnan(lazy_a); + AllEqual(b, lazy_b); + }); + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("lazy::isnan", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestExpand) { + torch::Tensor a = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = a.expand({2, 3, 4}, /*implicit=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = lazy_a.expand({2, 3, 4}, /*implicit=*/false); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestExpandBack) { + torch::Tensor a = torch::rand( + {3, 1}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = a.expand({3, 4}, /*implicit=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = lazy_a.expand({3, 4}, /*implicit=*/false); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestExpandAs) { + torch::Tensor a = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::native::expand_as(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::native::expand_as(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestEye) { + int n = 5; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor out = torch::eye( + n, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_out = + torch::eye(n, torch::TensorOptions(torch::kFloat).device(device)); + AllClose(out, lazy_out); + }); +} + +TEST_F(LazyOpsTest, TestEyeWide) { + int lines = 3; + int cols = 5; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor out = torch::eye( + lines, + cols, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_out = torch::eye( + lines, cols, torch::TensorOptions(torch::kFloat).device(device)); + AllClose(out, lazy_out); + }); +} + +TEST_F(LazyOpsTest, TestEyeNarrow) { + int lines = 5; + int cols = 3; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor out = torch::eye( + lines, + cols, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_out = torch::eye( + lines, cols, torch::TensorOptions(torch::kFloat).device(device)); + AllClose(out, lazy_out); + }); +} + +TEST_F(LazyOpsTest, TestBroadcastTensors) { + torch::Tensor a = torch::rand( + {2, 1, 1}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2, 1}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector c = torch::broadcast_tensors({a, b}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + std::vector lazy_c = + torch::broadcast_tensors({lazy_a, lazy_b}); + ASSERT_EQ(c.size(), lazy_c.size()); + for (size_t i = 0; i < c.size(); ++i) { + AllClose(c[i], lazy_c[i]); + } + }); +} + +TEST_F(LazyOpsTest, TestOneIndex) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor indices = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor result = torch::index(params, {indices}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices = CopyToDevice(indices, device); + torch::Tensor lazy_result = torch::index(lazy_params, {lazy_indices}); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestOneIndexTransfer) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor indices = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor result = torch::index(params, {indices}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_result = torch::index(lazy_params, {indices.cpu()}); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestNonzero) { + torch::Tensor a = torch::zeros( + {4, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + a[0][1] = 1.0; + a[1][0] = 2.0; + a[3][1] = 3.0; + torch::Tensor b = torch::nonzero(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::nonzero(lazy_a); + AllClose(b, lazy_b); + + if (DebugUtil::ExperimentEnabled("nonzero")) { + // If the nonzero support is enabled, we must not see any aten:: calls. + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + } + ResetCounters(); + }); +} + +TEST_F(LazyOpsTest, TestMaskedSelect) { + torch::Tensor a = torch::rand( + {3, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::randint( + 0, 2, {5}, torch::TensorOptions(torch::kBool).device(DefaultDevice())); + torch::Tensor c = torch::masked_select(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::masked_select(lazy_a, lazy_b); + AllClose(c, lazy_c); + + if (DebugUtil::ExperimentEnabled("masked_select")) { + // If the masked_select support is enabled, we must not see any aten:: + // calls. + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + } + ResetCounters(); + }); +} + +TEST_F(LazyOpsTest, TestMaskedScatter) { + torch::Tensor a = torch::rand( + {3, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::randint( + 0, 2, {3, 5}, torch::TensorOptions(torch::kBool).device(DefaultDevice())); + torch::Tensor c = torch::rand( + {15}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor d = torch::masked_scatter(a, b, c); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::masked_scatter(lazy_a, lazy_b, lazy_c); + AllClose(d, lazy_d); + + if (DebugUtil::ExperimentEnabled("masked_scatter")) { + // If the masked_select support is enabled, we must not see any aten:: + // calls. + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + } + ResetCounters(); + }); +} + +TEST_F(LazyOpsTest, TestMultiIndexHeadNull) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor indices_null; + torch::Tensor indices_0 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_1 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor result = + torch::index(params, {indices_null, indices_0, indices_1}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices_0 = CopyToDevice(indices_0, device); + torch::Tensor lazy_indices_1 = CopyToDevice(indices_1, device); + torch::Tensor lazy_result = torch::index( + lazy_params, {indices_null, lazy_indices_0, lazy_indices_1}); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestMultiIndexMiddleNull) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor indices_0 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_null; + torch::Tensor indices_1 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor result = + torch::index(params, {indices_0, indices_null, indices_1}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices_0 = CopyToDevice(indices_0, device); + torch::Tensor lazy_indices_1 = CopyToDevice(indices_1, device); + torch::Tensor lazy_result = torch::index( + lazy_params, {lazy_indices_0, indices_null, lazy_indices_1}); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestMultiIndexTailNull) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor indices_0 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_null; + torch::Tensor indices_1 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor result = + torch::index(params, {indices_0, indices_1, indices_null}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices_0 = CopyToDevice(indices_0, device); + torch::Tensor lazy_indices_1 = CopyToDevice(indices_1, device); + torch::Tensor lazy_result = torch::index( + lazy_params, {lazy_indices_0, lazy_indices_1, indices_null}); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestMultiIndexMiddleBroadcast) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor indices_0 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_1 = torch::randint( + -3, + 3, + {2, 1, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor result = torch::index(params, {indices_0, indices_1}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices_0 = CopyToDevice(indices_0, device); + torch::Tensor lazy_indices_1 = CopyToDevice(indices_1, device); + torch::Tensor lazy_result = + torch::index(lazy_params, {lazy_indices_0, lazy_indices_1}); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestMultiIndexTailBroadcast) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor indices_0 = torch::randint( + -3, + 3, + {2, 1, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_1 = torch::randint( + -3, + 3, + {2, 1}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor result = torch::index(params, {indices_0, indices_1}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices_0 = CopyToDevice(indices_0, device); + torch::Tensor lazy_indices_1 = CopyToDevice(indices_1, device); + torch::Tensor lazy_result = + torch::index(lazy_params, {lazy_indices_0, lazy_indices_1}); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestMaskIndex) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {2, 2}, torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {2, 2}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor indices = torch::randint( + 0, + 2, + {2, 2}, + torch::TensorOptions(torch::kBool).device(DefaultDevice())); + torch::Tensor result = torch::index(params, {indices}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices = CopyToDevice(indices, device); + torch::Tensor lazy_result = torch::index(lazy_params, {lazy_indices}); + AllEqual(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestOneIndexPut) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor indices = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor values = isFloatingType(scalar_type) + ? torch::rand( + {3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool accumulate : {false, true}) { + if (accumulate && IsCuda()) { + GTEST_SKIP(); + } + torch::Tensor result = + torch::index_put(params, {indices}, values, accumulate); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices = CopyToDevice(indices, device); + torch::Tensor lazy_values = CopyToDevice(values, device); + torch::Tensor lazy_result = torch::index_put( + lazy_params, {lazy_indices}, lazy_values, accumulate); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestOneIndexPutInPlace) { + torch::Tensor indices = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor values = torch::ones( + {3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool accumulate : {false, true}) { + if (accumulate && IsCuda()) { + GTEST_SKIP(); + } + ForEachDevice([&](const torch::Device& device) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor lazy_params = CopyToDevice(params.clone(), device); + torch::Tensor result = + torch::index_put_(params, {indices}, values, accumulate); + torch::Tensor lazy_indices = CopyToDevice(indices, device); + torch::Tensor lazy_values = CopyToDevice(values, device); + torch::Tensor lazy_result = torch::index_put_( + lazy_params, {lazy_indices}, lazy_values, accumulate); + AllEqual(result, lazy_result); + AllEqual(params, lazy_params); + }); + } + } +} + +TEST_F(LazyOpsTest, TestOneIndexPutTransfer) { + torch::Tensor indices = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor values = torch::ones( + {3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool accumulate : {false, true}) { + if (accumulate && IsCuda()) { + GTEST_SKIP(); + } + torch::Tensor result = + torch::index_put(params, {indices}, values, accumulate); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_values = CopyToDevice(values, device); + torch::Tensor lazy_result = + torch::index_put(lazy_params, {indices}, lazy_values, accumulate); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestMultiIndexPut) { + torch::Tensor indices_0 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_1 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor values = torch::ones( + {5, 6, 7}, torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool accumulate : {false, true}) { + if (accumulate && IsCuda()) { + GTEST_SKIP(); + } + torch::Tensor result = + torch::index_put(params, {indices_0, indices_1}, values, accumulate); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices_0 = CopyToDevice(indices_0, device); + torch::Tensor lazy_indices_1 = CopyToDevice(indices_1, device); + torch::Tensor lazy_values = CopyToDevice(values, device); + torch::Tensor lazy_result = torch::index_put( + lazy_params, + {lazy_indices_0, lazy_indices_1}, + lazy_values, + accumulate); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestMultiIndexPutHeadNull) { + torch::Tensor indices_0 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_null; + torch::Tensor indices_1 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 3, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 3, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor values = torch::ones( + {3, 6, 7}, torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool accumulate : {false, true}) { + if (accumulate && IsCuda()) { + GTEST_SKIP(); + } + torch::Tensor result = torch::index_put( + params, {indices_null, indices_0, indices_1}, values, accumulate); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices_0 = CopyToDevice(indices_0, device); + torch::Tensor lazy_indices_1 = CopyToDevice(indices_1, device); + torch::Tensor lazy_values = CopyToDevice(values, device); + torch::Tensor lazy_result = torch::index_put( + lazy_params, + {indices_null, lazy_indices_0, lazy_indices_1}, + lazy_values, + accumulate); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestMultiIndexPutMiddleNull) { + torch::Tensor indices_0 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_null; + torch::Tensor indices_1 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 3, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 3, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor values = torch::ones( + {3, 6, 7}, torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool accumulate : {false, true}) { + if (accumulate && IsCuda()) { + GTEST_SKIP(); + } + torch::Tensor result = torch::index_put( + params, {indices_0, indices_null, indices_1}, values, accumulate); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices_0 = CopyToDevice(indices_0, device); + torch::Tensor lazy_indices_1 = CopyToDevice(indices_1, device); + torch::Tensor lazy_values = CopyToDevice(values, device); + torch::Tensor lazy_result = torch::index_put( + lazy_params, + {lazy_indices_0, indices_null, lazy_indices_1}, + lazy_values, + accumulate); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestMultiIndexPutTailNull) { + torch::Tensor indices_0 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_1 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_null; + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 3, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 3, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor values = torch::ones( + {3, 6, 7}, torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool accumulate : {false, true}) { + if (accumulate && IsCuda()) { + GTEST_SKIP(); + } + torch::Tensor result = torch::index_put( + params, {indices_0, indices_1, indices_null}, values, accumulate); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices_0 = CopyToDevice(indices_0, device); + torch::Tensor lazy_indices_1 = CopyToDevice(indices_1, device); + torch::Tensor lazy_values = CopyToDevice(values, device); + torch::Tensor lazy_result = torch::index_put( + lazy_params, + {lazy_indices_0, lazy_indices_1, indices_null}, + lazy_values, + accumulate); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestMultiIndexPutMiddleBroadcast) { + torch::Tensor indices_0 = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_1 = torch::randint( + -3, + 3, + {2, 1, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor values = torch::ones( + {5, 6, 7}, torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool accumulate : {false, true}) { + if (accumulate && IsCuda()) { + GTEST_SKIP(); + } + torch::Tensor result = + torch::index_put(params, {indices_0, indices_1}, values, accumulate); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices_0 = CopyToDevice(indices_0, device); + torch::Tensor lazy_indices_1 = CopyToDevice(indices_1, device); + torch::Tensor lazy_values = CopyToDevice(values, device); + torch::Tensor lazy_result = torch::index_put( + lazy_params, + {lazy_indices_0, lazy_indices_1}, + lazy_values, + accumulate); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestMultiIndexPutTailBroadcast) { + torch::Tensor indices_0 = torch::randint( + -3, + 3, + {2, 1, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor indices_1 = torch::randint( + -3, + 3, + {2, 1}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor values = torch::ones( + {5, 6, 7}, torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool accumulate : {false, true}) { + if (accumulate && IsCuda()) { + GTEST_SKIP(); + } + torch::Tensor result = + torch::index_put(params, {indices_0, indices_1}, values, accumulate); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices_0 = CopyToDevice(indices_0, device); + torch::Tensor lazy_indices_1 = CopyToDevice(indices_1, device); + torch::Tensor lazy_values = CopyToDevice(values, device); + torch::Tensor lazy_result = torch::index_put( + lazy_params, + {lazy_indices_0, lazy_indices_1}, + lazy_values, + accumulate); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestMaskIndexPut) { + torch::Tensor indices = + torch::tensor( + {0, 1}, torch::TensorOptions(torch::kByte).device(DefaultDevice())) + .to(torch::kBool); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {2, 2}, torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {2, 2}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor values = torch::ones( + {2}, torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool accumulate : {false, true}) { + torch::Tensor result = + torch::index_put(params, {indices}, values, accumulate); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_params = CopyToDevice(params, device); + torch::Tensor lazy_indices = CopyToDevice(indices, device); + torch::Tensor lazy_values = CopyToDevice(values, device); + torch::Tensor lazy_result = torch::index_put( + lazy_params, {lazy_indices}, lazy_values, accumulate); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestIndexPutImpl) { + torch::Tensor indices = torch::randint( + -3, + 3, + {2, 4, 3}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor values = torch::ones( + {3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + for (bool accumulate : {false, true}) { + if (accumulate && IsCuda()) { + GTEST_SKIP(); + } + ForEachDevice([&](const torch::Device& device) { + torch::Tensor params = isFloatingType(scalar_type) + ? torch::rand( + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {4, 3, 5, 6, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor lazy_params = CopyToDevice(params.clone(), device); + torch::Tensor result = torch::_index_put_impl_( + params, {indices}, values, accumulate, /*unsafe=*/true); + torch::Tensor lazy_indices = CopyToDevice(indices, device); + torch::Tensor lazy_values = CopyToDevice(values, device); + torch::Tensor lazy_result = torch::_index_put_impl_( + lazy_params, + {lazy_indices}, + lazy_values, + accumulate, + /*unsafe=*/true); + AllEqual(result, lazy_result); + AllEqual(params, lazy_params); + }); + } + } +} + +TEST_F(LazyOpsTest, TestIndexFillWithScalar) { + torch::Tensor index = torch::tensor( + {0, 2}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Scalar value = 42; + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor base = isFloatingType(scalar_type) + ? torch::rand( + {3, 4, 5}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {3, 4, 5}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + int rank = base.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::index_fill(base, dim, index, value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base, device); + torch::Tensor lazy_index = CopyToDevice(index, device); + torch::Tensor lazy_result = + torch::index_fill(lazy_base, dim, lazy_index, value); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestIndexFillWithScalarInPlace) { + torch::Tensor index = torch::tensor( + {0, 2}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Scalar value = 42; + int rank = 3; + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + for (int dim = -rank; dim < rank; ++dim) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor base = isFloatingType(scalar_type) + ? torch::rand( + {3, 4, 5}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {3, 4, 5}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor lazy_base = CopyToDevice(base.clone(), device); + torch::Tensor result = base.index_fill_(dim, index, value); + torch::Tensor lazy_index = CopyToDevice(index, device); + torch::Tensor lazy_result = + lazy_base.index_fill_(dim, lazy_index, value); + AllEqual(result, lazy_result); + AllEqual(base, lazy_base); + }); + } + } +} + +TEST_F(LazyOpsTest, TestIndexFillWithTensor) { + torch::Tensor index = torch::tensor( + {0, 2}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor base = isFloatingType(scalar_type) + ? torch::rand( + {3, 4, 5}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {3, 4, 5}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor value = torch::scalar_tensor( + 42, torch::TensorOptions(scalar_type).device(DefaultDevice())); + int rank = base.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::index_fill(base, dim, index, value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base, device); + torch::Tensor lazy_index = CopyToDevice(index, device); + torch::Tensor lazy_value = CopyToDevice(value, device); + torch::Tensor lazy_result = + torch::index_fill(lazy_base, dim, lazy_index, lazy_value); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestIndexFillWithTensorInPlace) { + torch::Tensor index = torch::tensor( + {0, 2}, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor value = torch::scalar_tensor( + 42, torch::TensorOptions(scalar_type).device(DefaultDevice())); + int rank = 3; + for (int dim = -rank; dim < rank; ++dim) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor base = isFloatingType(scalar_type) + ? torch::rand( + {3, 4, 5}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {3, 4, 5}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor lazy_base = CopyToDevice(base.clone(), device); + torch::Tensor result = base.index_fill_(dim, index, value); + torch::Tensor lazy_index = CopyToDevice(index, device); + torch::Tensor lazy_value = CopyToDevice(value, device); + torch::Tensor lazy_result = + lazy_base.index_fill_(dim, lazy_index, lazy_value); + AllEqual(result, lazy_result); + AllEqual(base, lazy_base); + }); + } + } +} + +TEST_F(LazyOpsTest, TestIndexFillRank0) { + torch::Tensor index = torch::scalar_tensor( + 2, torch::TensorOptions(torch::kLong).device(DefaultDevice())); + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor base = isFloatingType(scalar_type) + ? torch::rand( + {3, 4, 5}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {3, 4, 5}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor value = torch::scalar_tensor( + 42, torch::TensorOptions(scalar_type).device(DefaultDevice())); + int rank = base.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::index_fill(base, dim, index, value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base, device); + torch::Tensor lazy_index = CopyToDevice(index, device); + torch::Tensor lazy_value = CopyToDevice(value, device); + torch::Tensor lazy_result = + torch::index_fill(lazy_base, dim, lazy_index, lazy_value); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestIndexAdd) { + int index_size = 10; + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor base = isFloatingType(scalar_type) + ? torch::rand( + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + int rank = base.dim(); + for (int dim = -rank; dim < rank; ++dim) { + for (torch::ScalarType index_scalar_type : {torch::kInt, torch::kLong}) { + torch::Tensor index = torch::randint( + 0, + base.size(dim), + {index_size}, + torch::TensorOptions(index_scalar_type).device(DefaultDevice())); + std::vector value_sizes( + base.sizes().begin(), base.sizes().end()); + int canonical_dim = dim < 0 ? dim + rank : dim; + value_sizes[canonical_dim] = index_size; + torch::Tensor value = isFloatingType(scalar_type) + ? torch::rand( + value_sizes, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + value_sizes, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor result = torch::index_add(base, dim, index, value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base, device); + torch::Tensor lazy_index = CopyToDevice(index, device); + torch::Tensor lazy_value = CopyToDevice(value, device); + torch::Tensor lazy_result = + torch::index_add(lazy_base, dim, lazy_index, lazy_value); + AllClose(result, lazy_result); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestIndexAddInPlace) { + int index_size = 10; + int rank = 3; + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + for (int dim = -rank; dim < rank; ++dim) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor base = isFloatingType(scalar_type) + ? torch::rand( + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor index = torch::randint( + 0, + base.size(dim), + {index_size}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + std::vector value_sizes( + base.sizes().begin(), base.sizes().end()); + int canonical_dim = dim < 0 ? dim + rank : dim; + value_sizes[canonical_dim] = index_size; + torch::Tensor value = isFloatingType(scalar_type) + ? torch::rand( + value_sizes, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + value_sizes, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor lazy_base = CopyToDevice(base.clone(), device); + torch::Tensor result = base.index_add_(dim, index, value); + torch::Tensor lazy_index = CopyToDevice(index, device); + torch::Tensor lazy_value = CopyToDevice(value, device); + torch::Tensor lazy_result = + lazy_base.index_add_(dim, lazy_index, lazy_value); + AllClose(result, lazy_result); + AllClose(base, lazy_base); + }); + } + } +} + +TEST_F(LazyOpsTest, TestIndexAddRank0) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor base = isFloatingType(scalar_type) + ? torch::rand( + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + int rank = base.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor index = torch::randint( + 0, + base.size(dim), + at::IntArrayRef{}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + std::vector value_sizes( + base.sizes().begin(), base.sizes().end()); + int canonical_dim = dim < 0 ? dim + rank : dim; + value_sizes[canonical_dim] = 1; + torch::Tensor value = isFloatingType(scalar_type) + ? torch::rand( + value_sizes, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + value_sizes, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor result = torch::index_add(base, dim, index, value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base, device); + torch::Tensor lazy_index = CopyToDevice(index, device); + torch::Tensor lazy_value = CopyToDevice(value, device); + torch::Tensor lazy_result = + torch::index_add(lazy_base, dim, lazy_index, lazy_value); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestIndexCopy) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor base = isFloatingType(scalar_type) + ? torch::rand( + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + int rank = base.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor index = torch::randperm( + base.size(dim), + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor value = isFloatingType(scalar_type) + ? torch::rand( + base.sizes(), + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + base.sizes(), + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor result = torch::index_copy(base, dim, index, value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base, device); + torch::Tensor lazy_index = CopyToDevice(index, device); + torch::Tensor lazy_value = CopyToDevice(value, device); + torch::Tensor lazy_result = + torch::index_copy(lazy_base, dim, lazy_index, lazy_value); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestIndexCopyInPlace) { + if (IsCuda()) { + GTEST_SKIP(); + } + int index_size = 10; + int rank = 3; + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + for (int dim = -rank; dim < rank; ++dim) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor base = isFloatingType(scalar_type) + ? torch::rand( + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor index = torch::randint( + 0, + base.size(dim), + {index_size}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + std::vector value_sizes( + base.sizes().begin(), base.sizes().end()); + int canonical_dim = dim < 0 ? dim + rank : dim; + value_sizes[canonical_dim] = index_size; + torch::Tensor value = isFloatingType(scalar_type) + ? torch::rand( + value_sizes, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + value_sizes, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor lazy_base = CopyToDevice(base.clone(), device); + torch::Tensor result = base.index_copy_(dim, index, value); + torch::Tensor lazy_index = CopyToDevice(index, device); + torch::Tensor lazy_value = CopyToDevice(value, device); + torch::Tensor lazy_result = + lazy_base.index_copy_(dim, lazy_index, lazy_value); + AllEqual(result, lazy_result); + AllEqual(base, lazy_base); + }); + } + } +} + +TEST_F(LazyOpsTest, TestIndexCopyRank0) { + for (torch::ScalarType scalar_type : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor base = isFloatingType(scalar_type) + ? torch::rand( + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + {5, 3, 7}, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + int rank = base.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor index = torch::randint( + 0, + base.size(dim), + at::IntArrayRef{}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + std::vector value_sizes( + base.sizes().begin(), base.sizes().end()); + int canonical_dim = dim < 0 ? dim + rank : dim; + value_sizes[canonical_dim] = 1; + torch::Tensor value = isFloatingType(scalar_type) + ? torch::rand( + value_sizes, + torch::TensorOptions(scalar_type).device(DefaultDevice())) + : torch::randint( + 100, + value_sizes, + torch::TensorOptions(scalar_type).device(DefaultDevice())); + torch::Tensor result = torch::index_copy(base, dim, index, value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base, device); + torch::Tensor lazy_index = CopyToDevice(index, device); + torch::Tensor lazy_value = CopyToDevice(value, device); + torch::Tensor lazy_result = + torch::index_copy(lazy_base, dim, lazy_index, lazy_value); + AllEqual(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestRelu) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::relu(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::relu(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestReluInPlace) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = torch::relu_(input); + torch::Tensor lazy_output = torch::relu_(lazy_input); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestHardshrink) { + torch::Tensor input = torch::randn( + {10}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::hardshrink(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::hardshrink(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestHardSigmoid) { + torch::Tensor input = torch::randn( + {10}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::hardsigmoid(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::hardsigmoid(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestHardSigmoidInPlace) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::randn( + {10}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = torch::hardsigmoid_(input); + torch::Tensor lazy_output = torch::hardsigmoid_(lazy_input); + AllClose(input, lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestHardSigmoidBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::hardsigmoid(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::randn( + {10}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestSoftshrink) { + torch::Tensor input = torch::randn( + {10}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::softshrink(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::softshrink(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestHardtanh) { + torch::Tensor input = torch::randn( + {10}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::hardtanh(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::hardtanh(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestHardtanhInPlace) { + torch::Tensor input = torch::randn( + {10}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = torch::hardtanh_(input); + torch::Tensor lazy_output = torch::hardtanh_(lazy_input); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestLeakyRelu) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + double negative_slope = 0.01; + torch::Tensor output = torch::leaky_relu(input, negative_slope); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::leaky_relu(lazy_input, negative_slope); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestLeakyReluInPlace) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + double negative_slope = 0.01; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = torch::leaky_relu_(input, negative_slope); + torch::Tensor lazy_output = torch::leaky_relu_(lazy_input, negative_slope); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestExp) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::exp(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::exp(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestExpm1) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::expm1(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::expm1(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestLog) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::log(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::log(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestLog2) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::log2(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::log2(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestLog10) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::log10(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::log10(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestLog1p) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::log1p(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::log1p(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestErf) { + torch::Tensor a = torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::erf(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::erf(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestErfc) { + torch::Tensor a = torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::erfc(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::erfc(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestErfinv) { + torch::Tensor a = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::erfinv(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::erfinv(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestSqrt) { + torch::Tensor a = torch::abs(torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice()))); + torch::Tensor b = torch::sqrt(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::sqrt(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestRsqrt) { + torch::Tensor a = torch::abs(torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice()))); + torch::Tensor b = torch::rsqrt(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::rsqrt(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestReciprocal) { + torch::Tensor a = torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::reciprocal(a); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::reciprocal(lazy_a); + AllClose(b, lazy_b, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestPowTensorScalar) { + torch::Tensor base = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar exponent = 4.09; + torch::Tensor result = torch::pow(base, exponent); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base, device); + torch::Tensor lazy_result = torch::pow(lazy_base, exponent); + AllClose(result, lazy_result, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestPowTensorScalarInPlace) { + torch::Tensor base = torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar exponent = 4.09; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base.clone(), device); + torch::Tensor result = base.pow_(exponent); + torch::Tensor lazy_result = lazy_base.pow_(exponent); + AllClose(result, lazy_result, /*rtol=*/1e-3, /*atol=*/1e-5); + AllClose(base, lazy_base, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestPowTensorTensor) { + torch::Tensor base = torch::abs(torch::rand( + {4, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice()))); + torch::Tensor exponent = torch::rand( + {4, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor result = torch::pow(base, exponent); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base, device); + torch::Tensor lazy_exponent = CopyToDevice(exponent, device); + torch::Tensor lazy_result = torch::pow(lazy_base, lazy_exponent); + AllClose(result, lazy_result, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestPowTensorTensorInPlace) { + torch::Tensor base = torch::abs(torch::rand( + {4, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice()))); + torch::Tensor exponent = torch::rand( + {4, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base.clone(), device); + torch::Tensor result = base.pow_(exponent); + torch::Tensor lazy_exponent = CopyToDevice(exponent, device); + torch::Tensor lazy_result = lazy_base.pow_(lazy_exponent); + AllClose(result, lazy_result, /*rtol=*/1e-3, /*atol=*/1e-5); + AllClose(base, lazy_base, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestPowTensorTensorBroadcast) { + torch::Tensor base = torch::abs(torch::rand( + {4, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice()))); + torch::Tensor exponent = torch::rand( + {4, 1}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor result = torch::pow(base, exponent); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base, device); + torch::Tensor lazy_exponent = CopyToDevice(exponent, device); + torch::Tensor lazy_result = torch::pow(lazy_base, lazy_exponent); + AllClose(result, lazy_result, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestPowScalarTensor) { + torch::Scalar base = 3.5; + torch::Tensor exponent = torch::rand({4, 2}); + torch::Tensor result = torch::pow(base, exponent); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_exponent = CopyToDevice(exponent, device); + torch::Tensor lazy_result = torch::pow(base, lazy_exponent); + AllClose(result, lazy_result, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestPowIntExponent) { + torch::Tensor base = torch::abs(torch::rand( + {4, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice()))); + torch::Scalar exponent = 3; + torch::Tensor result = torch::pow(base, exponent); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_base = CopyToDevice(base, device); + torch::Tensor lazy_result = torch::pow(lazy_base, exponent); + AllClose(result, lazy_result, /*rtol=*/1e-3, /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestFmodScalar) { + torch::Tensor a = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Scalar divisor = 2.0; + torch::Tensor b = torch::fmod(a, divisor); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::fmod(lazy_a, divisor); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestFmodScalarInPlace) { + torch::Scalar divisor = 2.0; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = + torch::rand( + {2, 2}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor b = a.fmod_(divisor); + torch::Tensor lazy_b = lazy_a.fmod_(divisor); + AllClose(b, lazy_b); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestFmodTensor) { + torch::Tensor a = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 10.0; + torch::Tensor c = torch::fmod(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::fmod(lazy_a, lazy_b); + AllClose(c, lazy_c); + }); +} + +TEST_F(LazyOpsTest, TestFmodTensorInPlace) { + torch::Tensor b = + torch::rand( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 10.0; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = + torch::rand( + {2, 2}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor c = a.fmod_(b); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = lazy_a.fmod_(lazy_b); + AllClose(c, lazy_c); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestRemainderScalar) { + torch::Tensor a = + torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Scalar divisor = -2.0; + torch::Tensor b = torch::remainder(a, divisor); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = torch::remainder(lazy_a, divisor); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestRemainderScalarInPlace) { + torch::Scalar divisor = -2.0; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = + torch::randn( + {2, 2}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor b = a.remainder_(divisor); + torch::Tensor lazy_b = lazy_a.remainder_(divisor); + AllClose(b, lazy_b); + AllClose(a, lazy_a); + }); +} + +TEST_F(LazyOpsTest, TestRemainderTensor) { + torch::Tensor a = + torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor b = + torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 10.0; + torch::Tensor c = torch::remainder(a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = torch::remainder(lazy_a, lazy_b); + AllClose(c, lazy_c, /*rtol=*/1e-4, /*atol=*/1e-6); + }); +} + +TEST_F(LazyOpsTest, TestRemainderTensorInPlace) { + torch::Tensor b = + torch::randn( + {2, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 10.0; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor a = + torch::randn( + {2, 2}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())) * + 100.0; + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor c = a.remainder_(b); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = lazy_a.remainder_(lazy_b); + AllClose(c, lazy_c, /*rtol=*/1e-4, /*atol=*/1e-6); + AllClose(a, lazy_a, /*rtol=*/1e-4, /*atol=*/1e-6); + }); +} + +TEST_F(LazyOpsTest, TestWhere) { + torch::Tensor a = torch::rand( + {3, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {3, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::empty( + {3, 3}, torch::TensorOptions(torch::kByte).device(DefaultDevice())); + for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 3; ++j) { + c[i][j] = i == j; + } + } + torch::Tensor d = torch::where(c, a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::where(lazy_c, lazy_a, lazy_b); + AllClose(d, lazy_d); + }); +} + +TEST_F(LazyOpsTest, TestWhereBroadcast) { + torch::Tensor a = torch::rand( + {3, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::zeros( + {}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::empty( + {3, 3}, torch::TensorOptions(torch::kByte).device(DefaultDevice())); + for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 3; ++j) { + c[i][j] = i == j; + } + } + torch::Tensor d = torch::where(c, a, b); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = torch::where(lazy_c, lazy_a, lazy_b); + AllClose(d, lazy_d); + }); +} + +TEST_F(LazyOpsTest, TestThreshold) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + float threshold = 0.4; + float value = 20; + torch::Tensor output = torch::threshold(input, threshold, value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::threshold(lazy_input, threshold, value); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestThresholdBackward) { + float threshold = 0.4; + float value = 20; + + auto testFunction = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::threshold(inputs[0], threshold, value); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testFunction); + }); +} + +TEST_F(LazyOpsTest, TestThresholdInPlace) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = input.clone(); + float threshold = 0.4; + float value = 20; + torch::threshold_(output, threshold, value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_output = CopyToDevice(input, device); + torch::threshold_(lazy_output, threshold, value); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestElu) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar alpha = 0.5; + torch::Scalar scale = 2.5; + torch::Scalar input_scale = 1.5; + torch::Tensor output = torch::elu(input, alpha, scale, input_scale); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = + torch::elu(lazy_input, alpha, scale, input_scale); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestEluInPlace) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar alpha = 0.5; + torch::Scalar scale = 2.5; + torch::Scalar input_scale = 1.5; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = torch::elu_(input, alpha, scale, input_scale); + torch::Tensor lazy_output = + torch::elu_(lazy_input, alpha, scale, input_scale); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestSelu) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::selu(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::selu(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestSeluInPlace) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = torch::selu_(input); + torch::Tensor lazy_output = torch::selu_(lazy_input); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestCelu) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar alpha = 2.5; + torch::Tensor output = torch::celu(input, alpha); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::celu(lazy_input, alpha); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestCeluInPlace) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar alpha = 2.5; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = torch::celu_(input, alpha); + torch::Tensor lazy_output = torch::celu_(lazy_input, alpha); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestGelu) { + torch::Tensor input = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::gelu(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::gelu(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestAddMatMul) { + int in_channels = 32; + int out_channels = 320; + int labels = 50; + torch::Tensor input = torch::rand( + {in_channels, out_channels}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {out_channels, labels}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor bias = torch::rand( + {labels}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test beta != 1. through the CPU interop. + for (double beta : {1., 2.}) { + torch::Tensor output = torch::addmm(bias, input, weight, /*beta=*/beta); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_weight = CopyToDevice(weight, device); + torch::Tensor lazy_bias = CopyToDevice(bias, device); + torch::Tensor lazy_output = + torch::addmm(lazy_bias, lazy_input, lazy_weight, /*beta=*/beta); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestEmbedding) { + torch::Tensor a = torch::rand( + {32, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor i = torch::randint( + 0, + 31, + {3, 4}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor b = torch::embedding( + a, + i, + /*padding_idx=*/0, + /*scale_grad_by_freq=*/false, + /*sparse=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_i = CopyToDevice(i, device); + torch::Tensor lazy_b = torch::embedding( + lazy_a, + lazy_i, + /*padding_idx=*/0, + /*scale_grad_by_freq=*/false, + /*sparse=*/false); + AllClose(b, lazy_b); + }); +} + +TEST_F(LazyOpsTest, TestOneHot) { + int num_classes = 5; + torch::Tensor input = torch::randint( + 0, + num_classes, + {10}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor output = torch::one_hot(input, num_classes); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::one_hot(lazy_input, num_classes); + AllEqual(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestTranspose) { + torch::Tensor input = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::t(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::t(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestTransposeInPlace) { + torch::Tensor input = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = input.t_(); + torch::Tensor lazy_output = lazy_input.t_(); + EXPECT_EQ(lazy_output.sizes(), output.sizes()); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestReshape) { + torch::Tensor input = torch::rand( + {32, 20, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::reshape(input, {-1, 320}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::reshape(lazy_input, {-1, 320}); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestResize) { + // Testing a resize_() with target size bigger than original size is not + // possible, as we fill with zeros, while pytorch fills with random garbage. + torch::Tensor input = torch::rand( + {2, 2, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor saved_input = input.clone(); + input.resize_({3, 3}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(saved_input, device); + lazy_input.resize_({3, 3}); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestViewResize) { + torch::Tensor input = torch::zeros( + {8, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor saved_input = input.clone(); + torch::Tensor output = input.view({4, 4}); + output.resize_({3, 3}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(saved_input, device); + torch::Tensor lazy_output = lazy_input.view({4, 4}); + lazy_output.resize_({3, 3}); + AllClose(input, lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestView) { + torch::Tensor input = torch::rand( + {32, 20, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = input.view({-1, 320}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = lazy_input.view({-1, 320}); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestViewMod) { + torch::Tensor input = torch::zeros( + {32, 20, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor one = torch::tensor( + 1.0, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = input.view({-1, 320}); + output.add_(one, 1.0); + input.add_(one, 1.0); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor xinput = torch::zeros( + {32, 20, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(xinput, device); + torch::Tensor lazy_one = CopyToDevice(one, device); + torch::Tensor lazy_output = lazy_input.view({-1, 320}); + lazy_output.add_(lazy_one, 1.0); + lazy_input.add_(lazy_one, 1.0); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestViewModComplex) { + torch::Tensor input = torch::zeros( + {32, 20, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor one = torch::tensor( + 1.0, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output1 = input.view({-1, 320}); + output1.add_(one, 1.0); + torch::Tensor output2 = input.view({-1, 160}); + output2.add_(one, 1.0); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor xinput = torch::zeros( + {32, 20, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(xinput, device); + torch::Tensor lazy_one = CopyToDevice(one, device); + torch::Tensor lazy_output1 = lazy_input.view({-1, 320}); + lazy_output1.add_(lazy_one, 1.0); + torch::Tensor lazy_output2 = lazy_input.view({-1, 160}); + lazy_output2.add_(lazy_one, 1.0); + AllClose(output1, lazy_output1); + AllClose(output2, lazy_output2); + }); +} + +TEST_F(LazyOpsTest, TestViewOfViewMod) { + torch::Tensor input = torch::zeros( + {32, 20, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor one = torch::tensor( + 1.0, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output1 = input.view({-1, 320}); + output1.add_(one, 1.0); + torch::Tensor output2 = output1.view({-1, 160}); + output2.add_(one, 1.0); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor xinput = torch::zeros( + {32, 20, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(xinput, device); + torch::Tensor lazy_one = CopyToDevice(one, device); + torch::Tensor lazy_output1 = lazy_input.view({-1, 320}); + lazy_output1.add_(lazy_one, 1.0); + torch::Tensor lazy_output2 = lazy_output1.view({-1, 160}); + lazy_output2.add_(lazy_one, 1.0); + AllClose(output1, lazy_output1); + AllClose(output2, lazy_output2); + }); +} + +TEST_F(LazyOpsTest, TestViewSqueezeAddInPlace) { + torch::Tensor input = torch::zeros( + {2, 3, 1}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector view_size = {2, 3, 1, 1}; + int squeeze_dim = 2; + torch::Tensor one = torch::tensor( + 1.0, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = input.view(view_size); + output.squeeze_(squeeze_dim); + output.add_(one, 1.0); + torch::Tensor lazy_one = CopyToDevice(one, device); + torch::Tensor lazy_output = lazy_input.view(view_size); + lazy_output.squeeze_(squeeze_dim); + lazy_output.add_(lazy_one, 1.0); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestUnsafeView) { + torch::Tensor input = torch::rand( + {32, 20, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::_unsafe_view(input, {-1, 320}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::_unsafe_view(lazy_input, {-1, 320}); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestNarrow) { + torch::Tensor a = torch::rand( + {8, 10, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int64_t dim : {1, -3}) { + for (int64_t start : {2, -8}) { + torch::Tensor b = a.narrow(dim, start, 6); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = lazy_a.narrow(dim, start, 6); + AllClose(b, lazy_b); + }); + } + } +} + +TEST_F(LazyOpsTest, TestNarrowUpdate) { + for (int64_t dim : {1, -2}) { + for (int64_t start : {2, -6}) { + torch::Tensor a = torch::rand( + {3, 8, 3}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor a_copy = a.clone(); + torch::Tensor b = torch::rand( + {3, 4, 3}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = a.narrow(dim, start, 4); + c.add_(b, 1.0); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a_copy, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = lazy_a.narrow(dim, start, 4); + lazy_c.add_(lazy_b, 1.0); + AllClose(c, lazy_c); + }); + } + } +} + +TEST_F(LazyOpsTest, TestNarrowUpdateBaseCheck) { + for (int64_t dim : {0, -2}) { + for (int64_t start : {2, -6}) { + torch::Tensor a = torch::zeros( + {8, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor a_copy = a.clone(); + torch::Tensor b = torch::ones( + {4, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = a.narrow(dim, start, 4); + c.add_(b, 1.0); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a_copy, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = lazy_a.narrow(dim, start, 4); + lazy_c.add_(lazy_b, 1.0); + AllClose(a, lazy_a); + }); + } + } +} + +TEST_F(LazyOpsTest, TestNarrowUpdateTwoSlices) { + for (int64_t dim : {0, -2}) { + for (int64_t start0 : {2, -6}) { + for (int64_t start1 : {6, -2}) { + torch::Tensor a = torch::zeros( + {8, 3}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor a_copy = a.clone(); + torch::Tensor b = torch::ones( + {2, 3}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = b + 1; + torch::Tensor d = a.narrow(dim, start0, 2); + torch::Tensor e = a.narrow(dim, start1, 2); + d.add_(b, 1.0); + e.add_(c, 1.0); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a_copy, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + torch::Tensor lazy_d = lazy_a.narrow(dim, start0, 2); + torch::Tensor lazy_e = lazy_a.narrow(dim, start1, 2); + lazy_d.add_(lazy_b, 1.0); + lazy_e.add_(lazy_c, 1.0); + AllClose(d, lazy_d); + AllClose(e, lazy_e); + AllClose(a, lazy_a); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestNarrowUpdateView) { + for (int64_t dim : {0, -3}) { + for (int64_t start : {2, -6}) { + torch::Tensor a = torch::rand( + {8, 2, 3}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor a_copy = a.clone(); + torch::Tensor b = torch::rand( + {4, 6}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = a.narrow(dim, start, 4); + torch::Tensor d = c.view({4, 6}); + d.add_(b, 1.0); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a_copy, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = lazy_a.narrow(dim, start, 4); + torch::Tensor lazy_d = lazy_c.view({4, 6}); + lazy_d.add_(lazy_b, 1.0); + AllClose(d, lazy_d); + }); + } + } +} + +TEST_F(LazyOpsTest, TestNarrowInNarrowUpdate) { + for (int64_t dim : {1, -2}) { + for (int64_t start0 : {1, -7}) { + for (int64_t start1 : {1, -5}) { + torch::Tensor a = torch::rand( + {3, 8, 3}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor a_copy = a.clone(); + torch::Tensor b = torch::rand( + {3, 2, 3}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = a.narrow(dim, start0, 6); + torch::Tensor d = c.narrow(dim, start1, 2); + d.add_(b, 1.0); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a_copy, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = lazy_a.narrow(dim, start0, 6); + torch::Tensor lazy_d = lazy_c.narrow(dim, start1, 2); + lazy_d.add_(lazy_b, 1.0); + AllClose(a, lazy_a); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestNarrowCopy) { + for (int64_t dim : {1, -3}) { + for (int64_t start : {2, -8}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::rand( + {8, 10, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor result = input.narrow_copy(dim, start, 6); + input.add_(1); + torch::Tensor lazy_result = lazy_input.narrow_copy(dim, start, 6); + lazy_input.add_(1); + AllClose(result, lazy_result); + }); + } + } +} + +TEST_F(LazyOpsTest, TestViewAs) { + torch::Tensor input = torch::rand( + {32, 20, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor empty = torch::empty({32, 320}); + torch::Tensor output = input.view_as(empty); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_empty = CopyToDevice(empty, device); + torch::Tensor lazy_output = lazy_input.view_as(lazy_empty); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestLogSoftmax) { + torch::Tensor input = torch::rand( + {5, 3, 4, 2}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor output = torch::log_softmax(input, dim); + torch::Tensor lazy_output = torch::log_softmax(lazy_input, dim); + AllClose(output, lazy_output, /*rtol=*/1e-3); + } + }); +} + +TEST_F(LazyOpsTest, TestLogSoftmaxCast) { + torch::Tensor input = torch::rand( + {5, 3, 4, 2}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor output = torch::log_softmax(input, dim, torch::kDouble); + torch::Tensor lazy_output = + torch::log_softmax(lazy_input, dim, torch::kDouble); + AllClose(output, lazy_output, /*rtol=*/1e-3); + } + }); +} + +TEST_F(LazyOpsTest, TestLogSoftmaxWrapper) { + torch::Tensor input = torch::rand( + {10, 2, 6, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor output = + torch::_log_softmax(input, dim, /*half_to_float=*/false); + torch::Tensor lazy_output = + torch::_log_softmax(lazy_input, dim, /*half_to_float=*/false); + AllClose(output, lazy_output, /*rtol=*/1e-3); + } + }); +} + +TEST_F(LazyOpsTest, TestSoftmax) { + torch::Tensor input = torch::rand( + {10, 2, 6, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor output = torch::softmax(input, dim); + torch::Tensor lazy_output = torch::softmax(lazy_input, dim); + AllClose(output, lazy_output, /*rtol=*/1e-3); + } + }); +} + +TEST_F(LazyOpsTest, TestSoftmaxCast) { + torch::Tensor input = torch::rand( + {10, 2, 6, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor output = torch::softmax(input, dim, torch::kDouble); + torch::Tensor lazy_output = + torch::softmax(lazy_input, dim, torch::kDouble); + AllClose(output, lazy_output, /*rtol=*/1e-3); + } + }); +} + +TEST_F(LazyOpsTest, TestSoftmaxWrapper) { + torch::Tensor input = torch::rand( + {10, 2, 6, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor output = + torch::_softmax(input, dim, /*half_to_float=*/false); + torch::Tensor lazy_output = + torch::_softmax(lazy_input, dim, /*half_to_float=*/false); + AllClose(output, lazy_output, /*rtol=*/1e-3); + } + }); +} + +TEST_F(LazyOpsTest, TestSoftplus) { + torch::Tensor input = torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::softplus(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::softplus(lazy_input); + AllClose(output, lazy_output, /*rtol=*/1e-4); + }); +} + +TEST_F(LazyOpsTest, TestMaxPool1D) { + torch::Tensor input = torch::rand( + {1, 16, 56}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output = torch::max_pool1d( + input, + /*kernel_size=*/{kernel_size}, + /*stride=*/{stride}, + /*padding=*/{padding}, + /*dilation=*/{dilation}, + /*ceil_mode=*/ceil_mode); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::max_pool1d( + lazy_input, + /*kernel_size=*/{kernel_size}, + /*stride=*/{stride}, + /*padding=*/{padding}, + /*dilation=*/{dilation}, + /*ceil_mode=*/ceil_mode); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool2D) { + torch::Tensor input = torch::rand( + {1, 4, 14, 14}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output = torch::max_pool2d( + input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{dilation, dilation}, + /*ceil_mode=*/ceil_mode); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::max_pool2d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{dilation, dilation}, + /*ceil_mode=*/ceil_mode); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool2DWithIndices) { + torch::Tensor input = torch::rand( + {1, 4, 14, 14}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + auto outputs = torch::max_pool2d_with_indices( + input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{dilation, dilation}, + /*ceil_mode=*/ceil_mode); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + auto lazy_outputs = torch::max_pool2d_with_indices( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{dilation, dilation}, + /*ceil_mode=*/ceil_mode); + AllClose(std::get<0>(outputs), std::get<0>(lazy_outputs)); + AllClose(std::get<1>(outputs), std::get<1>(lazy_outputs)); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool2DNonSquare) { + torch::Tensor input = torch::rand( + {1, 4, 14, 14}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 4; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output = torch::max_pool2d( + input, + /*kernel_size=*/{kernel_size, kernel_size + 1}, + /*stride=*/{stride, stride + 1}, + /*padding=*/{padding, padding + 1}, + /*dilation=*/{dilation, dilation}, + /*ceil_mode=*/ceil_mode); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::max_pool2d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size + 1}, + /*stride=*/{stride, stride + 1}, + /*padding=*/{padding, padding + 1}, + /*dilation=*/{dilation, dilation}, + /*ceil_mode=*/ceil_mode); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool3D) { + torch::Tensor input = torch::rand( + {1, 1, 8, 8, 8}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output = torch::max_pool3d( + input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::max_pool3d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool3DWithIndices) { + torch::Tensor input = torch::rand( + {1, 1, 8, 8, 8}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + auto outputs = torch::max_pool3d_with_indices( + input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + auto lazy_outputs = torch::max_pool3d_with_indices( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + + AllClose(std::get<0>(outputs), std::get<0>(lazy_outputs)); + AllClose(std::get<1>(outputs), std::get<1>(lazy_outputs)); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool3DIncompleteAttributes) { + torch::Tensor input = torch::rand( + {1, 1, 8, 8, 8}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output = torch::max_pool3d( + input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{}, + /*padding=*/{padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::max_pool3d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{}, + /*padding=*/{padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool3DNonSquare) { + torch::Tensor input = torch::rand( + {1, 1, 8, 8, 8}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 4; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output = torch::max_pool3d( + input, + /*kernel_size=*/{kernel_size, kernel_size + 1, kernel_size}, + /*stride=*/{stride, stride + 1, stride}, + /*padding=*/{padding, padding + 1, padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::max_pool3d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size + 1, kernel_size}, + /*stride=*/{stride, stride + 1, stride}, + /*padding=*/{padding, padding + 1, padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool2DNoBatch) { + torch::Tensor input = torch::rand( + {4, 14, 14}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output = torch::max_pool2d( + input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{dilation, dilation}, + /*ceil_mode=*/ceil_mode); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::max_pool2d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{dilation, dilation}, + /*ceil_mode=*/ceil_mode); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool3DNoBatch) { + torch::Tensor input = torch::rand( + {1, 8, 8, 8}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output = torch::max_pool3d( + input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::max_pool3d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAvgPool1D) { + torch::Tensor input = torch::rand( + {4, 1, 28}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 2; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + torch::Tensor output = torch::avg_pool1d( + input, + /*kernel_size=*/{kernel_size}, + /*stride=*/{stride}, + /*padding=*/{padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::avg_pool1d( + lazy_input, + /*kernel_size=*/{kernel_size}, + /*stride=*/{stride}, + /*padding=*/{padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAvgPool2D) { + torch::Tensor input = torch::rand( + {2, 1, 14, 14}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 2; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + torch::Tensor output = torch::avg_pool2d( + input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + ForEachDevice([&](const torch::Device& device) { + // torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::avg_pool2d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + AllClose(output, lazy_output.to(torch::kCPU)); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAvgPool2DNonSquare) { + torch::Tensor input = torch::rand( + {2, 1, 14, 14}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 4; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + torch::Tensor output = torch::avg_pool2d( + input, + /*kernel_size=*/{kernel_size, kernel_size + 1}, + /*stride=*/{stride, stride + 1}, + /*padding=*/{padding, padding + 1}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::avg_pool2d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size + 1}, + /*stride=*/{stride, stride + 1}, + /*padding=*/{padding, padding + 1}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAvgPool3D) { + torch::Tensor input = torch::rand( + {1, 1, 7, 7, 7}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 2; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + torch::Tensor output = torch::avg_pool3d( + input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::avg_pool3d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAvgPool3DIncompleteAttributes) { + torch::Tensor input = torch::rand( + {1, 1, 7, 7, 7}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 2; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + torch::Tensor output = torch::avg_pool3d( + input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{}, + /*padding=*/{padding, padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::avg_pool3d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{}, + /*padding=*/{padding, padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAvgPool3DNonSquare) { + torch::Tensor input = torch::rand( + {1, 1, 7, 7, 7}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 4; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + torch::Tensor output = torch::avg_pool3d( + input, + /*kernel_size=*/{kernel_size, kernel_size + 1, kernel_size}, + /*stride=*/{stride, stride + 1, stride}, + /*padding=*/{padding, padding + 1, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::avg_pool3d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size + 1, kernel_size}, + /*stride=*/{stride, stride + 1, stride}, + /*padding=*/{padding, padding + 1, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAvgPool2DNoBatch) { + torch::Tensor input = torch::rand( + {1, 7, 7}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 2; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + torch::Tensor output = torch::avg_pool2d( + input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::avg_pool2d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAvgPool3DNoBatch) { + torch::Tensor input = torch::rand( + {1, 7, 7, 7}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int kernel_size = 2; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + torch::Tensor output = torch::avg_pool3d( + input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::avg_pool3d( + lazy_input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAdaptiveAvgPool2D) { + torch::Tensor input = torch::rand( + {4, 1, 28, 28}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int64_t output_size : {7, 4}) { + torch::Tensor output = + torch::adaptive_avg_pool2d(input, {output_size, output_size}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = + torch::adaptive_avg_pool2d(lazy_input, {output_size, output_size}); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestAdaptiveAvgPool3D) { + torch::Tensor input = torch::rand( + {9, 4, 56, 28, 28}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int64_t output_size : {7, 4}) { + torch::Tensor output = torch::adaptive_avg_pool3d( + input, {output_size, output_size, output_size}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::adaptive_avg_pool3d( + lazy_input, {output_size, output_size, output_size}); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestAdaptiveAvgPool3DNoBatch) { + torch::Tensor input = torch::rand( + {3, 56, 28, 28}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int64_t output_size : {7, 4}) { + torch::Tensor output = torch::adaptive_avg_pool3d( + input, {output_size, output_size, output_size}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::adaptive_avg_pool3d( + lazy_input, {output_size, output_size, output_size}); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestAdaptiveAvgPool2DNoBatch) { + torch::Tensor input = torch::rand( + {1, 56, 56}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int64_t output_size : {7, 8}) { + torch::Tensor output = + torch::adaptive_avg_pool2d(input, {output_size, output_size}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = + torch::adaptive_avg_pool2d(lazy_input, {output_size, output_size}); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestMaxUnpool2D) { + int kernel_size = 2; + torch::Tensor input = torch::rand( + {2, 2, 8, 8}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output; + torch::Tensor indices; + std::tie(output, indices) = torch::max_pool2d_with_indices( + input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{dilation, dilation}, + /*ceil_mode=*/ceil_mode); + + std::vector output_size({input.size(2), input.size(3)}); + at::Tensor utensor = + torch::max_unpool2d(output, indices, output_size); + + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_output = CopyToDevice(output, device); + torch::Tensor lazy_indices = CopyToDevice(indices, device); + at::Tensor lazy_utensor = + torch::max_unpool2d(lazy_output, lazy_indices, output_size); + AllClose(utensor, lazy_utensor); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxUnpool3D) { + int kernel_size = 2; + torch::Tensor input = torch::rand( + {1, 1, 4, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + // Test dilation through the CPU interop. + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output; + torch::Tensor indices; + std::tie(output, indices) = torch::max_pool3d_with_indices( + input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + + std::vector output_size( + {input.size(2), input.size(3), input.size(4)}); + at::Tensor utensor = torch::max_unpool3d( + output, + indices, + output_size, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}); + + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_output = CopyToDevice(output, device); + torch::Tensor lazy_indices = CopyToDevice(indices, device); + at::Tensor lazy_utensor = torch::max_unpool3d( + lazy_output, + lazy_indices, + output_size, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}); + AllClose(utensor, lazy_utensor); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestNllLoss) { + int batch = 6; + int classes = 2; + // TODO(asuhan): Fix the torch::kDouble case. + for (auto dtype : {torch::kFloat}) { + for (int ignore_index : {-1, 0, 1, 5}) { + for (bool def_weight : {false, true}) { + torch::Tensor input = torch::rand( + {batch, classes}, + torch::TensorOptions(dtype).device(DefaultDevice())); + torch::Tensor target = torch::randint( + std::min(ignore_index, 0), + classes, + {batch}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor weight; + if (def_weight) { + weight = torch::rand( + {classes}, torch::TensorOptions(dtype).device(DefaultDevice())); + } + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, + torch::Reduction::Sum, + torch::Reduction::None}) { + torch::Tensor output = torch::nll_loss( + /*self=*/input, + /*target=*/target, + /*weight=*/weight, + /*reduction=*/reduction, + /*ignore_index=*/ignore_index); + + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_target = CopyToDevice(target, device); + torch::Tensor lazy_weight = + def_weight ? CopyToDevice(weight, device) : torch::Tensor(); + torch::Tensor lazy_output = torch::nll_loss( + /*self=*/lazy_input, + /*target=*/lazy_target, + /*weight=*/lazy_weight, + /*reduction=*/reduction, + /*ignore_index=*/ignore_index); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestNllLoss2d) { + int batch = 6; + int classes = 2; + int height = 3; + int width = 3; + // TODO(asuhan): Fix the torch::kDouble case. + for (auto dtype : {torch::kFloat}) { + for (int ignore_index : {-1, 0, 1, 5}) { + for (bool def_weight : {false, true}) { + torch::Tensor input = torch::rand( + {batch, classes, height, width}, + torch::TensorOptions(dtype).device(DefaultDevice())); + torch::Tensor target = torch::randint( + std::min(ignore_index, 0), + classes, + {batch, height, width}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor weight; + if (def_weight) { + weight = torch::rand( + {classes}, torch::TensorOptions(dtype).device(DefaultDevice())); + } + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, + torch::Reduction::Sum, + torch::Reduction::None}) { + torch::Tensor output = torch::nll_loss2d( + /*self=*/input, + /*target=*/target, + /*weight=*/weight, + /*reduction=*/reduction, + /*ignore_index=*/ignore_index); + + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_target = CopyToDevice(target, device); + torch::Tensor lazy_weight = + def_weight ? CopyToDevice(weight, device) : torch::Tensor(); + torch::Tensor lazy_output = torch::nll_loss2d( + /*self=*/lazy_input, + /*target=*/lazy_target, + /*weight=*/lazy_weight, + /*reduction=*/reduction, + /*ignore_index=*/ignore_index); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestSmoothL1Loss) { + torch::Tensor input = torch::randn( + {2, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor target = torch::randn( + {2, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (torch::Reduction::Reduction reduction : + {torch::Reduction::None, + torch::Reduction::Mean, + torch::Reduction::Sum}) { + for (double beta : {0.25, 1.}) { + torch::Tensor output = + torch::smooth_l1_loss(input, target, reduction, beta); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_target = CopyToDevice(target, device); + torch::Tensor lazy_output = + torch::smooth_l1_loss(lazy_input, lazy_target, reduction, beta); + AllClose(output, lazy_output); + }); + } + } +} + +TEST_F(LazyOpsTest, TestL1Loss) { + torch::Tensor input = torch::randn( + {2, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor target = torch::randn( + {2, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (torch::Reduction::Reduction reduction : + {torch::Reduction::None, + torch::Reduction::Mean, + torch::Reduction::Sum}) { + torch::Tensor output = torch::l1_loss(input, target, reduction); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_target = CopyToDevice(target, device); + torch::Tensor lazy_output = + torch::l1_loss(lazy_input, lazy_target, reduction); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestL1LossBackward) { + for (torch::Reduction::Reduction reduction : + {torch::Reduction::None, + torch::Reduction::Mean, + torch::Reduction::Sum}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::l1_loss(inputs[0], inputs[1], reduction); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 4}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)), + torch::rand( + {2, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice()))}, + device, + testfn); + }); + } +} + +TEST_F(LazyOpsTest, TestMseLoss) { + torch::Tensor input = torch::randn( + {2, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor target = torch::randn( + {2, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (torch::Reduction::Reduction reduction : + {torch::Reduction::None, + torch::Reduction::Mean, + torch::Reduction::Sum}) { + torch::Tensor output = torch::mse_loss(input, target, reduction); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_target = CopyToDevice(target, device); + torch::Tensor lazy_output = + torch::mse_loss(lazy_input, lazy_target, reduction); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestMseLossBackward) { + for (torch::Reduction::Reduction reduction : + {torch::Reduction::None, + torch::Reduction::Mean, + torch::Reduction::Sum}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::mse_loss(inputs[0], inputs[1], reduction); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 4}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)), + torch::rand( + {2, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice()))}, + device, + testfn); + }); + } +} + +TEST_F(LazyOpsTest, TestBatchNorm1D) { + int num_features = 3; + torch::Tensor input = torch::rand( + {2, num_features, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor bias = torch::rand( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor running_mean = torch::zeros( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor running_var = torch::ones( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + double momentum = 0.1; + double eps = 0.5; + torch::Tensor undef; + for (bool training : {true, false}) { + for (bool undef_weight_bias : {false, true}) { + torch::Tensor output = torch::batch_norm( + /*input=*/input, + /*weight=*/undef_weight_bias ? undef : weight, + /*bias=*/undef_weight_bias ? undef : bias, + /*running_mean=*/running_mean, + /*running_var=*/running_var, + /*training=*/training, + /*momentum=*/momentum, + /*eps=*/eps, + /*cudnn_enabled=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_weight = + undef_weight_bias ? undef : CopyToDevice(weight, device); + torch::Tensor lazy_bias = + undef_weight_bias ? undef : CopyToDevice(bias, device); + torch::Tensor lazy_running_mean = CopyToDevice(running_mean, device); + torch::Tensor lazy_running_var = CopyToDevice(running_var, device); + torch::Tensor lazy_output = torch::batch_norm( + /*input=*/lazy_input, + /*weight=*/lazy_weight, + /*bias=*/lazy_bias, + /*running_mean=*/lazy_running_mean, + /*running_var=*/lazy_running_var, + /*training=*/training, + /*momentum=*/momentum, + /*eps=*/eps, + /*cudnn_enabled=*/false); + AllClose(output, lazy_output, /*rtol=*/1e-3, /*atol=*/1e-5); + }); + } + } +} + +TEST_F(LazyOpsTest, TestBatchNorm2D) { + int num_features = 3; + torch::Tensor input = torch::rand( + {2, num_features, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor bias = torch::rand( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor running_mean = torch::zeros( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor running_var = torch::ones( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + double momentum = 0.1; + double eps = 0.5; + torch::Tensor undef; + for (bool training : {true, false}) { + for (bool undef_weight_bias : {false, true}) { + torch::Tensor output = torch::batch_norm( + /*input=*/input, + /*weight=*/undef_weight_bias ? undef : weight, + /*bias=*/undef_weight_bias ? undef : bias, + /*running_mean=*/running_mean, + /*running_var=*/running_var, + /*training=*/training, + /*momentum=*/momentum, + /*eps=*/eps, + /*cudnn_enabled=*/false); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_weight = + undef_weight_bias ? undef : CopyToDevice(weight, device); + torch::Tensor lazy_bias = + undef_weight_bias ? undef : CopyToDevice(bias, device); + torch::Tensor lazy_running_mean = CopyToDevice(running_mean, device); + torch::Tensor lazy_running_var = CopyToDevice(running_var, device); + torch::Tensor lazy_output = torch::batch_norm( + /*input=*/lazy_input, + /*weight=*/lazy_weight, + /*bias=*/lazy_bias, + /*running_mean=*/lazy_running_mean, + /*running_var=*/lazy_running_var, + /*training=*/training, + /*momentum=*/momentum, + /*eps=*/eps, + /*cudnn_enabled=*/false); + AllClose(output, lazy_output, /*rtol=*/1e-3, /*atol=*/1e-5); + }); + } + } +} + +TEST_F(LazyOpsTest, TestDim) { + torch::Tensor input = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + EXPECT_EQ(input.dim(), lazy_input.dim()); + }); +} + +TEST_F(LazyOpsTest, TestContiguous) { + torch::Tensor input = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::native::contiguous(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::native::contiguous(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestSqueezeAll) { + torch::Tensor input = torch::rand( + {2, 1, 3, 1}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::squeeze(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::squeeze(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestSqueezeAllInPlace) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::rand( + {2, 1, 3, 1}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = input.squeeze_(); + torch::Tensor lazy_output = lazy_input.squeeze_(); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + ASSERT_EQ(input.dim(), lazy_input.dim()); + for (int64_t dim_idx = 0; dim_idx < input.dim(); ++dim_idx) { + ASSERT_EQ(input.size(dim_idx), lazy_input.size(dim_idx)); + } + }); +} + +TEST_F(LazyOpsTest, TestSqueezeOne) { + torch::Tensor input = torch::rand( + {2, 1, 3, 1}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor output = torch::squeeze(input, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::squeeze(lazy_input, dim); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestSqueezeOneInPlace) { + int rank = 4; + for (int dim = -rank; dim < rank; ++dim) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::rand( + {2, 1, 3, 1}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = input.squeeze_(dim); + torch::Tensor lazy_output = lazy_input.squeeze_(dim); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + ASSERT_EQ(input.dim(), lazy_input.dim()); + for (int64_t dim_idx = 0; dim_idx < input.dim(); ++dim_idx) { + ASSERT_EQ(input.size(dim_idx), lazy_input.size(dim_idx)); + } + }); + } +} + +TEST_F(LazyOpsTest, TestUnsqueeze) { + torch::Tensor input = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim() + 1; + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor output = torch::unsqueeze(input, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::unsqueeze(lazy_input, dim); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestUnsqueezeInPlace) { + torch::Tensor input = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim() + 1; + for (int dim = -rank; dim < rank; ++dim) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = input.unsqueeze_(dim); + torch::Tensor lazy_output = lazy_input.unsqueeze_(dim); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + ASSERT_EQ(input.dim(), lazy_input.dim()); + for (int64_t dim_idx = 0; dim_idx < input.dim(); ++dim_idx) { + ASSERT_EQ(input.size(dim_idx), lazy_input.size(dim_idx)); + } + }); + } +} + +TEST_F(LazyOpsTest, TestMaskedFill) { + torch::Tensor input = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor mask = torch::randint( + 0, 2, {2, 3}, torch::TensorOptions(torch::kBool).device(DefaultDevice())); + torch::Scalar value(42); + torch::Tensor result = torch::masked_fill(input, mask, value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_mask = CopyToDevice(mask, device); + torch::Tensor lazy_result = + torch::masked_fill(lazy_input, lazy_mask, value); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestMaskedFillInPlace) { + torch::Scalar value(42); + torch::Tensor mask = torch::randint( + 0, 2, {2, 3}, torch::TensorOptions(torch::kBool).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::rand( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_mask = CopyToDevice(mask, device); + torch::Tensor result = input.masked_fill_(mask, value); + torch::Tensor lazy_result = lazy_input.masked_fill_(lazy_mask, value); + AllClose(result, lazy_result); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestMaskedFillBroadcast) { + torch::Tensor input = torch::rand( + {2, 5, 4, 3}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor mask = torch::randint( + 0, 2, {4, 1}, torch::TensorOptions(torch::kBool).device(DefaultDevice())); + torch::Scalar value(42); + torch::Tensor result = torch::masked_fill(input, mask, value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_mask = CopyToDevice(mask, device); + torch::Tensor lazy_result = + torch::masked_fill(lazy_input, lazy_mask, value); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestFill) { + torch::Scalar value(42); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::empty( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor result = torch::fill_(input, value); + torch::Tensor lazy_result = torch::fill_(lazy_input, value); + AllClose(result, lazy_result); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestFillWithRank0) { + torch::Tensor value = torch::scalar_tensor(42); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::empty( + {2, 3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor result = torch::fill_(input, value); + torch::Tensor lazy_value = CopyToDevice(value, device); + torch::Tensor lazy_result = torch::fill_(lazy_input, value); + AllClose(result, lazy_result); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestPermute) { + torch::Tensor input = torch::rand( + {2, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector> dims_permutations = { + {0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}}; + int rank = input.dim(); + for (std::vector dims_permutation : dims_permutations) { + for (bool negative_dims : {false, true}) { + if (negative_dims) { + std::for_each( + dims_permutation.begin(), + dims_permutation.end(), + [rank](int64_t& dim) { dim -= rank; }); + } + torch::Tensor output = input.permute(dims_permutation); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = lazy_input.permute(dims_permutation); + AllClose(output, lazy_output); + }); + } + } +} + +TEST_F(LazyOpsTest, TestPermuteMod) { + std::vector> dims_permutations = { + {0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}}; + std::vector input_sizes = {2, 3, 4}; + int rank = input_sizes.size(); + for (std::vector dims_permutation : dims_permutations) { + for (bool negative_dims : {false, true}) { + if (negative_dims) { + std::for_each( + dims_permutation.begin(), + dims_permutation.end(), + [rank](int64_t& dim) { dim -= rank; }); + } + torch::Tensor input = torch::zeros( + input_sizes, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor one = torch::tensor( + 1.0, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = input.permute(dims_permutation); + output.add_(one, 1.0); + input.add_(one, 1.0); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor xinput = torch::zeros( + input_sizes, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(xinput, device); + torch::Tensor lazy_one = CopyToDevice(one, device); + torch::Tensor lazy_output = lazy_input.permute(dims_permutation); + lazy_output.add_(lazy_one, 1.0); + lazy_input.add_(lazy_one, 1.0); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); + } + } +} + +TEST_F(LazyOpsTest, TestFlip) { + torch::Tensor input = torch::rand( + {2, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector> dim_powerset = { + {0}, {1}, {2}, {0, 1}, {1, 2}, {2, 0}, {0, 1, 2}}; + for (std::vector flip_dims : dim_powerset) { + for (bool negative_dims : {false, true}) { + if (negative_dims) { + std::for_each( + flip_dims.begin(), flip_dims.end(), [](int64_t& dim) { dim -= 3; }); + } + torch::Tensor output = torch::flip(input, flip_dims); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::flip(lazy_input, flip_dims); + AllClose(output, lazy_output); + }); + } + } +} + +TEST_F(LazyOpsTest, TestPixelShuffle) { + torch::Tensor input = torch::rand( + {5, 18, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int upscale_factor = 3; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = torch::pixel_shuffle(input, upscale_factor); + torch::Tensor lazy_output = + torch::pixel_shuffle(lazy_input, upscale_factor); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestSumToSize) { + torch::Tensor input = torch::rand( + {4, 6, 3, 7}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector out_size = {4, 1, 1, 7}; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = input.sum_to_size(out_size); + torch::Tensor lazy_output = lazy_input.sum_to_size(out_size); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestTransposeDims) { + torch::Tensor input = torch::rand( + {2, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int dim0 = 0; + int dim1 = 2; + torch::Tensor output = torch::transpose(input, dim0, dim1); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::transpose(lazy_input, dim0, dim1); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestTransposeDimsMod) { + std::vector input_sizes = {2, 3, 4}; + int dim0 = 0; + int dim1 = 2; + torch::Tensor input = torch::zeros( + input_sizes, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor one = torch::tensor( + 1.0, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::transpose(input, dim0, dim1); + output.add_(one, 1.0); + input.add_(one, 1.0); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor xinput = torch::zeros( + input_sizes, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(xinput, device); + torch::Tensor lazy_one = CopyToDevice(one, device); + torch::Tensor lazy_output = torch::transpose(lazy_input, dim0, dim1); + lazy_output.add_(lazy_one, 1.0); + lazy_input.add_(lazy_one, 1.0); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestTransposeDimsInPlace) { + torch::Tensor input = torch::rand( + {2, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int dim0 = 0; + int dim1 = 2; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = input.transpose_(dim0, dim1); + torch::Tensor lazy_output = lazy_input.transpose_(dim0, dim1); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestSplit) { + torch::Tensor input = torch::rand( + {7, 8, 9}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (int split_size : {2, 3}) { + for (int dim = -rank; dim < rank; ++dim) { + std::vector outputs = torch::split(input, split_size, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + std::vector lazy_outputs = + torch::split(lazy_input, split_size, dim); + ASSERT_EQ(outputs.size(), lazy_outputs.size()); + for (size_t i = 0; i < outputs.size(); ++i) { + AllClose(outputs[i], lazy_outputs[i]); + } + }); + } + } +} + +TEST_F(LazyOpsTest, TestSplitEmpty) { + torch::Tensor input = torch::rand( + {0}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int split_size = 0; + int dim = 0; + std::vector outputs = torch::split(input, split_size, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + std::vector lazy_outputs = + torch::split(lazy_input, split_size, dim); + ASSERT_EQ(outputs.size(), lazy_outputs.size()); + for (size_t i = 0; i < outputs.size(); ++i) { + AllClose(outputs[i], lazy_outputs[i]); + } + }); +} + +TEST_F(LazyOpsTest, TestSplitWithSizes) { + torch::Tensor input = torch::rand( + {15, 15, 15}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = input.dim(); + for (int dim = -rank; dim < rank; ++dim) { + std::vector outputs = + torch::split_with_sizes(input, {4, 5, 6}, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + std::vector lazy_outputs = + torch::split_with_sizes(lazy_input, {4, 5, 6}, dim); + ASSERT_EQ(outputs.size(), lazy_outputs.size()); + for (size_t i = 0; i < outputs.size(); ++i) { + AllClose(outputs[i], lazy_outputs[i]); + } + }); + } +} + +TEST_F(LazyOpsTest, TestCrossImplicitDim) { + std::vector> dim_sizes = { + {4, 5, 3}, {4, 3, 5}, {3, 4, 5}}; + for (auto dim_size : dim_sizes) { + torch::Tensor input = torch::rand( + dim_size, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor other = torch::rand( + dim_size, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor result = torch::cross(input, other); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_other = CopyToDevice(other, device); + torch::Tensor lazy_result = torch::cross(lazy_input, lazy_other); + AllClose(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestCrossExplicitDim) { + std::vector dim_size = {3, 3}; + torch::Tensor input = torch::rand( + dim_size, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor other = torch::rand( + dim_size, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + int rank = dim_size.size(); + for (int dim = -rank; dim < rank; ++dim) { + torch::Tensor result = torch::cross(input, other, dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_other = CopyToDevice(other, device); + torch::Tensor lazy_result = torch::cross(lazy_input, lazy_other, dim); + AllClose(result, lazy_result); + }); + } +} + +TEST_F(LazyOpsTest, TestCrossZeroDim) { + torch::Tensor input = torch::rand( + {0, 1, 3, 0}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor result = torch::cross(input, input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::cross(lazy_input, lazy_input); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestTriu) { + int size = 5; + torch::Tensor input = torch::rand( + {size, size}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + torch::Tensor output = torch::triu(input, diagonal); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::triu(lazy_input, diagonal); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestTriuNonSquare) { + int size = 5; + torch::Tensor input = torch::rand( + {size, size + 1}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + torch::Tensor output = torch::triu(input, diagonal); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::triu(lazy_input, diagonal); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestTriuBatch) { + int size = 5; + int batch_size = 3; + torch::Tensor input = torch::rand( + {batch_size, size, size}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + torch::Tensor output = torch::triu(input, diagonal); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::triu(lazy_input, diagonal); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestTril) { + int size = 5; + torch::Tensor input = torch::rand( + {size, size}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + torch::Tensor output = torch::tril(input, diagonal); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::tril(lazy_input, diagonal); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestTrilNonSquare) { + int size = 5; + torch::Tensor input = torch::rand( + {size, size + 1}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + torch::Tensor output = torch::tril(input, diagonal); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::tril(lazy_input, diagonal); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestTrilBatch) { + int size = 5; + int batch_size = 3; + torch::Tensor input = torch::rand( + {batch_size, size, size}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + torch::Tensor output = torch::tril(input, diagonal); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::tril(lazy_input, diagonal); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestTriuInPlace) { + int size = 5; + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::rand( + {size, size}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = input.triu_(diagonal); + torch::Tensor lazy_output = lazy_input.triu_(diagonal); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); + } +} + +TEST_F(LazyOpsTest, TestTrilInPlace) { + int size = 5; + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::rand( + {size, size}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = input.tril_(diagonal); + torch::Tensor lazy_output = lazy_input.tril_(diagonal); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); + } +} + +TEST_F(LazyOpsTest, TestTrace) { + int n = 5; + torch::Tensor input = torch::rand( + {n, n}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::trace(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::trace(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestTraceWide) { + int lines = 3; + int cols = 5; + torch::Tensor input = torch::rand( + {lines, cols}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::trace(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::trace(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestTraceNarrow) { + int lines = 5; + int cols = 3; + torch::Tensor input = torch::rand( + {lines, cols}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor output = torch::trace(input); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::trace(lazy_input); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestDiagRank1) { + int size = 7; + torch::Tensor input = torch::rand( + {size}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -2 * size; diagonal <= 2 * size; ++diagonal) { + torch::Tensor output = torch::diag(input, diagonal); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::diag(lazy_input, diagonal); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestDiagRank2) { + int size = 7; + torch::Tensor input = torch::rand( + {size, size}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + torch::Tensor output = torch::diag(input, diagonal); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::diag(lazy_input, diagonal); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestDiagFlat) { + torch::Tensor input = torch::rand( + {4, 3, 6, 7}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int diagonal = -10; diagonal < 10; ++diagonal) { + torch::Tensor output = torch::diagflat(input, diagonal); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::diagflat(lazy_input, diagonal); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestDiagonal) { + int size = 5; + torch::Tensor input = torch::rand( + {size, size}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + torch::Tensor output = torch::diagonal(input, diagonal); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::diagonal(lazy_input, diagonal); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestDiagonalUpdate) { + int size = 5; + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + auto input = torch::rand( + {size, size}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + auto input_clone = input.clone(); + auto output = torch::diagonal(input, diagonal); + output.add_(1); + + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input_clone, device); + torch::Tensor lazy_output = torch::diagonal(lazy_input, diagonal); + lazy_output.add_(1); + + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); + } +} + +TEST_F(LazyOpsTest, TestDiagonalNonSquare) { + int size = 5; + torch::Tensor input = torch::rand( + {size, size + 1}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + torch::Tensor output = torch::diagonal(input, diagonal); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::diagonal(lazy_input, diagonal); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestDiagonalBatch) { + int size = 5; + int batch_size = 3; + int dim1 = 1; + int dim2 = 2; + torch::Tensor input = torch::rand( + {batch_size, size, size}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + // Test all diagonals and out of bounds (must be no-op). + for (int diagonal = -size; diagonal <= size; ++diagonal) { + torch::Tensor output = + torch::diagonal(input, diagonal, /*dim1=*/dim1, /*dim1=*/dim2); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = + torch::diagonal(lazy_input, diagonal, /*dim1=*/dim1, /*dim1=*/dim2); + AllClose(output, lazy_output); + }); + } +} + +TEST_F(LazyOpsTest, TestFlatten) { + torch::Tensor input = torch::rand({4, 7, 5, 3}); + int rank = input.dim(); + for (int pos_start_dim = 0; pos_start_dim < rank; ++pos_start_dim) { + for (int pos_end_dim = pos_start_dim; pos_end_dim < rank; ++pos_end_dim) { + for (bool negative_start_dim : {false, true}) { + for (bool negative_end_dim : {false, true}) { + int start_dim = + negative_start_dim ? pos_start_dim - rank : pos_start_dim; + int end_dim = negative_end_dim ? pos_end_dim - rank : pos_end_dim; + torch::Tensor output = torch::flatten(input, start_dim, end_dim); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = + torch::flatten(lazy_input, start_dim, end_dim); + AllClose(output, lazy_output); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestLogicalAnd) { + for (torch::ScalarType scalar_type1 : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor lhs = isFloatingType(scalar_type1) + ? torch::rand({3, 4}, torch::TensorOptions(scalar_type1)) + : torch::randint(0, 100, {3, 4}, torch::TensorOptions(scalar_type1)); + for (torch::ScalarType scalar_type2 : + {torch::kFloat, + torch::kByte, + torch::kChar, + torch::kShort, + torch::kInt, + torch::kLong}) { + torch::Tensor rhs = isFloatingType(scalar_type2) + ? torch::rand({3, 4}, torch::TensorOptions(scalar_type2)) + : torch::randint(1, 100, {3, 4}, torch::TensorOptions(scalar_type2)); + torch::Tensor result = torch::logical_and(lhs, rhs); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor lazy_rhs = CopyToDevice(rhs, device); + torch::Tensor lazy_result = torch::logical_and(lazy_lhs, lazy_rhs); + AllEqual(result, lazy_result); + }); + } + } + + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("xla::logical_and_out", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestBitwiseAnd) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Tensor rhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Tensor result = lhs.__and__(rhs); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor lazy_rhs = CopyToDevice(rhs, device); + torch::Tensor lazy_result = lazy_lhs.__and__(lazy_rhs); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseAndInPlace) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Tensor rhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor result = lhs.__iand__(rhs); + torch::Tensor lazy_rhs = CopyToDevice(rhs, device); + torch::Tensor lazy_result = lazy_lhs.__iand__(lazy_rhs); + AllEqual(result, lazy_result); + AllEqual(lhs, lazy_lhs); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseAndScalar) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Scalar rhs(123456789); + torch::Tensor result = lhs.__and__(rhs); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor lazy_result = lazy_lhs.__and__(rhs); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseAndScalarInPlace) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Scalar rhs(123456789); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor result = lhs.__iand__(rhs); + torch::Tensor lazy_result = lazy_lhs.__iand__(rhs); + AllEqual(result, lazy_result); + AllEqual(lhs, lazy_lhs); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseAndPromotion) { + torch::Tensor input = torch::rand( + {4, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor view = input.reshape(-1); + torch::Tensor result = torch::__and__(view.gt(0), view.ne(0)); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_view = lazy_input.reshape(-1); + torch::Tensor lazy_result = + torch::__and__(lazy_view.gt(0), lazy_view.ne(0)); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseOr) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Tensor rhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Tensor result = lhs.__or__(rhs); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor lazy_rhs = CopyToDevice(rhs, device); + torch::Tensor lazy_result = lazy_lhs.__or__(lazy_rhs); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseOrInPlace) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Tensor rhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor result = lhs.__ior__(rhs); + torch::Tensor lazy_rhs = CopyToDevice(rhs, device); + torch::Tensor lazy_result = lazy_lhs.__ior__(lazy_rhs); + AllEqual(result, lazy_result); + AllEqual(lhs, lazy_lhs); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseOrScalar) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Scalar rhs(123456789); + torch::Tensor result = lhs.__or__(rhs); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor lazy_result = lazy_lhs.__or__(rhs); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseOrScalarInPlace) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Scalar rhs(123456789); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor result = lhs.__ior__(rhs); + torch::Tensor lazy_result = lazy_lhs.__ior__(rhs); + AllEqual(result, lazy_result); + AllEqual(lhs, lazy_lhs); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseXor) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Tensor rhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Tensor result = lhs.__xor__(rhs); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor lazy_rhs = CopyToDevice(rhs, device); + torch::Tensor lazy_result = lazy_lhs.__xor__(lazy_rhs); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseXorInPlace) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Tensor rhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor result = lhs.__ixor__(rhs); + torch::Tensor lazy_rhs = CopyToDevice(rhs, device); + torch::Tensor lazy_result = lazy_lhs.__ixor__(lazy_rhs); + AllEqual(result, lazy_result); + AllEqual(lhs, lazy_lhs); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseXorScalar) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Scalar rhs(123456789); + torch::Tensor result = lhs.__xor__(rhs); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor lazy_result = lazy_lhs.__xor__(rhs); + AllEqual(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestBitwiseXorScalarInPlace) { + torch::Tensor lhs = torch::randint( + 0, + std::numeric_limits::max(), + {4, 2}, + torch::TensorOptions(torch::kInt)); + torch::Scalar rhs(123456789); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_lhs = CopyToDevice(lhs, device); + torch::Tensor result = lhs.__ixor__(rhs); + torch::Tensor lazy_result = lazy_lhs.__ixor__(rhs); + AllEqual(result, lazy_result); + AllEqual(lhs, lazy_lhs); + }); +} + +TEST_F(LazyOpsTest, TestLshift) { + torch::Tensor input = torch::ones( + {4, 2}, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Tensor shift_amount = torch::randint( + 16, + input.sizes(), + torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Tensor result = torch::__lshift__(input, shift_amount); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_shift_amount = CopyToDevice(shift_amount, device); + torch::Tensor lazy_result = + torch::__lshift__(lazy_input, lazy_shift_amount); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestLshiftInPlace) { + torch::Tensor input = torch::ones( + {4, 2}, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor shift_amount = torch::randint( + 16, + input.sizes(), + torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Tensor result = input.__ilshift__(shift_amount); + torch::Tensor lazy_shift_amount = CopyToDevice(shift_amount, device); + torch::Tensor lazy_result = lazy_input.__ilshift__(lazy_shift_amount); + AllClose(result, lazy_result); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestLshiftScalar) { + torch::Tensor input = torch::ones( + {4, 2}, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Scalar shift_amount = 3; + torch::Tensor result = torch::__lshift__(input, shift_amount); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::__lshift__(lazy_input, shift_amount); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestLshiftScalarInPlace) { + torch::Tensor input = torch::ones( + {4, 2}, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Scalar shift_amount = 3; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor result = input.__ilshift__(shift_amount); + torch::Tensor lazy_result = lazy_input.__ilshift__(shift_amount); + AllClose(result, lazy_result); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestRshift) { + torch::Tensor input = torch::ones( + {4, 2}, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Tensor shift_amount = torch::randint( + 16, + input.sizes(), + torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Tensor result = torch::__rshift__(input, shift_amount); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_shift_amount = CopyToDevice(shift_amount, device); + torch::Tensor lazy_result = + torch::__rshift__(lazy_input, lazy_shift_amount); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestRshiftInPlace) { + torch::Tensor input = torch::ones( + {4, 2}, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor shift_amount = torch::randint( + 16, + input.sizes(), + torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Tensor result = input.__irshift__(shift_amount); + torch::Tensor lazy_shift_amount = CopyToDevice(shift_amount, device); + torch::Tensor lazy_result = lazy_input.__irshift__(lazy_shift_amount); + AllClose(result, lazy_result); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestRshiftScalar) { + torch::Tensor input = torch::ones( + {4, 2}, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Scalar shift_amount = 3; + torch::Tensor result = torch::__rshift__(input, shift_amount); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_result = torch::__rshift__(lazy_input, shift_amount); + AllClose(result, lazy_result); + }); +} + +TEST_F(LazyOpsTest, TestRshiftScalarInPlace) { + torch::Tensor input = torch::ones( + {4, 2}, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Scalar shift_amount = 3; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor result = input.__irshift__(shift_amount); + torch::Tensor lazy_result = lazy_input.__irshift__(shift_amount); + AllClose(result, lazy_result); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestMeshgrid) { + torch::Tensor a = torch::rand( + {3}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor b = torch::rand( + {2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor c = torch::rand( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + auto d = torch::meshgrid({a, b, c}); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_a = CopyToDevice(a, device); + torch::Tensor lazy_b = CopyToDevice(b, device); + torch::Tensor lazy_c = CopyToDevice(c, device); + auto lazy_d = torch::meshgrid({lazy_a, lazy_b, lazy_c}); + EXPECT_EQ(d.size(), lazy_d.size()); + for (size_t i = 0; i < d.size(); ++i) { + AllClose(d[i], lazy_d[i]); + } + }); +} + +TEST_F(LazyOpsTest, TestConstantPad) { + torch::Tensor input = torch::rand( + {4, 2, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector pad{1, 2, 3, 4, 5, 6}; + float pad_value = 5; + torch::Tensor output = torch::constant_pad_nd(input, pad, pad_value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = + torch::constant_pad_nd(lazy_input, pad, pad_value); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestConstantPadIncomplete) { + torch::Tensor input = torch::rand( + {4, 2, 5}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector pad{1, 2}; + float pad_value = 5; + torch::Tensor output = torch::constant_pad_nd(input, pad, pad_value); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = + torch::constant_pad_nd(lazy_input, pad, pad_value); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestReflectionPad2dRank3) { + torch::Tensor input = torch::rand( + {2, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector pad{2, 2, 2, 2}; + torch::Tensor output = torch::reflection_pad2d(input, pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::reflection_pad2d(lazy_input, pad); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestReflectionPad2dRank4) { + torch::Tensor input = torch::rand( + {2, 2, 3, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector pad{2, 2, 2, 2}; + torch::Tensor output = torch::reflection_pad2d(input, pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::reflection_pad2d(lazy_input, pad); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestReflectionPad2dBackward) { + std::vector pad{2, 3, 1, 2}; + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::reflection_pad2d(inputs[0], pad); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {1, 2, 4, 4}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestReplicationPad1d) { + torch::Tensor input = torch::rand( + {1, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector pad{1, 2}; + torch::Tensor output = torch::replication_pad1d(input, pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::replication_pad1d(lazy_input, pad); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestReplicationPad1dZeroPad) { + torch::Tensor input = torch::rand( + {1, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector pad{1, 0}; + torch::Tensor output = torch::replication_pad1d(input, pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::replication_pad1d(lazy_input, pad); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestReplicationPad1dBackward) { + std::vector pad{2, 3}; + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::replication_pad1d(inputs[0], pad); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 4}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestReplicationPad2d) { + torch::Tensor input = torch::rand( + {1, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector pad{1, 2, 2, 1}; + torch::Tensor output = torch::replication_pad2d(input, pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::replication_pad2d(lazy_input, pad); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestReplicationPad2dZeroPad) { + torch::Tensor input = torch::rand( + {1, 3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector pad{1, 0, 0, 1}; + torch::Tensor output = torch::replication_pad2d(input, pad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::replication_pad2d(lazy_input, pad); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestReplicationPad2dBackward) { + std::vector pad{2, 3, 1, 1}; + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::replication_pad2d(inputs[0], pad); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 3, 4}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestAsStrided) { + torch::Tensor input = torch::rand( + {128, 320}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector size = {128, 20, 4, 4}; + std::vector stride = {320, 16, 4, 1}; + torch::Tensor output = + torch::as_strided(input, /*size=*/size, /*stride=*/stride); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = + torch::as_strided(lazy_input, /*size=*/size, /*stride=*/stride); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestAsStridedInPlace) { + torch::Tensor input = torch::rand( + {128, 320}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector size = {128, 20, 4, 4}; + std::vector stride = {320, 16, 4, 1}; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor output = + torch::as_strided_(input, /*size=*/size, /*stride=*/stride); + torch::Tensor lazy_output = + torch::as_strided_(lazy_input, /*size=*/size, /*stride=*/stride); + AllClose(output, lazy_output); + AllClose(input, lazy_input); + }); +} + +TEST_F(LazyOpsTest, TestAsStridedWithOffset) { + torch::Tensor input = torch::rand( + {4, 8, 2}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector size = {4, 4, 2}; + std::vector stride = {8, 2, 1}; + int64_t storage_offset = 4; + torch::Tensor output = torch::as_strided( + input, + /*size=*/size, + /*stride=*/stride, + /*storage_offset=*/storage_offset); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_output = torch::as_strided( + lazy_input, + /*size=*/size, + /*stride=*/stride, + /*storage_offset=*/storage_offset); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestAsStridedWithInplaceCopy) { + torch::Tensor grad = torch::ones( + {4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + std::vector size = {4}; + std::vector stride = {1}; + torch::Tensor output = torch::zeros({4}, grad.options()); + output.as_strided(size, stride).copy_(grad); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_grad = CopyToDevice(grad, device); + torch::Tensor lazy_output = torch::zeros({4}, lazy_grad.options()); + lazy_output.as_strided(size, stride).copy_(lazy_grad); + AllClose(output, lazy_output); + }); +} + +TEST_F(LazyOpsTest, TestEmptyStrided) { + std::vector size = {4, 4, 2}; + std::vector stride = {8, 2, 1}; + torch::Tensor output = torch::empty_strided(/*size=*/size, /*stride=*/stride); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_output = + torch::empty_strided(/*size=*/size, /*stride=*/stride); + EXPECT_EQ(output.sizes(), lazy_output.sizes()); + EXPECT_EQ(output.strides(), lazy_output.strides()); + }); +} + +TEST_F(LazyOpsTest, TestAvgPool2DBackward) { + int kernel_size = 2; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::avg_pool2d( + inputs[0], + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {1, 1, 7, 7}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAvgPool3DBackward) { + int kernel_size = 2; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::avg_pool3d( + inputs[0], + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {1, 1, 7, 7, 7}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAvgPool2DNoBatchBackward) { + int kernel_size = 2; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::avg_pool2d( + inputs[0], + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {1, 7, 7}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAvgPool3DNoBatchBackward) { + int kernel_size = 2; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (bool count_include_pad : {true, false}) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::avg_pool3d( + inputs[0], + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*ceil_mode=*/ceil_mode, + /*count_include_pad=*/count_include_pad); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {1, 7, 7, 7}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestAdaptiveAvgPool3DNoBatchBackward) { + if (IsCuda()) { + GTEST_SKIP(); + } + for (int64_t output_size : {7, 4}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::adaptive_avg_pool3d( + inputs[0], {output_size, output_size, output_size}); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {1, 56, 28, 28}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } +} + +TEST_F(LazyOpsTest, TestAdaptiveAvgPool3DBackward) { + if (IsCuda()) { + GTEST_SKIP(); + } + for (int64_t output_size : {7, 4}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::adaptive_avg_pool3d( + inputs[0], {output_size, output_size, output_size}); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {4, 1, 56, 28, 28}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } +} + +TEST_F(LazyOpsTest, TestAdaptiveAvgPool2DBackward) { + for (int64_t output_size : {7, 8}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::adaptive_avg_pool2d(inputs[0], {output_size, output_size}); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {4, 1, 56, 56}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } +} + +TEST_F(LazyOpsTest, TestAdaptiveAvgPool2DNoBatchBackward) { + for (int64_t output_size : {7, 8}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::adaptive_avg_pool2d(inputs[0], {output_size, output_size}); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {1, 56, 56}, + torch::TensorOptions(torch::kFloat).requires_grad(true))}, + device, + testfn); + }); + } +} + +TEST_F(LazyOpsTest, TestConv2D) { + int in_channels = 4; + int out_channels = 4; + int kernel_size = 3; + for (int stride = 1; stride <= 3; ++stride) { + for (int padding = 0; padding <= 2; ++padding) { + for (bool with_bias : {true, false}) { + for (int dilation = 1; dilation <= 3; ++dilation) { + for (int groups : + {1, 2, 4}) { // covers normal, grouped, depthwise conv. + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::rand( + {1, in_channels, 7, 7}, + torch::TensorOptions(torch::kDouble).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {out_channels, + in_channels / groups, + kernel_size, + kernel_size}, + torch::TensorOptions(torch::kDouble).device(DefaultDevice())); + torch::Tensor bias = with_bias + ? torch::rand( + {out_channels}, + torch::TensorOptions(torch::kDouble) + .device(DefaultDevice())) + : torch::Tensor(); + + torch::Tensor lazy_input = CopyToDevice(input, device); + torch::Tensor lazy_weight = CopyToDevice(weight, device); + torch::Tensor lazy_bias = + with_bias ? CopyToDevice(bias, device) : torch::Tensor(); + + torch::Tensor output = torch::conv2d( + input, + weight, + bias, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{dilation, dilation}, + groups); + torch::Tensor lazy_output = torch::conv2d( + lazy_input, + lazy_weight, + lazy_bias, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{dilation, dilation}, + groups); + AllClose(output, lazy_output); + }); + } + } + } + } + } +} + +TEST_F(LazyOpsTest, TestConv2DBackward) { + int in_channels = 4; + int out_channels = 4; + int kernel_size = 3; + for (int stride = 1; stride <= 3; ++stride) { + for (int padding = 0; padding <= 2; ++padding) { + for (bool with_bias : {true, false}) { + for (int dilation = 1; dilation <= 3; ++dilation) { + for (int groups : + {1, 2, 4}) { // covers normal, grouped, depthwise conv. + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::conv2d( + inputs[0], + inputs[1], + inputs[2], + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{dilation, dilation}, + groups); + }; + + ForEachDevice([&](const torch::Device& device) { + torch::Tensor bias = with_bias + ? torch::rand( + {out_channels}, + torch::TensorOptions(torch::kDouble) + .device(DefaultDevice())) + : torch::Tensor(); + TestBackward( + {torch::rand( + {1, in_channels, 7, 7}, + torch::TensorOptions(torch::kDouble) + .device(DefaultDevice()) + .requires_grad(true)), + torch::rand( + {out_channels, + in_channels / groups, + kernel_size, + kernel_size}, + torch::TensorOptions(torch::kDouble) + .device(DefaultDevice()) + .requires_grad(true)), + bias}, + device, + testfn); + }); + } + }; + } + } + } +} + +TEST_F(LazyOpsTest, TestTransposedConv2DBackward) { + int in_channels = 4; + int out_channels = 4; + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (int dilation = 1; dilation <= 2; ++dilation) { + for (int output_padding = 0; + output_padding < std::max(stride, dilation); + ++output_padding) { + for (bool with_bias : {true, false}) { + for (int groups : + {1, 2, 4}) { // covers normal, grouped, depthwise conv. + auto testfn = [&](const std::vector& inputs) + -> torch::Tensor { + return torch::conv_transpose2d( + inputs[0], + inputs[1], + inputs[2], + /*stride=*/{stride, stride + 1}, + /*padding=*/{padding, padding + 1}, + /*output_padding=*/output_padding, + /*groups=*/groups, + /*dilation=*/{dilation, dilation + 1}); + }; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::rand( + {4, out_channels, 7, 7}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor weight = torch::rand( + {out_channels, + in_channels / groups, + kernel_size, + kernel_size}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor bias = with_bias + ? torch::rand( + {in_channels}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)) + : torch::Tensor(); + TestBackward( + {input, weight, bias}, + device, + testfn, + /*rtol=*/1e-5, + /*atol=*/1e-5); + }); + } + }; + } + } + } + } +} + +TEST_F(LazyOpsTest, TestConv3DBackward) { + int in_channels = 4; + int out_channels = 4; + int kernel_size = 3; + for (int stride = 1; stride <= 3; ++stride) { + for (int padding = 1; padding <= 2; ++padding) { + for (bool with_bias : {true, false}) { + for (int dilation = 1; dilation <= 2; ++dilation) { + for (int groups : + {1, 2, 4}) { // covers normal, grouped, depthwise conv. + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::conv3d( + inputs[0], + inputs[1], + inputs[2], + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*dilation=*/{dilation, dilation, dilation}, + groups); + }; + + ForEachDevice([&](const torch::Device& device) { + torch::Tensor bias = with_bias + ? torch::rand( + {out_channels}, + torch::TensorOptions(torch::kDouble) + .device(DefaultDevice())) + : torch::Tensor(); + TestBackward( + {torch::rand( + {4, in_channels, 7, 7, 7}, + torch::TensorOptions(torch::kDouble) + .device(DefaultDevice()) + .requires_grad(true)), + torch::rand( + {out_channels, + in_channels / groups, + kernel_size, + kernel_size, + kernel_size}, + torch::TensorOptions(torch::kDouble) + .device(DefaultDevice()) + .requires_grad(true)), + bias}, + device, + testfn); + }); + } + }; + } + } + } +} + +TEST_F(LazyOpsTest, TestTransposedConv3DBackward) { + int in_channels = 4; + int out_channels = 4; + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + for (int dilation = 1; dilation <= 2; ++dilation) { + for (int output_padding = 0; + output_padding < std::max(stride, dilation); + ++output_padding) { + for (bool with_bias : {true, false}) { + for (int groups : + {1, 2, 4}) { // covers normal, grouped, depthwise conv. + auto testfn = [&](const std::vector& inputs) + -> torch::Tensor { + return torch::conv_transpose3d( + inputs[0], + inputs[1], + inputs[2], + /*stride=*/{stride, stride + 1, stride}, + /*padding=*/{padding, padding + 1, stride}, + /*output_padding=*/output_padding, + /*groups=*/groups, + /*dilation=*/{dilation, dilation + 1, dilation}); + }; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::rand( + {4, out_channels, 7, 7, 7}, + torch::TensorOptions(torch::kDouble) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor weight = torch::rand( + {out_channels, + in_channels / groups, + kernel_size, + kernel_size, + kernel_size}, + torch::TensorOptions(torch::kDouble) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor bias = with_bias + ? torch::rand( + {in_channels}, + torch::TensorOptions(torch::kDouble) + .device(DefaultDevice()) + .requires_grad(true)) + : torch::Tensor(); + TestBackward({input, weight, bias}, device, testfn); + }); + } + }; + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool2DBackward) { + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::max_pool2d( + inputs[0], + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{1, 1}, + /*ceil_mode=*/ceil_mode); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {1, 2, 8, 8}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool3DBackward) { + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::max_pool3d( + inputs[0], + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*dilation=*/{1, 1, 1}, + /*ceil_mode=*/ceil_mode); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {1, 2, 4, 4, 4}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool2DNoBatchBackward) { + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::max_pool2d( + inputs[0], + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{1, 1}, + /*ceil_mode=*/ceil_mode); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 8, 8}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxPool3DNoBatchBackward) { + int kernel_size = 3; + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::max_pool3d( + inputs[0], + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*dilation=*/{1, 1, 1}, + /*ceil_mode=*/ceil_mode); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 4, 4, 4}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxUnpool2DBackward) { + int kernel_size = 2; + torch::Tensor input = torch::rand( + {2, 2, 8, 8}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output; + torch::Tensor indices; + std::tie(output, indices) = torch::max_pool2d_with_indices( + input, + /*kernel_size=*/{kernel_size, kernel_size}, + /*stride=*/{stride, stride}, + /*padding=*/{padding, padding}, + /*dilation=*/{dilation, dilation}, + /*ceil_mode=*/ceil_mode); + + std::vector output_size({input.size(2), input.size(3)}); + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::max_unpool2d(inputs[0], inputs[1], output_size); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {output.requires_grad_(true), indices}, device, testfn); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestMaxUnpool3DBackward) { + int kernel_size = 2; + torch::Tensor input = torch::rand( + {1, 1, 4, 4, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (int stride = 1; stride <= 2; ++stride) { + for (int padding = 0; padding <= 1; ++padding) { + // Test ceil_mode=true through the CPU interop. + for (bool ceil_mode : {false, true}) { + for (int dilation = 1; dilation <= 2; ++dilation) { + torch::Tensor output; + torch::Tensor indices; + std::tie(output, indices) = torch::max_pool3d_with_indices( + input, + /*kernel_size=*/{kernel_size, kernel_size, kernel_size}, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}, + /*dilation=*/{dilation, dilation, dilation}, + /*ceil_mode=*/ceil_mode); + + std::vector output_size( + {input.size(2), input.size(3), input.size(4)}); + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::max_unpool3d( + inputs[0], + inputs[1], + output_size, + /*stride=*/{stride, stride, stride}, + /*padding=*/{padding, padding, padding}); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {output.requires_grad_(true), indices}, device, testfn); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestTanhBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::tanh(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 2}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn, + /*rtol=*/1e-3, + /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestSigmoidBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::sigmoid(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 2}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestLogSigmoidBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::log_sigmoid(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 2}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn, + /*rtol=*/1e-3, + /*atol=*/1e-5); + }); +} + +TEST_F(LazyOpsTest, TestLogSoftmaxBackward) { + for (int dim = -4; dim < 4; ++dim) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::log_softmax(inputs[0], dim); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {5, 3, 4, 2}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn, + /*rtol=*/1e-3, + /*atol=*/1e-4); + }); + } +} + +TEST_F(LazyOpsTest, TestSoftmaxBackward) { + for (int dim = -4; dim < 4; ++dim) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::softmax(inputs[0], dim); + }; + + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {5, 3, 4, 2}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn, + /*rtol=*/1e-3, + /*atol=*/1e-4); + }); + } +} + +TEST_F(LazyOpsTest, TestSoftplusBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::softplus(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn, + /*rtol=*/1e-4); + }); +} + +TEST_F(LazyOpsTest, TestReluBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::relu(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestRreluBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::rrelu(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestHardshrinkBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::hardshrink(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::randn( + {100}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestSoftshrinkBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::softshrink(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::randn( + {100}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestHardtanhBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::hardtanh(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::randn( + {100}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestEluBackward) { + torch::Scalar alpha = 0.5; + torch::Scalar scale = 2.5; + torch::Scalar input_scale = 1.5; + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::elu(inputs[0], alpha, scale, input_scale); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestGeluBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::gelu(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 3}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + ExpectCounterChanged("lazy::gelu_backward", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestLeakyReluBackward) { + double negative_slope = 0.01; + auto testfn = [=](const std::vector& inputs) -> torch::Tensor { + return torch::leaky_relu(inputs[0], negative_slope); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 1, 4, 6}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestTransposeBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::t(inputs[0]); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {2, 3}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestAddMatMulBackward) { + int in_channels = 32; + int out_channels = 320; + int labels = 50; + // Test beta != 1. through the CPU interop. + for (double beta : {1., 2.}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::addmm(inputs[0], inputs[1], inputs[2], /*beta=*/beta); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {labels}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)), + torch::rand( + {in_channels, out_channels}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)), + torch::rand( + {out_channels, labels}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); + } +} + +TEST_F(LazyOpsTest, TestBinaryCrossEntropyBackward) { + int batch = 6; + int classes = 2; + // TODO(asuhan): Fix the torch::kDouble case. + for (auto dtype : {torch::kFloat}) { + for (bool def_weight : {false, true}) { + torch::Tensor input = torch::rand( + {batch, classes}, torch::TensorOptions(dtype).requires_grad(true)); + torch::Tensor target = + torch::rand({batch, classes}, torch::TensorOptions(dtype)); + torch::Tensor weight; + if (def_weight) { + weight = torch::rand({batch, classes}, torch::TensorOptions(dtype)); + } + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, + torch::Reduction::Sum, + torch::Reduction::None}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::binary_cross_entropy( + /*self=*/inputs[0], + /*target=*/inputs[1], + /*weight=*/inputs[2], + /*reduction=*/reduction); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {input, target, weight}, + device, + testfn, + /*rtol=*/1e-4, + /*atol=*/1e-7); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestNllLossBackward) { + int batch = 6; + int classes = 2; + // TODO(asuhan): Fix the torch::kDouble case. + for (auto dtype : {torch::kFloat}) { + for (int ignore_index : {-1, 0, 1, 5}) { + for (bool def_weight : {false, true}) { + torch::Tensor input = torch::rand( + {batch, classes}, + torch::TensorOptions(dtype) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor target = torch::randint( + std::min(ignore_index, 0), + classes, + {batch}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor weight; + if (def_weight) { + weight = torch::rand( + {classes}, torch::TensorOptions(dtype).device(DefaultDevice())); + } + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, + torch::Reduction::Sum, + torch::Reduction::None}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::nll_loss( + /*self=*/inputs[0], + /*target=*/inputs[1], + /*weight=*/inputs[2], + /*reduction=*/reduction, + /*ignore_index=*/ignore_index); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {input, target, weight}, + device, + testfn, + /*rtol=*/1e-5, + /*atol=*/1e-8); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestNllLoss2dBackward) { + int batch = 6; + int classes = 2; + int height = 3; + int width = 3; + // TODO(asuhan): Fix the torch::kDouble case. + for (auto dtype : {torch::kFloat}) { + for (int ignore_index : {-1, 0, 1, 5}) { + for (bool def_weight : {false, true}) { + torch::Tensor input = torch::rand( + {batch, classes, height, width}, + torch::TensorOptions(dtype) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor target = torch::randint( + std::min(ignore_index, 0), + classes, + {batch, height, width}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + torch::Tensor weight; + if (def_weight) { + weight = torch::rand( + {classes}, torch::TensorOptions(dtype).device(DefaultDevice())); + } + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, + torch::Reduction::Sum, + torch::Reduction::None}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::nll_loss2d( + /*self=*/inputs[0], + /*target=*/inputs[1], + /*weight=*/inputs[2], + /*reduction=*/reduction, + /*ignore_index=*/ignore_index); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {input, target, weight}, + device, + testfn, + /*rtol=*/1e-5, + /*atol=*/1e-8); + }); + } + } + } + } +} + +TEST_F(LazyOpsTest, TestSmoothL1LossBackward) { + torch::Tensor input = torch::randn( + {2, 4}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor target = torch::randn( + {2, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + for (torch::Reduction::Reduction reduction : + {torch::Reduction::None, + torch::Reduction::Mean, + torch::Reduction::Sum}) { + for (double beta : {0.25, 1.}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::smooth_l1_loss( + /*input=*/inputs[0], + /*target=*/inputs[1], + /*reduction=*/reduction, + /*beta=*/beta); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {input, target}, + device, + testfn, + /*rtol=*/1e-5, + /*atol=*/1e-8); + }); + } + } +} + +TEST_F(LazyOpsTest, TestViewBackward) { + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return inputs[0].view({-1, 320}); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {torch::rand( + {32, 20, 4, 4}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true))}, + device, + testfn); + }); +} + +TEST_F(LazyOpsTest, TestBatchNorm2DBackward) { + double momentum = 0.1; + double eps = 0.5; + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::batch_norm( + /*input=*/inputs[0], + /*weight=*/inputs[1], + /*bias=*/inputs[2], + /*running_mean=*/inputs[3], + /*running_var=*/inputs[4], + /*training=*/true, + /*momentum=*/momentum, + /*eps=*/eps, + /*cudnn_enabled=*/false); + }; + int num_features = 3; + torch::Tensor undef; + for (bool undef_weight_bias : {false, true}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::rand( + {2, num_features, 4, 4}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor weight = undef_weight_bias + ? undef + : torch::rand( + {num_features}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor bias = undef_weight_bias + ? undef + : torch::rand( + {num_features}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor running_mean = torch::zeros( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor running_var = torch::ones( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + TestBackward( + {input, weight, bias, running_mean, running_var}, + device, + testfn, + /*rtol=*/1e-3, + /*atol=*/1e-4); + }); + } +} + +TEST_F(LazyOpsTest, TestBatchNorm3DBackward) { + double momentum = 0.1; + double eps = 0.5; + auto testfn = [&](const std::vector& inputs) -> torch::Tensor { + return torch::batch_norm( + /*input=*/inputs[0], + /*weight=*/inputs[1], + /*bias=*/inputs[2], + /*running_mean=*/inputs[3], + /*running_var=*/inputs[4], + /*training=*/true, + /*momentum=*/momentum, + /*eps=*/eps, + /*cudnn_enabled=*/false); + }; + int num_features = 3; + torch::Tensor undef; + for (bool undef_weight_bias : {false, true}) { + ForEachDevice([&](const torch::Device& device) { + torch::Tensor input = torch::rand( + {2, num_features, 4, 4, 2}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor weight = undef_weight_bias + ? undef + : torch::rand( + {num_features}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor bias = undef_weight_bias + ? undef + : torch::rand( + {num_features}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor running_mean = torch::zeros( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor running_var = torch::ones( + {num_features}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + TestBackward( + {input, weight, bias, running_mean, running_var}, + device, + testfn, + /*rtol=*/1e-3, + /*atol=*/1e-3); + }); + } +} + +TEST_F(LazyOpsTest, TestBCEWithLogitsBackward) { + int batch = 10; + int classes = 5; + torch::Tensor undef; + for (torch::Reduction::Reduction reduction : + {torch::Reduction::None, + torch::Reduction::Mean, + torch::Reduction::Sum}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::binary_cross_entropy_with_logits( + /*input=*/inputs[0], + /*target=*/inputs[1], + /*weight=*/inputs[2], + /*pos_weight=*/inputs[3], + /*reduction=*/reduction); + }; + for (bool undef_weight : {false, true}) { + for (bool undef_pos_weight : {false, true}) { + torch::Tensor input = torch::rand( + {batch, classes}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor target = torch::rand( + {batch, classes}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor weight = undef_weight + ? undef + : torch::rand( + {classes}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor pos_weight = undef_pos_weight + ? undef + : torch::rand( + {classes}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {input, target, weight, pos_weight}, + device, + testfn, + /*rtol=*/1e-3, + /*atol=*/1e-5); + }); + } + } + } +} + +TEST_F(LazyOpsTest, TestKlDivBackward) { + torch::Tensor input = torch::rand( + {4, 3}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor target = torch::rand( + {4, 3}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + for (torch::Reduction::Reduction reduction : + {torch::Reduction::Mean, + torch::Reduction::Sum, + torch::Reduction::None}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::kl_div(/*self=*/inputs[0], /*target=*/inputs[1], reduction); + }; + ForEachDevice([&](const torch::Device& device) { + TestBackward( + {input, target}, + device, + testfn, + /*rtol=*/1e-4, + /*atol=*/1e-5); + }); + } +} + +TEST_F(LazyOpsTest, TestEmbeddingBackward) { + int num_weights = 32; + for (int padding_idx = -1; padding_idx < num_weights; ++padding_idx) { + for (bool scale_grad_by_freq : {false, true}) { + auto testfn = + [&](const std::vector& inputs) -> torch::Tensor { + return torch::embedding( + inputs[0], + inputs[1], + /*padding_idx=*/padding_idx, + /*scale_grad_by_freq=*/scale_grad_by_freq, + /*sparse=*/false); + }; + ForEachDevice([&](const torch::Device& device) { + torch::Tensor weight = torch::rand( + {num_weights, 7}, + torch::TensorOptions(torch::kFloat) + .device(DefaultDevice()) + .requires_grad(true)); + torch::Tensor indices = torch::randint( + num_weights, + {3, 9, 4}, + torch::TensorOptions(torch::kLong).device(DefaultDevice())); + TestBackward( + {weight, indices}, + device, + testfn, + /*rtol=*/1e-5, + /*atol=*/1e-8); + }); + } + } +} + +TEST_F(LazyOpsTest, TestAmpForeachNonFiniteCheckAndUnscale) { + if (IsCuda()) { + // TODO(whc) debug failure on cuda + GTEST_SKIP(); + } + + torch::Tensor grads0 = torch::tensor( + {1, 2, 3, 4}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor grads1 = torch::tensor( + {1.0, 2.0, std::nan("1"), 4.0}, + torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor inv_scale = torch::scalar_tensor( + 0.2, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor found_inf = torch::scalar_tensor( + 0, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor grads_output0 = grads0 * inv_scale; + torch::Tensor found_inf_output0 = torch::scalar_tensor( + 0, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor found_inf_output1 = torch::scalar_tensor( + 1, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ForEachDevice([&](const torch::Device& device) { + if (grads0.device() == at::kCPU) { + GTEST_SKIP(); + } + torch::Tensor lazy_grads0 = CopyToDevice(grads0, device); + torch::Tensor lazy_inv_scale = CopyToDevice(inv_scale, device); + torch::Tensor lazy_found_inf = CopyToDevice(found_inf, device); + torch::_amp_foreach_non_finite_check_and_unscale_( + lazy_grads0, lazy_found_inf, lazy_inv_scale); + AllClose(grads_output0, lazy_grads0, /*rtol=*/1e-2, /*atol=*/1e-4); + AllEqual(found_inf_output0, lazy_found_inf); + + torch::Tensor lazy_grads1 = CopyToDevice(grads1, device); + torch::_amp_foreach_non_finite_check_and_unscale_( + lazy_grads1, lazy_found_inf, lazy_inv_scale); + AllEqual(found_inf_output1, lazy_found_inf); + }); +} + +TEST_F(LazyOpsTest, TestAmpUpdateScale) { + torch::Tensor growth_tracker = torch::scalar_tensor( + 0, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Tensor current_scale = torch::scalar_tensor( + 4, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor found_inf = torch::scalar_tensor( + 1, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor not_found_inf = torch::scalar_tensor( + 0, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + float scale_growth_factor = 2.0; + float scale_backoff_factor = 0.5; + int growth_interval = 3; + + torch::Tensor growth_tracker_result0 = torch::scalar_tensor( + 1, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Tensor current_scale_result0 = torch::scalar_tensor( + 4, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor growth_tracker_result1 = torch::scalar_tensor( + 2, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Tensor current_scale_result1 = torch::scalar_tensor( + 4, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor growth_tracker_result2 = torch::scalar_tensor( + 0, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Tensor current_scale_result2 = torch::scalar_tensor( + 8, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor growth_tracker_result3 = torch::scalar_tensor( + 0, torch::TensorOptions(torch::kInt32).device(DefaultDevice())); + torch::Tensor current_scale_result3 = torch::scalar_tensor( + 4, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + + ForEachDevice([&](const torch::Device& device) { + if (growth_tracker.device() == at::kCPU) { + GTEST_SKIP(); + } + torch::Tensor lazy_growth_tracker = CopyToDevice(growth_tracker, device); + torch::Tensor lazy_current_scale = CopyToDevice(current_scale, device); + torch::Tensor lazy_found_inf = CopyToDevice(found_inf, device); + torch::Tensor lazy_not_found_inf = CopyToDevice(not_found_inf, device); + + torch::_amp_update_scale_( + lazy_current_scale, + lazy_growth_tracker, + lazy_not_found_inf, + scale_growth_factor, + scale_backoff_factor, + growth_interval); + AllClose( + current_scale_result0, + lazy_current_scale, + /*rtol=*/1e-2, + /*atol=*/1e-4); + AllEqual(growth_tracker_result0, lazy_growth_tracker); + + torch::_amp_update_scale_( + lazy_current_scale, + lazy_growth_tracker, + lazy_not_found_inf, + scale_growth_factor, + scale_backoff_factor, + growth_interval); + AllClose( + current_scale_result1, + lazy_current_scale, + /*rtol=*/1e-2, + /*atol=*/1e-4); + AllEqual(growth_tracker_result1, lazy_growth_tracker); + + // torch::_amp_update_scale_ returns the reference of current_scale + lazy_current_scale = torch::_amp_update_scale_( + lazy_current_scale, + lazy_growth_tracker, + lazy_not_found_inf, + scale_growth_factor, + scale_backoff_factor, + growth_interval); + AllClose( + current_scale_result2, + lazy_current_scale, + /*rtol=*/1e-2, + /*atol=*/1e-4); + AllEqual(growth_tracker_result2, lazy_growth_tracker); + + lazy_current_scale = torch::_amp_update_scale_( + lazy_current_scale, + lazy_growth_tracker, + lazy_found_inf, + scale_growth_factor, + scale_backoff_factor, + growth_interval); + AllClose( + current_scale_result3, + lazy_current_scale, + /*rtol=*/1e-2, + /*atol=*/1e-4); + AllEqual(growth_tracker_result3, lazy_growth_tracker); + }); + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("lazy::_amp_update_scale_", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestEarlySyncLiveTensors) { + torch::Tensor scalar_tensor = torch::scalar_tensor( + 1., torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar scalar1 = scalar_tensor.item(); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_scalar_tensor = CopyToDevice(scalar_tensor, device); + torch::Scalar scalar2 = lazy_scalar_tensor.item(); + ASSERT_EQ(scalar1.to(), scalar2.to()); + }); + if (DebugUtil::ExperimentEnabled("early_sync")) { + ExpectCounterChanged("EarlySyncLiveTensorsCount", GetIgnoredCounters()); + } else { + ExpectCounterNotChanged("EarlySyncLiveTensorsCount", GetIgnoredCounters()); + } + ExpectCounterChanged("aten::_local_scalar_dense", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestLerp) { + torch::Tensor start = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor end = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor res = torch::lerp(start, end, weight); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_start = CopyToDevice(start, device); + torch::Tensor lazy_end = CopyToDevice(end, device); + torch::Tensor lazy_weight = CopyToDevice(weight, device); + torch::Tensor lazy_res = torch::lerp(lazy_start, lazy_end, lazy_weight); + AllClose(res, lazy_res); + }); + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("lazy::lerp", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestLerpScalar) { + torch::Tensor start = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor end = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar weight = torch::Scalar(3.0); + torch::Tensor res = torch::lerp(start, end, weight); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_start = CopyToDevice(start, device); + torch::Tensor lazy_end = CopyToDevice(end, device); + torch::Tensor lazy_res = torch::lerp(lazy_start, lazy_end, weight); + AllClose(res, lazy_res); + }); + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("lazy::lerp", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestLerpInplace) { + torch::Tensor input = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor end = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor input_copy = input.clone(); + input.lerp_(end, weight); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input_copy, device); + torch::Tensor lazy_end = CopyToDevice(end, device); + torch::Tensor lazy_weight = CopyToDevice(weight, device); + lazy_input.lerp_(lazy_end, lazy_weight); + AllClose(lazy_input, input); + }); + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("lazy::lerp", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestLerpScalarInplace) { + torch::Tensor input = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor end = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar weight = torch::Scalar(3.0); + torch::Tensor input_copy = input.clone(); + input.lerp_(end, weight); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_input = CopyToDevice(input_copy, device); + torch::Tensor lazy_end = CopyToDevice(end, device); + lazy_input.lerp_(lazy_end, weight); + AllClose(lazy_input, input); + }); + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("lazy::lerp", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestLerpOut) { + torch::Tensor start = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor end = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor weight = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor res = torch::empty( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + ; + torch::lerp_out(res, start, end, weight); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_start = CopyToDevice(start, device); + torch::Tensor lazy_end = CopyToDevice(end, device); + torch::Tensor lazy_weight = CopyToDevice(weight, device); + torch::Tensor lazy_res = torch::empty({3, 4}, lazy_start.options()); + torch::lerp_out(lazy_res, lazy_start, lazy_end, lazy_weight); + AllClose(res, lazy_res); + }); + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("lazy::lerp", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, TestLerpScalarOut) { + torch::Tensor start = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Tensor end = torch::rand( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::Scalar weight = torch::Scalar(3.0); + torch::Tensor res = torch::empty( + {3, 4}, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + torch::lerp_out(res, start, end, weight); + ForEachDevice([&](const torch::Device& device) { + torch::Tensor lazy_start = CopyToDevice(start, device); + torch::Tensor lazy_end = CopyToDevice(end, device); + torch::Tensor lazy_res = torch::empty({3, 4}, lazy_start.options()); + torch::lerp_out(lazy_res, lazy_start, lazy_end, weight); + AllClose(res, lazy_res); + }); + ExpectCounterNotChanged("aten::.*", GetIgnoredCounters()); + ExpectCounterChanged("lazy::lerp", GetIgnoredCounters()); +} + +TEST_F(LazyOpsTest, IsAliasOf) { + auto a = torch::empty( + 4, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + auto b = torch::empty( + 4, torch::TensorOptions(torch::kFloat).device(DefaultDevice())); + + ForEachDevice([&](const torch::Device& device) { + auto lazy_a = CopyToDevice(a, device); + auto lazy_b = CopyToDevice(b, device); + EXPECT_EQ(!a.is_alias_of(b), !lazy_a.is_alias_of(lazy_b)); + + auto c = a.view({2, 2}); + auto lazy_c = lazy_a.view({2, 2}); + EXPECT_EQ(a.is_alias_of(c), lazy_a.is_alias_of(lazy_c)); + + auto d = c.view({1, 4}); + auto lazy_d = lazy_c.view({1, 4}); + EXPECT_EQ(d.is_alias_of(c), lazy_d.is_alias_of(lazy_c)); + EXPECT_EQ(d.is_alias_of(a), lazy_d.is_alias_of(lazy_a)); + }); +} + +#endif // FBCODE_CAFFE2 + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_ops_util.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_ops_util.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b383d6223af9490cc4db6b72c33608fca0195046 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_ops_util.cpp @@ -0,0 +1,202 @@ +#include + +#include +#include +#include +#include + +#include +#include + +namespace torch { +namespace lazy { +namespace { + +std::unordered_set* CreateIgnoredCounters() { + std::unordered_set* icounters = + new std::unordered_set(); + // Add below the counters whose name need to be ignored when doing + // is-any-counter-changed assertions. + icounters->insert("aten::rand"); + return icounters; +} + +} // namespace + +const std::unordered_set* GetIgnoredCounters() { + static const std::unordered_set* icounters = + CreateIgnoredCounters(); + return icounters; +} + +at::Tensor ToCpuTensor(const at::Tensor& tensor) { + // tensor.to() implicitly triggers a sync if t.device=torch::kLazy. + return tensor.to(torch::kCPU); +} + +torch::Tensor CopyToDevice( + const torch::Tensor& tensor, + const torch::Device& device) { + return tensor.clone().to(device, /*non_blocking=*/false, /*copy=*/true); +} + +bool EqualValues(at::Tensor tensor1, at::Tensor tensor2) { + tensor1 = ToCpuTensor(tensor1); + tensor2 = ToCpuTensor(tensor2); + if (torch::isnan(tensor1).any().item()) { + EXPECT_TRUE(EqualValues(torch::isnan(tensor1), torch::isnan(tensor2))); + tensor1.nan_to_num_(); + tensor2.nan_to_num_(); + } + if (tensor1.sizes() != tensor2.sizes() || + tensor1.dtype() != tensor2.dtype()) { + std::cerr << "Different shape:\n" + << tensor1.dtype() << ' ' << tensor1.sizes() << "\n-vs-\n" + << tensor2.dtype() << ' ' << tensor2.sizes() << '\n'; + return false; + } + at::ScalarType type1 = tensor1.scalar_type(); + at::ScalarType type2 = tensor2.scalar_type(); + if (type1 != type2) { + tensor1 = tensor1.toType(type2); + } + bool equal = tensor1.equal(tensor2); + return equal; +} + +bool EqualValuesNoElementTypeCheck(at::Tensor tensor1, at::Tensor tensor2) { + tensor1 = ToCpuTensor(tensor1); + tensor2 = ToCpuTensor(tensor2); + if (tensor1.sizes() != tensor2.sizes()) { + std::cerr << "Different shape:\n" + << tensor1.dtype() << ' ' << tensor1.sizes() << "\n-vs-\n" + << tensor2.dtype() << ' ' << tensor2.sizes() << '\n'; + return false; + } + at::ScalarType type1 = tensor1.scalar_type(); + at::ScalarType type2 = tensor2.scalar_type(); + if (type1 != type2) { + tensor1 = tensor1.toType(type2); + } + bool equal = tensor1.equal(tensor2); + return equal; +} + +void ForEachDevice(const std::function& devfn) { + // Currently TorchScript backend only supports one type of hardware per + // process, which is set by env. And the ordinal is always 0 given distributed + // training/ multi-device is not supported yet. + auto device = torch::lazy::BackendDevice(); + torch::Device torch_device = torch::lazy::backendDeviceToAtenDevice(device); + devfn(torch_device); +} + +bool CloseValues( + at::Tensor tensor1, + at::Tensor tensor2, + double rtol, + double atol) { + tensor1 = ToCpuTensor(tensor1); + tensor2 = ToCpuTensor(tensor2); + if (torch::isnan(tensor1).any().item()) { + EXPECT_TRUE(EqualValues(torch::isnan(tensor1), torch::isnan(tensor2))); + tensor1.nan_to_num_(); + tensor2.nan_to_num_(); + } + if (tensor1.sizes() != tensor2.sizes() || + tensor1.dtype() != tensor2.dtype()) { + std::cerr << "Different shape:\n" + << tensor1.dtype() << ' ' << tensor1.sizes() << "\n-vs-\n" + << tensor2.dtype() << ' ' << tensor2.sizes() << '\n'; + return false; + } + bool equal = tensor1.allclose(tensor2, rtol, atol); + return equal; +} + +std::string GetTensorTextGraph(at::Tensor tensor) { + torch::lazy::LazyTensorPtr lazy_tensor = torch::lazy::TryGetLtcTensor(tensor); + return torch::lazy::DumpUtil::ToText({lazy_tensor->GetIrValue().node.get()}); +} + +std::string GetTensorDotGraph(at::Tensor tensor) { + torch::lazy::LazyTensorPtr lazy_tensor = torch::lazy::TryGetLtcTensor(tensor); + return torch::lazy::DumpUtil::ToDot({lazy_tensor->GetIrValue().node.get()}); +} + +void TestBackward( + const std::vector& inputs, + const torch::Device& device, + const std::function&)>& + testfn, + double rtol, + double atol, + int derivative_level) { + std::vector input_vars; + std::vector xinput_vars; + std::vector inputs_w_grad; + std::vector xinputs_w_grad; + for (size_t i = 0; i < inputs.size(); ++i) { + const torch::Tensor& input = inputs[i]; + if (input.defined()) { + torch::Tensor oinput = + input.detach().clone().set_requires_grad(input.requires_grad()); + input_vars.push_back(oinput); + + torch::Tensor xinput = CopyToDevice(input, device) + .detach() + .set_requires_grad(input.requires_grad()); + xinput_vars.push_back(xinput); + if (input.requires_grad()) { + inputs_w_grad.push_back(oinput); + xinputs_w_grad.push_back(xinput); + } + } else { + input_vars.emplace_back(); + xinput_vars.emplace_back(); + } + } + + torch::Tensor output = testfn(input_vars); + torch::Tensor xoutput = testfn(xinput_vars); + torch::lazy::AllClose(output, xoutput, rtol, atol); + + std::vector outs = {output}; + std::vector xouts = {xoutput}; + for (int d = 1; d <= derivative_level; ++d) { + // Check grad of sum(outs) w.r.t inputs_w_grad. + torch::Tensor sum = torch::zeros_like(outs[0]).sum(); + torch::Tensor xsum = torch::zeros_like(xouts[0]).sum(); + for (size_t i = 0; i < outs.size(); ++i) { + if (outs[i].requires_grad()) { + sum += outs[i].sum(); + xsum += xouts[i].sum(); + } + } + // Calculating higher order derivative requires create_graph=true + bool create_graph = d != derivative_level; + outs = torch::autograd::grad( + {sum}, + inputs_w_grad, + /*grad_outputs=*/{}, + /*retain_graph=*/std::nullopt, + /*create_graph=*/create_graph, + /*allow_unused=*/true); + xouts = torch::autograd::grad( + {xsum}, + xinputs_w_grad, + /*grad_outputs=*/{}, + /*retain_graph=*/std::nullopt, + /*create_graph=*/create_graph, + /*allow_unused=*/true); + for (size_t i = 0; i < outs.size(); ++i) { + ASSERT_EQ(outs[i].defined(), xouts[i].defined()); + if (outs[i].defined()) { + AllClose(outs[i], xouts[i], rtol, atol); + } + } + } +} + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_ops_util.h b/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_ops_util.h new file mode 100644 index 0000000000000000000000000000000000000000..b3c0ae7637eedef569ac1303e4258e5b4dc3a855 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_lazy_ops_util.h @@ -0,0 +1,81 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace torch { +namespace lazy { + +const std::unordered_set* GetIgnoredCounters(); + +// Converts an at::Tensor(device=torch::kLazy) to at::Tensor(device=torch::kCPU) +// This at::Tensor can be torch::Tensor which is a Variable, or at::Tensor which +// know nothing about autograd. If the input tensor is already a CPU tensor, it +// will be returned. Needed because EqualValues and AllClose require CPU tensors +// on both sides. +at::Tensor ToCpuTensor(const at::Tensor& tensor); + +// Helper function to copy a tensor to device. +torch::Tensor CopyToDevice( + const torch::Tensor& tensor, + const torch::Device& device); + +bool EqualValues(at::Tensor tensor1, at::Tensor tensor2); + +bool EqualValuesNoElementTypeCheck(at::Tensor tensor1, at::Tensor tensor2); + +bool CloseValues( + at::Tensor tensor1, + at::Tensor tensor2, + double rtol = 1e-5, + double atol = 1e-8); + +static inline void AllClose( + at::Tensor tensor, + at::Tensor xla_tensor, + double rtol = 1e-5, + double atol = 1e-8) { + EXPECT_TRUE(CloseValues(tensor, xla_tensor, rtol, atol)); +} + +static inline void AllClose( + at::Tensor tensor, + torch::lazy::LazyTensor& xla_tensor, + double rtol = 1e-5, + double atol = 1e-8) { + EXPECT_TRUE( + CloseValues(tensor, xla_tensor.ToTensor(/*detached=*/false), rtol, atol)); +} + +static inline void AllEqual(at::Tensor tensor, at::Tensor xla_tensor) { + EXPECT_TRUE(EqualValues(tensor, xla_tensor)); +} + +void ForEachDevice(const std::function& devfn); + +std::string GetTensorTextGraph(at::Tensor tensor); + +std::string GetTensorDotGraph(at::Tensor tensor); + +std::string GetTensorHloGraph(at::Tensor tensor); + +void TestBackward( + const std::vector& inputs, + const torch::Device& device, + const std::function&)>& + testfn, + double rtol = 1e-5, + double atol = 1e-8, + int derivative_level = 1); + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_misc.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_misc.cpp new file mode 100644 index 0000000000000000000000000000000000000000..571bc79d4c775a67374e43cab195f32537fc8db8 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_misc.cpp @@ -0,0 +1,85 @@ +#include +#include + +#include +#include + +namespace torch { +namespace lazy { + +template +void test_hash_repeatable_sensitive(const T& example_a, const T& example_b) { + // repeatable + EXPECT_EQ(Hash(example_a), Hash(example_a)); + EXPECT_EQ(MHash(example_a), MHash(example_a)); + EXPECT_EQ(MHash(example_a, example_a), MHash(example_a, example_a)); + + // sensitive + EXPECT_NE(Hash(example_a), Hash(example_b)); + EXPECT_NE(MHash(example_a), MHash(example_b)); + EXPECT_NE(MHash(example_a, example_a), MHash(example_a, example_b)); +} + +TEST(HashTest, Scalar) { + GTEST_SKIP() + << "Broken test. See https://github.com/pytorch/pytorch/issues/99883"; + c10::Scalar a(0); + c10::Scalar b(0); + + // simulate some garbage in the unused bits of the + // the tagged union that is c10::Scalar, which is bigger + // than the size of the int64_t we're currently using it with + *((uint8_t*)&b) = 1; + // actual 'value' of the Scalar as a 64 bit int shouldn't have changed + EXPECT_EQ(a.toLong(), b.toLong()); + // and hash should ignore this garbage + EXPECT_EQ(Hash(a), Hash(b)); + EXPECT_EQ(MHash(a), MHash(b)); + EXPECT_EQ(MHash(a, a), MHash(a, b)); +} + +TEST(HashTest, Sanity) { + // String + test_hash_repeatable_sensitive( + std::string( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut at suscipit purus."), + std::string( + "Lorem Jpsum dolor sit amet, consectetur adipiscing elit. Ut at suscipit purus.")); + + // Number types + test_hash_repeatable_sensitive(true, false); + test_hash_repeatable_sensitive((int8_t)0xfa, (int8_t)0xfb); + test_hash_repeatable_sensitive((int16_t)0xface, (int16_t)0xfade); + test_hash_repeatable_sensitive((int32_t)0xfaceb000, (int32_t)0xfadeb000); + test_hash_repeatable_sensitive((int64_t)0x1faceb000, (int64_t)0x1fadeb000); + test_hash_repeatable_sensitive((uint8_t)0xfa, (uint8_t)0xfb); + test_hash_repeatable_sensitive((uint16_t)0xface, (uint16_t)0xfade); + test_hash_repeatable_sensitive((uint32_t)0xfaceb000, (uint32_t)0xfadeb000); + test_hash_repeatable_sensitive((uint64_t)0x1faceb000, (uint64_t)0x1fadeb000); + + // c10 types + test_hash_repeatable_sensitive(c10::ScalarType::Bool, c10::ScalarType::Byte); + test_hash_repeatable_sensitive(c10::Scalar(1.334), c10::Scalar(1.335)); + test_hash_repeatable_sensitive(c10::Scalar(true), c10::Scalar(false)); + test_hash_repeatable_sensitive(c10::Scalar(12345), c10::Scalar(12354)); + + // std::optional + test_hash_repeatable_sensitive( + std::optional("I have value!"), + std::optional(std::nullopt)); + + // Containers + auto a = std::vector({0, 1, 1, 2, 3, 5, 8}); + auto b = std::vector({1, 1, 2, 3, 5, 8, 12}); + test_hash_repeatable_sensitive(a, b); + test_hash_repeatable_sensitive( + c10::ArrayRef(a), c10::ArrayRef(b)); + + // vector is a special case bc it is implemented as vector + auto bool_a = std::vector({true, false, false, true}); + auto bool_b = std::vector({true, true, false, true}); + test_hash_repeatable_sensitive(bool_a, bool_b); +} + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_permutation_util.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_permutation_util.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4649de1b03192277b4a407997d89c3b7e1dc490d --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_permutation_util.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include + +namespace torch { +namespace lazy { + +TEST(PermutationUtilTest, TestInversePermutation) { + EXPECT_EQ(InversePermutation({0}), std::vector({0})); + EXPECT_EQ(InversePermutation({0, 1, 2}), std::vector({0, 1, 2})); + EXPECT_EQ( + InversePermutation({1, 3, 2, 0}), std::vector({3, 0, 2, 1})); + // Not a valid permutation + EXPECT_THROW(InversePermutation({-1}), c10::Error); + EXPECT_THROW(InversePermutation({1, 1}), c10::Error); +} + +TEST(PermutationUtilTest, TestIsPermutation) { + EXPECT_TRUE(IsPermutation({0})); + EXPECT_TRUE(IsPermutation({0, 1, 2, 3})); + EXPECT_FALSE(IsPermutation({-1})); + EXPECT_FALSE(IsPermutation({5, 3})); + EXPECT_FALSE(IsPermutation({1, 2, 3})); +} + +TEST(PermutationUtilTest, TestPermute) { + EXPECT_EQ( + PermuteDimensions({0}, std::vector({224})), + std::vector({224})); + EXPECT_EQ( + PermuteDimensions({1, 2, 0}, std::vector({3, 224, 224})), + std::vector({224, 224, 3})); + // Not a valid permutation + EXPECT_THROW( + PermuteDimensions({-1}, std::vector({244})), c10::Error); + EXPECT_THROW( + PermuteDimensions({3, 2}, std::vector({244})), c10::Error); + // Permutation size is different from the to-be-permuted vector size + EXPECT_THROW( + PermuteDimensions({0, 1}, std::vector({244})), c10::Error); + EXPECT_THROW( + PermuteDimensions({0}, std::vector({3, 244, 244})), c10::Error); +} + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_shape.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_shape.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9d7d9a84363e988da7214e6498a609c465461b0c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_shape.cpp @@ -0,0 +1,84 @@ +#include + +#include + +#include + +namespace torch { +namespace lazy { + +TEST(ShapeTest, Basic1) { + auto shape = Shape(); + + EXPECT_STREQ(shape.to_string().c_str(), "UNKNOWN_SCALAR[]"); + EXPECT_EQ(shape.scalar_type(), c10::ScalarType::Undefined); + EXPECT_EQ(shape.dim(), 0); + EXPECT_TRUE(shape.sizes().empty()); + EXPECT_THROW(shape.size(0), std::out_of_range); +} + +TEST(ShapeTest, Basic2) { + auto shape = Shape(c10::ScalarType::Float, {1, 2, 3}); + + EXPECT_EQ(shape.numel(), 6); + EXPECT_STREQ(shape.to_string().c_str(), "Float[1,2,3]"); + EXPECT_EQ(shape.scalar_type(), c10::ScalarType::Float); + EXPECT_EQ(shape.dim(), 3); + EXPECT_EQ(shape.sizes().size(), 3); + for (int64_t i = 0; i < shape.dim(); i++) { + EXPECT_EQ(shape.sizes()[i], i + 1); + EXPECT_EQ(shape.size(i), i + 1); + } +} + +TEST(ShapeTest, Basic3) { + auto shape = Shape(c10::ScalarType::Float, {}); + + EXPECT_STREQ(shape.to_string().c_str(), "Float[]"); + EXPECT_EQ(shape.scalar_type(), c10::ScalarType::Float); + EXPECT_EQ(shape.dim(), 0); + // this is surprising, but it's in line with how 0-D tensors behave + EXPECT_EQ(shape.numel(), 1); + EXPECT_TRUE(shape.sizes().empty()); + EXPECT_THROW(shape.size(0), std::out_of_range); +} + +TEST(ShapeTest, SetScalarType) { + auto shape = Shape(); + + shape.set_scalar_type(c10::ScalarType::Long); + EXPECT_EQ(shape.scalar_type(), c10::ScalarType::Long); +} + +TEST(ShapeTest, SetSize) { + auto shape1 = Shape(); + EXPECT_THROW(shape1.set_size(0, 0), std::out_of_range); + + auto shape2 = Shape(c10::ScalarType::Float, {1, 2, 3}); + shape2.set_size(0, 3); + EXPECT_EQ(shape2.sizes()[0], 3); + EXPECT_EQ(shape2.size(0), 3); +} + +TEST(ShapeTest, Equal) { + auto shape1 = Shape(c10::ScalarType::Float, {}); + auto shape2 = Shape(c10::ScalarType::Float, {1, 2, 3}); + auto shape3 = Shape(c10::ScalarType::Long, {1, 2, 3}); + auto shape4 = Shape(c10::ScalarType::Float, {1, 2, 3}); + + EXPECT_FALSE(shape1 == shape2); + EXPECT_FALSE(shape2 == shape3); + EXPECT_FALSE(shape1 == shape3); + EXPECT_TRUE(shape2 == shape2); +} + +TEST(ShapeTest, Ostream) { + auto shape = Shape(); + std::stringstream ss; + ss << shape; + + EXPECT_EQ(shape.to_string(), std::move(ss).str()); +} + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_tensor_impl.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_tensor_impl.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5833c72d8f253feaf3d65bf99f7d849ae4aeec3a --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_tensor_impl.cpp @@ -0,0 +1,23 @@ +#include + +#include +#include + +namespace torch { +namespace lazy { + +#ifdef FBCODE_CAFFE2 +// Lazy Tensor is disabled in FBCODE until addressing non-virtual methods (e.g. +// sizes) in TensorImpl +TEST(LazyTensorImplTest, BasicThrow) { + EXPECT_THROW( + { + auto input = torch::rand( + {0, 1, 3, 0}, torch::TensorOptions(torch::kFloat).device("lazy")); + }, + ::c10::Error); +} +#endif // FBCODE_CAFFE2 + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_trie_cache.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_trie_cache.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7b0d512a0360756bad525c3c209804c6dc216594 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_trie_cache.cpp @@ -0,0 +1,97 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace lazy { + +class TrieCacheNode : public Node { + public: + static OpKind ClassOpKind() { + return OpKind(); + } + + explicit TrieCacheNode(size_t id) + : Node(ClassOpKind(), /* num_outputs */ 1), id_(id), hash_(Hash(id_)) {} + ~TrieCacheNode() override = default; + + bool CanBeReused(size_t id) const { + return (id_ == id); + } + + void AddOperand(Value v) { + if (!v.node) { + return; + } + operands_as_outputs_.emplace_back(v.node.get(), v.index); + operands_.push_back(std::move(v.node)); + } + + hash_t hash() const override { + return hash_; + } + hash_t shapeHash() const override { + return hash_; + } + + private: + size_t id_; + hash_t hash_; +}; + +TEST(TrieCacheTest, TestSinglePath) { + FLAGS_torch_lazy_reuse_ir = true; + TrieCache::Get()->Clear(); + + NodePtr a = ReuseOrMakeNode(0); + NodePtr b = ReuseOrMakeNode(1); + NodePtr c = ReuseOrMakeNode(2); + TrieCache::Get()->ResetCurrent(); // MarkStep + + EXPECT_EQ(ReuseOrMakeNode(0).get(), a.get()); + EXPECT_EQ(ReuseOrMakeNode(1).get(), b.get()); + EXPECT_EQ(ReuseOrMakeNode(2).get(), c.get()); + TrieCache::Get()->ResetCurrent(); // MarkStep +} + +/* + * 0 + * | + * 1 + * / \ + * 2 3 + */ +TEST(TrieCacheTest, TestTwoPaths) { + FLAGS_torch_lazy_reuse_ir = true; + TrieCache::Get()->Clear(); + + NodePtr a = ReuseOrMakeNode(0); + NodePtr b = ReuseOrMakeNode(1); + NodePtr c = ReuseOrMakeNode(2); + TrieCache::Get()->ResetCurrent(); // MarkStep + + EXPECT_EQ(ReuseOrMakeNode(0).get(), a.get()); + EXPECT_EQ(ReuseOrMakeNode(1).get(), b.get()); + NodePtr d = ReuseOrMakeNode(3); + EXPECT_NE(d.get(), c.get()); + TrieCache::Get()->ResetCurrent(); // MarkStep + + EXPECT_EQ(ReuseOrMakeNode(0).get(), a.get()); + EXPECT_EQ(ReuseOrMakeNode(1).get(), b.get()); + EXPECT_EQ(ReuseOrMakeNode(3).get(), d.get()); + TrieCache::Get()->ResetCurrent(); // MarkStep + + EXPECT_EQ(ReuseOrMakeNode(0).get(), a.get()); + EXPECT_EQ(ReuseOrMakeNode(1).get(), b.get()); + EXPECT_EQ(ReuseOrMakeNode(2).get(), c.get()); + TrieCache::Get()->ResetCurrent(); // MarkStep +} + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lazy/test_util.cpp b/western_ai/usa/pytorch/test/cpp/lazy/test_util.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6a33486d03e9311b920ca91faf64acdc39682e9c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lazy/test_util.cpp @@ -0,0 +1,71 @@ +#include + +#include + +#include + +namespace torch { +namespace lazy { + +TEST(UtilTest, ExceptionCleanup) { + std::exception_ptr exception; + EXPECT_EQ(exception, nullptr); + + { + ExceptionCleanup cleanup( + [&](std::exception_ptr&& e) { exception = std::move(e); }); + + cleanup.SetStatus(std::make_exception_ptr(std::runtime_error("Oops!"))); + } + EXPECT_NE(exception, nullptr); + + try { + std::rethrow_exception(exception); + } catch (const std::exception& e) { + EXPECT_STREQ(e.what(), "Oops!"); + } + + exception = nullptr; + { + ExceptionCleanup cleanup( + [&](std::exception_ptr&& e) { exception = std::move(e); }); + + cleanup.SetStatus(std::make_exception_ptr(std::runtime_error(""))); + cleanup.Release(); + } + EXPECT_EQ(exception, nullptr); +} + +TEST(UtilTest, MaybeRef) { + std::string storage("String storage"); + MaybeRef refStorage(storage); + EXPECT_FALSE(refStorage.IsStored()); + EXPECT_EQ(*refStorage, storage); + + MaybeRef effStorage(std::string("Vanishing")); + EXPECT_TRUE(effStorage.IsStored()); + EXPECT_EQ(*effStorage, "Vanishing"); +} + +TEST(UtilTest, Iota) { + auto result = Iota(0); + EXPECT_TRUE(result.empty()); + + result = Iota(1); + EXPECT_EQ(result.size(), 1); + EXPECT_EQ(result[0], 0); + + result = Iota(2); + EXPECT_EQ(result.size(), 2); + EXPECT_EQ(result[0], 0); + EXPECT_EQ(result[1], 1); + + result = Iota(3, 1, 3); + EXPECT_EQ(result.size(), 3); + EXPECT_EQ(result[0], 1); + EXPECT_EQ(result[1], 4); + EXPECT_EQ(result[2], 7); +} + +} // namespace lazy +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..9e1abfdad4911ab5083073adccd312eaca519b65 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/CMakeLists.txt @@ -0,0 +1,38 @@ +set( + LITE_INTERPRETER_RUNTIME_TEST_DIR + "${TORCH_ROOT}/test/cpp/lite_interpreter_runtime") +set(LITE_INTERPRETER_RUNTIME_TEST_DIR + ${TORCH_ROOT}/test/cpp/lite_interpreter_runtime/main.cpp + ${TORCH_ROOT}/test/cpp/lite_interpreter_runtime/test_lite_interpreter_runtime.cpp + ${TORCH_ROOT}/test/cpp/lite_interpreter_runtime/test_mobile_profiler.cpp +) + +add_library(backend_with_compiler_runtime SHARED + ${TORCH_ROOT}/test/cpp/jit/test_backend_compiler_lib.cpp + ${TORCH_ROOT}/torch/csrc/jit/backends/backend_interface.cpp + ) +target_link_libraries(backend_with_compiler_runtime PRIVATE torch) + +add_executable( + test_lite_interpreter_runtime + ${LITE_INTERPRETER_RUNTIME_TEST_DIR}) +target_include_directories( + test_lite_interpreter_runtime PRIVATE + ${ATen_CPU_INCLUDE} +) + +target_link_libraries(test_lite_interpreter_runtime PRIVATE torch gtest_main backend_with_compiler_runtime) + +if(LINUX) + target_link_libraries(test_lite_interpreter_runtime PRIVATE "-Wl,--no-as-needed,$,--as-needed") +endif() + +if(INSTALL_TEST) + install(TARGETS test_lite_interpreter_runtime DESTINATION bin) + # Install PDB files for MSVC builds + if(MSVC AND BUILD_SHARED_LIBS) + install( + FILES $ + DESTINATION bin OPTIONAL) + endif() +endif() diff --git a/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/delegate_test.ptl b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/delegate_test.ptl new file mode 100644 index 0000000000000000000000000000000000000000..1e65c14e2ab7a81650766e4848ac9042f4d1caad Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/delegate_test.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/delegated_submodule_with_debug_info.ptl b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/delegated_submodule_with_debug_info.ptl new file mode 100644 index 0000000000000000000000000000000000000000..901724d82225b87b78f97590d209235e207b45ee Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/delegated_submodule_with_debug_info.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/light_model.ptl b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/light_model.ptl new file mode 100644 index 0000000000000000000000000000000000000000..d0783d3badd8c85af6101d349ba68e95a8a49252 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/light_model.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/main.cpp b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8ce406a1cb0f5e724c2c5749d840d4721afdf7a1 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/main.cpp @@ -0,0 +1,23 @@ +#include +#include +#include +#include +#include +#include + +std::string add_negative_flag(const std::string& flag) { + std::string filter = ::testing::GTEST_FLAG(filter); + if (filter.find('-') == std::string::npos) { + filter.push_back('-'); + } else { + filter.push_back(':'); + } + filter += flag; + return filter; +} +int main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + ::testing::GTEST_FLAG(filter) = add_negative_flag("*_CUDA:*_MultiCUDA"); + + return RUN_ALL_TESTS(); +} diff --git a/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/resources.h b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/resources.h new file mode 100644 index 0000000000000000000000000000000000000000..3c8b351a8018b2ab9309016e1c1baf815bcdd775 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/resources.h @@ -0,0 +1,41 @@ +#pragma once + +#include + +namespace torch { +namespace testing { + +namespace detail { +class Path; +} + +/// Gets the path to the resource identified by name. +/// +/// @param name identifies a resource, relative path starting from the +/// repo root +inline auto getResourcePath(std::string name) -> detail::Path; + +// End interface: implementation details follow. + +namespace detail { + +class Path { + public: + explicit Path(std::string rep) : rep_(std::move(rep)) {} + + auto string() const -> std::string const& { + return rep_; + } + + private: + std::string rep_; +}; + +} // namespace detail + +inline auto getResourcePath(std::string name) -> detail::Path { + return detail::Path(std::move(name)); +} + +} // namespace testing +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/sequence.ptl b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/sequence.ptl new file mode 100644 index 0000000000000000000000000000000000000000..1f7027007b97099c813703329b146e76326c0e19 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/sequence.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/test_backend_for_profiling.ptl b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/test_backend_for_profiling.ptl new file mode 100644 index 0000000000000000000000000000000000000000..290b21b15dc140f540d93107ae62695a04ca158a Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/test_backend_for_profiling.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/test_lite_interpreter_runtime.cpp b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/test_lite_interpreter_runtime.cpp new file mode 100644 index 0000000000000000000000000000000000000000..89ec0c48b0ba06ad24ea9d3b35dd8f079686b55e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/test_lite_interpreter_runtime.cpp @@ -0,0 +1,178 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace torch { +namespace jit { +namespace mobile { + +TEST(RunTimeTest, LoadAndForward) { + // Load check in model: sequence.ptl + std::string filePath(__FILE__); + auto testModelFile = filePath.substr(0, filePath.find_last_of("/\\") + 1); + testModelFile.append("sequence.ptl"); + + // sequence.ptl source code: + // class A(torch.nn.Module): + // def __init__(self) -> None: + // super().__init__() + // + // def forward(self, x): + // return x + 1 + // + // class B(torch.nn.Module): + // def __init__(self) -> None: + // super().__init__() + // + // def forward(self, x): + // return x + 2 + // + // class C(torch.nn.Module): + // def __init__(self) -> None: + // super().__init__() + // self.A0 = A() + // self.B0 = B() + // + // def forward(self, x): + // return self.A0.forward(self.B0.forward(x)) + + Module bc = _load_for_mobile(testModelFile); + auto forward_method = bc.find_method("forward"); + std::vector input{c10::IValue(at::tensor(1))}; + const auto result = bc.forward(input); + const auto expected_result = c10::IValue(at::tensor(4)); + ASSERT_EQ(result, expected_result); +} + +TEST(RunTimeTest, Delegate) { + std::string filePath(__FILE__); + auto testModelFile = filePath.substr(0, filePath.find_last_of("/\\") + 1); + // "delegate_test.ptl" is generated from test/cpp/jit/test_backend.cpp, + // BackendTest.TestCompiler. This test is on target runtime. It has + // model running capability, but no compilation and serialization. + // The mobile model delegated to the "backend_with_compiler_demo" backend + // The model is from the jit code: + // Module m("m"); + // m.define(R"( + // def forward(self, x, h): + // return x + h + // )"); + testModelFile.append("delegate_test.ptl"); + auto mlm = _load_for_mobile(testModelFile); + std::vector inputs; + inputs.emplace_back(2.0 * at::ones({})); + inputs.emplace_back(1.0 * at::ones({})); + + auto mres = mlm.forward(inputs); + AT_ASSERT(mres.toTensor().equal(3 * at::ones({}))); +} + +TEST(RunTimeTest, DelegateException) { + std::string filePath(__FILE__); + auto testModelFile = filePath.substr(0, filePath.find_last_of("/\\") + 1); + /* + * Model: delegated_submodule_with_debug_info.ptl + * Model structure: + * def AA(..): + * def forward(self, x, y): + * return x + y + * + * def A(..): + * def __init__(..): + * self.AA0 = AA() + * def forward(self, x, y): + * return self.AA0.forward(x, y) + 3 + * + * def B(..): + * def forward(self, x): + * return x + 2 + * + * def C(..): + * def __init__(..): + * self.A0 = A() + * self.B0 = B() + * def forward(self, x, y): + * return self.A0.forward(x, y) + self.B0.forward(x) + * + * std::vector inputs; + * inputs.emplace_back(torch::rand({2, 4})); + * inputs.emplace_back(torch::rand({13, 9})); + * Run with inputs and expect exception + * Error stack trace will look like this: + * Module hierarchy:top(C).A0(backend_with_compiler_demoLoweredModule).AA0(AA) + * Traceback of TorchScript (most recent call last): + * File "", line 3, in FunctionName_UNKNOWN + * + * def forward(self, x, y): + * return self.A0.forward(x, y) + self.B0.forward(x) + * ~~~~~~~~~~~~~~~ <--- HERE + * + * File "", line 5, in FunctionName_UNKNOWN + * typed_inputs: List[Any] = [x, y, ] + * if self.__backend.is_available() : + * _0, = self.__backend.execute(self.__handles["forward"], + * typed_inputs) + * ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + * assert isinstance(_0, Tensor) + * return _0 + * File "", line 3, in FunctionName_UNKNOWN + * + * def forward(self, x, y): + * return self.AA0.forward(x, y) + 3 + * ~~~~~~~~~~~~~~~~ <--- HERE + * + * File "", line 3, in FunctionName_UNKNOWN + * + * def forward(self, x, y): + * return x + y + * ~~~~~ <--- HERE + * + * + */ + testModelFile.append("delegated_submodule_with_debug_info.ptl"); + auto mlm = _load_for_mobile(testModelFile); + std::vector inputs; + inputs.emplace_back(torch::rand({2, 4})); + inputs.emplace_back(torch::rand({13, 9})); + + std::string error_pattern = R"( + Module hierarchy:top(C)::.A0(backend_with_compiler_demoLoweredModule)::forward.AA0(AA)::forward.aten::add +Traceback of TorchScript (most recent call last): + File "", line 3, in + + def forward(self, x, y): + return self.A0.forward(x, y) + self.B0.forward(x) + ~~~~~~~~~~~~~~~ <--- HERE + + File "", line 5, in forward + typed_inputs: List[Any] = [x, y, ] + if self.__backend.is_available() : + _0, = self.__backend.execute(self.__handles["forward"], typed_inputs) + ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + assert isinstance(_0, Tensor) + return _0 + File "", line 3, in + + def forward(self, x, y): + return self.AA0.forward(x, y) + 3 + ~~~~~~~~~~~~~~~~ <--- HERE + + File "", line 3, in forward + + def forward(self, x, y): + return x + y + ~~~~~ <--- HERE + )"; + ASSERT_THROWS_WITH_MESSAGE(mlm.forward(inputs), error_pattern); +} +} // namespace mobile +} // namespace jit +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/test_mobile_profiler.cpp b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/test_mobile_profiler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9925fe8d264875ff86b7d583a4a69645007e0531 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/test_mobile_profiler.cpp @@ -0,0 +1,215 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "test/cpp/lite_interpreter_runtime/resources.h" + +#ifdef EDGE_PROFILER_USE_KINETO +namespace torch { +namespace jit { +namespace mobile { + +namespace { +bool checkMetaData( + const std::string& op_name, + const std::string& metadata_name, + const std::string& metadata_val, + std::ifstream& trace_file) { + std::string line; + while (std::getline(trace_file, line)) { + if (line.find(op_name) != std::string::npos) { + while (std::getline(trace_file, line)) { + if (line.find(metadata_name) != std::string::npos) { + if (line.find(metadata_val) != std::string::npos || + !metadata_val.size()) { + /* if found the right metadata_val OR if expected + * metadata value is an empty string then ignore the metadata_val */ + return true; + } + } + } + } + } + return false; +} +} // namespace + +TEST(MobileProfiler, ModuleHierarchy) { + auto testModelFile = torch::testing::getResourcePath( + "test/cpp/lite_interpreter_runtime/to_be_profiled_module.ptl"); + + std::vector inputs; + inputs.emplace_back(at::rand({64, 64})); + inputs.emplace_back(at::rand({64, 64})); + std::string trace_file_name("/tmp/test_trace.trace"); + + mobile::Module bc = _load_for_mobile(testModelFile.string()); + { + KinetoEdgeCPUProfiler profiler( + bc, + trace_file_name, + false, // record input_shapes + false, // profile memory + true, // record callstack + false, // record flops + true, // record module hierarchy + {}, // events + false); // adjust_vulkan_timestamps + bc.forward(inputs); + } // End of profiler + std::ifstream trace_file(trace_file_name); + std::string line; + ASSERT_TRUE(trace_file.is_open()); + trace_file.seekg(0, std::ios_base::beg); + const std::string metadata_name("Module Hierarchy"); + ASSERT_TRUE(checkMetaData( + "aten::sub", + metadata_name, + "top(C)::.A0(A)::forward.aten::sub", + trace_file)); + trace_file.seekg(0, std::ios_base::beg); + ASSERT_TRUE(checkMetaData( + "aten::mul", + metadata_name, + "top(C)::.A0(A)::forward.SELF(A)::forward_impl_.SELF(A)::my_new_method.aten::mul", + trace_file)); + trace_file.seekg(0, std::ios_base::beg); + ASSERT_TRUE(checkMetaData( + "aten::add", + metadata_name, + "top(C)::.A0(A)::forward.SELF(A)::forward_impl_.aten::add", + trace_file)); + ASSERT_TRUE(checkMetaData( + "aten::add", + metadata_name, + "top(C)::.SELF(C)::call_b.B0(B)::forward.aten::add", + trace_file)); + ASSERT_TRUE(checkMetaData( + "aten::add", metadata_name, "top(C)::.aten::add", trace_file)); +} + +TEST(MobileProfiler, Backend) { + auto testModelFile = torch::testing::getResourcePath( + "test/cpp/lite_interpreter_runtime/test_backend_for_profiling.ptl"); + + std::vector inputs; + inputs.emplace_back(at::rand({64, 64})); + inputs.emplace_back(at::rand({64, 64})); + std::string trace_file_name("/tmp/test_trace_backend.trace"); + + mobile::Module bc = _load_for_mobile(testModelFile.string()); + { + KinetoEdgeCPUProfiler profiler( + bc, + trace_file_name, + false, // record input_shapes + false, // profile memory + true, // record callstack + false, // record flops + true); // record module hierarchy + bc.forward(inputs); + } // End of profiler + std::ifstream trace_file(trace_file_name); + std::string line; + ASSERT_TRUE(trace_file.is_open()); + trace_file.seekg(0, std::ios_base::beg); + std::string metadata_name("Module Hierarchy"); + ASSERT_TRUE(checkMetaData( + "aten::add", metadata_name, "top(m)::.aten::add", trace_file)); + trace_file.seekg(0, std::ios_base::beg); + metadata_name = "Backend"; + ASSERT_TRUE( + checkMetaData("aten::add", metadata_name, "test_backend", trace_file)); +} + +TEST(MobileProfiler, BackendMemoryEvents) { + auto testModelFile = torch::testing::getResourcePath( + "test/cpp/lite_interpreter_runtime/test_backend_for_profiling.ptl"); + + std::vector inputs; + inputs.emplace_back(at::rand({64, 64})); + inputs.emplace_back(at::rand({64, 64})); + std::string trace_file_name("/tmp/test_trace_backend_memory.trace"); + + mobile::Module bc = _load_for_mobile(testModelFile.string()); + { + mobile::KinetoEdgeCPUProfiler profiler( + bc, + trace_file_name, + false, // record input_shapes + true, // profile memory + true, // record callstack + false, // record flops + true); // record module hierarchy + bc.forward(inputs); + } + std::ifstream trace_file(trace_file_name); + std::string line; + ASSERT_TRUE(trace_file.is_open()); + trace_file.seekg(0, std::ios_base::beg); + std::string metadata_name("Bytes"); + ASSERT_TRUE(checkMetaData("[memory]", metadata_name, "16384", trace_file)); + trace_file.seekg(0, std::ios_base::beg); + metadata_name = "Total Reserved"; + ASSERT_TRUE(checkMetaData("[memory]", metadata_name, "49152", trace_file)); +} + +TEST(MobileProfiler, ProfilerEvent) { + auto testModelFile = torch::testing::getResourcePath( + "test/cpp/lite_interpreter_runtime/test_backend_for_profiling.ptl"); + + std::vector inputs; + inputs.emplace_back(at::rand({64, 64})); + inputs.emplace_back(at::rand({64, 64})); + std::string trace_file_name("/tmp/test_trace_profiler_event.trace"); + + std::vector events( + torch::profiler::ProfilerPerfEvents.begin(), + torch::profiler::ProfilerPerfEvents.end()); + + mobile::Module bc = _load_for_mobile(testModelFile.string()); + { + // Bail if something goes wrong here + try { + KinetoEdgeCPUProfiler profiler( + bc, + trace_file_name, + false, // record input_shapes + false, // profile memory + true, // record callstack + false, // record flops + true, // record module hierarchy + events); // performance events + bc.forward(inputs); + } catch (...) { + return; + } + } // End of profiler + std::ifstream trace_file(trace_file_name); + std::string line; + ASSERT_TRUE(trace_file.is_open()); + + for (auto& event : events) { + trace_file.seekg(0, std::ios_base::beg); + /* + * Just checking if the event entry exists in the chrometrace. + * Checking the value in a hardware independent matter is tricky. + */ + ASSERT_TRUE(checkMetaData("aten::__getitem__", event, "", trace_file)); + } +} + +} // namespace mobile +} // namespace jit +} // namespace torch +#endif diff --git a/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/to_be_profiled_module.ptl b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/to_be_profiled_module.ptl new file mode 100644 index 0000000000000000000000000000000000000000..243e41ec835006803aae1e9eba1ba5f559eeafa4 Binary files /dev/null and b/western_ai/usa/pytorch/test/cpp/lite_interpreter_runtime/to_be_profiled_module.ptl differ diff --git a/western_ai/usa/pytorch/test/cpp/monitor/test_counters.cpp b/western_ai/usa/pytorch/test/cpp/monitor/test_counters.cpp new file mode 100644 index 0000000000000000000000000000000000000000..41a76bdd40a9157f497b77a56ad284f51a35420f --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/monitor/test_counters.cpp @@ -0,0 +1,305 @@ +#include + +#include + +#include +#include + +using namespace torch::monitor; + +TEST(MonitorTest, CounterDouble) { + Stat a{ + "a", + {Aggregation::MEAN, Aggregation::COUNT}, + std::chrono::milliseconds(100000), + 2, + }; + a.add(5.0); + ASSERT_EQ(a.count(), 1); + a.add(6.0); + ASSERT_EQ(a.count(), 0); + + auto stats = a.get(); + std::unordered_map want = { + {Aggregation::MEAN, 5.5}, + {Aggregation::COUNT, 2.0}, + }; + ASSERT_EQ(stats, want); +} + +TEST(MonitorTest, CounterInt64Sum) { + Stat a{ + "a", + {Aggregation::SUM}, + std::chrono::milliseconds(100000), + 2, + }; + a.add(5); + a.add(6); + auto stats = a.get(); + std::unordered_map want = { + {Aggregation::SUM, 11}, + }; + ASSERT_EQ(stats, want); +} + +TEST(MonitorTest, CounterInt64Value) { + Stat a{ + "a", + {Aggregation::VALUE}, + std::chrono::milliseconds(100000), + 2, + }; + a.add(5); + a.add(6); + auto stats = a.get(); + std::unordered_map want = { + {Aggregation::VALUE, 6}, + }; + ASSERT_EQ(stats, want); +} + +TEST(MonitorTest, CounterInt64Mean) { + Stat a{ + "a", + {Aggregation::MEAN}, + std::chrono::milliseconds(100000), + 2, + }; + { + // zero samples case + auto stats = a.get(); + std::unordered_map want = { + {Aggregation::MEAN, 0}, + }; + ASSERT_EQ(stats, want); + } + + a.add(0); + a.add(10); + + { + auto stats = a.get(); + std::unordered_map want = { + {Aggregation::MEAN, 5}, + }; + ASSERT_EQ(stats, want); + } +} + +TEST(MonitorTest, CounterInt64Count) { + Stat a{ + "a", + {Aggregation::COUNT}, + std::chrono::milliseconds(100000), + 2, + }; + ASSERT_EQ(a.count(), 0); + a.add(0); + ASSERT_EQ(a.count(), 1); + a.add(10); + ASSERT_EQ(a.count(), 0); + + auto stats = a.get(); + std::unordered_map want = { + {Aggregation::COUNT, 2}, + }; + ASSERT_EQ(stats, want); +} + +TEST(MonitorTest, CounterInt64MinMax) { + Stat a{ + "a", + {Aggregation::MIN, Aggregation::MAX}, + std::chrono::milliseconds(100000), + 6, + }; + { + auto stats = a.get(); + std::unordered_map want = { + {Aggregation::MAX, 0}, + {Aggregation::MIN, 0}, + }; + ASSERT_EQ(stats, want); + } + + a.add(0); + a.add(5); + a.add(-5); + a.add(-6); + a.add(9); + a.add(2); + { + auto stats = a.get(); + std::unordered_map want = { + {Aggregation::MAX, 9}, + {Aggregation::MIN, -6}, + }; + ASSERT_EQ(stats, want); + } +} + +TEST(MonitorTest, CounterInt64WindowSize) { + Stat a{ + "a", + {Aggregation::COUNT, Aggregation::SUM}, + std::chrono::milliseconds(100000), + /*windowSize=*/3, + }; + a.add(1); + a.add(2); + ASSERT_EQ(a.count(), 2); + a.add(3); + ASSERT_EQ(a.count(), 0); + + // after logging max for window, should be zero + a.add(4); + ASSERT_EQ(a.count(), 0); + + auto stats = a.get(); + std::unordered_map want = { + {Aggregation::COUNT, 3}, + {Aggregation::SUM, 6}, + }; + ASSERT_EQ(stats, want); +} + +TEST(MonitorTest, CounterInt64WindowSizeHuge) { + Stat a{ + "a", + {Aggregation::COUNT, Aggregation::SUM}, + std::chrono::hours(24 * 365 * 10), // 10 years + /*windowSize=*/3, + }; + a.add(1); + a.add(2); + ASSERT_EQ(a.count(), 2); + a.add(3); + ASSERT_EQ(a.count(), 0); + + // after logging max for window, should be zero + a.add(4); + ASSERT_EQ(a.count(), 0); + + auto stats = a.get(); + std::unordered_map want = { + {Aggregation::COUNT, 3}, + {Aggregation::SUM, 6}, + }; + ASSERT_EQ(stats, want); +} + +template +struct TestStat : public Stat { + uint64_t mockWindowId{1}; + + TestStat( + std::string name, + std::initializer_list aggregations, + std::chrono::milliseconds windowSize, + int64_t maxSamples = std::numeric_limits::max()) + : Stat(name, aggregations, windowSize, maxSamples) {} + + uint64_t currentWindowId() const override { + return mockWindowId; + } +}; + +struct AggregatingEventHandler : public EventHandler { + std::vector events; + + void handle(const Event& e) override { + events.emplace_back(e); + } +}; + +template +struct HandlerGuard { + std::shared_ptr handler; + + HandlerGuard() : handler(std::make_shared()) { + registerEventHandler(handler); + } + + ~HandlerGuard() { + unregisterEventHandler(handler); + } +}; + +TEST(MonitorTest, Stat) { + HandlerGuard guard; + + Stat a{ + "a", + {Aggregation::COUNT, Aggregation::SUM}, + std::chrono::milliseconds(1), + }; + ASSERT_EQ(guard.handler->events.size(), 0); + + a.add(1); + ASSERT_LE(a.count(), 1); + + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + a.add(2); + ASSERT_LE(a.count(), 1); + + ASSERT_GE(guard.handler->events.size(), 1); + ASSERT_LE(guard.handler->events.size(), 2); +} + +TEST(MonitorTest, StatEvent) { + HandlerGuard guard; + + TestStat a{ + "a", + {Aggregation::COUNT, Aggregation::SUM}, + std::chrono::milliseconds(1), + }; + ASSERT_EQ(guard.handler->events.size(), 0); + + a.add(1); + ASSERT_EQ(a.count(), 1); + a.add(2); + ASSERT_EQ(a.count(), 2); + ASSERT_EQ(guard.handler->events.size(), 0); + + a.mockWindowId = 100; + + a.add(3); + ASSERT_LE(a.count(), 1); + + ASSERT_EQ(guard.handler->events.size(), 1); + Event e = guard.handler->events.at(0); + ASSERT_EQ(e.name, "torch.monitor.Stat"); + ASSERT_NE(e.timestamp, std::chrono::system_clock::time_point{}); + std::unordered_map data{ + {"a.sum", 3L}, + {"a.count", 2L}, + }; + ASSERT_EQ(e.data, data); +} + +TEST(MonitorTest, StatEventDestruction) { + HandlerGuard guard; + + { + TestStat a{ + "a", + {Aggregation::COUNT, Aggregation::SUM}, + std::chrono::hours(10), + }; + a.add(1); + ASSERT_EQ(a.count(), 1); + ASSERT_EQ(guard.handler->events.size(), 0); + } + ASSERT_EQ(guard.handler->events.size(), 1); + + Event e = guard.handler->events.at(0); + ASSERT_EQ(e.name, "torch.monitor.Stat"); + ASSERT_NE(e.timestamp, std::chrono::system_clock::time_point{}); + std::unordered_map data{ + {"a.sum", 1L}, + {"a.count", 1L}, + }; + ASSERT_EQ(e.data, data); +} diff --git a/western_ai/usa/pytorch/test/cpp/monitor/test_events.cpp b/western_ai/usa/pytorch/test/cpp/monitor/test_events.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b3371a919bad0688bb1b47903a88e11da48b16eb --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/monitor/test_events.cpp @@ -0,0 +1,38 @@ +#include + +#include + +using namespace torch::monitor; + +struct AggregatingEventHandler : public EventHandler { + std::vector events; + + void handle(const Event& e) override { + events.emplace_back(e); + } +}; + +TEST(EventsTest, EventHandler) { + Event e; + e.name = "test"; + e.timestamp = std::chrono::system_clock::now(); + e.data["string"] = "asdf"; + e.data["double"] = 1234.5678; + e.data["int"] = 1234L; + e.data["bool"] = true; + + // log to nothing + logEvent(e); + + auto handler = std::make_shared(); + registerEventHandler(handler); + + logEvent(e); + ASSERT_EQ(handler->events.size(), 1); + ASSERT_EQ(e, handler->events.at(0)); + + unregisterEventHandler(handler); + logEvent(e); + // handler unregister, didn't log + ASSERT_EQ(handler->events.size(), 1); +} diff --git a/western_ai/usa/pytorch/test/cpp/nativert/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/nativert/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..8fcec6a2970712ef6f458c27ce0e315f0afeba8c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/CMakeLists.txt @@ -0,0 +1,94 @@ +set(NATIVERT_TEST_ROOT ${TORCH_ROOT}/test/cpp/nativert) + +file(GLOB_RECURSE NATIVERT_ALL_TEST_FILES "${NATIVERT_TEST_ROOT}/test_*.cpp") + +# Build the cpp gtest binary containing the cpp-only tests. +set(NATIVERT_TEST_SRCS + ${NATIVERT_ALL_TEST_FILES} + ${TORCH_ROOT}/torch/nativert/ModelRunner.cpp + ${TORCH_ROOT}/torch/nativert/graph/TensorMeta.cpp + ${TORCH_ROOT}/torch/nativert/graph/Graph.cpp + ${TORCH_ROOT}/torch/nativert/graph/GraphPasses.cpp + ${TORCH_ROOT}/torch/nativert/graph/GraphSignature.cpp + ${TORCH_ROOT}/torch/nativert/graph/GraphUtils.cpp + ${TORCH_ROOT}/torch/nativert/graph/Serialization.cpp + ${TORCH_ROOT}/torch/nativert/executor/OpKernel.cpp + ${TORCH_ROOT}/torch/nativert/executor/PlacementUtils.cpp + ${TORCH_ROOT}/torch/nativert/executor/Weights.cpp + ${TORCH_ROOT}/torch/nativert/common/FileUtil.cpp + ${TORCH_ROOT}/torch/nativert/executor/memory/FunctionSchema.cpp + ${TORCH_ROOT}/torch/nativert/executor/ExecutionPlanner.cpp + ${TORCH_ROOT}/torch/nativert/detail/ITree.cpp + ${TORCH_ROOT}/torch/nativert/executor/ExecutionFrame.cpp + ${TORCH_ROOT}/torch/nativert/kernels/C10Kernel.cpp + ${TORCH_ROOT}/torch/nativert/executor/memory/GreedyBySize.cpp + ${TORCH_ROOT}/torch/nativert/executor/memory/Bump.cpp + ${TORCH_ROOT}/torch/nativert/executor/memory/DisjointStorageGroups.cpp + ${TORCH_ROOT}/torch/nativert/executor/memory/LayoutPlanner.cpp + ${TORCH_ROOT}/torch/nativert/executor/memory/LayoutManager.cpp + ${TORCH_ROOT}/torch/nativert/executor/memory/AliasAnalyzer.cpp + ${TORCH_ROOT}/torch/nativert/executor/Executor.cpp + ${TORCH_ROOT}/torch/nativert/kernels/KernelFactory.cpp + ${TORCH_ROOT}/torch/nativert/executor/ConstantFolder.cpp + ${TORCH_ROOT}/torch/nativert/executor/GraphExecutorBase.cpp + ${TORCH_ROOT}/torch/nativert/executor/SerialGraphExecutor.cpp + ${TORCH_ROOT}/torch/nativert/executor/ParallelGraphExecutor.cpp + ${TORCH_ROOT}/torch/nativert/kernels/AutoFunctionalizeKernel.cpp + ${TORCH_ROOT}/torch/nativert/kernels/CallTorchBindKernel.cpp + ${TORCH_ROOT}/torch/nativert/kernels/HigherOrderKernel.cpp + ${TORCH_ROOT}/torch/nativert/graph/passes/SubgraphRewriter.cpp + ${TORCH_ROOT}/torch/nativert/graph/passes/pass_manager/GraphPasses.cpp + ${TORCH_ROOT}/torch/nativert/graph/passes/pass_manager/PassManager.cpp + ${TORCH_ROOT}/torch/nativert/kernels/KernelHandlerRegistry.cpp + ${TORCH_ROOT}/torch/nativert/executor/triton/CpuTritonKernelManager.cpp + ${TORCH_ROOT}/torch/nativert/executor/triton/fb/MtiaTritonKernelManager.cpp + ${TORCH_ROOT}/torch/nativert/kernels/TritonKernel.cpp + ${TORCH_ROOT}/torch/nativert/executor/DelegateExecutor.cpp + ${TORCH_ROOT}/torch/nativert/executor/AOTInductorDelegateExecutor.cpp + ${TORCH_ROOT}/torch/nativert/kernels/ETCallDelegateKernel.cpp + ${TORCH_ROOT}/torch/csrc/inductor/aoti_torch/oss_proxy_executor.cpp +) + +if(USE_CUDA OR USE_ROCM) + list(APPEND NATIVERT_TEST_SRCS ${TORCH_ROOT}/torch/nativert/executor/triton/CudaTritonKernelManager.cpp) + list(APPEND NATIVERT_TEST_SRCS ${TORCH_ROOT}/torch/nativert/executor/AOTInductorModelContainerCudaShim.cpp) +endif() + +add_executable(test_nativert + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${NATIVERT_TEST_SRCS} +) + +if(MSVC) + target_compile_definitions(test_nativert PRIVATE NATIVERT_MSVC_TEST) +endif() + +# TODO temporary until we can delete the old gtest polyfills. +target_compile_definitions(test_nativert PRIVATE USE_GTEST) + +set(NATIVERT_TEST_DEPENDENCIES torch gtest_main) + +target_link_libraries(test_nativert PRIVATE ${CMAKE_DL_LIBS}) +target_link_libraries(test_nativert PRIVATE ${NATIVERT_TEST_DEPENDENCIES}) +target_link_libraries(test_nativert PRIVATE fmt::fmt-header-only) +target_include_directories(test_nativert PRIVATE ${ATen_CPU_INCLUDE}) + +if(USE_CUDA) + target_compile_definitions(test_nativert PRIVATE USE_CUDA) +elseif(USE_ROCM) + target_link_libraries(test_nativert PRIVATE + hiprtc::hiprtc + hip::amdhip64 + ${TORCH_CUDA_LIBRARIES}) + + target_compile_definitions(test_nativert PRIVATE USE_ROCM) +endif() + +if(INSTALL_TEST) + set_target_properties(test_nativert PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS test_nativert DESTINATION bin) + # Install PDB files for MSVC builds + if(MSVC AND BUILD_SHARED_LIBS) + install(FILES $ DESTINATION bin OPTIONAL) + endif() +endif() diff --git a/western_ai/usa/pytorch/test/cpp/nativert/static_kernel_test_utils.h b/western_ai/usa/pytorch/test/cpp/nativert/static_kernel_test_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..2f38a513034eb21c5965375c7c03d5aad54003c0 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/static_kernel_test_utils.h @@ -0,0 +1,158 @@ +#include + +#include +#include +#include +#include + +#include + +namespace torch::nativert { + +/* + * This is a lightweight version of ModelRunner that executes a model in + * interpreter mode given a string graph with no weights/attributes + */ +class SimpleTestModelRunner { + public: + SimpleTestModelRunner( + const std::string_view source, + const ExecutorConfig& config) { + register_kernel_handlers(); + graph_ = stringToGraph(source); + weights_ = std::make_shared(graph_.get()); + + executor_ = std::make_unique(config, graph_, weights_); + } + + std::vector run(const std::vector& inputs) const { + return executor_->execute(inputs); + } + + ProfileMetrics benchmarkIndividualNodes( + const std::vector& inputs) const { + return executor_->benchmarkIndividualNodes({inputs}, 10, 10); + } + + private: + std::shared_ptr graph_; + std::unique_ptr executor_; + std::shared_ptr weights_; +}; + +inline void compareIValue( + const c10::IValue& expected, + const c10::IValue& actual, + bool native = false) { + if (expected.isTensor()) { + EXPECT_TRUE(actual.isTensor()); + EXPECT_TRUE(torch::allclose( + expected.toTensor(), + actual.toTensor(), + 1e-5, + 1e-8, + /*equal_nan*/ true)); + if (!native) { + EXPECT_TRUE(expected.toTensor().strides() == actual.toTensor().strides()); + } + } else if (expected.isTuple()) { + EXPECT_TRUE(actual.isTuple()); + auto expected_tuple = expected.toTupleRef().elements(); + auto actual_tuple = actual.toTupleRef().elements(); + ASSERT_TRUE(expected_tuple.size() == actual_tuple.size()); + for (size_t i = 0; i < expected_tuple.size(); i++) { + compareIValue(expected_tuple[i], actual_tuple[i], native); + } + } else if (expected.isList()) { + EXPECT_TRUE(actual.isList()); + auto expected_list = expected.toList(); + auto actual_list = actual.toList(); + ASSERT_TRUE(expected_list.size() == actual_list.size()); + for (size_t i = 0; i < expected_list.size(); i++) { + compareIValue(expected_list[i], actual_list[i], native); + } + } else if (expected.isGenericDict()) { + EXPECT_TRUE(actual.isGenericDict()); + auto expected_dict = expected.toGenericDict(); + auto actual_dict = actual.toGenericDict(); + EXPECT_TRUE(expected_dict.size() == actual_dict.size()); + for (auto& expected_kv : expected_dict) { + auto actual_kv = actual_dict.find(expected_kv.key()); + ASSERT_FALSE(actual_kv == actual_dict.end()); + compareIValue(expected_kv.value(), actual_kv->value(), native); + } + } else { + // Fall back to default comparison from IValue + EXPECT_TRUE(expected == actual); + } +} + +void compareIValues( + std::vector expected, + std::vector actual, + bool native = false) { + ASSERT_TRUE(expected.size() == actual.size()); + for (size_t i = 0; i < expected.size(); i++) { + compareIValue(expected[i], actual[i], native); + } +} + +inline void testStaticKernelEqualityInternal( + const SimpleTestModelRunner& modelRunner, + const SimpleTestModelRunner& staticModelRunner, + const std::vector& args, + bool native = false) { + auto expected = modelRunner.run(args); + + auto output = staticModelRunner.run(args); + compareIValues(expected, output, native); + + // Run again to test the static kernel when outputs IValue are cached in the + // execution frame + auto output2 = staticModelRunner.run(args); + compareIValues(expected, output2, native); +} + +void testStaticKernelEquality( + const std::string_view source, + const std::vector& args, + bool native = false) { + ExecutorConfig config; + config.enableStaticCPUKernels = false; + SimpleTestModelRunner model(source, config); + + config.enableStaticCPUKernels = true; + SimpleTestModelRunner staticKernelModel(source, config); + + testStaticKernelEqualityInternal(model, staticKernelModel, args, native); +} + +inline void testGraphABEquality( + const std::string_view graph_a, + const std::string_view graph_b, + const std::vector& args, + const ExecutorConfig& config = {}, + bool native = false) { + SimpleTestModelRunner model_a(graph_a, config); + auto expected = model_a.run(args); + + SimpleTestModelRunner model_b(graph_b, config); + auto output = model_b.run(args); + + compareIValues(expected, output, native); +} + +inline void testGraphABPerf( + const std::string_view graph_a, + const std::string_view graph_b, + const std::vector& args, + const ExecutorConfig& config = {}) { + SimpleTestModelRunner model_a(graph_a, config); + auto resultA = model_a.benchmarkIndividualNodes(args); + + SimpleTestModelRunner model_b(graph_b, config); + auto resultB = model_b.benchmarkIndividualNodes(args); + ASSERT_TRUE(resultA.totalTime > resultB.totalTime); +} + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_alias_analyzer.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_alias_analyzer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8012c3f5f0ce40cffb759d402a03a60a48d29a91 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_alias_analyzer.cpp @@ -0,0 +1,182 @@ +#include + +#include + +#include +#include + +#include +#include + +using namespace ::testing; +using namespace torch::nativert; + +using AliasTestCase = std::tuple< + std::string /* value */, + AllocationLifetime, + bool /* is_alias */, + bool /* is_storage_associated_with_output */, + c10::FastSet /* source(s) */>; + +class AliasAnalyzerTests : public testing::Test { + void SetUp() override {} + + void TearDown() override { + test_cases.clear(); + model.clear(); + } + + public: + void setTestCases(std::vector cases) { + test_cases = std::move(cases); + } + + void setModel(std::string m) { + model = std::move(m); + } + + void run() { + EXPECT_FALSE(test_cases.empty()); + EXPECT_FALSE(model.empty()); + + ExecutorConfig cfg; + cfg.enableStaticCPUKernels = true; + + auto graph = stringToGraph(model); + auto kernels = + KernelFactory().initializeNodeKernels(*graph, nullptr, cfg, nullptr); + auto kernelSchemas = Executor::getKernelSchemas(kernels.nodeKernels); + + AliasAnalyzer analyzer(*graph, kernelSchemas); + + for ( + auto& [value, lifetime, is_alias, is_storage_associated_with_output, srcs] : + test_cases) { + LOG(INFO) << fmt::format( + "running test: value={}, lifetime=({}, {}), is_alias={}, is_storage_associated_with_output={}, src={}", + value, + lifetime.start, + lifetime.end, + is_alias, + is_storage_associated_with_output, + srcs.empty() ? "{}" + : std::accumulate( + srcs.begin(), + srcs.end(), + std::string{}, + [](std::string cur, const std::string& src) { + cur.push_back(','); + cur.append(src); + return cur; + })); + auto* v = graph->getValue(value); + EXPECT_EQ(analyzer.lifetime(v), lifetime); + EXPECT_EQ(analyzer.is_alias(v), is_alias); + EXPECT_EQ( + analyzer.is_storage_associated_with_output(v), + is_storage_associated_with_output); + const auto* resolved_srcs = analyzer.get_sources_of_alias(v); + if (resolved_srcs /* ensure set equality between *resolved_srcs and srcs */) { + EXPECT_FALSE(srcs.empty()); + EXPECT_EQ(resolved_srcs->size(), srcs.size()); + for (const auto& resolved_src : *resolved_srcs) { + EXPECT_TRUE(srcs.erase(std::string(resolved_src->name())) == 1); + } + EXPECT_TRUE(srcs.empty()); + } else { + EXPECT_TRUE(srcs.empty()); + } + } + } + + private: + std::string model; + std::vector test_cases; +}; + +TEST_F(AliasAnalyzerTests, TestNoAlias) { + setModel(R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %res = torch.ops.aten.clone.default(self=%out_t, memory_format=None) + return (%res))"); + + setTestCases({ + {"out_t", AllocationLifetime(1, 2), false, false, {}}, + {"res", AllocationLifetime(2, 3), false, true, {}}, + }); + + run(); +} + +TEST_F(AliasAnalyzerTests, TestSimpleAlias) { + setModel(R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %res = torch.ops.aten.slice.Tensor(self=%out_t, dim=1, start=0, end=0, step=1) + return (%res))"); + + setTestCases({ + {"out_t", AllocationLifetime(1, 3), false, true, {}}, + {"res", AllocationLifetime(2, 3), true, false, {"out_t"}}, + }); + + run(); +} + +TEST_F(AliasAnalyzerTests, TestDeepAlias) { + setModel(R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %a1 = torch.ops.aten.slice.Tensor(self=%out_t, dim=1, start=0, end=0, step=1) + %res = torch.ops.aten.slice.Tensor(self=%a1, dim=1, start=0, end=0, step=1) + return (%res))"); + + setTestCases({ + {"out_t", AllocationLifetime(1, 4), false, true, {}}, + {"a1", AllocationLifetime(2, 4), true, false, {"out_t"}}, + {"res", AllocationLifetime(3, 4), true, false, {"out_t"}}, + }); + + run(); +} + +TEST_F(AliasAnalyzerTests, TestPackedListUnpack) { + setModel(R"( + graph(%a, %b, %c, %d): + %input_list[] = prim.ListPack(l0=%a, l1=%b, l2=%c, l3=%d) + %x0, %x1, %x2, %x3 = prim.ListUnpack(input=%input_list) + return (%x1, %x3))"); + + setTestCases({ + {"a", AllocationLifetime(0, 2), false, false, {}}, + {"x0", AllocationLifetime(2, 2), true, false, {"a"}}, + {"b", AllocationLifetime(0, 3), false, true, {}}, + {"x1", AllocationLifetime(2, 3), true, false, {"b"}}, + {"c", AllocationLifetime(0, 2), false, false, {}}, + {"x2", AllocationLifetime(2, 2), true, false, {"c"}}, + {"d", AllocationLifetime(0, 3), false, true, {}}, + {"x3", AllocationLifetime(2, 3), true, false, {"d"}}, + }); + + run(); +} + +TEST_F(AliasAnalyzerTests, TestAmbiguousSourceOfAlias) { + setModel(R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %out_t2 = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %a1 = prim.VarStack(l0=%out_t, l1=%out_t2) + %res = torch.ops.aten.slice.Tensor(self=%a1, dim=1, start=0, end=0, step=1) + return (%res))"); + + setTestCases({ + {"out_t", AllocationLifetime(1, 5), false, true, {}}, + {"out_t2", AllocationLifetime(2, 5), false, true, {}}, + {"a1", AllocationLifetime(3, 5), true, false, {"out_t", "out_t2"}}, + {"res", AllocationLifetime(4, 5), true, false, {"out_t", "out_t2"}}, + }); + + run(); +} diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_aoti_model_container_registration.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_aoti_model_container_registration.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2b1be37dd72f7bb16b55969a5611c42ac41e2bad --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_aoti_model_container_registration.cpp @@ -0,0 +1,16 @@ +#include + +#include + +using namespace ::testing; +using namespace torch::nativert; + +TEST(AOTIModelContainerRegistrationTests, TestRegister) { + EXPECT_TRUE(AOTIModelContainerRunnerRegistry()->Has(at::kCPU)); + +#if defined(USE_CUDA) || defined(USE_ROCM) + EXPECT_TRUE(AOTIModelContainerRunnerRegistry()->Has(at::kCUDA)); +#else + EXPECT_FALSE(AOTIModelContainerRunnerRegistry()->Has(at::kCUDA)); +#endif // USE_CUDA +} diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_c10_kernel.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_c10_kernel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8bc28c94bff93335a7057ed71fb22a280014e308 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_c10_kernel.cpp @@ -0,0 +1,74 @@ +#include +#include +#include +#include +#include +#include + +namespace torch::nativert { + +at::Tensor foo_kernel(const at::Tensor& a, const at::Tensor& b) { + return a + b; +} + +TEST(C10KernelTest, computeInternal) { + auto registrar = c10::RegisterOperators().op( + "test::foo(Tensor a, Tensor b) -> Tensor", &foo_kernel); + + static constexpr std::string_view source = + R"(graph(%a, %b): +%x = test.foo.default(a=%a, b=%b) +return (%x) +)"; + + auto graph = stringToGraph(source); + const auto& nodes = graph->nodes(); + auto it = nodes.begin(); + std::advance(it, 1); + const Node& node = *it; + + auto a = at::randn({6, 6, 6}); + auto b = at::randn({6, 6, 6}); + + auto frame = ExecutionFrame(*graph); + frame.setIValue(graph->getValue("a")->id(), a); + frame.setIValue(graph->getValue("b")->id(), b); + + auto kernel = C10Kernel(&node); + + kernel.computeInternal(frame); + + at::Tensor expected = a + b; + EXPECT_TRUE( + torch::equal(frame.getTensor(graph->getValue("x")->id()), expected)); +} + +TEST(ScalarBinaryOpKernelTest, computeInternal) { + static constexpr std::string_view source = + R"(graph(%a, %b): +%x = _operator.add(a=%a, b=%b) +return (%x) +)"; + + auto graph = stringToGraph(source); + const auto& nodes = graph->nodes(); + auto it = nodes.begin(); + std::advance(it, 1); + const Node& node = *it; + + auto a = 1; + auto b = 2; + + auto frame = ExecutionFrame(*graph); + frame.setIValue(graph->getValue("a")->id(), a); + frame.setIValue(graph->getValue("b")->id(), b); + + auto kernel = ScalarBinaryOpKernel(&node); + + kernel.computeInternal(frame); + + auto expected = a + b; + EXPECT_EQ(frame.getIValue(graph->getValue("x")->id()).toInt(), expected); +} + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_execution_frame.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_execution_frame.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7dcfb419c82bdf2952cc0240f83968aef149090b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_execution_frame.cpp @@ -0,0 +1,100 @@ +#include + +#include +#include + +namespace torch::nativert { + +TEST(ExecutionFrameTest, CreateFrame) { + auto graph = stringToGraph(R"( + graph(%x, %y): + %a = foo(a=%x, b=%y) + %b = foo1(a=%x, b=%y) + %c = foo2(c=%a, d=%b) + return(%c) + )"); + + auto frame = ExecutionFrame(*graph); + + for (auto* v : graph->values()) { + frame.setIValue(v->id(), c10::IValue(at::tensor({v->id()}, at::kInt))); + auto& frame_v = frame.getIValue(v->id()); + EXPECT_EQ(frame_v.tagKind(), "Tensor"); + } + + auto outputs = frame.tryMoveUserOutputs(); + + EXPECT_EQ(outputs.size(), 1); + EXPECT_EQ(outputs[0].tagKind(), "Tensor"); + EXPECT_EQ(outputs[0].toTensor().item().toInt(), graph->getValue("c")->id()); +} + +TEST(ExecutionFrameTest, TestSetBorrowedValue) { + auto graph = stringToGraph(R"( + graph(%x, %y): + %a = foo(a=%x, b=%y) + %b = foo1(a=%x, b=%y) + %c = foo2(c=%a, d=%b) + return(%c) + )"); + + auto x = c10::IValue(at::tensor({1}, at::kInt)); + auto y = c10::IValue(at::tensor({2}, at::kInt)); + + { + auto frame = ExecutionFrame(*graph); + + frame.setBorrowedIValue( + graph->getValue("x")->id(), + c10::MaybeOwnedTraits::createBorrow(x)); + frame.setBorrowedIValue( + graph->getValue("y")->id(), + c10::MaybeOwnedTraits::createBorrow(y)); + + [[maybe_unused]] auto& w = frame.getIValue(graph->getValue("x")->id()); + [[maybe_unused]] auto& z = frame.getIValue(graph->getValue("y")->id()); + + EXPECT_EQ(x.use_count(), 1); + EXPECT_EQ(y.use_count(), 1); + + EXPECT_TRUE(c10::MaybeOwnedTraits{}.debugBorrowIsValid( + frame.getIValue(graph->getValue("x")->id()))); + EXPECT_TRUE(c10::MaybeOwnedTraits{}.debugBorrowIsValid( + frame.getIValue(graph->getValue("y")->id()))); + } + + EXPECT_EQ(x.use_count(), 1); + EXPECT_EQ(y.use_count(), 1); +} + +TEST(ExecutionFrameTest, TestPersistentValue) { + auto graph = stringToGraph(R"( + graph(%x, %y, %my_weight): + %a = foo(a=%x, b=%y) + %b = foo1(a=%x, b=%y) + %c = foo2(c=%a, d=%b) + return(%c) + )"); + + Weights weights(graph.get()); + weights.setValue("my_weight", at::tensor({1}, at::kInt)); + + auto new_sig = graph->signature(); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + const_cast>&>( + new_sig.inputsToWeights()) + .emplace_back("my_weight", "my_weight"); + graph->setSignature(new_sig); + + auto frame = ExecutionFrame(*graph, weights); + + EXPECT_EQ(frame.weightVersion(), 0); + auto wid = graph->getValue("my_weight")->id(); + + EXPECT_NO_THROW(frame.getTensor(wid)); + // can't release persistent value + frame.releaseValueIfNeeded(wid); + EXPECT_FALSE(frame.getIValue(wid).isNone()); +} + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_execution_planner.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_execution_planner.cpp new file mode 100644 index 0000000000000000000000000000000000000000..926288ae0010ec9f4a0f89702485ee9f5f07d5cc --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_execution_planner.cpp @@ -0,0 +1,47 @@ +#include +#include + +namespace torch::nativert { + +TEST(ExecutionPlannerTest, CreatePlan) { + auto graph = stringToGraph(R"( + graph(%x, %y): + %a = foo(a=%x, b=%y) + %b = foo1(a=%x, b=%y) + %c = foo2(c=%a, d=%b) + return(%c) + )"); + + { + auto plan = ExecutionPlanner{*graph}.createPlan(); + + auto& values_to_free = plan->valuesToFree; + EXPECT_EQ(values_to_free.size(), 5); + + for (const auto i : c10::irange(3)) { + EXPECT_TRUE(values_to_free[i].empty()); + } + + EXPECT_EQ(values_to_free[3].size(), 2); + std::set ids{values_to_free[3].begin(), values_to_free[3].end()}; + EXPECT_EQ( + ids, + std::set( + {graph->tryGetValue("a")->id(), graph->tryGetValue("b")->id()})); + + EXPECT_EQ(values_to_free[4].size(), 0); + } + + { + auto static_values = ExecutionPlanner::staticValues(*graph); + std::set static_ids{static_values.begin(), static_values.end()}; + EXPECT_EQ( + static_ids, + std::set( + {graph->tryGetValue("x")->id(), + graph->tryGetValue("y")->id(), + graph->tryGetValue("c")->id()})); + } +} + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_file_util.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_file_util.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7c47a7350e3967db3c33c52957f4d011e6108f15 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_file_util.cpp @@ -0,0 +1,111 @@ +#include +#include +#include + +namespace torch { +namespace nativert { + +TEST(FileUtilTest, OpenNoInt) { + // Create a temporary file + std::ofstream tmpFile("tmp_file.txt"); + tmpFile.close(); + + int fd = openNoInt("tmp_file.txt", O_RDONLY, 0); + ASSERT_GE(fd, 0); + + closeNoInt(fd); +} + +TEST(FileUtilTest, CloseNoInt) { + // Create a temporary file + std::ofstream tmpFile("tmp_file.txt"); + tmpFile.close(); + + int fd = openNoInt("tmp_file.txt", O_RDONLY, 0); + ASSERT_GE(fd, 0); + + int result = closeNoInt(fd); + ASSERT_EQ(result, 0); +} + +TEST(FileUtilTest, WriteFull) { + // Create a temporary file + std::ofstream tmpFile("tmp_file.txt"); + tmpFile.close(); + + int fd = openNoInt("tmp_file.txt", O_WRONLY | O_CREAT, 0644); + ASSERT_GE(fd, 0); + + const char* data = "Hello, World!"; + ssize_t bytesWritten = writeFull(fd, data, strlen(data)); + ASSERT_EQ(bytesWritten, strlen(data)); + + closeNoInt(fd); +} + +TEST(FileUtilTest, ReadFull) { + // Create a temporary file + std::ofstream tmpFile("tmp_file.txt"); + tmpFile << "Hello, World!"; + tmpFile.close(); + + int fd = openNoInt("tmp_file.txt", O_RDONLY, 0); + ASSERT_GE(fd, 0); + + char buffer[1024]; + ssize_t bytesRead = readFull(fd, buffer, 1024); + ASSERT_EQ(bytesRead, 13); // length of "Hello, World!" + + closeNoInt(fd); +} + +TEST(FileUtilTest, FileConstructor) { + // Create a temporary file + std::ofstream tmpFile("tmp_file.txt"); + tmpFile.close(); + + File file("tmp_file.txt", O_RDONLY, 0); + ASSERT_GE(file.fd(), 0); + + file.close(); +} + +TEST(FileUtilTest, FileMoveConstructor) { + // Create a temporary file + std::ofstream tmpFile("tmp_file.txt"); + tmpFile.close(); + + File file1("tmp_file.txt", O_RDONLY, 0); + File file2(std::move(file1)); + + ASSERT_GE(file2.fd(), 0); + ASSERT_EQ(file1.fd(), -1); + + file2.close(); +} + +TEST(FileUtilTest, FileAssignmentOperator) { + // Create a temporary file + std::ofstream tmpFile("tmp_file.txt"); + tmpFile.close(); + + File file1("tmp_file.txt", O_RDONLY, 0); + File file2; + + file2 = std::move(file1); + + ASSERT_GE(file2.fd(), 0); + ASSERT_EQ(file1.fd(), -1); + + file2.close(); +} + +TEST(FileUtilTest, TemporaryFile) { + File file = File::temporary(); + ASSERT_GE(file.fd(), 0); + + file.close(); +} + +} // namespace nativert +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_function_schema.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_function_schema.cpp new file mode 100644 index 0000000000000000000000000000000000000000..aa432ceb81ccffe75c747d71fa9c17061dbc070e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_function_schema.cpp @@ -0,0 +1,70 @@ +#include + +#include +#include +#include + +using namespace ::testing; + +int64_t increment_kernel(const at::Tensor& tensor, int64_t input) { + return input + 1; +} + +at::Tensor slice_kernel(const at::Tensor& tensor, int64_t dim) { + return tensor.slice(dim); +} + +TEST(TestFunctionSchema, testNoAlias) { + auto registrar = c10::RegisterOperators().op( + "_test::my_op(Tensor dummy, int input) -> int", &increment_kernel); + auto handle = c10::Dispatcher::singleton().findSchema({"_test::my_op", ""}); + + EXPECT_TRUE(handle.has_value()); + EXPECT_TRUE(handle->hasSchema()); + + auto nativert_schema = torch::nativert::FunctionSchema(handle->schema()); + + EXPECT_FALSE(nativert_schema.alias(0, 0)); + EXPECT_FALSE(nativert_schema.alias(1, 0)); + + // bounds check + EXPECT_THROW(nativert_schema.alias(2, 0), c10::Error); + EXPECT_THROW(nativert_schema.alias(1, 1), c10::Error); +} + +TEST(TestFunctionSchema, testAliasOverride) { + auto registrar = c10::RegisterOperators().op( + "_test::my_op(Tensor dummy, int input) -> int", &increment_kernel); + auto handle = c10::Dispatcher::singleton().findSchema({"_test::my_op", ""}); + + EXPECT_TRUE(handle.has_value()); + EXPECT_TRUE(handle->hasSchema()); + + auto nativert_schema = + torch::nativert::FunctionSchema(handle->schema(), {{0, 0}}); + + EXPECT_TRUE(nativert_schema.alias(0, 0)); + EXPECT_FALSE(nativert_schema.alias(1, 0)); + + // bounds check + EXPECT_THROW(nativert_schema.alias(2, 0), c10::Error); + EXPECT_THROW(nativert_schema.alias(1, 1), c10::Error); +} + +TEST(TestFunctionSchema, testAlias) { + auto registrar = c10::RegisterOperators().op( + "_test::my_op(Tensor(a) dummy, int input) -> Tensor(a)", &slice_kernel); + auto handle = c10::Dispatcher::singleton().findSchema({"_test::my_op", ""}); + + EXPECT_TRUE(handle.has_value()); + EXPECT_TRUE(handle->hasSchema()); + + auto nativert_schema = torch::nativert::FunctionSchema(handle->schema()); + + EXPECT_TRUE(nativert_schema.alias(0, 0)); + EXPECT_FALSE(nativert_schema.alias(1, 0)); + + // bounds check + EXPECT_THROW(nativert_schema.alias(2, 0), c10::Error); + EXPECT_THROW(nativert_schema.alias(1, 1), c10::Error); +} diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_graph.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_graph.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8130eed4fa30e4d8a2b645bcc449e7de061de112 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_graph.cpp @@ -0,0 +1,647 @@ +#include +#include +#include + +#include + +using namespace ::testing; + +namespace torch::nativert { +TEST(GraphTest, Basic) { + static constexpr std::string_view source = + R"(graph(%foo, %bar, %baz): +%o1, %o2 = aten.foo(self=%foo, target=%bar, alpha=0.1) +return(%o2, %baz) +)"; + auto graph = stringToGraph(source); + EXPECT_EQ(graph->inputs().size(), 3); + EXPECT_EQ(graph->inputs()[0]->name(), "foo"); + EXPECT_EQ(graph->inputs()[1]->name(), "bar"); + EXPECT_EQ(graph->inputs()[2]->name(), "baz"); + + const auto& nodes = graph->nodes(); + EXPECT_EQ(nodes.size(), 3); + // First node is the input node + auto it = nodes.begin(); + { + const auto& node = *it; + EXPECT_EQ(node.target(), "prim.Input"); + EXPECT_EQ(node.inputs().size(), 0); + EXPECT_EQ(node.outputs().size(), 3); + EXPECT_EQ(node.outputs()[0]->name(), "foo"); + EXPECT_EQ(node.outputs()[1]->name(), "bar"); + EXPECT_EQ(node.outputs()[2]->name(), "baz"); + } + { + std::advance(it, 1); + const auto& node = *it; + EXPECT_EQ(node.target(), "aten.foo"); + EXPECT_EQ(node.inputs().size(), 2); + EXPECT_EQ(node.inputs()[0].name, "self"); + EXPECT_EQ(node.inputs()[1].name, "target"); + + EXPECT_EQ(node.attributes().size(), 1); + EXPECT_EQ(node.attributes()[0].name, "alpha"); + } + { + std::advance(it, 1); + const auto& node = *it; + EXPECT_EQ(node.target(), "prim.Output"); + EXPECT_EQ(node.inputs().size(), 2); + EXPECT_EQ(node.inputs()[0].name, "o2"); + EXPECT_EQ(node.inputs()[1].name, "baz"); + } + EXPECT_EQ(graph->outputs().size(), 2); + EXPECT_EQ(graph->outputs()[0]->name(), "o2"); + EXPECT_EQ(graph->outputs()[1]->name(), "baz"); + + const auto& values = graph->values(); + EXPECT_EQ(values.size(), 5); + std::vector valueNames; + valueNames.reserve(values.size()); + for (const auto& v : values) { + valueNames.emplace_back(v->name()); + } + std::sort(valueNames.begin(), valueNames.end()); + + EXPECT_THAT( + valueNames, + ContainerEq(std::vector({"bar", "baz", "foo", "o1", "o2"}))); +} + +TEST(GraphTest, ValueProducer) { + static constexpr std::string_view source = + R"(graph(%foo, %bar, %baz): +%o1, %o2 = aten.foo(self=%foo, target=%bar, alpha=0.1) +return(%o2, %baz) +)"; + auto graph = stringToGraph(source); + auto foo = graph->getValue("foo"); + EXPECT_EQ(foo->producer()->target(), "prim.Input"); + auto o1 = graph->getValue("o1"); + EXPECT_EQ(o1->producer()->target(), "aten.foo"); +} + +TEST(GraphTest, InsertBeforeAfter) { + static constexpr std::string_view source = + R"(graph(%foo, %bar, %baz): +%o1, %o2 = aten.foo(self=%foo, target=%bar, alpha=0.1) +return(%o2, %baz) +)"; + auto graph = stringToGraph(source); + auto it = graph->nodes().begin(); + ++it; + auto& node = *it; + EXPECT_EQ(node.target(), "aten.foo"); + auto before = graph->createNode("before", {}); + auto after = graph->createNode("after", {}); + auto atEnd = graph->createNode("atEnd", {}); + + graph->insertBefore(before, &node); + graph->insertAfter(after, &node); + graph->insert(atEnd); + + static constexpr std::string_view expected = + R"(graph(%foo, %bar, %baz): + = before() +%o1, %o2 = aten.foo(self=%foo, target=%bar, alpha=0.1) + = after() + = atEnd() +return(%o2, %baz) +)"; + EXPECT_EQ(graphToString(*graph), expected); +} + +TEST(GraphTest, ValueUses) { + static constexpr std::string_view source = + R"(graph(%foo, %bar, %baz): +%o1, %o2 = aten.foo(self=%foo, target=%bar, alpha=0.1) +return(%o2, %baz) +)"; + auto graph = stringToGraph(source); + auto o2 = graph->getValue("o2"); + EXPECT_EQ(o2->users().size(), 1); + EXPECT_EQ(o2->users()[0]->target(), "prim.Output"); +} + +TEST(GraphTest, ApplyDevicePlacement) { + auto graph = Graph::createGraph(); + auto node1 = graph->insertNode("node1"); + auto node2 = graph->insertNode("node2"); + + node1->addAttribute({"a", c10::Device(c10::DeviceType::CPU)}); + node1->addAttribute({"b", c10::Device(c10::DeviceType::CUDA, 0)}); + node1->addAttribute({"c", c10::Device(c10::DeviceType::CUDA, 1)}); + + node2->addAttribute({"d", c10::Device(c10::DeviceType::CUDA, 0)}); + + graph->applyDevicePlacement( + Placement(std::unordered_map{ + {c10::Device(c10::DeviceType::CUDA, 0), + c10::Device(c10::DeviceType::CUDA, 1)}})); + + EXPECT_EQ( + std::get(node1->getAttribute("a").value), + c10::Device(c10::DeviceType::CPU)); + EXPECT_EQ( + std::get(node1->getAttribute("b").value), + c10::Device(c10::DeviceType::CUDA, 1)); + EXPECT_EQ( + std::get(node1->getAttribute("c").value), + c10::Device(c10::DeviceType::CUDA, 1)); + EXPECT_EQ( + std::get(node2->getAttribute("d").value), + c10::Device(c10::DeviceType::CUDA, 1)); +} + +TEST(GraphTest, ReplaceAllUses) { + static constexpr std::string_view source = + R"(graph(%foo, %bar, %baz): +%o1, %o2 = aten.foo(self=%foo, target=%bar, alpha=0.1) +return(%o2, %baz) +)"; + auto graph = stringToGraph(source); + auto o2 = graph->getValue("o2"); + auto bar = graph->getValue("bar"); + auto foo = graph->getValue("foo"); + + EXPECT_EQ(o2->users().size(), 1); + EXPECT_EQ(bar->users().size(), 1); + EXPECT_EQ(foo->users().size(), 1); + + graph->replaceAllUses(o2, bar); + EXPECT_EQ(o2->users().size(), 0); + EXPECT_EQ(bar->users().size(), 2); + + graph->replaceAllUses(bar, foo); + EXPECT_EQ(bar->users().size(), 0); + EXPECT_EQ(foo->users().size(), 2); + static constexpr std::string_view expected = + R"(graph(%foo, %bar, %baz): +%o1, %o2 = aten.foo(self=%foo, target=%foo, alpha=0.1) +return(%foo, %baz) +)"; + EXPECT_EQ(graphToString(*graph), expected); +} + +TEST(GraphTest, GetUniqueValueName) { + static constexpr std::string_view source = + R"(graph(%foo, %bar, %baz): +%o1, %o2 = aten.foo(self=%foo, target=%bar, alpha=0.1) +return(%o2, %bar) +)"; + auto graph = stringToGraph(source); + auto o2 = graph->getValue("o2"); + auto fooNode = o2->producer(); + auto v0 = graph->getUniqueValueName(); + graph->addValue(v0, Type::Kind::None, fooNode); + auto v1 = graph->getUniqueValueName(); + graph->addValue(v1, Type::Kind::None, fooNode); + auto v2 = graph->getUniqueValueName(); + EXPECT_EQ(v0, "v0"); + EXPECT_EQ(v1, "v1"); + EXPECT_EQ(v2, "v2"); +} + +TEST(GraphTest, ReplaceAllUsesMultiUse) { + static constexpr std::string_view source = + R"(graph(%foo, %bar): +%o1 = aten.foo(a=%foo, b=%foo, c=%bar) +return(%o1) +)"; + auto graph = stringToGraph(source); + auto foo = graph->getValue("foo"); + auto bar = graph->getValue("bar"); + graph->replaceAllUses(foo, bar); + + static constexpr std::string_view expected = + R"(graph(%foo, %bar): +%o1 = aten.foo(a=%bar, b=%bar, c=%bar) +return(%o1) +)"; + EXPECT_EQ(graphToString(*graph), expected); +} + +TEST(GraphTest, ReplaceAllUsesAfter) { + static constexpr std::string_view source = + R"(graph(%foo): +%o1 = aten.foo1(a=%foo) +%o2 = aten.foo2(a=%o1, b=%foo) +%o3 = aten.foo3(a=%o2, b=%o2, c=%foo) +return(%foo, %o1, %o2, %o3) +)"; + auto graph = stringToGraph(source); + auto foo = graph->getValue("foo"); + auto o1 = graph->getValue("o1"); + auto foo3Node = graph->getValue("o3")->producer(); + graph->replaceAllUsesAfterNode(foo, o1, foo3Node); + + static constexpr std::string_view expected = + R"(graph(%foo): +%o1 = aten.foo1(a=%foo) +%o2 = aten.foo2(a=%o1, b=%foo) +%o3 = aten.foo3(a=%o2, b=%o2, c=%foo) +return(%o1, %o1, %o2, %o3) +)"; + EXPECT_EQ(graphToString(*graph), expected); + EXPECT_EQ(foo->users().size(), 3); + EXPECT_EQ(o1->users().size(), 2); +} + +TEST(GraphTest, InsertingAfter) { + static constexpr std::string_view source = + R"(graph(%foo, %bar): +%o1 = aten.first(a=%foo) +%o2 = aten.foo(c=%bar) +return(%o1, %o2) +)"; + auto graph = stringToGraph(source); + auto origNode = graph->getValue("o1")->producer(); + { + InsertingAfter guard(origNode); + graph->insertNode("one"); + graph->insertNode("two"); + graph->insertNode("three"); + } + graph->insertNode("four"); + static constexpr std::string_view expected = + R"(graph(%foo, %bar): +%o1 = aten.first(a=%foo) + = one() + = two() + = three() +%o2 = aten.foo(c=%bar) + = four() +return(%o1, %o2) +)"; + EXPECT_EQ(graphToString(*graph), expected); +} + +TEST(NodeTest, GetInputAndAttribute) { + auto graph = Graph::createGraph(); + auto input1 = graph->addInput("input1", Type::Kind::Tensor); + auto input2 = graph->addInput("input2", Type::Kind::Tensor); + auto input3 = graph->addInput("input3", Type::Kind::Tensor); + auto node = graph->createNode("foo.bar"); + + node->addInput({"out_of_order", input1}); + node->addInput({"arg1", input2}); + node->addInput({"arg2", input3}); + + node->addAttribute({"b", static_cast(0)}); + node->addAttribute({"a", static_cast(2)}); + node->addAttribute({"c", static_cast(1)}); + { + const auto& input = node->getInput("out_of_order"); + EXPECT_EQ(input.name, "out_of_order"); + EXPECT_EQ(input.value, input1); + } + { + const auto& input = node->getInput("arg1"); + EXPECT_EQ(input.name, "arg1"); + EXPECT_EQ(input.value, input2); + } + { + const auto& input = node->getInput("arg2"); + EXPECT_EQ(input.name, "arg2"); + EXPECT_EQ(input.value, input3); + } + { + const auto& attr = node->getAttribute("a"); + EXPECT_EQ(attr.name, "a"); + EXPECT_EQ(attr.value, Constant(static_cast(2))); + } + { + const auto& attr = node->getAttribute("b"); + EXPECT_EQ(attr.name, "b"); + EXPECT_EQ(attr.value, Constant(static_cast(0))); + } + { + const auto& attr = node->getAttribute("c"); + EXPECT_EQ(attr.name, "c"); + EXPECT_EQ(attr.value, Constant(static_cast(1))); + } + + EXPECT_EQ(node->tryGetInput("doesnotexist"), nullptr); + EXPECT_EQ(node->tryGetAttribute("doesnotexist"), nullptr); +} + +TEST(NodeTest, NextPrev) { + static constexpr std::string_view source = + R"(graph(%foo): +%o1 = aten.foo1(a=%foo) +%o2 = aten.foo2(a=%o1, b=%foo) +%o3 = aten.foo3(a=%o2, b=%o2, c=%foo) +return(%foo, %o1, %o2, %o3) +)"; + auto graph = stringToGraph(source); + auto foo1 = graph->getValue("o1")->producer(); + auto foo2 = graph->getValue("o2")->producer(); + auto foo3 = graph->getValue("o3")->producer(); + EXPECT_EQ(foo1->next(), foo2); + EXPECT_EQ(foo2->next(), foo3); + EXPECT_EQ(foo3->prev(), foo2); + EXPECT_EQ(foo3->next(), graph->outputNode()); + EXPECT_EQ(foo2->prev(), foo1); + EXPECT_EQ(foo1->prev(), graph->inputNode()); + EXPECT_EQ(graph->inputNode()->prev(), nullptr); + EXPECT_EQ(graph->outputNode()->next(), nullptr); +} + +TEST(GraphTest, IsBefore) { + auto source = R"IR( + graph(%foo): + %o1 = aten.foo1(a=%foo) + %o2 = aten.foo2(a=%o1) + %o3 = aten.foo3(a=%o2) + return (%o3) + )IR"; + + auto graph = stringToGraph(source); + ASSERT_NE(graph, nullptr); + + auto* o1 = graph->tryGetValue("o1"); + auto* o2 = graph->tryGetValue("o2"); + auto* o3 = graph->tryGetValue("o3"); + + auto* foo1 = o1->producer(); + auto* foo2 = o2->producer(); + auto* foo3 = o3->producer(); + + EXPECT_TRUE(foo1->isBefore(foo2)) << "foo1 should appear before foo2"; + EXPECT_TRUE(foo2->isBefore(foo3)) << "foo2 should appear before foo3"; + EXPECT_TRUE(foo1->isBefore(foo3)) << "foo1 should appear before foo3"; + + EXPECT_FALSE(foo2->isBefore(foo1)) << "foo2 should not appear before foo1"; + EXPECT_FALSE(foo3->isBefore(foo2)) << "foo3 should not appear before foo2"; +} + +TEST(GraphTest, RemoveNodeWithUsers) { + // Check we shouldn't be able to remove a node that still has users + auto source = R"IR( + graph(%foo): + %o1 = aten.foo1(a=%foo) + %o2 = aten.foo2(a=%o1, b=%foo) + %o3 = aten.foo3(a=%o2, b=%o2, c=%foo) + return (%foo, %o1, %o3) + )IR"; + + auto graph = stringToGraph(source); + ASSERT_NE(graph, nullptr); + + auto* o2 = graph->tryGetValue("o2"); + auto* foo2 = o2->producer(); + + EXPECT_THROW(graph->removeNode(foo2), c10::Error); +} + +TEST(GraphTest, RemoveNodeUnused) { + // Check node removal works as expected + auto source = R"IR( + graph(%foo): + %o1 = aten.foo1(a=%foo) + %o2 = aten.foo2(a=%o1, b=%foo) + %unused = aten.fooUnused(a=%o2) + return(%foo, %o1, %o2) + )IR"; + auto graph = stringToGraph(source); + + auto* valUnused = graph->tryGetValue("unused"); + Node* nodeUnused = valUnused->producer(); + EXPECT_EQ(nodeUnused->target(), "aten.fooUnused"); + + graph->removeNode(nodeUnused); + graph->lint(); + + // %unused should now be gone + EXPECT_EQ(graph->tryGetValue("unused"), nullptr) + << "Value %unused should no longer exist in the graph"; + + for (const auto& node : graph->nodes()) { + EXPECT_NE(node.target(), "aten.fooUnused"); + for (const auto* output : node.outputs()) { + EXPECT_NE(output->name(), "unused") + << "Should not find %unused in any remaining node's outputs"; + } + } +} + +TEST(GraphTest, RemoveValue) { + auto source = R"IR( + graph(%foo): + %o1 = aten.foo1(a=%foo) + %o2 = aten.foo2(a=%o1, b=%foo) + %o3 = aten.foo3(a=%o2, b=%o2, c=%foo) + return (%foo, %o1, %o3) + )IR"; + + auto graph = stringToGraph(source); + auto* val_o1 = graph->tryGetValue("o1"); + + { + // Check we shouldn't be able to remove a value that still has users + EXPECT_THROW(graph->removeValue(val_o1), c10::Error); + } + + { + // Check value removal works as expected + graph->replaceAllUses(val_o1, graph->tryGetValue("foo")); + graph->removeValue(val_o1); + EXPECT_EQ(graph->tryGetValue("%o1"), nullptr); + } +} + +TEST(GraphTest, InsertGraph) { + auto source = R"IR( + graph(%foo): + %o1 = aten.foo1(a=%foo) + return (%o1) + )IR"; + + // Subgraph to be inserted + auto subgraphSource = R"IR( + graph(%x): + %s1 = aten.subFoo1(a=%x) + %s2 = aten.subFoo2(a=%s1) + return (%s2) + )IR"; + + auto mainGraph = stringToGraph(source); + auto subGraph = stringToGraph(subgraphSource); + + // Insert subGraph into mainGraph. Use %o1 as the subGraph's %x + auto val_o1 = mainGraph->tryGetValue("o1"); + std::unordered_map valueMap; + std::vector insertedOutputs = + mainGraph->insertGraph(*subGraph, {val_o1}, valueMap); + + EXPECT_EQ(insertedOutputs.size(), 1); + + // Check all new nodes are inserted correctly from the copied %s2 + auto* newS2 = insertedOutputs.front(); + + auto* newSubFoo2 = newS2->producer(); + EXPECT_EQ(newSubFoo2->target(), "aten.subFoo2"); + + auto* newS1 = newSubFoo2->inputs().front().value; + auto* newSubFoo1 = newS1->producer(); + EXPECT_EQ(newSubFoo1->target(), "aten.subFoo1"); + + EXPECT_EQ(newSubFoo1->inputs().front().value, val_o1); + + auto* subInputVal = subGraph->inputs().front(); + EXPECT_EQ(valueMap[subInputVal], val_o1); + for (const auto& [val1, val2] : valueMap) { + if (val1->name() == "s1") { + EXPECT_EQ(val2->name(), newS1->name()); + } + if (val1->name() == "s2") { + EXPECT_EQ(val2->name(), newS2->name()); + } + if (val1->name() == "x") { + EXPECT_EQ(val2->name(), val_o1->name()); + } + } + + mainGraph->lint(); +} + +TEST(GraphTest, CleanupDeadNodes) { + // %c is unused + const std::string source = R"( + graph(%x, %y): +%a = foo(a=%x, b=%y) +%b = foo1(c=%a) +%c = foo2(a=%b, b=%y) +return(%b) +)"; + auto graph = stringToGraph(source); + + // Verify that %c exists initially + auto* cVal = graph->tryGetValue("c"); + ASSERT_NE(nullptr, cVal); + size_t nodeCountBefore = graph->nodes().size(); + + graph->cleanupDeadNodes(); + + // %c should now be gone + EXPECT_EQ(nullptr, graph->tryGetValue("c")); + // %b should still be there + EXPECT_NE(nullptr, graph->tryGetValue("b")); + EXPECT_EQ(nodeCountBefore - 1, graph->nodes().size()); +} + +TEST(GraphTest, RenumberValues) { + const std::string source = R"( + graph(%x): +%a = foo(a=%x) +%b = foo1(a=%a) +return (%a) +)"; + auto graph = stringToGraph(source); + graph->cleanupDeadNodes(); + + // %b should now be gone + EXPECT_EQ(nullptr, graph->tryGetValue("b")); + + // %a should now be the last value + EXPECT_EQ(graph->tryGetValue("a")->id(), graph->numValues() - 1); + + // All values should be renumbered + size_t numVals = graph->numValues(); + std::unordered_set ids; + ids.reserve(numVals); + for (const auto* val : graph->values()) { + ASSERT_LT(val->id(), numVals); + ids.insert(val->id()); + } + + // Check ids are contiguous and unique b/w 0 and numVals + EXPECT_EQ(numVals, ids.size()); + for (size_t i = 0; i < numVals; ++i) { + EXPECT_NE(ids.end(), ids.find(i)); + } +} + +TEST(SerializationTest, RoundTrip) { + static constexpr std::string_view source = + R"(graph(%foo, %bar, %baz): +%o1 = aten.foo(self=%foo, target=%bar, alpha=0.1) +return(%o1, %baz) +)"; + const auto graph = stringToGraph(source); + const auto serialized = graphToString(*graph); + EXPECT_EQ(source, serialized); +} + +TEST(SerializationTest, EscapedStringConstant) { + const auto parsed = + std::get(convertAtomicConstant(R"("string_\"escape")")); + std::string expected = "string_\\\"escape"; + EXPECT_EQ(parsed, expected); +} + +TEST(SerializationTest, DeviceConstant) { + const auto device = + std::get(convertAtomicConstant("Device{cuda:1}")); + EXPECT_EQ(device.index(), 1); + EXPECT_EQ(device.type(), c10::DeviceType::CUDA); +} + +TEST(SerializationTest, TrueConstant) { + const auto parsedTrue = std::get(convertAtomicConstant("true")); + EXPECT_EQ(parsedTrue, true); + const auto parsedFalse = std::get(convertAtomicConstant("false")); + EXPECT_EQ(parsedFalse, false); +} + +TEST(SerializationTest, MemoryFormatConstant) { + const auto parsed = std::get( + convertAtomicConstant("MemoryFormat::ContiguousFormat")); + EXPECT_EQ(parsed, c10::MemoryFormat::Contiguous); +} + +TEST(SerializationTest, FloatConstant) { + const auto parsed = std::get(convertAtomicConstant("5.0")); + EXPECT_EQ(parsed, 5.0); +} + +TEST(SerializationTest, IntConstant) { + const auto parsed = std::get(convertAtomicConstant("5")); + EXPECT_EQ(parsed, 5); +} + +TEST(SerializationTest, FloatExponentConstant) { + const auto parsed = std::get(convertAtomicConstant("1e-05")); + EXPECT_EQ(parsed, 0.00001); +} + +TEST(SerializationTest, SingleElementListConstant) { + const auto parsed = + std::get>(convertListConstant("[1]")); + const auto expected = std::vector{1}; + EXPECT_EQ(parsed, expected); +} + +TEST(SerializationTest, IntListConstant) { + const auto parsed = + std::get>(convertListConstant("[1, 2, 3, 4]")); + const auto expected = std::vector{1, 2, 3, 4}; + EXPECT_EQ(parsed, expected); +} + +TEST(SerializationTest, FloatListConstant) { + const auto parsed = std::get>( + convertListConstant("[1.0, 2.0, 3.0, 4.0]")); + const auto expected = std::vector{1.0, 2.0, 3.0, 4.0}; + EXPECT_EQ(parsed, expected); +} + +TEST(SerializationTest, BoolListConstant) { + const auto parsed = + std::get>(convertListConstant("[false, true, false]")); + const auto expected = std::vector{false, true, false}; + EXPECT_EQ(parsed, expected); +} + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_graph_signature.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_graph_signature.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a91e8639a6f40754a2cae3292c7553ccf23bb4fd --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_graph_signature.cpp @@ -0,0 +1,584 @@ +#include +#include + +namespace torch::nativert { + +class GraphSignatureTest : public ::testing::Test { + protected: + // Member to hold the GraphSignature object + GraphSignature graph_sig; + + void SetUp() override { + torch::_export::TensorArgument param_tensor_arg; + param_tensor_arg.set_name("param"); + torch::_export::InputToParameterSpec param_input_spec; + param_input_spec.set_arg(param_tensor_arg); + param_input_spec.set_parameter_name("param"); + torch::_export::InputSpec input_spec_0; + input_spec_0.set_parameter(param_input_spec); + + torch::_export::TensorArgument input_tensor_arg; + input_tensor_arg.set_name("input"); + torch::_export::Argument input_arg; + input_arg.set_as_tensor(input_tensor_arg); + torch::_export::UserInputSpec user_input_spec; + user_input_spec.set_arg(input_arg); + torch::_export::InputSpec input_spec_1; + input_spec_1.set_user_input(user_input_spec); + + torch::_export::TensorArgument loss_tensor_arg; + loss_tensor_arg.set_name("loss"); + torch::_export::LossOutputSpec loss_output_spec; + loss_output_spec.set_arg(loss_tensor_arg); + torch::_export::OutputSpec output_spec_0; + output_spec_0.set_loss_output(loss_output_spec); + + torch::_export::TensorArgument output_tensor_arg; + output_tensor_arg.set_name("output"); + torch::_export::Argument output_arg; + output_arg.set_as_tensor(output_tensor_arg); + torch::_export::UserOutputSpec user_output_spec; + user_output_spec.set_arg(output_arg); + torch::_export::OutputSpec output_spec_1; + output_spec_1.set_user_output(user_output_spec); + + torch::_export::GraphSignature mock_storage; + mock_storage.set_input_specs({input_spec_0, input_spec_1}); + mock_storage.set_output_specs({output_spec_0, output_spec_1}); + + // Initialize the GraphSignature object + graph_sig = GraphSignature(mock_storage); + } +}; + +// Helper function to create a tensor argument +torch::_export::TensorArgument makeTensorArg(const std::string& name) { + torch::_export::TensorArgument arg; + arg.set_name(name); + return arg; +} + +// Helper function to create a SymInt argument with a name +torch::_export::SymIntArgument makeSymIntArg(const std::string& name) { + torch::_export::SymIntArgument arg; + arg.set_as_name(name); + return arg; +} + +// Helper function to create a SymInt argument with a constant int +torch::_export::SymIntArgument makeSymIntArgConst(int64_t value) { + torch::_export::SymIntArgument arg; + arg.set_as_int(value); + return arg; +} + +// Helper function to create a SymBool argument with a name +torch::_export::SymBoolArgument makeSymBoolArg(const std::string& name) { + torch::_export::SymBoolArgument arg; + arg.set_as_name(name); + return arg; +} + +// Helper function to create an OptionalTensorArgument with a tensor +torch::_export::OptionalTensorArgument makeOptionalTensorArg( + const std::string& name) { + torch::_export::OptionalTensorArgument arg; + arg.set_as_tensor(makeTensorArg(name)); + return arg; +} + +// Helper function to create an OptionalTensorArgument with None +torch::_export::OptionalTensorArgument makeOptionalTensorArgNone() { + torch::_export::OptionalTensorArgument arg; + arg.set_as_none({}); + return arg; +} + +// Test the constructor with a simple GraphSignature +TEST_F(GraphSignatureTest, ConstructorTest) { + std::vector expected_params = {"param"}; + EXPECT_EQ(graph_sig.parameters(), expected_params); + + std::vector expected_inputs = {"input"}; + EXPECT_EQ(graph_sig.userInputs(), expected_inputs); + + EXPECT_EQ(graph_sig.userInputs().size(), 1); + EXPECT_EQ(graph_sig.parameters().size(), 1); + EXPECT_EQ(graph_sig.lossOutput(), "loss"); + + std::vector> expected_outputs = {"output"}; + EXPECT_EQ(graph_sig.userOutputs(), expected_outputs); +} + +// Test the replaceAllUses method +TEST_F(GraphSignatureTest, ReplaceAllUsesTest) { + graph_sig.replaceAllUses("output", "new_output"); + std::vector> expected_outputs = {"new_output"}; + EXPECT_EQ(graph_sig.userOutputs(), expected_outputs); +} + +// Test AS_TENSORS user input handling - each tensor in the list should become +// a separate entry in userInputs +TEST(GraphSignatureInputTypesTest, UserInputAsTensors) { + // Create list of tensor arguments + std::vector tensor_list; + tensor_list.push_back(makeTensorArg("tensor_0")); + tensor_list.push_back(makeTensorArg("tensor_1")); + tensor_list.push_back(makeTensorArg("tensor_2")); + + torch::_export::Argument input_arg; + input_arg.set_as_tensors(tensor_list); + torch::_export::UserInputSpec user_input_spec; + user_input_spec.set_arg(input_arg); + torch::_export::InputSpec input_spec; + input_spec.set_user_input(user_input_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({input_spec}); + storage.set_output_specs({}); + + GraphSignature sig(storage); + + // All tensors in the list should be expanded to individual user inputs + std::vector expected_inputs = { + "tensor_0", "tensor_1", "tensor_2"}; + EXPECT_EQ(sig.userInputs(), expected_inputs); +} + +// Test AS_OPTIONAL_TENSOR user input handling +TEST(GraphSignatureInputTypesTest, UserInputAsOptionalTensor) { + // Create an optional tensor that has a value + torch::_export::Argument input_arg; + input_arg.set_as_optional_tensor(makeOptionalTensorArg("opt_tensor")); + torch::_export::UserInputSpec user_input_spec; + user_input_spec.set_arg(input_arg); + torch::_export::InputSpec input_spec; + input_spec.set_user_input(user_input_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({input_spec}); + storage.set_output_specs({}); + + GraphSignature sig(storage); + + std::vector expected_inputs = {"opt_tensor"}; + EXPECT_EQ(sig.userInputs(), expected_inputs); +} + +// Test AS_OPTIONAL_TENSOR with None value - should be skipped +TEST(GraphSignatureInputTypesTest, UserInputAsOptionalTensorNone) { + torch::_export::Argument input_arg; + input_arg.set_as_optional_tensor(makeOptionalTensorArgNone()); + torch::_export::UserInputSpec user_input_spec; + user_input_spec.set_arg(input_arg); + torch::_export::InputSpec input_spec; + input_spec.set_user_input(user_input_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({input_spec}); + storage.set_output_specs({}); + + GraphSignature sig(storage); + + // None optional tensor should be skipped + EXPECT_TRUE(sig.userInputs().empty()); +} + +// Test AS_OPTIONAL_TENSORS user input handling - mixed tensors and None values +TEST(GraphSignatureInputTypesTest, UserInputAsOptionalTensors) { + std::vector opt_tensors; + opt_tensors.push_back(makeOptionalTensorArg("opt_tensor_0")); + opt_tensors.push_back( + makeOptionalTensorArgNone()); // None - should be skipped + opt_tensors.push_back(makeOptionalTensorArg("opt_tensor_2")); + + torch::_export::Argument input_arg; + input_arg.set_as_optional_tensors(opt_tensors); + torch::_export::UserInputSpec user_input_spec; + user_input_spec.set_arg(input_arg); + torch::_export::InputSpec input_spec; + input_spec.set_user_input(user_input_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({input_spec}); + storage.set_output_specs({}); + + GraphSignature sig(storage); + + // Only non-None tensors should be in user inputs + std::vector expected_inputs = {"opt_tensor_0", "opt_tensor_2"}; + EXPECT_EQ(sig.userInputs(), expected_inputs); +} + +// Test AS_SYM_INT user input handling +TEST(GraphSignatureInputTypesTest, UserInputAsSymInt) { + torch::_export::Argument input_arg; + input_arg.set_as_sym_int(makeSymIntArg("sym_int_0")); + torch::_export::UserInputSpec user_input_spec; + user_input_spec.set_arg(input_arg); + torch::_export::InputSpec input_spec; + input_spec.set_user_input(user_input_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({input_spec}); + storage.set_output_specs({}); + + GraphSignature sig(storage); + + std::vector expected_inputs = {"sym_int_0"}; + EXPECT_EQ(sig.userInputs(), expected_inputs); +} + +// Test AS_SYM_INT with constant value - should be skipped +TEST(GraphSignatureInputTypesTest, UserInputAsSymIntConstant) { + torch::_export::Argument input_arg; + input_arg.set_as_sym_int(makeSymIntArgConst(42)); + torch::_export::UserInputSpec user_input_spec; + user_input_spec.set_arg(input_arg); + torch::_export::InputSpec input_spec; + input_spec.set_user_input(user_input_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({input_spec}); + storage.set_output_specs({}); + + GraphSignature sig(storage); + + // Constant symints should be skipped + EXPECT_TRUE(sig.userInputs().empty()); +} + +// Test AS_SYM_INTS user input handling - mixed names and constants +TEST(GraphSignatureInputTypesTest, UserInputAsSymInts) { + std::vector sym_ints; + sym_ints.push_back(makeSymIntArg("s0")); + sym_ints.push_back(makeSymIntArgConst(8)); // Constant - should be skipped + sym_ints.push_back(makeSymIntArg("s1")); + + torch::_export::Argument input_arg; + input_arg.set_as_sym_ints(sym_ints); + torch::_export::UserInputSpec user_input_spec; + user_input_spec.set_arg(input_arg); + torch::_export::InputSpec input_spec; + input_spec.set_user_input(user_input_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({input_spec}); + storage.set_output_specs({}); + + GraphSignature sig(storage); + + // Only named symints should be in user inputs + std::vector expected_inputs = {"s0", "s1"}; + EXPECT_EQ(sig.userInputs(), expected_inputs); +} + +// Test AS_TENSORS user output handling +TEST(GraphSignatureOutputTypesTest, UserOutputAsTensors) { + std::vector tensor_list; + tensor_list.push_back(makeTensorArg("out_tensor_0")); + tensor_list.push_back(makeTensorArg("out_tensor_1")); + + torch::_export::Argument output_arg; + output_arg.set_as_tensors(tensor_list); + torch::_export::UserOutputSpec user_output_spec; + user_output_spec.set_arg(output_arg); + torch::_export::OutputSpec output_spec; + output_spec.set_user_output(user_output_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({}); + storage.set_output_specs({output_spec}); + + GraphSignature sig(storage); + + std::vector> expected_outputs = { + "out_tensor_0", "out_tensor_1"}; + EXPECT_EQ(sig.userOutputs(), expected_outputs); +} + +// Test AS_OPTIONAL_TENSOR user output handling +TEST(GraphSignatureOutputTypesTest, UserOutputAsOptionalTensor) { + torch::_export::Argument output_arg; + output_arg.set_as_optional_tensor(makeOptionalTensorArg("opt_out_tensor")); + torch::_export::UserOutputSpec user_output_spec; + user_output_spec.set_arg(output_arg); + torch::_export::OutputSpec output_spec; + output_spec.set_user_output(user_output_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({}); + storage.set_output_specs({output_spec}); + + GraphSignature sig(storage); + + std::vector> expected_outputs = {"opt_out_tensor"}; + EXPECT_EQ(sig.userOutputs(), expected_outputs); +} + +// Test AS_OPTIONAL_TENSOR user output with None - should have nullopt name +TEST(GraphSignatureOutputTypesTest, UserOutputAsOptionalTensorNone) { + torch::_export::Argument output_arg; + output_arg.set_as_optional_tensor(makeOptionalTensorArgNone()); + torch::_export::UserOutputSpec user_output_spec; + user_output_spec.set_arg(output_arg); + torch::_export::OutputSpec output_spec; + output_spec.set_user_output(user_output_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({}); + storage.set_output_specs({output_spec}); + + GraphSignature sig(storage); + + // None optional tensor should result in nullopt + EXPECT_EQ(sig.userOutputs().size(), 1); + EXPECT_FALSE(sig.userOutputs()[0].has_value()); +} + +// Test AS_OPTIONAL_TENSORS user output handling - mixed tensors and None +TEST(GraphSignatureOutputTypesTest, UserOutputAsOptionalTensors) { + std::vector opt_tensors; + opt_tensors.push_back(makeOptionalTensorArg("opt_out_0")); + opt_tensors.push_back(makeOptionalTensorArgNone()); + opt_tensors.push_back(makeOptionalTensorArg("opt_out_2")); + + torch::_export::Argument output_arg; + output_arg.set_as_optional_tensors(opt_tensors); + torch::_export::UserOutputSpec user_output_spec; + user_output_spec.set_arg(output_arg); + torch::_export::OutputSpec output_spec; + output_spec.set_user_output(user_output_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({}); + storage.set_output_specs({output_spec}); + + GraphSignature sig(storage); + + EXPECT_EQ(sig.userOutputs().size(), 3); + EXPECT_EQ(sig.userOutputs()[0], "opt_out_0"); + EXPECT_FALSE(sig.userOutputs()[1].has_value()); // None becomes nullopt + EXPECT_EQ(sig.userOutputs()[2], "opt_out_2"); +} + +// Test AS_SYM_INT user output handling +TEST(GraphSignatureOutputTypesTest, UserOutputAsSymInt) { + torch::_export::Argument output_arg; + output_arg.set_as_sym_int(makeSymIntArg("sym_out_0")); + torch::_export::UserOutputSpec user_output_spec; + user_output_spec.set_arg(output_arg); + torch::_export::OutputSpec output_spec; + output_spec.set_user_output(user_output_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({}); + storage.set_output_specs({output_spec}); + + GraphSignature sig(storage); + + std::vector> expected_outputs = {"sym_out_0"}; + EXPECT_EQ(sig.userOutputs(), expected_outputs); +} + +// Test AS_SYM_INTS user output handling +TEST(GraphSignatureOutputTypesTest, UserOutputAsSymInts) { + std::vector sym_ints; + sym_ints.push_back(makeSymIntArg("sym_out_0")); + sym_ints.push_back(makeSymIntArgConst(10)); // Constant - should be skipped + sym_ints.push_back(makeSymIntArg("sym_out_1")); + + torch::_export::Argument output_arg; + output_arg.set_as_sym_ints(sym_ints); + torch::_export::UserOutputSpec user_output_spec; + user_output_spec.set_arg(output_arg); + torch::_export::OutputSpec output_spec; + output_spec.set_user_output(user_output_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({}); + storage.set_output_specs({output_spec}); + + GraphSignature sig(storage); + + // Only named symints should appear + std::vector> expected_outputs = { + "sym_out_0", "sym_out_1"}; + EXPECT_EQ(sig.userOutputs(), expected_outputs); +} + +// Test AS_SYM_BOOL user output handling +TEST(GraphSignatureOutputTypesTest, UserOutputAsSymBool) { + torch::_export::Argument output_arg; + output_arg.set_as_sym_bool(makeSymBoolArg("sym_bool_out")); + torch::_export::UserOutputSpec user_output_spec; + user_output_spec.set_arg(output_arg); + torch::_export::OutputSpec output_spec; + output_spec.set_user_output(user_output_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({}); + storage.set_output_specs({output_spec}); + + GraphSignature sig(storage); + + std::vector> expected_outputs = {"sym_bool_out"}; + EXPECT_EQ(sig.userOutputs(), expected_outputs); +} + +// Test AS_SYM_BOOLS user output handling +TEST(GraphSignatureOutputTypesTest, UserOutputAsSymBools) { + std::vector sym_bools; + sym_bools.push_back(makeSymBoolArg("sym_bool_0")); + sym_bools.push_back(makeSymBoolArg("sym_bool_1")); + + torch::_export::Argument output_arg; + output_arg.set_as_sym_bools(sym_bools); + torch::_export::UserOutputSpec user_output_spec; + user_output_spec.set_arg(output_arg); + torch::_export::OutputSpec output_spec; + output_spec.set_user_output(user_output_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({}); + storage.set_output_specs({output_spec}); + + GraphSignature sig(storage); + + std::vector> expected_outputs = { + "sym_bool_0", "sym_bool_1"}; + EXPECT_EQ(sig.userOutputs(), expected_outputs); +} + +// Test AS_SYM_FLOAT user output handling - should result in nullopt (unnamed) +TEST(GraphSignatureOutputTypesTest, UserOutputAsSymFloat) { + torch::_export::SymFloatArgument sym_float; + sym_float.set_as_name("sym_float_out"); + torch::_export::Argument output_arg; + output_arg.set_as_sym_float(sym_float); + torch::_export::UserOutputSpec user_output_spec; + user_output_spec.set_arg(output_arg); + torch::_export::OutputSpec output_spec; + output_spec.set_user_output(user_output_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({}); + storage.set_output_specs({output_spec}); + + GraphSignature sig(storage); + + // SymFloat outputs currently result in nullopt (unnamed) + EXPECT_EQ(sig.userOutputs().size(), 1); + EXPECT_FALSE(sig.userOutputs()[0].has_value()); +} + +// Test AS_SYM_FLOATS user output handling +TEST(GraphSignatureOutputTypesTest, UserOutputAsSymFloats) { + torch::_export::SymFloatArgument sym_float_0; + sym_float_0.set_as_name("sym_float_0"); + torch::_export::SymFloatArgument sym_float_1; + sym_float_1.set_as_name("sym_float_1"); + std::vector sym_floats = { + sym_float_0, sym_float_1}; + + torch::_export::Argument output_arg; + output_arg.set_as_sym_floats(sym_floats); + torch::_export::UserOutputSpec user_output_spec; + user_output_spec.set_arg(output_arg); + torch::_export::OutputSpec output_spec; + output_spec.set_user_output(user_output_spec); + + torch::_export::GraphSignature storage; + storage.set_input_specs({}); + storage.set_output_specs({output_spec}); + + GraphSignature sig(storage); + + // SymFloats outputs are treated as unnamed (nullopt) + EXPECT_EQ(sig.userOutputs().size(), 2); + for (const auto& output : sig.userOutputs()) { + EXPECT_FALSE(output.has_value()); + } +} + +// Test mixed input types in the same GraphSignature +TEST(GraphSignatureMixedTypesTest, MixedInputTypes) { + // First input: single tensor + torch::_export::Argument input_arg_0; + input_arg_0.set_as_tensor(makeTensorArg("single_tensor")); + torch::_export::UserInputSpec user_input_spec_0; + user_input_spec_0.set_arg(input_arg_0); + torch::_export::InputSpec input_spec_0; + input_spec_0.set_user_input(user_input_spec_0); + + // Second input: list of tensors + std::vector tensor_list; + tensor_list.push_back(makeTensorArg("list_tensor_0")); + tensor_list.push_back(makeTensorArg("list_tensor_1")); + torch::_export::Argument input_arg_1; + input_arg_1.set_as_tensors(tensor_list); + torch::_export::UserInputSpec user_input_spec_1; + user_input_spec_1.set_arg(input_arg_1); + torch::_export::InputSpec input_spec_1; + input_spec_1.set_user_input(user_input_spec_1); + + // Third input: symint + torch::_export::Argument input_arg_2; + input_arg_2.set_as_sym_int(makeSymIntArg("batch_size")); + torch::_export::UserInputSpec user_input_spec_2; + user_input_spec_2.set_arg(input_arg_2); + torch::_export::InputSpec input_spec_2; + input_spec_2.set_user_input(user_input_spec_2); + + torch::_export::GraphSignature storage; + storage.set_input_specs({input_spec_0, input_spec_1, input_spec_2}); + storage.set_output_specs({}); + + GraphSignature sig(storage); + + std::vector expected_inputs = { + "single_tensor", "list_tensor_0", "list_tensor_1", "batch_size"}; + EXPECT_EQ(sig.userInputs(), expected_inputs); +} + +// Test mixed output types in the same GraphSignature +TEST(GraphSignatureMixedTypesTest, MixedOutputTypes) { + // First output: single tensor + torch::_export::Argument output_arg_0; + output_arg_0.set_as_tensor(makeTensorArg("single_out")); + torch::_export::UserOutputSpec user_output_spec_0; + user_output_spec_0.set_arg(output_arg_0); + torch::_export::OutputSpec output_spec_0; + output_spec_0.set_user_output(user_output_spec_0); + + // Second output: list of tensors + std::vector tensor_list; + tensor_list.push_back(makeTensorArg("list_out_0")); + tensor_list.push_back(makeTensorArg("list_out_1")); + torch::_export::Argument output_arg_1; + output_arg_1.set_as_tensors(tensor_list); + torch::_export::UserOutputSpec user_output_spec_1; + user_output_spec_1.set_arg(output_arg_1); + torch::_export::OutputSpec output_spec_1; + output_spec_1.set_user_output(user_output_spec_1); + + // Third output: symint + torch::_export::Argument output_arg_2; + output_arg_2.set_as_sym_int(makeSymIntArg("out_size")); + torch::_export::UserOutputSpec user_output_spec_2; + user_output_spec_2.set_arg(output_arg_2); + torch::_export::OutputSpec output_spec_2; + output_spec_2.set_user_output(user_output_spec_2); + + torch::_export::GraphSignature storage; + storage.set_input_specs({}); + storage.set_output_specs({output_spec_0, output_spec_1, output_spec_2}); + + GraphSignature sig(storage); + + std::vector> expected_outputs = { + "single_out", "list_out_0", "list_out_1", "out_size"}; + EXPECT_EQ(sig.userOutputs(), expected_outputs); +} + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_itree.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_itree.cpp new file mode 100644 index 0000000000000000000000000000000000000000..43145578d4f52369e067d069b57cba4398064237 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_itree.cpp @@ -0,0 +1,1347 @@ +#include +#include + +#include + +#include +#include +#include + +namespace torch::nativert::detail { + +using torch::nativert::Graph; +using torch::nativert::stringToGraph; +using torch::nativert::Type; +using torch::nativert::Value; + +std::pair, std::vector> makeValues( + int count) { + auto graph = Graph::createGraph(); + std::vector values; + + for (int i = 0; i < count; i++) { + std::string name = fmt::format("v{}", i); + Value* value = graph->addValue(name, Type::Kind::None, nullptr); + values.push_back(value); + } + + return std::make_pair(std::move(graph), values); +} + +TEST(ITreeTest, Unflatten) { + // Original data: [(0, 1), 2, {"4": 7, "5": 8, "6": 9}, (10,), {"11": 12}] + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": "builtins.dict", + "context": "[\"4\", \"5\", \"6\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": "torch.fx.immutable_collections.immutable_list", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": "torch.fx.immutable_collections.immutable_dict", + "context": "[\"11\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + + auto [graph, valuePtrs] = makeValues(8); + + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + std::vector flats = { + c10::IValue(0), + c10::IValue(1), + c10::IValue(2), + c10::IValue(7), + c10::IValue(8), + c10::IValue(9), + c10::IValue(10), + c10::IValue(12), + }; + auto itree = itreeUnflatten(flats, spec); + EXPECT_TRUE(itree.isList()); + EXPECT_EQ(itree.toListRef().size(), 5); + + EXPECT_TRUE(itree.toListRef().at(0).isTuple()); + EXPECT_EQ(itree.toListRef().at(0).toTupleRef().elements()[0], c10::IValue(0)); + EXPECT_EQ(itree.toListRef().at(0).toTupleRef().elements()[1], c10::IValue(1)); + + EXPECT_TRUE(itree.toListRef().at(1).isInt()); + EXPECT_EQ(itree.toListRef().at(1), c10::IValue(2)); + + EXPECT_TRUE(itree.toListRef().at(2).isGenericDict()); + EXPECT_EQ(itree.toListRef().at(2).toGenericDict().at("4"), c10::IValue(7)); + EXPECT_EQ(itree.toListRef().at(2).toGenericDict().at("5"), c10::IValue(8)); + EXPECT_EQ(itree.toListRef().at(2).toGenericDict().at("6"), c10::IValue(9)); + + EXPECT_TRUE(itree.toListRef().at(3).isList()); + EXPECT_EQ(itree.toListRef().at(3).toListRef().at(0), c10::IValue(10)); + + EXPECT_TRUE(itree.toListRef().at(4).isGenericDict()); + EXPECT_EQ(itree.toListRef().at(4).toGenericDict().at("11"), c10::IValue(12)); + + const auto flattened = itreeFlatten(itree, spec); + EXPECT_EQ(flattened.size(), flats.size()); + for (size_t i = 0; i < flattened.size(); i++) { + EXPECT_EQ(flattened[i], flats[i]); + } +} + +TEST(ITreeTest, NoVersion) { + auto jsonSpec = R"( + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } + )"; + + auto [graph, valuePtrs] = makeValues(2); + EXPECT_THROW({ itreeSpecLoads(jsonSpec, valuePtrs); }, std::exception); +} + +TEST(ITreeTest, NoField) { + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "children_spec": [] + }, + { + "type": "builtins.dict", + "context": "[\"4\", \"5\", \"6\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + + auto [graph, valuePtrs] = makeValues(3); + EXPECT_THROW(itreeSpecLoads(jsonSpec, valuePtrs), std::exception); +} + +TEST(ITreeTest, NoContext) { + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": "builtins.dict", + "context": "[]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + auto [graph, valuePtrs] = makeValues(3); + auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + + std::vector flats = { + c10::IValue(7), + c10::IValue(8), + c10::IValue(9), + }; + EXPECT_THROW({ itreeUnflatten(flats, spec); }, c10::Error); +} + +TEST(ITreeTest, TooManyContext) { + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": "builtins.dict", + "context": "[\"4\", \"5\", \"6\", \"10\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + + auto [graph, valuePtrs] = makeValues(3); + auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + + std::vector flats = { + c10::IValue(7), + c10::IValue(8), + c10::IValue(9), + }; + EXPECT_THROW({ itreeUnflatten(flats, spec); }, c10::Error); +} + +TEST(ITreeTest, DoubleRegister) { + EXPECT_THROW( + { registerPytreeNode("builtins.dict", NodeDef{}); }, std::exception); +} + +TEST(ITreeTest, NotEnoughUnflatten) { + // Original data: [(0, 1), 2, {"4": 7, "5": 8, "6": 9}] + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": "builtins.dict", + "context": "[\"4\", \"5\", \"6\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + auto [graph, valuePtrs] = makeValues(6); + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + std::vector flats = { + c10::IValue(0), + c10::IValue(1), + c10::IValue(2), + c10::IValue(7), + }; + EXPECT_THROW({ itreeUnflatten(flats, spec); }, c10::Error); +} + +TEST(ITreeTest, TooManyUnflatten) { + // Original data: [(0, 1), 2, {"4": 7, "5": 8, "6": 9}] + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": "builtins.dict", + "context": "[\"4\", \"5\", \"6\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + auto [graph, valuePtrs] = makeValues(6); + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + std::vector flats = { + c10::IValue(0), + c10::IValue(1), + c10::IValue(2), + c10::IValue(7), + c10::IValue(0), + c10::IValue(1), + c10::IValue(2), + c10::IValue(7), + c10::IValue(0), + c10::IValue(1), + c10::IValue(2), + c10::IValue(7), + }; + EXPECT_THROW({ itreeUnflatten(flats, spec); }, c10::Error); +} + +TEST(ITreeTest, Flatten) { + // Original data: [(0, 1), 2, {"4": 7, "5": 8, "6": 9}, (10,), {"11": 12}] + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": "builtins.dict", + "context": "[\"4\", \"5\", \"6\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": "torch.fx.immutable_collections.immutable_list", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": "torch.fx.immutable_collections.immutable_dict", + "context": "[\"11\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + auto [graph, valuePtrs] = makeValues(8); + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + auto tup = c10::ivalue::Tuple::create({c10::IValue(0), c10::IValue(1)}); + c10::Dict dict( + c10::StringType::get(), c10::AnyType::get()); + dict.insert("4", c10::IValue(7)); + dict.insert("5", c10::IValue(8)); + dict.insert("6", c10::IValue(9)); + c10::List ilist(c10::AnyType::get()); + ilist.push_back(c10::IValue(10)); + c10::Dict idict( + c10::StringType::get(), c10::AnyType::get()); + idict.insert("11", c10::IValue(12)); + c10::List list(c10::AnyType::get()); + list.push_back(std::move(tup)); + list.push_back(c10::IValue(2)); + list.push_back(std::move(dict)); + list.push_back(std::move(ilist)); + list.push_back(std::move(idict)); + auto flats = itreeFlatten(c10::IValue{list}, spec); + std::vector expected = { + c10::IValue(0), + c10::IValue(1), + c10::IValue(2), + c10::IValue(7), + c10::IValue(8), + c10::IValue(9), + c10::IValue(10), + c10::IValue(12), + }; + for (const auto& [i, flat] : c10::enumerate(flats)) { + EXPECT_EQ(flat, expected.at(i)); + } +} + +TEST(ITreeTest, IValueApplyFromArgs) { + // inputSpec for testing is generated from E2ETestModelWithNestedDictInput + /* + args = ( + { + "a": ( + torch.rand(4, 4), + { + 123: (torch.rand(4, 4), torch.rand(4, 4)), + 234: (torch.rand(4, 4), torch.rand(4, 4)), + }, + ), + "b": ( + torch.rand(4, 4), + { + 345: (torch.rand(4, 4), torch.rand(4, 4)), + 456: (torch.rand(4, 4), torch.rand(4, 4)), + }, + ), + }, + )*/ + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": "builtins.dict", + "context": "[\"a\", \"b\"]", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": "builtins.dict", + "context": "[123, 234]", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } + ] + }, + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": "builtins.dict", + "context": "[345, 456]", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "type": "builtins.dict", + "context": "[]", + "children_spec": [] + } + ] + } +] + )"; + + auto tup_a1_123 = + c10::ivalue::Tuple::create({c10::IValue(1), c10::IValue(2)}); + auto tup_a1_234 = + c10::ivalue::Tuple::create({c10::IValue(3), c10::IValue(4)}); + c10::Dict dict_a1( + c10::StringType::get(), c10::AnyType::get()); + dict_a1.insert(123, tup_a1_123); + dict_a1.insert(234, tup_a1_234); + auto tup_a = + c10::ivalue::Tuple::create({c10::IValue(0), c10::IValue(dict_a1)}); + + auto tup_b1_345 = + c10::ivalue::Tuple::create({c10::IValue(6), c10::IValue(7)}); + auto tup_b1_456 = + c10::ivalue::Tuple::create({c10::IValue(8), c10::IValue(9)}); + c10::Dict dict_b1( + c10::StringType::get(), c10::AnyType::get()); + dict_b1.insert(345, tup_b1_345); + dict_b1.insert(456, tup_b1_456); + auto tup_b = + c10::ivalue::Tuple::create({c10::IValue(5), c10::IValue(dict_b1)}); + + c10::Dict dict( + c10::StringType::get(), c10::AnyType::get()); + dict.insert("a", tup_a); + dict.insert("b", tup_b); + std::vector args = {c10::IValue(dict)}; + + for (int usedIdx = 0; usedIdx < 10; usedIdx++) { + std::vector isUsed(10, false); + isUsed[usedIdx] = true; + std::stringstream ss; + for (int i = 0; i < 10; ++i) { + if (isUsed[i]) { + ss << fmt::format("%o1 = aten.foo(a=%a{})\n", i); + } + } + std::string source = fmt::format( + R"(graph(%a0, %a1, %a2, %a3, %a4, %a5, %a6, %a7, %a8, %a9): +{} +return(%o1) +)", + std::move(ss).str()); + + auto graph = stringToGraph(source); + std::vector userInputs( + graph->userInputs().begin(), graph->userInputs().end()); + + const auto spec = itreeSpecLoads(jsonSpec, userInputs); + + std::vector visited; + auto fn = [&](const c10::IValue& leaf, const Value* value) { + visited.push_back(value->id()); + }; + ivalueApplyFromArgs(fn, args, {}, spec); + + EXPECT_EQ(visited.size(), 1); + EXPECT_EQ(visited[0], usedIdx); + } +} + +TEST(ITreeTest, UnmatchedFlattenType) { + // Original data: [(0, 1), 2, {"4": 7, "5": 8, "6": 9}] + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": "builtins.dict", + "context": "[\"4\", \"5\", \"6\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + auto [graph, valuePtrs] = makeValues(6); + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + auto tup = c10::ivalue::Tuple::create({c10::IValue(0), c10::IValue(1)}); + c10::Dict dict( + c10::StringType::get(), c10::AnyType::get()); + dict.insert("4", c10::IValue(7)); + dict.insert("5", c10::IValue(8)); + dict.insert("6", c10::IValue(9)); + EXPECT_THROW( + { itreeFlatten(c10::IValue{std::move(dict)}, spec); }, std::exception); +} + +TEST(ITreeTest, UnmatchedDictFlatten) { + // Original data: [(0, 1), 2, {"4": 7, "5": 8, "6": 9}] + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": "builtins.dict", + "context": "[\"4\", \"5\", \"6\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + auto [graph, valuePtrs] = makeValues(6); + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + auto tup = c10::ivalue::Tuple::create({c10::IValue(0), c10::IValue(1)}); + c10::Dict dict( + c10::StringType::get(), c10::AnyType::get()); + dict.insert("4", c10::IValue(7)); + dict.insert("5", c10::IValue(8)); + dict.insert("100", c10::IValue(8)); + dict.insert("101", c10::IValue(8)); + c10::List list(c10::AnyType::get()); + list.push_back(std::move(tup)); + list.push_back(c10::IValue(2)); + list.push_back(std::move(dict)); + EXPECT_THROW( + { itreeFlatten(c10::IValue{std::move(list)}, spec); }, c10::Error); +} + +TEST(ITreeTest, DictFlattenTest) { + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": "builtins.dict", + "context": "[\"4\", \"5\", \"6\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + auto [graph, valuePtrs] = makeValues(3); + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + c10::Dict dict( + c10::StringType::get(), c10::AnyType::get()); + // allow dict.size < context + // test dict.size=2 , context,size=3, + dict.insert("4", c10::IValue(7)); + dict.insert("5", c10::IValue(8)); + c10::List list(c10::AnyType::get()); + list.push_back(std::move(dict)); + itreeFlatten(c10::IValue{std::move(list)}, spec); +} + +TEST(ITreeTest, UnmatchedTupleFlatten) { + // Original data: [(0, 1), 2, {"4": 7, "5": 8, "6": 9}] + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": "builtins.dict", + "context": "[\"4\", \"5\", \"6\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + auto [graph, valuePtrs] = makeValues(6); + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + auto tup = c10::ivalue::Tuple::create({c10::IValue(0)}); + c10::Dict dict( + c10::StringType::get(), c10::AnyType::get()); + dict.insert("4", c10::IValue(7)); + dict.insert("5", c10::IValue(8)); + dict.insert("6", c10::IValue(8)); + c10::List list(c10::AnyType::get()); + list.push_back(std::move(tup)); + list.push_back(c10::IValue(2)); + list.push_back(std::move(dict)); + EXPECT_THROW( + { itreeFlatten(c10::IValue{std::move(list)}, spec); }, c10::Error); +} + +TEST(ITreeTest, ToAtenType) { + // Original data: ((0, 1), 2, {"4": 7, "5": 8}, [10], {6: 9}) + auto jsonSpec = R"( +[ + 1, + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": "builtins.tuple", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": "builtins.dict", + "context": "[\"4\", \"5\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": "builtins.list", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + } + ] + }, + { + "type": "builtins.dict", + "context": "[6]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } + ] + } +] + )"; + auto [graph, valuePtrs] = makeValues(7); + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + auto atenType = spec.toAtenType(); + + // Root level is tuple. + EXPECT_EQ(atenType->kind(), c10::TypeKind::TupleType); + const c10::TupleType& rootType = atenType->expectRef(); + EXPECT_EQ(rootType.elements().size(), 5); + + at::TypePtr elementType = rootType.elements()[0]; + EXPECT_EQ(elementType->kind(), c10::TypeKind::TupleType); + EXPECT_EQ( + elementType->expectRef().elements()[0]->kind(), + c10::TypeKind::AnyType); + EXPECT_EQ( + elementType->expectRef().elements()[1]->kind(), + c10::TypeKind::AnyType); + + elementType = rootType.elements()[1]; + EXPECT_EQ(elementType->kind(), c10::TypeKind::AnyType); + + elementType = rootType.elements()[2]; + EXPECT_EQ(elementType->kind(), c10::TypeKind::DictType); + EXPECT_EQ( + elementType->expectRef().getKeyType()->kind(), + c10::TypeKind::StringType); + EXPECT_EQ( + elementType->expectRef().getValueType()->kind(), + c10::TypeKind::AnyType); + + elementType = rootType.elements()[3]; + EXPECT_EQ(elementType->kind(), c10::TypeKind::ListType); + EXPECT_EQ( + elementType->expectRef().getElementType()->kind(), + c10::TypeKind::AnyType); + + elementType = rootType.elements()[4]; + EXPECT_EQ(elementType->kind(), c10::TypeKind::DictType); + EXPECT_EQ( + elementType->expectRef().getKeyType()->kind(), + c10::TypeKind::IntType); + EXPECT_EQ( + elementType->expectRef().getValueType()->kind(), + c10::TypeKind::AnyType); +} + +TEST(ITreeTest, KeyedJaggedTensorUnflatten) { + // Test KeyedJaggedTensor pytree node registration + // KeyedJaggedTensor has 6 tensor fields: _values, _weights, _lengths, + // _offsets, _stride_per_key_per_rank, _inverse_indices + auto jsonSpec = R"( +[ + 1, + { + "type": "torchrec.sparse.jagged_tensor.KeyedJaggedTensor", + "context": "[\"key1\", \"key2\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } +] + )"; + + auto [graph, valuePtrs] = makeValues(6); + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + + // Create mock tensor values for the 6 fields + std::vector flats = { + c10::IValue(1), // _values + c10::IValue(2), // _weights + c10::IValue(3), // _lengths + c10::IValue(4), // _offsets + c10::IValue(5), // _stride_per_key_per_rank + c10::IValue(6), // _inverse_indices tensor part + }; + + // Test unflatten - this will create a generic tuple since we don't have + // the actual KeyedJaggedTensor constructor available in tests + auto itree = itreeUnflatten(flats, spec); + EXPECT_TRUE(itree.isTuple()); + EXPECT_EQ(itree.toTupleRef().elements().size(), 6); + + // Verify the values match what we put in + for (size_t i = 0; i < 6; i++) { + EXPECT_EQ(itree.toTupleRef().elements()[i], flats[i]); + } + + // Verify spec has correct number of children and structure + EXPECT_EQ(spec.children().size(), 6); + EXPECT_EQ(spec.numIValues(), 6); + EXPECT_FALSE(spec.isIValue()); + EXPECT_EQ( + spec.uniformName(), "torchrec.sparse.jagged_tensor.KeyedJaggedTensor"); +} + +TEST(ITreeTest, KeyedJaggedTensorNodeRegistration) { + // Test that KeyedJaggedTensor pytree node is properly registered + + // Verify the KeyedJaggedTensor node is in the registry by attempting + // to load a spec that references it + auto jsonSpec = R"( +[ + 1, + { + "type": "torchrec.sparse.jagged_tensor.KeyedJaggedTensor", + "context": "[\"key1\", \"key2\"]", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } +] + )"; + + auto [graph, valuePtrs] = makeValues(6); + + // This should not throw - if KeyedJaggedTensor wasn't registered, + // we'd get an exception about "Unknown pytree node type" + EXPECT_NO_THROW({ + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + + // Verify the spec loaded correctly + EXPECT_FALSE(spec.isIValue()); + EXPECT_EQ( + spec.uniformName(), "torchrec.sparse.jagged_tensor.KeyedJaggedTensor"); + EXPECT_EQ(spec.children().size(), 6); + EXPECT_EQ(spec.numIValues(), 6); + + // Verify context is parsed correctly + EXPECT_FALSE(spec.context().is_null()); + EXPECT_TRUE(spec.context().is_array()); + EXPECT_EQ(spec.context().size(), 2); + }); +} + +TEST(ITreeTest, JaggedTensorNodeRegistration) { + // Test that JaggedTensor pytree node is also properly registered + + auto jsonSpec = R"( +[ + 1, + { + "type": "torchrec.sparse.jagged_tensor.JaggedTensor", + "context": "null", + "children_spec": [ + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + }, + { + "type": null, + "context": null, + "children_spec": [] + } + ] + } +] + )"; + + auto [graph, valuePtrs] = makeValues(4); + + // This should not throw - if JaggedTensor wasn't registered, + // we'd get an exception about "Unknown pytree node type" + EXPECT_NO_THROW({ + const auto spec = itreeSpecLoads(jsonSpec, valuePtrs); + + // Verify the spec loaded correctly + EXPECT_FALSE(spec.isIValue()); + EXPECT_EQ(spec.uniformName(), "torchrec.sparse.jagged_tensor.JaggedTensor"); + EXPECT_EQ(spec.children().size(), 4); + EXPECT_EQ(spec.numIValues(), 4); + }); +} + +} // namespace torch::nativert::detail diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_layout_planner.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_layout_planner.cpp new file mode 100644 index 0000000000000000000000000000000000000000..664292d3fa666e7e4e23cd1660d59cc16317a341 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_layout_planner.cpp @@ -0,0 +1,498 @@ +#include + +#include + +#define LayoutPlannerTests_TEST_FRIENDS \ + friend class LayoutPlannerCtorTests; \ + FRIEND_TEST(LayoutPlannerCtorTests, TestConstruct); \ + FRIEND_TEST(LayoutPlannerCtorTests, TestConstructSymbolicShape); \ + FRIEND_TEST(LayoutPlannerCtorTests, TestConstructNoMetadata); \ + FRIEND_TEST(LayoutPlannerCtorTests, TestConstructPlanWithOverlap); \ + FRIEND_TEST(LayoutPlannerCtorTests, TestConstructPlanNoOverlap); \ + FRIEND_TEST(LayoutPlannerCtorTests, TestConstructNoOutVariant); \ + FRIEND_TEST(LayoutPlannerCtorTests, TestConstructOutputAlias); \ + FRIEND_TEST( \ + LayoutPlannerCtorTests, TestConstructPlanWithMaybeAliasingToCopy); \ + FRIEND_TEST(LayoutPlannerCtorTests, TestConstructListPackNoUnpack); \ + FRIEND_TEST(LayoutPlannerCtorTests, TestConstructTensorList); + +#include // @manual + +#include // @manual +#include // @manual +#include // @manual +#include // @manual +#include // @manual +#include // @manual +#include // @manual + +using namespace ::testing; + +namespace torch::nativert /* must be same as namespace that includes TEST_FRIEND + declarations */ +{ + +class LayoutPlannerCtorTests : public testing::Test { + public: + void SetUp() override { + // register static dispatch kernel handler + register_kernel_handlers(); + } + void TearDown() override { + executor_config.reset(); + graph.reset(); + executor.reset(); + } + + void createPlannerForModel( + const std::string& model, + const ExecutorConfig& cfg = {}, + const std::unordered_map& + tensorMeta = {}) { + executor_config = std::make_unique(cfg); + + graph = stringToGraph(model); + + if (!tensorMeta.empty()) { + graph->setTensorValuesMeta(tensorMeta); + } + + auto kernels = KernelFactory().initializeNodeKernels( + *graph, nullptr, *executor_config, nullptr); + + auto kernelSchemas = Executor::getKernelSchemas(kernels.nodeKernels); + + planner = std::make_unique( + *graph, + kernelSchemas, + ExecutionFrame::getPersistentValueMask(*graph), + executor_config->layoutPlannerSettings); + + frame = std::make_unique( + *graph, Weights(graph.get()), *executor_config, planner.get()); + + executor = std::make_unique( + *graph, std::move(kernels.nodeKernels), *executor_config); + } + + torch::_export::TensorMeta createSymbolicTensorMeta( + const std::vector& dims, + std::string device = "cpu", + torch::_export::ScalarType dtype = torch::_export::ScalarType::FLOAT) { + torch::_export::TensorMeta out_meta; + + torch::_export::Device d; + d.set_type(std::move(device)); + out_meta.set_device(d); + + std::vector symvec; + for (size_t i = 0; i < dims.size(); ++i) { + torch::_export::SymInt symint; + torch::_export::SymExpr symexpr; + symexpr.set_expr_str(std::string("s") + std::to_string(i)); + symint.set_as_expr(symexpr); + symvec.push_back(symint); + } + + out_meta.set_sizes(symvec); + out_meta.set_dtype(dtype); + out_meta.set_layout(torch::_export::Layout::Strided); + + { + torch::_export::SymInt i; + i.set_as_int(0); + out_meta.set_storage_offset(i); + } + + return out_meta; + } + + torch::_export::TensorMeta createTensorMeta( + const std::vector& dims, + std::string device = "cpu", + torch::_export::ScalarType dtype = torch::_export::ScalarType::FLOAT) { + torch::_export::TensorMeta out_meta; + + torch::_export::Device d; + d.set_type(std::move(device)); + out_meta.set_device(d); + + std::vector symvec; + for (const auto dim : dims) { + torch::_export::SymInt symint; + symint.set_as_int(dim); + symvec.push_back(symint); + } + + out_meta.set_sizes(symvec); + out_meta.set_dtype(dtype); + out_meta.set_layout(torch::_export::Layout::Strided); + + { + torch::_export::SymInt i; + i.set_as_int(0); + out_meta.set_storage_offset(i); + } + + return out_meta; + } + + protected: + std::unique_ptr graph; + std::unique_ptr frame; + std::unique_ptr executor; + std::unique_ptr planner; + std::unique_ptr executor_config; +}; + +namespace { +ExecutorConfig create_enabled_executor_config() { + ExecutorConfig cfg; + cfg.enableStaticCPUKernels = true; + cfg.layoutPlannerSettings = + LayoutPlannerSettings() + .setAlgorithmType(LayoutPlannerAlgorithmType::GreedyBySize) + .setEnabled(true) + .setLayoutManagerSettings( + LayoutManagerSettings().setDeallocateBetweenRequests(false)); + return cfg; +}; +} // namespace + +TEST_F(LayoutPlannerCtorTests, TestConstructOutputAlias) { + auto model = R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + return (%out_t))"; + + createPlannerForModel(model, create_enabled_executor_config()); + // no outputs + EXPECT_EQ(planner->get_planned_values().size(), 0); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructNoOutVariant) { + std::unordered_map meta = { + {"out_t", createTensorMeta({10, 10, 10})}}; + + auto model = R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %res = torch.ops.aten.clone.default(self=%out_t, memory_format=None) + return (%res))"; + + auto executor_config = create_enabled_executor_config(); + executor_config.enableStaticCPUKernels = false; + + createPlannerForModel(model, executor_config, meta); + // no out variant (static dispatch disabled) + EXPECT_EQ(planner->get_planned_values().size(), 0); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructTensorList) { + auto model = R"( + graph(%y0, %y1): + %out_t0 = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %out_t1 = torch.ops.aten.matmul.default(self=%y0, other=%y1) + + %l[] = prim.ListPack(l0=%out_t0, l1=%out_t1) + %x0, %x1 = prim.ListUnpack(self=%l) + + %res0 = torch.ops.aten.clone.default(self=%x0, memory_format=None) + %res1 = torch.ops.aten.clone.default(self=%x1, memory_format=None) + return (%res0, %res1))"; + + createPlannerForModel(model, create_enabled_executor_config(), {}); + + EXPECT_EQ(planner->get_planned_values().size(), 2); + + auto& out_t0_lifetime = planner->planned_allocation_specs_[0].lifetime; + auto& out_t1_lifetime = planner->planned_allocation_specs_[1].lifetime; + + EXPECT_EQ( + std::abs( + static_cast(out_t0_lifetime.start) - + static_cast(out_t1_lifetime.start)), + 1); + EXPECT_EQ( + std::abs( + static_cast(out_t0_lifetime.end) - + static_cast(out_t1_lifetime.end)), + 1); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructListPackNoUnpack) { + auto model = R"(graph(%weight1, %weight2): +%weight1_plannable = torch.ops.aten.clone.default(self=%weight1, memory_format=None) +%weights_list[] = prim.ListPack(l0=%weight1_plannable, l1=%weight2) +%weights_cat = torch.ops.aten.cat.default(tensors=%weights_list, dim=0) +return (%weights_cat) +)"; + + createPlannerForModel(model, create_enabled_executor_config(), {}); + + auto& weight1_plannable_lifetime = + planner->planned_allocation_specs_[0].lifetime; + EXPECT_EQ(weight1_plannable_lifetime.start, 1); + EXPECT_EQ(weight1_plannable_lifetime.end, 3); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructReturnTensorListValues) { + auto model = R"( + graph(%y0, %y1): + %out_t0 = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %out_t1 = torch.ops.aten.matmul.default(self=%y0, other=%y1) + + %l[] = prim.ListPack(l0=%out_t0, l1=%out_t1) + %x0, %x1 = prim.ListUnpack(self=%l) + return (%x0, %x1))"; + createPlannerForModel(model, create_enabled_executor_config(), {}); + + EXPECT_EQ(planner->get_planned_values().size(), 0); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructInputTensorList) { + auto model = R"( + graph(%y0, %y1): + %l[] = prim.ListPack(l0=%y0, l1=%y1) + %x0, %x1 = prim.ListUnpack(self=%l) + + %res0 = torch.ops.aten.clone.default(self=%x0, memory_format=None) + %res1 = torch.ops.aten.clone.default(self=%x1, memory_format=None) + return (%res0, %res1))"; + createPlannerForModel(model, create_enabled_executor_config(), {}); + + EXPECT_EQ(planner->get_planned_values().size(), 0); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructReturnTensorList) { + auto model = R"( + graph(%y0, %y1): + %y0_clone = torch.ops.aten.clone.default(self=%y0, memory_format=None) + %y1_clone = torch.ops.aten.clone.default(self=%y1, memory_format=None) + + %l[] = prim.ListPack(l0=%y0_clone, l1=%y1_clone) + return (%l))"; + createPlannerForModel(model, create_enabled_executor_config(), {}); + + EXPECT_EQ(planner->get_planned_values().size(), 0); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructUnsupportedDevice) { + std::unordered_map meta = { + {"out_t", createTensorMeta({10, 10, 10})}}; + + { + torch::_export::Device d; + d.set_type("cuda"); + meta["out_t"].set_device(std::move(d)); + } + + auto model = R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %res = torch.ops.aten.clone.default(self=%out_t, memory_format=None) + return (%res))"; + createPlannerForModel(model, create_enabled_executor_config(), meta); + + // not cpu + EXPECT_EQ(planner->get_planned_values().size(), 0); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructNoMetadata) { + auto model = R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %res = torch.ops.aten.clone.default(self=%out_t, memory_format=None) + return (%res))"; + + createPlannerForModel(model, create_enabled_executor_config()); + // no metadata + + planner->create_plan(); + EXPECT_EQ(planner->planned_allocation_specs_.size(), 1); + EXPECT_EQ(planner->get_planned_values().size(), 1); + auto& spec = planner->planned_allocation_specs_[0]; + EXPECT_EQ(spec.size, 0); + EXPECT_EQ(spec.lifetime.start, 1); + EXPECT_EQ(spec.lifetime.end, 2); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructSymbolicShape) { + std::unordered_map meta = { + {"out_t", createSymbolicTensorMeta({10, 10, 10})}}; + + auto model = R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %res = torch.ops.aten.clone.default(self=%out_t, memory_format=None) + return (%res))"; + + createPlannerForModel(model, create_enabled_executor_config(), meta); + EXPECT_EQ(planner->get_planned_values().size(), 1); + EXPECT_EQ(planner->planned_allocation_specs_.size(), 1); + EXPECT_EQ( + planner->planned_allocation_specs_[0].size, + 0 /* haven't populated IValues yet */); +} + +TEST_F(LayoutPlannerCtorTests, TestConstruct) { + std::unordered_map meta = { + {"out_t", createTensorMeta({10, 10, 10})}}; + + auto model = R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %res = torch.ops.aten.clone.default(self=%out_t, memory_format=None) + return (%res))"; + createPlannerForModel(model, create_enabled_executor_config(), meta); + + auto& specs = planner->planned_allocation_specs_; + + EXPECT_EQ(specs.size(), 1); + EXPECT_EQ(specs[0].lifetime.start, 1); + EXPECT_EQ(specs[0].lifetime.end, 2); + + c10::IValue tensor = c10::IValue(torch::rand({10, 10, 10})); + + executor->execute(*frame, {tensor, tensor}); + + // 10 * 10 * 10 * 4 rounded up to the nearest multiple of 64 ==> 64 * 63 = + // 4032 + auto aligned_size = LayoutManager::get_aligned_nbytes( + 10 * 10 * 10 * at::elementSize(at::ScalarType::Float)); + EXPECT_EQ(specs[0].size, aligned_size); + EXPECT_EQ(specs[0].size, 4032); + + planner->with_plan([&](const LayoutPlan& plan) { + EXPECT_EQ(plan.total_size, 4032); + EXPECT_EQ(plan.allocations.size(), 1); + EXPECT_EQ(plan.allocations[0].size, 4032); + EXPECT_EQ(plan.allocations[0].offset, 0); + return; + }); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructPlanNoOverlap) { + std::unordered_map meta = { + {"out_t", createTensorMeta({10, 10, 10})}, + {"out2_t", createTensorMeta({10, 10, 10})}}; + + auto model = R"( + graph(%y0, %y1): + %out1_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %res1 = torch.ops.aten.clone.default(self=%out1_t, memory_format=None) + %out2_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %res2 = torch.ops.aten.clone.default(self=%out2_t, memory_format=None) + return (%res1, %res2))"; + createPlannerForModel(model, create_enabled_executor_config(), meta); + + c10::IValue tensor = c10::IValue(torch::rand({10, 10, 10})); + + executor->execute(*frame, {tensor, tensor}); + + planner->with_plan([&](const LayoutPlan& plan) { + EXPECT_EQ(plan.total_size, 4032); + EXPECT_EQ(plan.allocations.size(), 2); + EXPECT_EQ(plan.allocations[0].size, 4032); + EXPECT_EQ(plan.allocations[0].offset, 0); + EXPECT_EQ(plan.allocations[1].size, 4032); + EXPECT_EQ(plan.allocations[1].offset, 0); + return; + }); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructPlanWithOverlap) { + std::unordered_map meta = { + {"out_t", createTensorMeta({10, 10, 10})}, + {"out2_t", createTensorMeta({10, 10, 10})}}; + + auto model = R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %out2_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %res = torch.ops.aten.clone.default(self=%out2_t, memory_format=None) + %res1 = torch.ops.aten.clone.default(self=%out_t, memory_format=None) + return (%res, %res1))"; + createPlannerForModel(model, create_enabled_executor_config(), meta); + + c10::IValue tensor = c10::IValue(torch::rand({10, 10, 10})); + + executor->execute(*frame, {tensor, tensor}); + + planner->with_plan([&](const LayoutPlan& plan) { + EXPECT_EQ(plan.total_size, 8064); + EXPECT_EQ(plan.allocations.size(), 2); + EXPECT_EQ(plan.allocations[0].size, 4032); + EXPECT_EQ(plan.allocations[0].offset, 0); + EXPECT_EQ(plan.allocations[1].offset, 4032); + EXPECT_EQ(plan.allocations[1].size, 4032); + }); +} + +TEST_F(LayoutPlannerCtorTests, TestConstructPlanWithMaybeAliasingToCopy) { + auto model = R"(graph(%input): + %i1 = torch.ops.aten._to_copy.default(self=%input, dtype=ScalarType::FLOAT, memory_format=None) + %i2 = torch.ops.aten._to_copy.default(self=%input, dtype=ScalarType::FLOAT, memory_format=None) + %out_t = torch.ops.aten.matmul.default(self=%i1, other=%i2) + return (%out_t))"; + + createPlannerForModel(model, create_enabled_executor_config()); + + c10::IValue tensor = c10::IValue(torch::rand({10, 10, 10})); + + executor->execute(*frame, {tensor}); + + // i1 and i2 could alias input, so we should be safe and not plan them + planner->with_plan([&](const LayoutPlan& plan) { + EXPECT_EQ(plan.total_size, 0); + EXPECT_EQ(plan.allocations.size(), 0); + return; + }); +} + +TEST_F(LayoutPlannerCtorTests, TestCreateMultiplePlanners) { + auto executor_config = create_enabled_executor_config(); + + auto model = R"( + graph(%y0, %y1): + %out_t = torch.ops.aten.matmul.default(self=%y0, other=%y1) + %res = torch.ops.aten.clone.default(self=%out_t, memory_format=None) + return (%res))"; + + graph = stringToGraph(model); + + std::vector, + std::vector>>> + planners; + for ([[maybe_unused]] const auto _ : c10::irange(2)) { + auto kernels = KernelFactory().initializeNodeKernels( + *graph, nullptr, executor_config, nullptr); + auto kernelSchemas = Executor::getKernelSchemas(kernels.nodeKernels); + planners.emplace_back( + std::make_unique( + *graph, + kernelSchemas, + ExecutionFrame::getPersistentValueMask(*graph), + executor_config.layoutPlannerSettings), + std::move(kernels.nodeKernels)); + } + + c10::IValue tensor = c10::IValue(torch::rand({10, 10, 10})); + for (auto& [layout_planner, kernels] : planners) { + ExecutionFrame execution_frame( + *graph, Weights(graph.get()), executor_config, layout_planner.get()); + SerialGraphExecutor graph_executor( + *graph, std::move(kernels), executor_config); + graph_executor.execute(execution_frame, {tensor, tensor}); + layout_planner->with_plan([&](const LayoutPlan& plan) { + EXPECT_EQ(plan.total_size, 4032); + EXPECT_EQ(plan.allocations.size(), 1); + EXPECT_EQ(plan.allocations[0].size, 4032); + EXPECT_EQ(plan.allocations[0].offset, 0); + return; + }); + } +} + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_layout_planner_algorithm.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_layout_planner_algorithm.cpp new file mode 100644 index 0000000000000000000000000000000000000000..19157777d854a42dcd0b2304aba456a9f0efef58 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_layout_planner_algorithm.cpp @@ -0,0 +1,86 @@ +#include +#include + +#include +#include +#include + +using namespace ::testing; +using namespace torch::nativert; + +std::vector create_test_allocation_specs() { + std::vector specs; + + const std::vector> test_cases = { + {0, 1, 32}, + {1, 4, 28}, + {2, 5, 36}, + {3, 5, 16}, + {4, 5, 8}, + {5, 7, 64}, + {6, 8, 10}, + {7, 8, 40}, + }; + + specs.reserve(test_cases.size()); + for (const auto& [l_start, l_end, size] : test_cases) { + specs.push_back(AllocationSpec{AllocationLifetime(l_start, l_end), size}); + }; + + return specs; +} + +// figure 6 -- https://arxiv.org/pdf/2001.03288 +TEST(LayoutPlannerAlgorithmTests, TestGreedyBySize) { + auto result = GreedyBySizeAllocationPlanner(create_test_allocation_specs()); + + EXPECT_EQ(result.total_size, 124); + + auto& allocations = result.allocations; + + EXPECT_EQ(allocations[0].offset, 0); + EXPECT_EQ(allocations[1].offset, 32); + EXPECT_EQ(allocations[2].offset, 64); + EXPECT_EQ(allocations[3].offset, 100); + EXPECT_EQ(allocations[4].offset, 116); + EXPECT_EQ(allocations[5].offset, 0); + EXPECT_EQ(allocations[6].offset, 104); + EXPECT_EQ(allocations[7].offset, 64); +} + +TEST(LayoutPlannerAlgorithmTests, TestBump) { + auto specs = create_test_allocation_specs(); + auto result = BumpAllocationPlanner(create_test_allocation_specs()); + + auto& allocations = result.allocations; + + size_t offset = 0; + for (auto&& [i, spec] : c10::enumerate(specs)) { + EXPECT_EQ(allocations[i].offset, offset); + offset += spec.size; + } + + EXPECT_EQ(result.total_size, offset); +} + +TEST(LayoutPlannerAlgorithmTests, TestStorageGroup) { + auto specs = create_test_allocation_specs(); + auto result = DisjointStorageGroupsPlanner(create_test_allocation_specs()); + + auto& allocations = result.allocations; + + EXPECT_EQ(allocations[0].offset, 0); + EXPECT_EQ(allocations[1].offset, 36); + EXPECT_EQ(allocations[2].offset, 0); + EXPECT_EQ(allocations[3].offset, 100); + EXPECT_EQ(allocations[4].offset, 140); + EXPECT_EQ(allocations[5].offset, 36); + EXPECT_EQ(allocations[6].offset, 140); + EXPECT_EQ(allocations[7].offset, 100); + + for (auto&& [i, spec] : c10::enumerate(specs)) { + EXPECT_EQ(allocations[i].size, spec.size); + } + + EXPECT_EQ(result.total_size, 150); +} diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_mpmc_queue.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_mpmc_queue.cpp new file mode 100644 index 0000000000000000000000000000000000000000..261dde9d7a3a8f8a1fc83797f55e50fb83fced68 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_mpmc_queue.cpp @@ -0,0 +1,134 @@ +#include +#include + +#include + +#include + +using torch::nativert::detail::MPMCQueue; + +TEST(MPMCQueueTest, EmptyQueue) { + MPMCQueue queue(5); + int out = 0; + EXPECT_FALSE(queue.readIfNotEmpty(out)); +} + +TEST(MPMCQueueTest, SingleElement) { + MPMCQueue queue(5); + EXPECT_TRUE(queue.writeIfNotFull(10)); + int out = 0; + EXPECT_TRUE(queue.readIfNotEmpty(out)); + EXPECT_EQ(out, 10); +} + +TEST(MPMCQueueTest, MultipleElements) { + MPMCQueue queue(5); + for (int i = 0; i < 5; ++i) { + EXPECT_TRUE(queue.writeIfNotFull(i)); + } + for (int i = 0; i < 5; ++i) { + int out = 0; + EXPECT_TRUE(queue.readIfNotEmpty(out)); + EXPECT_EQ(out, i); + } +} + +TEST(MPMCQueueTest, FullQueue) { + MPMCQueue queue(5); + for (int i = 0; i < 5; ++i) { + EXPECT_TRUE(queue.writeIfNotFull(i)); + } + EXPECT_FALSE(queue.writeIfNotFull(10)); +} + +TEST(MPMCQueueTest, ConcurrentAccess) { + MPMCQueue queue(10); + std::thread writer([&queue]() { + for (int i = 0; i < 5; ++i) { + queue.writeIfNotFull(i); + } + }); + std::thread reader([&queue]() { + for (int i = 0; i < 5; ++i) { + int out = 0; + while (!queue.readIfNotEmpty(out)) { + // Wait until an element is available + // TODO We could provide a blocking version of read() instead of + // looping here. We only provide a non blocking wait API because + // for now the queue is paired with a semaphore in executor. + std::this_thread::yield(); + } + EXPECT_LT(out, 5); + } + }); + writer.join(); + reader.join(); +} + +TEST(MPMCQueueTest, MPMCConcurrentAccess) { + const size_t queueCapacity = 100000; + const size_t numWriters = 5; + const size_t numReaders = 5; + const size_t numElementsPerWriter = 10000; + MPMCQueue queue(queueCapacity); + // Writer threads + std::vector writers; + writers.reserve(numWriters); + for (size_t i = 0; i < numWriters; ++i) { + writers.emplace_back([&]() { + for (size_t j = 0; j < numElementsPerWriter; ++j) { + size_t value = i * numElementsPerWriter + j; + while (!queue.writeIfNotFull(static_cast(value))) { + // Retry until the queue has space + // TODO We could provide a blocking version of read() instead of + // looping here. We only provide a non blocking wait API because + // for now the queue is paired with a semaphore in executor. + std::this_thread::yield(); + } + } + }); + } + // Reader threads + std::vector readers; + std::atomic totalReadCount{0}; + readers.reserve(numReaders); + for (size_t i = 0; i < numReaders; ++i) { + readers.emplace_back([&]() { + int value = 0; + while (totalReadCount < numWriters * numElementsPerWriter) { + if (queue.readIfNotEmpty(value)) { + ++totalReadCount; + } else { + // TODO We could provide a blocking version of read() instead of + // looping here. We only provide a non blocking wait API because + // for now the queue is paired with a semaphore in executor. + std::this_thread::yield(); + } + } + }); + } + // Join all threads + for (auto& writer : writers) { + writer.join(); + } + for (auto& reader : readers) { + reader.join(); + } + // Verify that all elements were read + EXPECT_EQ(totalReadCount, numWriters * numElementsPerWriter); +} + +TEST(MPMCQueueTest, MoveOnlyType) { + struct MoveOnly { + MoveOnly() = default; + MoveOnly(const MoveOnly&) = delete; + MoveOnly& operator=(const MoveOnly&) = delete; + MoveOnly(MoveOnly&&) = default; + MoveOnly& operator=(MoveOnly&&) = default; + ~MoveOnly() = default; + }; + MPMCQueue queue(5); + EXPECT_TRUE(queue.writeIfNotFull(MoveOnly())); + MoveOnly out; + EXPECT_TRUE(queue.readIfNotEmpty(out)); +} diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_op_kernel.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_op_kernel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ffbf2eb17d67c6bc2947f7d0297803db86a2d071 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_op_kernel.cpp @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include + +namespace torch::nativert { + +int64_t increment_kernel(const at::Tensor& tensor, int64_t input) { + return input + 1; +} + +TEST(OpKernelTest, GetOperatorForTargetValid) { + auto registrar = c10::RegisterOperators().op( + "test::foo(Tensor dummy, int input) -> int", &increment_kernel); + std::string target = "test.foo.default"; + EXPECT_NO_THROW({ + c10::OperatorHandle handle = getOperatorForTarget(target); + EXPECT_TRUE(handle.hasSchema()); + EXPECT_EQ(handle.operator_name().name, "test::foo"); + EXPECT_EQ(handle.operator_name().overload_name, ""); + }); +} + +TEST(OpKernelTest, GetOperatorForTargetInvalid) { + std::string target = "invalid.target"; + EXPECT_THROW(getOperatorForTarget(target), c10::Error); +} + +TEST(OpKernelTest, GetReadableArgs) { + c10::FunctionSchema schema = c10::FunctionSchema( + "test_op", + "", + {c10::Argument("tensor_arg"), + c10::Argument("tensor_list_arg"), + c10::Argument("int_arg"), + c10::Argument("none_arg")}, + {}); + std::vector stack = { + at::tensor({1, 2, 3}), + c10::IValue( + std::vector{at::tensor({1, 2}), at::tensor({3, 4})}), + c10::IValue(1), + c10::IValue(), + }; + std::string expected = + "arg0 tensor_arg: Tensor int[3]cpu\n" + "arg1 tensor_list_arg: GenericList [int[2]cpu, int[2]cpu, ]\n" + "arg2 int_arg: Int 1\n" + "arg3 none_arg: None \n"; + + std::string result = readableArgs(schema, stack); + EXPECT_EQ(result, expected); +} + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_pass_manager.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_pass_manager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8d7d06ae283857e16d4d875dfe789a4f68c790d5 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_pass_manager.cpp @@ -0,0 +1,33 @@ +#include + +#include +#include + +#include + +using namespace ::testing; +using namespace torch::nativert; + +TEST(PassManagerTest, TestEmptyPass) { + GraphPassManager manager({"EmptyPass"}); + EXPECT_FALSE(manager.run(Graph::createGraph().get())); +} + +TEST(PassPipelineTest, TestConcat) { + GraphPassPipeline p1({"test"}); + EXPECT_EQ(p1.size(), 1); + EXPECT_EQ(p1.at(0), "test"); + p1.concat({"test1", "test2"}); + EXPECT_EQ(p1.at(0), "test"); + EXPECT_EQ(p1.at(1), "test1"); + EXPECT_EQ(p1.at(2), "test2"); +} + +TEST(PassPipelineTest, TestPushFront) { + GraphPassPipeline p1({"test"}); + EXPECT_EQ(p1.size(), 1); + EXPECT_EQ(p1.at(0), "test"); + p1.push_front("test1"); + EXPECT_EQ(p1.at(0), "test1"); + EXPECT_EQ(p1.at(1), "test"); +} diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_placement.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_placement.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f083513b36586c0dd86390cbffa2d68a2510ea15 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_placement.cpp @@ -0,0 +1,87 @@ + +#include +#include +#include + +#include + +using namespace ::testing; + +namespace torch::nativert { + +TEST(PlacementTest, IsSameDevice) { + c10::Device cpuDevice = c10::Device(c10::DeviceType::CPU); + c10::Device cpuDevice1 = c10::Device(c10::DeviceType::CPU); + cpuDevice1.set_index(1); + + EXPECT_TRUE(isSameDevice(cpuDevice, cpuDevice)); + EXPECT_TRUE(isSameDevice(cpuDevice, cpuDevice1)); + + c10::Device cudaDevice = c10::Device(c10::DeviceType::CUDA); + c10::Device cudaDevice0 = c10::Device(c10::DeviceType::CUDA, 0); + c10::Device cudaDevice1 = c10::Device(c10::DeviceType::CUDA, 1); + EXPECT_TRUE(isSameDevice(cudaDevice, cudaDevice0)); + EXPECT_FALSE(isSameDevice(cudaDevice0, cudaDevice1)); + + EXPECT_FALSE(isSameDevice(cudaDevice0, cpuDevice)); +} + +TEST(PlacementTest, PlacementDefaultOnly) { + Placement placement(c10::Device(c10::DeviceType::CUDA, 0)); + + std::ostringstream os; + os << placement; + EXPECT_EQ(os.str(), "|cuda:0"); + + c10::Device cuda0 = c10::Device(c10::DeviceType::CUDA, 0); + c10::Device cuda1 = c10::Device(c10::DeviceType::CUDA, 1); + c10::Device cuda2 = c10::Device(c10::DeviceType::CUDA, 2); + + EXPECT_EQ(placement.getMappedDevice(cuda0), cuda0); + EXPECT_EQ(placement.getMappedDevice(cuda1), cuda0); + EXPECT_EQ(placement.getMappedDevice(cuda2), cuda0); +} + +TEST(PlacementTest, PlacementBasic) { + Placement placement( + {{c10::Device(c10::DeviceType::CPU), c10::Device(c10::DeviceType::CPU)}, + {c10::Device(c10::DeviceType::CUDA, 0), + c10::Device(c10::DeviceType::CUDA, 1)}, + {c10::Device(c10::DeviceType::CUDA, 1), + c10::Device(c10::DeviceType::CUDA, 2)}}, + c10::Device(c10::DeviceType::CUDA, 0)); + + std::ostringstream os; + os << placement; + EXPECT_EQ(os.str(), "cpu|cpu,cuda:0|cuda:1,cuda:1|cuda:2,|cuda:0"); + + c10::Device cpu = c10::Device(c10::DeviceType::CPU); + c10::Device cuda0 = c10::Device(c10::DeviceType::CUDA, 0); + c10::Device cuda1 = c10::Device(c10::DeviceType::CUDA, 1); + c10::Device cuda2 = c10::Device(c10::DeviceType::CUDA, 2); + c10::Device cuda3 = c10::Device(c10::DeviceType::CUDA, 3); + + EXPECT_EQ(placement.getMappedDevice(cpu), cpu); + EXPECT_EQ(placement.getMappedDevice(cuda0), cuda1); + EXPECT_EQ(placement.getMappedDevice(cuda1), cuda2); + EXPECT_EQ(placement.getMappedDevice(cuda2), cuda0); + EXPECT_EQ(placement.getMappedDevice(cuda3), cuda0); +} + +TEST(PlacementTest, Placement) { + std::unordered_map deviceMap1 = { + {c10::Device("cuda:0"), c10::Device("cuda:1")}}; + Placement p1(deviceMap1); + EXPECT_EQ(p1.getMappedDevice(c10::Device("cpu")), c10::Device("cpu")); + EXPECT_EQ(p1.getMappedDevice(c10::Device("cuda")), c10::Device("cuda")); + EXPECT_EQ(p1.getMappedDevice(c10::Device("cuda:0")), c10::Device("cuda:1")); + + std::unordered_map deviceMap2 = { + {c10::Device("cpu"), c10::Device("cuda:0")}}; + Placement p2(deviceMap2); + EXPECT_EQ(p2.getMappedDevice(c10::Device("cpu")), c10::Device("cuda:0")); + EXPECT_EQ(p2.getMappedDevice(c10::Device("cuda:0")), c10::Device("cuda:0")); + EXPECT_EQ(p2.getMappedDevice(c10::Device("cuda:1")), c10::Device("cuda:1")); +} + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_serialization.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_serialization.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e8afa06c4f3919432660dbd01ff1ea2cf6ad1de3 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_serialization.cpp @@ -0,0 +1,417 @@ +#include +#include + +namespace torch::nativert { + +// Helper to create tensor argument +torch::_export::TensorArgument makeTensorArg(const std::string& name) { + torch::_export::TensorArgument arg; + arg.set_name(name); + return arg; +} + +// Helper to create SymInt argument with name +torch::_export::SymIntArgument makeSymIntArg(const std::string& name) { + torch::_export::SymIntArgument arg; + arg.set_as_name(name); + return arg; +} + +// Helper to create SymInt argument with constant int +torch::_export::SymIntArgument makeSymIntArgConst(int64_t value) { + torch::_export::SymIntArgument arg; + arg.set_as_int(value); + return arg; +} + +// Helper to create SymBool argument +torch::_export::SymBoolArgument makeSymBoolArg(const std::string& name) { + torch::_export::SymBoolArgument arg; + arg.set_as_name(name); + return arg; +} + +// Helper to create OptionalTensorArgument with tensor +torch::_export::OptionalTensorArgument makeOptionalTensorArg( + const std::string& name) { + torch::_export::OptionalTensorArgument arg; + arg.set_as_tensor(makeTensorArg(name)); + return arg; +} + +// Helper to create OptionalTensorArgument with None +torch::_export::OptionalTensorArgument makeOptionalTensorArgNone() { + torch::_export::OptionalTensorArgument arg; + arg.set_as_none({}); + return arg; +} + +TEST(SerializationTest, CheckIsSymbolic) { + torch::_export::TensorArgument tensor_arg; + torch::_export::Argument as_tensor_arg; + as_tensor_arg.set_as_tensor(tensor_arg); + EXPECT_TRUE(isSymbolic(as_tensor_arg)); + + std::vector tensor_args; + torch::_export::Argument as_tensors_arg; + as_tensors_arg.set_as_tensors(tensor_args); + EXPECT_TRUE(isSymbolic(as_tensors_arg)); + + torch::_export::SymIntArgument sym_int_arg; + torch::_export::Argument as_sym_int_arg; + as_sym_int_arg.set_as_sym_int(sym_int_arg); + EXPECT_TRUE(isSymbolic(as_sym_int_arg)); + + torch::_export::Argument as_int_arg; + as_int_arg.set_as_int(static_cast(1)); + EXPECT_FALSE(isSymbolic(as_int_arg)); + + torch::_export::Argument as_bool_arg; + as_bool_arg.set_as_bool(true); + EXPECT_FALSE(isSymbolic(as_bool_arg)); + + torch::_export::Argument as_string_arg; + as_string_arg.set_as_string("test_string"); + EXPECT_FALSE(isSymbolic(as_string_arg)); +} + +// Test isSymbolic for AS_OPTIONAL_TENSOR +TEST(SerializationTest, CheckIsSymbolicOptionalTensor) { + torch::_export::Argument arg; + arg.set_as_optional_tensor(makeOptionalTensorArg("opt_tensor")); + EXPECT_TRUE(isSymbolic(arg)); + + // Also test with None optional tensor + torch::_export::Argument arg_none; + arg_none.set_as_optional_tensor(makeOptionalTensorArgNone()); + EXPECT_TRUE(isSymbolic(arg_none)); +} + +// Test isSymbolic for AS_OPTIONAL_TENSORS +TEST(SerializationTest, CheckIsSymbolicOptionalTensors) { + std::vector opt_tensors; + opt_tensors.push_back(makeOptionalTensorArg("opt_0")); + opt_tensors.push_back(makeOptionalTensorArgNone()); + + torch::_export::Argument arg; + arg.set_as_optional_tensors(opt_tensors); + EXPECT_TRUE(isSymbolic(arg)); +} + +// Test isSymbolic for AS_SYM_INTS +TEST(SerializationTest, CheckIsSymbolicSymInts) { + std::vector sym_ints; + sym_ints.push_back(makeSymIntArg("s0")); + sym_ints.push_back(makeSymIntArgConst(8)); + + torch::_export::Argument arg; + arg.set_as_sym_ints(sym_ints); + EXPECT_TRUE(isSymbolic(arg)); +} + +// Test isSymbolic for AS_SYM_BOOL +TEST(SerializationTest, CheckIsSymbolicSymBool) { + torch::_export::Argument arg; + arg.set_as_sym_bool(makeSymBoolArg("sym_bool")); + EXPECT_TRUE(isSymbolic(arg)); +} + +// Test isSymbolic for AS_SYM_BOOLS +TEST(SerializationTest, CheckIsSymbolicSymBools) { + std::vector sym_bools; + sym_bools.push_back(makeSymBoolArg("b0")); + sym_bools.push_back(makeSymBoolArg("b1")); + + torch::_export::Argument arg; + arg.set_as_sym_bools(sym_bools); + EXPECT_TRUE(isSymbolic(arg)); +} + +// Test isSymbolic for AS_SYM_FLOAT +TEST(SerializationTest, CheckIsSymbolicSymFloat) { + torch::_export::SymFloatArgument sym_float; + sym_float.set_as_name("sym_float"); + + torch::_export::Argument arg; + arg.set_as_sym_float(sym_float); + EXPECT_TRUE(isSymbolic(arg)); +} + +// Test isSymbolic for AS_SYM_FLOATS +TEST(SerializationTest, CheckIsSymbolicSymFloats) { + torch::_export::SymFloatArgument sym_float_0; + sym_float_0.set_as_name("sym_float_0"); + torch::_export::SymFloatArgument sym_float_1; + sym_float_1.set_as_name("sym_float_1"); + std::vector sym_floats = { + sym_float_0, sym_float_1}; + + torch::_export::Argument arg; + arg.set_as_sym_floats(sym_floats); + EXPECT_TRUE(isSymbolic(arg)); +} + +// Test isSymbolic for AS_CUSTOM_OBJ +TEST(SerializationTest, CheckIsSymbolicCustomObj) { + torch::_export::CustomObjArgument custom_obj; + custom_obj.set_name("my_custom_obj"); + custom_obj.set_class_fqn("my.custom.Class"); + + torch::_export::Argument arg; + arg.set_as_custom_obj(custom_obj); + EXPECT_TRUE(isSymbolic(arg)); +} + +// Test that non-symbolic types return false +TEST(SerializationTest, CheckIsSymbolicNonSymbolicTypes) { + // AS_FLOAT + torch::_export::Argument as_float; + torch::_export::F64 f64_val; + f64_val.set(3.14); + as_float.set_as_float(f64_val); + EXPECT_FALSE(isSymbolic(as_float)); + + // AS_INTS + torch::_export::Argument as_ints; + as_ints.set_as_ints({1, 2, 3}); + EXPECT_FALSE(isSymbolic(as_ints)); + + // AS_FLOATS + torch::_export::Argument as_floats; + torch::_export::F64 f64_1, f64_2, f64_3; + f64_1.set(1.0); + f64_2.set(2.0); + f64_3.set(3.0); + as_floats.set_as_floats({f64_1, f64_2, f64_3}); + EXPECT_FALSE(isSymbolic(as_floats)); + + // AS_BOOLS + torch::_export::Argument as_bools; + as_bools.set_as_bools({true, false, true}); + EXPECT_FALSE(isSymbolic(as_bools)); + + // AS_NONE + torch::_export::Argument as_none; + as_none.set_as_none({}); + EXPECT_FALSE(isSymbolic(as_none)); + + // AS_STRINGS + torch::_export::Argument as_strings; + as_strings.set_as_strings({"a", "b", "c"}); + EXPECT_FALSE(isSymbolic(as_strings)); +} + +TEST(SerializationTest, ConstantToValue) { + torch::_export::Argument as_int_arg; + as_int_arg.set_as_int(static_cast(42)); + auto value = constantToValue(as_int_arg, false); + EXPECT_EQ(value, Constant(static_cast(42))); + + torch::_export::Argument as_bool_arg; + as_bool_arg.set_as_bool(true); + value = constantToValue(as_bool_arg, false); + EXPECT_EQ(value, Constant(true)); + + torch::_export::Argument as_string_arg; + as_string_arg.set_as_string("test_string"); + value = constantToValue(as_string_arg, false); + EXPECT_EQ(value, Constant("test_string")); +} + +// Test constantToValue for AS_FLOAT +TEST(SerializationTest, ConstantToValueFloat) { + torch::_export::Argument arg; + torch::_export::F64 f64_val; + f64_val.set(3.14159); + arg.set_as_float(f64_val); + auto value = constantToValue(arg, false); + EXPECT_EQ(value, Constant(3.14159)); +} + +// Test constantToValue for AS_INTS +TEST(SerializationTest, ConstantToValueInts) { + torch::_export::Argument arg; + arg.set_as_ints({1, 2, 3, 4, 5}); + auto value = constantToValue(arg, false); + std::vector expected = {1, 2, 3, 4, 5}; + EXPECT_EQ(value, Constant(expected)); +} + +// Test constantToValue for AS_FLOATS +TEST(SerializationTest, ConstantToValueFloats) { + torch::_export::Argument arg; + torch::_export::F64 f64_1, f64_2, f64_3; + f64_1.set(1.0); + f64_2.set(2.5); + f64_3.set(3.14); + arg.set_as_floats({f64_1, f64_2, f64_3}); + auto value = constantToValue(arg, false); + std::vector expected = {1.0, 2.5, 3.14}; + EXPECT_EQ(value, Constant(expected)); +} + +// Test constantToValue for AS_BOOLS +TEST(SerializationTest, ConstantToValueBools) { + torch::_export::Argument arg; + arg.set_as_bools({true, false, true}); + auto value = constantToValue(arg, false); + std::vector expected = {true, false, true}; + EXPECT_EQ(value, Constant(expected)); +} + +// Test constantToValue for AS_NONE +TEST(SerializationTest, ConstantToValueNone) { + torch::_export::Argument arg; + arg.set_as_none({}); + auto value = constantToValue(arg, false); + EXPECT_EQ(value, Constant(None())); +} + +// Test constantToValue for AS_STRINGS +TEST(SerializationTest, ConstantToValueStrings) { + torch::_export::Argument arg; + arg.set_as_strings({"hello", "world"}); + auto value = constantToValue(arg, false); + std::vector expected = {"hello", "world"}; + EXPECT_EQ(value, Constant(expected)); +} + +// Test that symbolic types throw when passed to constantToValue +TEST(SerializationTest, ConstantToValueThrowsOnSymbolicTypes) { + // AS_TENSOR should throw + torch::_export::Argument as_tensor; + as_tensor.set_as_tensor(makeTensorArg("tensor")); + EXPECT_THROW(constantToValue(as_tensor, false), std::exception); + + // AS_TENSORS should throw + torch::_export::Argument as_tensors; + as_tensors.set_as_tensors({makeTensorArg("t1"), makeTensorArg("t2")}); + EXPECT_THROW(constantToValue(as_tensors, false), std::exception); + + // AS_OPTIONAL_TENSORS should throw + torch::_export::Argument as_opt_tensors; + as_opt_tensors.set_as_optional_tensors( + {makeOptionalTensorArg("opt_t1"), makeOptionalTensorArgNone()}); + EXPECT_THROW(constantToValue(as_opt_tensors, false), std::exception); + + // AS_SYM_INT should throw + torch::_export::Argument as_sym_int; + as_sym_int.set_as_sym_int(makeSymIntArg("s0")); + EXPECT_THROW(constantToValue(as_sym_int, false), std::exception); + + // AS_SYM_INTS should throw + torch::_export::Argument as_sym_ints; + as_sym_ints.set_as_sym_ints({makeSymIntArg("s0"), makeSymIntArg("s1")}); + EXPECT_THROW(constantToValue(as_sym_ints, false), std::exception); + + // AS_SYM_BOOL should throw + torch::_export::Argument as_sym_bool; + as_sym_bool.set_as_sym_bool(makeSymBoolArg("b0")); + EXPECT_THROW(constantToValue(as_sym_bool, false), std::exception); + + // AS_SYM_BOOLS should throw + torch::_export::Argument as_sym_bools; + as_sym_bools.set_as_sym_bools({makeSymBoolArg("b0"), makeSymBoolArg("b1")}); + EXPECT_THROW(constantToValue(as_sym_bools, false), std::exception); + + // AS_CUSTOM_OBJ should throw + torch::_export::CustomObjArgument custom_obj; + custom_obj.set_name("obj"); + custom_obj.set_class_fqn("MyClass"); + torch::_export::Argument as_custom_obj; + as_custom_obj.set_as_custom_obj(custom_obj); + EXPECT_THROW(constantToValue(as_custom_obj, false), std::exception); + + // AS_OPTIONAL_TENSOR should throw + torch::_export::Argument as_opt_tensor; + as_opt_tensor.set_as_optional_tensor(makeOptionalTensorArg("opt")); + EXPECT_THROW(constantToValue(as_opt_tensor, false), std::exception); +} + +// Verify that None-typed input values deserialized from JSON have nullptr +// producer. Previously, they were incorrectly assigned the consuming node as +// producer, which caused dangling pointer crashes in cleanupDeadNodes() when +// graph passes like FuseListUnpack destroyed the consuming node. +TEST(SerializationTest, NoneInputValueHasNullProducer) { + // Build a minimal JSON graph: + // graph(%data): + // %out = some.op(data=%data, optional_arg=None) + // return (%out) + torch::_export::Graph jsonGraph; + + // Graph input: a tensor named "data" + torch::_export::Argument graphInput; + graphInput.set_as_tensor(makeTensorArg("data")); + jsonGraph.set_inputs({graphInput}); + + // Node: some.op with a tensor input and a None input + torch::_export::Node node; + node.set_target("some.op.default"); + + torch::_export::NamedArgument dataInput; + dataInput.set_name("data"); + torch::_export::Argument dataArg; + dataArg.set_as_tensor(makeTensorArg("data")); + dataInput.set_arg(dataArg); + + torch::_export::NamedArgument noneInput; + noneInput.set_name("optional_arg"); + torch::_export::Argument noneArg; + noneArg.set_as_none({}); + noneInput.set_arg(noneArg); + + node.set_inputs({dataInput, noneInput}); + + // Output: a tensor named "out" + torch::_export::Argument nodeOutput; + nodeOutput.set_as_tensor(makeTensorArg("out")); + node.set_outputs({nodeOutput}); + + jsonGraph.set_nodes({node}); + + // Graph output + torch::_export::Argument graphOutput; + graphOutput.set_as_tensor(makeTensorArg("out")); + jsonGraph.set_outputs({graphOutput}); + + // Build signature with proper input/output specs + torch::_export::GraphSignature sig; + + torch::_export::UserInputSpec userInput; + userInput.set_arg(graphInput); + torch::_export::InputSpec inputSpec; + inputSpec.set_user_input(userInput); + sig.set_input_specs({inputSpec}); + + torch::_export::UserOutputSpec userOutput; + userOutput.set_arg(graphOutput); + torch::_export::OutputSpec outputSpec; + outputSpec.set_user_output(userOutput); + sig.set_output_specs({outputSpec}); + + torch::_export::GraphModule graphModule; + graphModule.set_graph(jsonGraph); + graphModule.set_signature(sig); + + auto graph = jsonToGraph(graphModule); + + // Find the deserialized node and check its None input + for (const auto& n : graph->nodes()) { + if (n.target() == "some.op.default") { + for (const auto& inp : n.inputs()) { + if (inp.name == "optional_arg") { + // The None value should have nullptr producer, not the consuming node + EXPECT_EQ(inp.value->type().kind(), Type::Kind::None); + EXPECT_EQ(inp.value->producer(), nullptr) + << "None input value should have nullptr producer, not the " + "consuming node. This was fixed to prevent dangling pointer " + "crashes in cleanupDeadNodes()."; + return; + } + } + } + } + FAIL() << "Could not find the None input 'optional_arg' on 'some.op.default'"; +} + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_static_dispatch_kernel_registration.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_static_dispatch_kernel_registration.cpp new file mode 100644 index 0000000000000000000000000000000000000000..28d838eb5e75720800e895fb28d67356dd90ae13 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_static_dispatch_kernel_registration.cpp @@ -0,0 +1,15 @@ +#include + +#include +#include + +using namespace ::testing; +using namespace torch::nativert; + +TEST(StaticDispatchKernelRegistrationTests, TestRegistration) { + EXPECT_FALSE(KernelFactory::isHandlerRegistered("static_cpu")); + register_kernel_handlers(); + EXPECT_TRUE(KernelFactory::isHandlerRegistered("static_cpu")); + // try to re-register, which should be a no-op + register_kernel_handlers(); +} diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_static_kernel_ops.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_static_kernel_ops.cpp new file mode 100644 index 0000000000000000000000000000000000000000..470fb08388597836d3d87fd46a4f6ff803e6c44c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_static_kernel_ops.cpp @@ -0,0 +1,539 @@ +#include +#include +#include +#include +#include "test/cpp/nativert/static_kernel_test_utils.h" // @manual + +namespace torch::nativert { + +namespace { +std::vector generateArgsForQuantizedEmbeddingBag() { + // Set seed for reproducibility + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution int_dis(0, 15); // num_embeddings - 1 + int num_embeddings = 16; + int embedding_dim = 32; + int num_lengths = 10; + + auto weight = + at::randint(0, 255, {num_embeddings, embedding_dim}).to(at::kByte); + + // Generate random lengths + std::vector np_lengths(num_lengths); + for (auto& length : np_lengths) { + length = int_dis(gen); + } + int total_length = 0; + for (const auto& length : np_lengths) { + total_length += length; + } + // Generate random indices + at::Tensor indices = + torch::empty({total_length}, torch::dtype(torch::kInt32)); + auto indices_accessor = indices.accessor(); + for (int i = 0; i < total_length; ++i) { + indices_accessor[i] = int_dis(gen); + } + // Create lengths tensor + at::Tensor lengths = torch::from_blob( + np_lengths.data(), {num_lengths}, torch::dtype(torch::kInt32)); + // Calculate offsets + at::Tensor offsets = torch::cat( + {torch::zeros({1}, torch::dtype(torch::kInt32)), + torch::cumsum(lengths, 0)}); + offsets = offsets.to(torch::dtype(torch::kInt32)); + + at::Tensor per_sample_weights = at::randn(indices.sizes()); + + std::vector args{weight, indices, offsets, per_sample_weights}; + return args; +} + +std::vector generateArgsForEmbeddingBag(bool include_padding_idx) { + torch::Tensor weight = torch::randn({10, 3}, torch::dtype(torch::kFloat32)); + torch::Tensor indices = + torch::randint(0, 10, {20}, torch::dtype(torch::kInt64)); + torch::Tensor offsets = + torch::tensor({0, 5, 10, 15, 20}, torch::dtype(torch::kInt64)); + torch::Tensor per_sample_weights = + torch::rand({20}, torch::dtype(torch::kFloat32)); + // Define the padding_idx + int64_t padding_idx = 1; + // Create a vector of IValues to store the arguments + std::vector args; + args.emplace_back(weight); + args.emplace_back(indices); + args.emplace_back(offsets); + args.emplace_back(per_sample_weights); + if (include_padding_idx) { + args.emplace_back(padding_idx); + } + return args; +} +} // namespace + +TEST(StaticKernelTest, QuantizedEmbeddingBagByteRowwiseOffsets) { + const std::string graph = + R"(graph(%weight, %indices, %offsets, %per_sample_weights): +%out = torch.ops.quantized.embedding_bag_byte_rowwise_offsets.default(weight=%weight, indices=%indices, offsets=%offsets, scale_grad_by_freq=false, mode=0, pruned_weights=false, per_sample_weights=%per_sample_weights, compressed_indices_mapping=None, include_last_offset=true) +%res = torch.ops.aten.clone.default(self=%out, memory_format=None) +return (%res) +)"; + + std::vector args = generateArgsForQuantizedEmbeddingBag(); + + testStaticKernelEquality(graph, args); +} + +TEST(StaticKernelTest, QuantizedEmbeddingBag4BitRowwiseOffsets) { + const std::string graph = + R"(graph(%weight, %indices, %offsets, %per_sample_weights): +%out = torch.ops.quantized.embedding_bag_4bit_rowwise_offsets.default(weight=%weight, indices=%indices, offsets=%offsets, scale_grad_by_freq=false, mode=0, pruned_weights=false, per_sample_weights=%per_sample_weights, compressed_indices_mapping=None, include_last_offset=true) +%res = torch.ops.aten.clone.default(self=%out, memory_format=None) +return (%res) +)"; + std::vector args = generateArgsForQuantizedEmbeddingBag(); + + testStaticKernelEquality(graph, args); +} + +TEST(StaticKernelTest, EmbeddingBag) { + const std::string graph = + R"(graph(%weight, %indices, %offsets, %per_sample_weights): +%out0, %out1, %out2, %out3 = torch.ops.aten.embedding_bag.default(weight=%weight, indices=%indices, offsets=%offsets, scale_grad_by_freq=false, mode=0, sparse=false, per_sample_weights=%per_sample_weights, include_last_offset=true) +%res1 = torch.ops.aten.clone.default(self=%out0, memory_format=None) +%res2 = torch.ops.aten.clone.default(self=%out1, memory_format=None) +%res3 = torch.ops.aten.clone.default(self=%out2, memory_format=None) +%res4 = torch.ops.aten.clone.default(self=%out3, memory_format=None) +return (%res1, %res2, %res3, %res4) +)"; + std::vector args = generateArgsForEmbeddingBag(false); + testStaticKernelEquality(graph, args); + + // Test use_max_indices False + const std::string graph2 = + R"(graph(%weight, %indices, %offsets, %per_sample_weights): +%out0, %out1, %out2, %out3 = torch.ops.aten.embedding_bag.default(weight=%weight, indices=%indices, offsets=%offsets, scale_grad_by_freq=false, mode=0, sparse=false, per_sample_weights=%per_sample_weights, include_last_offset=true) +%res1 = torch.ops.aten.clone.default(self=%out0, memory_format=None) +%res2 = torch.ops.aten.clone.default(self=%out1, memory_format=None) +%res3 = torch.ops.aten.clone.default(self=%out2, memory_format=None) +return (%res1, %res2, %res3, %out2) +)"; + std::vector args2 = generateArgsForEmbeddingBag(false); + testStaticKernelEquality(graph2, args2); +} + +TEST(StaticKernelTest, EmbeddingBagPaddingIdx) { + const std::string graph = + R"(graph(%weight, %indices, %offsets, %per_sample_weights, %padding_idx): +%out0, %out1, %out2, %out3 = torch.ops.aten.embedding_bag.padding_idx(weight=%weight, indices=%indices, offsets=%offsets, scale_grad_by_freq=false, mode=0, sparse=false, per_sample_weights=%per_sample_weights, include_last_offset=true, padding_idx=%padding_idx) +%res1 = torch.ops.aten.clone.default(self=%out0, memory_format=None) +%res2 = torch.ops.aten.clone.default(self=%out1, memory_format=None) +%res3 = torch.ops.aten.clone.default(self=%out2, memory_format=None) +%res4 = torch.ops.aten.clone.default(self=%out3, memory_format=None) +return (%res1, %res2, %res3, %res4) +)"; + std::vector args = generateArgsForEmbeddingBag(true); + testStaticKernelEquality(graph, args); + + // Test use_max_indices False + const std::string graph2 = + R"(graph(%weight, %indices, %offsets, %per_sample_weights, %padding_idx): +%out0, %out1, %out2, %out3 = torch.ops.aten.embedding_bag.padding_idx(weight=%weight, indices=%indices, offsets=%offsets, scale_grad_by_freq=false, mode=0, sparse=false, per_sample_weights=%per_sample_weights, include_last_offset=true, padding_idx=%padding_idx) +%res1 = torch.ops.aten.clone.default(self=%out0, memory_format=None) +%res2 = torch.ops.aten.clone.default(self=%out1, memory_format=None) +%res3 = torch.ops.aten.clone.default(self=%out2, memory_format=None) +return (%res1, %res2, %res3, %out2) +)"; + std::vector args2 = generateArgsForEmbeddingBag(true); + testStaticKernelEquality(graph2, args2); +} + +TEST(StaticKernelTest, Aten_ToCopy) { + for (auto& target_dtype : + {"None", + "ScalarType::FLOAT", + "ScalarType::DOUBLE", + "ScalarType::HALF", + "ScalarType::INT", + "ScalarType::LONG"}) { + for (auto& target_memory_format : { + "None", + "MemoryFormat::PreserveFormat", + "MemoryFormat::ContiguousFormat", + }) { + for (auto& input_dtype : + {at::kLong, at::kInt, at::kFloat, at::kDouble, at::kHalf}) { + for (auto& permute_input : {true, false}) { + const std::string graph = fmt::format( + R"(graph(%input): +%out = torch.ops.aten._to_copy.default(self=%input, dtype={}, memory_format={}) +return (%out) +)", + target_dtype, + target_memory_format); + at::Tensor input = + at::randint(0, 128, {8, 8, 8, 8}, at::kLong).to(input_dtype); + if (permute_input) { + input = input.permute({1, 0, 3, 2}); + } + + testStaticKernelEquality(graph, {input}); + } + } + } + } +} + +TEST(StaticKernelTest, Aten_ToCopy_Aliasing) { + const std::string graph = + R"(graph(%input): + %out = torch.ops.aten._to_copy.default(self=%input, dtype=ScalarType::FLOAT, memory_format=None) + return (%out))"; + + at::Tensor input = + at::randint(0, 128, {8, 8, 8, 8}, at::kLong).to(at::kFloat); + + torch::nativert::ExecutorConfig config; + config.enableStaticCPUKernels = true; + SimpleTestModelRunner runner(graph, config); + + // try standard aliasing case + auto output = runner.run({input}); + EXPECT_TRUE(output[0].toTensor().storage().is_alias_of(input.storage())); + EXPECT_EQ(output[0].toTensor().dim(), 4); + EXPECT_EQ(output[0].toTensor().numel(), 8 * 8 * 8 * 8); + output = runner.run({input}); + EXPECT_TRUE(output[0].toTensor().storage().is_alias_of(input.storage())); + EXPECT_EQ(output[0].toTensor().dim(), 4); + EXPECT_EQ(output[0].toTensor().numel(), 8 * 8 * 8 * 8); + + // try swap out input storage between runs + at::Storage original_storage = input.storage(); + input.unsafeGetTensorImpl()->set_storage_keep_dtype( + at::randint(0, 128, {8, 8, 8, 8}, at::kLong).to(at::kFloat).storage()); + output = runner.run({input}); + EXPECT_TRUE(output[0].toTensor().storage().is_alias_of(input.storage())); + EXPECT_FALSE(output[0].toTensor().storage().is_alias_of(original_storage)); + EXPECT_EQ(output[0].toTensor().dim(), 4); + EXPECT_EQ(output[0].toTensor().numel(), 8 * 8 * 8 * 8); + + // try to upsize between runs + input.resize_({16, 16, 16, 16, 16}); + output = runner.run({input}); + EXPECT_TRUE(output[0].toTensor().storage().is_alias_of(input.storage())); + EXPECT_EQ(output[0].toTensor().dim(), 5); + EXPECT_EQ(output[0].toTensor().numel(), 16 * 16 * 16 * 16 * 16); + + // try to downsize between runs + input.resize_({4}); + output = runner.run({input}); + EXPECT_TRUE(output[0].toTensor().storage().is_alias_of(input.storage())); + EXPECT_EQ(output[0].toTensor().dim(), 1); + EXPECT_EQ(output[0].toTensor().numel(), 4); + + // try to restride between runs + input.as_strided_({3, 2}, {3, 6}).random_(); + output = runner.run({input}); + EXPECT_TRUE(output[0].toTensor().storage().is_alias_of(input.storage())); + EXPECT_EQ(output[0].toTensor().dim(), 2); + EXPECT_EQ(output[0].toTensor().numel(), 3 * 2); + for (int i = 0; i < 3; i += 1) { + for (int j = 0; j < 2; j += 1) { + EXPECT_EQ( + output[0].toTensor().index({i, j}).item().toFloat(), + input.index({i, j}).item().toFloat()); + } + } +} + +TEST(StaticKernelTest, MulScalar) { + const std::string graph = R"(graph(%in0_t, %in1_t): + %out = torch.ops.aten.mul.Scalar(self=%in0_t, other=%in1_t) + return (%out) + )"; + + std::vector>> test_cases = { + {at::rand({3, 4}), {2.0, -2.0, -2, 2, 0.0, 1e6, 1e-6, NAN, INFINITY}}, + {at::rand({2, 3, 4}), {2.0}}, + {at::rand({3, 4}, at::kFloat), {3.0}}, // fp32 tensor with int scalar + {at::randint(0, 10, {3, 4}, at::kInt), + {2.0}}, // int32 tensor with double scalar + {at::rand({3, 4}, at::kHalf), {2.0}}, // half tensor with float scalar + {at::rand({3, 4}, at::kBFloat16), {2.0}}, // bf16 tensor with float scalar + {at::randint(0, 10, {3, 4}, at::kInt), {2}}, // int tensor with int scalar + {at::randint(0, 10, {3, 4}, at::kLong), + {2}}, // int64 tensor with int64 scalar, + {at::rand({3, 4, 5}, at::kFloat).permute({2, 0, 1}), + {2}}, // int64 strided tensor with int64 scalar + {at::rand({3, 4}, at::kFloat).t(), + {2}}, // int64 strided tensor with int64 scalar + {at::rand({3, 4, 5}, at::kFloat).permute({2, 0, 1}), + {2}}, // int64 strided tensor with int64 scalar + {at::rand({3, 4}, at::kFloat).t(), + {2}}, // int64 strided tensor with int64 scalar + }; + + for (const auto& [tensor, scalars] : test_cases) { + for (double scalar : scalars) { + std::vector inputs = {tensor, scalar}; + testStaticKernelEquality(graph, inputs); + } + } +} + +TEST(StaticKernelTest, SymSizeInt) { + const std::string graph = R"(graph(%self, %dim): + %out = torch.ops.aten.sym_size.int(self=%self, dim=%dim) + return (%out) + )"; + + // Define test cases with different tensors + std::vector test_cases = { + at::rand({3, 4, 5}), // standard 3D tensor + at::rand({0, 4, 5}), // empty tensor + at::rand({1}), // single-element tensor + at::rand({2, 3, 4, 5, 6}), // high-dimensional tensor + at::rand({3, 1, 5}) // tensor with one dimension as 1 + }; + + // Iterate over each test case + for (const auto& tensor : test_cases) { + for (int64_t dim = 0; dim < tensor.dim(); ++dim) { + std::vector inputs = {tensor, dim}; + testStaticKernelEquality(graph, inputs); + } + } +} + +TEST(StaticKernelTest, BucketizeTensor) { + const std::string graph = + R"(graph(%input, %boundaries, %out_int32, %right): +%out = torch.ops.aten.bucketize.Tensor(self=%input, boundaries=%boundaries, out_int32=%out_int32, right=%right) +return (%out) +)"; + + std::vector> test_cases = { + {false, false}, {true, false}, {false, true}, {true, true}}; + + for (const auto& [out_int32, right] : test_cases) { + at::Tensor input = at::tensor({0.1, 2.5, 3.0, 4.5, 5.0}, at::kFloat); + at::Tensor boundaries = at::tensor({1.0, 2.0, 3.0, 4.0}, at::kFloat); + + std::vector args = {input, boundaries, out_int32, right}; + + testStaticKernelEquality(graph, args); + } +} + +TEST(StaticKernelTest, SliceScatter) { + const std::string graph = + R"(graph(%self, %src, %dim, %start, %end, %step): +%out = torch.ops.aten.slice_scatter.default(self=%self, src=%src, dim=%dim, start=%start, end=%end, step=%step) +return (%out) +)"; + + // Create input tensors + at::Tensor self = at::rand({5, 5}, at::kFloat); + at::Tensor src = at::rand({2, 5}, at::kFloat); + int64_t dim = 0; + int64_t start = 1; + int64_t end = 3; + int64_t step = 1; + + // Create a vector of IValues to pass as inputs + std::vector inputs = {self, src, dim, start, end, step}; + + // Run the kernel and verify the output + testStaticKernelEquality(graph, inputs); +} + +TEST(StaticKernelTest, QuantizedEmbeddingBagBytePrepack) { + const std::string graph = R"( + graph(%input): + %weight = torch.ops.quantized.embedding_bag_byte_prepack.default(weight=%input) + %res = torch.ops.aten.clone.default(self=%weight, memory_format=None) + return (%res) + )"; + + at::Tensor args1 = torch::randn({8, 16}, at::ScalarType::Float); + + testStaticKernelEquality(graph, {args1}); +} + +TEST(StaticKernelTest, QuantizedEmbeddingBagByteUnpack) { + const std::string graph = R"( + graph(%input): + %weight = torch.ops.quantized.embedding_bag_byte_prepack.default(weight=%input) + %output = torch.ops.quantized.embedding_bag_byte_unpack.default(weight=%weight) + %res = torch.ops.aten.clone.default(self=%output, memory_format=None) + return (%res) + )"; + + at::Tensor args1 = torch::randn({8, 16}, at::ScalarType::Float); + + testStaticKernelEquality(graph, {args1}); +} + +TEST(StaticKernelTest, QuantizedLinear) { + const std::string graph = R"( + graph(%input, %weights): + %packed_params = torch.ops.quantized.linear_prepack.default(W=%weights, B=None) + %1254 = torch.ops.quantized.linear.default(X=%input, W_prepack=%packed_params, Y_scale_i=1.0, Y_zero_point_i=1) + %res = torch.ops.aten.dequantize.self(self=%1254) + return (%res) + )"; + + at::Tensor input = + at::quantize_per_tensor(torch::randn({3, 2}), 2, 3, torch::kQUInt8); + at::Tensor weight = + at::quantize_per_tensor(torch::randn({3, 2}), 2, 3, torch::kQInt8); + + testStaticKernelEquality(graph, {input, weight}); +} + +TEST(NativeKernelTest, View) { + const std::string source = + R"(graph(%self): +%ret = torch.ops.aten.view.default(self=%self, size=[36]) +%cloned = torch.ops.aten.clone.default(self=%ret, memory_format=None) +return (%cloned) +)"; + + auto self0 = at::rand({6, 6}); + std::vector args{self0}; + testStaticKernelEquality(source, args, true); +} + +TEST(NativeKernelTest, Permute) { + const std::string source = + R"(graph(%self): +%ret = torch.ops.aten.permute.default(self=%self, dims=[1, 0]) +%cloned = torch.ops.aten.clone.default(self=%ret, memory_format=None) +return (%cloned) +)"; + + auto self0 = at::rand({2, 3}); + std::vector args{self0}; + testStaticKernelEquality(source, args, true); +} + +TEST(NativeKernelTest, Reshape) { + const std::string source = + R"(graph(%self): +%ret = torch.ops.aten.reshape.default(self=%self, shape=[9, 4]) +%cloned = torch.ops.aten.clone.default(self=%ret, memory_format=None) +return (%cloned) +)"; + + auto self0 = at::rand({3, 3, 4}); + std::vector args{self0}; + testStaticKernelEquality(source, args, true); +} + +TEST(NativeKernelTest, Select) { + static constexpr std::string_view source = + R"(graph(%self): +%ret = torch.ops.aten.select.int(self=%self, dim=1, index=0) +%cloned = torch.ops.aten.clone.default(self=%ret, memory_format=None) +return (%cloned) +)"; + + auto self0 = at::rand({3, 3, 3}); + std::vector args{self0}; + testStaticKernelEquality(source, args, true); +} + +TEST(NativeKernelTest, Slice) { + const std::string graph = + R"(graph(%self): +%ret = torch.ops.aten.slice.Tensor(self=%self, dim=0, start=1, end=3, step=1) +%cloned = torch.ops.aten.clone.default(self=%ret, memory_format=None) +return (%cloned) +)"; + + auto self0 = at::rand({5, 5}); + std::vector args{self0}; + testStaticKernelEquality(graph, args, true); +} + +TEST(NativeKernelTest, Split) { + const std::string graph = + R"(graph(%self): +%ret = torch.ops.aten.split.Tensor(self=%self, split_size=2, dim=0) +return (%ret) +)"; + + auto self0 = at::rand({6, 6}); + std::vector args{self0}; + testStaticKernelEquality(graph, args, true); +} + +TEST(NativeKernelTest, SplitWithSizes) { + const std::string graph = + R"(graph(%self): +%ret = torch.ops.aten.split_with_sizes.default(self=%self, split_sizes=[2, 4], dim=0) +return (%ret) +)"; + + auto self0 = at::rand({6, 6}); + std::vector args{self0}; + testStaticKernelEquality(graph, args, true); +} + +TEST(NativeKernelTest, TensorSplitSections) { + const std::string graph = + R"(graph(%self): +%ret = torch.ops.aten.tensor_split.sections(self=%self, sections=3, dim=0) +return (%ret) +)"; + + auto self0 = at::rand({9, 3}); + std::vector args{self0}; + testStaticKernelEquality(graph, args, true); +} + +TEST(StaticKernelTest, Stack) { + const std::string graph = + R"(graph(%tensors): +%ret = torch.ops.aten.stack.default(tensors=%tensors, dim=0) +return (%ret) +)"; + + auto tensor1 = at::rand({2, 3}); + auto tensor2 = at::rand({2, 3}); + auto tensor3 = at::rand({2, 3}); + std::vector args{ + std::vector{tensor1, tensor2, tensor3}}; + testStaticKernelEquality(graph, args, true); +} + +TEST(NativeKernelTest, Item) { + const std::string graph = + R"(graph(%self): +%ret = torch.ops.aten.item.default(self=%self) +return (%ret) +)"; + + auto self0 = at::tensor({42.0}); + std::vector args{self0}; + testStaticKernelEquality(graph, args, true); +} + +TEST(NativeKernelTest, Narrow) { + const std::string graph = + R"(graph(%self, %dim, %start, %length): +%ret = torch.ops.aten.narrow.default(self=%self, dim=%dim, start=%start, length=%length) +%cloned = torch.ops.aten.clone.default(self=%ret, memory_format=None) +return (%cloned) +)"; + + auto self = at::rand({5, 5}); + int64_t dim = 1; + int64_t start = 1; + int64_t length = 3; + std::vector args{self, dim, start, length}; + testStaticKernelEquality(graph, args, true); +} +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_tensor_meta.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_tensor_meta.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5660f20f4c9e1d62c4f78d6c9bbc7565048d0607 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_tensor_meta.cpp @@ -0,0 +1,62 @@ +#include +#include + +namespace torch::nativert { +TEST(TensorMetaTest, ScalarTypeConversion) { + EXPECT_EQ( + convertJsonScalarType(torch::_export::ScalarType::FLOAT), + c10::ScalarType::Float); + EXPECT_EQ( + convertJsonScalarType(torch::_export::ScalarType::INT), + c10::ScalarType::Int); + EXPECT_EQ( + convertJsonScalarType(torch::_export::ScalarType::HALF), + c10::ScalarType::Half); + EXPECT_EQ( + convertJsonScalarType(torch::_export::ScalarType::COMPLEXHALF), + c10::ScalarType::ComplexHalf); + EXPECT_EQ( + convertJsonScalarType(torch::_export::ScalarType::BFLOAT16), + c10::ScalarType::BFloat16); + EXPECT_THROW( + convertJsonScalarType(static_cast(100)), + c10::Error); +} +TEST(TensorMetaTest, MemoryFormatConversion) { + EXPECT_EQ( + convertJsonMemoryFormat(torch::_export::MemoryFormat::ContiguousFormat), + c10::MemoryFormat::Contiguous); + EXPECT_EQ( + convertJsonMemoryFormat(torch::_export::MemoryFormat::ChannelsLast), + c10::MemoryFormat::ChannelsLast); + EXPECT_EQ( + convertJsonMemoryFormat(torch::_export::MemoryFormat::PreserveFormat), + c10::MemoryFormat::Preserve); + EXPECT_THROW( + convertJsonMemoryFormat(static_cast(100)), + c10::Error); +} + +TEST(TensorMetaTest, LayoutConversion) { + EXPECT_EQ( + convertJsonLayout(torch::_export::Layout::Strided), c10::Layout::Strided); + EXPECT_EQ( + convertJsonLayout(torch::_export::Layout::SparseCsr), + c10::Layout::SparseCsr); + EXPECT_EQ( + convertJsonLayout(torch::_export::Layout::_mkldnn), c10::Layout::Mkldnn); + EXPECT_THROW( + convertJsonLayout(static_cast(100)), c10::Error); +} +TEST(TensorMetaTest, DeviceConversion) { + torch::_export::Device cpu_device; + cpu_device.set_type("cpu"); + EXPECT_EQ(convertJsonDevice(cpu_device), c10::Device(c10::DeviceType::CPU)); + torch::_export::Device cuda_device; + cuda_device.set_type("cuda"); + cuda_device.set_index(0); + EXPECT_EQ( + convertJsonDevice(cuda_device), c10::Device(c10::DeviceType::CUDA, 0)); +} + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_triton_kernel_manager_registration.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_triton_kernel_manager_registration.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8bc736733b394d7ee9d6d610eef4a5ff550b546a --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_triton_kernel_manager_registration.cpp @@ -0,0 +1,25 @@ +#include + +#include + +using namespace ::testing; +using namespace torch::nativert; + +TEST(TritonKernelManagerRegistrationTests, TestRegister) { + EXPECT_TRUE(TritonKernelManagerRegistry()->Has(at::kCPU)); + +#ifdef USE_CUDA +#ifdef USE_ROCM + EXPECT_TRUE(TritonKernelManagerRegistry()->Has(at::kHIP)); + EXPECT_FALSE(TritonKernelManagerRegistry()->Has(at::kCUDA)); + +#else + EXPECT_TRUE(TritonKernelManagerRegistry()->Has(at::kCUDA)); + EXPECT_FALSE(TritonKernelManagerRegistry()->Has(at::kHIP)); + +#endif // USE_ROCM +#else + EXPECT_FALSE(TritonKernelManagerRegistry()->Has(at::kCUDA)); + EXPECT_FALSE(TritonKernelManagerRegistry()->Has(at::kHIP)); +#endif // USE_CUDA +} diff --git a/western_ai/usa/pytorch/test/cpp/nativert/test_weights.cpp b/western_ai/usa/pytorch/test/cpp/nativert/test_weights.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8747ff91184b946ebe3f25a5d59f6d95e91f463d --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/nativert/test_weights.cpp @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace torch::nativert { +class WeightsTest : public ::testing::Test { + protected: + void SetUp() override { + static constexpr std::string_view source = + R"(graph(%foo, %bar, %baz): +%o1, %o2 = aten.foo(self=%foo, target=%bar, alpha=0.1) +return(%o2, %baz) +)"; + graph = stringToGraph(source); + placement = std::make_unique(c10::Device(c10::DeviceType::CPU)); + } + std::shared_ptr graph; + std::unique_ptr placement; +}; +TEST_F(WeightsTest, ConstructEmptyStateDict) { + std::unordered_map stateDict; + Weights weights(graph.get(), stateDict); + // Check that weights are initialized correctly + EXPECT_TRUE(weights.parameters().empty()); + EXPECT_TRUE(weights.buffers().empty()); + EXPECT_FALSE(weights.contains("non_existent_weight")); +} +TEST_F(WeightsTest, SetAndGetValue) { + std::unordered_map stateDict; + Weights weights(graph.get(), stateDict); + at::Tensor tensor = at::ones({2, 2}); + weights.setValue("added_weight", tensor); + EXPECT_TRUE(weights.contains("added_weight")); + EXPECT_EQ(weights.at("added_weight").sizes(), tensor.sizes()); +} + +} // namespace torch::nativert + +using namespace ::testing; +struct ContainsTensorDict : torch::CustomClassHolder { + explicit ContainsTensorDict(at::Tensor t) : t_(t) {} + + explicit ContainsTensorDict(c10::Dict dict) { + t_ = dict.at(std::string("init_tensor")); + } + + c10::Dict serialize() const { + c10::Dict dict; + dict.insert(std::string("init_tensor"), t_); + return dict; + } + + at::Tensor t_; +}; + +static auto reg = + torch::class_("testing", "ContainsTensorDict") + .def(torch::init()) + .def_pickle( + // __getstate__ + [](const c10::intrusive_ptr& self) + -> c10::Dict { + return self->serialize(); + }, + // __setstate__ + [](c10::Dict data) + -> c10::intrusive_ptr { + return c10::make_intrusive(std::move(data)); + }); + +TEST(CustomWeightsTest, TestCustomObjWithContainedTensor) { + // Save + auto customObj = + c10::make_intrusive(torch::tensor({1, 2, 3})); + const auto bytes = torch::jit::pickle_save(c10::IValue(std::move(customObj))); + + // Load + const auto loadedCustomObj = + torch::jit::pickle_load_obj(std::string{bytes.begin(), bytes.end()}); + EXPECT_TRUE(loadedCustomObj.isObject()); + EXPECT_EQ( + loadedCustomObj.to>() + ->t_[0] + .item(), + 1); +} diff --git a/western_ai/usa/pytorch/test/cpp/profiler/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/profiler/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..f8f4de537eb3a4fc8a09968d880c7efc899f7f17 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/profiler/CMakeLists.txt @@ -0,0 +1,100 @@ +if(USE_KINETO) + set(PROFILER_TEST_ROOT ${TORCH_ROOT}/test/cpp/profiler) + + set(PROFILER_TEST_DEPENDENCIES torch gtest_main fmt::fmt-header-only) + + add_executable(test_privateuse1_profiler + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${PROFILER_TEST_ROOT}/test_privateuse1_profiler.cpp + ) + + target_compile_definitions(test_privateuse1_profiler PRIVATE USE_GTEST USE_KINETO) + + target_link_libraries(test_privateuse1_profiler PRIVATE ${PROFILER_TEST_DEPENDENCIES}) + target_include_directories(test_privateuse1_profiler PRIVATE + ${ATen_CPU_INCLUDE} + ${KINETO_SOURCE_DIR}/include + ) + + if(INSTALL_TEST) + set_target_properties(test_privateuse1_profiler PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS test_privateuse1_profiler DESTINATION bin) + endif() + + add_executable(test_profiler_collection + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${PROFILER_TEST_ROOT}/test_profiler_collection.cpp + ) + + target_compile_definitions(test_profiler_collection PRIVATE USE_GTEST USE_KINETO) + + # Link libkineto explicitly. The test calls libkineto::api() and uses + # libkineto::GenericTraceActivity as a value type (which references its + # vtable, including GenericTraceActivity::log). Needed for Windows. + target_link_libraries(test_profiler_collection PRIVATE ${PROFILER_TEST_DEPENDENCIES} kineto) + target_include_directories(test_profiler_collection PRIVATE + ${ATen_CPU_INCLUDE} + ${KINETO_SOURCE_DIR}/include + ) + + if(INSTALL_TEST) + set_target_properties(test_profiler_collection PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS test_profiler_collection DESTINATION bin) + endif() + + add_executable(test_global_state_manager + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${PROFILER_TEST_ROOT}/test_global_state_manager.cpp + ) + + target_compile_definitions(test_global_state_manager PRIVATE USE_GTEST USE_KINETO) + + target_link_libraries(test_global_state_manager PRIVATE ${PROFILER_TEST_DEPENDENCIES}) + target_include_directories(test_global_state_manager PRIVATE + ${ATen_CPU_INCLUDE} + ${KINETO_SOURCE_DIR}/include + ) + + if(INSTALL_TEST) + set_target_properties(test_global_state_manager PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS test_global_state_manager DESTINATION bin) + endif() + + add_executable(test_profiler_global_callback_stale_session + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${PROFILER_TEST_ROOT}/profiler_global_callback_stale_session_test.cpp + ) + + target_compile_definitions(test_profiler_global_callback_stale_session PRIVATE USE_GTEST USE_KINETO) + + target_link_libraries(test_profiler_global_callback_stale_session PRIVATE ${PROFILER_TEST_DEPENDENCIES}) + target_include_directories(test_profiler_global_callback_stale_session PRIVATE + ${ATen_CPU_INCLUDE} + ${KINETO_SOURCE_DIR}/include + ) + + if(INSTALL_TEST) + set_target_properties(test_profiler_global_callback_stale_session PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS test_profiler_global_callback_stale_session DESTINATION bin) + endif() + + if(USE_DISTRIBUTED) + add_executable(test_comms_id + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${PROFILER_TEST_ROOT}/comms_id.cpp + ) + + target_compile_definitions(test_comms_id PRIVATE USE_GTEST USE_KINETO USE_DISTRIBUTED) + + target_link_libraries(test_comms_id PRIVATE ${PROFILER_TEST_DEPENDENCIES}) + target_include_directories(test_comms_id PRIVATE + ${ATen_CPU_INCLUDE} + ${KINETO_SOURCE_DIR}/include + ) + + if(INSTALL_TEST) + set_target_properties(test_comms_id PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS test_comms_id DESTINATION bin) + endif() + endif() +endif() diff --git a/western_ai/usa/pytorch/test/cpp/profiler/comms_id.cpp b/western_ai/usa/pytorch/test/cpp/profiler/comms_id.cpp new file mode 100644 index 0000000000000000000000000000000000000000..dca5ce9e39b91d5b09de3c29b9a184660d190d01 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/profiler/comms_id.cpp @@ -0,0 +1,155 @@ +#include + +#include +#include +#include + +using torch::ParamCommsDebugInfo; +using namespace torch::profiler::impl; + +namespace { + +std::shared_ptr makeDebugInfo( + const std::string& pgName, + const std::string& pgDesc, + int rank, + std::string collName, + int worldSize, + int64_t seqNumber, + bool isP2P, + int globalRankStart = 0, + int globalRankStride = 1) { + auto info = std::make_shared( + std::make_tuple(pgName, pgDesc), + rank, + std::move(collName), + /*inNelems=*/1024, + /*outNelems=*/1024, + /*dType=*/at::kFloat, + /*inSplitSizes=*/std::vector{}, + /*outSplitSizes=*/std::vector{}, + globalRankStart, + globalRankStride, + worldSize); + info->setSequenceInfo(seqNumber, isP2P); + return info; +} + +// Helper to get comms_id from saveNcclMeta for a given ParamCommsDebugInfo. +std::string getCommsIdViaSaveNcclMeta( + const std::shared_ptr& debugInfo) { + c10::DebugInfoGuard guard(c10::DebugInfoKind::PARAM_COMMS_INFO, debugInfo); + at::RecordFunction fn(at::RecordScope::USER_SCOPE); + fn._setAsync(); + auto meta = saveNcclMeta(fn); + if (meta.count(kCommsId) == 0) { + return ""; + } + return meta.at(kCommsId); +} + +} // namespace + +TEST(CommsIdTest, ParamCommsDebugInfoStoresSeqNumberAndIsP2P) { + auto info = + makeDebugInfo("pg_uid_123", "default_pg", 0, "allreduce", 8, 42, false); + + EXPECT_EQ(info->getSequenceNumber(), 42); + EXPECT_FALSE(info->getIsP2P()); + EXPECT_EQ(info->getProcessGroupName(), "pg_uid_123"); + EXPECT_EQ(info->getCollectiveName(), "allreduce"); + EXPECT_EQ(info->getWorldSize(), 8); +} + +TEST(CommsIdTest, ParamCommsDebugInfoP2PFlag) { + auto info = makeDebugInfo("pg_uid_456", "custom_pg", 3, "send", 4, 7, true); + + EXPECT_EQ(info->getSequenceNumber(), 7); + EXPECT_TRUE(info->getIsP2P()); +} + +TEST(CommsIdTest, ParamCommsDebugInfoDefaultSeqNumberAndIsP2P) { + auto info = std::make_shared( + std::make_tuple(std::string("pg_uid_789"), std::string("default_pg")), + /*rank=*/1, + /*collName=*/std::string("allgather"), + /*inNelems=*/256, + /*outNelems=*/512, + /*dType=*/at::kFloat, + /*inSplitSizes=*/std::vector{}, + /*outSplitSizes=*/std::vector{}, + /*globalRankStart=*/0, + /*globalRankStride=*/1, + /*worldSize=*/2); + + EXPECT_EQ(info->getSequenceNumber(), -1); + EXPECT_FALSE(info->getIsP2P()); +} + +TEST(CommsIdTest, SaveNcclMetaEmitsCommsId) { + auto debugInfo = + makeDebugInfo("pg_uid_123", "default_pg", 0, "allreduce", 8, 42, false); + + auto commsId = getCommsIdViaSaveNcclMeta(debugInfo); + EXPECT_FALSE(commsId.empty()); + + // Verify determinism: same input produces the same comms_id + auto commsId2 = getCommsIdViaSaveNcclMeta(debugInfo); + EXPECT_EQ(commsId, commsId2); +} + +TEST(CommsIdTest, SaveNcclMetaOmitsCommsIdWhenSeqNotSet) { + auto debugInfo = std::make_shared( + std::make_tuple(std::string("pg_uid_no_seq"), std::string("default_pg")), + /*rank=*/0, + /*collName=*/std::string("allreduce"), + /*inNelems=*/1024, + /*outNelems=*/1024, + /*dType=*/at::kFloat, + /*inSplitSizes=*/std::vector{}, + /*outSplitSizes=*/std::vector{}, + /*globalRankStart=*/0, + /*globalRankStride=*/1, + /*worldSize=*/8); + + auto commsId = getCommsIdViaSaveNcclMeta(debugInfo); + EXPECT_TRUE(commsId.empty()); +} + +TEST(CommsIdTest, CommsIdDiffersForDifferentSeqNumbers) { + auto id1 = getCommsIdViaSaveNcclMeta( + makeDebugInfo("pg", "desc", 0, "allreduce", 8, 1, false)); + auto id2 = getCommsIdViaSaveNcclMeta( + makeDebugInfo("pg", "desc", 0, "allreduce", 8, 2, false)); + EXPECT_NE(id1, id2); +} + +TEST(CommsIdTest, CommsIdDiffersForDifferentPGNames) { + auto id1 = getCommsIdViaSaveNcclMeta( + makeDebugInfo("pg_A", "desc", 0, "allreduce", 8, 42, false)); + auto id2 = getCommsIdViaSaveNcclMeta( + makeDebugInfo("pg_B", "desc", 0, "allreduce", 8, 42, false)); + EXPECT_NE(id1, id2); +} + +TEST(CommsIdTest, CommsIdDiffersForP2PvsCollective) { + auto id_collective = getCommsIdViaSaveNcclMeta( + makeDebugInfo("pg", "desc", 0, "allreduce", 8, 42, false)); + auto id_p2p = getCommsIdViaSaveNcclMeta( + makeDebugInfo("pg", "desc", 0, "send", 8, 42, true)); + EXPECT_NE(id_collective, id_p2p); +} + +TEST(CommsIdTest, CommsIdDiffersForDifferentTopology) { + auto id1 = getCommsIdViaSaveNcclMeta( + makeDebugInfo("pg", "desc", 0, "allreduce", 8, 42, false, 0, 1)); + auto id2 = getCommsIdViaSaveNcclMeta( + makeDebugInfo("pg", "desc", 0, "allreduce", 4, 42, false, 0, 1)); + EXPECT_NE(id1, id2); + + auto id3 = getCommsIdViaSaveNcclMeta( + makeDebugInfo("pg", "desc", 0, "allreduce", 4, 42, false, 0, 2)); + auto id4 = getCommsIdViaSaveNcclMeta( + makeDebugInfo("pg", "desc", 0, "allreduce", 4, 42, false, 1, 2)); + EXPECT_NE(id3, id4); +} diff --git a/western_ai/usa/pytorch/test/cpp/profiler/containers.cpp b/western_ai/usa/pytorch/test/cpp/profiler/containers.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a4d6d9d0c57acea9269ec12d44ae470076477732 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/profiler/containers.cpp @@ -0,0 +1,90 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +TEST(ProfilerTest, AppendOnlyList) { + const int n = 4096; + torch::profiler::impl::AppendOnlyList list; + for (const auto i : c10::irange(n)) { + list.emplace_back(i); + ASSERT_EQ(list.size(), i + 1); + } + + int expected = 0; + for (const auto i : list) { + ASSERT_EQ(i, expected++); + } + ASSERT_EQ(expected, n); + + list.clear(); + ASSERT_EQ(list.size(), 0); +} + +TEST(ProfilerTest, AppendOnlyList_ref) { + const int n = 512; + torch::profiler::impl::AppendOnlyList, 64> list; + std::vector*> refs; + for ([[maybe_unused]] const auto _ : c10::irange(n)) { + refs.push_back(list.emplace_back()); + } + + for (const auto i : c10::irange(n)) { + *refs.at(i) = {i, 0}; + } + + int expected = 0; + for (const auto& i : list) { + ASSERT_EQ(i.first, expected++); + } +} + +// Test that we can convert TSC measurements back to wall clock time. +TEST(ProfilerTest, clock_converter) { + const int n = 10001; + c10::ApproximateClockToUnixTimeConverter converter; + std::vector< + c10::ApproximateClockToUnixTimeConverter::UnixAndApproximateTimePair> + pairs; + for ([[maybe_unused]] const auto i : c10::irange(n)) { + pairs.push_back(c10::ApproximateClockToUnixTimeConverter::measurePair()); + } + auto count_to_ns = converter.makeConverter(); + std::vector deltas; + for (const auto& i : pairs) { + deltas.push_back(i.t_ - count_to_ns(i.approx_t_)); + } + std::sort(deltas.begin(), deltas.end()); + + // In general it's not a good idea to put clocks in unit tests as it leads + // to flakiness. We mitigate this by: + // 1) Testing the clock itself. While the time to complete a task may + // vary, two clocks measuring the same time should be much more + // consistent. + // 2) Only testing the interquartile range. Context switches between + // calls to the two timers do occur and can result in hundreds of + // nanoseconds of noise, but such switches are only a few percent + // of cases. + // 3) We're willing to accept a somewhat large bias which can emerge from + // differences in the cost of calling each clock. + EXPECT_LT(std::abs(deltas[n / 2]), 200); + EXPECT_LT(deltas[n * 3 / 4] - deltas[n / 4], 50); +} + +TEST(ProfilerTest, soft_assert) { + EXPECT_TRUE(SOFT_ASSERT(true)); + torch::profiler::impl::setSoftAssertRaises(true); + EXPECT_ANY_THROW(SOFT_ASSERT(false)); + torch::profiler::impl::setSoftAssertRaises(false); + EXPECT_NO_THROW(SOFT_ASSERT(false)); + // Reset soft assert behavior to default + torch::profiler::impl::setSoftAssertRaises(std::nullopt); + EXPECT_NO_THROW(SOFT_ASSERT(false)); +} diff --git a/western_ai/usa/pytorch/test/cpp/profiler/perf_events.cpp b/western_ai/usa/pytorch/test/cpp/profiler/perf_events.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3aa843311970b79a77d363fdb557ad5c3c1a4834 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/profiler/perf_events.cpp @@ -0,0 +1,248 @@ + +#include + +#include +#include + +double calc_pi() { + volatile double pi = 1.0; + for (int i = 3; i < 100000; i += 2) { + pi += (((i + 1) >> 1) % 2) ? 1.0 / i : -1.0 / i; + } + return pi * 4.0; +} + +TEST(ProfilerTest, LinuxPerf) { + torch::profiler::impl::linux_perf::PerfProfiler profiler; + + std::vector standard_events( + std::begin(torch::profiler::ProfilerPerfEvents), + std::end(torch::profiler::ProfilerPerfEvents)); + torch::profiler::perf_counters_t counters; + counters.resize(standard_events.size(), 0); + + // Use try..catch HACK to check TORCH_CHECK because we don't yet fail + // gracefully if the syscall were to fail + try { + profiler.Configure(standard_events); + + profiler.Enable(); + calc_pi(); + profiler.Disable(counters); + } catch (const c10::Error&) { + // Bail here if something bad happened during the profiling, we don't want + // to make the test fail + return; + } catch (...) { + // something else went wrong - this should be reported + ASSERT_EQ(0, 1); + } + + // Should have counted something if worked, so lets test that + // And if it not supported the counters should be zeros. +#if defined(__ANDROID__) || defined(__linux__) + for (auto counter : counters) { + ASSERT_GT(counter, 0); + } +#else /* __ANDROID__ || __linux__ */ + for (auto counter : counters) { + ASSERT_EQ(counter, 0); + } +#endif /* __ANDROID__ || __linux__ */ +} + +TEST(ProfilerTest, LinuxPerfNestedDepth) { + torch::profiler::impl::linux_perf::PerfProfiler profiler; + + // Only monotonically increasing events will work + std::vector standard_events( + std::begin(torch::profiler::ProfilerPerfEvents), + std::end(torch::profiler::ProfilerPerfEvents)); + + torch::profiler::perf_counters_t counters_A; + torch::profiler::perf_counters_t counters_B; + torch::profiler::perf_counters_t counters_C; + + counters_A.resize(standard_events.size(), 0); + counters_B.resize(standard_events.size(), 0); + counters_C.resize(standard_events.size(), 0); + + // Use try..catch HACK to check TORCH_CHECK because we don't yet fail + // gracefully if the syscall were to fail + try { + profiler.Configure(standard_events); + + // * = work kernel calc_pi() + // + // A --*---+ +--*-- A + // | | + // | | + // B +-*--+ +--*-+ B + // | | + // | | + // C +-*--+ C + // + + profiler.Enable(); + calc_pi(); + + profiler.Enable(); + calc_pi(); + + profiler.Enable(); + calc_pi(); + profiler.Disable(counters_C); + + calc_pi(); + profiler.Disable(counters_B); + + calc_pi(); + profiler.Disable(counters_A); + } catch (const c10::Error&) { + // Bail here if something bad happened during the profiling, we don't want + // to make the test fail + return; + } catch (...) { + // something else went wrong - this should be reported + ASSERT_EQ(0, 1); + } + +// for each counter, assert A > B > C +#if defined(__ANDROID__) || defined(__linux__) + for (auto i = 0; i < standard_events.size(); ++i) { + ASSERT_GT(counters_A[i], counters_B[i]); + ASSERT_GT(counters_A[i], counters_C[i]); + ASSERT_GT(counters_B[i], counters_C[i]); + ASSERT_GT(counters_A[i], counters_B[i] + counters_C[i]); + } +#else /* __ANDROID__ || __linux__ */ + for (auto i = 0; i < standard_events.size(); ++i) { + ASSERT_EQ(counters_A[i], 0); + ASSERT_EQ(counters_B[i], 0); + ASSERT_EQ(counters_C[i], 0); + } +#endif /* __ANDROID__ || __linux__ */ +} + +TEST(ProfilerTest, LinuxPerfNestedMultiple) { + torch::profiler::impl::linux_perf::PerfProfiler profiler; + + // Only monotonically increasing events will work + std::vector standard_events( + std::begin(torch::profiler::ProfilerPerfEvents), + std::end(torch::profiler::ProfilerPerfEvents)); + + torch::profiler::perf_counters_t counters_A; + torch::profiler::perf_counters_t counters_B; + torch::profiler::perf_counters_t counters_C; + + counters_A.resize(standard_events.size(), 0); + counters_B.resize(standard_events.size(), 0); + counters_C.resize(standard_events.size(), 0); + + // Use try..catch HACK to check TORCH_CHECK because we don't yet fail + // gracefully if the syscall were to fail + try { + profiler.Configure(standard_events); + + // * = work kernel calc_pi() + // + // A --*---+ +---*----+ +--*-- A + // | | | | + // | | | | + // B +-**-+ B C +-*--+ C + + profiler.Enable(); + calc_pi(); + + profiler.Enable(); + calc_pi(); + calc_pi(); + profiler.Disable(counters_B); + + calc_pi(); + + profiler.Enable(); + calc_pi(); + profiler.Disable(counters_C); + + calc_pi(); + profiler.Disable(counters_A); + } catch (const c10::Error&) { + // Bail here if something bad happened during the profiling, we don't want + // to make the test fail + return; + } catch (...) { + // something else went wrong - this should be reported + ASSERT_EQ(0, 1); + } + +// for each counter, assert A > B > C +#if defined(__ANDROID__) || defined(__linux__) + for (auto i = 0; i < standard_events.size(); ++i) { + ASSERT_GT(counters_A[i], counters_B[i]); + ASSERT_GT(counters_A[i], counters_C[i]); + ASSERT_GT(counters_B[i], counters_C[i]); + ASSERT_GT(counters_A[i], counters_B[i] + counters_C[i]); + } +#else /* __ANDROID__ || __linux__ */ + for (auto i = 0; i < standard_events.size(); ++i) { + ASSERT_EQ(counters_A[i], 0); + ASSERT_EQ(counters_B[i], 0); + ASSERT_EQ(counters_C[i], 0); + } +#endif /* __ANDROID__ || __linux__ */ +} + +TEST(ProfilerTest, LinuxPerfNestedSingle) { + torch::profiler::impl::linux_perf::PerfProfiler profiler; + + // Only monotonically increasing events will work + std::vector standard_events( + std::begin(torch::profiler::ProfilerPerfEvents), + std::end(torch::profiler::ProfilerPerfEvents)); + + torch::profiler::perf_counters_t counters_A; + torch::profiler::perf_counters_t counters_B; + torch::profiler::perf_counters_t counters_C; + + counters_A.resize(standard_events.size(), 0); + counters_B.resize(standard_events.size(), 0); + counters_C.resize(standard_events.size(), 0); + + // Use try..catch HACK to check TORCH_CHECK because we don't yet fail + // gracefully if the syscall were to fail + try { + profiler.Configure(standard_events); + + profiler.Enable(); + profiler.Enable(); + profiler.Enable(); + calc_pi(); + profiler.Disable(counters_C); + profiler.Disable(counters_B); + profiler.Disable(counters_A); + } catch (const c10::Error&) { + // Bail here if something bad happened during the profiling, we don't want + // to make the test fail + return; + } catch (...) { + // something else went wrong - this should be reported + ASSERT_EQ(0, 1); + } + +// for each counter, assert A > B > C +#if defined(__ANDROID__) || defined(__linux__) + for (auto i = 0; i < standard_events.size(); ++i) { + ASSERT_GE(counters_A[i], counters_B[i]); + ASSERT_GE(counters_A[i], counters_C[i]); + ASSERT_GE(counters_B[i], counters_C[i]); + } +#else /* __ANDROID__ || __linux__ */ + for (auto i = 0; i < standard_events.size(); ++i) { + ASSERT_EQ(counters_A[i], 0); + ASSERT_EQ(counters_B[i], 0); + ASSERT_EQ(counters_C[i], 0); + } +#endif /* __ANDROID__ || __linux__ */ +} diff --git a/western_ai/usa/pytorch/test/cpp/profiler/profiler_global_callback_stale_session_test.cpp b/western_ai/usa/pytorch/test/cpp/profiler/profiler_global_callback_stale_session_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..386436a5e7e13a231dc5572a1bb63e6d4a727bc1 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/profiler/profiler_global_callback_stale_session_test.cpp @@ -0,0 +1,104 @@ +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +// Reproducer for a cross-session use-after-free in the global RecordFunction +// callback path. The narrow write-section guard keeps a callback's critical +// section from overlapping teardown, but it intentionally does not span the op +// body (that would deadlock on the GIL). So an op can: +// +// 1. Enter in session A. +// 2. Have A torn down (freeing its RecordQueue event) while in-flight. +// 3. Exit while a LATER session B is active. +// +// The gate is open for B, so the exit is not skipped and onFunctionExitGlobal +// dereferences A's freed event. We tag each ObserverContext with the session +// generation and drop the exit when it no longer matches. +// +// at::RecordFunction's destructor runs the captured exit callback even after +// A's callback was removed. + +namespace { +using torch::profiler::impl::ActivityType; +using torch::profiler::impl::ExperimentalConfig; +using torch::profiler::impl::ProfilerConfig; +using torch::profiler::impl::ProfilerState; + +ProfilerConfig makeGlobalCallbackCpuConfig() { + ExperimentalConfig experimental_config; + experimental_config.profile_all_threads = true; + return ProfilerConfig( + ProfilerState::KINETO, + /*report_input_shapes=*/false, + /*profile_memory=*/false, + /*with_stack=*/false, + /*with_flops=*/false, + /*with_modules=*/false, + experimental_config); +} + +void enableGlobalProfiler() { + const auto config = makeGlobalCallbackCpuConfig(); + const std::set activities{ActivityType::CPU}; + const std::unordered_set scopes{at::RecordScope::FUNCTION}; + torch::autograd::profiler::prepareProfiler(config, activities); + torch::autograd::profiler::enableProfiler(config, activities, scopes); +} +} // namespace + +TEST( + ProfilerGlobalCallbackStaleSessionTest, + ExitAfterSessionTornDownIsDropped) { + at::clearCallbacks(); + + std::mutex mtx; + std::condition_variable cv; + bool worker_entered = false; // guarded by mtx + bool release_worker = false; // guarded by mtx + + enableGlobalProfiler(); // session A + + std::thread worker([&]() { + at::RecordFunction guard(at::RecordScope::FUNCTION); + guard.before("stale_probe_op"); // onFunctionEnterGlobal: ctx tagged gen(A) + { + std::unique_lock lock(mtx); + worker_entered = true; + cv.notify_all(); + cv.wait(lock, [&] { return release_worker; }); + } + // guard destructor -> onFunctionExitGlobal, now during session B + }); + + { + std::unique_lock lock(mtx); + cv.wait(lock, [&] { return worker_entered; }); + } + + // Tear down A (frees its RecordQueue, including the in-flight op's event), + // then bring up a fresh session B so the pending exit lands with the gate + // open. + torch::autograd::profiler::disableProfiler(); + enableGlobalProfiler(); // session B + + // Release the op: its exit runs in B with a ctx tagged gen(A). This should + // be safe because of generation checks. + { + std::lock_guard lock(mtx); + release_worker = true; + } + cv.notify_all(); + worker.join(); + + torch::autograd::profiler::disableProfiler(); + EXPECT_FALSE(at::hasGlobalCallbacks()); +} diff --git a/western_ai/usa/pytorch/test/cpp/profiler/record_function.cpp b/western_ai/usa/pytorch/test/cpp/profiler/record_function.cpp new file mode 100644 index 0000000000000000000000000000000000000000..282cdd1acc7dd92ebd91577a9fc14f55caca00fd --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/profiler/record_function.cpp @@ -0,0 +1,329 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +// Test that we can add and remove callbacks (both global and thread local.) +TEST(RecordFunctionTest, AddRemove) { + at::clearCallbacks(); + ASSERT_FALSE(at::hasCallbacks()); + + auto start_callback = + [](const at::RecordFunction& fn) -> std::unique_ptr { + return nullptr; + }; + auto end_callback = [](const at::RecordFunction& fn, at::ObserverContext*) {}; + + auto handle = at::addThreadLocalCallback( + at::RecordFunctionCallback(start_callback, end_callback)); + + ASSERT_TRUE(at::hasCallbacks()); + ASSERT_TRUE(at::hasThreadLocalCallbacks()); + ASSERT_FALSE(at::hasGlobalCallbacks()); + + at::removeCallback(handle); + ASSERT_FALSE(at::hasCallbacks()); + + handle = at::addGlobalCallback( + at::RecordFunctionCallback(start_callback, end_callback)); + + ASSERT_TRUE(at::hasCallbacks()); + ASSERT_FALSE(at::hasThreadLocalCallbacks()); + ASSERT_TRUE(at::hasGlobalCallbacks()); + + at::removeCallback(handle); + ASSERT_FALSE(at::hasCallbacks()); +} + +// Test that the callbacks that we register are actually run. +TEST(RecordFunctionTest, ThreadLocalState) { + at::clearCallbacks(); + ASSERT_FALSE(at::hasCallbacks()); + + static int tls_test_start_counter; + static int tls_test_end_counter; + tls_test_start_counter = 0; + tls_test_end_counter = 0; + + auto start_callback = + [](const at::RecordFunction&) -> std::unique_ptr { + ++tls_test_start_counter; + return nullptr; + }; + auto end_callback = [](const at::RecordFunction&, at::ObserverContext*) { + ++tls_test_end_counter; + }; + + auto handle = at::addThreadLocalCallback( + at::RecordFunctionCallback(start_callback, end_callback)); + + { + at::RecordFunction guard(at::RecordScope::USER_SCOPE); + guard.before("Test"); + EXPECT_EQ(tls_test_start_counter, 1); + EXPECT_EQ(tls_test_end_counter, 0); + } + EXPECT_EQ(tls_test_start_counter, 1); + EXPECT_EQ(tls_test_end_counter, 1); + + { + tls_test_start_counter = 0; + tls_test_end_counter = 0; + at::DisableRecordFunctionGuard no_profile_guard; + at::RecordFunction guard(at::RecordScope::USER_SCOPE); + guard.before("Test"); + EXPECT_EQ(tls_test_start_counter, 0); + EXPECT_EQ(tls_test_end_counter, 0); + } + EXPECT_EQ(tls_test_start_counter, 0); + EXPECT_EQ(tls_test_end_counter, 0); + + { + tls_test_start_counter = 0; + tls_test_end_counter = 0; + RECORD_FUNCTION("Test", {}); + EXPECT_EQ(tls_test_start_counter, 1); + EXPECT_EQ(tls_test_end_counter, 0); + } + EXPECT_EQ(tls_test_start_counter, 1); + EXPECT_EQ(tls_test_end_counter, 1); + + at::removeCallback(handle); + ASSERT_FALSE(at::hasCallbacks()); +} + +// Test that callbacks are run in the order that they are registered. +TEST(RecordFunctionTest, CallOrder) { + at::clearCallbacks(); + ASSERT_FALSE(at::hasCallbacks()); + + static int current_index; + current_index = 0; + + static std::array expected_order = { + "Start Callback 0 Outer", + "Start Callback 1 Outer", + "Start Callback 0 Inner", + "Start Callback 1 Inner", + "End Callback 0 Inner", + "End Callback 1 Inner", + "End Callback 0 Outer", + "End Callback 1 Outer", + }; + +#define REGISTER_CALLBACK(index) \ + at::addThreadLocalCallback( \ + at::RecordFunctionCallback( \ + [](const at::RecordFunction& fn) \ + -> std::unique_ptr { \ + EXPECT_EQ( \ + fmt::format("Start Callback {} {}", index, fn.name()), \ + expected_order[current_index++]); \ + return nullptr; \ + }, \ + [](const at::RecordFunction& fn, at::ObserverContext*) { \ + EXPECT_EQ( \ + fmt::format("End Callback {} {}", index, fn.name()), \ + expected_order[current_index++]); \ + }) \ + .scopes({at::RecordScope::FUNCTION})) + + REGISTER_CALLBACK(0); + REGISTER_CALLBACK(1); +#undef REGISTER_CALLBACK + + RECORD_FUNCTION("Outer", {}); + { + RECORD_FUNCTION("Inner", {}); + } + + at::clearCallbacks(); + ASSERT_FALSE(at::hasCallbacks()); +} + +// Make sure TLS migrates when tasks are launched. +TEST(RecordFunctionTest, ThreadMigration) { + at::clearCallbacks(); + ASSERT_FALSE(at::hasCallbacks()); + + static int call_count; + call_count = 0; + + auto handle = at::addThreadLocalCallback( + at::RecordFunctionCallback( + [](const at::RecordFunction&) + -> std::unique_ptr { return nullptr; }, + [](const at::RecordFunction&, at::ObserverContext*) { ++call_count; }) + .scopes({at::RecordScope::FUNCTION})); + + EXPECT_EQ(call_count, 0); + + std::condition_variable cv; + std::mutex lock; + at::launch([&cv]() { + RECORD_FUNCTION("Test", {}); + cv.notify_all(); + }); + auto guard = std::unique_lock(lock); + cv.wait(guard, [] { return call_count > 0; }); + + EXPECT_EQ(call_count, 1); + + at::removeCallback(handle); + ASSERT_FALSE(at::hasCallbacks()); +} + +// Test sampling logic and validate that callbacks fire at the correct times. +TEST(RecordFunctionTest, Sampling) { + at::clearCallbacks(); + ASSERT_FALSE(at::hasCallbacks()); + + static int sample_test_counter; + sample_test_counter = 0; + + uint32_t seed = 12345; + double p = 0.25; + + at::set_record_function_seed_for_testing(seed); + std::mt19937 generator; + generator.seed(seed); + auto dist = std::geometric_distribution(p); + + // Make sure we know which steps should fire. + auto outcomes = std::array{7, 0, 0, 6, 2}; + for (const auto i : c10::irange(outcomes.size())) { + ASSERT_EQ(dist(generator), outcomes[i]); + } + + std::vector expected_counts; + int running_count = 0; + for (const auto i : c10::irange(outcomes.size())) { + for ([[maybe_unused]] const auto j : c10::irange(outcomes[i])) { + expected_counts.push_back(running_count); + } + expected_counts.push_back(++running_count); + } + + auto start_callback = + [](const at::RecordFunction& fn) -> std::unique_ptr { + ++sample_test_counter; + return nullptr; + }; + auto end_callback = [](const at::RecordFunction& fn, at::ObserverContext*) {}; + + auto handle = at::addThreadLocalCallback( + at::RecordFunctionCallback(start_callback, end_callback) + .samplingProb(p) + .scopes({at::RecordScope::FUNCTION})); + + for (const auto i : c10::irange(expected_counts.size())) { + RECORD_FUNCTION("Test", {}); + EXPECT_EQ(sample_test_counter, expected_counts[i]); + } + + at::removeCallback(handle); + ASSERT_FALSE(at::hasCallbacks()); +} + +// Validate sampling against a simple reference implementation for a complex set +// of registered callbacks. +TEST(RecordFunctionTest, MultipleCallbacks) { + at::clearCallbacks(); + ASSERT_FALSE(at::hasCallbacks()); + + uint32_t seed = 54321; + + std::mt19937 generator; + generator.seed(seed); + + auto sample = [&](double p) { + return (p < 1.0 ? std::geometric_distribution(p)(generator) : 0) + 1; + }; + + std::array probabilities{0.1, 1.0, 1.0, 0.3}; + std::array next_call; + std::array counts; + static std::array counts_from_rec_fn; + counts_from_rec_fn.fill(0); + + auto end_callback = [](const at::RecordFunction& fn, at::ObserverContext*) {}; + +#define REGISTER_CALLBACK(register_fn, index) \ + register_fn(at::RecordFunctionCallback( \ + [](const at::RecordFunction& fn) \ + -> std::unique_ptr { \ + ++counts_from_rec_fn[index]; \ + return nullptr; \ + }, \ + end_callback) \ + .samplingProb(probabilities[index]) \ + .scopes({at::RecordScope::FUNCTION})) + + REGISTER_CALLBACK(at::addGlobalCallback, 0); + REGISTER_CALLBACK(at::addGlobalCallback, 1); + REGISTER_CALLBACK(at::addThreadLocalCallback, 2); + + // The RecordFunction machinery will rebuild callbacks whenever a new observer + // is registered, so we need to wait until the last callback to seed the + // random number generator. + at::set_record_function_seed_for_testing(seed); + REGISTER_CALLBACK(at::addThreadLocalCallback, 3); +#undef REGISTER_CALLBACK + + for (const auto i : c10::irange(probabilities.size())) { + next_call[i] = sample(probabilities[i]); + } + + for ([[maybe_unused]] const auto i : c10::irange(50)) { + RECORD_FUNCTION("Test", {}); + for (const auto j : c10::irange(next_call.size())) { + if (!(--next_call[j])) { + ++counts[j]; + next_call[j] = sample(probabilities[j]); + } + EXPECT_EQ(counts[j], counts_from_rec_fn[j]); + } + } + + at::clearCallbacks(); + ASSERT_FALSE(at::hasCallbacks()); +} + +// Test that KwargsOnly callbacks are run in USER_SCOPE. +TEST(RecordFunctionTest, KwargsOnly) { + at::clearCallbacks(); + ASSERT_FALSE(at::hasCallbacks()); + static const std::unordered_map myMap = { + {"a", 1}, {"b", 2.5}}; + +#define REGISTER_CALLBACK() \ + at::addThreadLocalCallback( \ + at::RecordFunctionCallback( \ + [](const at::RecordFunction& fn) \ + -> std::unique_ptr { \ + EXPECT_EQ(myMap, fn.kwinputs()); \ + return nullptr; \ + }, \ + [](const at::RecordFunction& fn, at::ObserverContext*) {}) \ + .needsInputs(true) \ + .scopes({at::RecordScope::USER_SCOPE})) + + REGISTER_CALLBACK(); +#undef REGISTER_CALLBACK + + RECORD_USER_SCOPE_WITH_KWARGS_ONLY("Test", &myMap); + + at::clearCallbacks(); + ASSERT_FALSE(at::hasCallbacks()); +} diff --git a/western_ai/usa/pytorch/test/cpp/profiler/test_global_state_manager.cpp b/western_ai/usa/pytorch/test/cpp/profiler/test_global_state_manager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7f59c030ae9a0c27e87989b82bdca82689fea5e4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/profiler/test_global_state_manager.cpp @@ -0,0 +1,77 @@ +// Unit tests for GlobalStateManager in torch/csrc/profiler/util.h. +// +// The profiler use-after-free fix rests on GlobalStateManager::get() returning +// an owning shared_ptr: a caller that has already fetched the state keeps it +// alive even after the manager drops its own reference in pop(). These tests +// pin that ownership contract. They are single threaded; the contract is what +// makes the concurrent teardown path safe, so it can be verified without +// threads or a sanitizer. + +#include + +#include + +#include + +namespace { + +// Each test uses a distinct tag type so the per-type process-global singleton +// inside GlobalStateManager stays isolated across tests, which may run in any +// order within the binary. +template +struct Tracked { + explicit Tracked(int* destroyed_count) : destroyed_count_(destroyed_count) {} + ~Tracked() { + ++(*destroyed_count_); + } + int* destroyed_count_; +}; + +} // namespace + +using torch::profiler::impl::GlobalStateManager; + +TEST(GlobalStateManagerTest, GetReturnsOwningHandleThatOutlivesPop) { + int destroyed = 0; + GlobalStateManager>::push( + std::make_shared>(&destroyed)); + + auto held = GlobalStateManager>::get(); + ASSERT_NE(held, nullptr); + + // pop() drops the manager reference, but held still owns the object. + auto popped = GlobalStateManager>::pop(); + ASSERT_EQ(held.get(), popped.get()); + + popped.reset(); + EXPECT_EQ(destroyed, 0); // still alive: held still owns it + + held.reset(); + EXPECT_EQ(destroyed, 1); // last owner released + + EXPECT_EQ(GlobalStateManager>::get(), nullptr); +} + +TEST(GlobalStateManagerTest, GetAndPopOnEmptyReturnNull) { + // Tracked<1>'s singleton is never pushed to, so it starts empty. + EXPECT_EQ(GlobalStateManager>::get(), nullptr); + EXPECT_EQ(GlobalStateManager>::pop(), nullptr); +} + +TEST(GlobalStateManagerTest, PushWhenAlreadySetIsNoOp) { + int first_destroyed = 0; + auto first = std::make_shared>(&first_destroyed); + auto* first_raw = first.get(); + GlobalStateManager>::push(std::move(first)); + + // A second push must not replace the state already held by the manager. + int second_destroyed = 0; + GlobalStateManager>::push( + std::make_shared>(&second_destroyed)); + EXPECT_EQ(second_destroyed, 1); // the ignored push drops its object + EXPECT_EQ(GlobalStateManager>::get().get(), first_raw); + + // Release the stored object so it does not outlive first_destroyed. + GlobalStateManager>::pop(); + EXPECT_EQ(first_destroyed, 1); +} diff --git a/western_ai/usa/pytorch/test/cpp/profiler/test_privateuse1_profiler.cpp b/western_ai/usa/pytorch/test/cpp/profiler/test_privateuse1_profiler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9c2d008c35d4deb784cd184f9e365b5b249c926e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/profiler/test_privateuse1_profiler.cpp @@ -0,0 +1,181 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifdef USE_KINETO + +#include + +#include +#include +#include + +#include +#include + +#include +#include + +namespace { + +// Flag to track if Kineto actually called our mock profiler's configure() +// This proves the full flow: registration → Kineto stores it → Kineto uses it +std::atomic g_configure_called{false}; + +// Mock IActivityProfilerSession implementation +class MockProfilerSession : public libkineto::IActivityProfilerSession { + public: + MockProfilerSession() = default; + + void start() override { + status_ = libkineto::TraceStatus::RECORDING; + } + + void stop() override { + status_ = libkineto::TraceStatus::PROCESSING; + } + + std::vector errors() override { + return {}; + } + + void processTrace( + [[maybe_unused]] libkineto::ActivityLogger& logger) override {} + + std::unique_ptr getDeviceInfo() override { + return std::make_unique( + /*id=*/0, + /*sortIndex=*/0, + /*name=*/"MockDevice", + /*label=*/"mock_device"); + } + + std::vector getResourceInfos() override { + return {}; + } + + std::unique_ptr getTraceBuffer() override { + return nullptr; + } +}; + +// Mock IActivityProfiler implementation for testing +class MockPrivateUse1Profiler : public libkineto::IActivityProfiler { + public: + MockPrivateUse1Profiler() = default; + + const std::string& name() const override { + return name_; + } + + const std::set& availableActivities() + const override { + return activities_; + } + + std::unique_ptr configure( + [[maybe_unused]] const std::set& activity_types, + [[maybe_unused]] const libkineto::Config& config) override { + g_configure_called = true; + return std::make_unique(); + } + + std::unique_ptr configure( + [[maybe_unused]] int64_t ts_ms, + [[maybe_unused]] int64_t duration_ms, + const std::set& activity_types, + const libkineto::Config& config) override { + return configure(activity_types, config); + } + + private: + std::string name_{"mock_privateuse1"}; + std::set activities_{ + libkineto::ActivityType::CPU_OP}; +}; + +} // namespace + +// Test that the registry singleton works correctly +TEST(PrivateUse1ProfilerTest, RegistrySingleton) { + auto& registry1 = + torch::profiler::impl::PrivateUse1ProfilerRegistry::instance(); + auto& registry2 = + torch::profiler::impl::PrivateUse1ProfilerRegistry::instance(); + EXPECT_EQ(®istry1, ®istry2); +} + +// Test that registering a factory works +TEST(PrivateUse1ProfilerTest, RegisterFactory) { + auto& registry = + torch::profiler::impl::PrivateUse1ProfilerRegistry::instance(); + + registry.registerFactory( + []() { return std::make_unique(); }); + + EXPECT_TRUE(registry.hasFactory()); +} + +// Test that onKinetoInit triggers registration when factory is registered +TEST(PrivateUse1ProfilerTest, OnKinetoInitForwarding) { + auto& registry = + torch::profiler::impl::PrivateUse1ProfilerRegistry::instance(); + + // Register factory if not already registered + if (!registry.hasFactory()) { + registry.registerFactory( + []() { return std::make_unique(); }); + } + + registry.onKinetoInit(); + + // Verify the factory was forwarded to Kineto + EXPECT_TRUE(registry.hasFactory()); + EXPECT_TRUE(registry.isRegisteredWithKineto()); +} + +// End-to-end test: Start a profiling session and verify mock profiler is used +TEST(PrivateUse1ProfilerTest, EndToEndProfiling) { + using namespace torch::autograd::profiler; + using namespace torch::profiler::impl; + + // Reset the flag before the test + g_configure_called = false; + + auto& registry = PrivateUse1ProfilerRegistry::instance(); + if (!registry.hasFactory()) { + registry.registerFactory( + []() { return std::make_unique(); }); + } + + // Create profiler config with KINETO_PRIVATEUSE1 state + ProfilerConfig config( + ProfilerState::KINETO_PRIVATEUSE1, + /*report_input_shapes=*/false, + /*profile_memory=*/false, + /*with_stack=*/false, + /*with_flops=*/false, + /*with_modules=*/false); + + std::set activities{ActivityType::CPU}; + + // This triggers Kineto init and onKinetoInit + prepareProfiler(config, activities); + enableProfiler(config, activities, {at::RecordScope::FUNCTION}); + + EXPECT_TRUE(registry.isRegisteredWithKineto()); + + // Stop profiler + auto result = disableProfiler(); + + EXPECT_TRUE(registry.isRegisteredWithKineto()); + + // Verify Kineto-side: Kineto actually called our mock's configure() + EXPECT_TRUE(g_configure_called); +} + +#endif // USE_KINETO diff --git a/western_ai/usa/pytorch/test/cpp/profiler/test_profiler_collection.cpp b/western_ai/usa/pytorch/test/cpp/profiler/test_profiler_collection.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3591fce2d28a67528f81fca5f71df94b02010e8d --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/profiler/test_profiler_collection.cpp @@ -0,0 +1,266 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Tests for torch/csrc/profiler/collection.cpp. The PrivateUse1 profiler +// machinery is used here only as a vehicle for injecting synthetic kineto +// activities into a real profiling session. +// +// To add new tests in this file: +// - Register child profilers via libkineto::api().registerProfilerFactory() +// directly. PrivateUse1ProfilerRegistry::registerFactory() is one-shot +// (it forwards to libkineto on first call only), so it cannot layer +// additional mocks on top of whatever a prior test already installed. +// - Wrap assertions on TORCH_WARN_ONCE output in c10::WarningUtils:: +// WarnAlways(true) to defeat per-process rate limiting. +// - Capture warnings with CapturingWarningHandler under a +// WarningHandlerGuard for isolated capture. +// - Assert what your test PRODUCES, not what is ABSENT. Activities from +// prior tests' child profilers will still flow through setParents. +// +// If a future test genuinely needs a pristine Kineto state (e.g., asserts a +// negative, or relies on TORCH_WARN_ONCE rate limiting being in effect), +// split it out in .ci/pytorch/test.sh with a -k filter — same pattern that +// isolates EndToEndProfiling in test_privateuse1_profiler. + +#ifdef USE_KINETO + +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +// Mock session that emits two CUDA_RUNTIME activities sharing the same +// async-CPU-GPU flow start ID. Drives the duplicate-flow path in +// TransferEvents::setParents(). +class DuplicateFlowMockSession : public libkineto::IActivityProfilerSession { + public: + DuplicateFlowMockSession() { + span_ = libkineto::TraceSpan{0, 0, "duplicate_flow_span"}; + for (int i = 0; i < 2; ++i) { + libkineto::GenericTraceActivity a( + span_, + libkineto::ActivityType::CUDA_RUNTIME, + "duplicate_flow_launch"); + a.startTime = 100 + i; + a.endTime = 200 + i; + a.id = i; // distinct correlation IDs + a.flow.id = 42; // identical flow ID across both activities + a.flow.type = libkineto::kLinkAsyncCpuGpu; + a.flow.start = 1; // both are flow starts + activities_.push_back(std::move(a)); + } + } + + void start() override { + status_ = libkineto::TraceStatus::RECORDING; + } + void stop() override { + status_ = libkineto::TraceStatus::PROCESSING; + } + std::vector errors() override { + return {}; + } + + // Push the duplicate-flow activities into the logger so they reach the + // MemoryTraceLogger's activities_ list, which is what TransferEvents + // consumes via ActivityTrace::activities(). + void processTrace(libkineto::ActivityLogger& logger) override { + for (const auto& a : activities_) { + logger.handleGenericActivity(a); + } + } + + std::unique_ptr getDeviceInfo() override { + return std::make_unique( + /*id=*/1, + /*sortIndex=*/1, + /*name=*/"DuplicateFlowMockDevice", + /*label=*/"duplicate_flow_mock_device"); + } + + std::vector getResourceInfos() override { + return {}; + } + + std::unique_ptr getTraceBuffer() override { + return nullptr; + } + + private: + libkineto::TraceSpan span_{0, 0, ""}; + std::vector activities_; +}; + +class DuplicateFlowMockProfiler : public libkineto::IActivityProfiler { + public: + const std::string& name() const override { + return name_; + } + + const std::set& availableActivities() + const override { + return activities_; + } + + std::unique_ptr configure( + [[maybe_unused]] const std::set& activity_types, + [[maybe_unused]] const libkineto::Config& config) override { + return std::make_unique(); + } + + std::unique_ptr configure( + [[maybe_unused]] int64_t ts_ms, + [[maybe_unused]] int64_t duration_ms, + const std::set& activity_types, + const libkineto::Config& config) override { + return configure(activity_types, config); + } + + private: + std::string name_{"duplicate_flow_mock"}; + std::set activities_{ + libkineto::ActivityType::CUDA_RUNTIME}; +}; + +// Minimal IActivityProfiler used to ensure PrivateUse1ProfilerRegistry has +// a factory so libkineto initializes. Emits no activities. +class NoopPrivateUse1Profiler : public libkineto::IActivityProfiler { + public: + const std::string& name() const override { + return name_; + } + const std::set& availableActivities() + const override { + return activities_; + } + std::unique_ptr configure( + [[maybe_unused]] const std::set& activity_types, + [[maybe_unused]] const libkineto::Config& config) override { + class Session : public libkineto::IActivityProfilerSession { + public: + void start() override { + status_ = libkineto::TraceStatus::RECORDING; + } + void stop() override { + status_ = libkineto::TraceStatus::PROCESSING; + } + std::vector errors() override { + return {}; + } + void processTrace( + [[maybe_unused]] libkineto::ActivityLogger& logger) override {} + std::unique_ptr getDeviceInfo() override { + return nullptr; + } + std::vector getResourceInfos() override { + return {}; + } + std::unique_ptr getTraceBuffer() override { + return nullptr; + } + }; + return std::make_unique(); + } + std::unique_ptr configure( + [[maybe_unused]] int64_t ts_ms, + [[maybe_unused]] int64_t duration_ms, + const std::set& activity_types, + const libkineto::Config& config) override { + return configure(activity_types, config); + } + + private: + std::string name_{"noop_privateuse1"}; + std::set activities_{ + libkineto::ActivityType::CPU_OP}; +}; + +class CapturingWarningHandler : public c10::WarningHandler { + public: + void process(const c10::Warning& warning) override { + messages_.push_back(warning.msg()); + } + const std::vector& messages() const { + return messages_; + } + + private: + std::vector messages_; +}; + +} // namespace + +// A backend producing duplicate async-CPU-GPU flow start IDs must be handled +// gracefully. +TEST(ProfilerCollectionTest, DuplicateFlowIdHandledGracefully) { + using namespace torch::autograd::profiler; + using namespace torch::profiler::impl; + + // Capture warnings so the new diagnostic can be asserted. TORCH_WARN_ONCE + // is rate-limited per process; force every WARN_ONCE to behave like WARN + // for the duration of this test. + CapturingWarningHandler handler; + c10::WarningUtils::WarningHandlerGuard handler_guard(&handler); + c10::WarningUtils::WarnAlways warn_always_guard(true); + + // Register a child profiler directly with libkineto. The session it + // returns is what injects duplicate flow IDs into the trace. + libkineto::api().registerProfilerFactory( + []() { return std::make_unique(); }); + + // The PrivateUse1 registry needs a factory so prepareProfiler triggers + // libkineto init, which in turn materializes the factory registered above. + if (!PrivateUse1ProfilerRegistry::instance().hasFactory()) { + PrivateUse1ProfilerRegistry::instance().registerFactory( + []() { return std::make_unique(); }); + } + + ProfilerConfig config( + ProfilerState::KINETO_PRIVATEUSE1, + /*report_input_shapes=*/false, + /*profile_memory=*/false, + /*with_stack=*/false, + /*with_flops=*/false, + /*with_modules=*/false); + + std::set activities{ActivityType::CPU}; + + prepareProfiler(config, activities); + enableProfiler(config, activities, {at::RecordScope::FUNCTION}); + auto result = disableProfiler(); + EXPECT_NE(result, nullptr); + + bool saw_per_id_warning = false; + bool saw_summary_warning = false; + for (const auto& msg : handler.messages()) { + if (msg.find("Profiler produced duplicate flow start") != + std::string::npos) { + saw_per_id_warning = true; + } + if (msg.find("duplicate flow start ID") != std::string::npos && + msg.find("Profiler observed") != std::string::npos) { + saw_summary_warning = true; + } + } + EXPECT_TRUE(saw_per_id_warning); + EXPECT_TRUE(saw_summary_warning); +} + +#endif // USE_KINETO diff --git a/western_ai/usa/pytorch/test/cpp/rpc/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/rpc/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..1842d17edde6427c9eeb58a5baefa7392dd35530 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/rpc/CMakeLists.txt @@ -0,0 +1,46 @@ +set(TORCH_RPC_TEST_DIR "${TORCH_ROOT}/test/cpp/rpc") +set(TORCH_RPC_TEST_SOURCES + ${TORCH_ROOT}/test/cpp/common/main.cpp + ${TORCH_RPC_TEST_DIR}/e2e_test_base.cpp + ${TORCH_RPC_TEST_DIR}/test_wire_serialization.cpp +) +set(TORCH_RPC_TEST_DEPENDENCY_LIBS + torch gtest_main +) + +if(USE_GLOO) + list(APPEND TORCH_RPC_TEST_SOURCES + ${TORCH_RPC_TEST_DIR}/test_e2e_tensorpipe.cpp + ) +endif() + +if(USE_TENSORPIPE) + list(APPEND TORCH_RPC_TEST_SOURCES + ${TORCH_RPC_TEST_DIR}/test_tensorpipe_serialization.cpp + ) + list(APPEND TORCH_RPC_TEST_DEPENDENCY_LIBS + tensorpipe + ) +endif() + +add_executable(test_cpp_rpc ${TORCH_RPC_TEST_SOURCES}) +target_include_directories( + test_cpp_rpc PRIVATE + ${ATen_CPU_INCLUDE}) +target_include_directories( + test_cpp_rpc PRIVATE + $) +target_link_libraries(test_cpp_rpc PRIVATE ${TORCH_RPC_TEST_DEPENDENCY_LIBS}) + +if(USE_CUDA) + target_compile_definitions(test_cpp_rpc PRIVATE "USE_CUDA") +endif() + +if(INSTALL_TEST) + set_target_properties(test_cpp_rpc PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${_rpath_portable_origin}/../lib") + install(TARGETS test_cpp_rpc DESTINATION bin) + # Install PDB files for MSVC builds + if(MSVC AND BUILD_SHARED_LIBS) + install(FILES $ DESTINATION bin OPTIONAL) + endif() +endif() diff --git a/western_ai/usa/pytorch/test/cpp/rpc/e2e_test_base.cpp b/western_ai/usa/pytorch/test/cpp/rpc/e2e_test_base.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b7bb59e2898935aa1347fa33df115849c9f636cd --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/rpc/e2e_test_base.cpp @@ -0,0 +1,19 @@ +#include "e2e_test_base.h" + +namespace torch { +namespace distributed { +namespace rpc { + +DistAutogradContainer* getDistAutogradContainer() { + static DistAutogradContainer* autogradContainer = + &DistAutogradContainer::init(0); + return autogradContainer; +} + +const char* TestE2EBase::serverAddress = "127.0.0.1"; +const size_t TestE2EBase::numIters = 100; +const size_t TestE2EBase::numWorkers = 1; + +} // namespace rpc +} // namespace distributed +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/rpc/e2e_test_base.h b/western_ai/usa/pytorch/test/cpp/rpc/e2e_test_base.h new file mode 100644 index 0000000000000000000000000000000000000000..1afc905bd3bb671d5ad3b396297ebc6a037fbbc3 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/rpc/e2e_test_base.h @@ -0,0 +1,172 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace torch { +namespace distributed { +namespace rpc { + +using torch::distributed::autograd::DistAutogradContainer; +using torch::distributed::autograd::DistAutogradContext; + +DistAutogradContainer* getDistAutogradContainer(); + +class TestE2EBase : public ::testing::Test { + protected: + void SetUp() override { + // Setup distributed autograd. + autogradContainer = getDistAutogradContainer(); + + // Setup server store. + c10d::TCPStoreOptions opts{ + /* port */ 0, + /* isServer */ true, + numWorkers, + /* waitWorkers */ true, + /* timeout */ std::chrono::seconds(10)}; + + store = c10::make_intrusive(serverAddress, opts); + + buildRpcAgent(); + + rpcAgentPostProcessing(); + } + + void rpcAgentPostProcessing() { + RpcAgent::setCurrentRpcAgent(rpcAgent); + std::shared_ptr typeResolver = + std::make_shared([&](const c10::QualifiedName& qn) { + // For Dict that is used for device map. + auto pos = qn.name().find("Dict"); + if (pos != std::string::npos) { + return c10::StrongTypePtr( + nullptr, + c10::DictType::create( + c10::StringType::get(), c10::StringType::get())); + } + return c10::StrongTypePtr( + nullptr, c10::TensorType::create(at::Tensor())); + }); + rpcAgent->setTypeResolver(typeResolver); + rpcAgent->start(); + } + + void TearDown() override { + rpcAgent->join(); + rpcAgent->shutdown(); + RpcAgent::setCurrentRpcAgent(nullptr); + } + + c10::intrusive_ptr createRemoteRRef( + at::Tensor t1, + at::Tensor t2, + std::shared_ptr op) { + auto& ctx = RRefContext::getInstance(); + auto ownerRRef = ctx.createOwnerRRef(c10::TensorType::create(t1)); + // prevent this owner RRef being deleted due to other forks + ctx.addSelfAsFork(ownerRRef); + + ScriptRemoteCall scriptRemoteCall( + op, {t1, t2, 1}, ownerRRef->rrefId(), ownerRRef->rrefId()); + auto jitFuture = autograd::sendMessageWithAutograd( + *rpcAgent, + rpcAgent->getWorkerInfo("worker"), + std::move(scriptRemoteCall).toMessage(), + false); + + ownerRRef->registerOwnerCreationFuture(jitFuture); + + // Builtin operators does not return py::object, and hence does not require + // GIL for destructing the potentially deleted OwerRRef. + jitFuture->addCallback( + [ownerRRefId = ownerRRef->rrefId()](JitFuture& jitFuture) { + callback::finishCreatingOwnerRRef(jitFuture, ownerRRefId); + }); + return ownerRRef; + } + + at::Tensor remoteAdd( + at::Tensor t1, + at::Tensor t2, + std::shared_ptr op) { + ScriptCall scriptCall(op, {t1, t2, /* alpha */ 1}); + + // Send the RPC and return result. + auto response = autograd::sendMessageWithAutograd( + *rpcAgent, + rpcAgent->getWorkerInfo("worker"), + std::move(scriptCall).toMessage()); + response->waitAndThrow(); + + MessageType messageType = MessageType::FORWARD_AUTOGRAD_RESP; + auto wrappedResponse = deserializeResponse( + std::move(*response->value().toCustomClass()), messageType); + return static_cast(*wrappedResponse).value().toTensor(); + } + + virtual void buildRpcAgent() = 0; + + class AutogradContextGuard { + public: + explicit AutogradContextGuard() + : context(DistAutogradContainer::getInstance().newContext()) {} + + ~AutogradContextGuard() { + DistAutogradContainer::getInstance().releaseContext(context->contextId()); + } + + private: + std::shared_ptr context; + }; + + void runTrainingLoop() { + auto options = at::TensorOptions().requires_grad(true); + auto t1 = torch::ones({3, 3}, options); + auto t2 = torch::ones({3, 3}, options); + + c10::OperatorName full_name("aten::add", "Tensor"); + auto matchedOp = torch::jit::findOperatorFor(full_name); + ASSERT_TRUE(matchedOp); + + for (size_t i = 0; i < numIters; i++) { + // Create the autograd context guard. + AutogradContextGuard guard; + + // Multiple RPCs within one autograd context for the forward pass. + auto result = remoteAdd(t1, t2, matchedOp); + for (size_t j = 0; j < 5; j++) { + result = remoteAdd(t1, result, matchedOp); + } + + auto rref = createRemoteRRef(t1, result, matchedOp); + result = rref->getValue().toTensor(); + + // Run backward pass now. + autograd::DistEngine::getInstance().execute( + DistAutogradContainer::currentContextId(), + {torch::sum(result)}, + /* retainGraph */ false); + } + } + + DistAutogradContainer* autogradContainer; + std::shared_ptr rpcAgent; + static const size_t numIters; + static const size_t numWorkers; + c10::intrusive_ptr store; + static const char* serverAddress; +}; + +} // namespace rpc +} // namespace distributed +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/rpc/test_e2e_tensorpipe.cpp b/western_ai/usa/pytorch/test/cpp/rpc/test_e2e_tensorpipe.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e92b56bfa267558bc4538e33c68d573b2ce127af --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/rpc/test_e2e_tensorpipe.cpp @@ -0,0 +1,63 @@ +#include + +#include "e2e_test_base.h" + +#include +#include +#include +#include + +namespace torch { +namespace distributed { +namespace rpc { + +#ifdef USE_TENSORPIPE + +class TestE2ETensorPipe : public TestE2EBase { + protected: + void buildRpcAgent() override { + auto options = c10d::ProcessGroupGloo::Options::create(); + options->devices.push_back( + ::c10d::ProcessGroupGloo::createDeviceForHostname(serverAddress)); + float rpcTimeout = 30; + + TensorPipeRpcBackendOptions opts( + /*numWorkerThreads=*/std::max(16U, std::thread::hardware_concurrency()), + /*transports=*/std::nullopt, + /*channels=*/std::nullopt, + /*rpc_timeout=*/rpcTimeout, + /*init_method=*/"unused"); + + rpcAgent = std::make_shared( + store, + "worker", + 0, + numWorkers, + opts, + std::unordered_map{}, + std::vector{}, + std::make_unique()); + } +}; + +// End to end training loop test in C++ so that we can run LSAN on this test to +// catch memory leaks. Enabling LSAN with python multiprocessing has been +// challenging and we don't have a good solution yet. +TEST_F(TestE2ETensorPipe, TestTrainingLoop) { + runTrainingLoop(); + // Ensure the tensorpipe internal state is cleared up. + auto tensorpipeAgent = std::static_pointer_cast(rpcAgent); + + // Shutdown RPC agent for all RPCs to clean up. + tensorpipeAgent->join(); + tensorpipeAgent->shutdown(); + ASSERT_EQ(0, tensorpipeAgent->numPendingResponses()); + ASSERT_EQ(0, tensorpipeAgent->timeoutMapSize()); + ASSERT_EQ(0, tensorpipeAgent->messageIdToTimeoutMapSize()); +} + +#endif + +} // namespace rpc +} // namespace distributed +} // namespace torch diff --git a/western_ai/usa/pytorch/test/cpp/rpc/test_tensorpipe_serialization.cpp b/western_ai/usa/pytorch/test/cpp/rpc/test_tensorpipe_serialization.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2322651b6516af811c5180219b9b9e6b03d3f44b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/rpc/test_tensorpipe_serialization.cpp @@ -0,0 +1,173 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +TEST(TensorpipeSerialize, Base) { + // Sender serializes + at::Tensor t1 = torch::ones({1024}, at::ScalarType::Int); + at::Tensor t2 = torch::ones({1024}, at::ScalarType::Float); + std::vector tensors{t1, t2}; + std::vector payload = {'1', '2', '3'}; + std::vector payloadCopy = payload; // for testing + torch::distributed::rpc::MessageType mtype = + torch::distributed::rpc::MessageType::UNKNOWN; + int64_t mId = 100; + auto sendingRpcMessage = + c10::make_intrusive( + std::move(payload), std::move(tensors), mtype); + sendingRpcMessage->setId(mId); + auto [sendingTpMessage, sendingTpBuffers] = + torch::distributed::rpc::tensorpipeSerialize( + std::move(sendingRpcMessage), {}, {}); + + // Mimic receiving message descriptor: recvingTpDescriptor is a copy of + // sendingTpMessage except for the data pointers which are left null. + tensorpipe::Descriptor recvingTpDescriptor; + recvingTpDescriptor.metadata = sendingTpMessage.metadata; + recvingTpDescriptor.payloads.reserve(sendingTpMessage.payloads.size()); + for (auto& tpPayload : sendingTpMessage.payloads) { + tensorpipe::Descriptor::Payload p; + p.length = tpPayload.length; + p.metadata = tpPayload.metadata; + recvingTpDescriptor.payloads.push_back(std::move(p)); + } + EXPECT_EQ( + recvingTpDescriptor.payloads.size(), sendingTpMessage.payloads.size()); + recvingTpDescriptor.tensors.reserve(sendingTpMessage.tensors.size()); + for (auto& tpTensor : sendingTpMessage.tensors) { + tensorpipe::Descriptor::Tensor t; + t.length = tpTensor.length; + t.sourceDevice = tpTensor.buffer.device(); + t.targetDevice = tpTensor.targetDevice; + t.metadata = tpTensor.metadata; + recvingTpDescriptor.tensors.push_back(std::move(t)); + } + EXPECT_EQ( + recvingTpDescriptor.tensors.size(), sendingTpMessage.tensors.size()); + + // Mimic readDescriptor() callback: + // - Allocate buffers + // - Fill pointers in tensorpipe message + auto [recvingTpAllocation, recvingTpBuffers] = + torch::distributed::rpc::tensorpipeAllocate(recvingTpDescriptor, {}); + + // Mimic tensorpipe data transfer + EXPECT_EQ( + recvingTpAllocation.payloads.size(), sendingTpMessage.payloads.size()); + for (const auto i : c10::irange(recvingTpAllocation.payloads.size())) { + tensorpipe::Message::Payload& srcPayload = sendingTpMessage.payloads[i]; + tensorpipe::Allocation::Payload& dstPayload = + recvingTpAllocation.payloads[i]; + if (srcPayload.length) { + // Empty vector's data() can return nullptr, use the length to avoid + // copying into nullptr + memcpy(dstPayload.data, srcPayload.data, srcPayload.length); + } + } + EXPECT_EQ( + recvingTpAllocation.tensors.size(), sendingTpMessage.tensors.size()); + for (const auto i : c10::irange(recvingTpAllocation.tensors.size())) { + tensorpipe::Message::Tensor& srcTensor = sendingTpMessage.tensors[i]; + tensorpipe::Allocation::Tensor& dstTensor = recvingTpAllocation.tensors[i]; + memcpy( + dstTensor.buffer.unwrap().ptr, + srcTensor.buffer.unwrap().ptr, + srcTensor.length); + } + + // Mimic read() callback: + // - Unpickle + c10::intrusive_ptr recvingRpcMessage = + torch::distributed::rpc::tensorpipeDeserialize( + std::move(recvingTpDescriptor), std::move(recvingTpBuffers)); + + // Data is ready + EXPECT_EQ(mtype, recvingRpcMessage->type()); + EXPECT_EQ(payloadCopy, recvingRpcMessage->payload()); + EXPECT_EQ(mId, recvingRpcMessage->id()); + EXPECT_TRUE(torch::equal(t1, recvingRpcMessage->tensors()[0])); + EXPECT_TRUE(torch::equal(t2, recvingRpcMessage->tensors()[1])); +} + +TEST(TensorpipeSerialize, RecopySparseTensors) { + // Take a 1K row of a 1M tensors, and make sure we don't send across 1M rows. + constexpr size_t k1K = 1024; + at::Tensor main = torch::randn({k1K, k1K}); + at::Tensor tiny = main.select(0, 2); // Select a row in the middle + EXPECT_EQ(tiny.numel(), k1K); + EXPECT_EQ(tiny.storage().nbytes() / tiny.itemsize(), k1K * k1K); + + std::vector tensors{main, tiny}; + std::vector payload = {'1', '2', '3'}; + torch::distributed::rpc::MessageType mtype = + torch::distributed::rpc::MessageType::UNKNOWN; + auto sendingRpcMessage = + c10::make_intrusive( + std::move(payload), std::move(tensors), mtype); + + auto [sendingTpMessage, tpBuffers] = + torch::distributed::rpc::tensorpipeSerialize( + std::move(sendingRpcMessage), {}, {}); + + EXPECT_EQ(tpBuffers.tensors.size(), 2); + EXPECT_EQ(sendingTpMessage.tensors.size(), 2); + EXPECT_TRUE(torch::equal(main, tpBuffers.tensors[0])); + EXPECT_TRUE(torch::equal(tiny, tpBuffers.tensors[1])); + // Test cloned storage + EXPECT_EQ( + main.storage().data(), + sendingTpMessage.tensors[0].buffer.unwrap().ptr); + EXPECT_NE( + tiny.storage().data(), + sendingTpMessage.tensors[1].buffer.unwrap().ptr); + EXPECT_EQ(tiny.element_size() * k1K, sendingTpMessage.tensors[1].length); +} + +TEST(TensorpipeSerialize, NoDeleterTensors) { + std::vector blob1{.8, .2}; + std::vector blob2{.7, .5, .9}; + at::Tensor t1 = torch::from_blob((float*)(blob1.data()), blob1.size()); + at::Tensor t2 = torch::from_blob((float*)(blob2.data()), blob2.size()); + std::vector tensors{t1, t2}; + std::vector payload = {'1', '2', '3'}; + torch::distributed::rpc::MessageType mtype = + torch::distributed::rpc::MessageType::UNKNOWN; + auto sendingRpcMessage = + c10::make_intrusive( + std::move(payload), std::move(tensors), mtype); + + auto [sendingTpMessage, tpBuffers] = + torch::distributed::rpc::tensorpipeSerialize( + std::move(sendingRpcMessage), {}, {}); + + EXPECT_EQ(tpBuffers.copiedTensors.size(), 2); + EXPECT_EQ(sendingTpMessage.tensors.size(), 2); + EXPECT_EQ( + tpBuffers.copiedTensors[0].size(), sendingTpMessage.tensors[0].length); + EXPECT_EQ( + tpBuffers.copiedTensors[1].size(), sendingTpMessage.tensors[1].length); + EXPECT_EQ( + tpBuffers.copiedTensors[0].data(), + sendingTpMessage.tensors[0].buffer.unwrap().ptr); + EXPECT_EQ( + tpBuffers.copiedTensors[1].data(), + sendingTpMessage.tensors[1].buffer.unwrap().ptr); + EXPECT_TRUE( + memcmp( + tpBuffers.copiedTensors[0].data(), + t1.storage().data(), + sendingTpMessage.tensors[0].length) == 0); + EXPECT_TRUE( + memcmp( + tpBuffers.copiedTensors[1].data(), + t2.storage().data(), + sendingTpMessage.tensors[1].length) == 0); +} diff --git a/western_ai/usa/pytorch/test/cpp/rpc/test_wire_serialization.cpp b/western_ai/usa/pytorch/test/cpp/rpc/test_wire_serialization.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3f4888d9291f2fd37ce42e2deb243c33cf028034 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/rpc/test_wire_serialization.cpp @@ -0,0 +1,105 @@ +#include + +#include +#include +#include + +#include +#include +#include + +using ::testing::IsSubstring; + +TEST(WireSerialize, Base) { + auto run = [](const std::string& payload, + const std::vector& tensors) { + std::string serialized; + { + std::vector mpayload(payload.begin(), payload.end()); + std::vector mtensors = tensors; + serialized = torch::distributed::rpc::wireSerialize( + std::move(mpayload), std::move(mtensors)); + } + auto deser = torch::distributed::rpc::wireDeserialize( + serialized.data(), serialized.size()); + EXPECT_EQ(payload.size(), deser.first.size()); + EXPECT_EQ(tensors.size(), deser.second.size()); + if (payload.size() > 0) { + EXPECT_TRUE( + memcmp(deser.first.data(), payload.data(), payload.size()) == 0); + } + for (const auto i : c10::irange(tensors.size())) { + EXPECT_TRUE(torch::equal(tensors[i], deser.second[i])); + } + }; + run("", {}); + run("hi", {}); + run("", {torch::randn({5, 5})}); + run("hi", {torch::randn({5, 5})}); + run("more", {torch::randn({5, 5}), torch::rand({10, 10})}); +} + +TEST(WireSerialize, RecopySparseTensors) { + // Take a 1K row of a 1M tensors, and make sure we don't send across 1M rows. + constexpr size_t k1K = 1024; + at::Tensor main = torch::randn({k1K, k1K}); + at::Tensor tiny = main.select(0, 2); // Select a row in the middle + EXPECT_EQ(tiny.numel(), k1K); + EXPECT_EQ(tiny.storage().nbytes() / tiny.dtype().itemsize(), k1K * k1K); + auto ser = torch::distributed::rpc::wireSerialize({}, {tiny}); + auto deser = torch::distributed::rpc::wireDeserialize(ser.data(), ser.size()); + EXPECT_TRUE(torch::equal(tiny, deser.second[0])); + EXPECT_LT(ser.size(), (tiny.element_size() * k1K) + k1K); +} + +TEST(WireSerialize, CloneSparseTensors) { + constexpr size_t k1K = 1024; + at::Tensor big = torch::randn({k1K, k1K}); + auto v1 = torch::distributed::rpc::cloneSparseTensors({big}); + EXPECT_EQ(v1.get(0).storage(), big.storage()); // Not cloned + + at::Tensor tiny = big.select(0, 2); // Select a row in the middle + auto v2 = torch::distributed::rpc::cloneSparseTensors({tiny}); + EXPECT_NE(&v2.get(0).storage(), &tiny.storage()); // Cloned. + EXPECT_TRUE(torch::equal(v2.get(0), tiny)); + + at::Tensor sparse = at::empty({2, 3}, at::dtype().layout(at::kSparse)); + auto v3 = torch::distributed::rpc::cloneSparseTensors({sparse}); + // There is no storage() to compare, but at least confirm equality. + EXPECT_TRUE(v3.get(0).is_same(sparse)); +} + +TEST(WireSerialize, Errors) { + auto checkMessage = [](auto&& f, const char* msg) { + try { + f(); + FAIL(); + } catch (const std::exception& e) { + EXPECT_PRED_FORMAT2(IsSubstring, msg, e.what()); + } catch (...) { + FAIL(); + } + }; + checkMessage( + []() { (void)torch::distributed::rpc::wireDeserialize("", 0); }, + "failed parse"); + checkMessage( + []() { (void)torch::distributed::rpc::wireDeserialize(" ", 1); }, + "failed parse"); + auto serialized = + torch::distributed::rpc::wireSerialize({}, {torch::randn({5, 5})}); + checkMessage( + [&]() { + (void)torch::distributed::rpc::wireDeserialize( + serialized.data(), serialized.size() / 2); + }, + "failed bounds"); +} + +// Enable this once JIT Pickler supports sparse tensors. +TEST(WireSerialize, DISABLED_Sparse) { + at::Tensor main = at::empty({2, 3}, at::dtype().layout(at::kSparse)); + auto ser = torch::distributed::rpc::wireSerialize({}, {main.to(at::kSparse)}); + auto deser = torch::distributed::rpc::wireDeserialize(ser.data(), ser.size()); + EXPECT_TRUE(torch::equal(main, deser.second[0])); +} diff --git a/western_ai/usa/pytorch/test/cpp/shim/CMakeLists.txt b/western_ai/usa/pytorch/test/cpp/shim/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..802ecfb9146fd293dd239c5104ea10233219bd7e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/shim/CMakeLists.txt @@ -0,0 +1,31 @@ +# Skip on windows +if(WIN32) + return() +endif() + +set(TORCH_SHIM_TEST_ROOT ${TORCH_ROOT}/test/cpp/shim) + +# Build the cpp gtest binary containing the cpp-only tests. +set(TORCH_SHIM_TEST_SRCS + ${TORCH_SHIM_TEST_ROOT}/main.cpp + ${TORCH_SHIM_TEST_ROOT}/test_stable_ivalue.cpp +) + +add_executable(test_shim + ${TORCH_SHIM_TEST_SRCS} +) + +# TODO temporary until we can delete the old gtest polyfills. +target_compile_definitions(test_shim PRIVATE USE_GTEST) + + +target_link_libraries(test_shim PRIVATE gtest_main torch) +target_include_directories(test_shim PRIVATE ${ATen_CPU_INCLUDE}) + +if(INSTALL_TEST) + install(TARGETS test_shim DESTINATION bin) + # Install PDB files for MSVC builds + if(MSVC AND BUILD_SHARED_LIBS) + install(FILES $ DESTINATION bin OPTIONAL) + endif() +endif() diff --git a/western_ai/usa/pytorch/test/cpp/shim/README.md b/western_ai/usa/pytorch/test/cpp/shim/README.md new file mode 100644 index 0000000000000000000000000000000000000000..655dc99ab28f500609caaf1c60aa635fa5c5bb7e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/shim/README.md @@ -0,0 +1,13 @@ +# Shim Tests + +All shim functions expected to be used by a custom extension library should be tested end to end in +`test/cpp_extensions/test_libtorch_agnostic.py`. +Some C shims are not so directly called, for example C shims that are meant for lower level usage (e.g., binding to rust). +This directory is intended for tests that require finer grained testing than what can be provided with the end to end tests. + +Whenever making changes, verify valgrind runs clean against the tests with: +``` +./test/cpp/shim/run_test.sh ./build/bin/ +``` + +Tests relying only on the headeronly parts should go into `aoti_abi_check` instead. diff --git a/western_ai/usa/pytorch/test/cpp/shim/main.cpp b/western_ai/usa/pytorch/test/cpp/shim/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..04a10046efe488caf859d8228ef833becdd796b2 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/shim/main.cpp @@ -0,0 +1,6 @@ +#include + +int main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/western_ai/usa/pytorch/test/cpp/shim/run_test.sh b/western_ai/usa/pytorch/test/cpp/shim/run_test.sh new file mode 100644 index 0000000000000000000000000000000000000000..7842317797432920405a7ca87bd110bdeee552ff --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/shim/run_test.sh @@ -0,0 +1,14 @@ +#!/bin/bash -xe +# +# Default to on, unless set to off by the calling script. +VALGRIND=${VALGRIND:=ON} +export CPP_TESTS_DIR=$1 +if [ "$VALGRIND" == "ON" ]; then + + # Use the aten/tools/valgrind.sup valgrind suppression file. + SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + SUPPRESSION_FILE="${SCRIPT_DIR}/../../../aten/tools/valgrind.sup" + + # --leak-check=full is necessary to return 1 in case of definite and possible leaks. + valgrind --suppressions="${SUPPRESSION_FILE}" --error-exitcode=1 --leak-check=full ${CPP_TESTS_DIR}/test_shim +fi diff --git a/western_ai/usa/pytorch/test/cpp/shim/test_stable_ivalue.cpp b/western_ai/usa/pytorch/test/cpp/shim/test_stable_ivalue.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6c43ec6e6c163c5fbb5aea20036907ff340cb9a9 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp/shim/test_stable_ivalue.cpp @@ -0,0 +1,44 @@ +// Checks the behaviour of: +// torch_new_stable_ivalue +// torch_delete_stable_ivalue + +#include +#include +#include + +TEST(TorchStableIValue, TestStableIValueUse) { + StableIValue* a = nullptr; + ASSERT_EQ(torch_new_stable_ivalue(&a), AOTI_TORCH_SUCCESS); + // Check if it is now a valid pointer. + ASSERT_NE(a, nullptr); + // Assign to verify that we can actually write to the full u64 size, this will + // show up in valgrind if we were to allocate less than the required size. + *a = std::numeric_limits::max(); + + // Free the value again. + EXPECT_EQ(torch_delete_stable_ivalue(a), AOTI_TORCH_SUCCESS); + + // Freeing a nullptr should result in a failure. + StableIValue* b = nullptr; + ASSERT_EQ(torch_delete_stable_ivalue(b), AOTI_TORCH_FAILURE); + + // Trying to allocate with a pointer into which can't be assigned is a + // failure. + StableIValue** ret_value = nullptr; + ASSERT_EQ(torch_new_stable_ivalue(ret_value), AOTI_TORCH_FAILURE); +} + +TEST(TorchStableIValue, TestStableIValueLargeAmount) { + // Next, do a bunch of allocations and frees, such that if the delete isn't + // working but the allocation is we see a clear memory leak in valgrind. + StableIValue* v = nullptr; + const size_t total_allocation_bytes = 10'000'000; + const size_t number_of_allocations = + total_allocation_bytes / sizeof(StableIValue); + for (size_t i = 0; i < number_of_allocations; i++) { + ASSERT_EQ(torch_new_stable_ivalue(&v), AOTI_TORCH_SUCCESS); + ASSERT_NE(v, nullptr); + ASSERT_EQ(torch_delete_stable_ivalue(v), AOTI_TORCH_SUCCESS); + v = nullptr; + } +} diff --git a/western_ai/usa/pytorch/test/cpp_api_parity/__init__.py b/western_ai/usa/pytorch/test/cpp_api_parity/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/test/cpp_api_parity/functional_impl_check.py b/western_ai/usa/pytorch/test/cpp_api_parity/functional_impl_check.py new file mode 100644 index 0000000000000000000000000000000000000000..cb2db7a78f2e1eb791c098efc6ba41ed7181dbec --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_api_parity/functional_impl_check.py @@ -0,0 +1,326 @@ +# The purpose of this test is to check that we have implementation parity between +# a Python `torch.nn.functional` function and its corresponding C++ `torch::nn::functional` +# function. Concretely, this test does the following: +# +# 1. Get a test params dict from common_nn.py, run forward pass on the Python functional +# created using the test params. +# +# 2. Serialize the Python functional's forward input arguments, deserialize them +# in C++ and use them as input for the C++ functional's forward pass. +# +# 3. Run the forward pass on the C++ functional, and serialize the C++ functional's +# forward output. +# +# 4. Compare Python/C++ functional's forward output. If they are the same, then we +# have implementation parity between Python/C++ module. + +import os +import pprint +import re +import tempfile +from string import Template + +import torch +from cpp_api_parity.sample_functional import SAMPLE_FUNCTIONAL_CPP_SOURCE +from cpp_api_parity.utils import ( + add_test, + compile_cpp_code_inline, + compute_arg_dict, + compute_cpp_args_construction_stmts_and_forward_arg_symbols, + compute_temp_file_path, + decorate_test_fn, + generate_error_msg, + is_torch_nn_functional_test, + move_python_tensors_to_device, + serialize_arg_dict_as_script_module, + set_python_tensors_requires_grad, + TORCH_NN_COMMON_TEST_HARNESS, + TorchNNFunctionalTestParams, + try_remove_folder, +) + + +# Expected substitutions: +# +# ${functional_variant_name} (e.g. `BCELoss_no_reduce`) +# ${cpp_args_construction_stmts} +# ${cpp_function_call} +TORCH_NN_FUNCTIONAL_TEST_FORWARD = Template( + """ +void ${functional_variant_name}_test_forward( + const std::string& arg_dict_file_path, + const std::string& forward_output_file_path) { + pybind11::gil_scoped_release no_gil; + + namespace F = torch::nn::functional; + + // Declare arguments + auto arg_dict = load_dict_from_file(arg_dict_file_path); + ${cpp_args_construction_stmts}; + + // Some functionals (such as `F::rrelu`) create random tensors in their call path. + // To make sure the random tensors created are the same in Python/C++, we need + // to set the RNG seed manually. + torch::manual_seed(0); + + // Run function with arguments + auto cpp_output = ${cpp_function_call}; + + // Save the output into a file to be compared in Python later + write_ivalue_to_file(torch::IValue(cpp_output), forward_output_file_path); +} +""" +) + + +def run_forward(unit_test_class, test_params): + device = test_params.device + + inputs = set_python_tensors_requires_grad( + move_python_tensors_to_device( + [arg_value for _, arg_value in test_params.arg_dict["input"]], device + ) + ) + inputs += move_python_tensors_to_device( + [arg_value for _, arg_value in test_params.arg_dict["target"]], device + ) + inputs += move_python_tensors_to_device( + [arg_value for _, arg_value in test_params.arg_dict["extra_args"]], device + ) + + # Some functionals (such as `F.rrelu`) create random tensors in their call path. + # To make sure the random tensors created are the same in Python/C++, we need + # to set the RNG seed manually. + torch.manual_seed(0) + python_output = test_params.test_instance.constructor()(*inputs) + + return python_output + + +def test_forward(unit_test_class, test_params): + functional_variant_name = test_params.functional_variant_name + cpp_tmp_folder = test_params.cpp_tmp_folder + # Remove the temporary folder if it exists already + try_remove_folder(cpp_tmp_folder) + os.mkdir(cpp_tmp_folder) + + # Run forward on Python functional + python_output = run_forward(unit_test_class, test_params) + + # Save Python arguments to be used from C++ function + arg_dict_file_path = compute_temp_file_path( + cpp_tmp_folder, functional_variant_name, "arg_dict" + ) + serialize_arg_dict_as_script_module(test_params.arg_dict).save(arg_dict_file_path) + + cpp_test_name = f"{test_params.functional_variant_name}_test_forward" + cpp_test_fn = getattr( + unit_test_class.functional_impl_check_cpp_module, cpp_test_name + ) + + def run_cpp_test_fn_and_check_output(): + forward_output_file_path = compute_temp_file_path( + cpp_tmp_folder, functional_variant_name, "forward_output" + ) + + cpp_test_fn(arg_dict_file_path, forward_output_file_path) + cpp_output = torch.load(forward_output_file_path) + + # Check that forward outputs are equal + unit_test_class.assertEqual( + python_output, + cpp_output, + msg=generate_error_msg("forward output", cpp_output, python_output), + ) + + run_cpp_test_fn_and_check_output() + + # Remove temporary folder that stores C++ outputs + try_remove_folder(cpp_tmp_folder) + + +def compute_functional_name(test_params_dict): + def camel_case_to_snake_case(camel_case_str): + return re.sub(r"(?to(std::string("${device}")); + torch::load(module, module_file_path); + + // Some modules (such as `RReLU`) create random tensors in their forward pass. + // To make sure the random tensors created are the same in Python/C++, we need + // to set the RNG seed manually. + torch::manual_seed(0); + + // Forward pass + auto cpp_output = module(${cpp_forward_args_symbols}); + + // Save the output into a file to be compared in Python later + write_ivalue_to_file(torch::IValue(cpp_output), forward_output_file_path); + + // Backward pass + if (cpp_output.is_complex()) { + cpp_output.sum().abs().backward(); + } else { + cpp_output.sum().backward(); + } + + // Put all gradients into a c10::Dict, save it into a file to be compared in Python later + c10::Dict grad_dict; + for (const auto& param : module->named_parameters()) { + torch::Tensor grad = param.value().grad(); + if (grad.is_sparse()) { + grad_dict.insert(param.key() + "_grad_indices", grad.coalesce().indices()); + grad_dict.insert(param.key() + "_grad_values", grad.coalesce().values()); + } else { + grad_dict.insert(param.key() + "_grad", grad); + } + } + + write_ivalue_to_file(torch::IValue(grad_dict), backward_grad_dict_file_path); +} +""" +) + + +def run_python_forward_backward(unit_test_class, test_params): + device = test_params.device + module = test_params.test_instance.constructor( + *test_params.test_instance.constructor_args + ).to(device) + + inputs = set_python_tensors_requires_grad( + move_python_tensors_to_device( + [arg_value for _, arg_value in test_params.arg_dict["input"]], device + ) + ) + inputs += move_python_tensors_to_device( + [arg_value for _, arg_value in test_params.arg_dict["target"]], device + ) + inputs += move_python_tensors_to_device( + [arg_value for _, arg_value in test_params.arg_dict["extra_args"]], device + ) + + # Some modules (such as `RReLU`) create random tensors in their forward pass. + # To make sure the random tensors created are the same in Python/C++, we need + # to set the RNG seed manually. + torch.manual_seed(0) + + # Forward pass + python_output = module(*inputs) + + # NOTE: This is a workaround to allow any module to be traced. + # We can do this because we are only interested in transferring + # the Python module's parameters and buffers to the C++ module. + module.forward = types.MethodType(lambda self, input: input, module) + script_module = torch.jit.trace(module, torch.tensor(0)) + + # Backward pass + if python_output.dtype.is_complex: + python_output.sum().abs().backward() + else: + python_output.sum().backward() + + # Put all gradients into a dict, to be compared later + python_grad_dict = {} + for name, param in module.named_parameters(): + grad = param.grad + if grad.is_sparse: + python_grad_dict[name + "_grad_indices"] = grad.coalesce().indices() + python_grad_dict[name + "_grad_values"] = grad.coalesce().values() + else: + python_grad_dict[name + "_grad"] = grad + + return script_module, python_output, python_grad_dict + + +def test_forward_backward(unit_test_class, test_params): + module_variant_name = test_params.module_variant_name + cpp_tmp_folder = test_params.cpp_tmp_folder + # Remove the temporary folder if it exists already + try_remove_folder(cpp_tmp_folder) + os.mkdir(cpp_tmp_folder) + + # Run forward and backward on Python module + script_module, python_output, python_grad_dict = run_python_forward_backward( + unit_test_class, test_params + ) + + # Save Python module and arguments to be used from C++ function + module_file_path = compute_temp_file_path( + cpp_tmp_folder, module_variant_name, "module" + ) + arg_dict_file_path = compute_temp_file_path( + cpp_tmp_folder, module_variant_name, "arg_dict" + ) + script_module.save(module_file_path) + serialize_arg_dict_as_script_module(test_params.arg_dict).save(arg_dict_file_path) + + cpp_test_name = f"{test_params.module_variant_name}_test_forward_backward" + cpp_test_fn = getattr(unit_test_class.module_impl_check_cpp_module, cpp_test_name) + + def run_cpp_test_fn_and_check_output(): + forward_output_file_path = compute_temp_file_path( + cpp_tmp_folder, module_variant_name, "forward_output" + ) + backward_grad_dict_file_path = compute_temp_file_path( + cpp_tmp_folder, module_variant_name, "backward_grad_dict" + ) + + cpp_test_fn( + arg_dict_file_path, + module_file_path, + forward_output_file_path, + backward_grad_dict_file_path, + ) + cpp_output = torch.load(forward_output_file_path) + # weights_only: need GLOBAL torch.jit._pickle.restore_type_tag + with torch.serialization.safe_globals([restore_type_tag]): + cpp_grad_dict = torch.load(backward_grad_dict_file_path) + + # Check that forward outputs are equal + unit_test_class.assertEqual( + python_output, + cpp_output, + msg=generate_error_msg("forward output", cpp_output, python_output), + ) + + # Check that module parameter gradients are equal after backward pass + unit_test_class.assertEqual( + len(python_grad_dict), + len(cpp_grad_dict), + msg=generate_error_msg( + "# of parameters", len(cpp_grad_dict), len(python_grad_dict) + ), + ) + for key in python_grad_dict: + param_name = None + for suffix in ["_grad", "_grad_indices", "_grad_values"]: + if key.endswith(suffix): + param_name = key[: -len(suffix)] + break + if param_name is None: + raise AssertionError(f"Could not determine param_name from key '{key}'") + sparsity_str = ( + "sparse" if key.endswith(("_grad_indices", "_grad_values")) else "dense" + ) + + unit_test_class.assertTrue( + key in cpp_grad_dict, + msg=generate_error_msg( + f'"Does module have a parameter named `{param_name}` with {sparsity_str} gradient?"', + False, + True, + ), + ) + unit_test_class.assertEqual( + python_grad_dict[key], + cpp_grad_dict[key], + msg=generate_error_msg( + f"`{param_name}`'s {sparsity_str} gradient (`{key}`)", + cpp_grad_dict[key], + python_grad_dict[key], + ), + ) + + run_cpp_test_fn_and_check_output() + + # Remove temporary folder that stores C++ outputs + try_remove_folder(cpp_tmp_folder) + + +def compute_module_name(test_params_dict): + fullname = test_params_dict.get("fullname", None) + if fullname: + module_name = fullname.split("_")[0] + else: + module_name = test_params_dict.get("module_name") + return module_name + + +def process_test_params_for_module(test_params_dict, device, test_instance_class): + module_name = compute_module_name(test_params_dict) + test_params_dict["constructor"] = test_params_dict.get( + "constructor", getattr(torch.nn, module_name) + ) + test_instance = test_instance_class(**test_params_dict) + if not test_instance.get_name().startswith("test_"): + raise AssertionError( + f"Expected test name to start with 'test_', got '{test_instance.get_name()}'" + ) + # Example output: `BCELoss_weights_cuda` + module_variant_name = test_instance.get_name()[5:] + ( + ("_" + device) if device != "cpu" else "" + ) + + if "constructor_args" in test_params_dict: + if "cpp_constructor_args" not in test_params_dict: + raise AssertionError( + "If `constructor_args` is present in test params dict, to enable C++ API parity test, " + f"`cpp_constructor_args` must be present in:\n{pprint.pformat(test_params_dict)}" + "If you are interested in adding the C++ API parity test, please see:\n" + "NOTE [How to check NN module / functional API parity between Python and C++ frontends]. \n" + "If not, please add `test_cpp_api_parity=False` to the test params dict and file an issue about this." + ) + + return TorchNNModuleTestParams( + module_name=module_name, + module_variant_name=module_variant_name, + test_instance=test_instance, + cpp_constructor_args=test_params_dict.get("cpp_constructor_args", ""), + arg_dict=compute_arg_dict(test_params_dict, test_instance), + has_parity=test_params_dict.get("has_parity", True), + device=device, + cpp_tmp_folder=tempfile.mkdtemp(), + ) + + +def write_test_to_test_class( + unit_test_class, test_params_dict, test_instance_class, parity_table, devices +): + if is_torch_nn_functional_test(test_params_dict): + raise AssertionError("Expected non-functional test") + + module_name = compute_module_name(test_params_dict) + + if not hasattr(torch.nn, module_name): + raise AssertionError( + f"`torch.nn` doesn't have module `{module_name}`. " + "If you are adding a new test, please set `fullname` using format `ModuleName_desc` " + f"or set `module_name` using format `ModuleName` in the module test dict:\n{pprint.pformat(test_params_dict)}" + ) + + module_full_name = "torch::nn::" + module_name + + if module_full_name not in parity_table["torch::nn"]: + raise AssertionError( + f"Please add `{module_full_name}` entry to `torch::nn` section of `test/cpp_api_parity/parity-tracker.md`. " + f"(Discovered while processing\n{pprint.pformat(test_params_dict)}.)" + ) + + for device in devices: + test_params = process_test_params_for_module( + test_params_dict=test_params_dict, + device=device, + test_instance_class=test_instance_class, + ) + try_remove_folder(test_params.cpp_tmp_folder) + unit_test_name = f"test_torch_nn_{test_params.module_variant_name}" + unit_test_class.module_test_params_map[unit_test_name] = test_params + + def test_fn(self): + test_forward_backward( + unit_test_class=self, + test_params=unit_test_class.module_test_params_map[ + self._testMethodName + ], + ) + + test_fn = decorate_test_fn( + test_fn=test_fn, + test_cuda=test_params_dict.get("test_cuda", True), + has_impl_parity=parity_table["torch::nn"][module_full_name][0] + and test_params_dict.get("has_parity", True), + device=device, + ) + + add_test(unit_test_class, unit_test_name, test_fn) + + +def generate_test_cpp_sources(test_params, template): + device = test_params.device + + cpp_constructor_args = test_params.cpp_constructor_args + if cpp_constructor_args != "": + cpp_constructor_args = f"({cpp_constructor_args})" + + ( + cpp_args_construction_stmts, + cpp_forward_args_symbols, + ) = compute_cpp_args_construction_stmts_and_forward_arg_symbols(test_params) + + test_cpp_sources = template.substitute( + module_variant_name=test_params.module_variant_name, + module_qualified_name=f"torch::nn::{test_params.module_name}", + cpp_args_construction_stmts=";\n ".join(cpp_args_construction_stmts), + cpp_constructor_args=cpp_constructor_args, + cpp_forward_args_symbols=", ".join(cpp_forward_args_symbols), + device=device, + ) + return test_cpp_sources + + +# Build all C++ tests together, instead of once per test. +def build_cpp_tests(unit_test_class, print_cpp_source=False): + if len(unit_test_class.module_test_params_map) <= 0: + raise AssertionError( + "Expected module_test_params_map to have at least one entry" + ) + cpp_sources = TORCH_NN_COMMON_TEST_HARNESS + SAMPLE_MODULE_CPP_SOURCE + functions = [] + for test_params in unit_test_class.module_test_params_map.values(): + cpp_sources += generate_test_cpp_sources( + test_params=test_params, template=TORCH_NN_MODULE_TEST_FORWARD_BACKWARD + ) + functions.append(f"{test_params.module_variant_name}_test_forward_backward") + if print_cpp_source: + print(cpp_sources) + + cpp_module = compile_cpp_code_inline( + name="module_impl_check", cpp_sources=cpp_sources, functions=functions + ) + unit_test_class.module_impl_check_cpp_module = cpp_module diff --git a/western_ai/usa/pytorch/test/cpp_api_parity/parity-tracker.md b/western_ai/usa/pytorch/test/cpp_api_parity/parity-tracker.md new file mode 100644 index 0000000000000000000000000000000000000000..9e4c4fe39fa70b02fba7ecf547731310b99207f8 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_api_parity/parity-tracker.md @@ -0,0 +1,249 @@ +# C++ / Python API parity tracker + +## torch::nn +API | Implementation Parity | Doc Parity +------------- | ------------- | ------------- +torch::nn::Sequential|Yes|No +torch::nn::ModuleList|Yes|No +torch::nn::ModuleDict|No|No +torch::nn::ParameterList|No|No +torch::nn::ParameterDict|No|No +torch::nn::Conv1d|Yes|No +torch::nn::Conv2d|Yes|No +torch::nn::Conv3d|Yes|No +torch::nn::ConvTranspose1d|Yes|No +torch::nn::ConvTranspose2d|Yes|No +torch::nn::ConvTranspose3d|Yes|No +torch::nn::Unfold|Yes|No +torch::nn::Fold|Yes|No +torch::nn::MaxPool1d|Yes|No +torch::nn::MaxPool2d|Yes|No +torch::nn::MaxPool3d|Yes|No +torch::nn::MaxUnpool1d|Yes|No +torch::nn::MaxUnpool2d|Yes|No +torch::nn::MaxUnpool3d|Yes|No +torch::nn::AvgPool1d|Yes|No +torch::nn::AvgPool2d|Yes|No +torch::nn::AvgPool3d|Yes|No +torch::nn::FractionalMaxPool2d|Yes|No +torch::nn::FractionalMaxPool3d|Yes|No +torch::nn::LPPool1d|Yes|No +torch::nn::LPPool2d|Yes|No +torch::nn::LPPool3d|Yes|No +torch::nn::AdaptiveMaxPool1d|Yes|No +torch::nn::AdaptiveMaxPool2d|Yes|No +torch::nn::AdaptiveMaxPool3d|Yes|No +torch::nn::AdaptiveAvgPool1d|Yes|No +torch::nn::AdaptiveAvgPool2d|Yes|No +torch::nn::AdaptiveAvgPool3d|Yes|No +torch::nn::ReflectionPad1d|Yes|No +torch::nn::ReflectionPad2d|Yes|No +torch::nn::ReflectionPad3d|Yes|No +torch::nn::ReplicationPad1d|Yes|No +torch::nn::ReplicationPad2d|Yes|No +torch::nn::ReplicationPad3d|Yes|No +torch::nn::ZeroPad1d|Yes|No +torch::nn::ZeroPad2d|Yes|No +torch::nn::ZeroPad3d|Yes|No +torch::nn::ConstantPad1d|Yes|No +torch::nn::ConstantPad2d|Yes|No +torch::nn::ConstantPad3d|Yes|No +torch::nn::ELU|Yes|No +torch::nn::Hardshrink|Yes|No +torch::nn::Hardtanh|Yes|No +torch::nn::LeakyReLU|Yes|No +torch::nn::LogSigmoid|Yes|No +torch::nn::Mish|Yes|No +torch::nn::MultiheadAttention|No|No +torch::nn::PReLU|Yes|No +torch::nn::ReLU|Yes|No +torch::nn::ReLU6|Yes|No +torch::nn::RReLU|Yes|No +torch::nn::SELU|Yes|No +torch::nn::CELU|Yes|No +torch::nn::GELU|Yes|No +torch::nn::SiLU|Yes|No +torch::nn::Sigmoid|Yes|No +torch::nn::Softplus|Yes|No +torch::nn::Softshrink|Yes|No +torch::nn::Softsign|Yes|No +torch::nn::Tanh|Yes|No +torch::nn::Tanhshrink|Yes|No +torch::nn::Threshold|Yes|No +torch::nn::GLU|Yes|No +torch::nn::Softmin|Yes|No +torch::nn::Softmax|Yes|No +torch::nn::Softmax2d|Yes|No +torch::nn::LogSoftmax|Yes|No +torch::nn::AdaptiveLogSoftmaxWithLoss|Yes|No +torch::nn::BatchNorm1d|Yes|No +torch::nn::BatchNorm2d|Yes|No +torch::nn::BatchNorm3d|Yes|No +torch::nn::GroupNorm|Yes|No +torch::nn::SyncBatchNorm|No|No +torch::nn::InstanceNorm1d|Yes|No +torch::nn::InstanceNorm2d|Yes|No +torch::nn::InstanceNorm3d|Yes|No +torch::nn::LayerNorm|Yes|No +torch::nn::LocalResponseNorm|Yes|No +torch::nn::CrossMapLRN2d|Yes|No +torch::nn::RNN|Yes|No +torch::nn::LSTM|Yes|No +torch::nn::GRU|Yes|No +torch::nn::RNNCell|Yes|No +torch::nn::LSTMCell|Yes|No +torch::nn::GRUCell|Yes|No +torch::nn::Transformer|Yes|No +torch::nn::TransformerEncoder|No|No +torch::nn::TransformerDecoder|No|No +torch::nn::TransformerEncoderLayer|Yes|No +torch::nn::TransformerDecoderLayer|Yes|No +torch::nn::Identity|Yes|No +torch::nn::Linear|Yes|No +torch::nn::Bilinear|Yes|No +torch::nn::Flatten|Yes|No +torch::nn::Unflatten|Yes|No +torch::nn::Dropout|Yes|No +torch::nn::Dropout2d|Yes|No +torch::nn::Dropout3d|Yes|No +torch::nn::AlphaDropout|Yes|No +torch::nn::FeatureAlphaDropout|Yes|No +torch::nn::Embedding|Yes|No +torch::nn::EmbeddingBag|Yes|No +torch::nn::CosineSimilarity|Yes|No +torch::nn::PairwiseDistance|Yes|No +torch::nn::L1Loss|Yes|No +torch::nn::MSELoss|Yes|No +torch::nn::CrossEntropyLoss|Yes|No +torch::nn::CTCLoss|Yes|No +torch::nn::NLLLoss|Yes|No +torch::nn::PoissonNLLLoss|Yes|No +torch::nn::KLDivLoss|Yes|No +torch::nn::BCELoss|Yes|No +torch::nn::BCEWithLogitsLoss|Yes|No +torch::nn::MarginRankingLoss|Yes|No +torch::nn::HingeEmbeddingLoss|Yes|No +torch::nn::MultiLabelMarginLoss|Yes|No +torch::nn::SmoothL1Loss|Yes|No +torch::nn::HuberLoss|Yes|No +torch::nn::SoftMarginLoss|Yes|No +torch::nn::MultiLabelSoftMarginLoss|Yes|No +torch::nn::CosineEmbeddingLoss|Yes|No +torch::nn::MultiMarginLoss|Yes|No +torch::nn::TripletMarginLoss|Yes|No +torch::nn::PixelShuffle|Yes|No +torch::nn::PixelUnshuffle|Yes|No +torch::nn::Upsample|Yes|No +torch::nn::DataParallel|No|No +torch::nn::parallel::DistributedDataParallel|No|No +torch::nn::utils::clip_grad_norm_|Yes|No +torch::nn::utils::clip_grad_value_|Yes|No +torch::nn::utils::parameters_to_vector|Yes|No +torch::nn::utils::vector_to_parameters|Yes|No +torch::nn::utils::weight_norm|No|No +torch::nn::utils::remove_weight_norm|No|No +torch::nn::utils::spectral_norm|No|No +torch::nn::utils::remove_spectral_norm|No|No +torch::nn::utils::rnn::PackedSequence|Yes|No +torch::nn::utils::rnn::pack_padded_sequence|Yes|No +torch::nn::utils::rnn::pad_packed_sequence|Yes|No +torch::nn::utils::rnn::pad_sequence|Yes|No +torch::nn::utils::rnn::pack_sequence|Yes|No +torch::nn::SampleModule|Yes|Yes + +## torch::nn::functional + +API | Implementation Parity | Doc Parity +------------- | ------------- | ------------- +F::conv1d|Yes|No +F::conv2d|Yes|No +F::conv3d|Yes|No +F::conv_transpose1d|Yes|No +F::conv_transpose2d|Yes|No +F::conv_transpose3d|Yes|No +F::unfold|Yes|No +F::fold|Yes|No +F::avg_pool1d|Yes|No +F::avg_pool2d|Yes|No +F::avg_pool3d|Yes|No +F::max_pool1d|Yes|No +F::max_pool2d|Yes|No +F::max_pool3d|Yes|No +F::max_unpool1d|Yes|No +F::max_unpool2d|Yes|No +F::max_unpool3d|Yes|No +F::lp_pool1d|Yes|No +F::lp_pool2d|Yes|No +F::lp_pool3d|Yes|No +F::adaptive_max_pool1d|Yes|No +F::adaptive_max_pool2d|Yes|No +F::adaptive_max_pool3d|Yes|No +F::adaptive_avg_pool1d|Yes|No +F::adaptive_avg_pool2d|Yes|No +F::adaptive_avg_pool3d|Yes|No +F::threshold|Yes|No +F::relu|Yes|No +F::hardtanh|Yes|No +F::relu6|Yes|No +F::elu|Yes|No +F::selu|Yes|No +F::celu|Yes|No +F::leaky_relu|Yes|No +F::prelu|Yes|No +F::rrelu|Yes|No +F::glu|Yes|No +F::gelu|Yes|No +F::silu|Yes|No +F::mish|Yes|No +F::logsigmoid|Yes|No +F::hardshrink|Yes|No +F::tanhshrink|Yes|No +F::softsign|Yes|No +F::softplus|Yes|No +F::softmin|Yes|No +F::softmax|Yes|No +F::softshrink|Yes|No +F::gumbel_softmax|Yes|No +F::log_softmax|Yes|No +F::batch_norm|Yes|No +F::instance_norm|Yes|No +F::layer_norm|Yes|No +F::local_response_norm|Yes|No +F::normalize|Yes|No +F::linear|Yes|No +F::bilinear|Yes|No +F::dropout|Yes|No +F::alpha_dropout|Yes|No +F::dropout2d|Yes|No +F::dropout3d|Yes|No +F::embedding|Yes|No +F::embedding_bag|Yes|No +F::one_hot|Yes|No +F::pairwise_distance|Yes|No +F::cosine_similarity|Yes|No +F::pdist|Yes|No +F::binary_cross_entropy|Yes|No +F::binary_cross_entropy_with_logits|Yes|No +F::poisson_nll_loss|Yes|No +F::cosine_embedding_loss|Yes|No +F::cross_entropy|Yes|No +F::ctc_loss|Yes|No +F::hinge_embedding_loss|Yes|No +F::kl_div|Yes|No +F::l1_loss|Yes|No +F::mse_loss|Yes|No +F::margin_ranking_loss|Yes|No +F::multilabel_margin_loss|Yes|No +F::multilabel_soft_margin_loss|Yes|No +F::multi_margin_loss|Yes|No +F::nll_loss|Yes|No +F::smooth_l1_loss|Yes|No +F::huber_loss|Yes|No +F::soft_margin_loss|Yes|No +F::triplet_margin_loss|Yes|No +F::pixel_shuffle|Yes|No +F::pad|Yes|No +F::interpolate|Yes|No +F::grid_sample|Yes|No +F::affine_grid|Yes|No +F::sample_functional|Yes|No \ No newline at end of file diff --git a/western_ai/usa/pytorch/test/cpp_api_parity/parity_table_parser.py b/western_ai/usa/pytorch/test/cpp_api_parity/parity_table_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..a60adf760fab90a259d095bfb54715bf7bd999e2 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_api_parity/parity_table_parser.py @@ -0,0 +1,67 @@ +from collections import namedtuple + + +ParityStatus = namedtuple("ParityStatus", ["has_impl_parity", "has_doc_parity"]) + +""" +This function expects the parity tracker Markdown file to have the following format: + +``` +## package1_name + +API | Implementation Parity | Doc Parity +------------- | ------------- | ------------- +API_Name|No|No +... + +## package2_name + +API | Implementation Parity | Doc Parity +------------- | ------------- | ------------- +API_Name|No|No +... +``` + +The returned dict has the following format: + +``` +Dict[package_name] + -> Dict[api_name] + -> ParityStatus +``` +""" + + +def parse_parity_tracker_table(file_path): + def parse_parity_choice(str): + if str in ["Yes", "No"]: + return str == "Yes" + else: + raise RuntimeError( + f'{str} is not a supported parity choice. The valid choices are "Yes" and "No".' + ) + + parity_tracker_dict = {} + + with open(file_path) as f: + all_text = f.read() + packages = all_text.split("##") + for package in packages[1:]: + lines = [line.strip() for line in package.split("\n") if line.strip() != ""] + package_name = lines[0] + if package_name in parity_tracker_dict: + raise RuntimeError( + f"Duplicated package name `{package_name}` found in {file_path}" + ) + else: + parity_tracker_dict[package_name] = {} + for api_status in lines[3:]: + api_name, has_impl_parity_str, has_doc_parity_str = ( + x.strip() for x in api_status.split("|") + ) + parity_tracker_dict[package_name][api_name] = ParityStatus( + has_impl_parity=parse_parity_choice(has_impl_parity_str), + has_doc_parity=parse_parity_choice(has_doc_parity_str), + ) + + return parity_tracker_dict diff --git a/western_ai/usa/pytorch/test/cpp_api_parity/sample_functional.py b/western_ai/usa/pytorch/test/cpp_api_parity/sample_functional.py new file mode 100644 index 0000000000000000000000000000000000000000..7839b024c21bd64f5c801a432cdc25a0ec765417 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_api_parity/sample_functional.py @@ -0,0 +1,72 @@ +import torch +import torch.nn.functional as F +from torch.testing._internal.common_nn import wrap_functional + + +""" +`sample_functional` is used by `test_cpp_api_parity.py` to test that Python / C++ API +parity test harness works for `torch.nn.functional` functions. + +When `has_parity=true` is passed to `sample_functional`, behavior of `sample_functional` +is the same as the C++ equivalent. + +When `has_parity=false` is passed to `sample_functional`, behavior of `sample_functional` +is different from the C++ equivalent. +""" + + +def sample_functional(x, has_parity): + if has_parity: + return x * 2 + else: + return x * 4 + + +torch.nn.functional.sample_functional = sample_functional + +SAMPLE_FUNCTIONAL_CPP_SOURCE = """\n +namespace torch { +namespace nn { +namespace functional { + +struct C10_EXPORT SampleFunctionalFuncOptions { + SampleFunctionalFuncOptions(bool has_parity) : has_parity_(has_parity) {} + + TORCH_ARG(bool, has_parity); +}; + +Tensor sample_functional(Tensor x, SampleFunctionalFuncOptions options) { + return x * 2; +} + +} // namespace functional +} // namespace nn +} // namespace torch +""" + +functional_tests = [ + dict( + constructor=wrap_functional(F.sample_functional, has_parity=True), + cpp_options_args="F::SampleFunctionalFuncOptions(true)", + input_size=(1, 2, 3), + fullname="sample_functional_has_parity", + has_parity=True, + ), + dict( + constructor=wrap_functional(F.sample_functional, has_parity=False), + cpp_options_args="F::SampleFunctionalFuncOptions(false)", + input_size=(1, 2, 3), + fullname="sample_functional_no_parity", + has_parity=False, + ), + # This is to test that setting the `test_cpp_api_parity=False` flag skips + # the C++ API parity test accordingly (otherwise this test would run and + # throw a parity error). + dict( + constructor=wrap_functional(F.sample_functional, has_parity=False), + cpp_options_args="F::SampleFunctionalFuncOptions(false)", + input_size=(1, 2, 3), + fullname="sample_functional_THIS_TEST_SHOULD_BE_SKIPPED", + test_cpp_api_parity=False, + ), +] diff --git a/western_ai/usa/pytorch/test/cpp_api_parity/sample_module.py b/western_ai/usa/pytorch/test/cpp_api_parity/sample_module.py new file mode 100644 index 0000000000000000000000000000000000000000..51b95fba6d83b6ab1510e2604d57f741983ff5e1 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_api_parity/sample_module.py @@ -0,0 +1,108 @@ +import torch + + +""" +`SampleModule` is used by `test_cpp_api_parity.py` to test that Python / C++ API +parity test harness works for `torch.nn.Module` subclasses. + +When `SampleModule.has_parity` is true, behavior of `forward` / `backward` +is the same as the C++ equivalent. + +When `SampleModule.has_parity` is false, behavior of `forward` / `backward` +is different from the C++ equivalent. +""" + + +class SampleModule(torch.nn.Module): + def __init__(self, has_parity, has_submodule): + super().__init__() + self.has_parity = has_parity + if has_submodule: + self.submodule = SampleModule(self.has_parity, False) + + self.has_submodule = has_submodule + self.register_parameter("param", torch.nn.Parameter(torch.empty(3, 4))) + + self.reset_parameters() + + def reset_parameters(self): + with torch.no_grad(): + self.param.fill_(1) + + def forward(self, x): + submodule_forward_result = ( + self.submodule(x) if hasattr(self, "submodule") else 0 + ) + if self.has_parity: + return x + self.param * 2 + submodule_forward_result + else: + return x + self.param * 4 + submodule_forward_result + 3 + + +torch.nn.SampleModule = SampleModule + +SAMPLE_MODULE_CPP_SOURCE = """\n +namespace torch { +namespace nn { +struct C10_EXPORT SampleModuleOptions { + SampleModuleOptions(bool has_parity, bool has_submodule) : has_parity_(has_parity), has_submodule_(has_submodule) {} + + TORCH_ARG(bool, has_parity); + TORCH_ARG(bool, has_submodule); +}; + +struct C10_EXPORT SampleModuleImpl : public torch::nn::Cloneable { + explicit SampleModuleImpl(SampleModuleOptions options) : options(std::move(options)) { + if (options.has_submodule()) { + submodule = register_module( + "submodule", + std::make_shared(SampleModuleOptions(options.has_parity(), false))); + } + reset(); + } + void reset() { + param = register_parameter("param", torch::ones({3, 4})); + } + torch::Tensor forward(torch::Tensor x) { + return x + param * 2 + (submodule ? submodule->forward(x) : torch::zeros_like(x)); + } + SampleModuleOptions options; + torch::Tensor param; + std::shared_ptr submodule{nullptr}; +}; + +TORCH_MODULE(SampleModule); +} // namespace nn +} // namespace torch +""" + +module_tests = [ + dict( + module_name="SampleModule", + desc="has_parity", + constructor_args=(True, True), + cpp_constructor_args="torch::nn::SampleModuleOptions(true, true)", + input_size=(3, 4), + cpp_input_args=["torch::randn({3, 4})"], + has_parity=True, + ), + dict( + fullname="SampleModule_no_parity", + constructor=lambda: SampleModule(has_parity=False, has_submodule=True), + cpp_constructor_args="torch::nn::SampleModuleOptions(false, true)", + input_size=(3, 4), + cpp_input_args=["torch::randn({3, 4})"], + has_parity=False, + ), + # This is to test that setting the `test_cpp_api_parity=False` flag skips + # the C++ API parity test accordingly (otherwise this test would run and + # throw a parity error). + dict( + fullname="SampleModule_THIS_TEST_SHOULD_BE_SKIPPED", + constructor=lambda: SampleModule(False, True), + cpp_constructor_args="torch::nn::SampleModuleOptions(false, true)", + input_size=(3, 4), + cpp_input_args=["torch::randn({3, 4})"], + test_cpp_api_parity=False, + ), +] diff --git a/western_ai/usa/pytorch/test/cpp_api_parity/utils.py b/western_ai/usa/pytorch/test/cpp_api_parity/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6cc84b8852875bc4a90355da4d3d6bdbb07ac2be --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_api_parity/utils.py @@ -0,0 +1,399 @@ +import os +import shutil +import unittest +import warnings +from collections import namedtuple + +import torch +import torch.testing._internal.common_nn as common_nn +import torch.utils.cpp_extension +from torch.testing._internal.common_cuda import TEST_CUDA + + +# Note that this namedtuple is for C++ parity test mechanism's internal use. +# For guidance on how to add a new C++ parity test, please see +# NOTE [How to check NN module / functional API parity between Python and C++ frontends] +TorchNNModuleTestParams = namedtuple( + "TorchNNModuleTestParams", + [ + # NN module name (e.g. "BCELoss") + "module_name", + # Unique identifier for this module config (e.g. "BCELoss_weights_cuda") + "module_variant_name", + # An instance of an NN test class (e.g. `CriterionTest`) which stores + # necessary information (e.g. input / target / extra_args) for running the Python test + "test_instance", + # Constructor arguments passed to the C++ module constructor, which must be + # strictly equivalent to the Python module constructor arguments + # (e.g. `torch::nn::BCELossOptions().weight(torch::rand(10))`, + # which is strictly equivalent to passing `torch.rand(10)` to `torch.nn.BCELoss` + # constructor in Python) + "cpp_constructor_args", + # All arguments used in NN module's forward pass. + # Please see `compute_arg_dict` function for details on how we construct this dict. + # (e.g. + # ``` + # arg_dict = { + # 'input': [python_input_tensor], + # 'target': [python_target_tensor], + # 'extra_args': [], + # 'other': [], + # } + # ``` + # ) + "arg_dict", + # Whether we expect this NN module test to pass the Python/C++ parity test + # (e.g. `True`) + "has_parity", + # Device (e.g. "cuda") + "device", + # Temporary folder to store C++ outputs (to be compared with Python outputs later) + "cpp_tmp_folder", + ], +) + +# Note that this namedtuple is for C++ parity test mechanism's internal use. +# For guidance on how to add a new C++ parity test, please see +# NOTE [How to check NN module / functional API parity between Python and C++ frontends] +TorchNNFunctionalTestParams = namedtuple( + "TorchNNFunctionalTestParams", + [ + # NN functional name (e.g. "binary_cross_entropy") + "functional_name", + # Unique identifier for this functional config (e.g. "BCELoss_no_reduce_cuda") + "functional_variant_name", + # An instance of an NN test class (e.g. `NewModuleTest`) which stores + # necessary information (e.g. input / target / extra_args) for running the Python test + "test_instance", + # The C++ function call that is strictly equivalent to the Python function call + # (e.g. "F::binary_cross_entropy( + # i, t.to(i.options()),F::BinaryCrossEntropyFuncOptions().reduction(torch::kNone))", + # which is strictly equivalent to `F.binary_cross_entropy(i, t.type_as(i), reduction='none')` in Python) + "cpp_function_call", + # All arguments used in NN functional's function call. + # Please see `compute_arg_dict` function for details on how we construct this dict. + # (e.g. + # ``` + # arg_dict = { + # 'input': [python_input_tensor], + # 'target': [python_target_tensor], + # 'extra_args': [], + # 'other': [], + # } + # ``` + # ) + "arg_dict", + # Whether we expect this NN functional test to pass the Python/C++ parity test + # (e.g. `True`) + "has_parity", + # Device (e.g. "cuda") + "device", + # Temporary folder to store C++ outputs (to be compared with Python outputs later) + "cpp_tmp_folder", + ], +) + +CppArg = namedtuple("CppArg", ["name", "value"]) + +TORCH_NN_COMMON_TEST_HARNESS = """ +#include + +void write_ivalue_to_file(const torch::IValue& ivalue, const std::string& file_path) { + auto bytes = torch::jit::pickle_save(ivalue); + std::ofstream fout(file_path, std::ios::out | std::ios::binary); + fout.write(bytes.data(), bytes.size()); + fout.close(); +} + +c10::Dict load_dict_from_file(const std::string& file_path) { + c10::Dict arg_dict; + auto arg_dict_module = torch::jit::load(file_path); + for (const auto& p : arg_dict_module.named_buffers(/*recurse=*/false)) { + arg_dict.insert(p.name, p.value); + } + return arg_dict; +} + +// Generates rand tensor with non-equal values. This ensures that duplicate +// values won't be causing test failure for modules like MaxPooling. +// size should be small, otherwise randperm fails / long overflows. +torch::Tensor _rand_tensor_non_equal(torch::IntArrayRef size) { + int64_t total = 1; + for (int64_t elem : size) { + total *= elem; + } + return torch::randperm(total).view(size).to(torch::kDouble); +} +""" + + +def compile_cpp_code_inline(name, cpp_sources, functions): + cpp_module = torch.utils.cpp_extension.load_inline( + name=name, + cpp_sources=cpp_sources, + extra_cflags=[ + "-g" + ], # Enable debug symbols by default for debugging test failures. + functions=functions, + verbose=False, + ) + return cpp_module + + +def compute_temp_file_path(cpp_tmp_folder, variant_name, file_suffix): + return os.path.join(cpp_tmp_folder, f"{variant_name}_{file_suffix}.pt") + + +def is_torch_nn_functional_test(test_params_dict): + return "wrap_functional" in str(test_params_dict.get("constructor", "")) + + +def convert_to_list(python_input): + if isinstance(python_input, torch.Tensor): + return [python_input] + else: + return list(python_input) + + +def set_python_tensors_requires_grad(python_tensors): + return [ + tensor.requires_grad_(True) if tensor.dtype != torch.long else tensor + for tensor in python_tensors + ] + + +def move_python_tensors_to_device(python_tensors, device): + return [tensor.to(device) for tensor in python_tensors] + + +def has_test(unit_test_class, test_name): + return hasattr(unit_test_class, test_name) + + +def add_test(unit_test_class, test_name, test_fn): + if has_test(unit_test_class, test_name): + raise RuntimeError("Found two tests with the same name: " + test_name) + setattr(unit_test_class, test_name, test_fn) + + +def set_cpp_tensors_requires_grad(cpp_tensor_stmts, python_tensors): + if len(cpp_tensor_stmts) != len(python_tensors): + raise AssertionError( + f"Expected len(cpp_tensor_stmts) == len(python_tensors), got {len(cpp_tensor_stmts)} vs {len(python_tensors)}" + ) + return [ + f"{tensor_stmt}.requires_grad_(true)" + if tensor.dtype != torch.long + else tensor_stmt + for tensor_stmt, (_, tensor) in zip(cpp_tensor_stmts, python_tensors) + ] + + +def move_cpp_tensors_to_device(cpp_tensor_stmts, device): + return [f'{tensor_stmt}.to("{device}")' for tensor_stmt in cpp_tensor_stmts] + + +def is_criterion_test(test_instance): + return isinstance(test_instance, common_nn.CriterionTest) + + +# This function computes the following: +# - What variable declaration statements should show up in the C++ parity test function +# - What arguments should be passed into the C++ module/functional's forward function +# +# For example, for the "L1Loss" test, the return values from this function are: +# ``` +# // Note that `arg_dict` stores all tensor values we transfer from Python to C++ +# cpp_args_construction_stmts = [ +# "auto i0 = arg_dict.at("i0").to("cpu").requires_grad_(true)", +# "auto t0 = arg_dict.at("t0").to("cpu")", +# ], +# cpp_forward_args_symbols = [ +# "i0", +# "t0", +# ] +# ``` +def compute_cpp_args_construction_stmts_and_forward_arg_symbols(test_params): + device = test_params.device + cpp_forward_args_symbols = [] + + def add_cpp_forward_args(args): + args_stmts = [] + for arg_name, _ in args: + args_stmts.append(f'auto {arg_name} = arg_dict.at("{arg_name}")') + cpp_forward_args_symbols.append(arg_name) + return args_stmts + + cpp_forward_input_args_stmts = set_cpp_tensors_requires_grad( + move_cpp_tensors_to_device( + add_cpp_forward_args(test_params.arg_dict["input"]), device + ), + test_params.arg_dict["input"], + ) + cpp_forward_target_args_stmts = move_cpp_tensors_to_device( + add_cpp_forward_args(test_params.arg_dict["target"]), device + ) + cpp_forward_extra_args_stmts = move_cpp_tensors_to_device( + add_cpp_forward_args(test_params.arg_dict["extra_args"]), device + ) + + # Build the list of other arguments needed + cpp_other_args_stmts = [] + for arg_name, _ in test_params.arg_dict["other"]: + cpp_other_args_stmts.append(f'auto {arg_name} = arg_dict.at("{arg_name}")') + cpp_other_args_stmts = move_cpp_tensors_to_device(cpp_other_args_stmts, device) + + cpp_args_construction_stmts = ( + cpp_forward_input_args_stmts + + cpp_forward_target_args_stmts + + cpp_forward_extra_args_stmts + + cpp_other_args_stmts + ) + + return cpp_args_construction_stmts, cpp_forward_args_symbols + + +def serialize_arg_dict_as_script_module(arg_dict): + arg_dict_flat = dict( + arg_dict["input"] + + arg_dict["target"] + + arg_dict["extra_args"] + + arg_dict["other"] + ) + arg_dict_module = torch.nn.Module() + for arg_name, arg_value in arg_dict_flat.items(): + if not isinstance(arg_value, torch.Tensor): + raise AssertionError( + f"Expected arg_value to be Tensor, got {type(arg_value)} for arg '{arg_name}'" + ) + arg_dict_module.register_buffer(arg_name, arg_value) + + return torch.jit.script(arg_dict_module) + + +# NOTE: any argument symbol used in `cpp_constructor_args` / `cpp_options_args` / `cpp_function_call` +# must have a mapping in `cpp_var_map`. +# +# The mapping can take one of the following formats: +# +# 1. `argument_name` -> Python value +# 2. `argument_name` -> '_get_input()' (which means `argument_name` in C++ will be bound to `test_instance._get_input()`) +# +# For example: +# ``` +# def bceloss_weights_no_reduce_test(): +# t = torch.randn(15, 10).gt(0).double() +# weights = torch.rand(10) +# return dict( +# fullname='BCELoss_weights_no_reduce', +# constructor=wrap_functional( +# lambda i: F.binary_cross_entropy(i, t.type_as(i), +# weight=weights.type_as(i), reduction='none')), +# cpp_function_call='''F::binary_cross_entropy( +# i, t.to(i.options()), +# F::BinaryCrossEntropyFuncOptions() +# .weight(weights.to(i.options())) +# .reduction(torch::kNone))''', +# input_fn=lambda: torch.rand(15, 10).clamp_(2.8e-2, 1 - 2.8e-2), +# cpp_var_map={'i': '_get_input()', 't': t, 'weights': weights}, +# reference_fn=lambda i, p, m: -(t * i.log() + (1 - t) * (1 - i).log()) * weights, +# ) +# ``` +def compute_arg_dict(test_params_dict, test_instance): + arg_dict = { + "input": [], + "target": [], + "extra_args": [], + "other": [], + } + + def put_args_into_arg_dict(arg_type, arg_type_prefix, args): + for i, arg in enumerate(args): + arg_dict[arg_type].append(CppArg(name=arg_type_prefix + str(i), value=arg)) + + put_args_into_arg_dict("input", "i", convert_to_list(test_instance._get_input())) + if is_criterion_test(test_instance): + put_args_into_arg_dict( + "target", "t", convert_to_list(test_instance._get_target()) + ) + if test_instance.extra_args: + put_args_into_arg_dict( + "extra_args", "e", convert_to_list(test_instance.extra_args) + ) + + cpp_var_map = test_params_dict.get("cpp_var_map", {}) + for arg_name, arg_value in cpp_var_map.items(): + if isinstance(arg_value, str): + if arg_value == "_get_input()": + arg_dict["other"].append( + CppArg(name=arg_name, value=test_instance._get_input()) + ) + else: + raise RuntimeError( + f"`{arg_name}` has unsupported string value: {arg_value}" + ) + elif isinstance(arg_value, torch.Tensor): + arg_dict["other"].append(CppArg(name=arg_name, value=arg_value)) + else: + raise RuntimeError(f"`{arg_name}` has unsupported value: {arg_value}") + + return arg_dict + + +def decorate_test_fn(test_fn, test_cuda, has_impl_parity, device): + if device == "cuda": + test_fn = unittest.skipIf(not TEST_CUDA, "CUDA unavailable")(test_fn) + test_fn = unittest.skipIf(not test_cuda, "Excluded from CUDA tests")(test_fn) + + # If `Implementation Parity` entry in parity table for this module is `No`, + # or `has_parity` entry in test params dict is `False`, we mark the test as + # expected failure. + if not has_impl_parity: + test_fn = unittest.expectedFailure(test_fn) + + return test_fn + + +MESSAGE_HOW_TO_FIX_CPP_PARITY_TEST_FAILURE = """ +What should I do when C++ API parity test is failing? + +- If you are changing the implementation of an existing `torch.nn` module / `torch.nn.functional` function: +Answer: Ideally you should also change the C++ API implementation for that module / function +(you can start by searching for the module / function name in `torch/csrc/api/` folder). + +- If you are adding a new test for an existing `torch.nn` module / `torch.nn.functional` function: +Answer: Ideally you should fix the C++ API implementation for that module / function +to exactly match the Python API implementation (you can start by searching for the module / +function name in `torch/csrc/api/` folder). + +- If you are adding a test for a *new* `torch.nn` module / `torch.nn.functional` function: +Answer: Ideally you should add the corresponding C++ API implementation for that module / function, +and it should exactly match the Python API implementation. (We have done a large effort on this +which is tracked at https://github.com/pytorch/pytorch/issues/25883.) + +However, if any of the above is proven to be too complicated, you can just add +`test_cpp_api_parity=False` to any failing test in `torch/testing/_internal/common_nn.py`, +and the C++ API parity test will be skipped accordingly. Note that you should +also file an issue when you do this. + +For more details on how to add a C++ API parity test, please see: +NOTE [How to check NN module / functional API parity between Python and C++ frontends] +""" + + +def generate_error_msg(name, cpp_value, python_value): + return ( + f"Parity test failed: {name} in C++ has value: {cpp_value}, " + f"which does not match the corresponding value in Python: {python_value}.\n{MESSAGE_HOW_TO_FIX_CPP_PARITY_TEST_FAILURE}" + ) + + +def try_remove_folder(folder_path): + if os.path.exists(folder_path): + # Don't block the process if this fails, but show the error message as warning. + try: + shutil.rmtree(folder_path) + except Exception as e: + warnings.warn( + f"Non-blocking folder removal fails with the following error:\n{str(e)}" + ) diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cpp_c10d_extension.cpp b/western_ai/usa/pytorch/test/cpp_extensions/cpp_c10d_extension.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fcd97045e4be5f7be49c16e63d6429983bdcede0 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cpp_c10d_extension.cpp @@ -0,0 +1,122 @@ +#include "cpp_c10d_extension.hpp" + +#include + +namespace c10d { + +ProcessGroupTest::WorkTest::~WorkTest() {} + +bool ProcessGroupTest::WorkTest::isCompleted() { + return true; +} + +bool ProcessGroupTest::WorkTest::isSuccess() const { + return true; +} + +bool ProcessGroupTest::WorkTest::wait(std::chrono::milliseconds /* unused */) { + return true; +} + +ProcessGroupTest::ProcessGroupTest(int rank, int size) + : ProcessGroup(rank, size) {} + +ProcessGroupTest::~ProcessGroupTest() {} + +c10::intrusive_ptr ProcessGroupTest::broadcast( + std::vector& tensors, + const BroadcastOptions& opts) { + return c10::make_intrusive(); +} + +c10::intrusive_ptr ProcessGroupTest::allreduce( + std::vector& tensors, + const AllreduceOptions& opts) { + return c10::make_intrusive(); +} + +c10::intrusive_ptr ProcessGroupTest::allreduce_coalesced( + std::vector& tensors, + const AllreduceCoalescedOptions& opts) { + throw std::runtime_error("ProcessGroupTest does not support allreduce_coalesced"); +} + +c10::intrusive_ptr ProcessGroupTest::reduce( + std::vector& tensors, + const ReduceOptions& opts) { + throw std::runtime_error("ProcessGroupTest does not support reduce"); +} + +c10::intrusive_ptr ProcessGroupTest::allgather( + std::vector>& outputTensors, + std::vector& inputTensors, + const AllgatherOptions& opts) { + throw std::runtime_error("ProcessGroupTest does not support allgather"); +} + +c10::intrusive_ptr ProcessGroupTest::_allgather_base( + at::Tensor& outputBuffer, + at::Tensor& inputBuffer, + const AllgatherOptions& opts) { + throw std::runtime_error("ProcessGroupTest does not support _allgather_base"); +} + +c10::intrusive_ptr ProcessGroupTest::barrier( + const BarrierOptions& opts) { + return c10::make_intrusive(); +} + +c10::intrusive_ptr ProcessGroupTest::gather( + std::vector>& outputTensors, + std::vector& inputTensors, + const GatherOptions& opts) { + throw std::runtime_error("ProcessGroupTest does not support gather"); +} + +c10::intrusive_ptr ProcessGroupTest::scatter( + std::vector& outputTensors, + std::vector>& inputTensors, + const ScatterOptions& opts) { + throw std::runtime_error("ProcessGroupTest does not support scatter"); +} + +c10::intrusive_ptr ProcessGroupTest::reduce_scatter( + std::vector& outputTensors, + std::vector>& inputTensors, + const ReduceScatterOptions& opts) { + throw std::runtime_error("ProcessGroupTest does not support reduce_scatter"); +} + +c10::intrusive_ptr ProcessGroupTest::send( + std::vector& tensors, + int dstRank, + int tag) { + throw std::runtime_error("ProcessGroupTest does not support send"); +} + +c10::intrusive_ptr ProcessGroupTest::recv( + std::vector& tensors, + int srcRank, + int tag) { + throw std::runtime_error("ProcessGroupTest does not support recv"); +} + +c10::intrusive_ptr ProcessGroupTest::recvAnysource( + std::vector& tensor, + int tag) { + throw std::runtime_error("ProcessGroupTest does not support recvAnysource"); +} + +c10::intrusive_ptr ProcessGroupTest::createProcessGroupTest( + const c10::intrusive_ptr<::c10d::Store>& store, + int rank, + int size, + const std::chrono::duration& timeout) { + return c10::make_intrusive(rank, size); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("createProcessGroupTest", &ProcessGroupTest::createProcessGroupTest); +} + +} // namespace c10d diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cpp_c10d_extension.hpp b/western_ai/usa/pytorch/test/cpp_extensions/cpp_c10d_extension.hpp new file mode 100644 index 0000000000000000000000000000000000000000..8f8adaac6872329b7ec30dd646cbf330f69b9d63 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cpp_c10d_extension.hpp @@ -0,0 +1,123 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace c10d { + +// +// ProcessGroupTest implements dummy bindings for c10d. +// + +class ProcessGroupTest : public ProcessGroup { + public: + class WorkTest : public Work { + public: + WorkTest() {} + + virtual ~WorkTest(); + bool isCompleted() override; + bool isSuccess() const override; + bool wait(std::chrono::milliseconds timeout) override; + + protected: + friend class ProcessGroupTest; + }; + + explicit ProcessGroupTest(int rank = -1, int size = -1); + virtual ~ProcessGroupTest(); + + c10::intrusive_ptr broadcast( + std::vector& data, + const BroadcastOptions& opts = BroadcastOptions()) override; + + c10::intrusive_ptr allreduce( + std::vector& tensors, + const AllreduceOptions& opts = AllreduceOptions()) override; + + c10::intrusive_ptr allreduce_coalesced( + std::vector& tensors, + const AllreduceCoalescedOptions& opts = AllreduceCoalescedOptions()) override; + + c10::intrusive_ptr reduce( + std::vector& tensors, + const ReduceOptions& opts = ReduceOptions()) override; + + c10::intrusive_ptr allgather( + std::vector>& outputTensors, + std::vector& inputTensors, + const AllgatherOptions& opts = AllgatherOptions()) override; + + c10::intrusive_ptr _allgather_base( + at::Tensor& outputBuffer, + at::Tensor& inputBuffer, + const AllgatherOptions& opts = AllgatherOptions()) override; + + c10::intrusive_ptr barrier( + const BarrierOptions& opts = BarrierOptions()) override; + + c10::intrusive_ptr gather( + std::vector>& outputTensors, + std::vector& inputTensors, + const GatherOptions& opts = GatherOptions()) override; + + c10::intrusive_ptr scatter( + std::vector& outputTensors, + std::vector>& inputTensors, + const ScatterOptions& opts = ScatterOptions()) override; + + c10::intrusive_ptr reduce_scatter( + std::vector& outputTensors, + std::vector>& inputTensors, + const ReduceScatterOptions& opts = ReduceScatterOptions()) override; + + c10::intrusive_ptr send( + std::vector& tensors, + int dstRank, + int tag) override; + + c10::intrusive_ptr recv( + std::vector& tensors, + int srcRank, + int tag) override; + + c10::intrusive_ptr recvAnysource( + std::vector& tensor, + int tag) override; + + // Create a new ProcessGroupTest instance + static c10::intrusive_ptr createProcessGroupTest( + const c10::intrusive_ptr<::c10d::Store>& store, + int rank, + int size, + const std::chrono::duration& timeout); + + static void ProcessGroupTestConstructor() __attribute__((constructor)) { + py::object module = py::module::import("torch.distributed"); + py::object register_backend = module.attr("Backend").attr("register_backend"); + // The first parameter is the backend name used by user in invoking + // torch.distributed.init_process_group(). + // Note it could be different with module name. For example, the module + // name is "torch_test" but the backend name is "test". + // The second parameter is the instantiation function. + register_backend("test", py::cpp_function(createProcessGroupTest)); + } + +}; + +} // namespace c10d diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cpp_frontend_extension.cpp b/western_ai/usa/pytorch/test/cpp_extensions/cpp_frontend_extension.cpp new file mode 100644 index 0000000000000000000000000000000000000000..819dafd87aad5d35802ffff698782027a4fe18c4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cpp_frontend_extension.cpp @@ -0,0 +1,54 @@ +#include + +#include +#include + +struct Net : torch::nn::Cloneable { + Net(int64_t in, int64_t out) : in_(in), out_(out) { + reset(); + } + + void reset() override { + fc = register_module("fc", torch::nn::Linear(in_, out_)); + buffer = register_buffer("buf", torch::eye(5)); + } + + torch::Tensor forward(torch::Tensor x) { + return fc->forward(x); + } + + void set_bias(torch::Tensor bias) { + torch::NoGradGuard guard; + fc->bias.set_(bias); + } + + torch::Tensor get_bias() const { + return fc->bias; + } + + void add_new_parameter(const std::string& name, torch::Tensor tensor) { + register_parameter(name, tensor); + } + + void add_new_buffer(const std::string& name, torch::Tensor tensor) { + register_buffer(name, tensor); + } + + void add_new_submodule(const std::string& name) { + register_module(name, torch::nn::Linear(fc->options)); + } + + int64_t in_, out_; + torch::nn::Linear fc{nullptr}; + torch::Tensor buffer; +}; + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + torch::python::bind_module(m, "Net") + .def(py::init()) + .def("set_bias", &Net::set_bias) + .def("get_bias", &Net::get_bias) + .def("add_new_parameter", &Net::add_new_parameter) + .def("add_new_buffer", &Net::add_new_buffer) + .def("add_new_submodule", &Net::add_new_submodule); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cublas_extension.cpp b/western_ai/usa/pytorch/test/cpp_extensions/cublas_extension.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6bb01b338d15a0bfdc1d3d40d1d0e5c812d5a6fe --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cublas_extension.cpp @@ -0,0 +1,17 @@ +#include + +#include +#include + +#include + +torch::Tensor noop_cublas_function(torch::Tensor x) { + cublasHandle_t handle; + TORCH_CUDABLAS_CHECK(cublasCreate(&handle)); + TORCH_CUDABLAS_CHECK(cublasDestroy(handle)); + return x; +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("noop_cublas_function", &noop_cublas_function, "a cublas function"); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension.cpp b/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fcf9651627fb5fbd4bebe5d37081814170e03f50 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension.cpp @@ -0,0 +1,21 @@ +#include + +// Declare the function from cuda_dlink_extension.cu. +void add_cuda(const float* a, const float* b, float* output, int size); + +at::Tensor add(at::Tensor a, at::Tensor b) { + TORCH_CHECK(a.device().is_cuda(), "a is a cuda tensor"); + TORCH_CHECK(b.device().is_cuda(), "b is a cuda tensor"); + TORCH_CHECK(a.dtype() == at::kFloat, "a is a float tensor"); + TORCH_CHECK(b.dtype() == at::kFloat, "b is a float tensor"); + TORCH_CHECK(a.sizes() == b.sizes(), "a and b should have same size"); + + at::Tensor output = at::empty_like(a); + add_cuda(a.data_ptr(), b.data_ptr(), output.data_ptr(), a.numel()); + + return output; +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("add", &add, "a + b"); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension_add.cu b/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension_add.cu new file mode 100644 index 0000000000000000000000000000000000000000..b746242fd08979b5615454ec485cdc309069460f --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension_add.cu @@ -0,0 +1,6 @@ +#include +#include + +__device__ void add(const float* a, const float* b, float* output) { + *output = *a + *b; +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension_add.cuh b/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension_add.cuh new file mode 100644 index 0000000000000000000000000000000000000000..7c283de4c98967387a66abbde0ffd11e38f018c6 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension_add.cuh @@ -0,0 +1,6 @@ +#pragma once + +#include +#include + +__device__ void add(const float* a, const float* b, float* output); diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension_kernel.cu b/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..ba73d56873d162fae014ae18e96f2f3779341ea1 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cuda_dlink_extension_kernel.cu @@ -0,0 +1,22 @@ +#include +#include +#include + +#include + +#include "cuda_dlink_extension_add.cuh" + +__global__ void add_kernel(const float* a, const float* b, float* output, int size) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + add(a + i, b + i, output + i); + } +} + +// output = a * b + c +void add_cuda(const float* a, const float* b, float* output, int size) { + const int threads = 1024; + const int blocks = (size + threads - 1) / threads; + add_kernel<<>>(a, b, output, size); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension.cpp b/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension.cpp new file mode 100644 index 0000000000000000000000000000000000000000..351b9ba098f5110e047b9752a499843d54670ac0 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension.cpp @@ -0,0 +1,19 @@ +#include + +// Declare the function from cuda_extension.cu. It will be compiled +// separately with nvcc and linked with the object file of cuda_extension.cpp +// into one shared library. +void sigmoid_add_cuda(const float* x, const float* y, float* output, int size); + +torch::Tensor sigmoid_add(torch::Tensor x, torch::Tensor y) { + TORCH_CHECK(x.device().is_cuda(), "x must be a CUDA tensor"); + TORCH_CHECK(y.device().is_cuda(), "y must be a CUDA tensor"); + auto output = torch::zeros_like(x); + sigmoid_add_cuda( + x.data_ptr(), y.data_ptr(), output.data_ptr(), output.numel()); + return output; +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("sigmoid_add", &sigmoid_add, "sigmoid(x) + sigmoid(y)"); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension.cu b/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension.cu new file mode 100644 index 0000000000000000000000000000000000000000..8fde798d0b7439aca16cc9004bc215b71c890e0c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension.cu @@ -0,0 +1,31 @@ +// NOTE: This is a copy of cuda_extension_kernel.cu. It's kept here to test +// collision handling when a C++ file and CUDA file share the same filename. +// Setuptools can't deal with this at all, so the setup.py-based test uses +// cuda_extension_kernel.cu and the JIT test uses this file. Symlinks don't +// work well on Windows, so this is the most thorough solution right now. + +#include +#include +#include + +#include + +__global__ void sigmoid_add_kernel( + const float* __restrict__ x, + const float* __restrict__ y, + float* __restrict__ output, + const int size) { + const int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < size) { + const float sigmoid_x = 1.0f / (1.0f + __expf(-x[index])); + const float sigmoid_y = 1.0f / (1.0f + __expf(-y[index])); + output[index] = sigmoid_x + sigmoid_y; + } +} + +void sigmoid_add_cuda(const float* x, const float* y, float* output, int size) { + const int threads = 1024; + const int blocks = (size + threads - 1) / threads; + sigmoid_add_kernel<<>>(x, y, output, size); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension_kernel.cu b/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..d32ce397a4cc893befdf0463c9ccc625c5123196 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension_kernel.cu @@ -0,0 +1,25 @@ +#include +#include +#include + +#include + +__global__ void sigmoid_add_kernel( + const float* __restrict__ x, + const float* __restrict__ y, + float* __restrict__ output, + const int size) { + const int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < size) { + const float sigmoid_x = 1.0f / (1.0f + __expf(-x[index])); + const float sigmoid_y = 1.0f / (1.0f + __expf(-y[index])); + output[index] = sigmoid_x + sigmoid_y; + } +} + +void sigmoid_add_cuda(const float* x, const float* y, float* output, int size) { + const int threads = 1024; + const int blocks = (size + threads - 1) / threads; + sigmoid_add_kernel<<>>(x, y, output, size); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension_kernel2.cu b/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension_kernel2.cu new file mode 100644 index 0000000000000000000000000000000000000000..f831ff74750837bb09353e9054d010df2cfaea61 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cuda_extension_kernel2.cu @@ -0,0 +1,25 @@ +#include +#include +#include + +#include + +__global__ void tanh_add_kernel( + const float* __restrict__ x, + const float* __restrict__ y, + float* __restrict__ output, + const int size) { + const int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < size) { + const float tanh_x = 2.0f / (1.0f + __expf(-2.0f * x[index])) - 1; + const float tanh_y = 2.0f / (1.0f + __expf(-2.0f * y[index])) - 1; + output[index] = tanh_x + tanh_y; + } +} + +void tanh_add_cuda(const float* x, const float* y, float* output, int size) { + const int threads = 1024; + const int blocks = (size + threads - 1) / threads; + tanh_add_kernel<<>>(x, y, output, size); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cudnn_extension.cpp b/western_ai/usa/pytorch/test/cpp_extensions/cudnn_extension.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ddaa17d1262a2e2e64ec90ede7fdbc5d94e37847 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cudnn_extension.cpp @@ -0,0 +1,80 @@ +/* + * CuDNN ReLU extension. Simple function but contains the general structure of + * most CuDNN extensions: + * 1) Check arguments. torch::check* functions provide a standard way to + * validate input and provide pretty errors. 2) Create descriptors. Most CuDNN + * functions require creating and setting a variety of descriptors. 3) Apply the + * CuDNN function. 4) Destroy your descriptors. 5) Return something (optional). + */ + +#include + +#include // for CUDNN_CHECK +#include // for TensorDescriptor +#include // for getCudnnHandle + +// Name of function in python module and name used for error messages by +// torch::check* functions. +const char* cudnn_relu_name = "cudnn_relu"; + +// Check arguments to cudnn_relu +void cudnn_relu_check( + const torch::Tensor& inputs, + const torch::Tensor& outputs) { + // Create TensorArgs. These record the names and positions of each tensor as a + // parameter. + torch::TensorArg arg_inputs(inputs, "inputs", 0); + torch::TensorArg arg_outputs(outputs, "outputs", 1); + // Check arguments. No need to return anything. These functions with throw an + // error if they fail. Messages are populated using information from + // TensorArgs. + torch::checkContiguous(cudnn_relu_name, arg_inputs); + torch::checkScalarType(cudnn_relu_name, arg_inputs, torch::kFloat); + torch::checkBackend(cudnn_relu_name, arg_inputs.tensor, torch::Backend::CUDA); + torch::checkContiguous(cudnn_relu_name, arg_outputs); + torch::checkScalarType(cudnn_relu_name, arg_outputs, torch::kFloat); + torch::checkBackend( + cudnn_relu_name, arg_outputs.tensor, torch::Backend::CUDA); + torch::checkSameSize(cudnn_relu_name, arg_inputs, arg_outputs); +} + +void cudnn_relu(const torch::Tensor& inputs, const torch::Tensor& outputs) { + // Most CuDNN extensions will follow a similar pattern. + // Step 1: Check inputs. This will throw an error if inputs are invalid, so no + // need to check return codes here. + cudnn_relu_check(inputs, outputs); + // Step 2: Create descriptors + cudnnHandle_t cuDnn = torch::native::getCudnnHandle(); + // Note: 4 is minimum dim for a TensorDescriptor. Input and output are same + // size and type and contiguous, so one descriptor is sufficient. + torch::native::TensorDescriptor input_tensor_desc(inputs, 4); + cudnnActivationDescriptor_t activationDesc; + // Note: Always check return value of cudnn functions using CUDNN_CHECK + AT_CUDNN_CHECK(cudnnCreateActivationDescriptor(&activationDesc)); + AT_CUDNN_CHECK(cudnnSetActivationDescriptor( + activationDesc, + /*mode=*/CUDNN_ACTIVATION_RELU, + /*reluNanOpt=*/CUDNN_PROPAGATE_NAN, + /*coef=*/1.)); + // Step 3: Apply CuDNN function + float alpha = 1.; + float beta = 0.; + AT_CUDNN_CHECK(cudnnActivationForward( + cuDnn, + activationDesc, + &alpha, + input_tensor_desc.desc(), + inputs.data_ptr(), + &beta, + input_tensor_desc.desc(), // output descriptor same as input + outputs.data_ptr())); + // Step 4: Destroy descriptors + AT_CUDNN_CHECK(cudnnDestroyActivationDescriptor(activationDesc)); + // Step 5: Return something (optional) +} + +// Create the pybind11 module +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + // Use the same name as the check functions so error messages make sense + m.def(cudnn_relu_name, &cudnn_relu, "CuDNN ReLU"); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/cusolver_extension.cpp b/western_ai/usa/pytorch/test/cpp_extensions/cusolver_extension.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0a855c177f33aff03a21d596e021b603371061e3 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/cusolver_extension.cpp @@ -0,0 +1,17 @@ +#include +#include + +#include + + +torch::Tensor noop_cusolver_function(torch::Tensor x) { + cusolverDnHandle_t handle; + TORCH_CUSOLVER_CHECK(cusolverDnCreate(&handle)); + TORCH_CUSOLVER_CHECK(cusolverDnDestroy(handle)); + return x; +} + + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("noop_cusolver_function", &noop_cusolver_function, "a cusolver function"); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/dangling_impl_extension.cpp b/western_ai/usa/pytorch/test/cpp_extensions/dangling_impl_extension.cpp new file mode 100644 index 0000000000000000000000000000000000000000..466329dc07b94f5d104d9dfa8f008f6f16a0a341 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/dangling_impl_extension.cpp @@ -0,0 +1,11 @@ +#include + +void foo() { } + +TORCH_LIBRARY_IMPL(__test, CPU, m) { + m.impl("foo", foo); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("bar", foo); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/doubler.h b/western_ai/usa/pytorch/test/cpp_extensions/doubler.h new file mode 100644 index 0000000000000000000000000000000000000000..0a88df5b1b4e784c4684b0100da10b2bb86a1bac --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/doubler.h @@ -0,0 +1,17 @@ +#include + +struct Doubler { + Doubler(int A, int B) { + tensor_ = + torch::ones({A, B}, torch::dtype(torch::kFloat64).requires_grad(true)); + } + torch::Tensor forward() { + return tensor_ * 2; + } + torch::Tensor get() const { + return tensor_; + } + + private: + torch::Tensor tensor_; +}; diff --git a/western_ai/usa/pytorch/test/cpp_extensions/extension.cpp b/western_ai/usa/pytorch/test/cpp_extensions/extension.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1f29533e4667977a50183ab92ceef33b924177b2 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/extension.cpp @@ -0,0 +1,73 @@ +#include +#include + +// test include_dirs in setuptools.setup with relative path +#include +#include + +torch::Tensor sigmoid_add(torch::Tensor x, torch::Tensor y) { + return x.sigmoid() + y.sigmoid(); +} + +struct MatrixMultiplier { + MatrixMultiplier(int A, int B) { + tensor_ = + torch::ones({A, B}, torch::dtype(torch::kFloat64).requires_grad(true)); + } + torch::Tensor forward(torch::Tensor weights) { + return tensor_.mm(weights); + } + torch::Tensor get() const { + return tensor_; + } + + private: + torch::Tensor tensor_; +}; + +bool function_taking_optional(std::optional tensor) { + return tensor.has_value(); +} + +torch::Tensor random_tensor() { + return torch::randn({1}); +} + +at::ScalarType get_math_type(at::ScalarType other) { + return at::toOpMathType(other); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("sigmoid_add", &sigmoid_add, "sigmoid(x) + sigmoid(y)"); + m.def( + "function_taking_optional", + &function_taking_optional, + "function_taking_optional"); + py::class_(m, "MatrixMultiplier") + .def(py::init()) + .def("forward", &MatrixMultiplier::forward) + .def("get", &MatrixMultiplier::get); + + m.def("get_complex", []() { return c10::complex(1.0, 2.0); }); + m.def("get_device", []() { return at::device_of(random_tensor()).value(); }); + m.def("get_generator", []() { return at::detail::getDefaultCPUGenerator(); }); + // ArrayRef is a non-owning view; constructing it from a brace-init-list + // would point at a stack-temporary initializer_list and dangle once the + // lambda returns (caught by ASAN as stack-use-after-return). Back the + // values with static storage so the view stays valid. + m.def("get_intarrayref", []() { + static const int64_t data[] = {1, 2, 3}; + return at::IntArrayRef(data, 3); + }); + m.def("get_memory_format", []() { return c10::get_contiguous_memory_format(); }); + m.def("get_storage", []() { return random_tensor().storage(); }); + m.def("get_symfloat", []() { return c10::SymFloat(1.0); }); + m.def("get_symint", []() { return c10::SymInt(1); }); + m.def("get_symintarrayref", []() { + static const c10::SymInt data[] = {c10::SymInt(1), c10::SymInt(2), c10::SymInt(3)}; + return at::SymIntArrayRef(data, 3); + }); + m.def("get_tensor", []() { return random_tensor(); }); + m.def("get_math_type", &get_math_type); + m.def("roundtrip_layout", [](c10::Layout layout) -> c10::Layout { return layout; }); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/identity.cpp b/western_ai/usa/pytorch/test/cpp_extensions/identity.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c8894306c78cbc0cee8084b62facb005857695c2 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/identity.cpp @@ -0,0 +1,23 @@ +#include +#include + +using namespace torch::autograd; + +class Identity : public Function { + public: + static torch::Tensor forward(AutogradContext* ctx, torch::Tensor input) { + return input; + } + + static tensor_list backward(AutogradContext* ctx, tensor_list grad_outputs) { + return {grad_outputs[0]}; + } +}; + +torch::Tensor identity(torch::Tensor input) { + return Identity::apply(input); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("identity", &identity, "identity"); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/jit_extension.cpp b/western_ai/usa/pytorch/test/cpp_extensions/jit_extension.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6cb5534d25108eb2a58045a8d6760d37068c11da --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/jit_extension.cpp @@ -0,0 +1,20 @@ +#include + +#include "doubler.h" + +using namespace at; + +Tensor exp_add(Tensor x, Tensor y); + +Tensor tanh_add(Tensor x, Tensor y) { + return x.tanh() + y.tanh(); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("tanh_add", &tanh_add, "tanh(x) + tanh(y)"); + m.def("exp_add", &exp_add, "exp(x) + exp(y)"); + py::class_(m, "Doubler") + .def(py::init()) + .def("forward", &Doubler::forward) + .def("get", &Doubler::get); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/jit_extension2.cpp b/western_ai/usa/pytorch/test/cpp_extensions/jit_extension2.cpp new file mode 100644 index 0000000000000000000000000000000000000000..036cc5a4840d97c0440452b9bb071156036fa79e --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/jit_extension2.cpp @@ -0,0 +1,7 @@ +#include + +using namespace at; + +Tensor exp_add(Tensor x, Tensor y) { + return x.exp() + y.exp(); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/get_any_data_ptr.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/get_any_data_ptr.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fac8838037c3e6cd988368ee500505d15aa5b51f --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/get_any_data_ptr.cpp @@ -0,0 +1,20 @@ +#include +#include + +using torch::stable::Tensor; + +uint64_t get_any_data_ptr(Tensor t, bool mutable_) { + if (mutable_) { + return reinterpret_cast(t.mutable_data_ptr()); + } else { + return reinterpret_cast(t.const_data_ptr()); + } +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("get_any_data_ptr(Tensor t, bool mutable_) -> int"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("get_any_data_ptr", TORCH_BOX(&get_any_data_ptr)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/get_template_any_data_ptr.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/get_template_any_data_ptr.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1a66e7766724020a7430984b00f77d416ae64342 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/get_template_any_data_ptr.cpp @@ -0,0 +1,38 @@ +#include +#include +#include +#include + +using torch::stable::Tensor; + +uint64_t get_template_any_data_ptr(Tensor t, torch::headeronly::ScalarType dtype, bool mutable_) { +#define DEFINE_CASE(T, name) \ + case name: { \ + if (mutable_) { \ + return reinterpret_cast(t.mutable_data_ptr()); \ + } else { \ + return reinterpret_cast(t.const_data_ptr()); \ + } \ + } + switch (dtype) { + // per aten/src/ATen/templates/TensorMethods.cpp: + AT_FORALL_SCALAR_TYPES_V2( + AT_WRAP(DEFINE_CASE), + AT_EXPAND(AT_ALL_SCALAR_TYPES_WITH_COMPLEX), + torch::headeronly::ScalarType::UInt16, + torch::headeronly::ScalarType::UInt32, + torch::headeronly::ScalarType::UInt64 + ) + default: + return 0; + } +#undef DEFINE_CASE +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("get_template_any_data_ptr(Tensor t, ScalarType dtype, bool mutable_) -> int"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("get_template_any_data_ptr", TORCH_BOX(&get_template_any_data_ptr)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/make_tensor_clones_and_call_foreach.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/make_tensor_clones_and_call_foreach.cpp new file mode 100644 index 0000000000000000000000000000000000000000..584ecaa55f2aa5489fa89d0b3aad87fa5e36ea29 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/make_tensor_clones_and_call_foreach.cpp @@ -0,0 +1,36 @@ +#include +#include +#include + +#include + +using torch::stable::Tensor; + +// Declare my__foreach_mul (defined in my__foreach_mul.cpp) +extern std::vector my__foreach_mul( + const torch::headeronly::HeaderOnlyArrayRef& self, + torch::headeronly::HeaderOnlyArrayRef other); + +std::vector make_tensor_clones_and_call_foreach(Tensor t1, Tensor t2) { + // This function tests that my__foreach_mul can take in std::initializer_lists + // in addition to std::vectors. + Tensor t1_1 = clone(t1); + Tensor t1_2 = clone(t1); + Tensor t2_1 = clone(t2); + Tensor t2_2 = clone(t2); + return my__foreach_mul({t1_1, t2_1}, {t1_2, t2_2}); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def( + "make_tensor_clones_and_call_foreach(Tensor t1, Tensor t2) -> Tensor[]"); +} + +STABLE_TORCH_LIBRARY_IMPL( + STABLE_LIB_NAME, + CompositeExplicitAutograd, + m) { + m.impl( + "make_tensor_clones_and_call_foreach", + TORCH_BOX(&make_tensor_clones_and_call_foreach)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my__foreach_mul.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my__foreach_mul.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cf1798b2b81075ac2701d9bd7627a82086b091e4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my__foreach_mul.cpp @@ -0,0 +1,21 @@ +#include +#include +#include +#include + +using torch::stable::Tensor; + +// This is used to test const torch::headeronly::HeaderOnlyArrayRef& with TORCH_BOX +std::vector my__foreach_mul(const torch::headeronly::HeaderOnlyArrayRef& self, torch::headeronly::HeaderOnlyArrayRef other) { + std::array stack = {torch::stable::detail::from(self), torch::stable::detail::from(other)}; + aoti_torch_call_dispatcher("aten::_foreach_mul", "List", stack.data()); + return torch::stable::detail::to>(stack[0]); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my__foreach_mul(Tensor[] self, Tensor[] other) -> Tensor[]"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my__foreach_mul", TORCH_BOX(&my__foreach_mul)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my__foreach_mul_.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my__foreach_mul_.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e477840f9cb11d06d0390234edaba30a4e3b98c7 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my__foreach_mul_.cpp @@ -0,0 +1,19 @@ +#include +#include +#include +#include + +using torch::stable::Tensor; + +void my__foreach_mul_(torch::headeronly::HeaderOnlyArrayRef self, torch::headeronly::HeaderOnlyArrayRef other) { + std::array stack = {torch::stable::detail::from(self), torch::stable::detail::from(other)}; + aoti_torch_call_dispatcher("aten::_foreach_mul_", "List", stack.data()); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my__foreach_mul_(Tensor(a!)[] self, Tensor[] other) -> ()"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my__foreach_mul_", TORCH_BOX(&my__foreach_mul_)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my__foreach_mul_vec.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my__foreach_mul_vec.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4302c6b39a730e6d007406c0e05605c71f5e4cbe --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my__foreach_mul_vec.cpp @@ -0,0 +1,27 @@ +#include +#include +#include +#include + +using torch::stable::Tensor; + +// This is used to test const std::vector& with TORCH_BOX +std::vector my__foreach_mul_vec( + const std::vector& self, + const std::vector& other) { + std::array stack = { + torch::stable::detail::from(self), torch::stable::detail::from(other)}; + aoti_torch_call_dispatcher("aten::_foreach_mul", "List", stack.data()); + return torch::stable::detail::to>(stack[0]); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my__foreach_mul_vec(Tensor[] self, Tensor[] other) -> Tensor[]"); +} + +STABLE_TORCH_LIBRARY_IMPL( + STABLE_LIB_NAME, + CompositeExplicitAutograd, + m) { + m.impl("my__foreach_mul_vec", TORCH_BOX(&my__foreach_mul_vec)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_contiguous.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_contiguous.cpp new file mode 100644 index 0000000000000000000000000000000000000000..de9241fe0221d4f8f17d76e9e501a79209c0df4b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_contiguous.cpp @@ -0,0 +1,27 @@ +#include +#include +#include + +using torch::stable::Tensor; + +// Test contiguous with default memory format +Tensor my_contiguous(Tensor self) { + return torch::stable::contiguous(self); +} + +// Test contiguous with specified memory format +Tensor my_contiguous_memory_format( + Tensor self, + torch::headeronly::MemoryFormat memory_format) { + return torch::stable::contiguous(self, memory_format); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_contiguous(Tensor self) -> Tensor"); + m.def("my_contiguous_memory_format(Tensor self, MemoryFormat memory_format) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_contiguous", TORCH_BOX(&my_contiguous)); + m.impl("my_contiguous_memory_format", TORCH_BOX(&my_contiguous_memory_format)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_empty.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_empty.cpp new file mode 100644 index 0000000000000000000000000000000000000000..df80efdfb0a4d43949c65bf45bc0c80baaa53e26 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_empty.cpp @@ -0,0 +1,27 @@ +#include +#include +#include +#include + +#include + +using torch::stable::Tensor; + +Tensor my_empty( + torch::headeronly::HeaderOnlyArrayRef size, + std::optional dtype, + std::optional& layout, + const std::optional& device, + std::optional pin_memory, + std::optional memory_format) { + return empty(size, dtype, layout, device, pin_memory, memory_format); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def( + "my_empty(int[] size, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_empty", TORCH_BOX(&my_empty)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_from_blob.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_from_blob.cpp new file mode 100644 index 0000000000000000000000000000000000000000..32bb62cee46bf0b24759c24e8aabc707b54e2163 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_from_blob.cpp @@ -0,0 +1,32 @@ +#include +#include +#include +#include + +using torch::stable::Tensor; + +// Wrapper for torch::stable::from_blob with all parameters +// Note: We pass data_ptr as int64_t since we can't pass void* through the +// dispatcher +Tensor my_from_blob( + int64_t data_ptr, + torch::headeronly::HeaderOnlyArrayRef sizes, + torch::headeronly::HeaderOnlyArrayRef strides, + torch::stable::Device device, + torch::headeronly::ScalarType dtype) { + void* data = reinterpret_cast(data_ptr); + return torch::stable::from_blob( + data, sizes, strides, device, dtype); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def( + "my_from_blob(int data_ptr, int[] sizes, int[] strides, Device device, ScalarType dtype) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL( + STABLE_LIB_NAME, + CompositeExplicitAutograd, + m) { + m.impl("my_from_blob", TORCH_BOX(&my_from_blob)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_full.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_full.cpp new file mode 100644 index 0000000000000000000000000000000000000000..27c1922c0dc79324613e98bdab8b1fa97cf590ad --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_full.cpp @@ -0,0 +1,27 @@ +#include +#include +#include +#include + +#include + +using torch::stable::Tensor; + +Tensor my_full( + std::vector size, + double fill_value, + std::optional dtype, + std::optional layout, + std::optional device, + std::optional pin_memory) { + return torch::stable::full(size, fill_value, dtype, layout, device, pin_memory); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def( + "my_full(int[] size, float fill_value, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_full", TORCH_BOX(&my_full)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_new_empty_zeros.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_new_empty_zeros.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a973486a0e86a6a65cf131969023bf92f5cc6303 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_new_empty_zeros.cpp @@ -0,0 +1,40 @@ +#include +#include +#include + +using torch::stable::Device; +using torch::stable::Tensor; + +// Test new_empty with all kwargs +Tensor my_new_empty( + Tensor self, + torch::headeronly::IntHeaderOnlyArrayRef size, + std::optional dtype, + std::optional layout, + std::optional device, + std::optional pin_memory) { + return torch::stable::new_empty(self, size, dtype, layout, device, pin_memory); +} + +// Test new_zeros with all kwargs +Tensor my_new_zeros( + Tensor self, + torch::headeronly::IntHeaderOnlyArrayRef size, + std::optional dtype, + std::optional layout, + std::optional device, + std::optional pin_memory) { + return torch::stable::new_zeros(self, size, dtype, layout, device, pin_memory); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def( + "my_new_empty(Tensor self, int[] size, ScalarType? dtype, Layout? layout, Device? device, bool? pin_memory) -> Tensor"); + m.def( + "my_new_zeros(Tensor self, int[] size, ScalarType? dtype, Layout? layout, Device? device, bool? pin_memory) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_new_empty", TORCH_BOX(&my_new_empty)); + m.impl("my_new_zeros", TORCH_BOX(&my_new_zeros)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_reshape.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_reshape.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1250ea81471c2def3a621c807d370769eb188035 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_reshape.cpp @@ -0,0 +1,17 @@ +#include +#include +#include + +using torch::stable::Tensor; + +Tensor my_reshape(Tensor t, torch::headeronly::HeaderOnlyArrayRef shape) { + return reshape(t, shape); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_reshape(Tensor t, int[] shape) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_reshape", TORCH_BOX(&my_reshape)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_set_requires_grad.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_set_requires_grad.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1eb5f9d1fde01c9d7756599dc5280baf8c0d84e5 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_set_requires_grad.cpp @@ -0,0 +1,17 @@ +#include +#include + +using torch::stable::Tensor; + +Tensor my_set_requires_grad(Tensor t, bool requires_grad) { + t.set_requires_grad(requires_grad); + return t; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_set_requires_grad(Tensor t, bool requires_grad) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_set_requires_grad", TORCH_BOX(&my_set_requires_grad)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_shape.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_shape.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0f24ca9a56ec54c1d69871cb127abbb6fefd6335 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_shape.cpp @@ -0,0 +1,20 @@ +#include +#include +#include + +using torch::stable::Tensor; + +torch::headeronly::HeaderOnlyArrayRef my_shape(Tensor t) { + return t.sizes(); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_shape(Tensor t) -> int[]"); +} + +STABLE_TORCH_LIBRARY_IMPL( + STABLE_LIB_NAME, + CompositeExplicitAutograd, + m) { + m.impl("my_shape", TORCH_BOX(&my_shape)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_stable_error_check.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_stable_error_check.cpp new file mode 100644 index 0000000000000000000000000000000000000000..607bcd06a09ec8c6c8560586728c6c0854f892ac --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_stable_error_check.cpp @@ -0,0 +1,44 @@ +#include +#include +#include +#include + +using torch::stable::Tensor; + +// torch::stable::subtract using STABLE_TORCH_ERROR_CODE_CHECK (detailed error +// message via the stable ABI) instead of TORCH_ERROR_CODE_CHECK (simple message). +// +// This lives in the 2.10 extension on purpose: built at a target older than 2.13, +// STABLE_TORCH_ERROR_CODE_CHECK reaches torch_exception_get_what* through a +// runtime symbol lookup (TORCH_DYNAMIC_VERSION_CALL), so it exercises the dynamic +// version call path. Built at 2.13+ the same op takes the direct-call shortcut. +inline torch::stable::Tensor our_subtract_stable_error_check( + const torch::stable::Tensor& self, + const torch::stable::Tensor& other, + double alpha = 1.0) { + AtenTensorHandle ret0; + STABLE_TORCH_ERROR_CODE_CHECK( + aoti_torch_aten_subtract_Tensor(self.get(), other.get(), alpha, &ret0)); + return torch::stable::Tensor(ret0); +} + +// Same op but with the less detailed TORCH_ERROR_CODE_CHECK, for comparison. +inline torch::stable::Tensor our_subtract_torch_error_check( + const torch::stable::Tensor& self, + const torch::stable::Tensor& other, + double alpha = 1.0) { + AtenTensorHandle ret0; + TORCH_ERROR_CODE_CHECK( + aoti_torch_aten_subtract_Tensor(self.get(), other.get(), alpha, &ret0)); + return torch::stable::Tensor(ret0); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("our_subtract_stable_error_check(Tensor self, Tensor other, float alpha=1.0) -> Tensor"); + m.def("our_subtract_torch_error_check(Tensor self, Tensor other, float alpha=1.0) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("our_subtract_stable_error_check", TORCH_BOX(&our_subtract_stable_error_check)); + m.impl("our_subtract_torch_error_check", TORCH_BOX(&our_subtract_torch_error_check)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_string_op.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_string_op.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8e107fba0fc8d620f6439eb550588793f8a3fa71 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_string_op.cpp @@ -0,0 +1,33 @@ +#include +#include +#include + +#include +#include +#include + +using torch::stable::Tensor; + +std::tuple, int64_t> my_string_op(Tensor t, std::string_view accessor, std::string passthru) { + int64_t res; + if (accessor == "dim") { + res = t.dim(); + } else if (accessor == "size") { + res = t.size(0); + } else if (accessor == "stride") { + res = t.stride(0); + } else { + STD_TORCH_CHECK(false, "Unsupported accessor value: ", std::string(accessor).c_str()) + } + + auto vec = std::vector({std::string(accessor), std::to_string(res), passthru}); + return std::make_tuple(std::move(vec), res); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_string_op(Tensor t, str accessor, str passthru) -> (str[], int)"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_string_op", TORCH_BOX(&my_string_op)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_string_op_variants.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_string_op_variants.cpp new file mode 100644 index 0000000000000000000000000000000000000000..807d18387bfaf9256e11a74d257be066bae578c9 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_string_op_variants.cpp @@ -0,0 +1,65 @@ +// This file is intended to test (const) std::string& and const std::string_view& arguments with TORCH_BOX +#include +#include +#include + +#include +#include +#include + +using torch::stable::Tensor; + +// Helper function to process accessor +static int64_t process_accessor(Tensor t, std::string_view accessor) { + if (accessor == "dim") { + return t.dim(); + } else if (accessor == "size") { + return t.size(0); + } else if (accessor == "stride") { + return t.stride(0); + } else { + STD_TORCH_CHECK(false, "Unsupported accessor value: ", std::string(accessor).c_str()) + } +} + +// Test const std::string& +std::tuple, int64_t> my_string_op_const_string_ref( + Tensor t, + const std::string& accessor, + const std::string& passthru) { + int64_t res = process_accessor(t, accessor); + auto vec = std::vector({accessor, std::to_string(res), passthru}); + return std::make_tuple(std::move(vec), res); +} + +// Test const std::string_view& +std::tuple, int64_t> my_string_op_const_string_view_ref( + Tensor t, + const std::string_view& accessor, + const std::string_view& passthru) { + int64_t res = process_accessor(t, accessor); + auto vec = std::vector({std::string(accessor), std::to_string(res), std::string(passthru)}); + return std::make_tuple(std::move(vec), res); +} + +// Test std::string& (non-const) +std::tuple, int64_t> my_string_op_string_ref( + Tensor t, + std::string& accessor, + std::string& passthru) { + int64_t res = process_accessor(t, accessor); + auto vec = std::vector({accessor, std::to_string(res), passthru}); + return std::make_tuple(std::move(vec), res); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_string_op_const_string_ref(Tensor t, str accessor, str passthru) -> (str[], int)"); + m.def("my_string_op_const_string_view_ref(Tensor t, str accessor, str passthru) -> (str[], int)"); + m.def("my_string_op_string_ref(Tensor t, str accessor, str passthru) -> (str[], int)"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_string_op_const_string_ref", TORCH_BOX(&my_string_op_const_string_ref)); + m.impl("my_string_op_const_string_view_ref", TORCH_BOX(&my_string_op_const_string_view_ref)); + m.impl("my_string_op_string_ref", TORCH_BOX(&my_string_op_string_ref)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_subtract.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_subtract.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7049b4e8705c8b85056dedd8247f7dab30b28382 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_subtract.cpp @@ -0,0 +1,17 @@ +#include +#include +#include + +using torch::stable::Tensor; + +Tensor my_subtract(const Tensor& self, const Tensor& other, double alpha) { + return torch::stable::subtract(self, other, alpha); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_subtract(Tensor self, Tensor other, float alpha=1.0) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_subtract", TORCH_BOX(&my_subtract)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_sum.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_sum.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3b3881ce1cd5006d23de3c9e55d2af8369c7da6a --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_sum.cpp @@ -0,0 +1,45 @@ +#include +#include +#include + +using torch::stable::Tensor; + +Tensor my_sum( + Tensor self, + std::optional> dim, + bool keepdim, + std::optional dtype) { + return torch::stable::sum(self, dim, keepdim, dtype); +} + +// Tests that sum(t) works (independent from the STABLE_TORCH_LIBRARY +// registration which passes a default) +Tensor my_sum_all(Tensor self) { + return torch::stable::sum(self); +} + +// Test op that takes only a tensor and passes [1] as dim +// (sums along dimension 1) +Tensor my_sum_dim1(Tensor self) { + return torch::stable::sum( + self, + std::make_optional(torch::headeronly::IntHeaderOnlyArrayRef({1})), + false, + std::nullopt); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def( + "my_sum(Tensor self, int[]? dim=None, bool keepdim=False, ScalarType? dtype=None) -> Tensor"); + m.def("my_sum_all(Tensor self) -> Tensor"); + m.def("my_sum_dim1(Tensor self) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL( + STABLE_LIB_NAME, + CompositeExplicitAutograd, + m) { + m.impl("my_sum", TORCH_BOX(&my_sum)); + m.impl("my_sum_all", TORCH_BOX(&my_sum_all)); + m.impl("my_sum_dim1", TORCH_BOX(&my_sum_dim1)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_sum_out.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_sum_out.cpp new file mode 100644 index 0000000000000000000000000000000000000000..61f8b815d8f31b48dd57fa4f9a9dcc980dce4e89 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_sum_out.cpp @@ -0,0 +1,25 @@ +#include +#include +#include + +using torch::stable::Tensor; + +Tensor my_sum_out( + Tensor out, + Tensor self, + std::optional> dim, + bool keepdim = false, + std::optional dtype = std::nullopt) { + return sum_out(out, self, dim, keepdim, dtype); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_sum_out(Tensor(a!) out, Tensor self, int[]? dim=None, bool keepdim=False, ScalarType? dtype=None) -> Tensor(a!)"); +} + +STABLE_TORCH_LIBRARY_IMPL( + STABLE_LIB_NAME, + CompositeExplicitAutograd, + m) { + m.impl("my_sum_out", TORCH_BOX(&my_sum_out)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_to.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_to.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b3b4ca2b3891fb8ec2cd5c2a32dc14994d2fb04d --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_to.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +using torch::stable::Device; +using torch::stable::Tensor; + +// Test to(device) convenience overload +Tensor my_to_device(Tensor self, Device device) { + return torch::stable::to(self, device); +} + +// Test to(dtype) +Tensor my_to_dtype(Tensor self, torch::headeronly::ScalarType dtype) { + return torch::stable::to(self, dtype); +} + +// Test the full to.dtype_layout op with all parameters +Tensor my_to_dtype_layout( + Tensor self, + std::optional dtype, + std::optional layout, + std::optional device, + std::optional pin_memory, + bool non_blocking, + bool copy, + std::optional memory_format) { + return torch::stable::to( + self, + dtype, + layout, + device, + pin_memory, + non_blocking, + copy, + memory_format); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_to_device(Tensor self, Device device) -> Tensor"); + m.def("my_to_dtype(Tensor self, ScalarType dtype) -> Tensor"); + m.def( + "my_to_dtype_layout(Tensor self, ScalarType? dtype, Layout? layout, Device? device, bool? pin_memory, bool non_blocking, bool copy, MemoryFormat? memory_format) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL( + STABLE_LIB_NAME, + CompositeExplicitAutograd, + m) { + m.impl("my_to_device", TORCH_BOX(&my_to_device)); + m.impl("my_to_dtype", TORCH_BOX(&my_to_dtype)); + m.impl("my_to_dtype_layout", TORCH_BOX(&my_to_dtype_layout)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_view.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_view.cpp new file mode 100644 index 0000000000000000000000000000000000000000..547a9d7816a34de674c79105b3cad262fd2cd6eb --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/my_view.cpp @@ -0,0 +1,20 @@ +#include +#include +#include + +using torch::stable::Tensor; + +Tensor my_view(Tensor t, torch::headeronly::HeaderOnlyArrayRef size) { + return view(t, size); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_view(Tensor t, int[] size) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL( + STABLE_LIB_NAME, + CompositeExplicitAutograd, + m) { + m.impl("my_view", TORCH_BOX(&my_view)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_cublas_handle.cu b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_cublas_handle.cu new file mode 100644 index 0000000000000000000000000000000000000000..c1ad29607501e9ffc6aea305a780bb8a74732baa --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_cublas_handle.cu @@ -0,0 +1,16 @@ +#include +#include + +void* my_get_curr_cuda_blas_handle() { + void* ret_handle; + TORCH_ERROR_CODE_CHECK(torch_get_current_cuda_blas_handle(&ret_handle)); + return ret_handle; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_get_curr_cuda_blas_handle() -> int"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_get_curr_cuda_blas_handle", TORCH_BOX(&my_get_curr_cuda_blas_handle)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_cuda_stream.cu b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_cuda_stream.cu new file mode 100644 index 0000000000000000000000000000000000000000..39e0beea3c08bcdcfa9c074e651f165993f17a1f --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_cuda_stream.cu @@ -0,0 +1,36 @@ +#include +#include + +void* my_get_current_cuda_stream(int32_t device_index) { + void* ret_stream; + TORCH_ERROR_CODE_CHECK(aoti_torch_get_current_cuda_stream(device_index, &ret_stream)); + return ret_stream; +} + +void my_set_current_cuda_stream(void* stream, int32_t device_index) { + TORCH_ERROR_CODE_CHECK(torch_set_current_cuda_stream(stream, device_index)); +} + +void* my_get_cuda_stream_from_pool(bool isHighPriority, int32_t device_index) { + void* ret_stream; + TORCH_ERROR_CODE_CHECK(torch_get_cuda_stream_from_pool(isHighPriority, device_index, &ret_stream)); + return ret_stream; +} + +void my_cuda_stream_synchronize(void* stream, int32_t device_index) { + TORCH_ERROR_CODE_CHECK(torch_cuda_stream_synchronize(stream, device_index)); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_get_current_cuda_stream(int device_index) -> int"); + m.def("my_set_current_cuda_stream(int stream, int device_index) -> ()"); + m.def("my_get_cuda_stream_from_pool(bool isHighPriority, int device_index) -> int"); + m.def("my_cuda_stream_synchronize(int stream, int device_index) -> ()"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_get_current_cuda_stream", TORCH_BOX(&my_get_current_cuda_stream)); + m.impl("my_set_current_cuda_stream", TORCH_BOX(&my_set_current_cuda_stream)); + m.impl("my_get_cuda_stream_from_pool", TORCH_BOX(&my_get_cuda_stream_from_pool)); + m.impl("my_cuda_stream_synchronize", TORCH_BOX(&my_cuda_stream_synchronize)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_constructor.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_constructor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..43f45d80dbaa056d325fbcb8fde9ffcf7d611c0b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_constructor.cpp @@ -0,0 +1,37 @@ +#include +#include + +#include + +torch::stable::Device test_device_constructor( + bool is_cuda, + torch::stable::DeviceIndex index, + bool use_str) { + using torch::stable::Device; + using torch::stable::DeviceType; + + if (use_str) { + std::string device_str; + if (is_cuda) { + device_str = "cuda:" + std::to_string(index); + } else { + device_str = "cpu"; + } + return Device(device_str); + } else { + if (is_cuda) { + return Device(DeviceType::CUDA, index); + } else { + return Device(DeviceType::CPU); + } + } +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def( + "test_device_constructor(bool is_cuda, DeviceIndex index, bool use_str) -> Device"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("test_device_constructor", TORCH_BOX(&test_device_constructor)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_equality.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_equality.cpp new file mode 100644 index 0000000000000000000000000000000000000000..db92d65e8be78dfb296d476f34c8c7a55b17695c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_equality.cpp @@ -0,0 +1,14 @@ +#include +#include + +bool test_device_equality(torch::stable::Device d1, torch::stable::Device d2) { + return d1 == d2; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_device_equality(Device d1, Device d2) -> bool"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("test_device_equality", TORCH_BOX(&test_device_equality)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_index.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_index.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a298c7b51a7b1cb0089514f6cef6186760d86611 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_index.cpp @@ -0,0 +1,14 @@ +#include +#include + +torch::stable::DeviceIndex test_device_index(torch::stable::Device device) { + return device.index(); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_device_index(Device device) -> DeviceIndex"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("test_device_index", TORCH_BOX(&test_device_index)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_is_cpu.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_is_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ced61ff17826483a3c2b7e3b235d9a3a79f05f61 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_is_cpu.cpp @@ -0,0 +1,15 @@ +#include +#include + +// This is used to test torch::stable::Device& with TORCH_BOX +bool test_device_is_cpu(torch::stable::Device& device) { + return device.is_cpu(); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_device_is_cpu(Device device) -> bool"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("test_device_is_cpu", TORCH_BOX(&test_device_is_cpu)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_is_cuda.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_is_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b11c479f3ce5dc94719d9ad7aba4b0989e4ec783 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_is_cuda.cpp @@ -0,0 +1,15 @@ +#include +#include + +// This is used to test const torch::stable::Device& with TORCH_BOX +bool test_device_is_cuda(const torch::stable::Device& device) { + return device.is_cuda(); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_device_is_cuda(Device device) -> bool"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("test_device_is_cuda", TORCH_BOX(&test_device_is_cuda)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_set_index.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_set_index.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8ba69dc2156412c1a5d9329852c69fa651e2d837 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_device_set_index.cpp @@ -0,0 +1,17 @@ +#include +#include + +torch::stable::Device test_device_set_index( + torch::stable::Device device, + torch::stable::DeviceIndex index) { + device.set_index(index); + return device; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_device_set_index(Device device, DeviceIndex index) -> Device"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("test_device_set_index", TORCH_BOX(&test_device_set_index)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_get_num_threads.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_get_num_threads.cpp new file mode 100644 index 0000000000000000000000000000000000000000..842731408e255a34075da87d0e01277be2c9b547 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_get_num_threads.cpp @@ -0,0 +1,14 @@ +#include +#include + +uint32_t test_get_num_threads() { + return torch::stable::get_num_threads(); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_get_num_threads() -> int"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("test_get_num_threads", TORCH_BOX(&test_get_num_threads)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_parallel_for.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_parallel_for.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4333e4ed0448535ceae4c3ec4a1d705542895bc6 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_parallel_for.cpp @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include +#include + +using torch::stable::Tensor; + +Tensor test_parallel_for(int64_t size, int64_t grain_size) { + AtenTensorHandle tensor_handle; + int64_t stride = 1; + + aoti_torch_empty_strided( + 1, + &size, + &stride, + aoti_torch_dtype_int64(), + aoti_torch_device_type_cpu(), + 0, + &tensor_handle); + + Tensor tensor(tensor_handle); + int64_t* data_ptr = reinterpret_cast(tensor.data_ptr()); + + torch::stable::zero_(tensor); + + // Use parallel_for to fill each element with its index + // If using a parallel path, the thread id is encoded in the upper 32 bits + torch::stable::parallel_for( + 0, size, grain_size, [data_ptr](int64_t begin, int64_t end) { + for (auto i = begin; i < end; i++) { + STD_TORCH_CHECK(i <= UINT32_MAX); + uint32_t thread_id; + torch_get_thread_idx(&thread_id); + data_ptr[i] = i | (static_cast(thread_id) << 32); + } + }); + + return tensor; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_parallel_for(int size, int grain_size) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("test_parallel_for", TORCH_BOX(&test_parallel_for)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_std_cuda_check.cu b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_std_cuda_check.cu new file mode 100644 index 0000000000000000000000000000000000000000..f434dccc88223cb048d1baf6b4cdbcbae9505080 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_std_cuda_check.cu @@ -0,0 +1,63 @@ +#include +#include +#include + +__global__ void dummy_kernel(int /*unused*/) { + // Intentionally empty +} + +__global__ void invalid_kernel(int /*unused*/) { + // This kernel itself is fine, but we'll launch it with invalid config +} + +int test_std_cuda_check_success() { + // cudaGetDevice should succeed if CUDA is available + int device; + STD_CUDA_CHECK(cudaGetDevice(&device)); + return device; +} + +void test_std_cuda_check_error() { + // cudaSetDevice with an invalid device ID should fail + // Using 99999 as an invalid device ID to trigger an error + STD_CUDA_CHECK(cudaSetDevice(99999)); +} + +void test_std_cuda_kernel_launch_check_success() { + // Launch a simple kernel with valid configuration + dummy_kernel<<<1, 1>>>(0); + + STD_CUDA_KERNEL_LAUNCH_CHECK(); +} + +void test_std_cuda_kernel_launch_check_error() { + // Launch a kernel with invalid configuration + // Using more blocks than allowed (2^31) will trigger a launch error + // HIP does not error on invalid grid size, but errors on invalid block size + + invalid_kernel<<<2147483648, 2048>>>(0); + + STD_CUDA_KERNEL_LAUNCH_CHECK(); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_std_cuda_check_success() -> int"); + m.def("test_std_cuda_check_error() -> ()"); + m.def("test_std_cuda_kernel_launch_check_success() -> ()"); + m.def("test_std_cuda_kernel_launch_check_error() -> ()"); +} + +STABLE_TORCH_LIBRARY_IMPL( + STABLE_LIB_NAME, + CompositeExplicitAutograd, + m) { + m.impl( + "test_std_cuda_check_success", TORCH_BOX(&test_std_cuda_check_success)); + m.impl("test_std_cuda_check_error", TORCH_BOX(&test_std_cuda_check_error)); + m.impl( + "test_std_cuda_kernel_launch_check_success", + TORCH_BOX(&test_std_cuda_kernel_launch_check_success)); + m.impl( + "test_std_cuda_kernel_launch_check_error", + TORCH_BOX(&test_std_cuda_kernel_launch_check_error)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_tensor_device.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_tensor_device.cpp new file mode 100644 index 0000000000000000000000000000000000000000..602acef96c49e7848151cbcb10a3243ece40a46b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/csrc/test_tensor_device.cpp @@ -0,0 +1,17 @@ +#include +#include +#include + +using torch::stable::Tensor; + +torch::stable::Device test_tensor_device(torch::stable::Tensor tensor) { + return tensor.device(); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_tensor_device(Tensor t) -> Device"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("test_tensor_device", TORCH_BOX(&test_tensor_device)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/libtorch_agn_2_10/__init__.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/libtorch_agn_2_10/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b0ee6e97df5744e2ae3253016dce5324eb4484a4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/libtorch_agn_2_10/__init__.py @@ -0,0 +1,25 @@ +import ctypes +import sys +from pathlib import Path + +import torch + + +so_files = list( + Path(__file__).parent.glob("_C*" + (".pyd" if sys.platform == "win32" else ".so")) +) +if len(so_files) != 1: + raise AssertionError(f"Expected one _C*.{{so,pyd}} file, found {len(so_files)}") + +# use ctypes.CDLL instead of load_library to be able to test the unload logic +# below code is reduced from the load_library code +with torch._ops.dl_open_guard(): + loaded_lib = ctypes.CDLL(str(so_files[0])) + +from . import ops + + +__all__ = [ + "loaded_lib", + "ops", +] diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/libtorch_agn_2_10/ops.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/libtorch_agn_2_10/ops.py new file mode 100644 index 0000000000000000000000000000000000000000..85bd69710910f712e6a4ccf00e65e482b5c26391 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/libtorch_agn_2_10/ops.py @@ -0,0 +1,706 @@ +import torch +from torch import Tensor + + +def my__foreach_mul_(tensors, others) -> (): + """ + Updates tensors to be the result of pointwise multiplying with others. + + Args: + tensors: list of tensors + others: list of tensors (with the same corresponding shapes as tensors) + + Returns: nothing, tensors is updated in place. + """ + torch.ops.libtorch_agn_2_10.my__foreach_mul_.default(tensors, others) + + +def my__foreach_mul(tensors, others) -> list[Tensor]: + """ + Returns a list of tensors that are the results of pointwise multiplying + tensors and others. + + Args: + tensors: list of tensors + others: list of tensors (with the same corresponding shapes as tensors) + + Returns: list of multiplied tensors + """ + return torch.ops.libtorch_agn_2_10.my__foreach_mul.default(tensors, others) + + +def make_tensor_clones_and_call_foreach(t1, t2) -> list[Tensor]: + """ + Returns a list of 2 tensors corresponding to the square of the inputs. + + Args: + t1: Tensor + t2: Tensor + + Returns: list of [t1^2, t2^2] + """ + return torch.ops.libtorch_agn_2_10.make_tensor_clones_and_call_foreach.default( + t1, t2 + ) + + +def test_tensor_device(t): + """ + Tests Tensor device() method. + + Args: + t: Tensor - tensor to get device from + + Returns: Device - device of the tensor + """ + return torch.ops.libtorch_agn_2_10.test_tensor_device.default(t) + + +def test_device_constructor(is_cuda, index, use_str): + """ + Tests creating a Device from DeviceType and index, or from a string. + + Args: + is_cuda: bool - if True, creates CUDA device; if False, creates CPU device + index: int - device index + use_str: bool - if True, constructs from string; if False, constructs from DeviceType + + Returns: Device - A device with the specified type and index + """ + return torch.ops.libtorch_agn_2_10.test_device_constructor.default( + is_cuda, index, use_str + ) + + +def test_device_equality(d1, d2) -> bool: + """ + Tests Device equality operator. + + Args: + d1: Device - first device + d2: Device - second device + + Returns: bool - True if devices are equal + """ + return torch.ops.libtorch_agn_2_10.test_device_equality.default(d1, d2) + + +def test_device_set_index(device, index): + """ + Tests Device set_index() method. + + Args: + device: Device - device to modify + index: int - new device index + + Returns: Device - device with updated index + """ + return torch.ops.libtorch_agn_2_10.test_device_set_index.default(device, index) + + +def test_device_index(device) -> int: + """ + Tests Device index() method. + + Args: + device: Device - device to query + + Returns: int - device index + """ + return torch.ops.libtorch_agn_2_10.test_device_index.default(device) + + +def test_device_is_cuda(device) -> bool: + """ + Tests Device is_cuda() method. + + Args: + device: Device - device to check + + Returns: bool - True if device is CUDA + """ + return torch.ops.libtorch_agn_2_10.test_device_is_cuda.default(device) + + +def test_device_is_cpu(device) -> bool: + """ + Tests Device is_cpu() method. + + Args: + device: Device - device to check + + Returns: bool - True if device is CPU + """ + return torch.ops.libtorch_agn_2_10.test_device_is_cpu.default(device) + + +def test_parallel_for(size, grain_size) -> Tensor: + """ + Tests the parallel_for functionality by using it to fill a tensor with indices. + Args: + size: int - size of the tensor to create + grain_size: int - grain size for parallel_for + Returns: Tensor - a 1D int64 tensor where each element contains its index + (if multiple threads are used the threadid will be encoded in the upper 32 bits) + """ + return torch.ops.libtorch_agn_2_10.test_parallel_for.default(size, grain_size) + + +def test_get_num_threads() -> int: + """ + Tests the get_num_threads functionality by returning the number of threads + for the parallel backend. + + Returns: int - the number of threads for the parallel backend + """ + return torch.ops.libtorch_agn_2_10.test_get_num_threads.default() + + +def my_empty( + size, dtype=None, layout=None, device=None, pin_memory=None, memory_format=None +) -> Tensor: + """ + Creates an empty tensor with the specified size, dtype, layout, device, pin_memory, and memory_format. + + Args: + size: list[int] - size of the tensor to create + dtype: ScalarType or None - data type of the tensor + layout: Layout or None - layout of the tensor + device: Device or None - device on which to create the tensor + pin_memory: bool or None - whether to use pinned memory + memory_format: MemoryFormat or None - memory format of the tensor + + Returns: Tensor - an uninitialized tensor with the specified properties + """ + return torch.ops.libtorch_agn_2_10.my_empty.default( + size, dtype, layout, device, pin_memory, memory_format + ) + + +def my_reshape(t, shape) -> Tensor: + """ + Returns a tensor with the same data but different shape. + + Args: + t: Tensor - tensor to reshape + shape: list[int] - new shape for the tensor + + Returns: Tensor - reshaped tensor + """ + return torch.ops.libtorch_agn_2_10.my_reshape.default(t, shape) + + +def my_view(t, size) -> Tensor: + """ + Returns a new tensor with the same data as the input tensor but of a different shape. + + Args: + t: Tensor - tensor to view + size: list[int] - new size for the tensor + + Returns: Tensor - tensor with new view + """ + return torch.ops.libtorch_agn_2_10.my_view.default(t, size) + + +def my_shape(t) -> tuple[int]: + """ + Returns a shape of the input tensor. + + Args: + t: Tensor - input tensor + + Returns: tuple - shape of the input tensor. + """ + return torch.ops.libtorch_agn_2_10.my_shape.default(t) + + +def get_any_data_ptr(t, mutable) -> int: + """ + Return data pointer value of the tensor. + Args: + t: Input tensor + mutable: whether data pointer qualifier is mutable or const + Returns: int - pointer value + """ + return torch.ops.libtorch_agn_2_10.get_any_data_ptr.default(t, mutable) + + +def get_template_any_data_ptr(t, dtype, mutable) -> int: + """ + Return data pointer value of the tensor iff it has dtype. + Args: + t: Input tensor + dtype: Input dtype + mutable: whether data pointer qualifier is mutable or const + Returns: int - pointer value + Raises RuntimeError when t.dtype() != dtype. + """ + return torch.ops.libtorch_agn_2_10.get_template_any_data_ptr.default( + t, dtype, mutable + ) + + +def my_get_curr_cuda_blas_handle() -> int: + """ + Return the current cuBlasHandle_t pointer value. + """ + return torch.ops.libtorch_agn_2_10.my_get_curr_cuda_blas_handle.default() + + +def my_string_op(t, accessor, passthru) -> tuple[list[str], int]: + """ + The purpose of this op is to test inputting and outputting strings in a + stable custom op. This particular op takes in a Tensor, a string denoting + which tensor metadata API to call, and a pass through string to return a + string list and the value of the tensor metadata. + + If accessor is "size" or "stride", query along the 0th dim. + + Args: + t: Tensor - input tensor to query + accessor: str - which property to access ("dim", "size", or "stride") + passthru: str - a string that gets returned as the last element of the list + + Returns: tuple - (list of [accessor, value, passthru] as strings, value) + """ + return torch.ops.libtorch_agn_2_10.my_string_op.default(t, accessor, passthru) + + +def my_get_current_cuda_stream(device_index: int) -> int: + """ + Return the current cudaStream_t pointer value. + + Args: + device_index: int - device index + """ + return torch.ops.libtorch_agn_2_10.my_get_current_cuda_stream.default(device_index) + + +def my_set_current_cuda_stream(stream: int, device_index: int): + """ + Set the current stream to cudaStream_t pointer value. + + Args: + stream: int - cudaStream_t pointer value + device_index: int - device index + """ + return torch.ops.libtorch_agn_2_10.my_set_current_cuda_stream.default( + stream, device_index + ) + + +def my_get_cuda_stream_from_pool(high_priority: bool, device_index: int) -> int: + """ + Return the cudaStream_t pointer value from pool. + + Args: + high_priority: bool - if true, return a stream with high priority + device_index: int - device index + """ + return torch.ops.libtorch_agn_2_10.my_get_cuda_stream_from_pool.default( + high_priority, device_index + ) + + +def my_cuda_stream_synchronize(stream: int, device_index: int): + """ + Synchronize cuda stream. + + Args: + stream: int - cudaStream_t pointer value + device_index: int - device index + """ + return torch.ops.libtorch_agn_2_10.my_cuda_stream_synchronize(stream, device_index) + + +def my_from_blob(data_ptr, sizes, strides, device, dtype) -> Tensor: + """ + Creates a Tensor from existing memory using torch::stable::from_blob. + + Args: + data_ptr: int - pointer to the data buffer + sizes: tuple[int] - size of the tensor + strides: tuple[int] - strides of the tensor + device: Device - device on which the tensor resides + dtype: ScalarType - data type of the tensor + storage_offset: int - offset in the storage + layout: Layout - layout of the tensor + + Returns: Tensor - tensor wrapping the existing memory + """ + return torch.ops.libtorch_agn_2_10.my_from_blob.default( + data_ptr, sizes, strides, device, dtype + ) + + +def test_std_cuda_check_success() -> int: + """ + Test STD_CUDA_CHECK macro with a successful CUDA operation. + Returns the current CUDA device index. + """ + return torch.ops.libtorch_agn_2_10.test_std_cuda_check_success.default() + + +def test_std_cuda_check_error() -> None: + """ + Test STD_CUDA_CHECK macro with a failing CUDA operation. + This should raise a RuntimeError with the CUDA error message. + """ + torch.ops.libtorch_agn_2_10.test_std_cuda_check_error.default() + + +def test_std_cuda_kernel_launch_check_success() -> None: + """ + Test STD_CUDA_KERNEL_LAUNCH_CHECK macro with a successful kernel launch. + Launches a simple kernel and checks for errors. + """ + torch.ops.libtorch_agn_2_10.test_std_cuda_kernel_launch_check_success.default() + + +def test_std_cuda_kernel_launch_check_error() -> None: + """ + Test STD_CUDA_KERNEL_LAUNCH_CHECK macro with an invalid kernel launch. + This should raise a RuntimeError with the CUDA kernel launch error message. + """ + torch.ops.libtorch_agn_2_10.test_std_cuda_kernel_launch_check_error.default() + + +def my__foreach_mul_vec(tensors, others) -> list[Tensor]: + """ + Returns a list of tensors that are the results of pointwise multiplying + tensors and others. This variant tests const std::vector& parameters. + + Args: + tensors: list of tensors + others: list of tensors (with the same corresponding shapes as tensors) + + Returns: list of multiplied tensors + """ + return torch.ops.libtorch_agn_2_10.my__foreach_mul_vec.default(tensors, others) + + +def my_string_op_const_string_ref(t, accessor, passthru) -> tuple[list[str], int]: + """ + Tests TORCH_BOX with const std::string& parameters. + + Args: + t: Tensor - input tensor to query + accessor: str - which property to access ("dim", "size", or "stride") + passthru: str - a string that gets returned as the last element of the list + + Returns: tuple - (list of [accessor, value, passthru] as strings, value) + """ + return torch.ops.libtorch_agn_2_10.my_string_op_const_string_ref.default( + t, accessor, passthru + ) + + +def my_string_op_const_string_view_ref(t, accessor, passthru) -> tuple[list[str], int]: + """ + Tests TORCH_BOX with const std::string_view& parameters. + + Args: + t: Tensor - input tensor to query + accessor: str - which property to access ("dim", "size", or "stride") + passthru: str - a string that gets returned as the last element of the list + + Returns: tuple - (list of [accessor, value, passthru] as strings, value) + """ + return torch.ops.libtorch_agn_2_10.my_string_op_const_string_view_ref.default( + t, accessor, passthru + ) + + +def my_string_op_string_ref(t, accessor, passthru) -> tuple[list[str], int]: + """ + Tests TORCH_BOX with std::string& (non-const) parameters. + + Args: + t: Tensor - input tensor to query + accessor: str - which property to access ("dim", "size", or "stride") + passthru: str - a string that gets returned as the last element of the list + + Returns: tuple - (list of [accessor, value, passthru] as strings, value) + """ + return torch.ops.libtorch_agn_2_10.my_string_op_string_ref.default( + t, accessor, passthru + ) + + +def my_set_requires_grad(t, requires_grad) -> Tensor: + """ + Sets the requires_grad attribute on a tensor. + + Args: + t: Tensor - input tensor + requires_grad: bool - whether the tensor requires gradient + + Returns: Tensor - the input tensor with requires_grad set + """ + return torch.ops.libtorch_agn_2_10.my_set_requires_grad.default(t, requires_grad) + + +def my_to_device(t, device) -> Tensor: + """ + Moves a tensor to the specified device. + + Args: + t: Tensor - input tensor + device: Device - target device + + Returns: Tensor - tensor on the new device + """ + return torch.ops.libtorch_agn_2_10.my_to_device.default(t, device) + + +def my_to_dtype(t, dtype) -> Tensor: + """ + Converts a tensor to the specified dtype. + + Args: + t: Tensor - input tensor + dtype: ScalarType - target dtype (e.g., torch.float64) + + Returns: Tensor - tensor with the new dtype + """ + return torch.ops.libtorch_agn_2_10.my_to_dtype.default(t, dtype) + + +def my_contiguous(t) -> Tensor: + """ + Returns a contiguous tensor with the default memory format. + + Args: + t: Tensor - input tensor + + Returns: Tensor - contiguous tensor + """ + return torch.ops.libtorch_agn_2_10.my_contiguous.default(t) + + +def my_contiguous_memory_format(t, memory_format) -> Tensor: + """ + Returns a contiguous tensor with the specified memory format. + + Args: + t: Tensor - input tensor + memory_format: MemoryFormat - memory format (e.g., torch.channels_last) + + Returns: Tensor - contiguous tensor with specified memory format + """ + return torch.ops.libtorch_agn_2_10.my_contiguous_memory_format.default( + t, memory_format + ) + + +def my_to_dtype_layout( + t, + dtype=None, + layout=None, + device=None, + pin_memory=None, + non_blocking=False, + copy=False, + memory_format=None, +) -> Tensor: + """ + Converts a tensor to the specified dtype, layout, device, and/or memory format. + + Args: + t: Tensor - input tensor + dtype: ScalarType or None - target dtype (e.g., torch.float64) + layout: Layout or None - target layout (e.g., torch.strided) + device: Device or None - target device + pin_memory: bool or None - whether to use pinned memory + non_blocking: bool - if True, try to perform the operation asynchronously + copy: bool - if True, always copy the tensor + memory_format: MemoryFormat or None - target memory format (e.g., torch.channels_last) + + Returns: Tensor - converted tensor + """ + return torch.ops.libtorch_agn_2_10.my_to_dtype_layout.default( + t, dtype, layout, device, pin_memory, non_blocking, copy, memory_format + ) + + +def my_new_empty( + self, + size, + dtype=None, + layout=None, + device=None, + pin_memory=None, +) -> Tensor: + """ + Creates a new uninitialized tensor with the specified size and properties. + + Args: + self: Tensor - input tensor (used to infer default properties) + size: list[int] - size of the new tensor + dtype: ScalarType or None - data type of the new tensor + layout: Layout or None - layout of the new tensor + device: Device or None - device for the new tensor + pin_memory: bool or None - whether to use pinned memory + + Returns: Tensor - new uninitialized tensor + """ + return torch.ops.libtorch_agn_2_10.my_new_empty.default( + self, size, dtype, layout, device, pin_memory + ) + + +def my_new_zeros( + self, + size, + dtype=None, + layout=None, + device=None, + pin_memory=None, +) -> Tensor: + """ + Creates a new tensor filled with zeros with the specified size and properties. + + Args: + self: Tensor - input tensor (used to infer default properties) + size: list[int] - size of the new tensor + dtype: ScalarType or None - data type of the new tensor + layout: Layout or None - layout of the new tensor + device: Device or None - device for the new tensor + pin_memory: bool or None - whether to use pinned memory + + Returns: Tensor - new tensor filled with zeros + """ + return torch.ops.libtorch_agn_2_10.my_new_zeros.default( + self, size, dtype, layout, device, pin_memory + ) + + +def my_sum(self, dim=None, keepdim=False, dtype=None) -> Tensor: + """ + Returns the sum of the tensor along the specified dimensions. + + Args: + self: Tensor - input tensor + dim: list[int] - dimensions to sum over + keepdim: bool - whether to keep the reduced dimensions + dtype: ScalarType or None - the desired data type of returned tensor + + Returns: Tensor - the sum of the tensor along the specified dimensions + """ + return torch.ops.libtorch_agn_2_10.my_sum.default(self, dim, keepdim, dtype) + + +def my_sum_out(out, self, dim=None, keepdim=False, dtype=None) -> Tensor: + """ + Computes the sum of the tensor along the specified dimensions and writes + the result to the output tensor. + + Args: + out: Tensor - output tensor (modified in-place) + self: Tensor - input tensor + dim: list[int] - dimensions to sum over + keepdim: bool - whether to keep the reduced dimensions + dtype: ScalarType or None - the desired data type of returned tensor + + Returns: Tensor - the output tensor (same as out parameter) + """ + return torch.ops.libtorch_agn_2_10.my_sum_out.default( + out, self, dim, keepdim, dtype + ) + + +def my_sum_all(self) -> Tensor: + """ + Returns the sum of all elements in the tensor (reduces to a scalar). + + Args: + self: Tensor - input tensor + + Returns: Tensor - scalar tensor containing the sum of all elements + """ + return torch.ops.libtorch_agn_2_10.my_sum_all.default(self) + + +def my_sum_dim1(self) -> Tensor: + """ + Returns the sum of the tensor along dimension 1. + + Args: + self: Tensor - input tensor + + Returns: Tensor - the sum of the tensor along dimension 1 + """ + return torch.ops.libtorch_agn_2_10.my_sum_dim1.default(self) + + +def my_full( + size, fill_value, dtype=None, layout=None, device=None, pin_memory=None +) -> Tensor: + """ + Creates a tensor filled with a scalar value. + + Args: + size: list[int] - size of the tensor to create + fill_value: float - value to fill the tensor with + dtype: ScalarType or None - data type of the tensor + layout: Layout or None - layout of the tensor + device: Device or None - device on which to create the tensor + pin_memory: bool or None - whether to use pinned memory + + Returns: Tensor - a tensor filled with fill_value + """ + return torch.ops.libtorch_agn_2_10.my_full.default( + size, fill_value, dtype, layout, device, pin_memory + ) + + +def my_subtract(self, other, alpha=1.0) -> Tensor: + """ + Subtracts other from self, scaled by alpha. + + Computes: self - alpha * other + + Args: + self: Tensor - input tensor + other: Tensor - tensor to subtract + alpha: float - scaling factor for other (default: 1.0) + + Returns: Tensor - result of subtraction + """ + return torch.ops.libtorch_agn_2_10.my_subtract.default(self, other, alpha) + + +# ============================================================================= +# Proxy for inherited ops (from libtorch_agn_2_9_extension/csrc/) +# +# Ops compiled from previous versions' csrc directories are accessible via +# the module-level __getattr__. For example: +# libtorch_agn_2_10.ops.sgd_out_of_place(...) +# libtorch_agn_2_10.ops.identity(...) +# ============================================================================= + +_NAMESPACE = "libtorch_agn_2_10" + + +def __getattr__(name): + """Proxy for inherited ops from previous versions.""" + if name.startswith("_"): + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + ops_namespace = getattr(torch.ops, _NAMESPACE) + op = getattr(ops_namespace, name, None) + if op is None: + raise AttributeError(f"No op named '{name}' in {_NAMESPACE}") + return op.default + + +def __dir__(): + """List all available ops (native + inherited).""" + native = [ + name + for name in globals() + if not name.startswith("_") and callable(globals().get(name)) + ] + ops_namespace = getattr(torch.ops, _NAMESPACE) + inherited = [n for n in dir(ops_namespace) if not n.startswith("_")] + return sorted(set(native + inherited)) diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/setup.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..3c7fa81ae0de4a8d09d52b2946ff8e360e90f8eb --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/setup.py @@ -0,0 +1,101 @@ +import distutils.command.clean +import shutil +from pathlib import Path + +from setuptools import find_packages, setup + +import torch +from torch.utils.cpp_extension import ( + BuildExtension, + CppExtension, + CUDAExtension, + IS_WINDOWS, +) + + +ROOT_DIR = Path(__file__).parent +CSRC_DIR = ROOT_DIR / "csrc" + +# Include csrc from previous versions for forward compatibility testing +PREV_CSRC_DIRS = [ + ROOT_DIR.parent / "libtorch_agn_2_9_extension" / "csrc", +] + + +class clean(distutils.command.clean.clean): + def run(self): + # Run default behavior first + distutils.command.clean.clean.run(self) + + # Remove extension + for path in (ROOT_DIR / "libtorch_agn_2_10").glob("**/*.so"): + path.unlink() + # Remove build and dist and egg-info directories + dirs = [ + ROOT_DIR / "build", + ROOT_DIR / "dist", + ROOT_DIR / "libtorch_agn_2_10.egg-info", + ] + for path in dirs: + if path.exists(): + shutil.rmtree(str(path), ignore_errors=True) + + +def get_extension(): + extra_compile_args = { + "cxx": [ + "-DTORCH_TARGET_VERSION=0x020a000000000000", + "-DSTABLE_LIB_NAME=libtorch_agn_2_10", + ], + } + if not IS_WINDOWS: + extra_compile_args["cxx"].append("-fdiagnostics-color=always") + + # Collect sources from this version and all previous versions + sources = list(CSRC_DIR.glob("**/*.cpp")) + for prev_dir in PREV_CSRC_DIRS: + sources.extend(prev_dir.glob("**/*.cpp")) + + extension = CppExtension + # allow including + if torch.cuda.is_available(): + extra_compile_args["cxx"].append("-DLAE_USE_CUDA") + extra_compile_args["nvcc"] = [ + "-O2", + "-DUSE_CUDA", + "-DTORCH_TARGET_VERSION=0x020a000000000000", + "-DSTABLE_LIB_NAME=libtorch_agn_2_10", + ] + extension = CUDAExtension + sources.extend(CSRC_DIR.glob("**/*.cu")) + for prev_dir in PREV_CSRC_DIRS: + sources.extend(prev_dir.glob("**/*.cu")) + + return [ + extension( + "libtorch_agn_2_10._C", + sources=sorted(str(s) for s in sources), + py_limited_api=True, + extra_compile_args=extra_compile_args, + extra_link_args=[], + ) + ] + + +setup( + name="libtorch_agn_2_10", + version="0.0", + author="PyTorch Core Team", + description="Example of libtorch agnostic extension for PyTorch 2.10+", + packages=find_packages(exclude=("test",)), + package_data={"libtorch_agn_2_10": ["*.dll", "*.dylib", "*.so"]}, + install_requires=[ + "torch", + ], + ext_modules=get_extension(), + cmdclass={ + "build_ext": BuildExtension.with_options(no_python_abi_suffix=True), + "clean": clean, + }, + options={"bdist_wheel": {"py_limited_api": "cp39"}}, +) diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/test_version_compatibility.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/test_version_compatibility.py new file mode 100644 index 0000000000000000000000000000000000000000..065951ab3bf6f2db75f4c5ddbb05452494537158 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_10_extension/test_version_compatibility.py @@ -0,0 +1,316 @@ +# Owner(s): ["module: cpp"] + +""" +Unit tests to verify that each function file requires PyTorch 2.10+. + +This test suite compiles each .cpp file in the csrc directory with +TORCH_TARGET_VERSION=2.9.0 and expects compilation to fail. +If compilation succeeds, it means that either + +(1) The test function works with 2.9.0 and should not be in this directory. +(2) The test function tests APIs that do not have proper TORCH_FEATURE_VERSION + guards. If this is the case, and you incorrectly move the test function into + libtorch_agn_2_9_extension the libtorch_agnostic_targetting CI workflow + will catch this. + +Run this script with VERSION_COMPAT_DEBUG=1 to see compilation errors. +""" + +import os +import subprocess +import tempfile +from pathlib import Path + +from torch.testing._internal.common_utils import IS_WINDOWS, run_tests, TestCase +from torch.utils.cpp_extension import ( + CUDA_HOME, + get_cxx_compiler, + include_paths as torch_include_paths, + ROCM_HOME, +) + + +GPU_HOME = CUDA_HOME or ROCM_HOME + +# TODO: Fix this error in Windows: +# numba.cuda.cudadrv.driver:driver.py:384 Call to cuInit results in CUDA_ERROR_NO_DEVICE +if not IS_WINDOWS: + + class FunctionVersionCompatibilityTest(TestCase): + """Test that all function files require PyTorch 2.10+.""" + + @classmethod + def setUpClass(cls): + """Set up test environment once for all tests.""" + cls.csrc_dir = Path(__file__).parent / "libtorch_agn_2_10" / "csrc" + cls.build_dir = Path(tempfile.mkdtemp(prefix="version_check_")) + + cls.pytorch_includes = [ + f"-I{path}" for path in torch_include_paths(device_type="cpu") + ] + cls.cuda_includes = [] + if GPU_HOME: + cuda_include_path = os.path.join(GPU_HOME, "include") + if os.path.exists(cuda_include_path): + cls.cuda_includes = [f"-I{cuda_include_path}"] + + cls.cuda_available = cls._check_cuda_available() + + @classmethod + def tearDownClass(cls): + """Clean up build directory.""" + import shutil + + if cls.build_dir.exists(): + shutil.rmtree(cls.build_dir) + + @staticmethod + def _check_cuda_available() -> bool: + """Check if CUDA is available.""" + try: + import torch + + return torch.cuda.is_available() + except ImportError: + return False + + def _compile_cpp_file( + self, source_file: Path, output_file: Path + ) -> tuple[bool, str]: + """ + Compile a C++ file with TORCH_TARGET_VERSION=2.9.0. + Returns (success, error_message). + """ + torch_version_2_9 = "0x0209000000000000" + + cmd = [ + get_cxx_compiler(), + "-c", + "-std=c++20", + f"-DTORCH_TARGET_VERSION={torch_version_2_9}", + f"-I{source_file.parent}", # For includes in same directory + *self.pytorch_includes, + ] + + # Add CUDA flags if available + if self.cuda_available: + cmd.extend(self.cuda_includes) + + cmd.extend([str(source_file), "-o", str(output_file)]) + + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + + if result.returncode == 0: + return True, "" + else: + return False, result.stderr + + def _compile_cu_file( + self, source_file: Path, output_file: Path + ) -> tuple[bool, str]: + """ + Compile a CUDA file with TORCH_TARGET_VERSION=2.9.0. + Returns (success, error_message). + """ + if not GPU_HOME: + return False, "one of CUDA_HOME and ROCM_HOME should be set but is not" + + torch_version_2_9 = "0x0209000000000000" + + cmd = [ + os.path.join(GPU_HOME, "bin", "nvcc" if CUDA_HOME else "hipcc"), + "-c", + "-std=c++20", + f"-DTORCH_TARGET_VERSION={torch_version_2_9}", + f"-I{source_file.parent}", # For includes in same directory + *self.pytorch_includes, + *self.cuda_includes, + ] + + cmd.extend(["-DUSE_CUDA"]) + if ROCM_HOME: + cmd.extend(["-DUSE_ROCM=1"]) + + cmd.extend([str(source_file), "-o", str(output_file)]) + + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + + if result.returncode == 0: + return True, "" + else: + return False, result.stderr + + def _test_function_file(self, source_file: Path): + """Test that a function file fails to compile with TORCH_TARGET_VERSION=2.9.0.""" + func_name = source_file.stem + obj_file = self.build_dir / f"{func_name}.o" + + # Choose the appropriate compiler based on file extension + if source_file.suffix == ".cu": + if not self.cuda_available: + self.skipTest(f"CUDA not available, skipping {source_file.name}") + success, error_msg = self._compile_cu_file(source_file, obj_file) + else: + success, error_msg = self._compile_cpp_file(source_file, obj_file) + + obj_file.unlink(missing_ok=True) + + # Print error details for debugging + if not success: + relevant_errors = self._extract_relevant_errors(error_msg) + if relevant_errors: + print(f"\n Compilation errors for {func_name} (requires 2.10+):") + for err in relevant_errors: + print(f" {err}") + + self.assertFalse( + success, + f"Function {func_name} compiled successfully with TORCH_TARGET_VERSION=2.9.0. " + f"This could mean two things.\n\t1. It should run with 2.9.0 and should be " + "moved to libtorch_agn_2_9_extension\n\t2. The function(s) it tests do not use the " + "proper TORCH_FEATURE_VERSION guards\n\nThe libtorch_agnostic_targetting CI workflow will " + "verify if you incorrectly move this to the 2_9 extension instead of adding " + "the appropriate version guards.", + ) + + def test_mv_tensor_accessor_cpu_works_with_2_9(self): + """Test that mv_tensor_accessor_cpu.cpp compiles successfully with 2.9.0. + + This is a negative test - it ensures that a file we expect to work with 2.9.0 + actually does compile. This validates that our test infrastructure correctly + distinguishes between files that require 2.10+ and those that don't. + """ + cpp_file = ( + Path(self.csrc_dir).parent + / "libtorch_agn_2_9_extension" + / "csrc" + / "mv_tensor_accessor_cpu.cpp" + ) + + if not cpp_file.exists(): + self.skipTest(f"{cpp_file} not found - this is a test file only") + + obj_file = self.build_dir / "mv_tensor_accessor_cpu.o" + success, error_msg = self._compile_cpp_file(cpp_file, obj_file) + + # Clean up + obj_file.unlink(missing_ok=True) + + if not success: + relevant_errors = self._extract_relevant_errors(error_msg) + if relevant_errors: + print( + "\n Unexpected compilation errors for mv_tensor_accessor_cpu:" + ) + for err in relevant_errors: + print(f"{err}") + + self.assertTrue( + success, + f"mv_tensor_accessor_cpu.cpp failed to compile with TORCH_TARGET_VERSION=2.9.0. " + f"This file is expected to work with 2.9.0 since it doesn't use 2.10+ features. " + f"Error: {error_msg}", + ) + + def test_cuda_kernel_works_with_2_9(self): + """Test that cuda_kernel.cu compiles successfully with 2.9.0. + + This is a negative test - it ensures that a .cu file we expect to work with 2.9.0 + actually does compile. This validates that our test infrastructure correctly + compiles CUDA files and distinguishes between files that require 2.10+ and those + that don't. + """ + if not self.cuda_available: + self.skipTest("CUDA not available, skipping cuda_kernel.cu test") + + cu_file = ( + Path(self.csrc_dir).parent + / "libtorch_agn_2_9_extension" + / "csrc" + / "cuda_kernel.cu" + ) + + if not cu_file.exists(): + self.skipTest(f"{cu_file} not found - this is a test file only") + + obj_file = self.build_dir / "cuda_kernel.o" + success, error_msg = self._compile_cu_file(cu_file, obj_file) + + # Clean up + obj_file.unlink(missing_ok=True) + + if not success: + relevant_errors = self._extract_relevant_errors(error_msg) + if relevant_errors: + print("\n Unexpected compilation errors for cuda_kernel.cu:") + for err in relevant_errors: + print(f"{err}") + + self.assertTrue( + success, + f"cuda_kernel.cu failed to compile with TORCH_TARGET_VERSION=2.9.0. " + f"This file is expected to work with 2.9.0 since it doesn't use 2.10+ features. " + f"Error: {error_msg}", + ) + + @staticmethod + def _extract_relevant_errors(error_msg: str) -> list[str]: + """Extract the most relevant error messages.""" + error_lines = error_msg.strip().split("\n") + relevant_errors = [] + + for line in error_lines: + line_lower = line.lower() + if ( + "error:" in line_lower + or "undefined" in line_lower + or "undeclared" in line_lower + or "no member named" in line_lower + ): + relevant_errors.append(line.strip()) + + return relevant_errors + + # Dynamically create test methods for each .cpp and .cu file + + def _create_test_method_for_file(source_file: Path): + """Create a test method for a specific source file.""" + + def test_method_impl(self): + self._test_function_file(source_file) + + # Set a descriptive name and docstring + func_name = source_file.stem + file_ext = source_file.suffix + test_method_impl.__name__ = f"test_{func_name}_requires_2_10" + test_method_impl.__doc__ = ( + f"Test that {func_name}{file_ext} requires PyTorch 2.10+" + ) + + return test_method_impl + + # Test discovery: generate a test for each .cpp and .cu file + _csrc_dir = Path(__file__).parent / "csrc" + if not _csrc_dir.exists(): + raise AssertionError(f"Expected csrc directory to exist at {_csrc_dir}") + # Collect both .cpp and .cu files, excluding those used for negative test + # already defined above + _source_files = sorted( + [f for f in _csrc_dir.rglob("*.cpp") if f.name != "mv_tensor_accessor_cpu.cpp"] + + list(_csrc_dir.rglob("*.cu")) + ) + + for _source_file in _source_files: + _test_method = _create_test_method_for_file(_source_file) + setattr(FunctionVersionCompatibilityTest, _test_method.__name__, _test_method) + + del ( + _create_test_method_for_file, + _csrc_dir, + _source_files, + _source_file, + _test_method, + ) + +if __name__ == "__main__": + run_tests() diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/csrc/my_from_blob_with_deleter.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/csrc/my_from_blob_with_deleter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9ee4784d36a392d811e05474f7322b3814152a26 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/csrc/my_from_blob_with_deleter.cpp @@ -0,0 +1,100 @@ +#include +#include +#include +#include + +#ifdef LAE_USE_CUDA +#include +#endif + +using torch::stable::Tensor; + +// Global counter to track deleter calls for testing +static int64_t g_deleter_call_count = 0; + +static void test_deleter(void* /*data*/) { + g_deleter_call_count++; +} + +// Wrapper for from_blob with deleter - uses a test deleter that increments +// a global counter +Tensor my_from_blob_with_deleter( + int64_t data_ptr, + torch::headeronly::HeaderOnlyArrayRef sizes, + torch::headeronly::HeaderOnlyArrayRef strides, + torch::stable::Device device, + torch::headeronly::ScalarType dtype) { + void* data = reinterpret_cast(data_ptr); + return torch::stable::from_blob( + data, sizes, strides, device, dtype, test_deleter); +} + +int64_t get_deleter_call_count() { + return g_deleter_call_count; +} + +void reset_deleter_call_count() { + g_deleter_call_count = 0; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def( + "my_from_blob_with_deleter(int data_ptr, int[] sizes, int[] strides, Device device, ScalarType dtype) -> Tensor"); + m.def("get_deleter_call_count() -> int"); + m.def("reset_deleter_call_count() -> ()"); +} + +STABLE_TORCH_LIBRARY_IMPL( + STABLE_LIB_NAME, + CompositeExplicitAutograd, + m) { + m.impl("my_from_blob_with_deleter", TORCH_BOX(&my_from_blob_with_deleter)); + m.impl("get_deleter_call_count", TORCH_BOX(&get_deleter_call_count)); + m.impl("reset_deleter_call_count", TORCH_BOX(&reset_deleter_call_count)); +} + +#ifdef LAE_USE_CUDA + +// Wrapper for cudaFree since it returns cudaError_t, not void +static void cuda_deleter(void* data) { + cudaFree(data); +} + +// Creates a tensor that owns its CUDA memory via cudaMalloc. +// When the tensor is destroyed, the deleter will call cudaFree. +// This tests that from_blob's deleter properly frees memory. +Tensor my_from_blob_with_cuda_deleter( + int64_t numel, + torch::stable::Device device) { + size_t size_bytes = numel * sizeof(float); + + void* data = nullptr; + cudaError_t err = cudaMalloc(&data, size_bytes); + if (err != cudaSuccess) { + throw std::runtime_error("cudaMalloc failed"); + } + + // Zero the memory + cudaMemset(data, 0, size_bytes); + + std::array sizes = {numel}; + std::array strides = {1}; + + return torch::stable::from_blob( + data, + torch::headeronly::HeaderOnlyArrayRef(sizes.data(), sizes.size()), + torch::headeronly::HeaderOnlyArrayRef(strides.data(), strides.size()), + device, + torch::headeronly::ScalarType::Float, + cuda_deleter); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_from_blob_with_cuda_deleter(int numel, Device device) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_from_blob_with_cuda_deleter", TORCH_BOX(&my_from_blob_with_cuda_deleter)); +} + +#endif // LAE_USE_CUDA diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/csrc/my_from_blob_with_lambda_deleter.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/csrc/my_from_blob_with_lambda_deleter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b9f674e5fd47aa0b11f31e8fd6d4fd07115761f4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/csrc/my_from_blob_with_lambda_deleter.cpp @@ -0,0 +1,101 @@ +#include +#include +#include +#include + +#ifdef LAE_USE_CUDA +#include +#endif + +using torch::stable::Tensor; + +// Global counter to track lambda deleter calls for testing +static int64_t g_lambda_deleter_call_count = 0; + +// Wrapper for from_blob with a capturing-lambda deleter. +// The lambda captures a pointer to the global counter and increments it, +// which exercises the capturing-lambda code path in torch_from_blob. +Tensor my_from_blob_with_lambda_deleter( + int64_t data_ptr, + torch::headeronly::HeaderOnlyArrayRef sizes, + torch::headeronly::HeaderOnlyArrayRef strides, + torch::stable::Device device, + torch::headeronly::ScalarType dtype) { + void* data = reinterpret_cast(data_ptr); + int64_t* counter = &g_lambda_deleter_call_count; + auto deleter = [counter](void* /*data*/) { (*counter)++; }; + return torch::stable::from_blob(data, sizes, strides, device, dtype, deleter); +} + +int64_t get_lambda_deleter_call_count() { + return g_lambda_deleter_call_count; +} + +void reset_lambda_deleter_call_count() { + g_lambda_deleter_call_count = 0; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def( + "my_from_blob_with_lambda_deleter(int data_ptr, int[] sizes, int[] strides, Device device, ScalarType dtype) -> Tensor"); + m.def("get_lambda_deleter_call_count() -> int"); + m.def("reset_lambda_deleter_call_count() -> ()"); +} + +STABLE_TORCH_LIBRARY_IMPL( + STABLE_LIB_NAME, + CompositeExplicitAutograd, + m) { + m.impl( + "my_from_blob_with_lambda_deleter", + TORCH_BOX(&my_from_blob_with_lambda_deleter)); + m.impl( + "get_lambda_deleter_call_count", + TORCH_BOX(&get_lambda_deleter_call_count)); + m.impl( + "reset_lambda_deleter_call_count", + TORCH_BOX(&reset_lambda_deleter_call_count)); +} + +#ifdef LAE_USE_CUDA + +// Same as my_from_blob_with_cuda_deleter (from 2.11) but uses a non-capturing +// lambda deleter. +Tensor my_from_blob_with_cuda_lambda_deleter( + int64_t numel, + torch::stable::Device device) { + size_t size_bytes = numel * sizeof(float); + + void* data = nullptr; + cudaError_t err = cudaMalloc(&data, size_bytes); + if (err != cudaSuccess) { + throw std::runtime_error("cudaMalloc failed"); + } + + // Zero the memory + cudaMemset(data, 0, size_bytes); + + std::array sizes = {numel}; + std::array strides = {1}; + + // This lambda doesn't capture anything, but capture is tested above in + // my_from_blob_with_lambda_deleter + auto deleter = [](void* data) { cudaFree(data); }; + return torch::stable::from_blob( + data, + torch::headeronly::HeaderOnlyArrayRef(sizes.data(), sizes.size()), + torch::headeronly::HeaderOnlyArrayRef(strides.data(), strides.size()), + device, + torch::headeronly::ScalarType::Float, + deleter); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_from_blob_with_cuda_lambda_deleter(int numel, Device device) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_from_blob_with_cuda_lambda_deleter", TORCH_BOX(&my_from_blob_with_cuda_lambda_deleter)); +} + +#endif // LAE_USE_CUDA diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/libtorch_agn_2_11/__init__.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/libtorch_agn_2_11/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b0ee6e97df5744e2ae3253016dce5324eb4484a4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/libtorch_agn_2_11/__init__.py @@ -0,0 +1,25 @@ +import ctypes +import sys +from pathlib import Path + +import torch + + +so_files = list( + Path(__file__).parent.glob("_C*" + (".pyd" if sys.platform == "win32" else ".so")) +) +if len(so_files) != 1: + raise AssertionError(f"Expected one _C*.{{so,pyd}} file, found {len(so_files)}") + +# use ctypes.CDLL instead of load_library to be able to test the unload logic +# below code is reduced from the load_library code +with torch._ops.dl_open_guard(): + loaded_lib = ctypes.CDLL(str(so_files[0])) + +from . import ops + + +__all__ = [ + "loaded_lib", + "ops", +] diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/libtorch_agn_2_11/ops.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/libtorch_agn_2_11/ops.py new file mode 100644 index 0000000000000000000000000000000000000000..effdb2fca70fa0bba403c45c775073a18ea15a2b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/libtorch_agn_2_11/ops.py @@ -0,0 +1,145 @@ +import torch +from torch import Tensor + + +def my_from_blob_with_deleter(data_ptr, sizes, strides, device, dtype) -> Tensor: + """ + Creates a Tensor from existing memory with a deleter callback. + + The deleter will be called when the tensor's storage is deallocated. For + this test, the deleter just updates a global call count, which allows us to + assert that is was called from get_deleter_call_count(). + + Args: + data_ptr: int - pointer to the data buffer + sizes: tuple[int] - size of the tensor + strides: tuple[int] - strides of the tensor + device: Device - device on which the tensor resides + dtype: ScalarType - data type of the tensor + + Returns: Tensor - tensor wrapping the existing memory + """ + return torch.ops.libtorch_agn_2_11.my_from_blob_with_deleter.default( + data_ptr, sizes, strides, device, dtype + ) + + +def get_deleter_call_count() -> int: + """ + Returns the number of times the test deleter has been called. + """ + return torch.ops.libtorch_agn_2_11.get_deleter_call_count.default() + + +def reset_deleter_call_count() -> None: + """ + Resets the deleter call counter to zero. + """ + torch.ops.libtorch_agn_2_11.reset_deleter_call_count.default() + + +def my_from_blob_with_cuda_deleter(numel: int, device) -> Tensor: + """ + Creates a CUDA tensor that owns its memory via cudaMalloc. + + The tensor's memory is allocated with cudaMalloc and will be freed + with cudaFree when the tensor is destroyed (via from_blob's deleter). + This is useful for testing that the deleter properly frees memory. + + Args: + numel: int - number of elements in the tensor + device: Device - CUDA device + + Returns: Tensor - a 1D float32 tensor of zeros + """ + return torch.ops.libtorch_agn_2_11.my_from_blob_with_cuda_deleter.default( + numel, device + ) + + +def my_from_blob_with_lambda_deleter(data_ptr, sizes, strides, device, dtype) -> Tensor: + """ + Creates a Tensor from existing memory with a capturing-lambda deleter. + + The deleter is a capturing lambda that updates a global call count, + exercising the capturing-lambda code path in torch_from_blob. + + Args: + data_ptr: int - pointer to the data buffer + sizes: tuple[int] - size of the tensor + strides: tuple[int] - strides of the tensor + device: Device - device on which the tensor resides + dtype: ScalarType - data type of the tensor + + Returns: Tensor - tensor wrapping the existing memory + """ + return torch.ops.libtorch_agn_2_11.my_from_blob_with_lambda_deleter.default( + data_ptr, sizes, strides, device, dtype + ) + + +def get_lambda_deleter_call_count() -> int: + """ + Returns the number of times the lambda test deleter has been called. + """ + return torch.ops.libtorch_agn_2_11.get_lambda_deleter_call_count.default() + + +def reset_lambda_deleter_call_count() -> None: + """ + Resets the lambda deleter call counter to zero. + """ + torch.ops.libtorch_agn_2_11.reset_lambda_deleter_call_count.default() + + +def my_from_blob_with_cuda_lambda_deleter(numel: int, device) -> Tensor: + """ + Creates a CUDA tensor that owns its memory via cudaMalloc, using a lambda deleter. + + Similar to my_from_blob_with_cuda_deleter but uses the capturing-lambda + code path in torch_from_blob. + + Args: + numel: int - number of elements in the tensor + device: Device - CUDA device + + Returns: Tensor - a 1D float32 tensor of zeros + """ + return torch.ops.libtorch_agn_2_11.my_from_blob_with_cuda_lambda_deleter.default( + numel, device + ) + + +# ============================================================================= +# Proxy for inherited ops (from libtorch_agn_2_9 and libtorch_agn_2_10 csrc/) +# +# Ops compiled from previous versions' csrc directories are accessible via +# the module-level __getattr__. For example: +# libtorch_agn_2_11.ops.sgd_out_of_place(...) # from 2.9 +# libtorch_agn_2_11.ops.my_sum(...) # from 2.10 +# ============================================================================= + +_NAMESPACE = "libtorch_agn_2_11" + + +def __getattr__(name): + """Proxy for inherited ops from previous versions.""" + if name.startswith("_"): + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + ops_namespace = getattr(torch.ops, _NAMESPACE) + op = getattr(ops_namespace, name, None) + if op is None: + raise AttributeError(f"No op named '{name}' in {_NAMESPACE}") + return op.default + + +def __dir__(): + """List all available ops (native + inherited).""" + native = [ + name + for name in globals() + if not name.startswith("_") and callable(globals().get(name)) + ] + ops_namespace = getattr(torch.ops, _NAMESPACE) + inherited = [n for n in dir(ops_namespace) if not n.startswith("_")] + return sorted(set(native + inherited)) diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/setup.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..ef2078b5f9f779544fadee14845ccd4ec13794a1 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_11_extension/setup.py @@ -0,0 +1,102 @@ +import distutils.command.clean +import shutil +from pathlib import Path + +from setuptools import find_packages, setup + +import torch +from torch.utils.cpp_extension import ( + BuildExtension, + CppExtension, + CUDAExtension, + IS_WINDOWS, +) + + +ROOT_DIR = Path(__file__).parent +CSRC_DIR = ROOT_DIR / "csrc" + +# Include csrc from previous versions for forward compatibility testing +PREV_CSRC_DIRS = [ + ROOT_DIR.parent / "libtorch_agn_2_9_extension" / "csrc", + ROOT_DIR.parent / "libtorch_agn_2_10_extension" / "csrc", +] + + +class clean(distutils.command.clean.clean): + def run(self): + # Run default behavior first + distutils.command.clean.clean.run(self) + + # Remove extension + for path in (ROOT_DIR / "libtorch_agn_2_11").glob("**/*.so"): + path.unlink() + # Remove build and dist and egg-info directories + dirs = [ + ROOT_DIR / "build", + ROOT_DIR / "dist", + ROOT_DIR / "libtorch_agn_2_11.egg-info", + ] + for path in dirs: + if path.exists(): + shutil.rmtree(str(path), ignore_errors=True) + + +def get_extension(): + extra_compile_args = { + "cxx": [ + "-DTORCH_TARGET_VERSION=0x020b000000000000", + "-DSTABLE_LIB_NAME=libtorch_agn_2_11", + ], + } + if not IS_WINDOWS: + extra_compile_args["cxx"].append("-fdiagnostics-color=always") + + # Collect sources from this version and all previous versions + sources = list(CSRC_DIR.glob("**/*.cpp")) + for prev_dir in PREV_CSRC_DIRS: + sources.extend(prev_dir.glob("**/*.cpp")) + + extension = CppExtension + # allow including + if torch.cuda.is_available(): + extra_compile_args["cxx"].append("-DLAE_USE_CUDA") + extra_compile_args["nvcc"] = [ + "-O2", + "-DUSE_CUDA", + "-DTORCH_TARGET_VERSION=0x020b000000000000", + "-DSTABLE_LIB_NAME=libtorch_agn_2_11", + ] + extension = CUDAExtension + sources.extend(CSRC_DIR.glob("**/*.cu")) + for prev_dir in PREV_CSRC_DIRS: + sources.extend(prev_dir.glob("**/*.cu")) + + return [ + extension( + "libtorch_agn_2_11._C", + sources=sorted(str(s) for s in sources), + py_limited_api=True, + extra_compile_args=extra_compile_args, + extra_link_args=[], + ) + ] + + +setup( + name="libtorch_agn_2_11", + version="0.0", + author="PyTorch Core Team", + description="Example of libtorch agnostic extension for PyTorch 2.11+", + packages=find_packages(exclude=("test",)), + package_data={"libtorch_agn_2_11": ["*.dll", "*.dylib", "*.so"]}, + install_requires=[ + "torch", + ], + ext_modules=get_extension(), + cmdclass={ + "build_ext": BuildExtension.with_options(no_python_abi_suffix=True), + "clean": clean, + }, + options={"bdist_wheel": {"py_limited_api": "cp39"}}, +) diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/csrc/tag_test.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/csrc/tag_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..96b86bf1e8671aa7695ea77e259b919349aaf20c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/csrc/tag_test.cpp @@ -0,0 +1,20 @@ +#include +#include +#include +#include + +using torch::stable::Tensor; + +Tensor tagged_identity(Tensor t) { + return t; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("tagged_identity(Tensor t) -> Tensor", + std::vector{ + at::Tag::pointwise, at::Tag::pt2_compliant_tag, at::Tag::core}); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("tagged_identity", TORCH_BOX(&tagged_identity)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/libtorch_agn_2_12/__init__.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/libtorch_agn_2_12/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b0ee6e97df5744e2ae3253016dce5324eb4484a4 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/libtorch_agn_2_12/__init__.py @@ -0,0 +1,25 @@ +import ctypes +import sys +from pathlib import Path + +import torch + + +so_files = list( + Path(__file__).parent.glob("_C*" + (".pyd" if sys.platform == "win32" else ".so")) +) +if len(so_files) != 1: + raise AssertionError(f"Expected one _C*.{{so,pyd}} file, found {len(so_files)}") + +# use ctypes.CDLL instead of load_library to be able to test the unload logic +# below code is reduced from the load_library code +with torch._ops.dl_open_guard(): + loaded_lib = ctypes.CDLL(str(so_files[0])) + +from . import ops + + +__all__ = [ + "loaded_lib", + "ops", +] diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/libtorch_agn_2_12/ops.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/libtorch_agn_2_12/ops.py new file mode 100644 index 0000000000000000000000000000000000000000..231c3688c5865c31232dfc9ae31da5539186f2a1 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/libtorch_agn_2_12/ops.py @@ -0,0 +1,36 @@ +import torch + + +# ============================================================================= +# Proxy for inherited ops (from libtorch_agn_2_9, 2_10, and 2_11 csrc/) +# +# Ops compiled from previous versions' csrc directories are accessible via +# the module-level __getattr__. For example: +# libtorch_agn_2_12.ops.sgd_out_of_place(...) # from 2.9 +# libtorch_agn_2_12.ops.my_sum(...) # from 2.10 +# ============================================================================= + +_NAMESPACE = "libtorch_agn_2_12" + + +def __getattr__(name): + """Proxy for inherited ops from previous versions.""" + if name.startswith("_"): + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + ops_namespace = getattr(torch.ops, _NAMESPACE) + op = getattr(ops_namespace, name, None) + if op is None: + raise AttributeError(f"No op named '{name}' in {_NAMESPACE}") + return op.default + + +def __dir__(): + """List all available ops (native + inherited).""" + native = [ + name + for name in globals() + if not name.startswith("_") and callable(globals().get(name)) + ] + ops_namespace = getattr(torch.ops, _NAMESPACE) + inherited = [n for n in dir(ops_namespace) if not n.startswith("_")] + return sorted(set(native + inherited)) diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/setup.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..90d5817513e4b47b6987fda41c417eef11edaa3c --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_12_extension/setup.py @@ -0,0 +1,103 @@ +import distutils.command.clean +import shutil +from pathlib import Path + +from setuptools import find_packages, setup + +import torch +from torch.utils.cpp_extension import ( + BuildExtension, + CppExtension, + CUDAExtension, + IS_WINDOWS, +) + + +ROOT_DIR = Path(__file__).parent +CSRC_DIR = ROOT_DIR / "csrc" + +# Include csrc from previous versions for forward compatibility testing +PREV_CSRC_DIRS = [ + ROOT_DIR.parent / "libtorch_agn_2_9_extension" / "csrc", + ROOT_DIR.parent / "libtorch_agn_2_10_extension" / "csrc", + ROOT_DIR.parent / "libtorch_agn_2_11_extension" / "csrc", +] + + +class clean(distutils.command.clean.clean): + def run(self): + # Run default behavior first + distutils.command.clean.clean.run(self) + + # Remove extension + for path in (ROOT_DIR / "libtorch_agn_2_12").glob("**/*.so"): + path.unlink() + # Remove build and dist and egg-info directories + dirs = [ + ROOT_DIR / "build", + ROOT_DIR / "dist", + ROOT_DIR / "libtorch_agn_2_12.egg-info", + ] + for path in dirs: + if path.exists(): + shutil.rmtree(str(path), ignore_errors=True) + + +def get_extension(): + extra_compile_args = { + "cxx": [ + "-DTORCH_TARGET_VERSION=0x020c000000000000", + "-DSTABLE_LIB_NAME=libtorch_agn_2_12", + ], + } + if not IS_WINDOWS: + extra_compile_args["cxx"].append("-fdiagnostics-color=always") + + # Collect sources from this version and all previous versions + sources = list(CSRC_DIR.glob("**/*.cpp")) + for prev_dir in PREV_CSRC_DIRS: + sources.extend(prev_dir.glob("**/*.cpp")) + + extension = CppExtension + # allow including + if torch.cuda.is_available(): + extra_compile_args["cxx"].append("-DLAE_USE_CUDA") + extra_compile_args["nvcc"] = [ + "-O2", + "-DUSE_CUDA", + "-DTORCH_TARGET_VERSION=0x020c000000000000", + "-DSTABLE_LIB_NAME=libtorch_agn_2_12", + ] + extension = CUDAExtension + sources.extend(CSRC_DIR.glob("**/*.cu")) + for prev_dir in PREV_CSRC_DIRS: + sources.extend(prev_dir.glob("**/*.cu")) + + return [ + extension( + "libtorch_agn_2_12._C", + sources=sorted(str(s) for s in sources), + py_limited_api=True, + extra_compile_args=extra_compile_args, + extra_link_args=[], + ) + ] + + +setup( + name="libtorch_agn_2_12", + version="0.0", + author="PyTorch Core Team", + description="Example of libtorch agnostic extension for PyTorch 2.12+", + packages=find_packages(exclude=("test",)), + package_data={"libtorch_agn_2_12": ["*.dll", "*.dylib", "*.so"]}, + install_requires=[ + "torch", + ], + ext_modules=get_extension(), + cmdclass={ + "build_ext": BuildExtension.with_options(no_python_abi_suffix=True), + "clean": clean, + }, + options={"bdist_wheel": {"py_limited_api": "cp39"}}, +) diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/my_exception_what.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/my_exception_what.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f9d22e0cef725ea2b1ce9ebf81317bcaabb384b3 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/my_exception_what.cpp @@ -0,0 +1,25 @@ +#include +#include + +#include + +// Direct calls to the 2.13 exception getter shims. Also tested in +// our_subtract_stable_error_check (in the 2.10 extension, which reaches these +// shims dynamically via TORCH_DYNAMIC_VERSION_CALL when built at an older +// target). +std::string my_exception_what() { + return std::string(torch_exception_get_what()); +} +std::string my_exception_get_what_without_backtrace() { + return std::string(torch_exception_get_what_without_backtrace()); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_exception_what() -> str"); + m.def("my_exception_get_what_without_backtrace() -> str"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_exception_what", TORCH_BOX(&my_exception_what)); + m.impl("my_exception_get_what_without_backtrace", TORCH_BOX(&my_exception_get_what_without_backtrace)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/my_rand_generator.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/my_rand_generator.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6a9e8b0e1c3a00e428aeca1a462d1e303abd7b99 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/my_rand_generator.cpp @@ -0,0 +1,43 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +using torch::stable::Generator; +using torch::stable::Tensor; + +// Calls aten::rand.generator through torch_call_dispatcher, threading a +// Generator passed from Python and deriving the output device from the +// generator itself (which exercises Generator::device()). +Tensor my_rand_with_generator(std::vector size, Generator generator) { + torch::stable::Device device = generator.device(); + + // aten::rand.generator(SymInt[] size, *, Generator? generator, + // ScalarType? dtype=None, Layout? layout=None, Device? device=None, + // bool? pin_memory=None) -> Tensor + StableIValue stack[6]; + stack[0] = torch::stable::detail::from(size); + stack[1] = torch::stable::detail::from(std::optional(generator)); + stack[2] = torch::stable::detail::from(std::nullopt); // dtype + stack[3] = torch::stable::detail::from(std::nullopt); // layout + stack[4] = + torch::stable::detail::from(std::optional(device)); + stack[5] = torch::stable::detail::from(std::nullopt); // pin_memory + STABLE_TORCH_ERROR_CODE_CHECK(torch_call_dispatcher( + "aten::rand", "generator", stack, TORCH_ABI_VERSION)); + return torch::stable::detail::to(stack[0]); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_rand_with_generator(int[] size, Generator generator) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_rand_with_generator", TORCH_BOX(&my_rand_with_generator)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/set_python_module_test.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/set_python_module_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..10f6f2c300f6ad40278b138fa96ad4dd494401cb --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/set_python_module_test.cpp @@ -0,0 +1,33 @@ +#include +#include +#include + +using torch::stable::Tensor; + +Tensor identity_with_fake_module(Tensor t) { + return t; +} + +Tensor identity_w_nonexistent_fake_module(Tensor t) { + return t; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.set_python_module("libtorch_agn_2_13"); + m.def("identity_with_fake_module(Tensor t) -> Tensor"); +} + +// Separate fragment whose set_python_module points at a module that doesn't +// exist, to exercise the missing-module error path. +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.set_python_module("libtorch_agn_2_13_nonexistent_module"); + m.def("identity_w_nonexistent_fake_module(Tensor t) -> Tensor"); +} + +// Register on CPU key so there's no Meta kernel to fall back to. +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CPU, m) { + m.impl("identity_with_fake_module", TORCH_BOX(&identity_with_fake_module)); + m.impl( + "identity_w_nonexistent_fake_module", + TORCH_BOX(&identity_w_nonexistent_fake_module)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/stream_kernel_test.cu b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/stream_kernel_test.cu new file mode 100644 index 0000000000000000000000000000000000000000..84ce01e2cfa79ed2537ae922efdd7bd4db78f9e5 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/stream_kernel_test.cu @@ -0,0 +1,47 @@ +#include +#include +#include +#include +#include +#include + +using torch::stable::Tensor; + +__global__ void test_kernel_on_stream(int* output, int n, int fill_value) { + const int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx < n) { + output[idx] = fill_value; + } +} + +Tensor test_kernel_launch_on_stream(Tensor input, int fill_value) { + STD_TORCH_CHECK( + input.scalar_type() == torch::headeronly::ScalarType::Int, + "input must be int32 dtype"); + + const auto device_index = input.get_device_index(); + torch::stable::accelerator::DeviceGuard device_guard(device_index); + + Tensor output = torch::stable::new_empty(input, input.sizes()); + + cudaStream_t stream = static_cast( + torch::stable::accelerator::getCurrentStream(device_index).nativeHandle()); + + const int64_t n = input.numel(); + const int block_size = 256; + const int grid_size = static_cast((n + block_size - 1) / block_size); + + int* output_ptr = reinterpret_cast(output.data_ptr()); + test_kernel_on_stream<<>>( + output_ptr, static_cast(n), fill_value); + + return output; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_kernel_launch_on_stream(Tensor input, int fill_value) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CUDA, m) { + m.impl("test_kernel_launch_on_stream", TORCH_BOX(&test_kernel_launch_on_stream)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/stream_native_handle_test.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/stream_native_handle_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ccb28b71b38c5a6dd76bc9fe7f0f3b123f5ff9f3 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/csrc/stream_native_handle_test.cpp @@ -0,0 +1,25 @@ +#include +#include + +#include +#include +#include + +int64_t test_stream_native_handle(int32_t device_index) { + STD_TORCH_CHECK( + device_index >= std::numeric_limits::min() && + device_index <= std::numeric_limits::max(), + "Device index is out of range of DeviceIndex (int32_t)."); + + void* native_handle = + torch::stable::accelerator::getCurrentStream(device_index).nativeHandle(); + return static_cast(reinterpret_cast(native_handle)); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_stream_native_handle(int device_index) -> int"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("test_stream_native_handle", TORCH_BOX(&test_stream_native_handle)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/libtorch_agn_2_13/__init__.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/libtorch_agn_2_13/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5602c65a44f51add9dec7a1f6eac8c5ea0b5d73b --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/libtorch_agn_2_13/__init__.py @@ -0,0 +1,33 @@ +import ctypes +import sys +from pathlib import Path + +import torch + + +so_files = list( + Path(__file__).parent.glob("_C*" + (".pyd" if sys.platform == "win32" else ".so")) +) +if len(so_files) != 1: + raise AssertionError(f"Expected one _C*.{{so,pyd}} file, found {len(so_files)}") + +# use ctypes.CDLL instead of load_library to be able to test the unload logic +# below code is reduced from the load_library code +with torch._ops.dl_open_guard(): + loaded_lib = ctypes.CDLL(str(so_files[0])) + +from . import ops + + +# Fake impl for the op the C++ side declared with +# m.set_python_module("libtorch_agn_2_13"). Registering here means the fake +# impl is available as soon as the extension is imported. +@torch.library.register_fake("libtorch_agn_2_13::identity_with_fake_module") +def _(t): + return torch.empty_like(t) + + +__all__ = [ + "loaded_lib", + "ops", +] diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/libtorch_agn_2_13/ops.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/libtorch_agn_2_13/ops.py new file mode 100644 index 0000000000000000000000000000000000000000..3babf4a73fd218ceaebde0130c8bf479e33a6a65 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/libtorch_agn_2_13/ops.py @@ -0,0 +1,56 @@ +import torch + + +# ============================================================================= +# Proxy for inherited ops (from libtorch_agn_2_9, 2_10, 2_11, and 2_12 csrc/) +# +# Ops compiled from previous versions' csrc directories are accessible via +# the module-level __getattr__. For example: +# libtorch_agn_2_13.ops.sgd_out_of_place(...) # from 2.9 +# libtorch_agn_2_13.ops.my_sum(...) # from 2.10 +# +# Ops defined in this package's csrc/ (e.g. test_stream_native_handle) are +# registered on ``torch.ops.libtorch_agn_2_13`` and wrapped explicitly below. +# ============================================================================= + +_NAMESPACE = "libtorch_agn_2_13" + + +def test_stream_native_handle(device_index) -> int: + """Stable ``Stream::nativeHandle`` vs ``torch.accelerator`` (2.13+).""" + return torch.ops.libtorch_agn_2_13.test_stream_native_handle.default(device_index) + + +def test_kernel_launch_on_stream(input, fill_value: int): + """Launch CUDA kernel using stream from nativeHandle().""" + return torch.ops.libtorch_agn_2_13.test_kernel_launch_on_stream.default( + input, fill_value + ) + + +def my_rand_with_generator(size, generator): + """rand via torch_call_dispatcher threading a Generator (2.13+).""" + return torch.ops.libtorch_agn_2_13.my_rand_with_generator.default(size, generator) + + +def __getattr__(name): + """Proxy for inherited ops from previous versions.""" + if name.startswith("_"): + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + ops_namespace = getattr(torch.ops, _NAMESPACE) + op = getattr(ops_namespace, name, None) + if op is None: + raise AttributeError(f"No op named '{name}' in {_NAMESPACE}") + return op.default + + +def __dir__(): + """List all available ops (native + inherited).""" + native = [ + name + for name in globals() + if not name.startswith("_") and callable(globals().get(name)) + ] + ops_namespace = getattr(torch.ops, _NAMESPACE) + inherited = [n for n in dir(ops_namespace) if not n.startswith("_")] + return sorted(set(native + inherited)) diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/setup.py b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..0560c747c8b282be01dc2a11a8ce98a8c1eec597 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_13_extension/setup.py @@ -0,0 +1,104 @@ +import distutils.command.clean +import shutil +from pathlib import Path + +from setuptools import find_packages, setup + +import torch +from torch.utils.cpp_extension import ( + BuildExtension, + CppExtension, + CUDAExtension, + IS_WINDOWS, +) + + +ROOT_DIR = Path(__file__).parent +CSRC_DIR = ROOT_DIR / "csrc" + +# Include csrc from previous versions for forward compatibility testing +PREV_CSRC_DIRS = [ + ROOT_DIR.parent / "libtorch_agn_2_9_extension" / "csrc", + ROOT_DIR.parent / "libtorch_agn_2_10_extension" / "csrc", + ROOT_DIR.parent / "libtorch_agn_2_11_extension" / "csrc", + ROOT_DIR.parent / "libtorch_agn_2_12_extension" / "csrc", +] + + +class clean(distutils.command.clean.clean): + def run(self): + # Run default behavior first + distutils.command.clean.clean.run(self) + + # Remove extension + for path in (ROOT_DIR / "libtorch_agn_2_13").glob("**/*.so"): + path.unlink() + # Remove build and dist and egg-info directories + dirs = [ + ROOT_DIR / "build", + ROOT_DIR / "dist", + ROOT_DIR / "libtorch_agn_2_13.egg-info", + ] + for path in dirs: + if path.exists(): + shutil.rmtree(str(path), ignore_errors=True) + + +def get_extension(): + extra_compile_args = { + "cxx": [ + "-DTORCH_TARGET_VERSION=0x020d000000000000", + "-DSTABLE_LIB_NAME=libtorch_agn_2_13", + ], + } + if not IS_WINDOWS: + extra_compile_args["cxx"].append("-fdiagnostics-color=always") + + # Collect sources from this version and all previous versions + sources = list(CSRC_DIR.glob("**/*.cpp")) + for prev_dir in PREV_CSRC_DIRS: + sources.extend(prev_dir.glob("**/*.cpp")) + + extension = CppExtension + # allow including + if torch.cuda.is_available(): + extra_compile_args["cxx"].append("-DLAE_USE_CUDA") + extra_compile_args["nvcc"] = [ + "-O2", + "-DUSE_CUDA", + "-DTORCH_TARGET_VERSION=0x020d000000000000", + "-DSTABLE_LIB_NAME=libtorch_agn_2_13", + ] + extension = CUDAExtension + sources.extend(CSRC_DIR.glob("**/*.cu")) + for prev_dir in PREV_CSRC_DIRS: + sources.extend(prev_dir.glob("**/*.cu")) + + return [ + extension( + "libtorch_agn_2_13._C", + sources=sorted(str(s) for s in sources), + py_limited_api=True, + extra_compile_args=extra_compile_args, + extra_link_args=[], + ) + ] + + +setup( + name="libtorch_agn_2_13", + version="0.0", + author="PyTorch Core Team", + description="Example of libtorch agnostic extension for PyTorch 2.13+", + packages=find_packages(exclude=("test",)), + package_data={"libtorch_agn_2_13": ["*.dll", "*.dylib", "*.so"]}, + install_requires=[ + "torch", + ], + ext_modules=get_extension(), + cmdclass={ + "build_ext": BuildExtension.with_options(no_python_abi_suffix=True), + "clean": clean, + }, + options={"bdist_wheel": {"py_limited_api": "cp39"}}, +) diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_9_extension/csrc/cuda_kernel.cu b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_9_extension/csrc/cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..9c0fdfda45c7a78e4dcb9359bfdfa4283d32c084 --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_9_extension/csrc/cuda_kernel.cu @@ -0,0 +1,61 @@ +#include "kernel.h" + +#ifdef USE_ROCM +#include +#else +#include +#endif +#include +#include +#include +#include + +using torch::stable::Tensor; + +Tensor mv_tensor_accessor_cuda(Tensor m, Tensor v) { + STD_TORCH_CHECK(m.dim() == 2, "m must be 2D"); + STD_TORCH_CHECK(v.dim() == 1, "v must be 1D"); + STD_TORCH_CHECK(m.size(1) == v.size(0), "m.shape[1] == v.shape[0] must hold"); + STD_TORCH_CHECK( + m.scalar_type() == v.scalar_type(), "m and v must have the same dtype"); + STD_TORCH_CHECK( + m.device() == v.device(), "m and v must be on the same device"); + const auto device_index = m.get_device_index(); + torch::stable::accelerator::DeviceGuard device_guard(device_index); + Tensor res = new_empty(m, {m.size(0)}); + + void* raw_stream = nullptr; + TORCH_ERROR_CODE_CHECK( + aoti_torch_get_current_cuda_stream(device_index, &raw_stream)); +#ifdef USE_ROCM + auto stream = static_cast(raw_stream); +#else + auto stream = static_cast(raw_stream); +#endif + + THO_DISPATCH_V2( + m.scalar_type(), + "mv_tensor_accessor_cuda", + AT_WRAP(([&]() { + auto resa = Accessor_cuda( + reinterpret_cast(res.data_ptr()), + res.sizes().data(), + res.strides().data()); + auto ma = Accessor_cuda( + reinterpret_cast(m.data_ptr()), + m.sizes().data(), + m.strides().data()); + auto va = Accessor_cuda( + reinterpret_cast(v.data_ptr()), + v.sizes().data(), + v.strides().data()); + mv_tensor_accessor_kernel + <<<1, 1, 0, stream>>>(resa, ma, va); + })), + AT_FLOATING_TYPES); + return res; +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CUDA, m) { + m.impl("mv_tensor_accessor", TORCH_BOX(&mv_tensor_accessor_cuda)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_9_extension/csrc/kernel.cpp b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_9_extension/csrc/kernel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b7c0077c8d564b67a3400bd1b3139dc6fc1e4ced --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_9_extension/csrc/kernel.cpp @@ -0,0 +1,547 @@ +#include "kernel.h" + +#include +#include +#include +#include +#include +#include + +#ifdef LAE_USE_CUDA +#include +#include +#endif + +#include + +void inline sgd_math( + float* param_ptr, + float* grad_ptr, + float* out_ptr, + const float weight_decay, + const double lr, + const bool maximize, + int64_t size +){ + int64_t d = 0; + for (; d < size; d++) { + float grad_val = grad_ptr[d]; + if (maximize) grad_val = -grad_val; + if (weight_decay != 0.0){ + grad_val += param_ptr[d] * weight_decay; + } + out_ptr[d] = param_ptr[d] - grad_val * float(lr); + } +} + +using torch::stable::Tensor; + +Tensor sgd_out_of_place( + const Tensor param, + const Tensor grad, + const double weight_decay, + const double lr, + const bool maximize) { + STD_TORCH_CHECK(param.dim() == 1, "param must be 1D"); + + // these test the get_device() and get_device_index() methods + // while ascertaining that we are still on CPU + STD_TORCH_CHECK(param.get_device() == -1, "CPU device index = -1"); + STD_TORCH_CHECK(param.get_device_index() == -1, "CPU device index = -1"); + + // testing Tensor strides + stride + STD_TORCH_CHECK(param.strides()[0] == param.stride(0)); + + auto out = new_empty(param, param.sizes()); + + sgd_math( + reinterpret_cast(param.data_ptr()), + reinterpret_cast(grad.data_ptr()), + reinterpret_cast(out.data_ptr()), + float(weight_decay), + lr, + maximize, + param.numel() + ); + + return out; +} + +void boxed_sgd_out_of_place(StableIValue* stack, uint64_t num_args, uint64_t num_outputs) { + Tensor res = sgd_out_of_place( + torch::stable::detail::to(stack[0]), + torch::stable::detail::to(stack[1]), + float(torch::stable::detail::to(stack[2])), + torch::stable::detail::to(stack[3]), + torch::stable::detail::to(stack[4])); + + stack[0] = torch::stable::detail::from(res); +} + +STABLE_TORCH_LIBRARY(STABLE_LIB_NAME, m) { + m.def("sgd_out_of_place(Tensor param, Tensor grad, float weight_decay, float lr, bool maximize) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CPU, m) { + m.impl("sgd_out_of_place", &boxed_sgd_out_of_place); +} + +Tensor identity(Tensor t) { + return t; +} + + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("identity(Tensor t) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CUDA, m) { + m.impl("identity", TORCH_BOX(&identity)); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CPU, m) { + m.impl("identity", TORCH_BOX(&identity)); +} + +Tensor my_abs(Tensor t) { + const auto num_args = 1; + StableIValue stack[num_args]; + stack[0] = torch::stable::detail::from(t); + aoti_torch_call_dispatcher("aten::abs", "", stack); + return torch::stable::detail::to(stack[0]); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_abs(Tensor t) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_abs", TORCH_BOX(&my_abs)); +} + +Tensor my_ones_like(Tensor t, StableIValue device) { + const auto num_args = 6; + StableIValue stack[num_args]; + + auto mf = aoti_torch_memory_format_contiguous_format(); + + stack[0] = torch::stable::detail::from(t); + stack[1] = torch::stable::detail::from(std::optional(t.scalar_type())); // dtype + stack[2] = torch::stable::detail::from(std::nullopt); // layout + stack[3] = torch::stable::detail::from(std::optional(device)); // device + stack[4] = torch::stable::detail::from(std::optional(false)); // pin_memory + stack[5] = torch::stable::detail::from(std::optional(mf)); // memory_format + + aoti_torch_call_dispatcher("aten::ones_like", "", stack); + + return torch::stable::detail::to(stack[0]); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_ones_like(Tensor t, Device d) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_ones_like", TORCH_BOX(&my_ones_like)); +} + +std::tuple exp_neg_is_leaf(Tensor t1, Tensor t2, Tensor t3) { + StableIValue stack_exp[1]; + stack_exp[0] = torch::stable::detail::from(t1); + aoti_torch_call_dispatcher("aten::exp", "", stack_exp); + + StableIValue stack_neg[1]; + stack_neg[0] = torch::stable::detail::from(t2); + aoti_torch_call_dispatcher("aten::neg", "", stack_neg); + + StableIValue stack_is_leaf[1]; + stack_is_leaf[0] = torch::stable::detail::from(t3); + aoti_torch_call_dispatcher("aten::is_leaf", "", stack_is_leaf); + + return std::make_tuple( + torch::stable::detail::to(stack_exp[0]), + torch::stable::detail::to(stack_neg[0]), + torch::stable::detail::to(stack_is_leaf[0])); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("exp_neg_is_leaf(Tensor t1, Tensor t2, Tensor t3) -> (Tensor, Tensor, bool)"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("exp_neg_is_leaf", TORCH_BOX(&exp_neg_is_leaf)); +} + +Tensor neg_exp(Tensor t) { + StableIValue stack[1]; + stack[0] = torch::stable::detail::from(t); + aoti_torch_call_dispatcher("aten::exp", "", stack); + aoti_torch_call_dispatcher("aten::neg", "", stack); + return torch::stable::detail::to(stack[0]); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("neg_exp(Tensor t) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("neg_exp", TORCH_BOX(&neg_exp)); +} + +Tensor divide_neg_exp(Tensor t) { + StableIValue stack_neg[1]; + stack_neg[0] = torch::stable::detail::from(t); + + StableIValue stack_exp[1]; + stack_exp[0] = torch::stable::detail::from(t); + aoti_torch_call_dispatcher("aten::exp", "", stack_exp); + aoti_torch_call_dispatcher("aten::neg", "", stack_neg); + + StableIValue stack_div[2]; + stack_div[0] = stack_neg[0]; + stack_div[1] = stack_exp[0]; + aoti_torch_call_dispatcher("aten::divide", "Tensor", stack_div); + return torch::stable::detail::to(stack_div[0]); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("divide_neg_exp(Tensor t) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("divide_neg_exp", TORCH_BOX(÷_neg_exp)); +} + +bool is_contiguous(Tensor t) { + return t.is_contiguous(); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("is_contiguous(Tensor t) -> bool"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("is_contiguous", TORCH_BOX(&is_contiguous)); +} + +Tensor my_transpose(Tensor t, int64_t dim0, int64_t dim1) { + return transpose(t, dim0, dim1); +} + +// This is used to test const torch::stable::Tensor& with TORCH_BOX +Tensor my_empty_like(const Tensor& t) { + return empty_like(t); +} + +// This is used to test torch::stable::Tensor& with TORCH_BOX +bool my_is_cpu(Tensor& t) { + return t.is_cpu(); +} + +Tensor fill_infinity(Tensor t) { + auto value = std::numeric_limits::infinity(); + return fill_(t, value); +} + +Tensor my_pad(Tensor t) { + std::string mode = "constant"; + double value = 0.0; + return pad(t, {1, 2, 2, 1}, mode, value); +} + +Tensor my_narrow(Tensor t, int64_t dim, int64_t start, int64_t length) { + return narrow(t, dim, start, length); +} + +Tensor my_narrow_symint(Tensor t, int64_t dim, int64_t start, int64_t length) { + return clone(narrow(t, dim, start, length)); +} + +Tensor my_new_empty_dtype_variant(Tensor t) { + // Still using a std::vector below even though people can just pass in an + // initializer list (which will be implicitly converted to an HeaderOnlyArrayRef) + // directly. + // This is to test that passing in a std::vector works for BC. (It gets + // implicitly converted to HeaderOnlyArrayRef too!) + std::vector sizes = {2, 5}; + auto dtype = std::make_optional(torch::headeronly::ScalarType::BFloat16); + return new_empty(t, sizes, dtype); +} + +Tensor my_new_zeros_dtype_variant(Tensor t) { + auto dtype = std::make_optional(at::ScalarType::Float); + return new_zeros(t, {2, 5}, dtype); +} + +Tensor my_copy_(Tensor dst, Tensor src, bool non_blocking) { + return copy_(dst, src, non_blocking); +} + +Tensor my_clone(Tensor t) { + return clone(t); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_transpose(Tensor t, int dim0, int dim1) -> Tensor"); + m.def("my_empty_like(Tensor t) -> Tensor"); + m.def("fill_infinity(Tensor(a!) t) -> Tensor(a!)"); + m.def("my_pad(Tensor t) -> Tensor"); + m.def("my_narrow(Tensor t, int dim, int start, int length) -> Tensor"); + m.def("my_narrow_symint(Tensor t, int dim, SymInt start, SymInt length) -> Tensor"); + m.def("my_new_empty_dtype_variant(Tensor t) -> Tensor"); + m.def("my_new_zeros_dtype_variant(Tensor t) -> Tensor"); + m.def("my_copy_(Tensor dst, Tensor src, bool non_blocking) -> Tensor"); + m.def("my_clone(Tensor t) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_transpose", TORCH_BOX(&my_transpose)); + m.impl("my_empty_like", TORCH_BOX(&my_empty_like)); + m.impl("fill_infinity", TORCH_BOX(&fill_infinity)); + m.impl("my_is_cpu", TORCH_BOX(&my_is_cpu)); + m.impl("my_new_empty_dtype_variant", TORCH_BOX(&my_new_empty_dtype_variant)); + m.impl("my_new_zeros_dtype_variant", TORCH_BOX(&my_new_zeros_dtype_variant)); + m.impl("my_copy_", TORCH_BOX(&my_copy_)); + m.impl("my_clone", TORCH_BOX(&my_clone)); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeImplicitAutograd, m) { + m.impl("my_pad", TORCH_BOX(&my_pad)); + m.impl("my_narrow", TORCH_BOX(&my_narrow)); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CPU, m) { + m.impl("my_narrow_symint", TORCH_BOX(&my_narrow_symint)); +} + +Tensor my_zero_(Tensor t) { + return zero_(t); +} + +Tensor my_amax(Tensor t) { + return amax(t, 0, false); +} + +Tensor my_amax_vec(Tensor t) { + return amax(t, {0,1}, false); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_zero_(Tensor(a!) t) -> Tensor(a!)"); + m.def("my_amax(Tensor a) -> Tensor"); + m.def("my_amax_vec(Tensor a) -> Tensor"); + m.def("my_is_cpu(Tensor t) -> bool"); + m.def("test_default_constructor(bool undefined) -> bool"); +} + +bool test_default_constructor(bool defined) { + Tensor out; + if (defined) { + AtenTensorHandle defined_ath; + int64_t sizes[] = {2, 3}; + int64_t strides[] = {3, 1}; + aoti_torch_empty_strided( + 2, + sizes, + strides, + aoti_torch_dtype_float32(), + aoti_torch_device_type_cpu(), + 0, + &defined_ath); + out = Tensor(defined_ath); + } + return out.defined(); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_zero_", TORCH_BOX(&my_zero_)); + m.impl("my_amax", TORCH_BOX(&my_amax)); + m.impl("my_amax_vec", TORCH_BOX(&my_amax_vec)); + m.impl("test_default_constructor", TORCH_BOX(&test_default_constructor)); +} + +Tensor mv_tensor_accessor_cpu(Tensor m, Tensor v) { + STD_TORCH_CHECK(m.dim() == 2, "m must be 2D"); + STD_TORCH_CHECK(v.dim() == 1, "v must be 1D"); + STD_TORCH_CHECK(m.size(1) == v.size(0), "m.shape[1] == v.shape[0] must hold"); + STD_TORCH_CHECK(m.scalar_type() == v.scalar_type(), "m and v must have the same dtype"); + STD_TORCH_CHECK(m.device() == v.device(), "m and v must be on the same device"); + Tensor res = new_empty(m, {m.size(0)}); + THO_DISPATCH_V2(m.scalar_type(), "mv_tensor_accessor_cpu", + AT_WRAP(([&]() { + auto resa = Accessor_cpu(reinterpret_cast(res.data_ptr()), res.sizes().data(), res.strides().data()); + auto ma = Accessor_cpu(reinterpret_cast(m.data_ptr()), m.sizes().data(), m.strides().data()); + auto va = Accessor_cpu(reinterpret_cast(v.data_ptr()), v.sizes().data(), v.strides().data()); + mv_tensor_accessor_kernel(resa, ma, va); + })), + AT_FLOATING_TYPES); + return res; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("mv_tensor_accessor(Tensor m, Tensor v) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CPU, m) { + m.impl("mv_tensor_accessor", TORCH_BOX(&mv_tensor_accessor_cpu)); +} + +// Test functions for torch::stable::accelerator APIs + +#ifdef LAE_USE_CUDA +int64_t test_device_guard(int64_t device_index) { + using torch::stable::accelerator::DeviceGuard; + + STD_TORCH_CHECK( + device_index >= std::numeric_limits::min() && + device_index <= std::numeric_limits::max(), + "Device index is out of range of DeviceIndex (int32_t)."); + + DeviceGuard guard(device_index); + int currentDevice; + cudaError_t err = cudaGetDevice(¤tDevice); + STD_TORCH_CHECK(err == cudaSuccess); + return currentDevice; +} + +int64_t test_device_guard_set_index() { + using torch::stable::accelerator::DeviceGuard; + + DeviceGuard guard(1); + guard.set_index(0); + int currentDevice; + cudaError_t err = cudaGetDevice(¤tDevice); + STD_TORCH_CHECK(err == cudaSuccess); + return currentDevice; +} + +int64_t test_stream(int32_t device_index) { + STD_TORCH_CHECK( + device_index >= std::numeric_limits::min() && + device_index <= std::numeric_limits::max(), + "Device index is out of range of DeviceIndex (int32_t)."); + + return torch::stable::accelerator::getCurrentStream(device_index).id(); +} + +int64_t test_get_current_device_index() { + return torch::stable::accelerator::getCurrentDeviceIndex(); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("test_device_guard(int device_index) -> int"); + m.def("test_device_guard_set_index() -> int"); + m.def("test_stream(int device_index) -> int"); + m.def("test_get_current_device_index() -> int"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("test_device_guard", TORCH_BOX(&test_device_guard)); + m.impl("test_device_guard_set_index", TORCH_BOX(&test_device_guard_set_index)); + m.impl("test_stream", TORCH_BOX(&test_stream)); + m.impl("test_get_current_device_index", TORCH_BOX(&test_get_current_device_index)); +} + +#endif // LAE_USE_CUDA + +Tensor my_flatten(Tensor t, int64_t start_dim, int64_t end_dim) { + return flatten(t, start_dim, end_dim); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_flatten(Tensor t, int start_dim=0, int end_dim=-1) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_flatten", TORCH_BOX(&my_flatten)); +} + +// Test function for const std::optional& with TORCH_BOX +// Returns the tensor if present, otherwise returns a zeros tensor of specified size +Tensor my_optional_tensor_ref( + const std::optional& maybe_tensor, + int64_t default_size) { + if (maybe_tensor.has_value()) { + return maybe_tensor.value(); + } + // Create a zeros tensor as default + AtenTensorHandle zeros_ath; + int64_t sizes[] = {default_size}; + int64_t strides[] = {1}; + aoti_torch_empty_strided( + 1, + sizes, + strides, + aoti_torch_dtype_float32(), + aoti_torch_device_type_cpu(), + 0, + &zeros_ath); + Tensor zeros_tensor(zeros_ath); + return zero_(zeros_tensor); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_optional_tensor_ref(Tensor? maybe_tensor, int default_size) -> Tensor"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_optional_tensor_ref", TORCH_BOX(&my_optional_tensor_ref)); +} + +int64_t my_storage_offset(Tensor t) { + return t.storage_offset(); +} + +int64_t my_element_size(Tensor t) { + return static_cast(t.element_size()); +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_storage_offset(Tensor t) -> int"); + m.def("my_element_size(Tensor t) -> int"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_storage_offset", TORCH_BOX(&my_storage_offset)); + m.impl("my_element_size", TORCH_BOX(&my_element_size)); +} + +Tensor my_unsqueeze(const Tensor& t, int64_t dim) { + return torch::stable::unsqueeze(t, dim); +} + +Tensor my_squeeze(const Tensor& t, int64_t dim) { + return torch::stable::squeeze(t, dim); +} + +Tensor my_select(const Tensor& t, int64_t dim, int64_t index) { + return torch::stable::select(t, dim, index); +} + +Tensor my_matmul(const Tensor& self, const Tensor& other) { + return torch::stable::matmul(self, other); +} + +// Putting my_layout in libtorch_agn_2_9 because it can target 2.9 +// for runtime, but it can only be compiled against 2.11 cuz of +// the layout API. +using torch::headeronly::Layout; +bool my_layout(const Tensor& t, Layout layout) { + return t.layout() == layout; +} + +STABLE_TORCH_LIBRARY_FRAGMENT(STABLE_LIB_NAME, m) { + m.def("my_unsqueeze(Tensor t, int dim) -> Tensor"); + m.def("my_squeeze(Tensor t, int dim) -> Tensor"); + m.def("my_select(Tensor t, int dim, int index) -> Tensor"); + m.def("my_matmul(Tensor self, Tensor other) -> Tensor"); + m.def("my_layout(Tensor t, Layout layout) -> bool"); +} + +STABLE_TORCH_LIBRARY_IMPL(STABLE_LIB_NAME, CompositeExplicitAutograd, m) { + m.impl("my_unsqueeze", TORCH_BOX(&my_unsqueeze)); + m.impl("my_squeeze", TORCH_BOX(&my_squeeze)); + m.impl("my_select", TORCH_BOX(&my_select)); + m.impl("my_matmul", TORCH_BOX(&my_matmul)); + m.impl("my_layout", TORCH_BOX(&my_layout)); +} diff --git a/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_9_extension/csrc/kernel.h b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_9_extension/csrc/kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..5a8a0be891449b5a1b409dbb7e69b3cbae9807cc --- /dev/null +++ b/western_ai/usa/pytorch/test/cpp_extensions/libtorch_agn_2_9_extension/csrc/kernel.h @@ -0,0 +1,26 @@ +#include +#include + +template +using Accessor_cpu = torch::headeronly::HeaderOnlyTensorAccessor; + +#if defined(__CUDACC__) || defined(__HIPCC__) +#define MAYBE_GLOBAL __global__ + +template +using Accessor_cuda = torch::headeronly::HeaderOnlyGenericPackedTensorAccessor; + +#else +#define MAYBE_GLOBAL +#endif + +template