diff --git a/western_ai/usa/pytorch/torch/CMakeLists.txt b/western_ai/usa/pytorch/torch/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..40a42d09f83392a1236760b39406d29a77a89c7f --- /dev/null +++ b/western_ai/usa/pytorch/torch/CMakeLists.txt @@ -0,0 +1,568 @@ +# This file used to build libtorch.so. +# Now it only builds the Torch python bindings. + +if(NOT CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO) + cmake_minimum_required(VERSION 3.27 FATAL_ERROR) + project(torch CXX C) + find_package(torch REQUIRED) + option(USE_CUDA "Use CUDA" ON) + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +endif() + +if(NOT BUILD_PYTHON) + return() +endif() + +set(TORCH_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}") +set(TORCH_ROOT "${TORCH_SRC_DIR}/..") + +if(NOT TORCH_INSTALL_LIB_DIR) + set(TORCH_INSTALL_LIB_DIR lib) +endif() + +if(MSVC) + set(LIBSHM_SUBDIR libshm_windows) +else() + set(LIBSHM_SUBDIR libshm) +endif() + +set(LIBSHM_SRCDIR ${TORCH_SRC_DIR}/lib/${LIBSHM_SUBDIR}) +add_subdirectory(${LIBSHM_SRCDIR}) + +# Generate files +set(TOOLS_PATH "${TORCH_ROOT}/tools") + + +set(TORCH_PYTHON_SRCS + ${GENERATED_CXX_PYTHON} + ) +append_filelist("libtorch_python_core_sources" TORCH_PYTHON_SRCS) + +# NB: This has to match the condition under which the JIT test directory +# is included (at the time of writing that's in caffe2/CMakeLists.txt). +if(BUILD_TEST) + add_definitions(-DBUILDING_TESTS) + list(APPEND TORCH_PYTHON_SRCS + ${TORCH_ROOT}/test/cpp/jit/torch_python_test.cpp + ) +endif() + +set(TORCH_PYTHON_INCLUDE_DIRECTORIES + ${PYTHON_INCLUDE_DIR} + + ${TORCH_ROOT} + ${TORCH_ROOT}/aten/src + + ${CMAKE_BINARY_DIR} + ${CMAKE_BINARY_DIR}/aten/src + ${CMAKE_BINARY_DIR}/caffe2/aten/src + ${CMAKE_BINARY_DIR}/third_party + ${CMAKE_BINARY_DIR}/third_party/onnx + + ${TORCH_ROOT}/third_party/valgrind-headers + + ${TORCH_ROOT}/third_party/gloo + ${TORCH_ROOT}/third_party/onnx + ${TORCH_ROOT}/third_party/flatbuffers/include + ${TORCH_ROOT}/third_party/kineto/libkineto/include + ${TORCH_ROOT}/third_party/cpp-httplib + ${TORCH_ROOT}/third_party/nlohmann/include + + ${TORCH_SRC_DIR}/csrc + ${TORCH_SRC_DIR}/csrc/api/include + ${TORCH_SRC_DIR}/lib + ${TORCH_SRC_DIR}/standalone + ) + +list(APPEND TORCH_PYTHON_INCLUDE_DIRECTORIES ${LIBSHM_SRCDIR}) + +set(TORCH_PYTHON_LINK_LIBRARIES + Python::Module + pybind::pybind11 + httplib + nlohmann + moodycamel + shm + fmt::fmt-header-only + ATEN_CPU_FILES_GEN_LIB) + +if(USE_ASAN AND TARGET Sanitizer::address) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::address) +endif() +# Skip UBSan on ROCm: see comment in cmake/Dependencies.cmake (UBSan + ASAN +# on ROCm Clang produces misaligned ASAN global metadata that aborts at +# startup). +if(USE_ASAN AND TARGET Sanitizer::undefined AND NOT USE_ROCM) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::undefined) +endif() +if(USE_LSAN AND TARGET Sanitizer::leak) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::leak) +endif() +if(USE_TSAN AND TARGET Sanitizer::thread) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::thread) +endif() + +set(TORCH_PYTHON_COMPILE_DEFINITIONS) + +set(TORCH_PYTHON_COMPILE_OPTIONS) + +set(TORCH_PYTHON_LINK_FLAGS "") + +if(MSVC) + string(APPEND TORCH_PYTHON_LINK_FLAGS " /NODEFAULTLIB:LIBCMT.LIB") + # Python headers use #pragma comment(lib, "pythonXX") which makes MSVC search + # for the library by bare name. Add the directory so the linker can find it. + foreach(_pylib IN LISTS Python_LIBRARIES) + if(_pylib MATCHES "python[0-9].*\\.lib$") + get_filename_component(_pylib_dir "${_pylib}" DIRECTORY) + string(APPEND TORCH_PYTHON_LINK_FLAGS " /LIBPATH:\"${_pylib_dir}\"") + break() + endif() + endforeach() + list(APPEND TORCH_PYTHON_LINK_LIBRARIES ${Python_LIBRARIES} onnx_library) + if(NOT CMAKE_BUILD_TYPE MATCHES "Release") + string(APPEND TORCH_PYTHON_LINK_FLAGS " /DEBUG:FULL") + endif() +elseif(APPLE) + string(APPEND TORCH_PYTHON_LINK_FLAGS " -undefined dynamic_lookup") +else() + list(APPEND TORCH_PYTHON_COMPILE_OPTIONS + -fno-strict-aliasing + -Wno-strict-aliasing) +endif() + +if(USE_ITT) + list(APPEND TORCH_PYTHON_SRCS + ${TORCH_SRC_DIR}/csrc/itt.cpp + ) + list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_ITT) +endif() + +if(USE_CUDA) + include(${TORCH_ROOT}/cmake/public/cuda.cmake) + append_filelist("libtorch_python_cuda_core_sources" TORCH_PYTHON_SRCS) + + list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUDA) + if(USE_CUDNN) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::cudnn) + list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUDNN) + endif() + if(USE_CUSPARSELT) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::cusparselt) + list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUSPARSELT) + endif() + if(USE_CUFILE) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::cufile) + list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUFILE) + endif() + + if(TARGET torch::nvtx3) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::nvtx3) + else() + if(TARGET torch::nvtoolsext) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::nvtoolsext) + endif() + endif() +endif() + +if(USE_ROCM) + append_filelist("libtorch_python_cuda_core_sources" TORCH_PYTHON_SRCS) + + list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS + USE_ROCM + __HIP_PLATFORM_AMD__ + ) + if(NOT WIN32) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES ${ROCM_ROCTX_LIB}) + endif() +endif() + +if(USE_XPU) + include(${TORCH_ROOT}/cmake/public/xpu.cmake) + append_filelist("libtorch_python_xpu_sources" TORCH_PYTHON_SRCS) + list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_XPU) +endif() + +if(USE_CUDNN OR USE_ROCM) + list(APPEND TORCH_PYTHON_SRCS + ${TORCH_SRC_DIR}/csrc/cuda/shared/cudnn.cpp + ) + if(USE_STATIC_CUDNN) + set_source_files_properties( + ${TORCH_SRC_DIR}/csrc/cuda/shared/cudnn.cpp + PROPERTIES COMPILE_DEFINITIONS "USE_STATIC_CUDNN" + ) + endif() +endif() + +if(USE_CUSPARSELT) + list(APPEND TORCH_PYTHON_SRCS ${TORCH_SRC_DIR}/csrc/cuda/shared/cusparselt.cpp) +endif() + +if(USE_MPS) + list(APPEND TORCH_PYTHON_SRCS ${MPS_PYTHON_SRCS}) +endif() + +if(USE_VALGRIND AND NOT WIN32) + list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_VALGRIND) +endif() + +add_custom_target(torch_python_stubs DEPENDS + "${TORCH_SRC_DIR}/_C/__init__.pyi" + "${TORCH_SRC_DIR}/_C/_VariableFunctions.pyi" + "${TORCH_SRC_DIR}/nn/functional.pyi" + "${TORCH_SRC_DIR}/utils/data/datapipes/datapipe.pyi" +) + +file(GLOB_RECURSE torchgen_python CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/torchgen/*.py") +file(GLOB_RECURSE autograd_python CONFIGURE_DEPENDS "${TOOLS_PATH}/autograd/*.py") +file(GLOB_RECURSE pyi_python CONFIGURE_DEPENDS "${TOOLS_PATH}/pyi/*.py") +add_custom_command( + OUTPUT + "${TORCH_SRC_DIR}/_C/__init__.pyi" + "${TORCH_SRC_DIR}/_C/_VariableFunctions.pyi" + "${TORCH_SRC_DIR}/nn/functional.pyi" + COMMAND + "${Python_EXECUTABLE}" -mtools.pyi.gen_pyi + --native-functions-path "aten/src/ATen/native/native_functions.yaml" + --tags-path "aten/src/ATen/native/tags.yaml" + --deprecated-functions-path "tools/autograd/deprecated.yaml" + DEPENDS + "${TORCH_SRC_DIR}/_C/__init__.pyi.in" + "${TORCH_SRC_DIR}/_C/_VariableFunctions.pyi.in" + "${TORCH_SRC_DIR}/nn/functional.pyi.in" + "${TORCH_ROOT}/aten/src/ATen/native/native_functions.yaml" + "${TORCH_ROOT}/aten/src/ATen/native/tags.yaml" + "${TORCH_ROOT}/tools/autograd/deprecated.yaml" + "${TORCH_ROOT}/torch/_torch_docs.py" + "${TORCH_ROOT}/torch/_tensor_docs.py" + ${pyi_python} + ${autograd_python} + ${torchgen_python} + WORKING_DIRECTORY + "${TORCH_ROOT}" +) +file(GLOB_RECURSE datapipe_files CONFIGURE_DEPENDS "${TORCH_SRC_DIR}/utils/data/datapipes/*.py") +add_custom_command( + OUTPUT + "${TORCH_SRC_DIR}/utils/data/datapipes/datapipe.pyi" + COMMAND + ${CMAKE_COMMAND} -E env --modify PYTHONPATH=path_list_prepend:"${TORCH_ROOT}" -- + "${Python_EXECUTABLE}" ${TORCH_SRC_DIR}/utils/data/datapipes/gen_pyi.py + DEPENDS + "${TORCH_SRC_DIR}/utils/data/datapipes/datapipe.pyi.in" + ${datapipe_files} + WORKING_DIRECTORY + "${TORCH_ROOT}" +) +if(USE_DISTRIBUTED) + if(WIN32) + append_filelist("libtorch_python_distributed_core_sources" TORCH_PYTHON_SRCS) + else() + append_filelist("libtorch_python_distributed_sources" TORCH_PYTHON_SRCS) + endif() + # Disable certain warnings for GCC-9.X + if(CMAKE_COMPILER_IS_GNUCXX) + set_source_files_properties(${TORCH_SRC_DIR}/csrc/distributed/autograd/init.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type") + set_source_files_properties(${TORCH_SRC_DIR}/csrc/distributed/rpc/testing/init.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type") + set_source_files_properties(${TORCH_SRC_DIR}/csrc/distributed/c10d/init.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type") + endif() + # NCCL is a private dependency of libtorch, but libtorch_python includes + # some private headers of libtorch, which in turn include NCCL. As a hacky + # alternative to making NCCL a public dependency of libtorch, we make it + # a private dependency of libtorch_python as well. + if(USE_NCCL) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES __caffe2_nccl) + endif() + # Same for MPI. + if(USE_MPI) + list(APPEND TORCH_PYTHON_LINK_LIBRARIES MPI::MPI_CXX) + endif() + list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_C10D) + +endif() + +if(USE_NCCL AND NOT WIN32) + list(APPEND TORCH_PYTHON_SRCS + ${TORCH_SRC_DIR}/csrc/cuda/python_nccl.cpp) + list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_NCCL) +endif() + +# coreml +if(USE_COREML_DELEGATE) + list(APPEND TORCH_PYTHON_SRCS ${TORCH_SRC_DIR}/csrc/jit/backends/coreml/cpp/backend.cpp) + list(APPEND TORCH_PYTHON_SRCS ${TORCH_SRC_DIR}/csrc/jit/backends/coreml/cpp/preprocess.cpp) +endif() + + +add_library(torch_python SHARED ${TORCH_PYTHON_SRCS}) +torch_compile_options(torch_python) # see cmake/public/utils.cmake +if(APPLE) + target_compile_options(torch_python PRIVATE + $<$: -fvisibility=default>) +endif() + +if(CAFFE2_USE_MKL AND BUILD_LIBTORCHLESS) + + # Use the RPATH of the linked libraries + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + # we need to explicitly link caffe2::mkl in order to have the + # correct RPATH in torch_python for the split build + target_link_libraries(torch_python PRIVATE caffe2::mkl) +endif() + +add_dependencies(torch_python onnx_proto) +# Avoid numpy for the DEPLOY build +if(USE_NUMPY) + target_link_libraries(torch_python PRIVATE Python::NumPy) + target_compile_definitions(torch_python PRIVATE USE_NUMPY) +endif() + +if(USE_CUFILE AND NOT USE_ROCM) + target_compile_definitions(torch_python PRIVATE USE_CUFILE) +endif() + +if(BUILD_LAZY_TS_BACKEND) + target_compile_definitions(torch_python PRIVATE BUILD_LAZY_TS_BACKEND) +endif() + +if(HAVE_SOVERSION) + set_target_properties(torch_python PROPERTIES + VERSION ${TORCH_VERSION} SOVERSION ${TORCH_SOVERSION}) +endif() + +# in case of the split build we need to add compile definitions +if(BUILD_LIBTORCHLESS) + + if(USE_UCC AND USE_C10D_UCC) + target_compile_definitions(torch_python PRIVATE USE_C10D_UCC) + endif() + + if(USE_NCCL AND USE_C10D_NCCL) + target_compile_definitions(torch_python PRIVATE USE_C10D_NCCL) + endif() + + if(USE_DISTRIBUTED) + target_compile_definitions(torch_python PRIVATE USE_DISTRIBUTED) + endif() + + if(USE_MPI AND USE_C10D_MPI) + target_compile_definitions(torch_python PRIVATE USE_C10D_MPI) + endif() + + if(USE_GLOO AND USE_C10D_GLOO) + target_compile_definitions(torch_python PRIVATE USE_C10D_GLOO) + endif() + + if(NOT WIN32) + target_compile_definitions(torch_python PRIVATE USE_RPC) + endif() + + if(USE_TENSORPIPE) + target_compile_definitions(torch_python PRIVATE USE_TENSORPIPE) + endif() + + set(EXPERIMENTAL_SINGLE_THREAD_POOL "0" CACHE STRING + "Experimental option to use a single thread pool for inter- and intra-op parallelism") + if("${EXPERIMENTAL_SINGLE_THREAD_POOL}") + target_compile_definitions(torch_python PRIVATE "-DAT_EXPERIMENTAL_SINGLE_THREAD_POOL=1") + endif() + +endif() + +add_dependencies(torch_python torch_python_stubs) +add_dependencies(torch_python flatbuffers) + + +if(USE_PRECOMPILED_HEADERS) + target_precompile_headers(torch_python PRIVATE + "$<$:ATen/ATen.h>") +endif() + +# Required workaround for generated sources +# See https://samthursfield.wordpress.com/2015/11/21/cmake-dependencies-between-targets-and-files-and-custom-commands/#custom-commands-in-different-directories +add_dependencies(torch_python generate-torch-sources) +set_source_files_properties( + ${GENERATED_CXX_PYTHON} + PROPERTIES GENERATED TRUE + ) + +# Disable certain warnings for GCC-9.X +if(CMAKE_COMPILER_IS_GNUCXX) + set_source_files_properties(${TORCH_SRC_DIR}/csrc/Module.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type") + set_source_files_properties(${TORCH_SRC_DIR}/csrc/autograd/python_variable.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type") +endif() + +# Preserve CUDA_GENCODE flags +if(USE_CUDA) + torch_cuda_get_nvcc_gencode_flag(_ARCH_FLAGS) + set_source_files_properties(${TORCH_SRC_DIR}/csrc/cuda/Module.cpp PROPERTIES COMPILE_FLAGS "-DCUDA_ARCH_FLAGS=\"${_ARCH_FLAGS_readable}\"") +endif() + +# Preserve HIP arch flags +if(USE_ROCM) + string(REPLACE ";" " " PYTORCH_ROCM_ARCH_readable "${PYTORCH_ROCM_ARCH}") + set_source_files_properties(${TORCH_SRC_DIR}/csrc/cuda/Module.cpp PROPERTIES COMPILE_FLAGS "-DCUDA_ARCH_FLAGS=\"${PYTORCH_ROCM_ARCH_readable}\"") +endif() + +# Preserve XPU arch flags +if(USE_XPU) + string(REPLACE "," " " _ARCH_FLAGS_readable "${TORCH_XPU_ARCH_LIST}") + set_source_files_properties(${TORCH_SRC_DIR}/csrc/xpu/Module.cpp PROPERTIES COMPILE_FLAGS "-DXPU_ARCH_FLAGS=\"${_ARCH_FLAGS_readable}\"") +endif() + +target_compile_definitions(torch_python PRIVATE "-DTHP_BUILD_MAIN_LIB") + +target_link_libraries(torch_python PRIVATE ${TORCH_LIB} ${TORCH_PYTHON_LINK_LIBRARIES}) + +target_compile_definitions(torch_python PRIVATE ${TORCH_PYTHON_COMPILE_DEFINITIONS}) + +target_compile_options(torch_python PRIVATE ${TORCH_PYTHON_COMPILE_OPTIONS}) + +target_include_directories(torch_python PUBLIC ${TORCH_PYTHON_INCLUDE_DIRECTORIES}) + +if(USE_UCC) + target_link_libraries(torch_python PRIVATE __caffe2_ucc) + target_compile_definitions(torch_python PRIVATE USE_UCC) +endif() + +if(BUILD_ONEDNN_GRAPH) + target_compile_definitions(torch_python PRIVATE "-DBUILD_ONEDNN_GRAPH") + if(NOT BUILD_LIBTORCHLESS) + target_compile_definitions(torch_cpu PRIVATE "-DBUILD_ONEDNN_GRAPH") + endif() +endif() + +if(${CMAKE_BUILD_TYPE} STREQUAL "RelWithAssert") + # Workaround numerous decret-without-a-gil warnings from JIT + # see https://github.com/pytorch/pytorch/issues/130073 + target_compile_definitions(torch_python PRIVATE "-DPYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF") +endif() + +if(NOT TORCH_PYTHON_LINK_FLAGS STREQUAL "") + set_target_properties(torch_python PROPERTIES LINK_FLAGS "${TORCH_PYTHON_LINK_FLAGS}") +endif() + +install(TARGETS torch_python DESTINATION "${TORCH_INSTALL_LIB_DIR}") + +# Build the torch._C Python extension module from the thin stub that +# forwards to torch_python. This was historically built by setuptools; +# building it here lets both setuptools and scikit-build-core use the +# same code path. +if(BUILD_PYTHON AND NOT BUILD_LIBTORCH_WHL) + Python_add_library(_C MODULE WITH_SOABI "${TORCH_SRC_DIR}/csrc/stub.c") + target_link_libraries(_C PRIVATE torch_python) + set_target_properties(_C PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/torch" + # libtorch_python.so is installed to lib/ relative to _C, so use + # $ORIGIN/lib as the install RPATH (the global default is just $ORIGIN). + INSTALL_RPATH "${_rpath_portable_origin}/lib" + ) + if(NOT WIN32) + set_target_properties(_C PROPERTIES + C_VISIBILITY_PRESET "default" + ) + endif() + # Install at the install prefix root, which maps to torch/ in the wheel + # via wheel.install-dir = "torch". The import library (Windows _C.lib) goes + # alongside the other import libs in torch/lib/, matching the layout that + # setup.py used before _C moved to CMake. + install(TARGETS _C + RUNTIME DESTINATION "." + LIBRARY DESTINATION "." + ARCHIVE DESTINATION "lib" + ) + + # ---[ NCCL EP extension (torch._nccl_ep) + # Optional standalone extension holding the NCCL EP bindings, imported lazily + # by torch/distributed/_token_switch.py so libtorch_cuda never references + # ncclEp*. How it links libnccl_ep follows how torch links NCCL: + # - USE_SYSTEM_NCCL=OFF: __caffe2_nccl_ep is libnccl_ep.a, statically linked + # here -- self-contained (nccl* bind to torch_cuda's embedded NCCL; JIT + # headers baked from the submodule build dir; no nccl4py). + # - USE_SYSTEM_NCCL=ON: __caffe2_nccl_ep is libnccl_ep.so, NEEDED-linked here + # and resolved from the nccl4py wheel at runtime (RPATH to nccl/ep/lib; JIT + # headers also from nccl4py, so nothing is baked). + if(USE_NCCL_EP) + Python_add_library(_nccl_ep MODULE WITH_SOABI + "${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/nccl_ep.cu" + "${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/nccl_ep_pybind.cpp") + target_compile_definitions(_nccl_ep PRIVATE USE_NCCL_EP USE_C10D_NCCL) + # torch_python: pybind + Tensor/ProcessGroup casters; torch_cuda: + # ProcessGroupNCCL::getCommPtr, getNcclDataType, CUDA stream, embedded NCCL; + # __caffe2_nccl_ep: libnccl_ep (static .a or shared .so) + nccl_ep headers. + target_link_libraries(_nccl_ep PRIVATE + torch_python torch_cuda __caffe2_nccl_ep pybind::pybind11) + set_target_properties(_nccl_ep PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/torch") + if(USE_SYSTEM_NCCL) + # Dynamic: resolve nccl4py's libnccl_ep.so (site-packages/nccl/ep/lib, one + # dir up from torch/ via $ORIGIN/..) at import. + set_target_properties(_nccl_ep PROPERTIES + INSTALL_RPATH "${_rpath_portable_origin}/lib;${_rpath_portable_origin}/../nccl/ep/lib") + else() + # Static: only torch's own libs needed; bake the in-tree JIT header dir so + # the extension self-configures NCCL_EP_HOME / NCCL_HOME at import. + target_compile_definitions(_nccl_ep PRIVATE + "NCCL_EP_JIT_HOME=\"${NCCL_EP_JIT_HOME}\"") + set_target_properties(_nccl_ep PROPERTIES + INSTALL_RPATH "${_rpath_portable_origin}/lib") + endif() + if(NOT WIN32) + set_target_properties(_nccl_ep PROPERTIES C_VISIBILITY_PRESET "default") + endif() + install(TARGETS _nccl_ep RUNTIME DESTINATION "." LIBRARY DESTINATION ".") + endif() +endif() + +# Generate torch/version.py from the appropriate CMake cache variables. +if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") + set(TORCH_VERSION_DEBUG 1) +else() + set(TORCH_VERSION_DEBUG 0) +endif() + +set(CUDA_VERSION "") +if(CUDAToolkit_VERSION_MAJOR) + set(CUDA_VERSION "${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR}") +endif() + +add_custom_target( + gen_torch_version ALL + "${Python_EXECUTABLE}" -m tools.generate_torch_version + --is-debug=${TORCH_VERSION_DEBUG} + --cuda-version=${CUDA_VERSION} + --hip-version=${HIP_VERSION_CLEAN} + --rocm-version=${ROCM_VERSION_DEV} + --xpu-version=${SYCL_COMPILER_VERSION} + BYPRODUCTS ${TORCH_SRC_DIR}/version.py + COMMENT "Regenerating version file..." + WORKING_DIRECTORY ${TORCH_ROOT} +) +add_dependencies(torch_python gen_torch_version) +# version.py is gitignored (generated), so scikit-build-core won't include it +# via Python package collection. Install it explicitly via cmake instead. +install(FILES "${TORCH_SRC_DIR}/version.py" DESTINATION ".") + +# Skip building this library under MacOS, since it is currently failing to build on Mac +# Github issue #61930 +if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + # Add Android Nnapi delegate library + add_library(nnapi_backend SHARED + ${TORCH_SRC_DIR}/csrc/jit/backends/nnapi/nnapi_backend_lib.cpp + ${TORCH_SRC_DIR}/csrc/jit/backends/nnapi/nnapi_backend_preprocess.cpp + ) + # Pybind11 requires explicit linking of the torch_python library + if(BUILD_LIBTORCHLESS) + target_link_libraries(nnapi_backend PRIVATE ${TORCH_LIB}) + else() + target_link_libraries(nnapi_backend PRIVATE torch) + endif() + target_link_libraries(nnapi_backend PRIVATE torch_python pybind::pybind11 fmt::fmt-header-only) + if(MSVC AND TORCH_PYTHON_LINK_FLAGS) + set_target_properties(nnapi_backend PROPERTIES LINK_FLAGS "${TORCH_PYTHON_LINK_FLAGS}") + endif() +endif() + +set(TORCH_PYTHON_COMPILE_OPTIONS ${TORCH_PYTHON_COMPILE_OPTIONS} PARENT_SCOPE) +set(TORCH_PYTHON_LINK_FLAGS ${TORCH_PYTHON_LINK_FLAGS} PARENT_SCOPE) diff --git a/western_ai/usa/pytorch/torch/autograd/_functions/replace.vim b/western_ai/usa/pytorch/torch/autograd/_functions/replace.vim new file mode 100644 index 0000000000000000000000000000000000000000..33a2a24bc3d7bf5bf93d71f234951ead2a33fc78 --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/_functions/replace.vim @@ -0,0 +1,3 @@ +%s/self/ctx/g +%s/\s\+def forward/ @staticmethod\r def forward/g +%s/\s\+def backward/ @staticmethod\r @once_differentiable\r def backward/g diff --git a/western_ai/usa/pytorch/torch/autograd/_functions/tensor.py b/western_ai/usa/pytorch/torch/autograd/_functions/tensor.py new file mode 100644 index 0000000000000000000000000000000000000000..50acb8d2caaef6b1330d791bb5a3070fa6f7fbb0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/_functions/tensor.py @@ -0,0 +1,72 @@ +# mypy: allow-untyped-defs +import operator +from functools import reduce +from typing_extensions import deprecated + +import torch +import torch._utils +from torch.autograd.function import Function + + +class Type(Function): + @staticmethod + @deprecated( + "`torch.autograd._functions.Type` is deprecated as of PyTorch 2.1, " + "please use `torch.tensor.to(dtype=dtype)` instead.", + category=FutureWarning, + ) + # pyrefly: ignore [bad-override] + def forward(ctx, i, dest_type): + ctx.input_type = type(i) + ctx.input_device = -1 if not i.is_cuda else i.get_device() + return i.type(dest_type) + + @staticmethod + # pyrefly: ignore [bad-override] + def backward(ctx, grad_output): + if ctx.input_device == -1: + return grad_output.type(ctx.input_type), None + else: + with torch.accelerator.device_index(ctx.input_device): + return grad_output.type(ctx.input_type), None + + +# TODO: deprecate this +class Resize(Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx, tensor, sizes): + ctx.sizes = sizes + ctx.numel = reduce(operator.mul, sizes, 1) + if tensor.numel() != ctx.numel: + raise RuntimeError( + ( + "requested resize to {} ({} elements in total), " + "but the given tensor has a size of {} ({} elements). " + "autograd's resize can only change the shape of a given " + "tensor, while preserving the number of elements. " + ).format( + "x".join(map(str, sizes)), + ctx.numel, + "x".join(map(str, tensor.size())), + tensor.numel(), + ) + ) + ctx.input_sizes = tensor.size() + if tensor.is_quantized: + tensor.copy_(tensor) + return tensor.contiguous().view(*sizes) + if tensor.is_contiguous(): + result = tensor.new(tensor).contiguous().view(*sizes) + return result + else: + return tensor.contiguous().view(*sizes) + + @staticmethod + # pyrefly: ignore [bad-override] + def backward(ctx, grad_output): + if grad_output.numel() != ctx.numel: + raise AssertionError( + f"Expected grad_output to have {ctx.numel} elements, but got {grad_output.numel()}" + ) + return grad_output.contiguous().view(ctx.input_sizes), None diff --git a/western_ai/usa/pytorch/torch/autograd/_functions/utils.py b/western_ai/usa/pytorch/torch/autograd/_functions/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b1c95ec2e37b1468fc5dbb1b8ee454fdb3fa0918 --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/_functions/utils.py @@ -0,0 +1,26 @@ +# mypy: allow-untyped-defs + + +def maybe_view(tensor, size, check_same_size=True): + if check_same_size and tensor.size() == size: + return tensor + return tensor.contiguous().view(size) + + +def maybe_unexpand(tensor, old_size, check_same_size=True): + if check_same_size and tensor.size() == old_size: + return tensor + num_unsqueezed = tensor.dim() - len(old_size) + expanded_dims = [ + dim + for dim, (expanded, original) in enumerate( + zip(tensor.size()[num_unsqueezed:], old_size) + ) + if expanded != original + ] + + for _ in range(num_unsqueezed): + tensor = tensor.sum(0, keepdim=False) + for dim in expanded_dims: + tensor = tensor.sum(dim, keepdim=True) + return tensor diff --git a/western_ai/usa/pytorch/torch/autograd/anomaly_mode.py b/western_ai/usa/pytorch/torch/autograd/anomaly_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..d18df091935f0a1318b9b67f39ddaf2f11cf2184 --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/anomaly_mode.py @@ -0,0 +1,123 @@ +# mypy: allow-untyped-defs +r"""Autograd anomaly mode.""" + +import warnings + +import torch + + +__all__ = ["detect_anomaly", "set_detect_anomaly"] + + +class detect_anomaly: + r"""Context-manager that enable anomaly detection for the autograd engine. + + This does two things: + + - Running the forward pass with detection enabled will allow the backward + pass to print the traceback of the forward operation that created the failing + backward function. + - If ``check_nan`` is ``True``, any backward computation that generate "nan" + value will raise an error. Default ``True``. + + .. warning:: + This mode should be enabled only for debugging as the different tests + will slow down your program execution. + + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_ANOMALY) + >>> import torch + >>> from torch import autograd + >>> class MyFunc(autograd.Function): + ... @staticmethod + ... def forward(ctx, inp): + ... return inp.clone() + ... + ... @staticmethod + ... def backward(ctx, gO): + ... # Error during the backward pass + ... raise RuntimeError("Some error in backward") + ... return gO.clone() + >>> def run_fn(a): + ... out = MyFunc.apply(a) + ... return out.sum() + >>> inp = torch.rand(10, 10, requires_grad=True) + >>> out = run_fn(inp) + >>> out.backward() + Traceback (most recent call last): + File "", line 1, in + File "/your/pytorch/install/torch/_tensor.py", line 93, in backward + torch.autograd.backward(self, gradient, retain_graph, create_graph) + File "/your/pytorch/install/torch/autograd/__init__.py", line 90, in backward + allow_unreachable=True) # allow_unreachable flag + File "/your/pytorch/install/torch/autograd/function.py", line 76, in apply + return self._forward_cls.backward(self, *args) + File "", line 8, in backward + RuntimeError: Some error in backward + >>> with autograd.detect_anomaly(): + ... inp = torch.rand(10, 10, requires_grad=True) + ... out = run_fn(inp) + ... out.backward() + Traceback of forward call that caused the error: + File "tmp.py", line 53, in + out = run_fn(inp) + File "tmp.py", line 44, in run_fn + out = MyFunc.apply(a) + Traceback (most recent call last): + File "", line 4, in + File "/your/pytorch/install/torch/_tensor.py", line 93, in backward + torch.autograd.backward(self, gradient, retain_graph, create_graph) + File "/your/pytorch/install/torch/autograd/__init__.py", line 90, in backward + allow_unreachable=True) # allow_unreachable flag + File "/your/pytorch/install/torch/autograd/function.py", line 76, in apply + return self._forward_cls.backward(self, *args) + File "", line 8, in backward + RuntimeError: Some error in backward + + """ + + def __init__(self, check_nan=True) -> None: + self.prev = torch.is_anomaly_enabled() + self.check_nan = check_nan + self.prev_check_nan = torch.is_anomaly_check_nan_enabled() + warnings.warn( + "Anomaly Detection has been enabled. " + "This mode will increase the runtime " + "and should only be enabled for debugging.", + stacklevel=2, + ) + + def __enter__(self) -> None: + torch.set_anomaly_enabled(True, self.check_nan) + + def __exit__(self, *args: object) -> None: + torch.set_anomaly_enabled(self.prev, self.prev_check_nan) + + +class set_detect_anomaly: + r"""Context-manager that sets the anomaly detection for the autograd engine on or off. + + ``set_detect_anomaly`` will enable or disable the autograd anomaly detection + based on its argument :attr:`mode`. + It can be used as a context-manager or as a function. + + See ``detect_anomaly`` above for details of the anomaly detection behaviour. + + Args: + mode (bool): Flag whether to enable anomaly detection (``True``), + or disable (``False``). + check_nan (bool): Flag whether to raise an error when the backward + generate "nan" + + """ + + def __init__(self, mode: bool, check_nan: bool = True) -> None: + self.prev = torch.is_anomaly_enabled() + self.prev_check_nan = torch.is_anomaly_check_nan_enabled() + torch.set_anomaly_enabled(mode, check_nan) + + def __enter__(self) -> None: + pass + + def __exit__(self, *args: object) -> None: + torch.set_anomaly_enabled(self.prev, self.prev_check_nan) diff --git a/western_ai/usa/pytorch/torch/autograd/forward_ad.py b/western_ai/usa/pytorch/torch/autograd/forward_ad.py new file mode 100644 index 0000000000000000000000000000000000000000..af2c39bbbd2654c0fe727b68c65247c3700989c6 --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/forward_ad.py @@ -0,0 +1,241 @@ +# mypy: allow-untyped-defs +import os +from typing import Any, NamedTuple + +import torch + +from .grad_mode import _DecoratorContextManager + + +__all__ = [ + "UnpackedDualTensor", + "enter_dual_level", + "exit_dual_level", + "make_dual", + "unpack_dual", + "dual_level", +] + +# Global variable used to make the python API simpler to use +_current_level = -1 + + +def enter_dual_level(): + r"""Enter a new forward grad level. + + This level can be used to make and unpack dual Tensors to compute + forward gradients. + + This function also updates the current level that is used by default + by the other functions in this API. + """ + from torch._functorch.predispatch import ( + _enter_dual_level as _predispatch_enter_dual_level, + ) + + global _current_level + new_level = _predispatch_enter_dual_level() + if new_level != _current_level + 1: + raise RuntimeError( + "Entering a new forward AD level but the current level " + "is not valid. Make sure you did not modify it directly." + ) + _current_level = new_level + return new_level + + +def exit_dual_level(*, level=None): + r"""Exit a forward grad level. + + This function deletes all the gradients associated with this + level. Only deleting the latest entered level is allowed. + + This function also updates the current level that is used by default + by the other functions in this API. + """ + global _current_level + if level is None: + level = _current_level + if level != _current_level: + raise RuntimeError( + "Trying to exit a forward AD level that was not the last one " + "that was created. This is not supported." + ) + from torch._functorch.predispatch import ( + _exit_dual_level as _predispatch_exit_dual_level, + ) + + _predispatch_exit_dual_level(level=level) + _current_level = level - 1 + + +def _maybe_load_decompositions(): + if os.environ.get("PYTORCH_JIT", "1") == "1" and __debug__: + from torch._decomp import decompositions_for_jvp # noqa: F401 + + +def make_dual(tensor, tangent, *, level=None): + r"""Associate a tensor value with its tangent to create a "dual tensor" for forward AD gradient computation. + + The result is a new tensor aliased to :attr:`tensor` with :attr:`tangent` embedded + as an attribute as-is if it has the same storage layout or copied otherwise. + The tangent attribute can be recovered with :func:`unpack_dual`. + + This function is backward differentiable. + + Given a function `f` whose jacobian is `J`, it allows one to compute the Jacobian-vector product (`jvp`) + between `J` and a given vector `v` as follows. + + Example:: + + >>> # xdoctest: +SKIP("Undefined variables") + >>> with dual_level(): + ... inp = make_dual(x, v) + ... out = f(inp) + ... y, jvp = unpack_dual(out) + + Please see the `forward-mode AD tutorial `__ + for detailed steps on how to use this API. + + """ + # See NOTE: [forward-mode AD decompositions mechanism] + # + # Import from torch._decomp import decompositions_for_jvp to register + # decompositions for jvp to the jit registry + # + # FIXME: We specify that __debug__ must be True because + # if python is run with -OO or -O flags (i.e., __debug__ is False), we encounter the + # following error: + # + # Return value was annotated as having type Tuple[NoneType, NoneType] but is actually of + # type Tuple[Tensor, Tensor]: + # File ".../torch/_decomp/__init__.py", line 1585 + # else: + # buffer = z + # return min - torch.log1p(z), buffer + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + _maybe_load_decompositions() + + if level is None: + level = _current_level + + if level < 0: + raise RuntimeError( + "Trying to create a dual Tensor for forward AD but no level " + "exists, make sure to enter_dual_level() first." + ) + if not (tensor.is_floating_point() or tensor.is_complex()): + raise ValueError( + f"Expected primal to be floating point or complex, but got: {tensor.dtype}" + ) + if not (tangent.is_floating_point() or tangent.is_complex()): + raise ValueError( + f"Expected tangent to be floating point or complex, but got: {tangent.dtype}" + ) + + from torch._functorch.predispatch import _make_dual as _predispatch_make_dual + + return _predispatch_make_dual(tensor, tangent, level=level) + + +class UnpackedDualTensor(NamedTuple): + r"""Namedtuple returned by :func:`unpack_dual` containing the primal and tangent components of the dual tensor. + + See :func:`unpack_dual` for more details. + """ + + primal: torch.Tensor + tangent: torch.Tensor | None + + +def unpack_dual(tensor, *, level=None): + r"""Unpack a "dual tensor" to get both its Tensor value and its forward AD gradient. + + The result is a namedtuple ``(primal, tangent)`` where ``primal`` is a view of + :attr:`tensor`'s primal and ``tangent`` is :attr:`tensor`'s tangent as-is. + Neither of these tensors can be dual tensor of level :attr:`level`. + + This function is backward differentiable. + + Example:: + + >>> # xdoctest: +SKIP("Undefined variables") + >>> with dual_level(): + ... inp = make_dual(x, x_t) + ... out = f(inp) + ... y, jvp = unpack_dual(out) + ... jvp = unpack_dual(out).tangent + + Please see the `forward-mode AD tutorial `__ + for detailed steps on how to use this API. + """ + if level is None: + level = _current_level + + if level < 0: + return UnpackedDualTensor(tensor, None) + + from torch._functorch.predispatch import _unpack_dual as _predispatch_unpack_dual + + primal, dual = _predispatch_unpack_dual(tensor, level=level) + + return UnpackedDualTensor(primal, dual) + + +class dual_level(_DecoratorContextManager): + r"""Context-manager for forward AD, where all forward AD computation must occur within the ``dual_level`` context. + + .. Note:: + + The ``dual_level`` context appropriately enters and exit the dual level to + controls the current forward AD level, which is used by default by the other + functions in this API. + + We currently don't plan to support nested ``dual_level`` contexts, however, so + only a single forward AD level is supported. To compute higher-order + forward grads, one can use :func:`torch.func.jvp`. + + Example:: + + >>> # xdoctest: +SKIP("Undefined variables") + >>> x = torch.tensor([1]) + >>> x_t = torch.tensor([1]) + >>> with dual_level(): + ... inp = make_dual(x, x_t) + ... # Do computations with inp + ... out = your_fn(inp) + ... _, grad = unpack_dual(out) + >>> grad is None + False + >>> # After exiting the level, the grad is deleted + >>> _, grad_after = unpack_dual(out) + >>> grad is None + True + + Please see the `forward-mode AD tutorial `__ + for detailed steps on how to use this API. + """ + + def __enter__(self): + return enter_dual_level() + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + exit_dual_level() + + +# Private helper functions +_is_fwd_grad_enabled = torch._C._is_fwd_grad_enabled + + +# Private helper function to enable or disable fwd grad. +# If you're a user and want to use this, please file an issue to discuss the use case. +class _set_fwd_grad_enabled(_DecoratorContextManager): + def __init__(self, mode: bool) -> None: + self.prev = _is_fwd_grad_enabled() + torch._C._set_fwd_grad_enabled(mode) + + def __enter__(self) -> None: + pass + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_fwd_grad_enabled(self.prev) diff --git a/western_ai/usa/pytorch/torch/autograd/function.py b/western_ai/usa/pytorch/torch/autograd/function.py new file mode 100644 index 0000000000000000000000000000000000000000..c60f0d0e916b6386c6dd772ed882657a6cd29c45 --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/function.py @@ -0,0 +1,898 @@ +# mypy: allow-untyped-defs +import functools +import inspect +import itertools +import warnings +from collections import OrderedDict +from collections.abc import Callable +from typing import Any, Concatenate, TypeVar +from typing_extensions import deprecated, ParamSpec + +import torch +import torch._C as _C +import torch._functorch as _functorch +import torch.utils.hooks as hooks +from torch._C import _functions +from torch._functorch.autograd_function import custom_function_call + + +__all__ = [ + "FunctionCtx", + "BackwardCFunction", + "FunctionMeta", + "Function", + "once_differentiable", + "InplaceFunction", + "NestedIOFunction", +] + +# Unique id provider for each class inheriting from Function +# This is incremented in FunctionMeta during class definition +AUTOGRAD_FUNCTION_COUNTER = itertools.count() + +_T = TypeVar("_T") +_R = TypeVar("_R") +_P = ParamSpec("_P") + + +# Formerly known as: _ContextMethodMixin +class FunctionCtx: + def save_for_backward(self, *tensors: torch.Tensor): + r"""Save given tensors for a future call to :func:`~Function.backward`. + + ``save_for_backward`` should be called at most once, in either the + :func:`setup_context` or :func:`forward` methods, and only with tensors. + + All tensors intended to be used in the backward pass should be saved + with ``save_for_backward`` (as opposed to directly on ``ctx``) to prevent + incorrect gradients and memory leaks, and enable the application of saved + tensor hooks. See :class:`torch.autograd.graph.saved_tensors_hooks`. + See :ref:`extending-autograd` for more details. + + Note that if intermediary tensors, tensors that are neither inputs + nor outputs of :func:`forward`, are saved for backward, your custom Function + may not support double backward. + Custom Functions that do not support double backward should decorate their + :func:`backward` method with ``@once_differentiable`` so that performing + double backward raises an error. If you'd like to support double backward, + you can either recompute intermediaries based on the inputs during backward + or return the intermediaries as the outputs of the custom Function. See the + `double backward tutorial `_ + for more details. + + In :func:`backward`, saved tensors can be accessed through the :attr:`saved_tensors` + attribute. Before returning them to the user, a check is made to ensure + they weren't used in any in-place operation that modified their content. + + Arguments can also be ``None``. This is a no-op. + + See :ref:`extending-autograd` for more details on how to use this method. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class Func(Function): + >>> @staticmethod + >>> def forward(ctx, x: torch.Tensor, y: torch.Tensor, z: int): + >>> w = x * z + >>> out = x * y + y * z + w * y + >>> ctx.save_for_backward(x, y, w, out) + >>> ctx.z = z # z is not a tensor + >>> return out + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, grad_out): + >>> x, y, w, out = ctx.saved_tensors + >>> z = ctx.z + >>> gx = grad_out * (y + y * z) + >>> gy = grad_out * (x + z + w) + >>> gz = None + >>> return gx, gy, gz + >>> + >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double) + >>> b = torch.tensor(2., requires_grad=True, dtype=torch.double) + >>> c = 4 + >>> d = Func.apply(a, b, c) + + """ + self.to_save = tensors + + def save_for_forward(self, *tensors: torch.Tensor): + r"""Save given tensors for a future call to :func:`~Function.jvp`. + + ``save_for_forward`` should be called at most once, in either the + :func:`setup_context` or :func:`forward` methods, and all arguments + should be tensors. + + In :func:`jvp`, saved objects can be accessed through the :attr:`saved_tensors` + attribute. + + Arguments can also be ``None``. This is a no-op. + + See :ref:`extending-autograd` for more details on how to use this method. + + Example:: + + >>> # xdoctest: +SKIP + >>> class Func(torch.autograd.Function): + >>> @staticmethod + >>> def forward(ctx, x: torch.Tensor, y: torch.Tensor, z: int): + >>> ctx.save_for_backward(x, y) + >>> ctx.save_for_forward(x, y) + >>> ctx.z = z + >>> return x * y * z + >>> + >>> @staticmethod + >>> def jvp(ctx, x_t, y_t, _): + >>> x, y = ctx.saved_tensors + >>> z = ctx.z + >>> return z * (y * x_t + x * y_t) + >>> + >>> @staticmethod + >>> def vjp(ctx, grad_out): + >>> x, y = ctx.saved_tensors + >>> z = ctx.z + >>> return z * grad_out * y, z * grad_out * x, None + >>> + >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double) + >>> t = torch.tensor(1., dtype=torch.double) + >>> b = torch.tensor(2., requires_grad=True, dtype=torch.double) + >>> c = 4 + >>> + >>> with fwAD.dual_level(): + >>> a_dual = fwAD.make_dual(a, t) + >>> d = Func.apply(a_dual, b, c) + + """ + for tensor in tensors: + if not (isinstance(tensor, torch.Tensor) or tensor is None): + raise AssertionError( + "save_for_forward expects all arguments to be tensors; you should " + "save non-tensors as attributes on ctx." + ) + + self.saved_for_forward = tensors + + def mark_dirty(self, *args: torch.Tensor): + r"""Mark given tensors as modified in an in-place operation. + + This should be called at most once, in either the :func:`setup_context` + or :func:`forward` methods, and all arguments should be inputs. + + Every tensor that's been modified in-place in a call to :func:`forward` + should be given to this function, to ensure correctness of our checks. + It doesn't matter whether the function is called before or after + modification. + + Examples:: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class Inplace(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> x_npy = x.numpy() # x_npy shares storage with x + >>> x_npy += 1 + >>> ctx.mark_dirty(x) + >>> return x + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, grad_output): + >>> return grad_output + >>> + >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double).clone() + >>> b = a * a + >>> Inplace.apply(a) # This would lead to wrong gradients! + >>> # but the engine would not know unless we mark_dirty + >>> # xdoctest: +SKIP + >>> b.backward() # RuntimeError: one of the variables needed for gradient + >>> # computation has been modified by an inplace operation + + """ + self.dirty_tensors = args + + @deprecated( + "`mark_shared_storage` is deprecated. " + "Tensors with shared storages are automatically tracked. " + "Note that calls to `set_()` are not tracked", + category=FutureWarning, + ) + def mark_shared_storage(self, *pairs): + pass + + def mark_non_differentiable(self, *args: torch.Tensor): + r"""Mark outputs as non-differentiable. + + This should be called at most once, in either the :func:`setup_context` + or :func:`forward` methods, and all arguments should be tensor outputs. + + This will mark outputs as not requiring gradients, increasing the + efficiency of backward computation. You still need to accept a gradient + for each output in :meth:`~Function.backward`, but it's always going to + be a zero tensor with the same shape as the shape of a corresponding + output. + + This is used e.g. for indices returned from a sort. See example:: + >>> class Func(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> sorted, idx = x.sort() + >>> ctx.mark_non_differentiable(idx) + >>> ctx.save_for_backward(x, idx) + >>> return sorted, idx + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, g1, g2): # still need to accept g2 + >>> x, idx = ctx.saved_tensors + >>> grad_input = torch.zeros_like(x) + >>> grad_input.index_add_(0, idx, g1) + >>> return grad_input + + """ + self.non_differentiable = args + + def set_materialize_grads(self, value: bool): + r"""Set whether to materialize grad tensors. Default is ``True``. + + This should be called only from either the :func:`setup_context` or + :func:`forward` methods. + + If ``True``, undefined grad tensors will be expanded to tensors full of zeros + prior to calling the :func:`backward` and :func:`jvp` methods. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class SimpleFunc(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> return x.clone(), x.clone() + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, g1, g2): + >>> return g1 + g2 # No check for None necessary + >>> + >>> # We modify SimpleFunc to handle non-materialized grad outputs + >>> class Func(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> ctx.set_materialize_grads(False) + >>> ctx.save_for_backward(x) + >>> return x.clone(), x.clone() + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, g1, g2): + >>> x, = ctx.saved_tensors + >>> grad_input = torch.zeros_like(x) + >>> if g1 is not None: # We must check for None now + >>> grad_input += g1 + >>> if g2 is not None: + >>> grad_input += g2 + >>> return grad_input + >>> + >>> a = torch.tensor(1., requires_grad=True) + >>> b, _ = Func.apply(a) # induces g2 to be undefined + + """ + self.materialize_grads = value + + +# DO NOT USE: This is only defined to be able to load old serialized models +_ContextMethodMixin = FunctionCtx + + +class _HookMixin: + @staticmethod + def _register_hook(backward_hooks, hook): + if backward_hooks is None: + backward_hooks = OrderedDict() + handle = hooks.RemovableHandle(backward_hooks) + backward_hooks[handle.id] = hook + return backward_hooks, handle + + +class BackwardCFunction(_C._FunctionBase, FunctionCtx, _HookMixin): + r""" + This class is used for internal autograd work. Do not use. + """ + + def _get_user_fn(self): + backward_fn = self._forward_cls.backward # type: ignore[attr-defined] + vjp_fn = self._forward_cls.vjp # type: ignore[attr-defined] + if backward_fn is not Function.backward and vjp_fn is not Function.vjp: + raise RuntimeError( + "Implementing both 'backward' and 'vjp' for a custom " + "Function is not allowed. You should only implement one " + "of them." + ) + return vjp_fn if vjp_fn is not Function.vjp else backward_fn + + def apply(self, *args): + r""" + Apply method used when executing this Node during the backward. + + Called by the autograd engine (non-boxed path) and by direct + grad_fn.apply() calls. When boxed_grads_call is True, boxes + grads into a mutable list before calling user's backward. + """ + user_fn = self._get_user_fn() + fwd_cls = self._forward_cls # type: ignore[attr-defined] # pyrefly: ignore[missing-attribute] + if getattr(fwd_cls, "boxed_grads_call", False): + args = (list(args),) + return user_fn(self, *args) + + def apply_boxed(self, *args): + r""" + Apply method called by the autograd engine when boxed_grads_call + is True. Grads arrive as a single mutable list argument, allowing + backward to free individual grads mid-execution. + """ + return self._get_user_fn()(self, *args) + + def apply_jvp(self, *args): + r""" + Apply method used when executing forward mode AD during the forward + """ + # _forward_cls is defined by derived class + return self._forward_cls.jvp(self, *args) # type: ignore[attr-defined] + + def _compiled_autograd_key(self): + return self._forward_cls._compiled_autograd_key(self) # type: ignore[attr-defined] + + +class FunctionMeta(type): + """Function metaclass. + + This metaclass sets up the following properties: + _backward_cls: The Function class corresponding to the differentiated + version of this function (which is generated on the fly by this + metaclass). + """ + + def __init__(cls, name, bases, attrs): + backward_fn = type( + name + "Backward", (BackwardCFunction,), {"_forward_cls": cls} + ) + backward_fn._autograd_function_id = next(AUTOGRAD_FUNCTION_COUNTER) # type: ignore[attr-defined] + cls._backward_cls = backward_fn + + super().__init__(name, bases, attrs) + + +class _SingleLevelFunction( + _C._FunctionBase, FunctionCtx, _HookMixin, metaclass=FunctionMeta +): + @staticmethod + def forward(*args: Any, **kwargs: Any) -> Any: + r"""Define the forward of the custom autograd Function. + + This function is to be overridden by all subclasses. + There are two ways to define forward: + + Usage 1 (Combined forward and ctx):: + + @staticmethod + def forward(ctx: Any, *args: Any, **kwargs: Any) -> Any: + pass + + - It must accept a context ctx as the first argument, followed by any + number of arguments (tensors or other types). + - See :ref:`combining-forward-context` for more details + + Usage 2 (Separate forward and ctx):: + + @staticmethod + def forward(*args: Any, **kwargs: Any) -> Any: + pass + + + @staticmethod + def setup_context(ctx: Any, inputs: Tuple[Any, ...], output: Any) -> None: + pass + + - The forward no longer accepts a ctx argument. + - Instead, you must also override the :meth:`torch.autograd.Function.setup_context` + staticmethod to handle setting up the ``ctx`` object. + ``output`` is the output of the forward, ``inputs`` are a Tuple of inputs + to the forward. + - See :ref:`extending-autograd` for more details + + The context can be used to store arbitrary data that can be then + retrieved during the backward pass. Tensors should not be stored + directly on `ctx` (though this is not currently enforced for + backward compatibility). Instead, tensors should be saved either with + :func:`ctx.save_for_backward` if they are intended to be used in + ``backward`` (equivalently, ``vjp``) or :func:`ctx.save_for_forward` + if they are intended to be used for in ``jvp``. + """ + raise NotImplementedError( + "You must implement the forward function for custom autograd.Function." + ) + + @staticmethod + def setup_context(ctx: Any, inputs: tuple[Any, ...], output: Any) -> Any: + r"""There are two ways to define the forward pass of an autograd.Function. + + Either: + + 1. Override forward with the signature ``forward(ctx, *args, **kwargs)``. + ``setup_context`` is not overridden. Setting up the ctx for backward + happens inside the ``forward``. + 2. Override forward with the signature ``forward(*args, **kwargs)`` and + override ``setup_context``. Setting up the ctx for backward happens + inside ``setup_context`` (as opposed to inside the ``forward``) + + See :meth:`torch.autograd.Function.forward` and :ref:`extending-autograd` for more details. + """ + raise NotImplementedError("setup_context is not implemented.") + + @staticmethod + def backward(ctx: Any, *grad_outputs: Any) -> Any: + r"""Define a formula for differentiating the operation with backward mode automatic differentiation. + + This function is to be overridden by all subclasses. + (Defining this function is equivalent to defining the ``vjp`` function.) + + It must accept a context :attr:`ctx` as the first argument, followed by + as many outputs as the :func:`forward` returned (None will be passed in + for non tensor outputs of the forward function), + and it should return as many tensors, as there were inputs to + :func:`forward`. Each argument is the gradient w.r.t the given output, + and each returned value should be the gradient w.r.t. the + corresponding input. If an input is not a Tensor or is a Tensor not + requiring grads, you can just pass None as a gradient for that input. + + The context can be used to retrieve tensors saved during the forward + pass. It also has an attribute :attr:`ctx.needs_input_grad` as a tuple + of booleans representing whether each input needs gradient. E.g., + :func:`backward` will have ``ctx.needs_input_grad[0] = True`` if the + first input to :func:`forward` needs gradient computed w.r.t. the + output. + """ + raise NotImplementedError( + "You must implement either the backward or vjp method for " + "your custom autograd.Function to use it with backward " + "mode AD." + ) + + # vjp and backward are alias of each other + vjp = backward + + """ + Bool that specifies if PyTorch should clear saved tensors after the first + access to ``ctx.saved_tensors``. When set to True, accessing saved_tensors + clears the internal references, allowing the tensors to be cleared as soon + as the Tensor returned by saved_tensors is deleted. + + This is useful for reducing memory pressure in backward passes when you + only need to access saved tensors once. + + Default is False. + """ + clear_saved_tensors_on_access = False + + """ + Bool that specifies if backward should receive grads as a single mutable + list argument instead of individual args in an immutable tuple. This allows + backward to free individual grads mid-execution by removing them from the + list, reducing peak memory. + + When True, ``backward(ctx, grads)`` receives a single list instead of + ``backward(ctx, *grads)``. + + Default is False. + """ + boxed_grads_call = False + + @staticmethod + def jvp(ctx: Any, *grad_inputs: Any) -> Any: + r"""Define a formula for differentiating the operation with forward mode automatic differentiation. + + This function is to be overridden by all subclasses. + It must accept a context :attr:`ctx` as the first argument, followed by + as many inputs as the :func:`forward` got (None will be passed in + for non tensor inputs of the forward function), + and it should return as many tensors as there were outputs to + :func:`forward`. Each argument is the gradient w.r.t the given input, + and each returned value should be the gradient w.r.t. the + corresponding output. If an output is not a Tensor or the function is not + differentiable with respect to that output, you can just pass None as a + gradient for that input. + + You can use the :attr:`ctx` object to pass any value from the forward to this + functions. + """ + raise NotImplementedError( + "You must implement the jvp function for custom " + "autograd.Function to use it with forward mode AD." + ) + + +class Function(_SingleLevelFunction): + r"""Base class to create custom `autograd.Function`. + + To create a custom `autograd.Function`, subclass this class and implement + the :meth:`forward` and :meth:`backward` static methods. Then, to use your custom + op in the forward pass, call the class method ``apply``. Do not call + :meth:`forward` directly. + + To ensure correctness and best performance, make sure you are calling the + correct methods on ``ctx`` and validating your backward function using + :func:`torch.autograd.gradcheck`. + + See :ref:`extending-autograd` for more details on how to use this class. + + Examples:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class Exp(Function): + >>> @staticmethod + >>> def forward(ctx, i): + >>> result = i.exp() + >>> ctx.save_for_backward(result) + >>> return result + >>> + >>> @staticmethod + >>> def backward(ctx, grad_output): + >>> result, = ctx.saved_tensors + >>> return grad_output * result + >>> + >>> # Use it by calling the apply method: + >>> # xdoctest: +SKIP + >>> output = Exp.apply(input) + """ + + def __init__(self, *args, **kwargs): + warnings.warn( + f"{self.__class__} should not be instantiated. Methods on autograd functions " + "are all static, so you should invoke them on the class itself. " + "Instantiating an autograd function will raise an " + "error in a future version of PyTorch.", + DeprecationWarning, + stacklevel=2, + ) + + def __call__(self, *args, **kwargs): + raise RuntimeError( + "Legacy autograd function with non-static forward method is deprecated. " + "Please use new-style autograd function with static forward method. " + "(Example: https://pytorch.org/docs/stable/autograd.html#torch.autograd.Function)" + ) + + """ + Bool that specifies if PyTorch should attempt to autogenerate + :func:`torch.vmap` support for this autograd.Function. You may set this to + True only if this autograd.Function's forward, backward, and jvp (if they + exist) are written using PyTorch operations; otherwise, please override + :meth:`torch.autograd.Function.vmap` to add support for :func:`torch.vmap`. + + Please see :ref:`func-autograd-function` for more details. + """ + generate_vmap_rule = False + + @staticmethod + def vmap(info, in_dims, *args): + r"""Define the behavior for this autograd.Function underneath :func:`torch.vmap`. + + For a :func:`torch.autograd.Function` to support + :func:`torch.vmap`, you must either override this static method, or set + ``generate_vmap_rule`` to ``True`` (you may not do both). + + If you choose to override this staticmethod: it must accept + + - an ``info`` object as the first argument. ``info.batch_size`` + specifies the size of the dimension being vmapped over, + while ``info.randomness`` is the randomness option passed to + :func:`torch.vmap`. + - an ``in_dims`` tuple as the second argument. + For each arg in ``args``, ``in_dims`` has a corresponding + ``Optional[int]``. It is ``None`` if the arg is not a Tensor or if + the arg is not being vmapped over, otherwise, it is an integer + specifying what dimension of the Tensor is being vmapped over. + - ``*args``, which is the same as the args to :meth:`~Function.forward`. + + The return of the vmap staticmethod is a tuple of ``(output, out_dims)``. + Similar to ``in_dims``, ``out_dims`` should be of the same structure as + ``output`` and contain one ``out_dim`` per output that specifies if the + output has the vmapped dimension and what index it is in. + + Please see :ref:`func-autograd-function` for more details. + """ + raise NotImplementedError( + "To use autograd.Function with vmap, you must either override the " + "vmap staticmethod or set generate_vmap_rule=True." + ) + + @classmethod + def apply(cls, *args, **kwargs): + def bind_default_args(func, *args, **kwargs): + signature = inspect.signature(func) + bound_args = signature.bind(*args, **kwargs) + bound_args.apply_defaults() + + return bound_args.args, bound_args.kwargs + + is_setup_ctx_defined = _is_setup_context_defined(cls.setup_context) + if is_setup_ctx_defined: + args, kwargs = bind_default_args(cls.forward, *args, **kwargs) + + if not torch._C._are_functorch_transforms_active(): + # See NOTE: [functorch vjp and autograd interaction] + args = _functorch.utils.unwrap_dead_wrappers(args) + return super().apply(*args, **kwargs) # type: ignore[misc] + + if not is_setup_ctx_defined: + raise RuntimeError( + "In order to use an autograd.Function with functorch transforms " + "(vmap, grad, jvp, jacrev, ...), it must override the setup_context " + "staticmethod. For more details, please see " + "https://pytorch.org/docs/main/notes/extending.func.html" + ) + + return custom_function_call(cls, *args, **kwargs) + + @staticmethod + def _compiled_autograd_key(ctx): + return (ctx._autograd_function_id,) + + +def _is_setup_context_defined(fn): + return fn != _SingleLevelFunction.setup_context + + +def once_differentiable( + fn: Callable[Concatenate[_T, _P], _R], +) -> Callable[Concatenate[_T, _P], _R]: + @functools.wraps(fn) + def wrapper(ctx: _T, *args: _P.args, **kwargs: _P.kwargs) -> _R: + with torch.no_grad(): + outputs = fn(ctx, *args, **kwargs) + + if not torch.is_grad_enabled(): + return outputs + + # If any of the inputs have requires_grad=True, we force the outputs + # to have requires_grad=True but point to a grad_fn which throws an + # error message during (double) back-propagation. + # XXX: this is only an approximation of requires_grad - there's no way + # to figure out if fn didn't use ctx.saved_tensors and as a result + # some Tensors might require grad, even if no args do. + # Unfortunately, this leads to unexpected error messages ("no nodes + # require computing gradients"), but I don't have a better idea. + # These functions would raise an error in backward anyway. + requires_grad = any( + isinstance(arg, torch.Tensor) and arg.requires_grad for arg in args + ) + if not requires_grad: + return outputs + + if not isinstance(outputs, tuple): + outputs_ = (outputs,) + else: + outputs_ = outputs + + err_fn = _functions.DelayedError( + b"trying to differentiate twice a function that was marked " + b"with @once_differentiable", + len(outputs_), + ) + + # Create aliases of each output that has requires_grad=True. We need + # at least one of the inputs to err_fn to require grad so that the + # output will have a grad_fn. + def fake_requires_grad(var): + if var is not None: + var = var.detach() + var.requires_grad = True + return var + + return err_fn(*[fake_requires_grad(v) for v in outputs_]) # type: ignore[return-value] + + return wrapper + + +class InplaceFunction(Function): + r""" + This class is here only for backward compatibility reasons. + Use :class:`Function` instead of this for any new use case. + """ + + def __init__(self, inplace=False): + super().__init__() + self.inplace = inplace + + +def _nested_map(condition, fn, condition_msg=None): + def _map(obj): + if condition(obj): + return fn(obj) + elif obj is None: + return None + elif isinstance(obj, (list, tuple)): + mapped = (_map(x) for x in obj) + if hasattr(obj, "_fields"): + # obj is namedtuple + return type(obj)(*mapped) + return type(obj)(mapped) + elif isinstance(obj, dict): + return {x: _map(obj[x]) for x in obj} + else: + raise ValueError( + "Auto nesting doesn't know how to process " + "an input object of type " + + torch.typename(obj) + + ( + ". Accepted types: " + condition_msg + ", or lists/tuples of them" + if condition_msg + else "" + ) + ) + + return _map + + +def _jit_unwrap_structured(obj): + if hasattr(obj, "_jit_unwrap"): + return obj._jit_unwrap() + return obj + + +def _iter_filter(condition, allow_unknown=False, condition_msg=None, conversion=None): + def _iter(obj): + if conversion is not None: + obj = conversion(obj) + if condition(obj): + yield obj + elif obj is None: + return + elif isinstance(obj, (list, tuple)): + for o in obj: + yield from _iter(o) + elif isinstance(obj, dict): + # We only accept primitive key types, so we needn't inspect them + for o in obj.values(): + yield from _iter(o) + elif allow_unknown: + yield obj + else: + raise ValueError( + "Auto nesting doesn't know how to process " + "an input object of type " + + torch.typename(obj) + + ( + ". Accepted types: " + condition_msg + ", or lists/tuples of them" + if condition_msg + else "" + ) + ) + + return _iter + + +def _unflatten(input, proto): + # unflatten a list or tuple input into a nested list/tuple structure + # specified by proto + def unflatten_helper(input, proto): + res: list[torch.Tensor | None] = [] + if hasattr(proto, "_jit_wrap"): + return proto._jit_wrap(input) + if not isinstance(proto, (list, tuple)): + return input[0], input[1:] + for e in proto: + if e is None: + res.append(e) + else: + res_e, input = unflatten_helper(input, e) + res.append(res_e) + return type(proto)(res), input + + return unflatten_helper(input, proto)[0] + + +_iter_jit_values = _iter_filter( + lambda o: o is None or isinstance(o, torch._C.Value), + condition_msg="jit's Values or None", +) +_iter_tensors = _iter_filter( + lambda x: isinstance(x, torch.Tensor), + condition_msg="Tensors", + conversion=_jit_unwrap_structured, +) +_iter_tensors_permissive = _iter_filter( + lambda x: isinstance(x, torch.Tensor), + allow_unknown=True, + condition_msg="Tensors (permissive)", +) +_iter_None_tensors = _iter_filter( + lambda o: o is None or isinstance(o, torch.Tensor), condition_msg="Tensors or None" +) +_map_tensor_data = _nested_map( + lambda x: isinstance(x, torch.Tensor), lambda o: o.data, condition_msg="Tensors" +) + + +class NestedIOFunction(Function): + r""" + This class is here only for backward compatibility reasons. + Use :class:`Function` instead of this for any new use case. + """ + + # The 'type: ignore' statements are needed here because these functions are declared as '@staticmethod' in the + # superclass (Function) but are instance methods here, which mypy reports as incompatible. + + def _do_forward(self, *input): + self._nested_input = input + flat_input = tuple(_iter_tensors(input)) + flat_output = super()._do_forward(*flat_input) # type: ignore[misc] + nested_tensors = _unflatten(flat_output, self._nested_output) + return nested_tensors + + def _do_backward(self, gradients, retain_variables): + self.retain_variables = retain_variables + result = super()._do_backward(gradients, retain_variables) # type: ignore[misc] + if not retain_variables: + del self._nested_output + del self._to_save_nested + return result + + def backward(self, *gradients: Any) -> Any: # type: ignore[override] + r""" + Shared backward utility. + """ + nested_gradients = _unflatten(gradients, self._nested_output) + result = self.backward_extended(*nested_gradients) # type: ignore[func-returns-value] + return tuple(_iter_None_tensors(result)) + + __call__ = _do_forward + + def forward(self, *args: Any) -> Any: # type: ignore[override] + r""" + Shared forward utility. + """ + nested_tensors = _map_tensor_data(self._nested_input) + result = self.forward_extended(*nested_tensors) # type: ignore[func-returns-value] + del self._nested_input + self._nested_output = result + return tuple(_iter_tensors(result)) + + def save_for_backward(self, *args: Any) -> None: + r""" + See :meth:`Function.save_for_backward`. + """ + self.to_save = tuple(_iter_tensors(args)) + self._to_save_nested = args + + @property + def saved_tensors(self): # type: ignore[override] + r""" + See :meth:`Function.saved_tensors`. + """ + flat_tensors = super().saved_tensors # type: ignore[misc] + return _unflatten(flat_tensors, self._to_save_nested) + + def mark_dirty(self, *args: Any, **kwargs: Any) -> None: + r""" + See :meth:`Function.mark_dirty`. + """ + self.dirty_tensors = tuple(_iter_tensors((args, kwargs))) + + def mark_non_differentiable(self, *args: Any, **kwargs: Any) -> None: + r""" + See :meth:`Function.mark_non_differentiable`. + """ + self.non_differentiable = tuple(_iter_tensors((args, kwargs))) + + def forward_extended(self, *input: Any) -> None: + r""" + User defined forward. + """ + raise NotImplementedError + + def backward_extended(self, *grad_output: Any) -> None: + r""" + User defined backward. + """ + raise NotImplementedError diff --git a/western_ai/usa/pytorch/torch/autograd/functional.py b/western_ai/usa/pytorch/torch/autograd/functional.py new file mode 100644 index 0000000000000000000000000000000000000000..a71f9232eaede16e1714b958ae4fe0cd4e7a3b86 --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/functional.py @@ -0,0 +1,1212 @@ +# mypy: allow-untyped-defs + +import torch +from torch._vmap_internals import _vmap + +from . import forward_ad as fwAD + + +__all__ = ["vjp", "jvp", "jacobian", "hessian", "hvp", "vhp"] + +# Utility functions + + +def _as_tuple_nocheck(x): + if isinstance(x, tuple): + return x + elif isinstance(x, list): + return tuple(x) + else: + return (x,) + + +def _as_tuple(inp, arg_name=None, fn_name=None): + # Ensures that inp is a tuple of Tensors + # Returns whether or not the original inp was a tuple and the tupled version of the input + if arg_name is None and fn_name is None: + return _as_tuple_nocheck(inp) + + is_inp_tuple = True + if not isinstance(inp, tuple): + inp = (inp,) + is_inp_tuple = False + + for i, el in enumerate(inp): + if not isinstance(el, torch.Tensor): + if is_inp_tuple: + raise TypeError( + f"The {arg_name} given to {fn_name} must be either a Tensor or a tuple of Tensors but the" + f" value at index {i} has type {type(el)}." + ) + else: + raise TypeError( + f"The {arg_name} given to {fn_name} must be either a Tensor or a tuple of Tensors but the" + f" given {arg_name} has type {type(el)}." + ) + + return is_inp_tuple, inp + + +def _tuple_postprocess(res, to_unpack): + # Unpacks a potentially nested tuple of Tensors + # to_unpack should be a single boolean or a tuple of two booleans. + # It is used to: + # - invert _as_tuple when res should match the inp given to _as_tuple + # - optionally remove nesting of two tuples created by multiple calls to _as_tuple + if isinstance(to_unpack, tuple): + if len(to_unpack) != 2: + raise AssertionError("Expected to_unpack tuple to have exactly 2 elements") + if not to_unpack[1]: + res = tuple(el[0] for el in res) + if not to_unpack[0]: + res = res[0] + else: + if not to_unpack: + res = res[0] + return res + + +def _grad_preprocess(inputs, create_graph, need_graph): + # Preprocess the inputs to make sure they require gradient + # inputs is a tuple of Tensors to preprocess + # create_graph specifies if the user wants gradients to flow back to the Tensors in inputs + # need_graph specifies if we internally want gradients to flow back to the Tensors in res + # Note that we *always* create a new Tensor object to be able to see the difference between + # inputs given as arguments and the same Tensors automatically captured by the user function. + # Check this issue for more details on how that can happen: https://github.com/pytorch/pytorch/issues/32576 + res = [] + for inp in inputs: + if create_graph and inp.requires_grad: + # Create at least a new Tensor object in a differentiable way + if not inp.is_sparse: + # Use .view_as() to get a shallow copy + res.append(inp.view_as(inp)) + else: + # We cannot use view for sparse Tensors so we clone + res.append(inp.clone()) + else: + res.append(inp.detach().requires_grad_(need_graph)) + return tuple(res) + + +def _grad_postprocess(inputs, create_graph): + # Postprocess the generated Tensors to avoid returning Tensors with history when the user did not + # request it. + if isinstance(inputs[0], torch.Tensor): + if not create_graph: + return tuple(inp.detach() for inp in inputs) + else: + return inputs + else: + return tuple(_grad_postprocess(inp, create_graph) for inp in inputs) + + +def _validate_v(v, other, is_other_tuple): + # This assumes that other is the correct shape, and v should match + # Both are assumed to be tuples of Tensors + if len(other) != len(v): + if is_other_tuple: + raise RuntimeError( + f"v is a tuple of invalid length: should be {len(other)} but got {len(v)}." + ) + else: + raise RuntimeError("The given v should contain a single Tensor.") + + for idx, (el_v, el_other) in enumerate(zip(v, other)): + if el_v.size() != el_other.size(): + prepend = "" + if is_other_tuple: + prepend = f"Entry {idx} in " + raise RuntimeError( + f"{prepend}v has invalid size: should be {el_other.size()} but got {el_v.size()}." + ) + + +def _check_requires_grad(inputs, input_type, strict): + # Used to make all the necessary checks to raise nice errors in strict mode. + if not strict: + return + + if input_type not in ["outputs", "grad_inputs", "jacobian", "hessian"]: + raise RuntimeError("Invalid input_type to _check_requires_grad") + for i, inp in enumerate(inputs): + if inp is None: + # This can only be reached for grad_inputs. + raise RuntimeError( + f"The output of the user-provided function is independent of input {i}." + " This is not allowed in strict mode." + ) + if not inp.requires_grad: + if input_type == "hessian": + raise RuntimeError( + f"The hessian of the user-provided function with respect to input {i}" + " is independent of the input. This is not allowed in strict mode." + " You should ensure that your function is thrice differentiable and that" + " the hessian depends on the inputs." + ) + elif input_type == "jacobian": + raise RuntimeError( + "While computing the hessian, found that the jacobian of the user-provided" + f" function with respect to input {i} is independent of the input. This is not" + " allowed in strict mode. You should ensure that your function is twice" + " differentiable and that the jacobian depends on the inputs (this would be" + " violated by a linear function for example)." + ) + elif input_type == "grad_inputs": + raise RuntimeError( + f"The gradient with respect to input {i} is independent of the inputs of the" + " user-provided function. This is not allowed in strict mode." + ) + else: + raise RuntimeError( + f"Output {i} of the user-provided function does not require gradients." + " The outputs must be computed in a differentiable manner from the input" + " when running in strict mode." + ) + + +def _autograd_grad( + outputs, + inputs, + grad_outputs=None, + create_graph=False, + retain_graph=None, + is_grads_batched=False, +): + # Version of autograd.grad that accepts `None` in outputs and do not compute gradients for them. + # This has the extra constraint that inputs has to be a tuple + if not isinstance(outputs, tuple): + raise AssertionError("Expected outputs to be a tuple") + if grad_outputs is None: + grad_outputs = (None,) * len(outputs) + if not isinstance(grad_outputs, tuple): + raise AssertionError("Expected grad_outputs to be a tuple") + if len(outputs) != len(grad_outputs): + raise AssertionError( + f"Expected outputs and grad_outputs to have the same length, " + f"but got {len(outputs)} and {len(grad_outputs)}" + ) + + new_outputs: tuple[torch.Tensor, ...] = () + new_grad_outputs: tuple[torch.Tensor, ...] = () + for out, grad_out in zip(outputs, grad_outputs): + if out is not None and out.requires_grad: + new_outputs += (out,) + # pyrefly: ignore [bad-assignment] + new_grad_outputs += (grad_out,) + + if len(new_outputs) == 0: + # No differentiable output, we don't need to call the autograd engine + return (None,) * len(inputs) + else: + return torch.autograd.grad( + new_outputs, + inputs, + new_grad_outputs, + allow_unused=True, + create_graph=create_graph, + retain_graph=retain_graph, + is_grads_batched=is_grads_batched, + ) + + +def _fill_in_zeros(grads, refs, strict, create_graph, stage): + # Used to detect None in the grads and depending on the flags, either replace them + # with Tensors full of 0s of the appropriate size based on the refs or raise an error. + # strict and create graph allow us to detect when it is appropriate to raise an error + # stage gives us information of which backward call we consider to give good error message + if stage not in ["back", "back_trick", "double_back", "double_back_trick"]: + raise RuntimeError(f"Invalid stage argument '{stage}' to _fill_in_zeros") + + res: tuple[torch.Tensor, ...] = () + for i, grads_i in enumerate(grads): + if grads_i is None: + if strict: + if stage == "back": + raise RuntimeError( + "The output of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode." + ) + elif stage == "back_trick": + raise RuntimeError( + f"The gradient with respect to the input is independent of entry {i}" + " in the grad_outputs when using the double backward trick to compute" + " forward mode gradients. This is not allowed in strict mode." + ) + elif stage == "double_back": + raise RuntimeError( + "The jacobian of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode." + ) + else: + raise RuntimeError( + "The hessian of the user-provided function is independent of " + f"entry {i} in the grad_jacobian. This is not allowed in strict " + "mode as it prevents from using the double backward trick to " + "replace forward mode AD." + ) + + grads_i = torch.zeros_like(refs[i]) + else: + if strict and create_graph and not grads_i.requires_grad: + if "double" not in stage: + raise RuntimeError( + "The jacobian of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode when create_graph=True." + ) + else: + raise RuntimeError( + "The hessian of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode when create_graph=True." + ) + + res += (grads_i,) + + return res + + +# Public API + + +def vjp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between a vector ``v`` and the Jacobian of the given function at the point given by the inputs. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a tuple of Tensors or a Tensor. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the vector + Jacobian product is computed. Must be the same size as the output + of ``func``. This argument is optional when the output of ``func`` + contains a single element and (if it is not provided) will be set + as a Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result + will be computed in a differentiable way. Note that when ``strict`` + is ``False``, the result can not require gradients or be + disconnected from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + vjp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + vjp (tuple of Tensors or Tensor): result of the dot product with + the same shape as the inputs. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def exp_reducer(x): + ... return x.exp().sum(dim=1) + >>> inputs = torch.rand(4, 4) + >>> v = torch.ones(4) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> vjp(exp_reducer, inputs, v) + (tensor([5.7817, 7.2458, 5.7830, 6.7782]), + tensor([[1.4458, 1.3962, 1.3042, 1.6354], + [2.1288, 1.0652, 1.5483, 2.5035], + [2.2046, 1.1292, 1.1432, 1.3059], + [1.3225, 1.6652, 1.7753, 2.0152]])) + + >>> vjp(exp_reducer, inputs, v, create_graph=True) + (tensor([5.7817, 7.2458, 5.7830, 6.7782], grad_fn=), + tensor([[1.4458, 1.3962, 1.3042, 1.6354], + [2.1288, 1.0652, 1.5483, 2.5035], + [2.2046, 1.1292, 1.1432, 1.3059], + [1.3225, 1.6652, 1.7753, 2.0152]], grad_fn=)) + + >>> def adder(x, y): + ... return 2 * x + 3 * y + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = torch.ones(2) + >>> vjp(adder, inputs, v) + (tensor([2.4225, 2.3340]), + (tensor([2., 2.]), tensor([3., 3.]))) + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "vjp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "vjp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if v is not None: + _, v = _as_tuple(v, "v", "vjp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, outputs, is_outputs_tuple) + else: + if len(outputs) != 1 or outputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the " + "user-provided function returns " + "a single Tensor with a single element." + ) + + enable_grad = True if create_graph else torch.is_grad_enabled() + with torch.set_grad_enabled(enable_grad): + grad_res = _autograd_grad(outputs, inputs, v, create_graph=create_graph) + vjp = _fill_in_zeros(grad_res, inputs, strict, create_graph, "back") + + # Cleanup objects and return them to the user + outputs = _grad_postprocess(outputs, create_graph) + vjp = _grad_postprocess(vjp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + vjp, is_inputs_tuple + ) + + +def jvp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between the Jacobian of the given function at the point given by the inputs and a vector ``v``. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a tuple of Tensors or a Tensor. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the Jacobian + vector product is computed. Must be the same size as the input of + ``func``. This argument is optional when the input to ``func`` + contains a single element and (if it is not provided) will be set + as a Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result + will be computed in a differentiable way. Note that when ``strict`` + is ``False``, the result can not require gradients or be + disconnected from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + jvp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + jvp (tuple of Tensors or Tensor): result of the dot product with + the same shape as the output. + + Note: + ``autograd.functional.jvp`` computes the jvp by using the backward of + the backward (sometimes called the double backwards trick). This is not + the most performant way of computing the jvp. Please consider using + :func:`torch.func.jvp` or the + :ref:`low-level forward-mode AD API ` instead. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def exp_reducer(x): + ... return x.exp().sum(dim=1) + >>> inputs = torch.rand(4, 4) + >>> v = torch.ones(4, 4) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> jvp(exp_reducer, inputs, v) + (tensor([6.3090, 4.6742, 7.9114, 8.2106]), + tensor([6.3090, 4.6742, 7.9114, 8.2106])) + + >>> jvp(exp_reducer, inputs, v, create_graph=True) + (tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=), + tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=)) + + >>> def adder(x, y): + ... return 2 * x + 3 * y + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = (torch.ones(2), torch.ones(2)) + >>> jvp(adder, inputs, v) + (tensor([2.2399, 2.5005]), + tensor([5., 5.])) + + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jvp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + if v is not None: + _, v = _as_tuple(v, "v", "jvp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, inputs, is_inputs_tuple) + else: + if len(inputs) != 1 or inputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the input to " + "the user-provided function is a single Tensor " + "with a single element." + ) + + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "jvp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + # The backward is linear so the value of grad_outputs is not important as + # it won't appear in the double backward graph. We only need to ensure that + # it does not contain inf or nan. + grad_outputs = tuple( + torch.zeros_like(out, requires_grad=True) for out in outputs + ) + + grad_inputs = _autograd_grad(outputs, inputs, grad_outputs, create_graph=True) + _check_requires_grad(grad_inputs, "grad_inputs", strict=strict) + + if create_graph: + with torch.enable_grad(): + grad_res = _autograd_grad( + grad_inputs, grad_outputs, v, create_graph=create_graph + ) + jvp = _fill_in_zeros(grad_res, outputs, strict, create_graph, "back_trick") + else: + grad_res = _autograd_grad( + grad_inputs, grad_outputs, v, create_graph=create_graph + ) + jvp = _fill_in_zeros(grad_res, outputs, strict, create_graph, "back_trick") + + # Cleanup objects and return them to the user + outputs = _grad_postprocess(outputs, create_graph) + jvp = _grad_postprocess(jvp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + jvp, is_outputs_tuple + ) + + +def _construct_standard_basis_for( + tensors: tuple[torch.Tensor, ...], tensor_numels: tuple[int, ...] +) -> tuple[torch.Tensor, ...]: + # This function: + # - constructs a N=sum(tensor_numels) standard basis. i.e. an NxN identity matrix. + # - Splits the identity matrix into chunks with each chunk size determined by `tensor_numels`. + # - Each chunk corresponds to one tensor. The chunk has the same dtype and + # device as the tensor + # + # For example, with tensor_numels = [1, 2, 1], this function returns: + # ( tensor([[1], tensor([[0, 0], tensor([[0], + # [0], [1, 0], [0], + # [0], [0, 1], [0], + # [0]]) , [0, 0]]) , [1]]) ) + # + # Precondition: tensor_numels == tuple(tensor.numel() for tensor in tensors) + # Precondition: tensors always has at least one element. + # + # See NOTE: [Computing jacobian with vmap and grad for multiple tensors] + # for context behind this function. All the pre-conditions are guarded for + # in torch.autograd.functional.jacobian. + if len(tensors) != len(tensor_numels): + raise AssertionError( + f"Expected tensors and tensor_numels to have the same length, " + f"but got {len(tensors)} and {len(tensor_numels)}" + ) + if len(tensors) == 0: + raise AssertionError("Expected at least one tensor") + total_numel = sum(tensor_numels) + chunks = tuple( + tensor.new_zeros(total_numel, tensor_numel) + for tensor, tensor_numel in zip(tensors, tensor_numels) + ) + diag_start_idx = 0 + for chunk, numel in zip(chunks, tensor_numels): + chunk.diagonal(diag_start_idx).fill_(1) + diag_start_idx -= numel + return chunks + + +def _jacfwd(func, inputs, strict=False, vectorize=False): + if strict: + raise RuntimeError( + "torch.autograd.functional.jacobian: `strict=True` " + 'and `strategy="forward-mode"` are not supported together (yet). ' + "Please either set `strict=False` or " + '`strategy="reverse-mode"`.' + ) + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jacobian") + output_info = [] + + if vectorize: + # See NOTE: [Computing jacobian with vmap and grad for multiple outputs] + input_numels = tuple(input.numel() for input in inputs) + + # Step 1: Prepare tangents + tangents = _construct_standard_basis_for(inputs, input_numels) + + # Step 2: Compute vmap over computation with dual tensors + def jvp(tangents): + with fwAD.dual_level(): + dual_inputs = tuple( + fwAD.make_dual(input, tangent.view_as(input)) + for input, tangent in zip(inputs, tangents) + ) + _is_outputs_tuple, dual_outputs = _as_tuple( + func(*dual_inputs), "outputs" + ) + output_info.append(_is_outputs_tuple) + jv = [] + primal_outs = [] + for dual_out in dual_outputs: + primal, tangent = fwAD.unpack_dual(dual_out) + primal_outs.append(primal) + if tangent is not None: + jv.append(tangent) + else: + jv.append(torch.zeros_like(primal)) + output_info.append(primal_outs) + return tuple(jv) + + outputs_before_split = _vmap(jvp)(tangents) + is_outputs_tuple, outputs = output_info + # Step 3: for each of the output tangents, split along dim 0 + jacobian_input_output = [] + for jac_output_i, output_i in zip(outputs_before_split, outputs): + jacobian_output_i_output = [] + for jac, input_j in zip(jac_output_i.split(input_numels, dim=0), inputs): + # We need to transpose the Jacobian because in forward AD, the + # batch dimension represents that of the inputs + jacobian_input_i_output_j = jac.permute(*range(1, jac.ndim), 0).reshape( + (*output_i.shape, *input_j.shape) + ) + + jacobian_output_i_output.append(jacobian_input_i_output_j) + jacobian_input_output.append(jacobian_output_i_output) + + # Omit [Step 4] because everything is already transposed w/ forward AD + return _tuple_postprocess( + jacobian_input_output, (is_outputs_tuple, is_inputs_tuple) + ) + else: + raise NotImplementedError( + "Computing Jacobian using forward-AD or forward-over-reverse Hessian is" + "only implemented for `vectorize=True`." + ) + + +def jacobian( + func, + inputs, + create_graph=False, + strict=False, + vectorize=False, + strategy="reverse-mode", +): + r"""Compute the Jacobian of a given function. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a tuple of Tensors or a Tensor. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + create_graph (bool, optional): If ``True``, the Jacobian will be + computed in a differentiable manner. Note that when ``strict`` is + ``False``, the result can not require gradients or be disconnected + from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + jacobian for said inputs, which is the expected mathematical value. + Defaults to ``False``. + vectorize (bool, optional): This feature is experimental. + Please consider using :func:`torch.func.jacrev` or + :func:`torch.func.jacfwd` instead if you are looking for something + less experimental and more performant. + When computing the jacobian, usually we invoke + ``autograd.grad`` once per row of the jacobian. If this flag is + ``True``, we perform only a single ``autograd.grad`` call with + ``batched_grad=True`` which uses the vmap prototype feature. + Though this should lead to performance improvements in many cases, + because this feature is still experimental, there may be performance + cliffs. See :func:`torch.autograd.grad`'s ``batched_grad`` parameter for + more information. + strategy (str, optional): Set to ``"forward-mode"`` or ``"reverse-mode"`` to + determine whether the Jacobian will be computed with forward or reverse + mode AD. Currently, ``"forward-mode"`` requires ``vectorized=True``. + Defaults to ``"reverse-mode"``. If ``func`` has more outputs than + inputs, ``"forward-mode"`` tends to be more performant. Otherwise, + prefer to use ``"reverse-mode"``. + + Returns: + Jacobian (Tensor or nested tuple of Tensors): if there is a single + input and output, this will be a single Tensor containing the + Jacobian for the linearized inputs and output. If one of the two is + a tuple, then the Jacobian will be a tuple of Tensors. If both of + them are tuples, then the Jacobian will be a tuple of tuple of + Tensors where ``Jacobian[i][j]`` will contain the Jacobian of the + ``i``\th output and ``j``\th input and will have as size the + concatenation of the sizes of the corresponding output and the + corresponding input and will have same dtype and device as the + corresponding input. If strategy is ``forward-mode``, the dtype will be + that of the output; otherwise, the input. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def exp_reducer(x): + ... return x.exp().sum(dim=1) + >>> inputs = torch.rand(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> jacobian(exp_reducer, inputs) + tensor([[[1.4917, 2.4352], + [0.0000, 0.0000]], + [[0.0000, 0.0000], + [2.4369, 2.3799]]]) + + >>> jacobian(exp_reducer, inputs, create_graph=True) + tensor([[[1.4917, 2.4352], + [0.0000, 0.0000]], + [[0.0000, 0.0000], + [2.4369, 2.3799]]], grad_fn=) + + >>> def exp_adder(x, y): + ... return 2 * x.exp() + 3 * y + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> jacobian(exp_adder, inputs) + (tensor([[2.8052, 0.0000], + [0.0000, 3.3963]]), + tensor([[3., 0.], + [0., 3.]])) + + >>> def linear_model(x): + ... W = torch.tensor([[2.0, -1.0], [0.0, 1.0]]) + ... b = torch.tensor([1.0, 0.5]) + ... return x @ W.T + b + + >>> x = torch.randn(4, 2, requires_grad=True) + >>> jac = jacobian(linear_model, x, vectorize=True) + >>> jac.shape + torch.Size([4, 2, 4, 2]) + """ + if strategy not in ("forward-mode", "reverse-mode"): + raise AssertionError( + 'Expected strategy to be either "forward-mode" or "reverse-mode". Hint: If your ' + 'function has more outputs than inputs, "forward-mode" tends to be more performant. ' + 'Otherwise, prefer to use "reverse-mode".' + ) + if strategy == "forward-mode": + if create_graph: + raise NotImplementedError( + "torch.autograd.functional.jacobian: `create_graph=True` " + 'and `strategy="forward-mode"` are not supported together (yet). ' + "Please either set `create_graph=False` or " + '`strategy="reverse-mode"`.' + ) + return _jacfwd(func, inputs, strict, vectorize) + + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jacobian") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "jacobian" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if vectorize: + if strict: + raise RuntimeError( + "torch.autograd.functional.jacobian: `strict=True` " + "and `vectorized=True` are not supported together. " + "Please either set `strict=False` or " + "`vectorize=False`." + ) + # NOTE: [Computing jacobian with vmap and grad for multiple outputs] + # + # Let's consider f(x) = (x**2, x.sum()) and let x = torch.randn(3). + # It turns out we can compute the jacobian of this function with a single + # call to autograd.grad by using vmap over the correct grad_outputs. + # + # Firstly, one way to compute the jacobian is to stack x**2 and x.sum() + # into a 4D vector. E.g., use g(x) = torch.stack([x**2, x.sum()]) + # + # To get the first row of the jacobian, we call + # >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([1, 0, 0, 0])) + # To get the 2nd row of the jacobian, we call + # >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([0, 1, 0, 0])) + # and so on. + # + # Using vmap, we can vectorize all 4 of these computations into one by + # passing the standard basis for R^4 as the grad_output. + # vmap(partial(autograd.grad, g(x), x))(torch.eye(4)). + # + # Now, how do we compute the jacobian *without stacking the output*? + # We can just split the standard basis across the outputs. So to + # compute the jacobian of f(x), we'd use + # >>> autograd.grad(f(x), x, grad_outputs=_construct_standard_basis_for(...)) + # The grad_outputs looks like the following: + # ( torch.tensor([[1, 0, 0], + # [0, 1, 0], + # [0, 0, 1], + # [0, 0, 0]]), + # torch.tensor([[0], + # [0], + # [0], + # [1]]) ) + # + # But we're not done yet! + # >>> vmap(partial(autograd.grad(f(x), x, grad_outputs=...))) + # returns a Tensor of shape [4, 3]. We have to remember to split the + # jacobian of shape [4, 3] into two: + # - one of shape [3, 3] for the first output + # - one of shape [ 3] for the second output + + # Step 1: Construct grad_outputs by splitting the standard basis + output_numels = tuple(output.numel() for output in outputs) + grad_outputs = _construct_standard_basis_for(outputs, output_numels) + flat_outputs = tuple(output.reshape(-1) for output in outputs) + + # Step 2: Call vmap + autograd.grad + def vjp(grad_output): + vj = list( + _autograd_grad( + flat_outputs, + inputs, + grad_output, + create_graph=create_graph, + is_grads_batched=True, + ) + ) + for el_idx, vj_el in enumerate(vj): + if vj_el is not None: + continue + vj[el_idx] = torch.zeros_like(inputs[el_idx]).expand( + (sum(output_numels),) + inputs[el_idx].shape + ) + return tuple(vj) + + jacobians_of_flat_output = vjp(grad_outputs) + + # Step 3: The returned jacobian is one big tensor per input. In this step, + # we split each Tensor by output. + jacobian_input_output = [] + for jac_input_i, input_i in zip(jacobians_of_flat_output, inputs): + jacobian_input_i_output = [] + for jac, output_j in zip( + jac_input_i.split(output_numels, dim=0), outputs + ): + jacobian_input_i_output_j = jac.view(output_j.shape + input_i.shape) + jacobian_input_i_output.append(jacobian_input_i_output_j) + jacobian_input_output.append(jacobian_input_i_output) + + # Step 4: Right now, `jacobian` is a List[List[Tensor]]. + # The outer List corresponds to the number of inputs, + # the inner List corresponds to the number of outputs. + # We need to exchange the order of these and convert to tuples + # before returning. + jacobian_output_input = tuple(zip(*jacobian_input_output)) + + jacobian_output_input = _grad_postprocess( + jacobian_output_input, create_graph + ) + return _tuple_postprocess( + jacobian_output_input, (is_outputs_tuple, is_inputs_tuple) + ) + + jacobian: tuple[torch.Tensor, ...] = () + + for i, out in enumerate(outputs): + # mypy complains that expression and variable have different types due to the empty list + jac_i: tuple[list[torch.Tensor]] = tuple([] for _ in range(len(inputs))) # type: ignore[assignment] + for j in range(out.nelement()): + vj = _autograd_grad( + (out.reshape(-1)[j],), + inputs, + retain_graph=True, + create_graph=create_graph, + ) + + for el_idx, (jac_i_el, vj_el, inp_el) in enumerate( + zip(jac_i, vj, inputs) + ): + if vj_el is not None: + if strict and create_graph and not vj_el.requires_grad: + msg = ( + "The jacobian of the user-provided function is " + f"independent of input {i}. This is not allowed in " + "strict mode when create_graph=True." + ) + raise RuntimeError(msg) + jac_i_el.append(vj_el) + else: + if strict: + msg = ( + f"Output {i} of the user-provided function is " + f"independent of input {el_idx}. This is not allowed in " + "strict mode." + ) + raise RuntimeError(msg) + jac_i_el.append(torch.zeros_like(inp_el)) + + # pyrefly: ignore [bad-assignment] + jacobian += ( + tuple( + torch.stack(jac_i_el, dim=0).view( + out.size() + inputs[el_idx].size() # type: ignore[operator] + ) + for (el_idx, jac_i_el) in enumerate(jac_i) + ), + ) + + jacobian = _grad_postprocess(jacobian, create_graph) + + return _tuple_postprocess(jacobian, (is_outputs_tuple, is_inputs_tuple)) + + +def hessian( + func, + inputs, + create_graph=False, + strict=False, + vectorize=False, + outer_jacobian_strategy="reverse-mode", +): + r"""Compute the Hessian of a given scalar function. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor with a single element. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + create_graph (bool, optional): If ``True``, the Hessian will be computed in + a differentiable manner. Note that when ``strict`` is ``False``, the result can not + require gradients or be disconnected from the inputs. + Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we detect that there exists an input + such that all the outputs are independent of it. If ``False``, we return a Tensor of zeros as the + hessian for said inputs, which is the expected mathematical value. + Defaults to ``False``. + vectorize (bool, optional): This feature is experimental. + Please consider using :func:`torch.func.hessian` + instead if you are looking for something less experimental and more performant. + When computing the hessian, usually we invoke + ``autograd.grad`` once per row of the hessian. If this flag is + ``True``, we use the vmap prototype feature as the backend to + vectorize calls to ``autograd.grad`` so we only invoke it once + instead of once per row. This should lead to performance + improvements in many use cases, however, due to this feature + being incomplete, there may be performance cliffs. Please + use `torch._C._debug_only_display_vmap_fallback_warnings(True)` + to show any performance warnings and file us issues if + warnings exist for your use case. Defaults to ``False``. + outer_jacobian_strategy (str, optional): The Hessian is computed by + computing the Jacobian of a Jacobian. The inner Jacobian is always + computed in reverse-mode AD. Setting strategy to ``"forward-mode"`` + or ``"reverse-mode"`` determines whether the outer Jacobian will be + computed with forward or reverse mode AD. Currently, computing the outer + Jacobian in ``"forward-mode"`` requires ``vectorized=True``. Defaults + to ``"reverse-mode"``. + + Returns: + Hessian (Tensor or a tuple of tuple of Tensors): if there is a single input, + this will be a single Tensor containing the Hessian for the input. + If it is a tuple, then the Hessian will be a tuple of tuples where + ``Hessian[i][j]`` will contain the Hessian of the ``i``\th input + and ``j``\th input with size the sum of the size of the ``i``\th input plus + the size of the ``j``\th input. ``Hessian[i][j]`` will have the same + dtype and device as the corresponding ``i``\th input. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pow_reducer(x): + ... return x.pow(3).sum() + >>> inputs = torch.rand(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> hessian(pow_reducer, inputs) + tensor([[[[5.2265, 0.0000], + [0.0000, 0.0000]], + [[0.0000, 4.8221], + [0.0000, 0.0000]]], + [[[0.0000, 0.0000], + [1.9456, 0.0000]], + [[0.0000, 0.0000], + [0.0000, 3.2550]]]]) + + >>> hessian(pow_reducer, inputs, create_graph=True) + tensor([[[[5.2265, 0.0000], + [0.0000, 0.0000]], + [[0.0000, 4.8221], + [0.0000, 0.0000]]], + [[[0.0000, 0.0000], + [1.9456, 0.0000]], + [[0.0000, 0.0000], + [0.0000, 3.2550]]]], grad_fn=) + + + >>> def pow_adder_reducer(x, y): + ... return (2 * x.pow(2) + 3 * y.pow(2)).sum() + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> hessian(pow_adder_reducer, inputs) + ((tensor([[4., 0.], + [0., 4.]]), + tensor([[0., 0.], + [0., 0.]])), + (tensor([[0., 0.], + [0., 0.]]), + tensor([[6., 0.], + [0., 6.]]))) + """ + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "hessian") + if outer_jacobian_strategy not in ( + "forward-mode", + "reverse-mode", + ): + raise AssertionError( + 'Expected strategy to be either "forward-mode" or "reverse-mode".' + ) + + def ensure_single_output_function(*inp): + out = func(*inp) + is_out_tuple, t_out = _as_tuple( + out, "outputs of the user-provided function", "hessian" + ) + _check_requires_grad(t_out, "outputs", strict=strict) + + if is_out_tuple or not isinstance(out, torch.Tensor): + raise RuntimeError( + "The function given to hessian should return a single Tensor" + ) + + if out.nelement() != 1: + raise RuntimeError( + "The Tensor returned by the function given to hessian should contain a single element" + ) + + return out.squeeze() + + def jac_func(*inp): + if outer_jacobian_strategy == "forward-mode": + # _grad_preprocess requires create_graph=True and input to require_grad + # or else the input will be detached + inp = tuple(t.requires_grad_(True) for t in inp) + jac = jacobian(ensure_single_output_function, inp, create_graph=True) + _check_requires_grad(jac, "jacobian", strict=strict) + return jac + + res = jacobian( + jac_func, + inputs, + create_graph=create_graph, + strict=strict, + vectorize=vectorize, + strategy=outer_jacobian_strategy, + ) + return _tuple_postprocess(res, (is_inputs_tuple, is_inputs_tuple)) + + +def vhp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between vector ``v`` and Hessian of a given scalar function at a specified point. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor with a single element. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the vector Hessian + product is computed. Must be the same size as the input of + ``func``. This argument is optional when ``func``'s input contains + a single element and (if it is not provided) will be set as a + Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result + will be computed in a differentiable way. Note that when ``strict`` + is ``False``, the result can not require gradients or be + disconnected from the inputs. + Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + vhp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + vhp (tuple of Tensors or Tensor): result of the dot product with the + same shape as the inputs. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pow_reducer(x): + ... return x.pow(3).sum() + >>> inputs = torch.rand(2, 2) + >>> v = torch.ones(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> vhp(pow_reducer, inputs, v) + (tensor(0.5591), + tensor([[1.0689, 1.2431], + [3.0989, 4.4456]])) + >>> vhp(pow_reducer, inputs, v, create_graph=True) + (tensor(0.5591, grad_fn=), + tensor([[1.0689, 1.2431], + [3.0989, 4.4456]], grad_fn=)) + >>> def pow_adder_reducer(x, y): + ... return (2 * x.pow(2) + 3 * y.pow(2)).sum() + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = (torch.zeros(2), torch.ones(2)) + >>> vhp(pow_adder_reducer, inputs, v) + (tensor(4.8053), + (tensor([0., 0.]), + tensor([6., 6.]))) + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "vhp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + if v is not None: + _, v = _as_tuple(v, "v", "vhp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, inputs, is_inputs_tuple) + else: + if len(inputs) != 1 or inputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the input to the user-provided function " + "is a single Tensor with a single element." + ) + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "vhp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if is_outputs_tuple or not isinstance(outputs[0], torch.Tensor): + raise RuntimeError( + "The function given to vhp should return a single Tensor" + ) + + if outputs[0].nelement() != 1: + raise RuntimeError( + "The Tensor returned by the function given to vhp should contain a single element" + ) + + jac = _autograd_grad(outputs, inputs, create_graph=True) + _check_requires_grad(jac, "jacobian", strict=strict) + + enable_grad = True if create_graph else torch.is_grad_enabled() + with torch.set_grad_enabled(enable_grad): + grad_res = _autograd_grad(jac, inputs, v, create_graph=create_graph) + vhp = _fill_in_zeros(grad_res, inputs, strict, create_graph, "double_back") + + outputs = _grad_postprocess(outputs, create_graph) + vhp = _grad_postprocess(vhp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + vhp, is_inputs_tuple + ) + + +def hvp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between the scalar function's Hessian and a vector ``v`` at a specified point. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor with a single element. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the Hessian vector + product is computed. Must be the same size as the input of + ``func``. This argument is optional when ``func``'s input contains + a single element and (if it is not provided) will be set as a + Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result will be + computed in a differentiable way. Note that when ``strict`` is + ``False``, the result can not require gradients or be disconnected + from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + hvp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + hvp (tuple of Tensors or Tensor): result of the dot product with + the same shape as the inputs. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pow_reducer(x): + ... return x.pow(3).sum() + >>> inputs = torch.rand(2, 2) + >>> v = torch.ones(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> hvp(pow_reducer, inputs, v) + (tensor(0.1448), + tensor([[2.0239, 1.6456], + [2.4988, 1.4310]])) + + >>> hvp(pow_reducer, inputs, v, create_graph=True) + (tensor(0.1448, grad_fn=), + tensor([[2.0239, 1.6456], + [2.4988, 1.4310]], grad_fn=)) + + + >>> def pow_adder_reducer(x, y): + ... return (2 * x.pow(2) + 3 * y.pow(2)).sum() + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = (torch.zeros(2), torch.ones(2)) + >>> hvp(pow_adder_reducer, inputs, v) + (tensor(2.3030), + (tensor([0., 0.]), + tensor([6., 6.]))) + + Note: + + This function is significantly slower than `vhp` due to backward mode AD constraints. + If your functions is twice continuously differentiable, then hvp = vhp.t(). So if you + know that your function satisfies this condition, you should use vhp instead that is + much faster with the current implementation. + + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "hvp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + if v is not None: + _, v = _as_tuple(v, "v", "hvp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, inputs, is_inputs_tuple) + else: + if len(inputs) != 1 or inputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the input to the user-provided function " + "is a single Tensor with a single element." + ) + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "hvp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if is_outputs_tuple or not isinstance(outputs[0], torch.Tensor): + raise RuntimeError( + "The function given to hvp should return a single Tensor" + ) + + if outputs[0].nelement() != 1: + raise RuntimeError( + "The Tensor returned by the function given to hvp should contain a single element" + ) + + jac = _autograd_grad(outputs, inputs, create_graph=True) + _check_requires_grad(jac, "jacobian", strict=strict) + + grad_jac = tuple(torch.zeros_like(inp, requires_grad=True) for inp in inputs) + + double_back = _autograd_grad(jac, inputs, grad_jac, create_graph=True) + _check_requires_grad(jac, "hessian", strict=strict) + + enable_grad = True if create_graph else torch.is_grad_enabled() + with torch.set_grad_enabled(enable_grad): + grad_res = _autograd_grad(double_back, grad_jac, v, create_graph=create_graph) + hvp = _fill_in_zeros( + grad_res, inputs, strict, create_graph, "double_back_trick" + ) + + outputs = _grad_postprocess(outputs, create_graph) + hvp = _grad_postprocess(hvp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + hvp, is_inputs_tuple + ) diff --git a/western_ai/usa/pytorch/torch/autograd/grad_mode.py b/western_ai/usa/pytorch/torch/autograd/grad_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..90658e71ab461ff3d2bdf86f76bbf30194d8ab4b --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/grad_mode.py @@ -0,0 +1,489 @@ +# mypy: allow-untyped-defs +from typing import Any + +import torch +from torch.utils._contextlib import ( + _DecoratorContextManager, + _NoParamDecoratorContextManager, + F, +) + + +__all__ = [ + "no_grad", + "enable_grad", + "set_grad_enabled", + "inference_mode", + "set_multithreading_enabled", + "enforce_grad_layout_policy", +] + + +class no_grad(_NoParamDecoratorContextManager): + r"""Context-manager that disables gradient calculation. + + Disabling gradient calculation is useful for inference, when you are sure + that you will not call :meth:`Tensor.backward()`. It will reduce memory + consumption for computations that would otherwise have `requires_grad=True`. + + In this mode, the result of every computation will have + `requires_grad=False`, even when the inputs have `requires_grad=True`. + There is an exception! All factory functions, or functions that create + a new Tensor and take a requires_grad kwarg, will NOT be affected by + this mode. + + This context manager is thread local; it will not affect computation + in other threads. + + Also functions as a decorator. + + .. note:: + No-grad is one of several mechanisms that can enable or + disable gradients locally see :ref:`locally-disable-grad-doc` for + more information on how they compare. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + If you want to disable forward AD for a computation, you can unpack + your dual tensors. + + Example:: + >>> # xdoctest: +SKIP + >>> x = torch.tensor([1.], requires_grad=True) + >>> with torch.no_grad(): + ... y = x * 2 + >>> y.requires_grad + False + >>> @torch.no_grad() + ... def doubler(x): + ... return x * 2 + >>> z = doubler(x) + >>> z.requires_grad + False + >>> @torch.no_grad() + ... def tripler(x): + ... return x * 3 + >>> z = tripler(x) + >>> z.requires_grad + False + >>> # factory function exception + >>> with torch.no_grad(): + ... a = torch.nn.Parameter(torch.rand(10)) + >>> a.requires_grad + True + """ + + def __init__(self) -> None: + if not torch._jit_internal.is_scripting(): + super().__init__() + self.prev = False + + def __enter__(self) -> None: + self.prev = torch.is_grad_enabled() + torch.set_grad_enabled(False) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch.set_grad_enabled(self.prev) + + +class enable_grad(_NoParamDecoratorContextManager): + r"""Context-manager that enables gradient calculation. + + Enables gradient calculation, if it has been disabled via :class:`~no_grad` + or :class:`~set_grad_enabled`. + + This context manager is thread local; it will not affect computation + in other threads. + + Also functions as a decorator. + + .. note:: + enable_grad is one of several mechanisms that can enable or + disable gradients locally see :ref:`locally-disable-grad-doc` for + more information on how they compare. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + + Example:: + >>> # xdoctest: +SKIP + >>> x = torch.tensor([1.], requires_grad=True) + >>> with torch.no_grad(): + ... with torch.enable_grad(): + ... y = x * 2 + >>> y.requires_grad + True + >>> y.backward() + >>> x.grad + tensor([2.]) + >>> @torch.enable_grad() + ... def doubler(x): + ... return x * 2 + >>> with torch.no_grad(): + ... z = doubler(x) + >>> z.requires_grad + True + >>> @torch.enable_grad() + ... def tripler(x): + ... return x * 3 + >>> with torch.no_grad(): + ... z = tripler(x) + >>> z.requires_grad + True + + """ + + def __enter__(self) -> None: + self.prev = torch.is_grad_enabled() + torch._C._set_grad_enabled(True) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_grad_enabled(self.prev) + + +class set_grad_enabled(_DecoratorContextManager): + r"""Context-manager that sets gradient calculation on or off. + + ``set_grad_enabled`` will enable or disable grads based on its argument :attr:`mode`. + It can be used as a context-manager or as a function. + + This context manager is thread local; it will not affect computation + in other threads. + + Args: + mode (bool): Flag whether to enable grad (``True``), or disable + (``False``). This can be used to conditionally enable + gradients. + + .. note:: + set_grad_enabled is one of several mechanisms that can enable or + disable gradients locally see :ref:`locally-disable-grad-doc` for + more information on how they compare. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + + Example:: + >>> # xdoctest: +SKIP + >>> x = torch.tensor([1.], requires_grad=True) + >>> is_train = False + >>> with torch.set_grad_enabled(is_train): + ... y = x * 2 + >>> y.requires_grad + False + >>> _ = torch.set_grad_enabled(True) + >>> y = x * 2 + >>> y.requires_grad + True + >>> _ = torch.set_grad_enabled(False) + >>> y = x * 2 + >>> y.requires_grad + False + + """ + + def __init__(self, mode: bool) -> None: + self.prev = torch.is_grad_enabled() + self.mode = mode + torch._C._set_grad_enabled(mode) + + def __call__(self, orig_func: F) -> F: + torch._C._set_grad_enabled(self.prev) + return super().__call__(orig_func) + + def __enter__(self) -> None: + torch._C._set_grad_enabled(self.mode) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_grad_enabled(self.prev) + + def __str__(self) -> str: + return f"{torch.typename(self)}(mode={self.mode})" + + def __repr__(self) -> str: + return str(self) + + def clone(self) -> "set_grad_enabled": + r""" + Create a copy of this class + """ + return self.__class__(self.mode) + + +class inference_mode(_DecoratorContextManager): + r"""Context manager that enables or disables inference mode. + + InferenceMode is analogous to :class:`~no_grad` and should be used + when you are certain your operations will not interact with autograd + (e.g., during data loading or model evaluation). Compared to + :class:`~no_grad`, it removes additional overhead by disabling view + tracking and version counter bumps. It is also more restrictive, in + that tensors created in this mode cannot be used in computations + recorded by autograd. + + This context manager is thread-local; it does not affect computation + in other threads. + + Also functions as a decorator. + + .. note:: + Inference mode is one of several mechanisms that can locally enable + or disable gradients. See :ref:`locally-disable-grad-doc` for a + comparison. If avoiding the use of tensors created in inference mode + in autograd-tracked regions is difficult, consider benchmarking your + code with and without inference mode to weigh the performance benefits + against the trade-offs. You can always use :class:`~no_grad` instead. + + .. note:: + Unlike some other mechanisms that locally enable or disable grad, + entering inference_mode also disables :ref:`forward-mode AD `. + + .. warning:: + `inference_mode` does NOT automatically set the model to evaluation mode. + For proper inference behavior (e.g., disabling dropout, using running statistics + in batch normalization), you must explicitly set your model to evaluation mode using + `model.eval()` in addition to using this context manager. + + Args: + mode (bool or function): Either a boolean flag to enable or disable + inference mode, or a Python function to decorate with inference + mode enabled. + + Example:: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> import torch + >>> x = torch.ones(1, 2, 3, requires_grad=True) + >>> with torch.inference_mode(): + ... y = x * x + >>> y.requires_grad + False + >>> # xdoctest: +SKIP("want string isn't quite right") + >>> y._version + Traceback (most recent call last): + File "", line 1, in + RuntimeError: Inference tensors do not track version counter. + >>> @torch.inference_mode() + ... def func(x): + ... return x * x + >>> out = func(x) + >>> out.requires_grad + False + >>> @torch.inference_mode() + ... def doubler(x): + ... return x * 2 + >>> out = doubler(x) + >>> out.requires_grad + False + + """ + + def __init__(self, mode: bool = True) -> None: + if not torch._jit_internal.is_scripting(): + super().__init__() + self.mode = mode + + def __new__(cls, mode=True): + if isinstance(mode, bool): + return super().__new__(cls) + return cls()(mode) + + def __enter__(self) -> None: + self._inference_mode_context = torch._C._InferenceMode(self.mode) + self._inference_mode_context.__enter__() + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self._inference_mode_context.__exit__(exc_type, exc_value, traceback) + + def clone(self) -> "inference_mode": + r""" + Create a copy of this class + """ + return self.__class__(self.mode) + + +def _enter_inference_mode(mode): + mode_context = torch._C._InferenceMode(mode) + mode_context.__enter__() + return mode_context + + +def _exit_inference_mode(mode): + mode.__exit__(None, None, None) + + +class set_multithreading_enabled(_DecoratorContextManager): + r"""Context-manager that enables or disables multithreaded backward. + + Ordinarily, when :ref:`accelerator` devices are in use, + the backward pass runs on device-specific worker threads. The engine + creates these threads based on the number of available devices and + reuses them across iterations. + + When ``mode=False``, the backward pass runs on the calling thread + instead. ``mode=True`` restores the default behavior. + + This can be used as a context-manager or as a function. It is + thread-local and will not affect computation in other threads. + + Args: + mode (bool): Whether to enable multithreaded backward (``True``, + default) or disable (``False``). + + .. note:: + This API does not apply to :ref:`forward-mode AD `, + which never uses multithreading. + + """ + + def __init__(self, mode: bool) -> None: + self.prev = torch._C._is_multithreading_enabled() + torch._C._set_multithreading_enabled(mode) + self.mode = mode + + def __enter__(self) -> None: + pass + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_multithreading_enabled(self.prev) + + def clone(self) -> "set_multithreading_enabled": + r""" + Create a copy of this class + """ + return self.__class__(self.mode) + + +class enforce_grad_layout_policy(_DecoratorContextManager): + r"""Context-manager that controls the gradient layout policy enforcement. + + The gradient layout contract ensures that accumulated gradients have + strides matching their corresponding parameters (for non-overlapping + dense parameters) or are row-major contiguous (otherwise). This aids + performance in optimizers and distributed reducers. + + When ``enable=False``, the autograd engine relaxes this enforcement: + + - Stealable gradients whose layout does *not* match the parameter can + still be stolen directly (avoiding an extra copy). + - The "grad and param do not obey the gradient layout contract" warning + is suppressed. + + The logic that *creates* a brand-new gradient (e.g., cloning into the + correct layout when the gradient is not stealable) is **not** affected + by this flag. + + This can be used as a context-manager or as a function. It is + thread-local and will not affect computation in other threads. + + Args: + enable (bool): Whether to enforce the gradient layout contract + (``True``, default) or relax enforcement (``False``). + + Example:: + >>> # xdoctest: +SKIP + >>> import torch + >>> p = torch.empty(2, 3, 4).permute(2, 0, 1).requires_grad_() + >>> with torch.autograd.enforce_grad_layout_policy(False): + ... (p * 2).sum().backward() + ... # p.grad may now have the same strides as the incoming + ... # gradient rather than being forced to match p's strides. + """ + + def __init__(self, enable: bool = True) -> None: + self.prev = torch._C._is_grad_layout_enforcement_enabled() + torch._C._set_grad_layout_enforcement_enabled(enable) + self.mode = enable + + def __call__(self, orig_func: F) -> F: + torch._C._set_grad_layout_enforcement_enabled(self.prev) + return super().__call__(orig_func) + + def __enter__(self) -> None: + pass + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_grad_layout_enforcement_enabled(self.prev) + + def clone(self) -> "enforce_grad_layout_policy": + r""" + Create a copy of this class + """ + return self.__class__(self.mode) + + +class _force_original_view_tracking(_DecoratorContextManager): + r"""Context-manager that sets whether or not to always enable view-replay in autograd. + + ``set_view_replay_enabled`` will enable or disable view-replay based on its argument :attr:`mode`. + It can be used as a context-manager or as a function. + + This context manager is thread local; it will not affect computation + in other threads. + + When a tensor view is mutated, the autograd engine needs to decide whether or not + to regenerate the "updated view" by either replaying the chain of views from the updated base, + or with a single call to as_strided. + + If set_view_replay_enabled is set to True, then autograd will always use view replay. + Otherwise, it will fall back to its existing logic. + + Args: + mode (bool): Flag whether to enable view-replay (``True``), or disable + (``False``). + + """ + + def __init__(self, mode: bool) -> None: + self.prev = torch._C._is_view_replay_enabled() + self.mode = mode + torch._C._set_view_replay_enabled(mode) + + def __call__(self, orig_func: F) -> F: + torch._C._set_view_replay_enabled(self.prev) + return super().__call__(orig_func) + + def __enter__(self) -> None: + torch._C._set_view_replay_enabled(self.mode) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_view_replay_enabled(self.prev) + + def clone(self): + return self.__class__(self.mode) + + +class _unsafe_preserve_version_counter(_DecoratorContextManager): + r"""DO NOT USE THIS UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING. + + This context manager can lead to arbitrary silent-correctness issues in any other part of your code + (even the ones not touched directly by the context manager)! + + Ordinarily, autograd will track mutations to tensors by incrementing its `._version` attribute. + This is generally important for correctness, as for example, mutating a tensor that autograd has saved + for the backwards pass can result in incorrect gradients, and autograd uses the version counter to detect + and error out in this situation. + + However, there are rare instances where it might be useful to hide mutations from autograd. For example: + if a tensor is very large, and you'd like to free its memory by storing it elsewhere, and re-populate + the tensor right before it is needed by autograd. + + Args: + tensor (torch.Tensor): the tensor in question, that you would like to preserve the version counter of. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + + """ + + def __init__(self, tensors: torch.Tensor | tuple[torch.Tensor, ...]) -> None: + self.tensors = (tensors,) if isinstance(tensors, torch.Tensor) else tensors + if not isinstance(self.tensors, tuple): + raise AssertionError("Expected tensors to be a tuple") + self.prev_versions = tuple(t._version for t in self.tensors) + + def __enter__(self) -> None: + pass + + # pyrefly: ignore [bad-override] + def __exit__(self, *args) -> None: + torch._C._autograd._unsafe_set_version_counter(self.tensors, self.prev_versions) diff --git a/western_ai/usa/pytorch/torch/autograd/gradcheck.py b/western_ai/usa/pytorch/torch/autograd/gradcheck.py new file mode 100644 index 0000000000000000000000000000000000000000..eb0802eeb64b46b57d13670ed0cb9cde53412f3e --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/gradcheck.py @@ -0,0 +1,2324 @@ +# mypy: allow-untyped-defs +import collections +import functools +import warnings +from collections.abc import Callable, Iterable +from itertools import product +from typing_extensions import deprecated + +import torch +import torch.testing + +# pyrefly: ignore [deprecated] +from torch._vmap_internals import _vmap, vmap +from torch.overrides import is_tensor_like +from torch.types import _TensorOrOptionalTensors, _TensorOrTensors + + +# Note: `get_*_jacobian` functions are added here even though we didn't intend to make them public +# since they have been exposed from before we added `__all__` and we already maintain BC for them +# We should eventually deprecate them and remove them from `__all__` +__all__ = [ + "gradcheck", + "gradgradcheck", + "GradcheckError", + "get_numerical_jacobian", + "get_analytical_jacobian", + "get_numerical_jacobian_wrt_specific_input", +] + + +class GradcheckError(RuntimeError): + r"""Error raised by :func:`gradcheck` and :func:`gradgradcheck`.""" + + +def _is_sparse_compressed_tensor(obj: torch.Tensor): + return obj.layout in { + torch.sparse_csr, + torch.sparse_csc, + torch.sparse_bsr, + torch.sparse_bsc, + } + + +def _is_sparse_any_tensor(obj: torch.Tensor): + return _is_sparse_compressed_tensor(obj) or obj.layout is torch.sparse_coo + + +def _is_float_or_complex_tensor(obj): + return is_tensor_like(obj) and (obj.is_floating_point() or obj.is_complex()) + + +def _allocate_jacobians_with_inputs( + input_tensors: tuple, numel_output +) -> tuple[torch.Tensor, ...]: + # Makes zero-filled tensors from inputs. If `numel_output` is not None, for + # each tensor in `input_tensors`, returns a new zero-filled tensor with height + # of `t.numel` and width of `numel_output`. Otherwise, for each tensor, returns + # a 1-d tensor with size `(t.numel,)`. Each new tensor will be strided and have + # the same dtype and device as those of the corresponding input. + out: list[torch.Tensor] = [ + t.new_zeros((t.numel(), numel_output), layout=torch.strided) + for t in input_tensors + if _is_float_or_complex_tensor(t) and t.requires_grad + ] + return tuple(out) + + +def _allocate_jacobians_with_outputs( + output_tensors: tuple, numel_input, dtype=None, device=None +) -> tuple[torch.Tensor, ...]: + # Makes zero-filled tensors from outputs. If `dim` is not None, for each tensor + # in `output_tensors`, returns a new zero-filled tensor with height of `dim` and + # width of `t.numel`. Otherwise, for each tensor, returns a 1-d tensor with size + # (t.numel,). + options = {"dtype": dtype, "device": device, "layout": torch.strided} + out: list[torch.Tensor] = [ + t.new_zeros((numel_input, t.numel()), **options) + for t in output_tensors + if _is_float_or_complex_tensor(t) + ] + return tuple(out) + + +def _iter_tensors( + x: torch.Tensor | Iterable[torch.Tensor], only_requiring_grad: bool = False +) -> Iterable[torch.Tensor]: + if is_tensor_like(x): + # mypy doesn't narrow type of `x` to torch.Tensor + if x.requires_grad or not only_requiring_grad: # type: ignore[union-attr] + yield x # type: ignore[misc] + elif isinstance(x, collections.abc.Iterable) and not isinstance(x, str): + for elem in x: + yield from _iter_tensors(elem, only_requiring_grad) + + +def _densify(x): + # return a copy of sparse x with all unspecified elements + # "replaced" with zero-valued elements + if isinstance(x, (list, tuple)): + return type(x)(map(_densify, x)) + elif not is_tensor_like(x) or x.layout in {torch.strided, torch._mkldnn}: # type: ignore[attr-defined] # no attr _mkldnn + return x + elif x.layout is torch.sparse_coo: + device = x.device + indices_dtype = x._indices().dtype + tmp = torch.ones(x.shape[: x.sparse_dim()], dtype=torch.int8, device=device) + indices = tmp.nonzero().t().to(dtype=indices_dtype) + values = torch.zeros( + (tmp.numel(), *x.shape[x.sparse_dim() :]), dtype=x.dtype, device=device + ) + x_coalesced = x.detach().coalesce() + if x_coalesced.numel() > 0: + stride = tmp.stride() + flat_indices = ( + x_coalesced.indices() + .mul( + torch.tensor(stride, dtype=indices_dtype, device=device).unsqueeze( + 1 + ) + ) + .sum(0) + ) + values[flat_indices] = x_coalesced.values() + return ( + torch.sparse_coo_tensor(indices, values, x.shape) + ._coalesced_(True) + .requires_grad_(x.requires_grad) + ) + elif _is_sparse_compressed_tensor(x): + blocksize = ( + x.values().shape[1:3] + if x.layout in {torch.sparse_bsr, torch.sparse_bsc} + else None + ) + compressed_indices = ( + x.crow_indices() + if x.layout in {torch.sparse_csr, torch.sparse_bsr} + else x.ccol_indices() + ) + # We'll use intermediate sparse COO for simplicity + r = _densify(x.detach().to_sparse(layout=torch.sparse_coo)).to_sparse( + layout=x.layout, blocksize=blocksize + ) + # Check that all elements are specified also after `to_sparse` op: + dense_numel = r.values().numel() // max(1, r.values().shape[0]) + batch_numel = compressed_indices.numel() // compressed_indices.shape[-1] + sparse_numel = r.numel() // max(1, dense_numel * batch_numel) + if sparse_numel != r._nnz(): + raise AssertionError( + f"{x.layout} densify failed: expected nnz={sparse_numel} but got {r._nnz()}" + ) + return r.requires_grad_(x.requires_grad) + elif _is_sparse_any_tensor(x): + raise NotImplementedError(x.layout) + return x + + +def _iter_tensor(x_tensor): + # (Only used for slow gradcheck) Returns a generator that yields the following + # elements at each iteration: + # 1) a tensor: the same tensor is returned across all iterations. The tensor + # is not the same as the original x_tensor as given as input - it is + # prepared so that it can be modified in-place. Depending on whether the + # input tensor is strided, sparse, or dense, the returned tensor may or may + # not share storage with x_tensor. + # 2) a tuple of indices that can be used with advanced indexing (yielded in + # dictionary order) + # 3) flattened index that will be used to index into the Jacobian tensor + # + # For a tensor t with size (2, 2), _iter_tensor yields: + # `x, (0, 0), 0`, `x, (0, 1), 1`, `x, (1, 0), 2`, `x, (1, 1), 3` + # + # where x is the t.data of the original tensor. Perturbing the entry of x + # at index (1, 1) yields the 3rd column of the overall Jacobian matrix. + if _is_sparse_any_tensor(x_tensor): + + def get_stride(size): + dim = len(size) + tmp = 1 + stride = [0] * dim + for i in reversed(range(dim)): + stride[i] = tmp + tmp *= size[i] + return stride + + x_nnz = x_tensor._nnz() + x_size = list(x_tensor.size()) + if x_tensor.layout is torch.sparse_coo: + x_indices = x_tensor._indices().t() + x_values = x_tensor._values() + elif x_tensor.layout is torch.sparse_csr: + x_indices = torch._convert_indices_from_csr_to_coo( + x_tensor.crow_indices(), x_tensor.col_indices() + ).t() + x_values = x_tensor.values() + elif x_tensor.layout is torch.sparse_csc: + x_indices = torch._convert_indices_from_csr_to_coo( + x_tensor.ccol_indices(), x_tensor.row_indices(), transpose=True + ).t() + x_values = x_tensor.values() + elif x_tensor.layout is torch.sparse_bsr: + x_block_values = x_tensor.values() + x_blocksize = x_block_values.size()[1:3] + x_indices = ( + torch._convert_indices_from_csr_to_coo( + x_tensor.crow_indices(), x_tensor.col_indices() + ) + .repeat_interleave(x_blocksize[0] * x_blocksize[1], 1) + .mul_(torch.tensor(x_blocksize, device=x_tensor.device).reshape(2, 1)) + .add_( + torch.stack( + torch.where(torch.ones(x_blocksize, device=x_tensor.device)) + ).repeat(1, x_nnz) + ) + .t() + ) + x_values = x_block_values.flatten(0, 2) + x_nnz = x_values.size(0) + elif x_tensor.layout is torch.sparse_bsc: + x_block_values = x_tensor.values() + x_blocksize = x_block_values.size()[1:3] + x_indices = ( + torch._convert_indices_from_csr_to_coo( + x_tensor.ccol_indices(), x_tensor.row_indices(), transpose=True + ) + .repeat_interleave(x_blocksize[0] * x_blocksize[1], 1) + .mul_(torch.tensor(x_blocksize, device=x_tensor.device).reshape(2, 1)) + .add_( + torch.stack( + torch.where(torch.ones(x_blocksize, device=x_tensor.device)) + ).repeat(1, x_nnz) + ) + .t() + ) + x_values = x_block_values.flatten(0, 2) + x_nnz = x_values.size(0) + else: + raise NotImplementedError(f"_iter_tensor for {x_tensor.layout} input") + x_stride = get_stride(x_size) + # Use .data here to get around the version check + x_values = x_values.data + for i in range(x_nnz): + x_value = x_values[i] + for x_idx in product(*[range(m) for m in x_values.size()[1:]]): + indices = x_indices[i].tolist() + list(x_idx) + d_idx = sum(indices[k] * x_stride[k] for k in range(len(x_size))) + yield x_value, x_idx, d_idx + elif x_tensor.layout == torch._mkldnn: # type: ignore[attr-defined] + for d_idx, x_idx in enumerate(product(*[range(m) for m in x_tensor.size()])): + # this is really inefficient, but without indexing implemented, there's + # not really a better way than converting back and forth + x_tensor_dense = x_tensor.to_dense() + yield x_tensor_dense, x_idx, d_idx + else: + # Use .data here to get around the version check + x_tensor = x_tensor.data + for d_idx, x_idx in enumerate(product(*[range(m) for m in x_tensor.size()])): + yield x_tensor, x_idx, d_idx + + +def _get_numerical_jacobian( + fn, inputs, outputs=None, target=None, eps=1e-3, is_forward_ad=False +) -> list[tuple[torch.Tensor, ...]]: + """Compute the numerical Jacobian of `fn(inputs)` with respect to `target`. + + If not specified, targets are the input. Returns M * N Jacobians where N is the + number of tensors in target that require grad and M is the number of non-integral + outputs. + + Args: + fn: the function to compute the jacobian for + inputs: inputs to `fn` + outputs: provide precomputed outputs to avoid one extra invocation of fn + target: the Tensors wrt whom Jacobians are calculated (default=`inputs`) + eps: the magnitude of the perturbation during finite differencing + (default=`1e-3`) + is_forward_ad: if this numerical jacobian is computed to be checked wrt + forward AD gradients (this is used for error checking only) + + Returns: + A list of M N-tuples of tensors + + Note that `target` may not even be part of `input` to `fn`, so please be + **very careful** in this to not clone `target`. + """ + jacobians: list[tuple[torch.Tensor, ...]] = [] + if outputs is None: + outputs = _as_tuple(fn(*_as_tuple(inputs))) + if not is_forward_ad and any(o.is_complex() for o in outputs): + raise ValueError( + "Expected output to be non-complex. get_numerical_jacobian no " + "longer supports functions that return complex outputs." + ) + if target is None: + target = inputs + inp_indices = [ + i for i, a in enumerate(target) if is_tensor_like(a) and a.requires_grad + ] + for inp, inp_idx in zip(_iter_tensors(target, True), inp_indices): + jacobians += [ + get_numerical_jacobian_wrt_specific_input( + fn, + inp_idx, + inputs, + outputs, + eps, + input=inp, + is_forward_ad=is_forward_ad, + ) + ] + return jacobians + + +@deprecated( + "`get_numerical_jacobian` was part of PyTorch's private API and not " + "meant to be exposed. We are deprecating it and it will be removed " + "in a future version of PyTorch. If you have a specific use for " + "this or feature request for this to be a stable API, please file " + "us an issue at https://github.com/pytorch/pytorch/issues/new", + category=FutureWarning, +) +def get_numerical_jacobian(fn, inputs, target=None, eps=1e-3, grad_out=1.0): + """Compute the numerical Jacobian for a given fn and its inputs. + + This is a Deprecated API. + + Args: + fn: the function to compute the Jacobian for (must take inputs as a tuple) + inputs: input to `fn` + target: the Tensors wrt whom Jacobians are calculated (default=`input`) + eps: the magnitude of the perturbation during finite differencing + (default=`1e-3`) + grad_out: defaults to 1.0. + + Returns: + A list of Jacobians of `fn` (restricted to its first output) with respect to + each input or target, if provided. + + Note that `target` may not even be part of `input` to `fn`, so please be + **very careful** in this to not clone `target`. + """ + if ( + grad_out != 1.0 + ): # grad_out param is only kept for backward compatibility reasons + raise ValueError( + "Expected grad_out to be 1.0. get_numerical_jacobian no longer " + "supports values of grad_out != 1.0." + ) + + def fn_pack_inps(*inps): + return fn(inps) + + jacobians = _get_numerical_jacobian(fn_pack_inps, inputs, None, target, eps) + + return tuple(jacobian_for_each_output[0] for jacobian_for_each_output in jacobians) + + +def _compute_numerical_gradient(fn, entry, v, norm_v, nbhd_checks_fn): + # Computes numerical directional derivative as finite difference + # of function `fn` at input `entry`, perturbed by vector `v`. + if _is_sparse_compressed_tensor(entry): + # sparse compressed tensors don't implement sub/add/copy_ + # yet. However, in non-masked semantics context entry and v + # have the same sparse indices ... + if entry.layout != v.layout: + raise AssertionError( + f"Expected entry and v to have the same layout, but got {entry.layout} and {v.layout}" + ) + if entry._nnz() != v._nnz(): + raise AssertionError( + f"Expected entry and v to have the same nnz, but got {entry._nnz()} and {v._nnz()} " + f"with entry shape {entry.shape}" + ) + # ... the finite differencing can be performed on values only: + entry = entry.values() + v = v.values() + # we'll detach to avoid backward computations that sparse + # tensors have limited support for. + entry = entry.detach() + + orig = entry.clone() + entry.copy_(orig - v) + outa = fn() + entry.copy_(orig + v) + outb = fn() + entry.copy_(orig) + + def compute(a, b): + nbhd_checks_fn(a, b) + ret = (b - a) / (2 * norm_v) # use central difference approx + return ret.detach().reshape(-1) + + return tuple(compute(a, b) for (a, b) in zip(outa, outb)) + + +def _compute_numerical_jvps_wrt_specific_input( + jvp_fn, delta, input_is_complex, is_forward_ad=False +) -> list[torch.Tensor]: + # Computing the jacobian only works for real delta + # For details on the algorithm used here, refer: + # Section 3.5.3 https://arxiv.org/pdf/1701.00392.pdf + # s = fn(z) where z = x for real valued input + # and z = x + yj for complex valued input + jvps: list[torch.Tensor] = [] + ds_dx_tup = jvp_fn(delta[0] if isinstance(delta, tuple) else delta) + + if input_is_complex: # C -> R + ds_dy_tup = ( + jvp_fn(delta[1] * 1j) if isinstance(delta, tuple) else jvp_fn(delta * 1j) + ) + for ds_dx, ds_dy in zip(ds_dx_tup, ds_dy_tup): + if ds_dx.is_complex(): + raise AssertionError("Expected ds_dx to be real-valued, not complex") + # conjugate wirtinger derivative + conj_w_d = ds_dx + ds_dy * 1j + jvps.append(conj_w_d) + else: + for ds_dx in ds_dx_tup: # R -> R or (R -> C for the forward AD case) + if not is_forward_ad and ds_dx.is_complex(): + raise AssertionError("Expected ds_dx to be real-valued, not complex.") + jvps.append(ds_dx) + return jvps + + +def _combine_jacobian_cols( + jacobians_cols: dict[int, list[torch.Tensor]], outputs, input, numel +) -> tuple[torch.Tensor, ...]: + # jacobian_cols maps column_idx -> output_idx -> single column of jacobian Tensor + # we return a list that maps output_idx -> full jacobian Tensor + jacobians = _allocate_jacobians_with_outputs( + outputs, numel, dtype=input.dtype if input.dtype.is_complex else None + ) + for i, jacobian in enumerate(jacobians): + for k, v in jacobians_cols.items(): + jacobian[k] = v[i] + return jacobians + + +def _prepare_input( + input: torch.Tensor, maybe_perturbed_input: torch.Tensor | None, fast_mode=False +) -> torch.Tensor: + # Prepares the inputs to be passed into the function while including the new + # modified input. + if input.layout == torch._mkldnn: # type: ignore[attr-defined] # no attr _mkldnn + # Convert back to mkldnn + if maybe_perturbed_input is not None: + return maybe_perturbed_input.to_mkldnn() + else: + return input + elif _is_sparse_any_tensor(input): + if fast_mode and maybe_perturbed_input is not None: + # entry is already a "cloned" version of the original tensor + # thus changes to entry are not reflected in the input + return maybe_perturbed_input + else: + return input + else: + # We cannot use entry (input.data) if we want gradgrad to work because + # fn (in the gradgrad case) needs to compute grad wrt input + return input + + +def _check_outputs_same_dtype_and_shape(output1, output2, eps, idx=None) -> None: + # Check that the returned outputs don't have different dtype or shape when you + # perturb the input + on_index = f"on index {idx} " if idx is not None else "" + if output1.shape != output2.shape: + raise AssertionError( + f"Expected `func` to return outputs with the same shape" + f" when inputs are perturbed {on_index}by {eps}, but got:" + f" shapes {output1.shape} and {output2.shape}." + ) + if output1.dtype != output2.dtype: + raise AssertionError( + f"Expected `func` to return outputs with the same dtype" + f" when inputs are perturbed {on_index}by {eps}, but got:" + f" dtypes {output1.dtype} and {output2.dtype}." + ) + + +def get_numerical_jacobian_wrt_specific_input( + fn, input_idx, inputs, outputs, eps, input=None, is_forward_ad=False +) -> tuple[torch.Tensor, ...]: + # Computes the numerical jacobians wrt to a single input. Returns N jacobian + # tensors, where N is the number of outputs. We use a dictionary for + # jacobian_cols because indices aren't necessarily consecutive for sparse inputs + # When we perturb only a single element of the input tensor at a time, the jvp + # is equivalent to a single col of the Jacobian matrix of fn. + jacobian_cols: dict[int, list[torch.Tensor]] = {} + input = inputs[input_idx] if input is None else input + if not input.requires_grad: + raise AssertionError("Expected input to have requires_grad=True") + for x, idx, d_idx in _iter_tensor(input): + wrapped_fn = _with_prepare_inputs(fn, inputs, input_idx, x) + input_to_perturb = x[idx] + nbhd_checks_fn = functools.partial( + _check_outputs_same_dtype_and_shape, idx=idx, eps=eps + ) + jvp_fn = _get_numerical_jvp_fn( + wrapped_fn, input_to_perturb, eps, nbhd_checks_fn + ) + jacobian_cols[d_idx] = _compute_numerical_jvps_wrt_specific_input( + jvp_fn, eps, x.is_complex(), is_forward_ad + ) + return _combine_jacobian_cols(jacobian_cols, outputs, input, input.numel()) + + +def _get_analytical_jacobian_forward_ad( + fn, inputs, outputs, *, check_grad_dtypes=False, all_u=None +) -> tuple[tuple[torch.Tensor, ...], ...]: + """Compute the analytical Jacobian using forward mode AD of `fn(inputs)` using forward mode AD with respect to `target`. + + Return N * M Jacobians where N is the number of tensors in target that require grad and + M is the number of non-integral outputs. + Contrary to other functions here, this function requires "inputs" to actually be used by the function. + The computed value is expected to be wrong if the function captures the inputs by side effect instead of + using the passed ones (many torch.nn tests do this). + + Args: + fn: the function to compute the jacobian for + inputs: inputs to `fn` + outputs: provide precomputed outputs to avoid one extra invocation of fn + check_grad_dtypes: if True, will check that the gradient dtype are valid + all_u (optional): if provided, the Jacobian will be right multiplied with this vector + + Returns: + A tuple of M N-tuples of tensors + """ + # To avoid early import issues + fwAD = torch.autograd.forward_ad + + tensor_inputs = tuple(i for i in inputs if is_tensor_like(i) and i.requires_grad) + + if any(i.is_complex() for i in tensor_inputs): + raise ValueError( + "Expected inputs to be non-complex for _get_analytical_jacobian_forward_ad." + ) + + if all_u: + jacobians = tuple( + _allocate_jacobians_with_outputs(outputs, 1) for i in tensor_inputs + ) + else: + jacobians = tuple( + _allocate_jacobians_with_outputs(outputs, i.numel()) for i in tensor_inputs + ) + + with fwAD.dual_level(): + fw_grads = [] + dual_inputs = [] + for inp in inputs: + if is_tensor_like(inp) and inp.requires_grad: + if inp.layout == torch._mkldnn: # type: ignore[attr-defined] + raise ValueError( + "MKLDNN inputs are not support for forward AD gradcheck." + ) + + inp = fwAD.make_dual(inp.detach(), torch.zeros_like(inp)) + # If inp is a differentiable view, the dual might not be the tangent given to + # make_dual, so read it explicitly from the dual tensor + fw_grads.append(fwAD.unpack_dual(inp)[1]) + dual_inputs.append(inp) + + if all_u: + # Do the full reduction in one pass + # To be consistent with numerical evaluation, we actually compute one reduction per input + for i, (fw_grad, u) in enumerate(zip(fw_grads, all_u)): + fw_grad.copy_(u.view_as(fw_grad)) + raw_outputs = _as_tuple(fn(*dual_inputs)) + dual_outputs = filter(_is_float_or_complex_tensor, raw_outputs) + for index_o, d_o in enumerate(dual_outputs): + val, res = fwAD.unpack_dual(d_o) + if ( + check_grad_dtypes + and res is not None + and val.is_complex() != res.is_complex() + ): + raise GradcheckError("Forward AD gradient has dtype mismatch.") + + # Remove extra dimension of size 1 corresponding to the reduced input + jacobians[i][index_o].squeeze_(0) + if res is None: + jacobians[i][index_o].zero_() + else: + jacobians[i][index_o].copy_(res.reshape(-1)) + fw_grad.zero_() + else: + # Reconstruct the full Jacobian column by column + for i, fw_grad in enumerate(fw_grads): + for lin_idx, grad_idx in enumerate( + product(*[range(m) for m in fw_grad.size()]) + ): + fw_grad[grad_idx] = 1.0 + raw_outputs = _as_tuple(fn(*dual_inputs)) + dual_outputs = filter(_is_float_or_complex_tensor, raw_outputs) + for index_o, d_o in enumerate(dual_outputs): + val, res = fwAD.unpack_dual(d_o) + if ( + check_grad_dtypes + and res is not None + and val.is_complex() != res.is_complex() + ): + raise GradcheckError( + "Forward AD gradient has dtype mismatch." + ) + + if res is None: + jacobians[i][index_o][lin_idx].zero_() + else: + jacobians[i][index_o][lin_idx].copy_(res.reshape(-1)) + fw_grad[grad_idx] = 0.0 + + return jacobians + + +def _get_input_to_perturb(input): + # Prepare the input so that it can be modified in-place and do certain + # operations that require the tensor to have strides. If fast_mode=False, + # _iter_tensor would handle the below cases: + if input.layout == torch._mkldnn: # type: ignore[attr-defined] # no attr _mkldnn + # Convert to dense so we can perform operations that require strided tensors + input_to_perturb = input.to_dense() + elif _is_sparse_any_tensor(input): + # Clone because input may require grad, and copy_ calls resize_, + # which is not allowed for .data + input_to_perturb = input.clone() + else: + input_to_perturb = input.data + return input_to_perturb + + +def _with_prepare_inputs(fn, inputs, input_idx, input_to_perturb, fast_mode=False): + # Wraps `fn` so that its inputs are already supplied + def wrapped_fn(): + inp = tuple( + _prepare_input(a, input_to_perturb if i == input_idx else None, fast_mode) + if is_tensor_like(a) + else a + for i, a in enumerate(_as_tuple(inputs)) + ) + return tuple(a.clone() for a in _as_tuple(fn(*inp))) + + return wrapped_fn + + +def _get_numerical_jvp_fn(wrapped_fn, input_to_perturb, eps, nbhd_checks_fn): + # Wraps jvp_fn so that certain arguments are already supplied + def jvp_fn(delta): + return _compute_numerical_gradient( + wrapped_fn, input_to_perturb, delta, eps, nbhd_checks_fn + ) + + return jvp_fn + + +def _reshape_tensor_or_tuple(u, shape): + # We don't need to reshape when input corresponding to u is sparse + if isinstance(u, tuple): + if not _is_sparse_any_tensor(u[0]): + return (u[0].reshape(shape), u[1].reshape(shape)) + else: + if not _is_sparse_any_tensor(u): + return u.reshape(shape) + return u + + +def _mul_tensor_or_tuple(u, k): + if isinstance(u, tuple): + return (k * u[0], k * u[1]) + else: + return k * u + + +def _get_numerical_jvp_wrt_specific_input( + fn, input_idx, inputs, u, eps, is_forward_ad=False +) -> list[torch.Tensor]: + input = inputs[input_idx] + input_to_perturb = _get_input_to_perturb(input) + wrapped_fn = _with_prepare_inputs(fn, inputs, input_idx, input_to_perturb, True) + nbhd_checks_fn = functools.partial(_check_outputs_same_dtype_and_shape, eps=eps) + jvp_fn = _get_numerical_jvp_fn(wrapped_fn, input_to_perturb, eps, nbhd_checks_fn) + u = _reshape_tensor_or_tuple(u, input_to_perturb.shape) + u = _mul_tensor_or_tuple(u, eps) + return _compute_numerical_jvps_wrt_specific_input( + jvp_fn, u, input.is_complex(), is_forward_ad + ) + + +def _get_numerical_vJu( + fn, inputs, inp_indices, func_out, all_u, all_v, eps, is_forward_ad +): + # Note that all_v can also be None, in that case, this function only computes Ju. + reduced_jacobians: list[list[torch.Tensor]] = [] + for inp_idx, u in zip(inp_indices, all_u): + all_Ju = _get_numerical_jvp_wrt_specific_input( + fn, inp_idx, inputs, u, eps, is_forward_ad + ) + # Filter out the Ju for non floating point outputs + filtered_Ju = [] + func_out = _as_tuple(func_out) + if len(all_Ju) != len(func_out): + raise AssertionError( + f"Expected all_Ju and func_out to have the same length, " + f"but got {len(all_Ju)} and {len(func_out)}" + ) + for Ju, output in zip(all_Ju, func_out): + if _is_float_or_complex_tensor(output): + filtered_Ju.append(Ju) + else: + # TODO: handle the other Ju + pass + if all_v is not None: + jacobian_scalars: list[torch.Tensor] = [] + for v, Ju in zip(all_v, filtered_Ju): + jacobian_scalars.append(_dot_with_type_promotion(v, Ju)) + reduced_jacobians.append(jacobian_scalars) + else: + reduced_jacobians.append(filtered_Ju) + return reduced_jacobians + + +def _check_jacobians_equal(j1, j2, atol): + # Check whether the max difference between two Jacobian tensors are within some + # tolerance `atol`. + for j1_x, j2_x in zip(j1, j2): + if j1_x.numel() != 0 and (j1_x - j2_x).abs().max() > atol: + return False + return True + + +def _stack_and_check_tensors( + list_of_list_of_tensors, inputs, numel_outputs +) -> tuple[tuple[torch.Tensor, ...], bool, bool]: + # For the ith tensor in the inner list checks whether it has the same size and + # dtype as the ith differentiable input. + out_jacobians = _allocate_jacobians_with_inputs(inputs, numel_outputs) + diff_input_list = list(_iter_tensors(inputs, True)) + correct_grad_sizes = True + correct_grad_types = True + for i, tensor_list in enumerate(list_of_list_of_tensors): + inp = diff_input_list[i] + out_jacobian = out_jacobians[i] + for j, tensor in enumerate(tensor_list): + if tensor is not None and tensor.size() != inp.size(): + correct_grad_sizes = False + elif tensor is not None and tensor.dtype != inp.dtype: + correct_grad_types = False + if tensor is None: + out_jacobian[:, j].zero_() + else: + dense = tensor.to_dense() if tensor.layout != torch.strided else tensor + if out_jacobian[:, j].numel() != dense.numel(): + raise AssertionError( + f"Expected out_jacobian column to have {dense.numel()} elements, " + f"but got {out_jacobian[:, j].numel()}" + ) + out_jacobian[:, j] = dense.reshape(-1) + return out_jacobians, correct_grad_sizes, correct_grad_types + + +FAILED_NONDET_MSG = """\n +NOTE: If your op relies on non-deterministic operations i.e., it is listed here: +https://pytorch.org/docs/stable/generated/torch.use_deterministic_algorithms.html +this failure might be expected. + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `nondet_tol=` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `gradcheck_nondet_tol=`. +- is a Module test (e.g., in common_nn.py), then modify the corresponding + module_test entry to have `gradcheck_nondet_tol=` +""" + + +def _check_analytical_jacobian_attributes( + inputs, output, nondet_tol, check_grad_dtypes, fast_mode=False, v=None +) -> tuple[torch.Tensor, ...]: + # This is used by both fast and slow mode: + # - For slow mode, vjps[i][j] is the jth row of the Jacobian wrt the ith + # input. + # - For fast mode, vjps[i][0] is a linear combination of the rows + # of the Jacobian wrt the ith input + diff_input_list = list(_iter_tensors(inputs, True)) + + def vjp_fn(grad_output): + return torch.autograd.grad( + output, diff_input_list, grad_output, retain_graph=True, allow_unused=True + ) + + # Compute everything twice to check for nondeterminism (which we call reentrancy) + if fast_mode: + vjps1 = _get_analytical_vjps_wrt_specific_output(vjp_fn, output.clone(), v) + vjps2 = _get_analytical_vjps_wrt_specific_output(vjp_fn, output.clone(), v) + else: + vjps1 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + vjps2 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + + output_numel = output.numel() if not fast_mode else 1 + jacobians1, types_ok, sizes_ok = _stack_and_check_tensors( + vjps1, inputs, output_numel + ) + jacobians2, _, _ = _stack_and_check_tensors(vjps2, inputs, output_numel) + reentrant = _check_jacobians_equal(jacobians1, jacobians2, nondet_tol) + + if not types_ok and check_grad_dtypes: + raise GradcheckError("Gradient has dtype mismatch") + if not sizes_ok: + raise GradcheckError("Analytical gradient has incorrect size") + if not reentrant: + raise GradcheckError( + "Backward is not reentrant, i.e., running backward with " + "same input and grad_output multiple times gives different values, " + "although analytical gradient matches numerical gradient." + f"The tolerance for nondeterminism was {nondet_tol}." + FAILED_NONDET_MSG + ) + return jacobians1 + + +def _get_analytical_vJu_backward_mode( + inputs, outputs, nondet_tol, check_grad_dtypes, all_v, all_u +): + reduced_jacobians: list[list[torch.Tensor]] = [] + for output, v in zip(outputs, all_v): + all_vJ = _check_analytical_jacobian_attributes( + inputs, output, nondet_tol, check_grad_dtypes, fast_mode=True, v=v + ) + jacobian_scalars: list[torch.Tensor] = [] + for vJ, u in zip(all_vJ, all_u): + # Why do we need squeeze here? vJ is a 2-d tensor so that we can reuse + # the error checking logic from slow mode + vJ = vJ.T.squeeze(0) + if vJ.is_complex(): # C -> R + tv = torch.view_as_real(vJ.resolve_conj()) + tr = tv.select(-1, 0) + ti = tv.select(-1, 1) + jacobian_scalars.append(tr.dot(u[0]) + 1j * ti.dot(u[1])) + else: # R -> R + jacobian_scalars.append(vJ.dot(u)) + reduced_jacobians.append(jacobian_scalars) + return reduced_jacobians + + +@deprecated( + "`get_analytical_jacobian` was part of PyTorch's private API and not " + "meant to be exposed. We are deprecating it and it will be removed " + "in a future version of PyTorch. If you have a specific use for " + "this or feature request for this to be a stable API, please file " + "us an issue at https://github.com/pytorch/pytorch/issues/new", + category=FutureWarning, +) +def get_analytical_jacobian(inputs, output, nondet_tol=0.0, grad_out=1.0): + # Replicates the behavior of the old get_analytical_jacobian before the refactor + # This shares much of its code with _check_analytical_jacobian_attributes + if ( + grad_out != 1.0 + ): # grad_out param is only kept for backward compatibility reasons + raise ValueError( + "Expected grad_out to be 1.0. get_analytical_jacobian no longer " + "supports values of grad_out != 1.0." + ) + if output.is_complex(): + raise ValueError( + "Expected output to be non-complex. get_analytical_jacobian no " + "longer supports functions that return complex outputs." + ) + diff_input_list = list(_iter_tensors(inputs, True)) + + def vjp_fn(grad_output): + return torch.autograd.grad( + output, diff_input_list, grad_output, retain_graph=True, allow_unused=True + ) + + # Compute everything twice to check for nondeterminism (which we call reentrancy) + vjps1 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + vjps2 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + + output_numel = output.numel() + jacobians1, types_ok, sizes_ok = _stack_and_check_tensors( + vjps1, inputs, output_numel + ) + jacobians2, _, _ = _stack_and_check_tensors(vjps2, inputs, output_numel) + reentrant = _check_jacobians_equal(jacobians1, jacobians2, nondet_tol) + + return jacobians1, reentrant, sizes_ok, types_ok + + +def _get_analytical_jacobian(inputs, outputs, input_idx, output_idx): + # Computes the analytical Jacobian in slow mode for a single input-output pair. + # Forgoes performing checks on dtype, shape, and reentrancy. + jacobians = _check_analytical_jacobian_attributes( + inputs, outputs[output_idx], nondet_tol=float("inf"), check_grad_dtypes=False + ) + return jacobians[input_idx] + + +def _compute_analytical_jacobian_rows( + vjp_fn, sample_output +) -> list[list[torch.Tensor | None]]: + # Computes Jacobian row-by-row by projecting `vjp_fn` = v^T J on standard basis + # vectors: vjp_fn(e) = e^T J is a corresponding row of the Jacobian. + # NB: this function does not assume vjp_fn(v) to return tensors with the same + # number of elements for different v. This is checked when we later combine the + # rows into a single tensor. + grad_out_base = torch.zeros_like( + sample_output, memory_format=torch.legacy_contiguous_format + ) + flat_grad_out = grad_out_base.view(-1) + # jacobians_rows[i][j] is the Jacobian jth row for the ith input + jacobians_rows: list[list[torch.Tensor | None]] = [] + for j in range(flat_grad_out.numel()): + flat_grad_out.zero_() + flat_grad_out[j] = 1.0 # projection for jth row of Jacobian + grad_inputs = vjp_fn(grad_out_base) + for i, d_x in enumerate(grad_inputs): + if j == 0: + jacobians_rows.append([]) + jacobians_rows[i] += [ + d_x.clone() if isinstance(d_x, torch.Tensor) else None + ] + return jacobians_rows + + +def _get_analytical_vjps_wrt_specific_output( + vjp_fn, sample_output, v +) -> list[list[torch.Tensor | None]]: + grad_inputs = vjp_fn(v.reshape(sample_output.shape)) + vjps: list[list[torch.Tensor | None]] = [ + [vjp.clone() if isinstance(vjp, torch.Tensor) else None] for vjp in grad_inputs + ] + return vjps + + +def _check_inputs(tupled_inputs) -> bool: + # Make sure that gradients are saved for at least one input + any_input_requiring_grad = False + for idx, inp in enumerate(tupled_inputs): + if is_tensor_like(inp) and inp.requires_grad: + if not (inp.dtype == torch.float64 or inp.dtype == torch.complex128): + warnings.warn( + f"Input #{idx} requires gradient and " + "is not a double precision floating point or complex. " + "This check will likely fail if all the inputs are " + "not of double precision floating point or complex. ", + stacklevel=2, + ) + if inp.is_sparse: + content = inp._values() + elif _is_sparse_compressed_tensor(inp): + content = inp.values() + else: + content = inp + # TODO: To cover more problematic cases, replace stride = 0 check with + # "any overlap in memory" once we have a proper function to check it. + if content.layout is not torch._mkldnn: # type: ignore[attr-defined] + if not all( + st > 0 or sz <= 1 + for st, sz in zip(content.stride(), content.size()) + ): + raise RuntimeError( + f"The {idx}th input has a dimension with stride 0. gradcheck only " + "supports inputs that are non-overlapping to be able to " + "compute the numerical gradients correctly. You should call " + ".contiguous on the input before passing it to gradcheck." + ) + any_input_requiring_grad = True + + if not any_input_requiring_grad: + raise ValueError( + "gradcheck expects at least one input tensor to require gradient, " + "but none of them have requires_grad=True." + ) + return True + + +def _check_outputs(outputs) -> None: + if any(_is_sparse_any_tensor(t) for t in outputs if isinstance(t, torch.Tensor)): + # it is easier to call to_dense() on the sparse output than + # to modify analytical jacobian + raise ValueError( + "Sparse output is not supported at gradcheck yet. " + "Please call to_dense(masked_grad=...) on the output of fn for gradcheck." + ) + if any(t.layout == torch._mkldnn for t in outputs if isinstance(t, torch.Tensor)): # type: ignore[attr-defined] + raise ValueError( + "MKLDNN output is not supported at gradcheck yet. " + "Please call to_dense(masked_grad=...) on the output of fn for gradcheck." + ) + + +def _check_no_differentiable_outputs( + func, inputs, func_out, eps, *, is_forward_ad +) -> bool: + # When there are no differentiable outputs, numerical gradient for a function is + # expected to be zero. + jacobians_all_inputs_outputs = _get_numerical_jacobian( + func, inputs, func_out, eps=eps, is_forward_ad=is_forward_ad + ) + for jacobians_all_outputs_and_fixed_input in jacobians_all_inputs_outputs: + for jacobian in jacobians_all_outputs_and_fixed_input: + if torch.ne(jacobian, 0).sum() > 0: + raise GradcheckError( + "Numerical gradient for function expected to be zero" + ) + return True + + +def _check_no_differentiable_outputs_fast( + func, func_out, all_inputs, inputs_indices, all_u, eps, nondet_tol +): + for inp_idx, u in zip(inputs_indices, all_u): + jvps = _get_numerical_jvp_wrt_specific_input(func, inp_idx, all_inputs, u, eps) + for jvp in jvps: + if jvp.numel() == 0: + continue + if (jvp - torch.zeros_like(jvp)).abs().max() > nondet_tol: + raise GradcheckError( + "Numerical gradient for function expected to be zero" + ) + return True + + +FAILED_BATCHED_GRAD_MSG = """ +gradcheck or gradgradcheck failed while testing batched gradient computation. +This could have been invoked in a number of ways (via a test that calls +gradcheck/gradgradcheck directly or via an autogenerated test). + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `check_batched_grad=False` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `check_batched_grad=False` and/or `check_batched_gradgrad=False`. + +If you're modifying an existing operator that supports batched grad computation, +or wish to make a new operator work with batched grad computation, please read +the following. + +To compute batched grads (e.g., jacobians, hessians), we vmap over the backward +computation. The most common failure case is if there is a 'vmap-incompatible +operation' in the backward pass. Please see +NOTE: [How to write vmap-compatible backward formulas] +in the codebase for an explanation of how to fix this. +""".strip() + +FAILED_BATCHED_GRAD_MSG_FWD_AD = """ +gradcheck failed while testing batched gradient computation with forward-mode AD. +This test is enabled automatically when both `check_batched_grad=True` +and `check_forward_ad=True`, but can be disabled in the following ways +dependong on how the test was invoked (via a test that calls gradcheck +directly or via an autogenerated test). + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `check_batched_forward_grad=False` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `check_batched_forward_grad=False` +""" + + +def _get_failed_batched_grad_test_msg( + output_idx, input_idx, res, exp, is_forward_ad=False +): + return f""" +For output {output_idx} and input {input_idx}: + +{FAILED_BATCHED_GRAD_MSG_FWD_AD if is_forward_ad else FAILED_BATCHED_GRAD_MSG} + +Got: +{res} + +Expected: +{exp} +""".strip() + + +def _test_batched_grad_forward_ad(func, inputs) -> bool: + fwAD = torch.autograd.forward_ad # To avoid early import issues (do we need this?) + if not isinstance(inputs, tuple): + raise AssertionError("Expected inputs to be a tuple") + + for input_idx, current_input in enumerate(inputs): + if not (is_tensor_like(current_input) and current_input.requires_grad): + continue + + def jvp(tangent: torch.Tensor): + with fwAD.dual_level(): + dual = fwAD.make_dual(current_input.detach(), tangent) + inputs_with_dual = tuple( + dual + if idx == input_idx + else (inp.detach() if is_tensor_like(inp) else inp) + for idx, inp in enumerate(inputs) + ) + dual_outputs = _as_tuple(func(*inputs_with_dual)) + ret = [] + for dual_output in dual_outputs: + if dual_output is None: + continue + primal_out, tangent_out = fwAD.unpack_dual(dual_output) + if tangent_out is not None: + ret.append(tangent_out) + else: + ret.append( + torch.zeros( + [], dtype=primal_out.dtype, device=primal_out.device + ).expand(primal_out.shape) + ) + return tuple(ret) + + if not _is_float_or_complex_tensor(current_input): + continue + + tangents = [torch.randn_like(current_input) for _ in range(2)] + expected = [jvp(t) for t in tangents] + expected = [torch.stack(shards) for shards in zip(*expected)] + + try: + result = _vmap(jvp)(torch.stack(tangents)) + except RuntimeError as ex: + # Rethrow to provide a better error message + raise GradcheckError( + f"While computing batched gradients, got: {ex}\n\n{FAILED_BATCHED_GRAD_MSG_FWD_AD}" + ) from ex + + for input_idx, (res, exp) in enumerate(zip(result, expected)): + if torch.allclose(res, exp): + continue + raise GradcheckError( + _get_failed_batched_grad_test_msg( + input_idx, input_idx, res, exp, is_forward_ad=True + ) + ) + return True + + +def _test_batched_grad(input, output, output_idx) -> bool: + # NB: _test_batched_grad compares two autograd.grad invocations with a single + # vmap(autograd.grad) invocation. It's not exactly a "gradcheck" in the + # sense that we're not comparing an analytical jacobian with a numeric one, + # but it is morally similar (we could have computed a full analytic jac + # via vmap, but that is potentially slow) + diff_input_list = list(_iter_tensors(input, True)) + grad = functools.partial( + torch.autograd.grad, + output, + diff_input_list, + retain_graph=True, + allow_unused=True, + ) + + def vjp(v): + results = grad(v) + results = tuple( + grad + if grad is not None + else torch.zeros([], dtype=inp.dtype, device=inp.device).expand(inp.shape) + for grad, inp in zip(results, diff_input_list) + ) + return results + + grad_outputs = [torch.randn_like(output) for _ in range(2)] + + expected = [vjp(gO) for gO in grad_outputs] + expected = [torch.stack(shards) for shards in zip(*expected)] + + # Squash warnings since these are expected to happen in most cases + # NB: this doesn't work for CUDA tests: https://github.com/pytorch/pytorch/issues/50209 + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="There is a performance drop") + warnings.filterwarnings("ignore", message="Please use `torch.vmap`") + try: + result = vmap(vjp)(torch.stack(grad_outputs)) + except RuntimeError as ex: + # It's OK that we're not raising the error at the correct callsite. + # That's because the callsite is always going to inside the Python + # autograd.grad instead of the C++ traceback of what line in the + # backward formula + raise GradcheckError( + f"While computing batched gradients, got: {ex}\n\n{FAILED_BATCHED_GRAD_MSG}" + ) from ex + + for input_idx, (res, exp) in enumerate(zip(result, expected)): + if torch.allclose(res, exp): + continue + raise GradcheckError( + _get_failed_batched_grad_test_msg(output_idx, input_idx, res, exp) + ) + return True + + +def _test_backward_mul_by_grad_output(outputs, inputs, masked) -> bool: + # Tests that backward is multiplied by grad_output + diff_input_list: list[torch.Tensor] = list(_iter_tensors(inputs, True)) + if not diff_input_list: + raise GradcheckError("no Tensors requiring grad found in input") + grads_input = torch.autograd.grad( + outputs, + diff_input_list, + [ + torch.zeros_like(o, memory_format=torch.legacy_contiguous_format) + for o in outputs + ], + allow_unused=True, + ) + for gi, di in zip(grads_input, diff_input_list): + if gi is None: + continue + if isinstance(gi, torch.Tensor) and gi.layout != torch.strided: + if gi.layout != di.layout: + raise GradcheckError( + "grad is incorrect layout (" + + str(gi.layout) + + " is not " + + str(di.layout) + + ")" + ) + if _is_sparse_any_tensor(gi): + sparse_kind = str(gi.layout).replace("torch.", "").replace("_coo", "") + if gi.sparse_dim() != di.sparse_dim(): + raise GradcheckError( + f"grad is {sparse_kind} tensor, but has incorrect sparse_dim" + f" {gi.sparse_dim()}, expected {di.sparse_dim()}" + ) + if gi.dense_dim() != di.dense_dim(): + raise GradcheckError( + f"grad is {sparse_kind} tensor, but has incorrect dense_dim" + f" {gi.dense_dim()}, expected {di.dense_dim()}" + ) + gi = gi.to_dense() + di = di.to_dense() + if masked: + if not torch.allclose(gi, torch.zeros_like(gi)): + raise GradcheckError("backward not multiplied by grad_output") + elif not gi.eq(0).all(): + raise GradcheckError("backward not multiplied by grad_output") + if gi.dtype != di.dtype: + raise GradcheckError("grad is incorrect type") + if gi.device != di.device: + raise GradcheckError("grad is incorrect device") + if gi.size() != di.size(): + raise GradcheckError("grad is incorrect size") + return True + + +def _test_undefined_forward_mode(func, outputs, inputs): + fwAD = torch.autograd.forward_ad + + _inp_tensors_idx, inp_tensors = _get_inp_tensors(inputs) + _all_v, all_u, _all_u_dense = _make_vectors( + inp_tensors, outputs, use_forward_ad=True + ) + + with fwAD.dual_level(): + fw_grads = [] + dual_inputs = [] + tensor_indices = set() + for i, inp in enumerate(inputs): + if is_tensor_like(inp) and inp.requires_grad: + if inp.layout == torch._mkldnn: # type: ignore[attr-defined] + raise ValueError( + "MKLDNN inputs are not support for forward AD gradcheck." + ) + + inp = fwAD.make_dual(inp.detach(), torch.zeros_like(inp)) + # If inp is a differentiable view, the dual might not be the tangent given to + # make_dual, so read it explicitly from the dual tensor + fw_grads.append(fwAD.unpack_dual(inp)[1]) + tensor_indices.add(i) + dual_inputs.append(inp) + + for fw_grad, u in zip(fw_grads, all_u): + fw_grad.copy_(u.view_as(fw_grad)) + + for idx, inp in enumerate(inputs): + if idx not in tensor_indices: + continue + dual_inp_obj = dual_inputs[idx] + + # case 1 (Materialized Zero Tensor Tangent) + dual_inputs[idx] = fwAD.make_dual(inp.detach(), torch.zeros_like(inp)) + raw_outputs = _as_tuple(func(*dual_inputs)) + dual_outputs1 = filter(_is_float_or_complex_tensor, raw_outputs) + + # case 2 (Efficient Zero Tensor Tangent since we don't make a dual object and pass a regular tensor) + dual_inputs[idx] = inp.detach() + raw_outputs = _as_tuple(func(*dual_inputs)) + dual_outputs2 = filter(_is_float_or_complex_tensor, raw_outputs) + + # reset + dual_inputs[idx] = dual_inp_obj + + for index_o, (d_o1, d_o2) in enumerate(zip(dual_outputs1, dual_outputs2)): + _val1, res1 = fwAD.unpack_dual(d_o1) + _val2, res2 = fwAD.unpack_dual(d_o2) + + if not (res1 is None or res2 is None): + if not torch.allclose(res1, res2): + raise GradcheckError( + "Mismatch in tangent values for output with index: ", + index_o, + " when input: ", + inp, + " has an undefined tangent value. ", + " Got: ", + res1, + " but expected: ", + res2, + ) + return True + + +def _test_undefined_backward_mode(func, outputs, inputs) -> bool: + diff_input_list: list[torch.Tensor] = list(_iter_tensors(inputs, True)) + if not diff_input_list: + raise GradcheckError("no Tensors requiring grad found in input") + + def warn_bc_breaking(): + warnings.warn( + "Backwards compatibility: New undefined gradient support checking " + "feature is enabled by default, but it may break existing callers " + "of this function. If this is true for you, you can call this " + 'function with "check_undefined_grad=False" to disable the feature', + stacklevel=2, + ) + + def check_undefined_grad_support(output_to_check): + grads_output = [ + torch.zeros_like(o, memory_format=torch.legacy_contiguous_format) + for o in output_to_check + ] + try: + grads_input = torch.autograd.grad( + output_to_check, diff_input_list, grads_output, allow_unused=True + ) + except RuntimeError as e: + warn_bc_breaking() + raise GradcheckError( + "Expected backward function to handle undefined output grads. " + 'Please look at "Notes about undefined output gradients" in ' + '"tools/autograd/derivatives.yaml"' + ) from e + + for gi in grads_input: + if (gi is not None) and (not gi.eq(0).all()): + warn_bc_breaking() + raise GradcheckError( + "Expected all input grads to be undefined or zero when all output grads are undefined " + 'or zero. Please look at "Notes about undefined output gradients" in ' + '"tools/autograd/derivatives.yaml"' + ) + return True + + # All backward functions must work properly if all output grads are undefined + outputs_to_check = [ + [ + torch._C._functions.UndefinedGrad()(o) + for o in _differentiable_outputs(func(*inputs)) + # This check filters out Tensor-likes that aren't instances of Tensor. + if isinstance(o, torch.Tensor) + ] + ] + + # If there are multiple output grads, we should be able to undef one at a time without error + if len(outputs_to_check[0]) > 1: + for undef_grad_idx in range(len(outputs)): + output_to_check = _differentiable_outputs(func(*inputs)) + outputs_to_check.append( + [ + torch._C._functions.UndefinedGrad()(o) + if idx == undef_grad_idx + else o + for idx, o in enumerate(output_to_check) + ] + ) + + return all(check_undefined_grad_support(output) for output in outputs_to_check) + + +def _as_tuple(x): + if isinstance(x, tuple): + return x + elif isinstance(x, list): + return tuple(x) + else: + return (x,) + + +def _differentiable_outputs(x): + return tuple(o for o in _as_tuple(x) if o.requires_grad) + + +def _get_notallclose_msg( + analytical, + numerical, + output_idx, + input_idx, + complex_indices, + test_imag=False, + is_forward_ad=False, +) -> str: + out_is_complex = ( + (not is_forward_ad) and complex_indices and output_idx in complex_indices + ) + inp_is_complex = is_forward_ad and complex_indices and input_idx in complex_indices + part = "imaginary" if test_imag else "real" + element = "inputs" if is_forward_ad else "outputs" + prefix = ( + "" + if not (out_is_complex or inp_is_complex) + else f"While considering the {part} part of complex {element} only, " + ) + mode = "computed with forward mode " if is_forward_ad else "" + return ( + prefix + + f"Jacobian {mode}mismatch for output {output_idx:d} with respect to input {input_idx:d},\n" + f"numerical:{numerical}\nanalytical:{analytical}\n" + ) + + +def _transpose(matrix_of_tensors): + # returns list of tuples + return list(zip(*matrix_of_tensors)) + + +def _real_and_imag_output(fn): + # returns new functions real(fn), and imag(fn) where real(fn) and imag(fn) behave the same as + # the original fn, except torch.real or torch.imag are applied to the complex outputs + def apply_to_c_outs(fn, fn_to_apply): + def wrapped_fn(*inputs): + outs = _as_tuple(fn(*inputs)) + return tuple(fn_to_apply(o) if o.is_complex() else o for o in outs) + + return wrapped_fn + + return apply_to_c_outs(fn, torch.real), apply_to_c_outs(fn, torch.imag) + + +def _real_and_imag_input(fn, complex_inp_indices, tupled_inputs): + # returns new functions that take real inputs instead of complex inputs as + # (x, y) -> fn(x + y * 1j). And it computes: inp -> fn(inp + y * 1j) and inp -> fn(x + inp * 1j). + # In each case, the other part is considered constant. + # We do not use 0 for the constant here to make sure we always call the user function with a valid input. + def apply_to_c_inps(fn, fn_to_apply): + def wrapped_fn(*inputs): + new_inputs = list(inputs) + for should_be_complex in complex_inp_indices: + new_inputs[should_be_complex] = fn_to_apply( + new_inputs[should_be_complex], tupled_inputs[should_be_complex] + ) + return _as_tuple(fn(*new_inputs)) + + return wrapped_fn + + real_fn = apply_to_c_inps(fn, lambda inp, orig: inp + orig.imag * 1j) + imag_fn = apply_to_c_inps(fn, lambda inp, orig: orig.real + inp * 1j) + return real_fn, imag_fn + + +def _gradcheck_real_imag( + gradcheck_fn, + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + check_forward_ad, + check_backward_ad, + nondet_tol, + check_undefined_grad, +): + complex_out_indices = [i for i, o in enumerate(outputs) if o.is_complex()] + has_any_complex_output = any(o.is_complex() for o in _as_tuple(func_out)) + if check_backward_ad: + if has_any_complex_output: + real_fn, imag_fn = _real_and_imag_output(func) + + imag_func_out = imag_fn(*tupled_inputs) + imag_outputs = _differentiable_outputs(imag_func_out) + gradcheck_fn( + imag_fn, + imag_func_out, + tupled_inputs, + imag_outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_out_indices, + test_imag=True, + ) + + real_func_out = real_fn(*tupled_inputs) + real_outputs = _differentiable_outputs(real_func_out) + gradcheck_fn( + real_fn, + real_func_out, + tupled_inputs, + real_outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_out_indices, + ) + else: + gradcheck_fn( + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + ) + + if check_forward_ad: + complex_inp_indices = [ + i + for i, inp in enumerate(tupled_inputs) + if is_tensor_like(inp) and inp.is_complex() + ] + if complex_inp_indices: + real_fn, imag_fn = _real_and_imag_input( + func, complex_inp_indices, tupled_inputs + ) + + imag_inputs = [ + inp.imag if is_tensor_like(inp) and inp.is_complex() else inp + for inp in tupled_inputs + ] + imag_func_out = imag_fn(*imag_inputs) + diff_imag_func_out = _differentiable_outputs(imag_func_out) + gradcheck_fn( + imag_fn, + imag_func_out, + imag_inputs, + diff_imag_func_out, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_inp_indices, + test_imag=True, + use_forward_ad=True, + ) + + real_inputs = [ + inp.real if is_tensor_like(inp) and inp.is_complex() else inp + for inp in tupled_inputs + ] + real_func_out = real_fn(*real_inputs) + diff_real_func_out = _differentiable_outputs(real_func_out) + gradcheck_fn( + real_fn, + real_func_out, + real_inputs, + diff_real_func_out, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_inp_indices, + use_forward_ad=True, + ) + if check_undefined_grad: + _test_undefined_forward_mode(imag_fn, imag_func_out, imag_inputs) + _test_undefined_forward_mode(real_fn, real_func_out, real_inputs) + else: + gradcheck_fn( + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + use_forward_ad=True, + ) + if check_undefined_grad: + _test_undefined_forward_mode(func, outputs, tupled_inputs) + + +def _slow_gradcheck( + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + *, + use_forward_ad=False, + complex_indices=None, + test_imag=False, + masked=False, +): + func_out = _as_tuple(func_out) + if not outputs: + return _check_no_differentiable_outputs( + func, tupled_inputs, func_out, eps=eps, is_forward_ad=use_forward_ad + ) + tupled_inputs_numerical = tupled_inputs if masked else _densify(tupled_inputs) + + numerical = _transpose( + _get_numerical_jacobian( + func, + tupled_inputs_numerical, + func_out, + eps=eps, + is_forward_ad=use_forward_ad, + ) + ) + # Note: [numerical vs analytical output length] + # The numerical path returns jacobian quantity for all outputs, even if requires_grad of that + # output is False. This behavior is necessary for _check_no_differentiable_outputs to work. + numerical = [nj for o, nj in zip(func_out, numerical) if o.requires_grad] + if use_forward_ad: + analytical_forward = _get_analytical_jacobian_forward_ad( + func, tupled_inputs, func_out, check_grad_dtypes=check_grad_dtypes + ) + + for i, n_per_out in enumerate(numerical): + for j, n in enumerate(n_per_out): + a = analytical_forward[j][i] + if not _allclose_with_type_promotion(a, n.to(a.device), rtol, atol): + raise GradcheckError( + _get_notallclose_msg( + a, n, i, j, complex_indices, test_imag, is_forward_ad=True + ) + ) + else: + for i, o in enumerate(outputs): + analytical = _check_analytical_jacobian_attributes( + tupled_inputs, o, nondet_tol, check_grad_dtypes + ) + + for j, (a, n) in enumerate(zip(analytical, numerical[i])): + if not _allclose_with_type_promotion(a, n.to(a.device), rtol, atol): + raise GradcheckError( + _get_notallclose_msg(a, n, i, j, complex_indices, test_imag) + ) + + return True + + +def _dot_with_type_promotion(u, v): + if u.dim() != 1 or v.dim() != 1: + raise AssertionError( + f"Expected u and v to be 1D tensors, but got dims {u.dim()} and {v.dim()}" + ) + return (u * v).sum() + + +def _allclose_with_type_promotion(a, b, rtol, atol): + promoted_type = torch.promote_types(a.dtype, b.dtype) + a = a.to(dtype=promoted_type) + b = b.to(dtype=promoted_type) + return torch.allclose(a, b, rtol, atol) + + +def _to_real_dtype(dtype): + if dtype == torch.complex128: + return torch.float64 + elif dtype == torch.complex64: + return torch.float32 + else: + return dtype + + +def _vec_from_tensor(x, generator, downcast_complex=False): + # Create a random vector with the same number of elements as x and the same + # dtype/device. If x is complex and downcast_complex is False, we create a + # complex tensor with only real component. + if x.layout == torch.sparse_coo: + # For sparse, create a random sparse vec with random values in the same + # indices. Make sure size is set so that it isn't inferred to be smaller. + x_values = x._values() + dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype + values = ( + torch.rand(x_values.numel(), generator=generator) + .to(dtype=dtype, device=x.device) + .view(x_values.shape) + ) + values /= values.norm() + vec = torch.sparse_coo_tensor(x._indices(), values, x.size(), device=x.device) + elif _is_sparse_compressed_tensor(x): + if x.layout in {torch.sparse_csr, torch.sparse_bsr}: + compressed_indices, plain_indices = x.crow_indices(), x.col_indices() + else: + compressed_indices, plain_indices = x.ccol_indices(), x.row_indices() + x_values = x.values() + dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype + values = ( + torch.rand(x_values.numel(), generator=generator) + .to(dtype=dtype, device=x.device) + .view(x_values.shape) + ) + values /= values.norm() + vec = torch.sparse_compressed_tensor( + compressed_indices, + plain_indices, + values, + x.size(), + layout=x.layout, + device=x.device, + ) + else: + dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype + vec = torch.rand(x.numel(), generator=generator).to( + dtype=dtype, device=x.device + ) + vec /= vec.norm() + return vec + + +def _get_inp_tensors(tupled_inputs): + inp_idx_tup = [ + (i, t) + for i, t in enumerate(tupled_inputs) + if is_tensor_like(t) and t.requires_grad + ] + return [tup[0] for tup in inp_idx_tup], [tup[1] for tup in inp_idx_tup] + + +def _adjusted_atol(atol, u, v): + # In slow gradcheck, we compare A and B element-wise, i.e., for some a, b we + # allow: |a - b| < atol + rtol * b. But since we now compare q1 = v^T A u and + # q2 = v^T B u, we must allow |q1 - q2| < v^T E u + rtol * v^T B u, where E is + # the correctly sized matrix in which each entry is atol. + # + # We see that atol needs to be scaled by v^T M u (where M is an all-ones M x N + # matrix): v^T M u = \sum_{i} \sum_{j} u[i] * v[j] = sum(u) * sum(v). + + # For the case of complex inputs, u has re and im. components: u = (ur, ui). + # Let q = a - b = (qr, qi) in the above notation, eg. q is the difference between analytic and numerical + # Jacobians. Then the transformed tolerance checks being done in the torch.allclose ops + # are of form abs(Re{q_r u_r - q_i u_i}) < atol * (abs(u_r) + abs(u_i)), and equivalently + # for imaginary components abs(Im{q_r u_r - q_i ui}) < atol * (abs(u_r) + abs(u_i)). + # Since u is drawn randomly non-negative, the end effect is a factor + # (sum(u_r) + sum(u_i)) * sum(v) increase in atol for complex inputs, eg. + # a statistical factor of 2 as compared to the real case. + + sum_v = 1.0 if v is None else v.sum() + + if isinstance(u, tuple): + # case of complex input + ur, ui = u[0], u[1] + sum_ur = ur.sum() + sum_ui = ui.sum() + complex_modified_atol = atol * (float(sum_ur) + float(sum_ui)) * float(sum_v) + return complex_modified_atol + + # case of real input + sum_u = u.sum() + modified_atol = atol * float(sum_u) * float(sum_v) + return modified_atol + + +FAST_FAIL_SLOW_OK_MSG = """ +Fast gradcheck failed but element-wise differences are small. This means that the +test might've passed in slow_mode! + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck: + +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `fast_mode=False` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `gradcheck_fast_mode=False` +- is a Module test (e.g., in common_nn.py), then modify the corresponding + module_test entry to have `gradcheck_fast_mode=False` +""".strip() + + +def _run_slow_mode_and_get_error( + func, tupled_inputs, outputs, input_idx, output_idx, rtol, atol, eps, is_forward_ad +): + # Compute jacobians in slow mode for better error message + slow_numerical = _get_numerical_jacobian( + func, tupled_inputs, outputs, eps=eps, is_forward_ad=is_forward_ad + )[input_idx][output_idx] + if is_forward_ad: + + def new_fn(inp): + new_inputs = list(tupled_inputs) + new_inputs[input_idx] = inp + return _as_tuple(func(*new_inputs))[output_idx] + + slow_analytical = _get_analytical_jacobian_forward_ad( + new_fn, (tupled_inputs[input_idx],), (outputs[output_idx],) + )[0][0] + else: + slow_analytical = _get_analytical_jacobian( + tupled_inputs, outputs, input_idx, output_idx + ) + + # Assume jacobians are non-empty and have the same shape + slow_max_diff = (slow_numerical - slow_analytical).abs().max() + + slow_allclose = torch.allclose(slow_analytical, slow_numerical, rtol, atol) + msg = ( + "\nThe above quantities relating the numerical and analytical jacobians are computed \n" + "in fast mode. See: https://github.com/pytorch/pytorch/issues/53876 for more background \n" + "about fast mode. Below, we recompute numerical and analytical jacobians in slow mode:\n\n" + f"Numerical:\n {slow_numerical}\n" + f"Analytical:\n{slow_analytical}\n\n" + f"The max per-element difference (slow mode) is: {slow_max_diff}.\n" + ) + if slow_allclose: + # Slow gradcheck would've passed! + msg += FAST_FAIL_SLOW_OK_MSG + return msg + + +def _to_flat_dense_if_sparse(tensor): + if _is_sparse_any_tensor(tensor): + return tensor.to_dense().reshape(-1) + else: + return tensor + + +def _make_vectors(inp_tensors, outputs, *, use_forward_ad): + # Use our own generator to avoid messing with the user's RNG state + g_cpu = torch.Generator() + + def _vec_from_tensor_cpu(*args): + # Default allocate all tensors on CPU, so they are on the same device as the generator + # even if the user specified a default device + with torch.device("cpu"): + return _vec_from_tensor(*args) + + all_u = [] + all_u_dense = [] + for inp in inp_tensors: + ur = _vec_from_tensor_cpu(inp, g_cpu, True) + ur_dense = _to_flat_dense_if_sparse(ur) + if inp.is_complex(): + ui = _vec_from_tensor_cpu(inp, g_cpu, True) + all_u.append((ur, ui)) + ui_dense = _to_flat_dense_if_sparse(ui) + all_u_dense.append((ur_dense, ui_dense)) + else: + all_u.append(ur) + all_u_dense.append(ur_dense) + all_v = ( + None + if use_forward_ad + else [_vec_from_tensor_cpu(out, g_cpu) for out in outputs] + ) + return all_v, all_u, all_u_dense + + +def _check_analytical_numerical_equal( + all_analytical, + all_numerical, + complex_indices, + tupled_inputs, + outputs, + func, + all_v, + all_u, + rtol, + atol, + eps, + test_imag, + *, + is_forward_ad=False, +): + for i, all_numerical_for_input_i in enumerate(all_numerical): + for j, n in enumerate(all_numerical_for_input_i): + # Forward AD generates the transpose of what this function expects + if is_forward_ad: + a = all_analytical[i][j] + else: + a = all_analytical[j][i] + n = n.to(device=a.device) + updated_atol = _adjusted_atol(atol, all_u[i], all_v[j] if all_v else None) + if not _allclose_with_type_promotion(a, n.to(a.device), rtol, updated_atol): + jacobians_str = _run_slow_mode_and_get_error( + func, tupled_inputs, outputs, i, j, rtol, atol, eps, is_forward_ad + ) + raise GradcheckError( + _get_notallclose_msg( + a, n, j, i, complex_indices, test_imag, is_forward_ad + ) + + jacobians_str + ) + + +def _fast_gradcheck( + func, + func_out, + inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + *, + use_forward_ad=False, + complex_indices=None, + test_imag=False, + masked=False, +): + # See https://github.com/pytorch/pytorch/issues/53876 for details + inp_tensors_idx, inp_tensors = _get_inp_tensors(inputs) + # Backward mode computes v^T * J (VJP) + # Since we computed J * u (JVP) through finite difference method, we perform an equality check + # between VJP * u, v * JVP + # ---- + # Forward mode computes J * u (JVP) + # Since we already compute JVP through finite difference method, + # we don't need v for correctness check here as asserted below + all_v, all_u, all_u_dense = _make_vectors( + inp_tensors, outputs, use_forward_ad=use_forward_ad + ) + + inputs_numerical, all_u_numerical, all_v_numerical = ( + (inputs, all_u, all_v) if masked else _densify((inputs, all_u, all_v)) + ) + + numerical_vJu = _get_numerical_vJu( + func, + inputs_numerical, + inp_tensors_idx, + func_out, + all_u_numerical, + all_v_numerical, + eps, + is_forward_ad=use_forward_ad, + ) + # TODO: replicate https://github.com/pytorch/pytorch/pull/77743 for fast gradcheck as well + if use_forward_ad: + if all_v is not None: + raise AssertionError("Expected all_v to be None.") + analytical_vJu = _get_analytical_jacobian_forward_ad( + func, + inputs, + _as_tuple(func_out), + all_u=all_u, + check_grad_dtypes=check_grad_dtypes, + ) + else: + if not outputs: + _check_no_differentiable_outputs_fast( + func, func_out, inputs, inp_tensors_idx, all_u, eps, nondet_tol + ) + + analytical_vJu = _get_analytical_vJu_backward_mode( + inputs, outputs, nondet_tol, check_grad_dtypes, all_v, all_u_dense + ) + + _check_analytical_numerical_equal( + analytical_vJu, + numerical_vJu, + complex_indices, + inputs, + outputs, + func, + all_v, + all_u, + rtol, + atol, + eps, + test_imag, + is_forward_ad=use_forward_ad, + ) + + return True + + +# Note [VarArg of Tensors] +# ~~~~~~~~~~~~~~~~~~~~~~~~ +# 'func' accepts a vararg of tensors, which isn't expressible in the type system at the moment. +# If https://mypy.readthedocs.io/en/latest/additional_features.html?highlight=callable#extended-callable-types is accepted, +# the '...' first argument of Callable can be replaced with VarArg(Tensor). +# For now, we permit any input. +def gradcheck( + func: Callable[..., _TensorOrTensors], # See Note [VarArg of Tensors] + inputs: _TensorOrTensors, + *, + eps: float = 1e-6, + atol: float = 1e-5, + rtol: float = 1e-3, + raise_exception: bool = True, + nondet_tol: float = 0.0, + check_undefined_grad: bool = True, + check_grad_dtypes: bool = False, + check_batched_grad: bool = False, + check_batched_forward_grad: bool = False, + check_forward_ad: bool = False, + check_backward_ad: bool = True, + fast_mode: bool = False, + masked: bool | None = None, +) -> bool: + r"""Check gradients computed via small finite differences against analytical + gradients wrt tensors in :attr:`inputs` that are of floating point or complex type + and with ``requires_grad=True``. + + The check between numerical and analytical gradients uses :func:`~torch.allclose`. + + For most of the complex functions we consider for optimization purposes, no notion of + Jacobian exists. Instead, gradcheck verifies if the numerical and analytical values of + the Wirtinger and Conjugate Wirtinger derivatives are consistent. Because the gradient + computation is done under the assumption that the overall function has a real-valued + output, we treat functions with complex output in a special way. For these functions, + gradcheck is applied to two real-valued functions corresponding to taking the real + components of the complex outputs for the first, and taking the imaginary components + of the complex outputs for the second. For more details, check out + :ref:`complex_autograd-doc`. + + .. note:: + The default values are designed for :attr:`input` of double precision. + This check will likely fail if :attr:`input` is of less precision, e.g., + ``FloatTensor``. + + .. note:: + Gradcheck may fail when evaluated on non-differentiable points + because the numerically computed gradients via finite differencing may differ + those computed analytically (not necessarily because either is incorrect). + For more context, see :ref:`non-differentiable-func-grad`. + + .. warning:: + If any checked tensor in :attr:`input` has overlapping memory, i.e., + different indices pointing to the same memory address (e.g., from + :func:`torch.Tensor.expand`), this check will likely fail because the numerical + gradients computed by point perturbation at such indices will change + values at all other indices that share the same memory address. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor or a tuple of Tensors + inputs (tuple of Tensor or Tensor): inputs to the function + eps (float, optional): perturbation for finite differences + atol (float, optional): absolute tolerance + rtol (float, optional): relative tolerance + raise_exception (bool, optional): indicating whether to raise an exception if + the check fails. The exception gives more information about the + exact nature of the failure. This is helpful when debugging gradchecks. + nondet_tol (float, optional): tolerance for non-determinism. When running + identical inputs through the differentiation, the results must either match + exactly (default, 0.0) or be within this tolerance. + check_undefined_grad (bool, optional): if ``True``, check if undefined output grads + are supported and treated as zeros, for ``Tensor`` outputs. + check_batched_grad (bool, optional): if ``True``, check if we can compute + batched gradients using prototype vmap support. Defaults to False. + check_batched_forward_grad (bool, optional): if ``True``, checks if we can compute + batched forward gradients using forward ad and prototype vmap support. Defaults to ``False``. + check_forward_ad (bool, optional): if ``True``, check that the gradients computed with forward + mode AD match the numerical ones. Defaults to ``False``. + check_backward_ad (bool, optional): if ``False``, do not perform any checks that rely on + backward mode AD to be implemented. Defaults to ``True``. + fast_mode (bool, optional): Fast mode for gradcheck and gradgradcheck is currently only + implemented for R to R functions. If none of the inputs and outputs are complex + a faster implementation of gradcheck that no longer computes the entire jacobian + is run; otherwise, we fall back to the slow implementation. + masked (bool, optional): if ``True``, the gradients of unspecified elements of + sparse tensors are ignored. Defaults to ``False``. + Returns: + ``True`` if all differences satisfy allclose condition + + """ + if not (check_forward_ad or check_backward_ad): + raise AssertionError( + "Expected at least one of check_forward_ad or check_backward_ad to be True" + ) + if check_batched_grad and not check_backward_ad: + raise AssertionError( + "Setting check_batched_grad=True requires check_backward_ad to be True" + ) + if check_batched_forward_grad and not check_forward_ad: + raise AssertionError( + "Setting check_batched_forward_grad=True requires check_forward_ad to be True" + ) + args = locals().copy() + args.pop("raise_exception") + if not raise_exception: + try: + return _gradcheck_helper(**args) + except GradcheckError: + return False + else: + return _gradcheck_helper(**args) + + +def _gradcheck_helper( + func, + inputs, + eps, + atol, + rtol, + nondet_tol, + check_undefined_grad, + check_grad_dtypes, + check_batched_grad, + check_batched_forward_grad, + check_forward_ad, + check_backward_ad, + fast_mode, + masked, +): + tupled_inputs = _as_tuple(inputs) + _check_inputs(tupled_inputs) + + func_out = func(*tupled_inputs) + outputs = _differentiable_outputs(func_out) + _check_outputs(outputs) + + gradcheck_fn = functools.partial( + _fast_gradcheck if fast_mode else _slow_gradcheck, masked=masked + ) + _gradcheck_real_imag( + gradcheck_fn, + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + check_forward_ad=check_forward_ad, + check_backward_ad=check_backward_ad, + nondet_tol=nondet_tol, + check_undefined_grad=check_undefined_grad, + ) + + if check_batched_forward_grad: + _test_batched_grad_forward_ad(func, tupled_inputs) + + # Short circuit because remaining tests rely on backward AD to be implemented + if not check_backward_ad: + return True + + for i, o in enumerate(outputs): + if check_batched_grad: + _test_batched_grad(tupled_inputs, o, i) + + _test_backward_mul_by_grad_output(outputs, tupled_inputs, masked) + + if check_undefined_grad and check_backward_ad: + _test_undefined_backward_mode(func, outputs, tupled_inputs) + return True + + +def gradgradcheck( + func: Callable[..., _TensorOrTensors], # See Note [VarArg of Tensors] + inputs: _TensorOrTensors, + grad_outputs: _TensorOrOptionalTensors | None = None, + *, + eps: float = 1e-6, + atol: float = 1e-5, + rtol: float = 1e-3, + gen_non_contig_grad_outputs: bool = False, + raise_exception: bool = True, + nondet_tol: float = 0.0, + check_undefined_grad: bool = True, + check_grad_dtypes: bool = False, + check_batched_grad: bool = False, + check_fwd_over_rev: bool = False, + check_rev_over_rev: bool = True, + fast_mode: bool = False, + masked: bool = False, +) -> bool: + r"""Check gradients of gradients computed via small finite differences + against analytical gradients wrt tensors in :attr:`inputs` and + :attr:`grad_outputs` that are of floating point or complex type and with + ``requires_grad=True``. + + This function checks that backpropagating through the gradients computed + to the given :attr:`grad_outputs` are correct. + + The check between numerical and analytical gradients uses :func:`~torch.allclose`. + + .. note:: + The default values are designed for :attr:`input` and + :attr:`grad_outputs` of double precision. This check will likely fail if + they are of less precision, e.g., ``FloatTensor``. + + .. warning:: + If any checked tensor in :attr:`input` and :attr:`grad_outputs` has + overlapping memory, i.e., different indices pointing to the same memory + address (e.g., from :func:`torch.Tensor.expand`), this check will likely fail + because the numerical gradients computed by point perturbation at such + indices will change values at all other indices that share the same + memory address. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor or a tuple of Tensors + inputs (tuple of Tensor or Tensor): inputs to the function + grad_outputs (tuple of [Tensor or None] or Tensor, optional): The gradients with + respect to the function's outputs. + eps (float, optional): perturbation for finite differences + atol (float, optional): absolute tolerance + rtol (float, optional): relative tolerance + gen_non_contig_grad_outputs (bool, optional): if :attr:`grad_outputs` is + ``None`` and :attr:`gen_non_contig_grad_outputs` is ``True``, the + randomly generated gradient outputs are made to be noncontiguous + raise_exception (bool, optional): indicating whether to raise an exception if + the check fails. The exception gives more information about the + exact nature of the failure. This is helpful when debugging gradchecks. + nondet_tol (float, optional): tolerance for non-determinism. When running + identical inputs through the differentiation, the results must either match + exactly (default, 0.0) or be within this tolerance. Note that a small amount + of nondeterminism in the gradient will lead to larger inaccuracies in + the second derivative. + check_undefined_grad (bool, optional): if True, check if undefined output grads + are supported and treated as zeros + check_batched_grad (bool, optional): if True, check if we can compute + batched gradients using prototype vmap support. Defaults to False. + fast_mode (bool, optional): if True, run a faster implementation of gradgradcheck that + no longer computes the entire jacobian. + masked (bool, optional): if True, the gradients of unspecified elements of + sparse tensors are ignored (default, False). + Returns: + True if all differences satisfy allclose condition + """ + if not (check_fwd_over_rev or check_rev_over_rev): + raise AssertionError( + "Expected at least one of check_fwd_over_rev or check_rev_over_rev to be True" + ) + if check_undefined_grad and not check_rev_over_rev: + raise AssertionError( + "Setting check_undefined_grad=True requires check_rev_over_rev to be True" + ) + if check_batched_grad and not check_rev_over_rev: + raise AssertionError( + "Setting check_batched_grad=True requires check_rev_over_rev to be True" + ) + # TODO: do we want to test this too? + # assert not (check_batched_forward_grad and not check_fwd_over_rev), ( + # "Setting check_batched_forward_grad=True requires check_fwd_over_rev to be True") + tupled_inputs = _as_tuple(inputs) + + if grad_outputs is None: + # If grad_outputs is not specified, create random Tensors of the same shape, type, and device as the outputs + + outputs = _differentiable_outputs(func(*tupled_inputs)) + tupled_grad_outputs = tuple( + torch.testing.make_tensor( + x.shape, + dtype=x.dtype + if x.is_floating_point() or x.is_complex() + else torch.double, + device=x.device, + low=-1, + high=1, + requires_grad=True, + noncontiguous=gen_non_contig_grad_outputs, + ) + for x in outputs + ) + else: + tupled_grad_outputs = _as_tuple(grad_outputs) + + num_outputs = len(tupled_grad_outputs) + + # NB: We need to save the requires_grad information about the inputs here because gradcheck detaches inputs + # before running forward mode AD + diff_input_args_indices = { + i for i, x in enumerate(tupled_inputs) if is_tensor_like(x) and x.requires_grad + } + diff_grad_output_indices = { + i for i, x in enumerate(tupled_grad_outputs) if x.requires_grad + } + + def new_func(*args): + # Restore the requires_grad information + input_args = tuple( + x.requires_grad_() if i in diff_input_args_indices else x + for i, x in enumerate(args[:-num_outputs]) + ) + outputs = _differentiable_outputs(func(*input_args)) + grad_outputs = tuple( + x.requires_grad_() if i in diff_grad_output_indices else x + for i, x in enumerate(args[-num_outputs:]) + ) + diff_input_args = tuple( + x for i, x in enumerate(input_args) if i in diff_input_args_indices + ) + grad_inputs = torch.autograd.grad( + outputs, diff_input_args, grad_outputs, create_graph=True, allow_unused=True + ) + grad_inputs = tuple(g for g in grad_inputs if g is not None) + return grad_inputs + + return gradcheck( + new_func, + tupled_inputs + tupled_grad_outputs, + eps=eps, + atol=atol, + rtol=rtol, + raise_exception=raise_exception, + nondet_tol=nondet_tol, + check_undefined_grad=check_undefined_grad, + check_grad_dtypes=check_grad_dtypes, + check_batched_grad=check_batched_grad, + fast_mode=fast_mode, + check_forward_ad=check_fwd_over_rev, + check_backward_ad=check_rev_over_rev, + masked=masked, + ) diff --git a/western_ai/usa/pytorch/torch/autograd/graph.py b/western_ai/usa/pytorch/torch/autograd/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..2dc8d130b8796c0b625d49549504e072be57bce9 --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/graph.py @@ -0,0 +1,989 @@ +import abc +import contextlib +import contextvars +import functools +import logging +import threading +from collections import defaultdict, deque +from collections.abc import ( + Callable, + Generator, + Iterable, + Iterator, + MutableMapping, + Sequence, +) +from typing import ( + Any, + cast, + Literal, + NamedTuple, + Optional, + TYPE_CHECKING, + TypeAlias, + Union, +) +from weakref import WeakKeyDictionary, WeakValueDictionary + +import torch +from torch.autograd.variable import Variable +from torch.utils._python_dispatch import TorchDispatchMode +from torch.utils.hooks import RemovableHandle + + +if TYPE_CHECKING: + from torch._ops import OpOverload + + +__all__ = [ + "saved_tensors_hooks", + "save_on_cpu", + "disable_saved_tensors_hooks", + "register_multi_grad_hook", + "allow_mutation_on_saved_tensors", + "Node", + "GradientEdge", + "get_gradient_edge", + "increment_version", + "region_activation_memory_budget", + "set_warn_on_accumulate_grad_stream_mismatch", + "set_override_stale_capture_stream", +] + + +log = logging.getLogger(__name__) + + +class Node(abc.ABC): + @abc.abstractmethod + def name(self) -> str: + r"""Return the name. + + Example:: + + >>> import torch + >>> a = torch.tensor([0., 0., 0.], requires_grad=True) + >>> b = a.clone() + >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) + >>> print(b.grad_fn.name()) + CloneBackward0 + """ + raise NotImplementedError + + @property + @abc.abstractmethod + def next_functions(self) -> tuple[tuple[Optional["Node"], int], ...]: + raise NotImplementedError + + @abc.abstractmethod + def metadata(self) -> dict: + r"""Return the metadata.""" + raise NotImplementedError + + @abc.abstractmethod + def _sequence_nr(self) -> int: + raise NotImplementedError + + @property + @abc.abstractmethod + def _input_metadata(self) -> list[Any]: + raise NotImplementedError + + @abc.abstractmethod + def _register_hook_dict(self, tensor: torch.Tensor) -> None: + raise NotImplementedError + + @abc.abstractmethod + def register_hook(self, fn: Callable[..., Any]) -> RemovableHandle: + r"""Register a backward hook. + + The hook will be called every time a gradient with respect to the + Node is computed. The hook should have the following signature:: + + hook(grad_inputs: Tuple[Tensor], grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None + + + The hook should not modify its argument, but it can optionally return + a new gradient which will be used in place of :attr:`grad_inputs`. + + This function returns a handle with a method ``handle.remove()`` + that removes the hook from the module. + + .. note:: + See :ref:`backward-hooks-execution` for more information on how when this hook + is executed, and how its execution is ordered relative to other hooks. + + .. note:: + In the rare case where the hook is registered while the Node has already + begun execution, there is no longer any guarantee on :attr:`grad_outputs` + content (it might be as usual or empty depending on other factors). The + hook can still optionally return a new gradient to be used in place of + :attr:`grad_inputs` independent of :attr:`grad_outputs`. + + Example:: + + >>> import torch + >>> a = torch.tensor([0., 0., 0.], requires_grad=True) + >>> b = a.clone() + >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) + >>> handle = b.grad_fn.register_hook(lambda gI, gO: (gO[0] * 2,)) + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([2., 2., 2.]) + >>> handle.remove() # Removes the hook + >>> a.grad = None + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([1., 1., 1.]) + """ + raise NotImplementedError + + @abc.abstractmethod + def register_prehook(self, fn: Callable[..., Any]) -> RemovableHandle: + r"""Register a backward pre-hook. + + The hook will be called every time a gradient with respect to the + Node is computed. The hook should have the following signature:: + + hook(grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None + + The hook should not modify its argument, but it can optionally return + a new gradient which will be used in place of :attr:`grad_outputs`. + + This function returns a handle with a method ``handle.remove()`` + that removes the hook from the module. + + .. note:: + See :ref:`backward-hooks-execution` for more information on how when this hook + is executed, and how its execution is ordered relative to other hooks. + + Example:: + + >>> a = torch.tensor([0., 0., 0.], requires_grad=True) + >>> b = a.clone() + >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) + >>> handle = b.grad_fn.register_prehook(lambda gI: (gI[0] * 2,)) + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([2., 2., 2.]) + >>> handle.remove() + >>> a.grad = None + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([1., 1., 1.]) + """ + raise NotImplementedError + + @classmethod + def __subclasshook__(cls, subclass: type) -> bool: + if cls is Node and ( + ( + subclass is not None + and subclass is getattr(torch._C._functions, subclass.__name__, None) + ) + or issubclass(subclass, torch.autograd.function.BackwardCFunction) + ): + return True + return NotImplemented + + +def _get_grad_fn_or_grad_acc(t: Union[torch.Tensor, "GradientEdge"]) -> Node: + if isinstance(t, GradientEdge): + return t.node + if t.requires_grad and t.grad_fn is None: + with torch.enable_grad(): + node = t.view_as(t).grad_fn.next_functions[0][0] # type: ignore[union-attr] + else: + node = t.grad_fn + if node is None: + raise AssertionError("Expected gradient function to be set") + return node + + +class GradientEdge(NamedTuple): + """Object representing a given gradient edge within the autograd graph. + + To get the gradient edge where a given Tensor gradient will be computed, + you can do ``edge = autograd.graph.get_gradient_edge(tensor)``. + """ + + node: Node + output_nr: int + # This token can be used to ensure the graph stays alive when it cannot be + # done via the node field + ownership_token: Node | None = None + + +def get_gradient_edge(tensor: torch.Tensor) -> GradientEdge: + """Get the gradient edge for computing the gradient of the given Tensor. + + In particular, it is equivalent to call + ``g = autograd.grad(loss, input)`` and ``g = autograd.grad(loss, get_gradient_edge(input))``. + """ + if not tensor.requires_grad: + raise RuntimeError( + "It is not possible to get the gradient edge for a Tensor " + "that does not require gradients", + ) + grad_fn = _get_grad_fn_or_grad_acc(tensor) + + # Python-based Node are owned by the C++ side meaning the python grad_fn + # object we hold here does NOT keep the C++ graph alive. + # Create an ownership token by creating a new C++ node that own the graph + # we care about here. + token = None + if isinstance(grad_fn, torch._C._FunctionBase): + with torch.enable_grad(): + token = tensor.view_as(tensor).grad_fn + + # Note that output_nr default to 0 which is the right value + # for the AccumulateGrad node. + # pyrefly: ignore [bad-argument-type] + return GradientEdge(grad_fn, tensor.output_nr, ownership_token=token) + + +def increment_version(tensor: torch.Tensor | Iterable[torch.Tensor]) -> None: + """Update autograd metadata tracking whether the given Tensor was modified in place. + + This is to enable more accurate error checking within the autograd engine. + It is already done automatically by PyTorch functions and within custom Function + when mark_dirty() is called appropriately so you only need to call this explicitly + if you are doing inplace operation on the Tensor data in a way that Pytorch doesn't + know about. For example a custom kernel that reads the Tensor data_ptr and modifies + the memory inplace based on this pointer. Can accept either a tensor, or a list of tensors. + + Note that incrementing the version counter multiple times for a single inplace operation + is not problematic. + + Note that if you pass in tensor constructed under torch.inference_mode(), + we will not bump its version counter (because your tensor does not have one). + """ + if isinstance(tensor, torch.Tensor): + tensor = (tensor,) + torch._C._increment_version(tensor) + + +class saved_tensors_hooks: + """Context-manager that sets a pair of pack / unpack hooks for saved tensors. + + Use this context-manager to define how intermediary results of an operation + should be packed before saving, and unpacked on retrieval. + + In that context, the ``pack_hook`` function will be called every time an + operation saves a tensor for backward (this includes intermediary results + saved using + :func:`~torch.autograd.function._ContextMethodMixin.save_for_backward` but + also those recorded by a PyTorch-defined operation). The output of + ``pack_hook`` is then stored in the computation graph instead of the + original tensor. + + The ``unpack_hook`` is called when the saved tensor needs to be accessed, + namely when executing :func:`torch.Tensor.backward()` or + :func:`torch.autograd.grad()`. It takes as argument the *packed* object + returned by ``pack_hook`` and should return a tensor which has the same + content as the original tensor (passed as input to the corresponding + ``pack_hook``). + + The hooks should have the following signatures: + + pack_hook(tensor: Tensor) -> Any + + unpack_hook(Any) -> Tensor + + where the return value of ``pack_hook`` is a valid input to ``unpack_hook``. + + In general, you want ``unpack_hook(pack_hook(t))`` to be equal to ``t`` in terms + of value, size, dtype and device. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pack_hook(x): + ... print("Packing", x) + ... return x.detach() + >>> + >>> def unpack_hook(x): + ... print("Unpacking", x) + ... return x + >>> + >>> a = torch.ones(5, requires_grad=True) + >>> b = torch.ones(5, requires_grad=True) * 2 + >>> with torch.autograd.graph.saved_tensors_hooks(pack_hook, unpack_hook): + ... y = a * b + Packing tensor([1., 1., 1., 1., 1.], requires_grad=True) + Packing tensor([2., 2., 2., 2., 2.], grad_fn=) + >>> y.sum().backward() + Unpacking tensor([1., 1., 1., 1., 1.], requires_grad=True) + Unpacking tensor([2., 2., 2., 2., 2.], grad_fn=) + + .. warning :: + Performing an inplace operation on the input to either hooks may lead + to undefined behavior. + + .. warning :: + Only one pair of hooks is allowed at a time. When recursively nesting this + context-manager, only the inner-most pair of hooks will be applied. + + .. warning :: + To avoid reference cycle, the return value of ``pack_hook`` cannot hold a + reference to the input tensor. For example, use `lambda x: x.detach()` + instead of `lambda x: x` as the pack hook. + """ + + def __init__( + self, + pack_hook: Callable[[torch.Tensor], Any], + unpack_hook: Callable[[Any], torch.Tensor], + ) -> None: + self.pack_hook = pack_hook + self.unpack_hook = unpack_hook + + def __enter__(self) -> None: + torch._C._autograd._push_saved_tensors_default_hooks( + self.pack_hook, self.unpack_hook + ) + + def __exit__(self, *args: object) -> None: + torch._C._autograd._pop_saved_tensors_default_hooks() + + +class save_on_cpu(saved_tensors_hooks): + """Context manager under which tensors saved by the forward pass will be stored on cpu, then retrieved for backward. + + When performing operations within this context manager, intermediary + results saved in the graph during the forward pass will be moved to CPU, + then copied back to the original device when needed for the backward pass. + If the graph was already on CPU, no tensor copy is performed. + + Use this context-manager to trade compute for GPU memory usage (e.g. + when your model doesn't fit in GPU memory during training). + + .. warning:: + + When ``pin_memory=True``, the GPU to CPU copy during packing is + asynchronous. Accessing saved tensors on CPU (e.g. via + ``grad_fn._saved_self``) before the CUDA stream has finished may + yield incorrect data. Call :func:`torch.cuda.synchronize` first + if you need to read them. + + Args: + pin_memory (bool): If ``True`` tensors will be saved to CPU pinned memory + during packing and copied to GPU asynchronously during both + packing and unpacking. + Defaults to ``False``. + Also see :ref:`cuda-memory-pinning`. + + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> a = torch.randn(5, requires_grad=True, device="cuda") + >>> b = torch.randn(5, requires_grad=True, device="cuda") + >>> c = torch.randn(5, requires_grad=True, device="cuda") + >>> + >>> def f(a, b, c): + ... prod_1 = a * b # a and b are saved on GPU + ... with torch.autograd.graph.save_on_cpu(): + ... prod_2 = prod_1 * c # prod_1 and c are saved on CPU + ... y = prod_2 * a # prod_2 and a are saved on GPU + ... return y + >>> + >>> y = f(a, b, c) + >>> del a, b, c # for illustration only + >>> # the content of a, b, and prod_2 are still alive on GPU + >>> # the content of prod_1 and c only live on CPU + >>> y.sum().backward() # all CPU tensors are moved back to GPU, for backward + >>> # all intermediary tensors are released (deleted) after the call to backward + """ + + def __init__(self, pin_memory: bool = False, device_type: str = "cuda") -> None: + device_module = getattr(torch, device_type, torch.cuda) + + def pack_to_cpu(tensor: torch.Tensor) -> tuple[torch.device, torch.Tensor]: + if not pin_memory: + return (tensor.device, tensor.cpu()) + is_pinnable = device_module.is_available() and not tensor.is_sparse + packed = torch.empty( + tensor.size(), + dtype=tensor.dtype, + layout=tensor.layout, + pin_memory=is_pinnable, + ) + packed.copy_(tensor, non_blocking=is_pinnable) + return (tensor.device, packed) + + def unpack_from_cpu(packed: tuple[torch.device, torch.Tensor]) -> torch.Tensor: + device, tensor = packed + return tensor.to(device, non_blocking=pin_memory) + + super().__init__(pack_to_cpu, unpack_from_cpu) + + +@contextlib.contextmanager +def disable_saved_tensors_hooks(error_message: str) -> Generator[None, None, None]: + """Context-manager that disables the saved tensors default hooks feature. + + Useful for if you are creating a feature that does not work with saved + tensors default hooks. + + Args: + error_message (str): When saved tensors default hooks are used when they + have been disabled, a RuntimeError with this + error message gets raised. + + Example:: + + >>> # xdoctest: +SKIP(failing) + >>> message = "saved tensors default hooks are disabled" + >>> with torch.autograd.graph.disable_saved_tensors_hooks(message): + ... # Raises RuntimeError: saved tensors default hooks are disabled + ... with torch.autograd.graph.save_on_cpu(): + ... pass + """ + maybe_prev_message = None + try: + maybe_prev_message = ( + torch._C._autograd._saved_tensors_hooks_get_disabled_error_message() + ) + torch._C._autograd._saved_tensors_hooks_disable(error_message) + yield + finally: + # See NOTE: [disabled_error_message invariant] + if maybe_prev_message is None: + torch._C._autograd._saved_tensors_hooks_enable() + else: + torch._C._autograd._saved_tensors_hooks_disable(maybe_prev_message) + + +def region_activation_memory_budget( + budget: float, +) -> contextlib.AbstractContextManager[None]: + r"""Context-manager that sets the activation memory budget for the region of + a compiled forward traced under it. + + .. warning:: + This is a prototype feature and is subject to change. + + Under :func:`torch.compile`, the min-cut partitioner chooses which + activations to save versus recompute in the backward pass to stay under a + memory budget. ``budget`` is a ratio in ``[0, 1]``: ``0.0`` corresponds to + the activation memory from applying activation checkpointing to the full + region, and ``1.0`` corresponds to the activation memory from the default + runtime-optimized strategy. So ``0.4`` would result in a strategy that saves + 40% of the activations compared to the default strategy. It solves a 0-1 + knapsack to find the minimum recompute necessary to stay below the budget. + This overrides the global ``torch._functorch.config.activation_memory_budget`` + for the annotated region. + + .. note:: + Today the partitioner only supports a single budget per compiled graph, + so the annotation must cover every forward op in the graph (a partial + annotation is rejected rather than silently applied graph-wide), and all + annotated nodes must agree on the budget. To use different budgets for + different parts of a model, separate them with a graph break (e.g. + ``torch._dynamo.graph_break()``) so each part becomes its own graph. + + This only has an effect under :func:`torch.compile`; using it outside of a + compiled region raises a ``RuntimeError``. + + Args: + budget (float): Activation memory budget ratio in ``[0, 1]``. + + Example:: + + >>> # xdoctest: +SKIP + >>> with torch.autograd.graph.region_activation_memory_budget(0.0): + ... x = layer(x) # recompute this region's activations in backward + """ + import torch.fx.traceback as fx_traceback + + if isinstance(budget, bool) or not isinstance(budget, (int, float)): + raise TypeError( + f"torch.autograd.graph.region_activation_memory_budget: expects a " + f"float, got {type(budget).__name__}" + ) + if not 0.0 <= budget <= 1.0: + raise ValueError( + f"torch.autograd.graph.region_activation_memory_budget: must be in " + f"[0, 1], got {budget}" + ) + # The budget only takes effect when read back by the partitioner during + # compilation. Dynamo folds torch.compiler.is_compiling() to True while + # tracing this (inlined) call, so this guard only fires in eager mode. + if not torch.compiler.is_compiling(): + raise RuntimeError( + "torch.autograd.graph.region_activation_memory_budget can only be " + "used inside a torch.compile region; it has no effect in eager mode." + ) + return fx_traceback.annotate( + {fx_traceback.MEMORY_BUDGET_ANNOTATION_KEY: float(budget)} + ) + + +def set_warn_on_accumulate_grad_stream_mismatch(enabled: bool) -> None: + """Whether to warn when the AccumulateGrad node's stream does not match the stream + of the node that produced the incoming gradient. + """ + return torch._C._set_warn_on_accumulate_grad_stream_mismatch(enabled) + + +def set_override_stale_capture_stream(enabled: bool) -> None: + """Control behavior when autograd detects a stale non-capturing stream during + CUDA graph capture. + + During CUDA graph capture, autograd nodes may reference a stale stream + that is not part of the capture. With the flag disabled (the + process-initial state), autograd raises a ``RuntimeError`` when the stale + stream is the default stream (stream 0), because this case always + invalidates the capture: ``cudaStreamWaitEvent`` on the default stream + pulls a non-capturing stream into the graph. For non-default stale streams + the stream reference is left unchanged; the capture will succeed if the + user has joined the stream into the capture (e.g. via + ``capture_stream.wait_stream(stale_stream)``) and will otherwise fail with + a CUDA runtime error. + + When ``enabled=True``, any stale non-capturing stream (default or + non-default) is automatically overridden with the producer's capturing + stream, allowing the capture to proceed. This is a process-global setting + and is not thread-local. + + Args: + enabled (bool): If ``True``, override stale non-capturing streams with + the producer's capturing stream during CUDA graph capture. If + ``False`` (the process-initial state), raise an error only when the + stale stream is the default stream (stream 0); other stale streams + are left unchanged. + """ + return torch._C._set_override_stale_capture_stream(enabled) + + +class _MultiHandle(RemovableHandle): + handles: tuple[RemovableHandle, ...] + + def __init__(self, handles: tuple[RemovableHandle, ...]) -> None: + self.handles = handles + + def remove(self) -> None: + for handle in self.handles: + handle.remove() + + def __getstate__(self) -> tuple[RemovableHandle, ...]: + return self.handles + + def __setstate__(self, state: tuple[RemovableHandle, ...]) -> None: + self.handles = state + + +def register_multi_grad_hook( + tensors: Sequence[torch.Tensor], + fn: Callable[[Sequence[torch.Tensor | None]], None] + | Callable[[torch.Tensor], None], + *, + mode: Literal["all", "any"] = "all", +) -> RemovableHandle: + r"""Register a multi-grad backward hook. + + There are two supported modes: ``"all"`` and ``"any"``. + + Under the ``"all"`` mode, the hook will be called after gradients with respect to every tensor in + :attr:`tensors` have been computed. If a tensor is in :attr:`tensors` but + is not part of the graph, or if a tensor is not needed to compute the gradients + for any ``inputs`` specified for the current ``.backward()`` or ``.grad()`` call, + this tensor will be ignored and the hook will not wait for its gradient to be + computed. + + After every non-ignored tensor's gradient has been computed, :attr:`fn` will be + called with those gradients. ``None`` will be passed for tensors that did not + have their gradients computed. + + Under the ``"any"`` mode, the hook will be called after the first gradient + with respect to a tensor in :attr:`tensors` has been computed. The hook + will be called with that gradient as its argument. + + The hook should not modify its arguments. + + This function returns a handle with a method ``handle.remove()`` that removes the hook. + + .. note:: + See :ref:`backward-hooks-execution` for more information on how when this hook + is executed, and how its execution is ordered relative to other hooks. + + Example:: + + >>> import torch + >>> + >>> a = torch.rand(2, 3, requires_grad=True) + >>> b = torch.rand(2, 3, requires_grad=True) + >>> c = a * b + >>> d = a * b + >>> + >>> def fn(grads): + ... print([g is not None for g in grads]) + ... + >>> torch.autograd.graph.register_multi_grad_hook((a, b, c, d), fn) + >>> + >>> c.sum().backward(retain_graph=True) + [True, True, True, False] + >>> c.sum().backward(inputs=(a,), retain_graph=True) + [True, False, True, False] + >>> + """ + supported_modes = ("all", "any") + lock = threading.Lock() + + if mode not in supported_modes: + raise ValueError(f"Expects mode to be one of {supported_modes} but got {mode}") + + if mode == "all": + count: dict[int, int] = {} + nb_calls = None + buffer: dict[int, list[torch.Tensor | None]] = {} + + grad_fns = list(map(_get_grad_fn_or_grad_acc, tensors)) + len_tensors = len(tensors) + + def get_inner_hook(idx: int) -> Callable[[torch.Tensor], None]: + def inner_hook(grad: torch.Tensor) -> None: + nonlocal count, nb_calls, buffer, fn + id = torch._C._current_graph_task_id() + if id == -1: + raise AssertionError( + "expected this hook to be called inside a backward call" + ) + count[id] = count.get(id, 0) + # pyrefly: ignore [unsupported-operation] + buffer[id] = buffer.get(id, [None] * len_tensors) + + with lock: + curr_count, count[id] = count[id], count[id] + 1 + + if curr_count == 0: + # On the first call, compute the actual nb_calls and buffer + nb_calls = sum( + map(torch._C._will_engine_execute_node, grad_fns) + ) + + buffer[id][idx] = grad + + if nb_calls is None: + raise AssertionError("Expected nb_calls to be set") + if curr_count == nb_calls - 1: + fn = cast(Callable[[Sequence[torch.Tensor | None]], None], fn) + fn(buffer[id]) + del count[id] + del buffer[id] + + return inner_hook + + handles = tuple( + t.register_hook(get_inner_hook(i)) for i, t in enumerate(tensors) + ) + elif mode == "any": + fn = cast(Callable[[torch.Tensor], None], fn) + ran_hook: dict[int, bool] = defaultdict(bool) + + @functools.wraps(fn) + def wrapped_fn(grad: torch.Tensor) -> None: + nonlocal ran_hook + id = torch._C._current_graph_task_id() + if id == -1: + raise AssertionError( + "expected this hook to be called inside a backward call" + ) + with lock: + prev, ran_hook[id] = ran_hook[id], True + if prev: + return + fn(grad) + + handles = tuple( + tensor.register_hook(wrapped_fn) + for tensor in tensors + if tensor.requires_grad + ) + + return _MultiHandle(handles) # type: ignore[possibly-undefined] + + +# NOTE [Allow mutation on tensors saved for backward] +# +# 1. Tensor gets saved for backward +# - remember the python object id and the version of the tensor +# - remember aliasing information (data_ptr of base + version) +# - save the original so we control its lifetime +# 2. Any time a tensor gets in-placed +# - for each tensor aliased to it: +# - check using its object id and version to see if it has been saved +# - if it has been saved, clone it +# - delete the reference to the original +# 3. during backward +# - if the clone exists, the tensor must've been modified in-place +_allow_mutation_on_saved_tensors_enabled: bool = False + + +_TID: TypeAlias = tuple[int, int, int] +_SID: TypeAlias = tuple[int, int] + + +def _get_tid(tensor: torch.Tensor) -> _TID: + # FIXME: This is almost definitely a bug. + if isinstance( + tensor, + ( + torch._subclasses.fake_tensor.FakeTensor, + torch._subclasses.functional_tensor.FunctionalTensor, + ), + ): + data_ptr = 0 + else: + data_ptr = tensor.data_ptr() + return (id(tensor), data_ptr, tensor._version) + + +def _get_sid(tensor: torch.Tensor) -> _SID: + # FIXME: This is almost definitely a bug. + if isinstance( + tensor, + ( + torch._subclasses.fake_tensor.FakeTensor, + torch._subclasses.functional_tensor.FunctionalTensor, + ), + ): + data_ptr = 0 + else: + data_ptr = tensor.data_ptr() + return (data_ptr, tensor._version) + + +class _Handle: + pass + + +class _swap_with_cloned(saved_tensors_hooks): + def __init__(self, ctx: "_AllowMutationOnSavedContext") -> None: + def pack_hook(tensor: torch.Tensor) -> _Handle: + tid = _get_tid(tensor) + sid = _get_sid(tensor) + # Tensors saved for backward have an entry in _tid_to_weakhandle + handle: _Handle | None = None + + # Save aliasing information + ctx.sid_to_tid[sid].add(tid) + + # NB: The same tensor (of the same version) can be saved multiple times + if tid not in ctx.tid_to_weakhandle: + handle = _Handle() + ctx.tid_to_weakhandle[tid] = handle + ctx.original[handle] = tensor + else: + # Store an additional strong reference to the handle + handle = ctx.tid_to_weakhandle[tid] + return handle + + def unpack_hook(handle: _Handle) -> torch.Tensor: + error_msg = ( + "Trying to backward outside of the 'allow_mutation_on_saved_tensors' context" + "in which the graph was originally recorded." + ) + if not _allow_mutation_on_saved_tensors_enabled: + raise AssertionError(error_msg) + if handle in ctx.cloned: + res = ctx.cloned[handle] + else: + if handle not in ctx.original: + raise AssertionError(error_msg) + res = ctx.original[handle] + return res + + super().__init__(pack_hook, unpack_hook) + + +class _CloneArgBeforeMutateMode(TorchDispatchMode): + def __init__(self, ctx: "_AllowMutationOnSavedContext") -> None: + self.ctx = ctx + + def __torch_dispatch__( + self, + func: "OpOverload", + types: Iterable[type], + args: tuple[Any, ...] = (), + kwargs: dict[Any, Any] | None = None, + ) -> Any: + kwargs = kwargs or {} + + def maybe_clone(t: torch.Tensor) -> None: + tid = _get_tid(t) + sid = _get_sid(t) + ctx = self.ctx + if sid in ctx.sid_to_tid: + for tid in ctx.sid_to_tid[sid]: + if tid not in ctx.tid_to_weakhandle: + # We know that if tid is in sid_to_tid, then it must also be in + # tid_to_weakhandle. However, it is possible for the tensor to be + # saved at one point, but cleared by backward before it is modified + # in-place. Consider the following example: + # + # >>> a = torch.randn(2, 3, requires_grad=True).clone() + # >>> out = (a**2).sum() + # >>> out.backward() + # >>> a.sin_() + continue + handle = ctx.tid_to_weakhandle[tid] + if handle in ctx.cloned: + # The same exact tensor has been cloned already + continue + ctx.cloned[handle] = ctx.original[handle].clone() + del ctx.original[handle] + + for idx, arg in enumerate(func._schema.arguments): + if arg.alias_info is not None and arg.alias_info.is_write: + if arg.is_out: + maybe_clone(kwargs["out"]) + elif isinstance(args[idx], list): + # Foreach case. (Possible optimization: if most of the + # tensors need to be cloned, use a for each clone?) + for t in args[idx]: + maybe_clone(t) + else: + maybe_clone(args[idx]) + + return func(*args, **kwargs) + + +class _AllowMutationOnSavedContext: + def __init__(self) -> None: + self.cloned: MutableMapping[_Handle, torch.Tensor] = WeakKeyDictionary() + self.original: MutableMapping[_Handle, torch.Tensor] = WeakKeyDictionary() + self.tid_to_weakhandle: MutableMapping[_TID, _Handle] = WeakValueDictionary() + self.sid_to_tid: dict[_SID, set[_TID]] = defaultdict(set) + + def clear(self) -> None: + self.cloned.clear() + self.original.clear() + self.tid_to_weakhandle.clear() + self.sid_to_tid.clear() + + +@contextlib.contextmanager +def allow_mutation_on_saved_tensors() -> Generator[ + _AllowMutationOnSavedContext, None, None +]: + """Context manager under which mutating tensors saved for backward is allowed. + + Under this context manager, tensors saved for backward are cloned on mutation, + so the original version can still be used during backward. Normally, mutating a tensor + saved for backward will result in an error raised when it's used during backward. + + To ensure the correct behavior, both the forward and backward should be run under + the same context manager. + + Returns: + An _AllowMutationOnSavedContext object storing the state managed by this + context manager. This object can be useful for debugging purposes. The state + managed by the context manager is automatically cleared upon exiting. + + Example:: + + >>> import torch + >>> with torch.autograd.graph.allow_mutation_on_saved_tensors(): + ... # forward + ... a = torch.ones(2, 3, requires_grad=True) + ... b = a.clone() + ... out = (b**2).sum() + ... b.sin_() + ... # backward + ... out.sum().backward() + ... + tensor([[0.8415, 0.8415, 0.8415], + [0.8415, 0.8415, 0.8415]], grad_fn=) + """ + global _allow_mutation_on_saved_tensors_enabled + + ctx = _AllowMutationOnSavedContext() + + with _swap_with_cloned(ctx), _CloneArgBeforeMutateMode(ctx): + try: + if _allow_mutation_on_saved_tensors_enabled: + raise RuntimeError( + "allow_mutation_on_saved_tensors contexts cannot be nested" + ) + _allow_mutation_on_saved_tensors_enabled = True + yield ctx + finally: + ctx.clear() + _allow_mutation_on_saved_tensors_enabled = False + + +def _register_logging_hooks_on_whole_graph( + t_outputs: Sequence[torch.Tensor | GradientEdge], +) -> Callable[[], None]: + grad_fns = list(map(_get_grad_fn_or_grad_acc, t_outputs)) + + def iter_graph(roots: list[Node]) -> Iterator[Node]: + if not roots: + return + seen: set[Node] = set() + q: deque[Node] = deque() + for node in roots: + if node is not None: + seen.add(node) + q.append(node) + + while q: + node = q.popleft() + for fn, _ in node.next_functions: + if fn in seen or fn is None: + continue + seen.add(fn) + q.append(fn) + + yield node + + def fmt(t: torch.Tensor | None) -> str: + # Avoid circular import + from torch.utils._dtype_abbrs import dtype_abbrs + + if t is None: + return "None" + return f"{dtype_abbrs[t.dtype]}[{', '.join(map(str, t.shape))}]" + + def prehook(grad_outputs: Sequence[torch.Tensor | None]) -> None: + node = torch._C._current_autograd_node() + grad_outputs_str = f"[{','.join(fmt(t) for t in grad_outputs)}]" + log_str = f"Executing: {node} with grad_outputs: {grad_outputs_str}" + log.debug(log_str) + + handles = [node.register_prehook(prehook) for node in iter_graph(grad_fns)] + + def unregister_hooks() -> None: + for handle in handles: + handle.remove() + + return unregister_hooks + + +def _engine_run_backward( + t_outputs: Sequence[torch.Tensor | GradientEdge], + *args: Any, + **kwargs: Any, +) -> tuple[torch.Tensor, ...]: + attach_logging_hooks = log.getEffectiveLevel() <= logging.DEBUG + if attach_logging_hooks: + unregister_hooks = _register_logging_hooks_on_whole_graph(t_outputs) + + # Need to save the context so compiler config will be visible in device threads + torch._C._stash_obj_in_tls("context", contextvars.copy_context()) + + try: + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass + t_outputs, *args, **kwargs + ) # Calls into the C++ engine to run the backward pass + finally: + if attach_logging_hooks: + unregister_hooks() # type: ignore[possibly-undefined] + # Erase rather than overwrite-with-None so the thread_local map is + # truly empty. SafePyObject's destructor needs the GIL; if a thread + # exits while a SafePyObject is still in its thread_local, + # __call_tls_dtors fires the destructor → take_gil → deadlock. + torch._C._remove_obj_from_tls("context") diff --git a/western_ai/usa/pytorch/torch/autograd/profiler.py b/western_ai/usa/pytorch/torch/autograd/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..3f3c9ec063d7a98d068c74868e7df6494386fc98 --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/profiler.py @@ -0,0 +1,1799 @@ +# mypy: allow-untyped-defs +import ast +import copy +import json +import logging +import re +import uuid +from bisect import bisect_left +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +from time import perf_counter_ns +from typing import Any, Optional +from warnings import warn + + +log = logging.getLogger(__name__) + +import torch +import torch.cuda +from torch._C import _get_privateuse1_backend_name +from torch._C._profiler import _ExperimentalConfig +from torch.autograd import ( + _disable_profiler, + _enable_profiler, + _kineto_step, + _prepare_profiler, + _ProfilerResult, + _supported_activities, + _toggle_collection_dynamic, + DeviceType, + kineto_available, + ProfilerActivity, + ProfilerConfig, + ProfilerState, +) +from torch.autograd.profiler_util import ( + _filter_name, + _filter_stack_entry, + _rewrite_name, + EventList, + FunctionEvent, + MEMORY_EVENT_NAME, + MemRecordsAcc, + OUT_OF_MEMORY_EVENT_NAME, +) +from torch.futures import Future + + +__all__ = [ + "profile", + "record_function", + "emit_itt", + "emit_nvtx", + "load_nvprof", + "EnforceUnique", + "parse_nvprof_trace", + "KinetoStepTracker", + "EventList", + "FunctionEvent", + "MemRecordsAcc", +] + +try: + # Available in Python >= 3.2 + from contextlib import ContextDecorator as _ContextDecorator +except ImportError: + import functools + + class _ContextDecorator: # type: ignore[no-redef] + def __enter__(self): + raise NotImplementedError + + def __exit__(self, exc_type, exc_val, exc_tb): + raise NotImplementedError + + def __call__(self, func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + with self: + return func(*args, **kwargs) + + return wrapped + + +# global python state - whether profiler is currently enabled +# useful for fast python checks to reduce latency +_is_profiler_enabled: bool = False + +# https://github.com/pytorch/kineto/blob/a054a4be0db117c579a21747debf19c863631f26/libkineto/src/output_json.cpp#L559 +# Kernel and CUDA runtime API events are connected by ac2g; forward and backward +# ATen ops are connected by fwdbwd. +_profile_flow_types = {"fwdbwd", "ac2g"} + + +def _set_is_profiler_enabled(enable: bool): + global _is_profiler_enabled + _is_profiler_enabled = enable + + +def _run_on_profiler_start(): + _set_is_profiler_enabled(True) + + +def _run_on_profiler_stop(): + _set_is_profiler_enabled(False) + + +@dataclass +class _ProfilerStats: + "Profiler timing and stats used by developers to catch issues/regressions" + + profiling_window_duration_sec: float = 0 + number_of_events: int = 0 + profiler_prepare_call_duration_us: int = 0 + profiler_enable_call_duration_us: int = 0 + profiler_disable_call_duration_us: int = 0 + parse_kineto_call_duration_us: int = 0 + function_events_build_tree_call_duration_us: int = 0 + + +class _EventItem: + def __init__(self, event): + self.e = event + self.start, self.end = self._interval_of() + + def _interval_of(self): + start = self.e["ts"] + duration = self.e.get("dur", 0) + return start, start + duration + + +def _find_events_covered_in( + events: list[dict[str, Any]], + top_level_events: list[dict[str, Any]], +): + """Find events covered in top level events. + + Returns a dict mapping top-level event uid to covered event uids. + """ + top_level_by_tid = defaultdict(list) + for event in top_level_events: + top_level_by_tid[event["tid"]].append(_EventItem(event)) + for tid in top_level_by_tid: + top_level_by_tid[tid] = sorted(top_level_by_tid[tid], key=lambda x: x.start) + starts_by_tid = { + tid: [event.start for event in event_items] + for tid, event_items in top_level_by_tid.items() + } + top_level_id_to_events = defaultdict(set) + + for event in events: + if event.get("cat") not in { + "cuda_runtime", + "cuLaunchKernel", + "cpu_op", + "cuda_driver", + "gpu_memset", + "python_function", + }: + continue + if "CallFrom" in event.get("args", {}): + continue + tid = event.get("tid") + if tid not in top_level_by_tid: + continue + item = _EventItem(event) + event_items = top_level_by_tid[tid] + idx = bisect_left(starts_by_tid[tid], item.start) - 1 + if idx < 0 or idx >= len(event_items): + continue + top_level_item = event_items[idx] + if top_level_item.start <= item.start and item.end <= top_level_item.end: + top_level_id_to_events[top_level_item.e["uid"]].add(event["uid"]) + return top_level_id_to_events + + +class profile: + """Context manager that manages autograd profiler state and holds a summary of results. + + .. note:: + This is the backend, most people should use :mod:`torch.profiler` instead. + + Under the hood it just records events of functions being executed in C++ and + exposes those events to Python. You can wrap any code into it and it will + only report runtime of PyTorch functions. + Note: profiler is thread local and is automatically propagated into the async tasks + + Args: + enabled (bool, optional): Setting this to False makes this context manager a no-op. + + use_cuda (bool, optional): Enables timing of CUDA events as well + using the cudaEvent API. (will be deprecated) + + use_device (str, optional): Enables timing of device events. + Adds approximately 4us of overhead to each tensor operation when use cuda. + The valid devices options are 'cuda', 'xpu', 'mtia' and 'privateuseone'. + + record_shapes (bool, optional): If shapes recording is set, information + about input dimensions will be collected. This allows one to see which + dimensions have been used under the hood and further group by them + using prof.key_averages(group_by_input_shape=True). Please note that + shape recording might skew your profiling data. It is recommended to + use separate runs with and without shape recording to validate the timing. + Most likely the skew will be negligible for bottom most events (in a case + of nested function calls). But for higher level functions the total + self cpu time might be artificially increased because of the shape + collection. + + with_flops (bool, optional): If with_flops is set, the profiler will estimate + the FLOPs (floating point operations) value using the operator's input shape. + This allows one to estimate the hardware performance. Currently, + this option only works for the matrix multiplication and 2D convolution operators. + + profile_memory (bool, optional): track tensor memory allocation/deallocation. + + with_stack (bool, optional): record source information (file and line number) for the ops. + + with_modules (bool): record module hierarchy (including function names) + corresponding to the callstack of the op. e.g. If module A's forward call's + module B's forward which contains an aten::add op, + then aten::add's module hierarchy is A.B + Note that this support exist, at the moment, only for TorchScript models + and not eager mode models. + + use_kineto (bool, optional): experimental, enable profiling with Kineto profiler. + + use_cpu (bool, optional): profile CPU events; setting to ``False`` requires + ``use_kineto=True`` and can be used to lower the overhead for GPU-only profiling. + + experimental_config (_ExperimentalConfig) : A set of experimental options + used by profiler libraries like Kineto. Note, backward compatibility is not guaranteed. + + acc_events (bool): Enable the accumulation of FunctionEvents across multiple profiling cycles + + post_processing_timeout_s (float): Optional timeout in seconds for post-processing profiler + results. In this context, post-processing happens after the profiling itself has finished. + If specified, event parsing will stop after this duration and return partial results. Useful + for handling large traces that may take too long to process. + + + .. warning:: + Enabling memory profiling or source attribution incurs additional profiler + overhead + + .. warning:: + This context managers should not be called recursively, i.e. no nested + instances are allowed + + .. warning:: + Due to some CUDA multiprocessing limitations (see :ref:`multiprocessing-cuda-note`), + one cannot use the profiler with ``use_device = 'cuda'`` to benchmark + DataLoaders with ``num_workers > 0``. If you wish to benchmark data loading, + please use ``use_device = None`` or ``num_workers = 0``. + + Example: + >>> # xdoctest: +SKIP + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> x = torch.randn((1, 1), requires_grad=True) + >>> with torch.autograd.profiler.profile() as prof: + >>> for _ in range(100): # any normal python code, really! + >>> y = x ** 2 + >>> y.backward() + >>> # NOTE: some columns were removed for brevity + >>> print(prof.key_averages().table(sort_by="self_cpu_time_total")) + ----------------------------------- --------------- --------------- --------------- + Name Self CPU total CPU time avg Number of Calls + ----------------------------------- --------------- --------------- --------------- + mul 32.048ms 32.048ms 200 + pow 27.041ms 27.041ms 200 + PowBackward0 9.727ms 55.483ms 100 + torch::autograd::AccumulateGrad 9.148ms 9.148ms 100 + torch::autograd::GraphRoot 691.816us 691.816us 100 + ----------------------------------- --------------- --------------- --------------- + + """ + + def __init__( + self, + enabled=True, + *, + use_cuda=False, # Deprecated + use_device=None, + record_shapes=False, + with_flops=False, + profile_memory=False, + with_stack=False, + with_modules=False, + use_kineto=False, + use_cpu=True, + experimental_config=None, + acc_events=False, + custom_trace_id_callback=None, + post_processing_timeout_s: float | None = None, + activity_filters: dict[ProfilerActivity, set[str]] | None = None, + ): + self.enabled: bool = enabled + if not self.enabled: + return + self.use_cuda = use_cuda + if self.use_cuda: + warn( + "The attribute `use_cuda` will be deprecated soon, " + "please use ``use_device = 'cuda'`` instead.", + FutureWarning, + stacklevel=2, + ) + self.use_device: str | None = "cuda" + else: + self.use_device = use_device + # TODO Consider changing _function_events into data structure with size cap + self._function_events: EventList | None = None + self._old_function_events: EventList | None = None + # Function event processing is done lazily + self._needs_processing = False + self.entered = False + self.record_shapes = record_shapes + self.with_flops = with_flops + self.record_shapes |= self.with_flops + self.profile_memory = profile_memory + self.with_stack = with_stack + self.with_modules = with_modules + self.use_cpu = use_cpu + self.acc_events = acc_events + if experimental_config is None: + experimental_config = _ExperimentalConfig() + if experimental_config.trace_only and with_stack: + warn( + "trace_only=True is incompatible with with_stack=True " + "(stack traces require event post-processing). " + "Disabling trace_only." + ) + experimental_config = copy.copy(experimental_config) + experimental_config.trace_only = False + if ( + experimental_config.profiler_metrics + or experimental_config.profiler_measure_per_kernel + ): + warn( + "profiler_metrics and profiler_measure_per_kernel are deprecated " + "and ignored. These options will be removed in a future release.", + FutureWarning, + stacklevel=2, + ) + self.experimental_config = experimental_config + self.kineto_results: _ProfilerResult | None = None + self.profiling_start_time_ns = 0 + self.profiling_end_time_ns = 0 + self._stats = _ProfilerStats() + self.custom_trace_id_callback = custom_trace_id_callback + self.post_processing_timeout_s = post_processing_timeout_s + self.activity_filters = activity_filters or {} + self.trace_id = "" + if not self.use_cpu: + if not use_kineto: + raise AssertionError( + "Device-only events supported only with Kineto (use_kineto=True)" + ) + + if self.use_device is not None: + VALID_DEVICE_OPTIONS = ["cuda", "xpu", "mtia", "hpu"] + if _get_privateuse1_backend_name() != "privateuseone": + VALID_DEVICE_OPTIONS.append(_get_privateuse1_backend_name()) + if self.use_device not in VALID_DEVICE_OPTIONS: + warn( + f"The {self.use_device} is not a valid device option.", stacklevel=2 + ) + self.use_device = None + + if self.use_device == "cuda" and not torch.cuda.is_available(): + warn("CUDA is not available, disabling CUDA profiling", stacklevel=2) + self.use_cuda = False + self.use_device = None + + if self.use_device == "xpu" and not torch.xpu.is_available(): + warn("XPU is not available, disabling XPU profiling", stacklevel=2) + self.use_device = None + + if self.use_device == "hpu" and not ( + hasattr(torch, "hpu") and torch.hpu.is_available() + ): + warn("HPU is not available, disabling HPU profiling", stacklevel=2) + self.use_device = None + + self.kineto_activities = set() + if self.use_cpu: + self.kineto_activities.add(ProfilerActivity.CPU) + + self.profiler_kind = ProfilerState.KINETO + if self.use_device == "cuda": + if not use_kineto or ProfilerActivity.CUDA not in _supported_activities(): + if not self.use_cpu: + raise AssertionError("Legacy CUDA profiling requires use_cpu=True") + self.profiler_kind = ProfilerState.KINETO_GPU_FALLBACK + else: + self.kineto_activities.add(ProfilerActivity.CUDA) + elif self.use_device == "xpu": + if not (use_kineto and ProfilerActivity.XPU in _supported_activities()): + raise AssertionError( + "Legacy XPU profiling is not supported. Requires use_kineto=True on XPU devices." + ) + self.kineto_activities.add(ProfilerActivity.XPU) + elif self.use_device == "mtia": + if not (use_kineto and ProfilerActivity.MTIA in _supported_activities()): + raise AssertionError( + "Legacy MTIA profiling is not supported. Requires use_kineto=True on MTIA devices." + ) + self.kineto_activities.add(ProfilerActivity.MTIA) + elif self.use_device == "hpu": + if not (use_kineto and ProfilerActivity.HPU in _supported_activities()): + raise AssertionError( + "Legacy HPU profiling is not supported. Requires use_kineto=True on HPU devices." + ) + self.kineto_activities.add(ProfilerActivity.HPU) + elif self.use_device is not None and self.use_device != "privateuseone": + if use_kineto: + # Native tracing mode: use KINETO_PRIVATEUSE1 with registered IActivityProfiler + self.profiler_kind = ProfilerState.KINETO_PRIVATEUSE1 + if ProfilerActivity.PrivateUse1 in _supported_activities(): + self.kineto_activities.add(ProfilerActivity.PrivateUse1) + else: + # Marker-only mode: use fallback state + if not self.use_cpu: + raise AssertionError( + "Legacy privateuse1 profiling requires use_cpu=True" + ) + self.profiler_kind = ProfilerState.KINETO_PRIVATEUSE1_FALLBACK + + if len(self.kineto_activities) == 0: + raise AssertionError("No activities specified for the profiler") + + if ( + self.post_processing_timeout_s is not None + and self.post_processing_timeout_s < 0 + ): + raise ValueError("post_processing_timeout_s must be non-negative") + + def default_trace_id(self): + # Generate a UUID + uuid_raw = uuid.uuid4() + + return f"{uuid_raw.int:032X}" + + def create_trace_id(self): + if self.custom_trace_id_callback: + return self.custom_trace_id_callback() + return self.default_trace_id() + + def config(self, create_trace_id=False): + # only need to generate new trace id upon prepare trace not start trace + if create_trace_id: + trace_id = self.create_trace_id() + self.trace_id = trace_id + return ProfilerConfig( + self.profiler_kind, + self.record_shapes, + self.profile_memory, + self.with_stack, + self.with_flops, + self.with_modules, + self.experimental_config, + self.trace_id, + ) + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("Profiler context manager is not reentrant") + self._prepare_trace() + self._start_trace() + return self + + def _prepare_trace(self): + self.entered = True + t0 = perf_counter_ns() + _prepare_profiler( + self.config(create_trace_id=True), + self.kineto_activities, + activity_filter=self.activity_filters, + ) + t1 = perf_counter_ns() + self._stats.profiler_prepare_call_duration_us = int((t1 - t0) / 1000) + + def _start_trace(self): + self.entered = True + _run_on_profiler_start() + t0 = perf_counter_ns() + _enable_profiler(self.config(create_trace_id=False), self.kineto_activities) + t1 = perf_counter_ns() + self._stats.profiler_enable_call_duration_us = int((t1 - t0) / 1000) + self.profiling_start_time_ns = t1 + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + if self.use_device and hasattr(torch, self.use_device): + device_module = getattr(torch, self.use_device) + if hasattr(device_module, "synchronize"): + device_module.synchronize() + + if self._function_events and self.acc_events: + self._old_function_events = self._function_events + self._function_events = None + self._needs_processing = True + + t0 = perf_counter_ns() + + self.kineto_results = _disable_profiler() + t1 = perf_counter_ns() + self._stats.profiler_disable_call_duration_us = int((t1 - t0) / 1000) + self.profiling_end_time_ns = t0 + + _run_on_profiler_stop() + + self._stats.profiling_window_duration_sec = ( + (self.profiling_end_time_ns - self.profiling_start_time_ns) * 1.0 / 1e9 + ) + + # If we plan to accumulate events we should post process the function events + # right away to retain the state across multiple start/stop calls + if self.acc_events and not self.experimental_config.trace_only: + self._ensure_function_events() + return False + + def __repr__(self): + if self._needs_processing: + self._ensure_function_events() + if self._function_events is None: + return "" + return repr(self._function_events) + + def __str__(self): + if self._needs_processing: + self._ensure_function_events() + if self._function_events is None: + return "" + return str(self._function_events) + + def _ensure_function_events(self): + """Process function events lazily if required""" + if self.experimental_config.trace_only: + raise RuntimeError( + "events() is not available when trace_only=True in " + "ExperimentalConfig. Use export_chrome_trace() instead." + ) + if self._function_events is not None: + return + self._needs_processing = False + + t0 = perf_counter_ns() + parsed_results = [] + if self.kineto_results: + parsed_results = self._parse_kineto_results( + self.kineto_results, timeout_s=self.post_processing_timeout_s + ) + t1 = perf_counter_ns() + self._stats.parse_kineto_call_duration_us = int((t1 - t0) / 1000) + + self._function_events = EventList( + parsed_results, + use_device=self.use_device, + profile_memory=self.profile_memory, + with_flops=self.with_flops, + ) + t0 = perf_counter_ns() + self._function_events._build_tree() + t1 = perf_counter_ns() + self._stats.function_events_build_tree_call_duration_us = int((t1 - t0) / 1000) + self._stats.number_of_events = len(self._function_events) + + if self._old_function_events and self.acc_events: + for evt in self._old_function_events: + self._function_events.append(evt) + self._old_function_events = None + + @property + def function_events(self): + if self._function_events is None or self._needs_processing: + self._ensure_function_events() + return self._function_events + + def table( + self, + sort_by=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + header=None, + top_level_events_only=False, + ): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.table( + sort_by=sort_by, + row_limit=row_limit, + max_src_column_width=max_src_column_width, + max_name_column_width=max_name_column_width, + max_shapes_column_width=max_shapes_column_width, + header=header, + top_level_events_only=top_level_events_only, + ) + + table.__doc__ = EventList.table.__doc__ + + def _assign_uniq_id_to_event(self, trace): + if trace.get("uid_assigned"): + return {event["uid"]: event for event in trace["traceEvents"]} + uid_2_events = {} + for uid, event in enumerate(trace["traceEvents"]): + event["uid"] = uid + uid_2_events[uid] = event + trace["uid_assigned"] = True + return uid_2_events + + def _build_flow_mapping(self, trace, flow_events): + """Build src/dst event mappings from profiler flow events.""" + if not flow_events: + return {}, {} + + flow_pair: dict[int, list[int | None]] = {} + prev_event = None + for event in trace["traceEvents"]: + if ( + event.get("name") not in _profile_flow_types + and event.get("cat") not in _profile_flow_types + ): + prev_event = event + continue + if event.get("ph") == "s": + if prev_event is None: + continue + pair = flow_pair.setdefault(int(event["id"]), [None, None]) + pair[0] = prev_event["uid"] + elif event.get("ph") == "f": + if prev_event is None: + continue + pair = flow_pair.setdefault(int(event["id"]), [None, None]) + pair[1] = prev_event["uid"] + prev_event = event + + src2dst = {} + dst2src = {} + for src, dst in flow_pair.values(): + if src is None or dst is None: + continue + src2dst[src] = dst + dst2src[dst] = src + return src2dst, dst2src + + def _maybe_triton_call(self, kernel_name): + return kernel_name.startswith("triton_") + + def _has_kernel_info(self, compile_info, graph_key, kernel_name): + graph_info = compile_info.get(graph_key, {}) + if kernel_name in graph_info: + return True + kernel_prefix = kernel_name + ":" + return any(name.startswith(kernel_prefix) for name in graph_info) + + def _maybe_extern_call(self, event, compile_info, fwd_key, bw_keys, need_bw): + kernel_name = event["name"] + return not self._has_kernel_info(compile_info, fwd_key, kernel_name) and ( + not need_bw + or all( + not self._has_kernel_info(compile_info, bw_key, kernel_name) + for bw_key in bw_keys + ) + ) + + def add_to_chrome_trace(self, origin_trace): + """Add Inductor kernel stack information to a Chrome trace.""" + from torch._inductor.debug import get_kernel_information_jsons + + compile_linenos = get_kernel_information_jsons() + if not compile_linenos: + return origin_trace + + trace = origin_trace + uid_2_events = self._assign_uniq_id_to_event(trace) + + real_events = [] + flow_events = [] + compiled_events = [] + extern_events = [] + for event in trace["traceEvents"]: + name = event.get("name", "") + cat = event.get("cat", "") + if name in _profile_flow_types or cat in _profile_flow_types: + flow_events.append(event) + continue + + real_events.append(event) + if cat == "cpu_op" and ( + name == "CompiledFunctionBackward" or "Torch-Compiled Region" in name + ): + compiled_events.append(event) + if ( + name + and "extern_kernels" in name + and cat + in ( + "cpu_op", + "python_function", + ) + ): + extern_events.append(event) + + compiled_events = sorted(compiled_events, key=lambda x: _EventItem(x).start) + compiled_event_items = [_EventItem(event) for event in compiled_events] + compiled_event_starts = [item.start for item in compiled_event_items] + ops_in_compile_region = _find_events_covered_in(real_events, compiled_events) + ops_in_extern_region = _find_events_covered_in(real_events, extern_events) + src2dst, dst2src = self._build_flow_mapping(trace, flow_events) + kernel_uids_by_external_id = defaultdict(list) + for event in real_events: + if event.get("cat") != "kernel": + continue + external_id = event.get("args", {}).get("External id") + if external_id is not None: + kernel_uids_by_external_id[external_id].append(event["uid"]) + + def _related_compile_region(compile_event): + src_event = uid_2_events[dst2src[compile_event["uid"]]] + src_item = _EventItem(src_event) + idx = bisect_left(compiled_event_starts, src_item.start) - 1 + # Compiled regions are expected to be nearly nested around their + # source events, so the nearest preceding region usually matches. + while idx >= 0: + region_item = compiled_event_items[idx] + if ( + src_item.start > region_item.start + and src_item.end < region_item.end + ): + return region_item.e + idx -= 1 + raise ValueError(f"Cannot find compile region for {compile_event}") + + def _parse_compile_region_key(key): + try: + parsed_key = ast.literal_eval(key) + except (SyntaxError, ValueError): + return None + if ( + not isinstance(parsed_key, tuple) + or len(parsed_key) != 2 + or not isinstance(parsed_key[0], str) + or not isinstance(parsed_key[1], bool) + ): + return None + return parsed_key + + def _backward_graph_keys(compile_info, compile_name): + bw_graph_key = str((compile_name, True)) + keys = [bw_graph_key] + if bw_graph_key in compile_info: + return keys + + prefix = compile_name + "_" + for key, info in compile_info.items(): + if not info: + continue + parsed_key = _parse_compile_region_key(key) + if parsed_key is None: + continue + graph_name, is_backward = parsed_key + if is_backward and graph_name.startswith(prefix): + keys.append(key) + return keys + + def _stack_from_kernel_info(kernel_info): + if isinstance(kernel_info, dict): + return kernel_info.get("stack_traces") + return kernel_info + + def _stack_for_kernel(compile_info, graph_key, kernel_name): + graph_info = compile_info.get(graph_key, {}) + kernel_info = graph_info.get(kernel_name) + if kernel_info is None: + kernel_prefix = kernel_name + ":" + for name, info in graph_info.items(): + if name.startswith(kernel_prefix): + kernel_info = info + break + return _stack_from_kernel_info(kernel_info) + + def _single_stack_for_graph(compile_info, graph_key): + stacks = [ + stack + for stack in ( + _stack_from_kernel_info(kernel_info) + for kernel_info in compile_info.get(graph_key, {}).values() + ) + if stack is not None + ] + if len(stacks) == 1: + return stacks[0] + return None + + def _assign_stack(event, compile_info, fwd_key, bw_keys, kernel_name): + for bw_key in bw_keys: + stack = _stack_for_kernel(compile_info, bw_key, kernel_name) + if stack is not None: + event.setdefault("args", {})["stack"] = stack + return True + stack = _stack_for_kernel(compile_info, fwd_key, kernel_name) + if stack is not None: + event.setdefault("args", {})["stack"] = stack + return True + return False + + def _assign_single_stack(event, compile_info, fwd_key, bw_keys): + for bw_key in bw_keys: + stack = _single_stack_for_graph(compile_info, bw_key) + if stack is not None: + event.setdefault("args", {})["stack"] = stack + return True + stack = _single_stack_for_graph(compile_info, fwd_key) + if stack is not None: + event.setdefault("args", {})["stack"] = stack + return True + return False + + def _kernel_events_for_op(op_id): + if op_id in src2dst: + return [uid_2_events[src2dst[op_id]]] + external_id = uid_2_events[op_id].get("args", {}).get("External id") + if external_id is None: + return [] + return [ + uid_2_events[kernel_uid] + for kernel_uid in kernel_uids_by_external_id.get(external_id, []) + ] + + def _extern_kernel_name(name): + match = re.search(r"\b(extern\w+)\s*,", name) + return match.group(1) if match else name + + for compile_func_id, scoped_ops in ops_in_compile_region.items(): + compile_event = uid_2_events[compile_func_id] + compile_name = compile_event["name"] + need_bw = "Backward" in compile_name + + if need_bw: + try: + compile_name = _related_compile_region(compile_event)["name"] + except ValueError: + continue + + fw_graph_key = str((compile_name, False)) + bw_graph_keys = _backward_graph_keys(compile_linenos, compile_name) + if fw_graph_key not in compile_linenos and all( + bw_graph_key not in compile_linenos for bw_graph_key in bw_graph_keys + ): + continue + + for op_id in scoped_ops: + kernel_events = _kernel_events_for_op(op_id) + if not kernel_events: + continue + for kernel_event in kernel_events: + assigned_stack = False + warn_if_unrecognized = False + if self._maybe_extern_call( + kernel_event, + compile_linenos, + fw_graph_key, + bw_graph_keys, + need_bw, + ): + for extern_call, related_ops in ops_in_extern_region.items(): + if op_id in related_ops: + extern_event = uid_2_events[extern_call] + assigned_stack = _assign_stack( + kernel_event, + compile_linenos, + fw_graph_key, + bw_graph_keys, + _extern_kernel_name(extern_event.get("name", "")), + ) + if assigned_stack: + break + if not assigned_stack: + assigned_stack = _assign_single_stack( + kernel_event, + compile_linenos, + fw_graph_key, + bw_graph_keys, + ) + elif not self._maybe_triton_call(kernel_event.get("name", "")): + warn_if_unrecognized = True + assigned_stack = _assign_single_stack( + kernel_event, + compile_linenos, + fw_graph_key, + bw_graph_keys, + ) + else: + assigned_stack = _assign_stack( + kernel_event, + compile_linenos, + fw_graph_key, + bw_graph_keys, + kernel_event.get("name", ""), + ) + if not assigned_stack and warn_if_unrecognized: + log.warning( + "Kernel %s cannot be recognized as a custom kernel or triton kernel. " + "Please try profile with stack=True.", + kernel_event.get("name", ""), + ) + return trace + + def export_chrome_trace(self, path, metadata=None, use_python_export=False): + """ + Exports the collected trace in Chrome JSON format. If kineto is enabled, only + last cycle in schedule is exported. + """ + if use_python_export and kineto_available(): + from torch.profiler._chrome_trace_export import ( + export_chrome_trace as _export, + ) + + _export(self.kineto_results, path, metadata) # type: ignore[union-attr] + elif kineto_available(): + self.kineto_results.save(path) # type: ignore[union-attr] + else: + self._ensure_function_events() + return self._function_events.export_chrome_trace(path) # type: ignore[union-attr] + + import torch._inductor.config as inductor_config + + if inductor_config.trace.provenance_tracking_to_timeline: + from torch._inductor.debug import get_kernel_information_jsons + + try: + # Kineto owns the initial serialization path. Re-read the exported + # trace here so timeline provenance stays decoupled from Kineto's + # JSON writer and remains a Python-only post-processing step. + with open(path) as f: + trace = json.load(f) + + num_events = len(trace.get("traceEvents", [])) + max_events = inductor_config.trace.provenance_tracking_max_events + if max_events > 0 and num_events > max_events: + log.warning( + "Skipping provenance tracking: trace has %d events " + "(exceeds limit of %d). Set TORCH_COMPILE_DEBUG_MAX_EVENTS=0 " + "to disable this protection or increase the limit.", + num_events, + max_events, + ) + return + + if not inductor_config.triton.unique_kernel_names: + log.warning( + "Profiling trace does not contain Triton kernel stack traces " + "because TORCHINDUCTOR_UNIQUE_KERNEL_NAMES=0." + ) + if inductor_config.cpp_wrapper: + log.warning( + "Profiling trace does not contain compiled kernel stack traces " + "because cpp_wrapper is enabled." + ) + return + try: + log.info("Add stack trace to compiled kernel.") + trace = self.add_to_chrome_trace(trace) + with open(path, "w") as f: + json.dump(trace, f, indent=1) + except MemoryError: + log.error( + "MemoryError during add_to_chrome_trace. " + "Try increasing TORCH_COMPILE_DEBUG_MAX_EVENTS or disable provenance tracking." + ) + raise + except Exception: + log.exception("Failed to add stack trace to compiled kernel") + finally: + get_kernel_information_jsons().clear() + + export_chrome_trace.__doc__ = EventList.export_chrome_trace.__doc__ + + def export_stacks(self, path: str, metric: str = "self_cpu_time_total"): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + if not self.with_stack: + raise AssertionError("export_stacks() requires with_stack=True") + return self._function_events.export_stacks(path, metric) + + def toggle_collection_dynamic( + self, enabled: bool, activities: Iterable[ProfilerActivity] + ): + """ + Toggles the collection of activities for the current profiler instance. + """ + return _toggle_collection_dynamic(enabled, set(activities)) + + def key_averages( + self, + group_by_input_shape=False, + group_by_stack_n=0, + group_by_overload_name=False, + include_python_functions=False, + ): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.key_averages( + group_by_input_shape, + group_by_stack_n, + group_by_overload_name, + include_python_functions, + ) + + key_averages.__doc__ = EventList.key_averages.__doc__ + + def total_average(self): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.total_average() + + total_average.__doc__ = EventList.total_average.__doc__ + + @property + def self_cpu_time_total(self): + """Returns total time spent on CPU. + + The total time is a sum of all self times across all the events. + """ + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.self_cpu_time_total + + def _parse_kineto_results( + self, result: _ProfilerResult, timeout_s: float | None = None + ): + # result.events() has most of the events - PyTorch op-level and device-level events + + timeout_ns = int(timeout_s * 1e9) if timeout_s is not None else None + result_events = result.events() + if timeout_ns is not None and timeout_ns < 0: + raise ValueError("timeout_s must be non-negative") + start_time_ns = perf_counter_ns() + timed_out = False + + def _check_timeout() -> bool: + """Check if timeout has been exceeded. Returns True if timed out.""" + nonlocal timed_out + if timeout_ns is not None and not timed_out: + elapsed_ns = perf_counter_ns() - start_time_ns + if elapsed_ns >= timeout_ns: + timed_out = True + return timed_out + + trace_start_ns = result.trace_start_ns() + mem_records = [ + [evt, False] for evt in result_events if evt.name() == MEMORY_EVENT_NAME + ] + oom_records = [ + evt for evt in result_events if evt.name() == OUT_OF_MEMORY_EVENT_NAME + ] + mem_records_acc = MemRecordsAcc(mem_records) + + def _cpu_memory_usage(mem_record): + return ( + mem_record.nbytes() + if mem_record.device_type() + in [DeviceType.CPU, DeviceType.MKLDNN, DeviceType.IDEEP] + else 0 + ) + + def _device_memory_usage(mem_record): + return ( + mem_record.nbytes() + if mem_record.device_type() + in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.HIP, + DeviceType.XPU, + ] + else 0 + ) + + # Create and return FunctionEvent list, which contains all function events + # Here 2 function events are created: + # all_function_events contains all events associated with each kineto event from result + all_function_events = [] + # frontend_function_events contains the events in aten or torch frontend level, + # whose correlation id is 0 + frontend_function_events = [] + device_corr_map: dict[int, list[FunctionEvent]] = {} + max_evt_id = 0 + for kineto_event in result_events: + if _check_timeout(): + break + + if ( + _filter_name(kineto_event.name()) + or getattr(kineto_event, "is_hidden_event", lambda: False)() + ): + continue + rel_start_ns = kineto_event.start_ns() - trace_start_ns + rel_end_ns = kineto_event.end_ns() - trace_start_ns + + cpu_memory_usage = 0 + device_memory_usage = 0 + if kineto_event.device_type() == DeviceType.CPU: + # find the corresponding memory allocation events + for mem_record in mem_records_acc.in_interval( + kineto_event.start_ns(), kineto_event.end_ns() + ): + cpu_memory_usage += _cpu_memory_usage(mem_record[0]) + device_memory_usage += _device_memory_usage(mem_record[0]) + mem_record[1] = True + + is_async = kineto_event.is_async() or ( + kineto_event.start_thread_id() != kineto_event.end_thread_id() + ) + + fe = FunctionEvent( + id=kineto_event.correlation_id(), + name=_rewrite_name(name=kineto_event.name(), with_wildcard=True), + overload_name=kineto_event.overload_name(), + trace_name=_rewrite_name(name=kineto_event.name(), with_wildcard=False), + thread=kineto_event.start_thread_id(), + start_us=rel_start_ns / 1000, + end_us=rel_end_ns / 1000, + fwd_thread=kineto_event.fwd_thread_id(), + input_shapes=kineto_event.shapes(), + concrete_inputs=kineto_event.concrete_inputs(), + kwinputs=kineto_event.kwinputs(), + stack=[ + entry + for entry in kineto_event.stack() + if _filter_stack_entry(entry) + ], + scope=kineto_event.scope(), + use_device=self.use_device, + cpu_memory_usage=cpu_memory_usage, + device_memory_usage=device_memory_usage, + is_async=is_async, + sequence_nr=kineto_event.sequence_nr(), + device_type=kineto_event.device_type(), + device_index=kineto_event.device_index(), + device_resource_id=kineto_event.device_resource_id(), + flops=kineto_event.flops(), + is_user_annotation=kineto_event.is_user_annotation(), + is_python_function=kineto_event.is_python_function(), + activity_type=kineto_event.activity_type(), + metadata_json=kineto_event.metadata_json(), + extra_meta=kineto_event.extra_meta() or None, + flow_id=kineto_event.flow_id(), + flow_type=kineto_event.flow_type(), + flow_start=kineto_event.flow_start(), + external_id=kineto_event.external_id(), + linked_correlation_id=kineto_event.linked_correlation_id(), + structured_input_shapes=kineto_event.structured_input_shapes(), + structured_input_strides=kineto_event.structured_input_strides(), + input_dtypes=kineto_event.dtypes(), + python_id=kineto_event.python_id(), + python_parent_id=kineto_event.python_parent_id(), + python_module_id=kineto_event.python_module_id(), + ) + max_evt_id = max(max_evt_id, fe.id) + if fe.device_type == DeviceType.CPU and not fe.is_async: + if self.use_device == _get_privateuse1_backend_name(): + privateuse1_time = kineto_event.privateuse1_elapsed_us() + if privateuse1_time > 0: + fe.append_kernel(fe.name, fe.device_index, privateuse1_time) + fe.is_legacy = True + elif self.use_device == "cuda": + # Check if we have CUDA time as a fallback + cuda_time = kineto_event.cuda_elapsed_us() + if cuda_time > 0: + fe.append_kernel(fe.name, fe.device_index, cuda_time) + fe.is_legacy = True + all_function_events.append(fe) + corr_id = kineto_event.linked_correlation_id() + if corr_id > 0: + if corr_id not in device_corr_map: + device_corr_map[corr_id] = [] + device_corr_map[corr_id].append(fe) + elif corr_id == 0: + frontend_function_events.append(fe) + else: + raise RuntimeError( + f"Got negative correlation id {corr_id} in profiler post processing" + ) + + # associate device kernels and device runtime (CPU) with CPU events + for fe in frontend_function_events: + if ( + fe.device_type == DeviceType.CPU + and not fe.is_async + and fe.id in device_corr_map + ): + for f_evt in device_corr_map[fe.id]: + if f_evt.device_type in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.XPU, + ]: + fe.append_kernel( + f_evt.name, + f_evt.device_index, + f_evt.time_range.end - f_evt.time_range.start, + ) + elif f_evt.device_type == DeviceType.CPU: + # make sure that 'thread' of a CPU Kineto (e.g. Device Runtime) event is associated + # with the 'thread' of the corresponding linked PyTorch event to properly track + # parents and children + f_evt.thread = fe.thread + + def _create_function_event_for_memory_events(evt): + rel_start_ns = evt.start_ns() - trace_start_ns + fe = FunctionEvent( + id=max_evt_id, + name=evt.name(), + overload_name="", + trace_name=None, # not outputting in the trace + thread=evt.start_thread_id(), + start_us=rel_start_ns / 1000, + end_us=rel_start_ns / 1000, # no duration + fwd_thread=evt.start_thread_id(), + input_shapes=[], + stack=[], + scope=0, # RecordScope::FUNCTION + use_device=self.use_device, + cpu_memory_usage=_cpu_memory_usage(evt), + device_memory_usage=_device_memory_usage(evt), + is_async=False, + sequence_nr=-1, + device_type=DeviceType.CPU, + device_index=0, + ) + return fe + + # output top-level memory events + for mem_record in mem_records: + if _check_timeout(): + break + + if not mem_record[1]: + max_evt_id += 1 + fe = _create_function_event_for_memory_events(mem_record[0]) + all_function_events.append(fe) + + for oom_record in oom_records: + if _check_timeout(): + break + + max_evt_id += 1 + fe = _create_function_event_for_memory_events(oom_record) + all_function_events.append(fe) + + if timed_out: + log.warning( + "Profiler _parse_kineto_results timed out after %.3f seconds, " + "returning partial results with %d events", + timeout_s, + len(all_function_events), + ) + + all_function_events.sort( + key=lambda evt: [evt.time_range.start, -evt.time_range.end] + ) + return all_function_events + + +# Set by torch.profiler to the active cupti_monitor ProfilerObserver while a session is +# running (None otherwise). record_function routes regions to it via push/pop_annotation. +# Held as an opaque object -- NOT imported from the cupti package -- so record_function never +# pulls in the cupti chain on a non-cupti run, and there is a single "is a session active" +# signal (this reference) rather than a separate flag. +_active_cupti_profiler_observer: Any = None + + +def _set_active_cupti_profiler_observer(observer: Any) -> None: + global _active_cupti_profiler_observer + _active_cupti_profiler_observer = observer + + +class record_function(_ContextDecorator): # pyrefly: ignore [invalid-inheritance] + """Context manager/function decorator that adds a label to a code block/function when running autograd profiler. + Label will only appear if CPU activity tracing is enabled. + + It is useful when tracing the code profile. + + Args: + name (str): Label assigned to the block of code. + node_id (int): ID of node, for distributed profiling. Unset in + non-distributed cases. + + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> x = torch.randn((1, 1), requires_grad=True) + >>> with torch.autograd.profiler.profile() as prof: + ... y = x**2 + ... with torch.autograd.profiler.record_function( + ... "label-z" + ... ): # label the block + ... z = y**3 + ... y.backward() + >>> # xdoctest: +IGNORE_WANT + >>> # NOTE: some columns were removed for brevity + >>> print(prof.key_averages().table(sort_by="self_cpu_time_total")) + ----------------------------------- --------------- --------------- --------------- + Name Self CPU total % CPU time avg Number of Calls + ----------------------------------- --------------- --------------- --------------- + pow 60.77% 47.470us 3 + mul 21.73% 25.465us 2 + PowBackward0 12.03% 121.891us 1 + torch::autograd::AccumulateGrad 2.70% 6.324us 1 + label-z 2.13% 12.421us 1 + torch::autograd::GraphRoot 0.64% 1.503us 1 + ----------------------------------- --------------- --------------- --------------- + Self CPU time total: 234.344us + CUDA time total: 0.000us + + """ + + def __init__(self, name: str, args: str | None = None): + self.name: str = name + self.args: str | None = args + # Whether or not we should run record function's end callbacks when exiting. + self.run_callbacks_on_exit: bool = True + # TODO: TorchScript ignores standard type annotation here + # self.record: Optional["torch.classes.profiler._RecordFunction"] = None + self.record = torch.jit.annotate( + # pyrefly: ignore [not-a-type] + Optional["torch.classes.profiler._RecordFunction"], + None, + ) + self._cupti_monitor_external_id: int | None = None + + def __enter__(self): + self.record = torch.ops.profiler._record_function_enter_new( + self.name, self.args + ) + # Route the region to the active cupti_monitor observer, if any. The reference is + # None unless a cupti_monitor profile is running, so a non-cupti run never touches + # the cupti chain. Guarded by is_scripting() (the global access doesn't compile under + # TorchScript), and the global is read inside the guard so it is dead-code-eliminated. + if not torch.jit.is_scripting(): + observer = _active_cupti_profiler_observer + if observer is not None: + self._cupti_monitor_external_id = observer.push_annotation(self.name) + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any): + if not torch.jit.is_scripting(): + if self._cupti_monitor_external_id is not None: + observer = _active_cupti_profiler_observer + if observer is not None: + observer.pop_annotation() + self._cupti_monitor_external_id = None + if not self.run_callbacks_on_exit: + return + + # Local variable is needed by TorchScript to refine Optional[T] to T + record = self.record + if record is None: + raise AssertionError("Expected record to be set") + + # TODO: Too slow with __torch_function__ handling enabled + # See https://github.com/pytorch/pytorch/issues/76410 + if not torch.jit.is_scripting(): + with torch._C.DisableTorchFunctionSubclass(): + torch.ops.profiler._record_function_exit._RecordFunction(record) + else: + torch.ops.profiler._record_function_exit(record) + + def _call_end_callbacks_on_future(self, fut: Future[Any]) -> Future[Any]: + """Use for profiling async calls that return a future. + + Calling this function will extend recording beyond this scope, until the future is + satisfied. It is useful for profiling the end to end time of asynchronous calls. + This function should only be called once to attach the callback onto the future, and + will throw if called multiple times. + + Args: + fut: (torch._C.Future): future for which to schedule + callback for. + + Returns: + A future that completes with the value of the passed in future when + the profiling callbacks have ran. + + """ + # Throw if we have already attached a callback onto the future. + if not self.run_callbacks_on_exit: + raise RuntimeError("_call_end_callbacks_on_future can only be called once.") + + # We are scheduling to run this RecordFunction's end callbacks when the + # passed in future completes, so don't run end callbacks on exit. + self.run_callbacks_on_exit = False + + # Local variable is needed by TorchScript to refine Optional[T] to T + record = self.record + if record is None: + raise AssertionError("Expected record to be set") + + # TODO: Too slow with __torch_function__ handling enabled + # See https://github.com/pytorch/pytorch/issues/76410 + if not torch.jit.is_scripting(): + with torch._C.DisableTorchFunctionSubclass(): + profiled_future = ( + torch.ops.profiler._call_end_callbacks_on_jit_fut._RecordFunction( + record, fut + ) + ) + else: + profiled_future = torch.ops.profiler._call_end_callbacks_on_jit_fut( + record, fut + ) + return profiled_future + + +class emit_itt: + """Context manager that makes every autograd operation emit an ITT range. + + It is useful when running the program under Intel(R) VTune Profiler:: + + vtune <--vtune-flags> + + The Instrumentation and Tracing Technology (ITT) API enables your application to generate and + control the collection of trace data during its execution across different Intel tools. + This context manager is to annotate Intel(R) VTune Profiling trace. With help of this context manager, + you will be able to see labeled ranges in Intel(R) VTune Profiler GUI. + + .. warning: + This context manager should not be called recursively, i.e. at most one + instance should be enabled at any given time. + + Args: + enabled (bool, optional): Setting ``enabled=False`` makes this context manager a no-op. + Default: ``True``. + record_shapes (bool, optional): If ``record_shapes=True``, the itt range wrapping + each autograd op will append information about the sizes of Tensor arguments received + by that op, in the following format: + ``[[arg0.size(0), arg0.size(1), ...], [arg1.size(0), arg1.size(1), ...], ...]`` + Non-tensor arguments will be represented by ``[]``. + Arguments will be listed in the order they are received by the backend op. + Please note that this order may not match the order in which those arguments were passed + on the Python side. Also note that shape recording may increase the overhead of itt range creation. + Default: ``False`` + + Example: + >>> # xdoctest: +SKIP("Undefined variables") + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> with torch.autograd.profiler.emit_itt(): + ... model(x) + + """ + + def __init__(self, enabled=True, record_shapes=False): + self.enabled = enabled + self.entered = False + self.record_shapes = record_shapes + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("ITT annotation context manager is not reentrant") + self.entered = True + _run_on_profiler_start() + _enable_profiler( + ProfilerConfig( + ProfilerState.ITT, + self.record_shapes, + False, + False, + False, + False, + _ExperimentalConfig(), + ), + set(), + ) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + _disable_profiler() + _run_on_profiler_stop() + return False + + +class emit_nvtx: + """Context manager that makes every autograd operation emit an NVTX range. + + It is useful when running the program under nvprof:: + + nvprof --profile-from-start off -o trace_name.prof -- + + Unfortunately, there's no way to force nvprof to flush the data it collected + to disk, so for CUDA profiling one has to use this context manager to annotate + nvprof traces and wait for the process to exit before inspecting them. + Then, either NVIDIA Visual Profiler (nvvp) can be used to visualize the timeline, or + :func:`torch.autograd.profiler.load_nvprof` can load the results for inspection + e.g. in Python REPL. + + .. warning: + This context manager should not be called recursively, i.e. at most one + instance should be enabled at any given time. + + Args: + enabled (bool, optional): Setting ``enabled=False`` makes this context manager a no-op. + Default: ``True``. + record_shapes (bool, optional): If ``record_shapes=True``, the nvtx range wrapping + each autograd op will append information about the sizes of Tensor arguments received + by that op, in the following format: + ``[[arg0.size(0), arg0.size(1), ...], [arg1.size(0), arg1.size(1), ...], ...]`` + Non-tensor arguments will be represented by ``[]``. + Arguments will be listed in the order they are received by the backend op. + Please note that this order may not match the order in which those arguments were passed + on the Python side. Also note that shape recording may increase the overhead of nvtx range creation. + Default: ``False`` + + Example: + >>> # xdoctest: +SKIP("undefined variables") + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> with torch.cuda.profiler.profile(): + ... model(x) # Warmup CUDA memory allocator and profiler + ... with torch.autograd.profiler.emit_nvtx(): + ... model(x) + + **Forward-backward correlation** + + When viewing a profile created using :class:`emit_nvtx` in the Nvidia Visual Profiler, + correlating each backward-pass op with the corresponding forward-pass op can be difficult. + To ease this task, :class:`emit_nvtx` appends sequence number information to the ranges it + generates. + + During the forward pass, each function range is decorated with ``seq=``. ``seq`` is a running + counter, incremented each time a new backward Function object is created and stashed for backward. + Thus, the ``seq=`` annotation associated with each forward function range tells you that + if a backward Function object is created by this forward function, + the backward object will receive sequence number N. + During the backward pass, the top-level range wrapping each C++ backward Function's + ``apply()`` call is decorated with ``stashed seq=``. ``M`` is the sequence number that + the backward object was created with. By comparing ``stashed seq`` numbers in backward with ``seq`` + numbers in forward, you can track down which forward op created each backward Function. + + Any functions executed during the backward pass are also decorated with ``seq=``. During + default backward (with ``create_graph=False``) this information is irrelevant, and in fact, + ``N`` may simply be 0 for all such functions. Only the top-level ranges associated with + backward Function objects' ``apply()`` methods are useful, as a way to correlate these Function + objects with the earlier forward pass. + + **Double-backward** + + If, on the other hand, a backward pass with ``create_graph=True`` is underway (in other words, + if you are setting up for a double-backward), each function's execution during backward + is given a nonzero, useful ``seq=``. Those functions may themselves create Function objects + to be executed later during double-backward, just as the original functions in the forward pass did. + The relationship between backward and double-backward is conceptually the same as the relationship + between forward and backward: The functions still emit current-sequence-number-tagged ranges, + the Function objects they create still stash those sequence numbers, and during the eventual + double-backward, the Function objects' ``apply()`` ranges are still tagged with ``stashed seq`` + numbers, which can be compared to `seq` numbers from the backward pass. + + .. warning: + The sequence number is thread-local, and some forward functions don't create an associated + backward Function object (instead delegating that to sub-functions further down the call chain). + For these reasons, the correspondence of stashed sequence numbers in + backward Function ``apply()`` ranges with `seq` numbers in forward-pass ranges is + not guaranteed to be 1 to 1. The sequence numbers alone may not be enough to fully + disambiguate which forward function created which + backward Function object. You may need to make a judgment based on analytic knowledge of what + the expected correspondence should be. + """ + + def __init__(self, enabled=True, record_shapes=False): + self.enabled = enabled + self.entered = False + self.record_shapes = record_shapes + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("NVTX annotation context manager is not reentrant") + self.entered = True + torch.cuda.synchronize() + _run_on_profiler_start() + _enable_profiler( + ProfilerConfig( + ProfilerState.NVTX, + self.record_shapes, + False, + False, + False, + False, + _ExperimentalConfig(), + ), + set(), + ) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + torch.cuda.synchronize() + _disable_profiler() + _run_on_profiler_stop() + return False + + +def load_nvprof(path): + """Open an nvprof trace file and parses autograd annotations. + + Args: + path (str): path to nvprof trace + """ + return EventList(parse_nvprof_trace(path)) + + +class EnforceUnique: + """Raises an error if a key is seen more than once.""" + + def __init__(self): + self.seen = set() + + def see(self, *key): + r""" + Observe a key and raise an error if it is seen multiple times. + """ + if key in self.seen: + raise RuntimeError("duplicate key: " + str(key)) + self.seen.add(key) + + +def parse_nvprof_trace(path): + import sqlite3 + + conn = sqlite3.connect(path) + conn.row_factory = sqlite3.Row + + # Parse strings table + strings = {} + for r in conn.execute("SELECT _id_ as id, value FROM StringTable"): + strings[r["id"]] = torch._C._demangle(r["value"]) + + # First, find all functions and create FunctionEvents for them + marker_query = """ + SELECT + start.id AS marker_id, start.name, start.timestamp AS start_time, end.timestamp AS end_time + FROM + CUPTI_ACTIVITY_KIND_MARKER AS start INNER JOIN CUPTI_ACTIVITY_KIND_MARKER AS end + ON start.id = end.id + WHERE + start.name != 0 AND end.name = 0 + """ + functions = [] + functions_map = {} + unique = EnforceUnique() + for row in conn.execute(marker_query): + unique.see(row["marker_id"]) + evt = FunctionEvent( + id=row["marker_id"], + node_id=0, # missing a node_id when calling FunctionEvent. This is just to ensure + # that pytorch doesn't crash when creating a FunctionEvent() object + name=strings[row["name"]], + start_us=row["start_time"], + end_us=row["end_time"], + thread=0, + ) # TODO: find in sqlite database + functions.append(evt) + functions_map[evt.id] = evt + + # Now, correlate all kernels with FunctionEvents + kernel_query = """ + SELECT + start.id AS marker_id, start.name, start.timestamp, end.timestamp, + runtime._id_ AS runtime_id, runtime.cbid, runtime.start AS runtime_start, runtime.end AS runtime_end, + kernel.start AS kernel_start, kernel.end AS kernel_end, kernel.name AS kernel_name + FROM + CUPTI_ACTIVITY_KIND_MARKER AS start + INNER JOIN CUPTI_ACTIVITY_KIND_MARKER AS end + ON start.id = end.id + INNER JOIN CUPTI_ACTIVITY_KIND_RUNTIME as runtime + ON (start.timestamp < runtime.start AND runtime.end < end.timestamp) + INNER JOIN CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL AS kernel + ON kernel.correlationId = runtime.correlationId + """ + unique = EnforceUnique() + for row in conn.execute(kernel_query): + unique.see(row["marker_id"], row["runtime_id"]) + # 211 is cudaKernelLaunch for cuda >= 9.2 + if row["cbid"] != 211: + raise AssertionError(f"Expected cbid to be 211, but got {row['cbid']}") + evt = functions_map[row["marker_id"]] + evt.append_kernel( + row["kernel_name"], 0, row["kernel_end"] - row["kernel_start"] + ) + + functions.sort(key=lambda evt: evt.time_range.start) + return functions + + +class KinetoStepTracker: + """Provides an abstraction for incrementing the step count globally. + + Previously, we only had one place to mark that a step() has occurred + in the program via pytorch profiler step(). We will now add step hooks + in the Optimizer class https://github.com/pytorch/pytorch/issues/88446 + + - This could mean programs that already call profiler.step() every + iteration can end up double incrementing step count. + - If a model uses multiple optimizers we can also have double or more + counting of the step. + + We fix this by adding a layer of abstraction before calling step() + to the kineto library. The idea is to maintain steps per requester in a dict: + + .. code-block:: + + { + "ProfilerStep": 100, # triggered by profiler step() call + "Optimizer1Step": 100, # Optimizer 1 or 2 are just examples, could be SGD, Adam etc + "Optimizer2Step": 100, + } + + To figure out the global step count just take the max of dict values (100). + + If one of the count increments the max will go up. + + .. code-block:: + + { + "ProfilerStep": 100, + "Optimizer1Step": 101, # Optimizer1 got incremented first say + "Optimizer2Step": 100, + } + + Then global step count is 101 + We only call the kineto step() function when global count increments. + + NOTE: Please do not use the KinetoStepTracker in modules beside the Optimizer + for now. The result could be incorrect increments of the step count. + """ + + _current_step = 0 + _step_dict: dict[str, int] = defaultdict(int) + + @classmethod + def init_step_count(cls, requester: str): + r""" + Initialize for a given requester. + """ + cls._step_dict[requester] = cls._current_step + + @classmethod + def erase_step_count(cls, requester: str) -> bool: + r""" + Remove a given requester. + """ + return cls._step_dict.pop(requester, None) is not None + + @classmethod + def increment_step(cls, requester: str) -> int: + """Increments the step count for the requester. + + Additionally if the max over all step counts has incremented then + trigger the _kineto_step() returns global step count + """ + if requester not in cls._step_dict: + cls.init_step_count(requester) + cls._step_dict[requester] += 1 + + new_step = max(cls._step_dict.values()) + if new_step > cls._current_step: + delta = new_step - cls._current_step + if delta > 1: + warn( + "Profiler step count has increased more than 1 - " + f"current_step = {cls._current_step} step dict = {cls._step_dict}", + stacklevel=2, + ) + for _ in range(delta): + _kineto_step() + cls._current_step = new_step + return cls._current_step + + @classmethod + def current_step(cls) -> int: + r""" + Get the latest step for any requester + """ + return cls._current_step diff --git a/western_ai/usa/pytorch/torch/autograd/profiler_legacy.py b/western_ai/usa/pytorch/torch/autograd/profiler_legacy.py new file mode 100644 index 0000000000000000000000000000000000000000..04585860ce26c7320bab3dc6520e311e87f7eedf --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/profiler_legacy.py @@ -0,0 +1,327 @@ +# mypy: allow-untyped-defs +import itertools +import warnings +from typing_extensions import deprecated + +import torch +import torch.cuda +from torch.autograd import ( + _disable_profiler_legacy, + _enable_profiler_legacy, + DeviceType, + ProfilerConfig, + ProfilerState, +) +from torch.autograd.profiler_util import ( + _filter_name, + _filter_stack_entry, + _rewrite_name, + EventList, + FunctionEvent, + MEMORY_EVENT_NAME, +) + + +__all__ = ["profile"] + + +@deprecated( + "`torch.autograd.profiler_legacy.profile` is deprecated and will be removed in a future release. " + "Please use `torch.profiler` instead.", + category=None, # TODO: change to `FutureWarning` +) +class profile: + """DEPRECATED: use torch.profiler instead.""" + + def __init__( + self, + enabled=True, + *, + use_cuda=False, + record_shapes=False, + with_flops=False, + profile_memory=False, + with_stack=False, + with_modules=False, + ): + self.enabled: bool = enabled + if not self.enabled: + return + self.use_cuda = use_cuda + self.function_events = None + self.entered = False + self.record_shapes = record_shapes + self.with_flops = with_flops + self.record_shapes |= self.with_flops + self.profile_memory = profile_memory + self.with_stack = with_stack + self.with_modules = with_modules + + if self.use_cuda and not torch.cuda.is_available(): + warnings.warn( + "CUDA is not available, disabling CUDA profiling", + stacklevel=2, + ) + self.use_cuda = False + + if self.use_cuda: + self.profiler_kind = ProfilerState.CUDA + else: + self.profiler_kind = ProfilerState.CPU + + def config(self): + return ProfilerConfig( + self.profiler_kind, + self.record_shapes, + self.profile_memory, + self.with_stack, + self.with_flops, + self.with_modules, + # avoid exposing _ExperimentalConfig this in legacy public API + torch._C._profiler._ExperimentalConfig(), + ) + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("Profiler context manager is not reentrant") + self.entered = True + self._start_trace() + return self + + def _start_trace(self): + _enable_profiler_legacy(self.config()) + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + if self.use_cuda: + torch.cuda.synchronize() + + records = _disable_profiler_legacy() + parsed_results = _parse_legacy_records(records) + # pyrefly: ignore [bad-assignment] + self.function_events = EventList( + parsed_results, + use_device="cuda" if self.use_cuda else None, + profile_memory=self.profile_memory, + with_flops=self.with_flops, + ) + # pyrefly: ignore [missing-attribute] + self.function_events._build_tree() + return False + + def __repr__(self): + if self.function_events is None: + return "" + return repr(self.function_events) + + def __str__(self): + if self.function_events is None: + return "" + return str(self.function_events) + + def _check_finish(self): + if self.function_events is None: + raise RuntimeError("Profiler didn't finish running") + + def table( + self, + sort_by=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + header=None, + top_level_events_only=False, + ): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.table( + sort_by=sort_by, + row_limit=row_limit, + max_src_column_width=max_src_column_width, + max_name_column_width=max_name_column_width, + max_shapes_column_width=max_shapes_column_width, + header=header, + top_level_events_only=top_level_events_only, + ) + + table.__doc__ = EventList.table.__doc__ + + def export_chrome_trace(self, path): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.export_chrome_trace(path) + + export_chrome_trace.__doc__ = EventList.export_chrome_trace.__doc__ + + def export_stacks(self, path: str, metric: str = "self_cpu_time_total"): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + if not self.with_stack: + raise AssertionError("export_stacks() requires with_stack=True") + return self.function_events.export_stacks(path, metric) + + def key_averages(self, group_by_input_shape=False, group_by_stack_n=0): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.key_averages(group_by_input_shape, group_by_stack_n) + + key_averages.__doc__ = EventList.key_averages.__doc__ + + def total_average(self): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.total_average() + + total_average.__doc__ = EventList.total_average.__doc__ + + @property + def self_cpu_time_total(self): + """Return CPU time as the sum of self times across all events.""" + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.self_cpu_time_total + + +def _parse_legacy_records(thread_records): + def _get_record_key(record): + """Return a tuple for correlating start and end records in `_parse_legacy_records`.""" + return (record.handle(), record.node_id()) + + start_record = None + functions = [] + + # '__start_profile' is not guaranteed to be first, so we must find it here + for record in itertools.chain.from_iterable(thread_records): + name = record.name() + if start_record is None and name == "__start_profile": + start_record = record + + if start_record is None or start_record.is_remote(): + raise AssertionError("Expected a valid local start_record") + + for thread_record_list in thread_records: + # accumulated memory allocations per handle + cpu_memory_allocs = {} + cuda_memory_allocs = {} + # ranges per handle + range_starts = {} + + filtered_handles = set() + prev_record = None + for record in thread_record_list: + record_key = _get_record_key(record) + if _filter_name(record.name()) or record_key in filtered_handles: + filtered_handles.add(record_key) + continue + + if record.kind() == "push": + # workaround to reduce double logging from operator + # wrappers and redispatch + if prev_record is not None: + duplicate = ( + prev_record.name() == record.name() + and prev_record.kind() == record.kind() + and prev_record.node_id() == record.node_id() + ) + if duplicate: + filtered_handles.add(record_key) + continue + + range_starts[record_key] = record + cpu_memory_allocs[record_key] = 0 + cuda_memory_allocs[record_key] = 0 + elif record.kind() == "pop": + if record_key not in range_starts: + raise AssertionError( + f"Expected record with key {record_key} to exist in range_starts. " + "This means that the pop event did not have a corresponding push." + ) + + start = range_starts[record_key] + + cpu_memory_usage = cpu_memory_allocs[record_key] + cuda_memory_usage = cuda_memory_allocs[record_key] + is_async = start.is_async() or (start.thread_id() != record.thread_id()) + is_remote_event = record.is_remote() + start_flops = start.flops() + + fe = FunctionEvent( + id=record.handle(), + node_id=record.node_id(), + name=_rewrite_name(name=start.name(), with_wildcard=True), + trace_name=_rewrite_name(name=start.name(), with_wildcard=False), + thread=start.thread_id(), + start_us=start_record.cpu_elapsed_us(start), + end_us=start_record.cpu_elapsed_us(record), + fwd_thread=start.fwd_thread_id(), + input_shapes=start.shapes(), + stack=[ + entry for entry in start.stack() if _filter_stack_entry(entry) + ], + scope=start.scope(), + use_device="cuda" if start.has_cuda() else None, + cpu_memory_usage=cpu_memory_usage, + device_memory_usage=cuda_memory_usage, + is_async=is_async, + is_remote=is_remote_event, + sequence_nr=start.sequence_nr(), + device_type=DeviceType.CPU, + is_legacy=True, + flops=start_flops, + ) + # note: async events have only cpu total time + if not is_async and start.has_cuda(): + duration = start.cuda_elapsed_us(record) + if duration > 0: + fe.append_kernel(start.name(), start.device(), duration) + functions.append(fe) + del range_starts[record_key] + del cpu_memory_allocs[record_key] + del cuda_memory_allocs[record_key] + elif record.kind() == "memory_alloc": + num_open_handles_cpu = len(cpu_memory_allocs) + num_open_handles_cuda = len(cuda_memory_allocs) + if num_open_handles_cpu != num_open_handles_cuda: + raise AssertionError( + f"Expected CPU and CUDA memory allocation handles to match, " + f"but got {num_open_handles_cpu} CPU and {num_open_handles_cuda} CUDA" + ) + for handle in cpu_memory_allocs: + cpu_memory_allocs[handle] += record.cpu_memory_usage() + for handle in cuda_memory_allocs: + cuda_memory_allocs[handle] += record.cuda_memory_usage() + if num_open_handles_cpu == 0: + # output event as a top-level memory event + fe = FunctionEvent( + id=0, + name=MEMORY_EVENT_NAME, + trace_name=None, + thread=0, + start_us=0, + end_us=0, + stack=[], + cpu_memory_usage=record.cpu_memory_usage(), + device_memory_usage=record.cuda_memory_usage(), + is_legacy=True, + ) + functions.append(fe) + prev_record = record + + # Sort functions by start time then by end time ascending. + # This ensures that--in the case of nested events which + # have the same start time (which may happen due to the + # granularity of the given clock tick)--we always show + # the outermost nested call first. This adds stability + # in how FunctionEvents appear + functions.sort(key=lambda evt: [evt.time_range.start, -evt.time_range.end]) + return functions diff --git a/western_ai/usa/pytorch/torch/autograd/profiler_util.py b/western_ai/usa/pytorch/torch/autograd/profiler_util.py new file mode 100644 index 0000000000000000000000000000000000000000..705986117558233f039fa7216113f4f8712be1b5 --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/profiler_util.py @@ -0,0 +1,1617 @@ +# mypy: allow-untyped-defs +import bisect +import itertools +import json +import math +from collections import defaultdict, namedtuple +from collections.abc import Callable +from operator import attrgetter +from typing import Any, NamedTuple +from typing_extensions import deprecated + +import torch +from torch.autograd import DeviceType + + +__all__ = [ + "EventList", + "EventMetadata", + "FormattedTimesMixin", + "Interval", + "Kernel", + "FunctionEvent", + "FunctionEventAvg", + "StringTable", + "MemRecordsAcc", +] + + +class EventList(list): + """A list of profiling events with helper methods for analysis and visualization. + + EventList extends the standard Python list to provide specialized methods for + working with profiling events (FunctionEvent or FunctionEventAvg objects). + It includes utilities for aggregating statistics, formatting output tables, + and exporting profiling data. + + This class is typically returned by profiler methods and should not be + instantiated directly by users. + + Args: + *args: Standard list arguments. + use_device (str, optional): Device type for profiling ("cuda", "xpu", etc.). + profile_memory (bool, optional): Whether memory profiling was enabled. Default: False. + with_flops (bool, optional): Whether to include FLOP counts. Default: False. + + Attributes: + _use_device (str): Device type being profiled. + _profile_memory (bool): Whether memory profiling is enabled. + _with_flops (bool): Whether FLOP counting is enabled. + _tree_built (bool): Whether the event tree structure has been built. + + Key Methods: + table(...): Format events as a table string for display. + export_chrome_trace(path): Export to Chrome tracing format. + export_stacks(path, metric): Export stack traces with metrics. + key_averages(...): Compute averaged statistics grouped by operation name. + total_average(): Compute aggregate totals across all events (sums, not averages). + + Properties: + self_cpu_time_total: Sum of self CPU time across all events. + + Example:: + + import torch + from torch.profiler import profile, ProfilerActivity + + with profile(activities=[ProfilerActivity.CPU]) as prof: + x = torch.randn(100, 100) + y = torch.matmul(x, x) + + # EventList is returned by prof.events() + events = prof.events() + + # Display as formatted table + print( + events.table( + sort_by="cpu_time_total", row_limit=20, top_level_events_only=False + ) + ) + + # Export to Chrome tracing format + events.export_chrome_trace("trace.json") + + # Get averaged statistics + avg_events = events.key_averages() + print(avg_events.table()) + + # Export stack traces + events.export_stacks("stacks.txt", "self_cpu_time_total") + + See Also: + - :class:`FunctionEvent`: Individual profiling event + - :class:`FunctionEventAvg`: Averaged profiling statistics + - :meth:`table`: Format events as a readable table + - :meth:`key_averages`: Aggregate events by operation name + """ + + def __init__(self, *args, **kwargs): + use_device = kwargs.pop("use_device", None) + profile_memory = kwargs.pop("profile_memory", False) + with_flops = kwargs.pop("with_flops", False) + # pyrefly: ignore [not-iterable] + super().__init__(*args, **kwargs) + self._use_device = use_device + self._profile_memory = profile_memory + self._tree_built = False + self._with_flops = with_flops + + def _build_tree(self): + self._populate_cpu_children() + self._remove_dup_nodes() + self._set_backward_stacktraces() + self._tree_built = True + + def __str__(self): + return self.table() + + def _remove_dup_nodes(self): + while True: + to_delete = set() + + for idx in range(len(self)): + if ( + self[idx].cpu_parent is not None + and self[idx].cpu_parent.name == self[idx].name + and len(self[idx].cpu_parent.cpu_children) == 1 + ): + self[idx].cpu_parent.cpu_children = self[idx].cpu_children + self[idx].cpu_parent.kernels = self[idx].kernels # lift kernels up + for ch in self[idx].cpu_children: + ch.cpu_parent = self[idx].cpu_parent + to_delete.add(idx) + if len(to_delete) == 0: + break + + new_evts = [ev for ind, ev in enumerate(self) if ind not in to_delete] + + self.clear() + + self.extend(new_evts) + + def _populate_cpu_children(self): + """Populate child events into each underlying FunctionEvent object. + + One event is a child of another if [s1, e1) is inside [s2, e2). Where + s1 and e1 would be start and end of the child event's interval. And + s2 and e2 start and end of the parent event's interval + + Example: In event list [[0, 10], [1, 3], [3, 4]] would have make [0, 10] + be a parent of two other intervals. + + If for any reason two intervals intersect only partially, this function + will not record a parent child relationship between then. + """ + # Some events can be async (i.e. start and end on different threads), + # since it's generally undefined how to attribute children ranges to + # async ranges, we do not use them when calculating nested ranges and stats + sync_events = [ + evt + for evt in self + if not evt.is_async and evt.device_type == DeviceType.CPU + ] + events = sorted( + sync_events, + key=attrgetter("thread"), + ) + # Group by both thread and node_id, so that events that happen to have + # the same thread_id but are from different nodes aren't incorrectly + # grouped together. + threads = itertools.groupby( + events, key=lambda event: (event.thread, event.node_id) + ) + + # For each thread we keep a stack of current nested parents. + # We maintain the invariant that each interval is a subset of all other + # intervals lower in the stack. + # + # First we sort the intervals by their start time. Then we iterate over them. + # Every time we see a new interval we remove several parents from + # the top until we restore the invariant. Then parent child relationship + # if recorded if the stack is not empty. + # Finally we add new interval to the list + # + # Algorithm has O(N * log(N)) complexity where N is number of + # intervals + for _thread_id, thread_events in threads: + thread_events_ = sorted( + thread_events, + key=lambda event: [event.time_range.start, -event.time_range.end], + ) + current_events: list[FunctionEvent] = [] + for event in thread_events_: + while len(current_events) > 0: + parent = current_events[-1] + if ( + event.time_range.start >= parent.time_range.end + or event.time_range.end > parent.time_range.end + ): + # this can't be a parent + current_events.pop() + else: + parent.append_cpu_child(event) + if event.cpu_parent is not None: + raise AssertionError( + f"There is already a CPU parent event for {event.key}" + ) + event.set_cpu_parent(parent) + break + + current_events.append(event) + + def _set_backward_stacktraces(self): + def bw_parent(evt): + if evt is None: + return None + elif evt.scope == 1: # BACKWARD_FUNCTION + return evt + else: + return bw_parent(evt.cpu_parent) + + fwd_stacks = {} + for evt in self: + if bw_parent(evt) is None and evt.stack is not None: + t = (evt.sequence_nr, evt.thread) + if t not in fwd_stacks: + fwd_stacks[t] = evt.stack + + for evt in self: + p = bw_parent(evt) + if p is not None: + if p.fwd_thread is None: + raise AssertionError( + "Expected fwd_thread to be set for backward parent" + ) + t = (p.sequence_nr, p.fwd_thread) + evt.stack = fwd_stacks.get(t, []) + + @property + def self_cpu_time_total(self): + return sum(event.self_cpu_time_total for event in self) + + def table( + self, + sort_by=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + header=None, + top_level_events_only=False, + time_unit=None, + ): + """Print an EventList as a nicely formatted table. + + Args: + sort_by (str, optional): Attribute used to sort entries. By default + they are printed in the same order as they were registered. + Valid keys include: ``cpu_time``, ``cuda_time``, ``xpu_time``, + ``cpu_time_total``, ``cuda_time_total``, ``xpu_time_total``, + ``cpu_memory_usage``, ``cuda_memory_usage``, ``xpu_memory_usage``, + ``self_cpu_memory_usage``, ``self_cuda_memory_usage``, + ``self_xpu_memory_usage``, ``count``. + top_level_events_only(bool, optional): Boolean flag to determine the + selection of events to display. If true, the profiler will only + display events at top level like top-level invocation of python + `lstm`, python `add` or other functions, nested events like low-level + cpu/cuda/xpu ops events are omitted for profiler result readability. + time_unit(str, optional): A time unit to be used for all values in the + table. Valid options are: ``s``, ``ms`` and ``us``. + + Returns: + A string containing the table. + """ + return _build_table( + self, + sort_by=sort_by, + row_limit=row_limit, + max_src_column_width=max_src_column_width, + max_name_column_width=max_name_column_width, + max_shapes_column_width=max_shapes_column_width, + header=header, + profile_memory=self._profile_memory, + with_flops=self._with_flops, + top_level_events_only=top_level_events_only, + time_unit=time_unit, + ) + + def export_chrome_trace(self, path): + """Export an EventList as a Chrome tracing tools file. + + The checkpoint can be later loaded and inspected under ``chrome://tracing`` URL. + + Args: + path (str): Path where the trace will be written. + """ + import os + + device_name = "cuda" if not self._use_device else self._use_device + with open(path, "w") as f: + next_id = 0 + # Use file IO over using json.dump since JSON dumping is very slow and + # this technique is proven to give a 4x speedup. + f.write("[") + for evt in self: + if evt.trace_name is None: + continue + f.write( + '{{"name": "{}", ' + '"ph": "X", ' + '"ts": {}, ' + '"dur": {}, ' + '"tid": {}, ' + '"pid": "CPU functions", ' + '"args": {{}}}}, '.format( + evt.trace_name, + evt.time_range.start, + evt.time_range.elapsed_us(), + evt.thread + if not evt.is_remote + else f'" node_id:{evt.node_id}, thread_id:{evt.thread} "', + ) + ) + for _ in evt.kernels: + # 's' and 'f' draw Flow arrows from + # the CPU launch to the GPU kernel + f.write( + f'{{"name": "{evt.trace_name}", ' + '"ph": "s", ' + f'"ts": {evt.time_range.start}, ' + f'"tid": {evt.thread}, ' + '"pid": "CPU functions", ' + f'"id": {next_id}, ' + f'"cat": "cpu_to_{device_name}", ' + '"args": {}}, ' + ) + # Note: use torch.profiler to get device kernel trace + next_id += 1 + if len(self) > 0: + # remove trailing whitespace and comma + f.seek(f.tell() - 2, os.SEEK_SET) + f.truncate() + f.write("]") + + def supported_export_stacks_metrics(self): + return [ + "self_cpu_time_total", + "self_cuda_time_total", + "self_xpu_time_total", + "self_privateuse1_time_total", + ] + + def export_stacks(self, path: str, metric: str): + if metric not in self.supported_export_stacks_metrics(): + raise ValueError( + "metric should be one of: " + + str(self.supported_export_stacks_metrics()) + ) + translate_table = str.maketrans(" ;\t\n", "____") + with open(path, "w") as f: + for evt in self: + if evt.stack and len(evt.stack) > 0: + metric_value = getattr( + evt, + metric.replace("cuda", "device") + .replace("xpu", "device") + .replace("privateuse1", "device"), + ) + if int(metric_value) > 0: + stack_str = "" + for entry in reversed(evt.stack): + stack_str += entry.translate(translate_table) + stack_str += ";" + stack_str = stack_str[:-1] + " " + str(int(metric_value)) + f.write(stack_str + "\n") + + def key_averages( + self, + group_by_input_shapes=False, + group_by_stack_n=0, + group_by_overload_name=False, + include_python_functions=False, + ): + """Averages all function events over their keys. + + Args: + group_by_input_shapes: group entries by + (event name, input shapes) rather than just event name. + This is useful to see which input shapes contribute to the runtime + the most and may help with size-specific optimizations or + choosing the best candidates for quantization (aka fitting a roof line) + + group_by_stack_n: group by top n stack trace entries + + group_by_overload_name: Differentiate operators by their overload name e.g. aten::add.Tensor + and aten::add.out will be aggregated separately + + include_python_functions: include Python function events (e.g. individual + Python callsite entries captured with ``with_stack=True``) in the + averages. By default these are excluded because they tend to appear as + misleading hotspots (e.g. ``threading.py: wait``) that obscure the + real operator-level breakdown. Set to ``True`` to restore the raw + per-callsite view. + + Returns: + An EventList containing FunctionEventAvg objects. + """ + if not self._tree_built: + raise AssertionError( + "Expected tree to be built before calling key_averages" + ) + stats: dict[tuple[str, ...], FunctionEventAvg] = defaultdict(FunctionEventAvg) + + def get_key( + event, group_by_input_shapes, group_by_stack_n, group_by_overload_name + ) -> tuple[str, ...]: + key = [ + str(event.key), + str(event.node_id), + str(event.device_type), + str(event.is_legacy), + str(event.is_user_annotation), + ] + if group_by_overload_name: + key.append(evt.overload_name) + if group_by_input_shapes: + key.append(str(event.input_shapes)) + if group_by_stack_n > 0: + key += event.stack[:group_by_stack_n] + return tuple(key) + + for evt in self: + if evt.is_python_function and not include_python_functions: + continue + stats[ + get_key( + evt, group_by_input_shapes, group_by_stack_n, group_by_overload_name + ) + ].add(evt) + + avg_list = EventList( + stats.values(), + use_device=self._use_device, + profile_memory=self._profile_memory, + with_flops=self._with_flops, + ) + for evt in avg_list: + evt.stack = evt.stack[:group_by_stack_n] + if not group_by_input_shapes: + evt.input_shapes = "" + if not group_by_overload_name: + evt.overload_name = "" + return avg_list + + def total_average(self): + """Compute aggregate statistics across all events. + + Accumulates statistics from all events into a single FunctionEventAvg object. + This is primarily useful for computing total metrics (total CPU time, total + memory usage, etc.) across the entire profiling session, regardless of + operation type. + + Note: + This sums up times and counts across ALL different operations, so the + "average" metrics (like cpu_time) represent the average time per operation + call across the entire session, mixing all operation types together. + For per-operation averages, use :meth:`key_averages` instead. + + Returns: + FunctionEventAvg: A single aggregate object with key="Total" containing + accumulated statistics. + + """ + total_stat = FunctionEventAvg() + for evt in self: + total_stat += evt + total_stat.key = None + total_stat.key = "Total" + return total_stat + + +def _format_time(time_us): + """Define how to format time in FunctionEvent.""" + US_IN_SECOND = 1000.0 * 1000.0 + US_IN_MS = 1000.0 + if time_us >= US_IN_SECOND: + return f"{time_us / US_IN_SECOND:.3f}s" + if time_us >= US_IN_MS: + return f"{time_us / US_IN_MS:.3f}ms" + return f"{time_us:.3f}us" + + +def _format_time_share(time_us, total_time_us): + """Define how to format time in FunctionEvent.""" + if total_time_us == 0: + if time_us != 0: + raise AssertionError(f"Expected time_us == 0 but got {time_us}") + return "NaN" + return f"{time_us * 100.0 / total_time_us:.2f}%" + + +def _format_memory(nbytes): + """Return a formatted memory size string.""" + KB = 1024 + MB = 1024 * KB + GB = 1024 * MB + if abs(nbytes) >= GB: + return f"{nbytes * 1.0 / GB:.2f} GB" + elif abs(nbytes) >= MB: + return f"{nbytes * 1.0 / MB:.2f} MB" + elif abs(nbytes) >= KB: + return f"{nbytes * 1.0 / KB:.2f} KB" + else: + return str(nbytes) + " B" + + +def _attr_formatter(name): + return property(lambda self: _format_time(getattr(self, name))) + + +class FormattedTimesMixin: + """Helpers for FunctionEvent and FunctionEventAvg. + + The subclass should define `*_time_total` and `count` attributes. + """ + + cpu_time_str = _attr_formatter("cpu_time") + device_time_str = _attr_formatter("device_time") + cpu_time_total_str = _attr_formatter("cpu_time_total") + device_time_total_str = _attr_formatter("device_time_total") + self_cpu_time_total_str = _attr_formatter("self_cpu_time_total") + self_device_time_total_str = _attr_formatter("self_device_time_total") + + @property + def cpu_time(self): + return 0.0 if self.count == 0 else 1.0 * self.cpu_time_total / self.count # type: ignore[attr-defined] + + @property + def device_time(self): + return 0.0 if self.count == 0 else 1.0 * self.device_time_total / self.count # type: ignore[attr-defined] + + @property + @deprecated( + "`cuda_time` is deprecated, please use `device_time` instead.", + category=FutureWarning, + ) + def cuda_time(self): # To be deprecated + return self.device_time + + +class Interval: + def __init__(self, start, end): + self.start = start + self.end = end + + def elapsed_us(self): + r""" + Returns the length of the interval + """ + return self.end - self.start + + +Kernel = namedtuple("Kernel", ["name", "device", "duration"]) + + +class EventMetadata(NamedTuple): + # Kernel fields + registers_per_thread: int | None + shared_memory: int | None + grid: list[int] | None + block: list[int] | None + priority: int | None + blocks_per_sm: float | None + warps_per_sm: float | None + occupancy: dict[str, Any] | None + est_occupancy_pct: float | None + queued: int | None + graph_id: int | None + graph_node_id: int | None + stream: int | None + context: int | None + channel: int | None + channel_type: int | None + # Memory fields + bytes: int | None + bandwidth_gb_s: float | None + # NCCL fields + collective_name: str | None + dtype: str | None + in_msg_nelems: int | None + out_msg_nelems: int | None + in_split_size: str | None + out_split_size: str | None + global_rank_start: int | None + global_rank_stride: int | None + group_size: int | None + process_group_name: str | None + process_group_desc: str | None + group_ranks: str | None + rank: int | None + src_rank: int | None + dst_rank: int | None + seq: int | None + is_async: bool | None + + +def _to_str(v: str) -> str: + return v.strip('"') + + +def _to_bool(v: str) -> bool: + return v in ("1", "true") + + +# Kineto key → (EventMetadata field name, converter from string) +_EVENT_METADATA_KEYS: dict[str, tuple[str, Callable[[str], Any]]] = { + "registers per thread": ("registers_per_thread", int), + "shared memory": ("shared_memory", int), + "grid": ("grid", json.loads), # list[int] + "block": ("block", json.loads), # list[int] + "priority": ("priority", int), + "blocks per SM": ("blocks_per_sm", float), + "warps per SM": ("warps_per_sm", float), + "est. achieved occupancy %": ("est_occupancy_pct", float), + "occupancy": ("occupancy", json.loads), # dict[str, Any] + "queued": ("queued", int), + "graph id": ("graph_id", int), + "graph node id": ("graph_node_id", int), + "stream": ("stream", int), + "context": ("context", int), + "channel": ("channel", int), + "channel_type": ("channel_type", int), + "bytes": ("bytes", int), + "memory bandwidth (GB/s)": ("bandwidth_gb_s", float), + "Collective name": ("collective_name", _to_str), + "dtype": ("dtype", _to_str), + "In msg nelems": ("in_msg_nelems", int), + "Out msg nelems": ("out_msg_nelems", int), + "In split size": ("in_split_size", _to_str), + "Out split size": ("out_split_size", _to_str), + "Global rank start": ("global_rank_start", int), + "Global rank stride": ("global_rank_stride", int), + "Group size": ("group_size", int), + "Process Group Name": ("process_group_name", _to_str), + "Process Group Description": ("process_group_desc", _to_str), + "Process Group Ranks": ("group_ranks", _to_str), + "Rank": ("rank", int), + "Src Rank": ("src_rank", int), + "Dst Rank": ("dst_rank", int), + "Seq": ("seq", int), + "Is asynchronized op": ("is_async", _to_bool), +} + + +def _build_metadata(extra_meta): + fields: dict[str, Any] = {} + any_populated = False + for kineto_key, (field_name, convert) in _EVENT_METADATA_KEYS.items(): + v = extra_meta.get(kineto_key) + if v is not None: + fields[field_name] = convert(v) + any_populated = True + else: + fields[field_name] = None + return EventMetadata(**fields) if any_populated else None + + +class FunctionEvent(FormattedTimesMixin): + """Profiling information about a single function. + + FunctionEvent records the execution of a single operation during profiling. + These events are obtained from the profiler/kineto and contain detailed + timing and memory usage information. + + .. note:: + FunctionEvent objects are typically created by the profiler/kineto and should not + be instantiated directly by users. Access them through the profiler's output. + + Attributes: + id (int): Unique identifier for this event. + node_id (int): Node identifier for distributed profiling (-1 if not applicable). + name (str): Name of the profiled function/operator. + overload_name (str): Overload name for the operator (requires _ExperimentalConfig(capture_overload_names=True) set). + trace_name (str): Same as name, just changes ProfilerStep* to ProfilerStep# + time_range (Interval): Time interval containing start and end timestamps in microseconds. + thread (int): Thread ID where the operation started. + fwd_thread (int): Thread ID of the corresponding forward operation. + kernels (List[Kernel]): List of device kernels launched by this operation. + count (int): Number of times this event was called (usually 1). + cpu_children (List[FunctionEvent]): Direct CPU child operations. + cpu_parent (FunctionEvent): Direct CPU parent operation. + input_shapes (List[List[int]]): Shapes of input tensors (requires record_shapes=True). + For plain tensor inputs, each entry is a list of dimensions (e.g. ``[16, 16]``). + TensorList inputs are represented as an empty list ``[]``; use + ``structured_input_shapes`` to get per-element shapes for TensorList inputs. + concrete_inputs (List[Any]): Concrete input values (requires record_shapes=true). + kwinputs (Dict[str, Any]): Keyword arguments (requires record_shapes=true). + stack (List[str]): Python stack trace where the operation was called (requires with_stack=true). + scope (int): at::RecordScope identifier (0=forward, 1=backward, etc.). + use_device (str): Device type being profiled ("cuda", "xpu", etc.). + cpu_memory_usage (int): CPU memory allocated in bytes. + device_memory_usage (int): Device memory allocated in bytes. + is_async (bool): Whether this is an asynchronous operation. + is_remote (bool): Whether this operation occurred on a remote node. + sequence_nr (int): Sequence number for autograd operations. + device_type (DeviceType): Type of device (CPU, CUDA, XPU, PrivateUse1, etc.). + device_index (int): Index of the device (e.g., GPU 0, 1, 2). + device_resource_id (int): Resource ID on the device (ie. stream ID). + is_legacy (bool): Whether this is from the legacy profiler. + flops (int): Estimated floating point operations. + is_user_annotation (bool): Whether this is a user-annotated region. + metadata_json (str): Deprecated. Use event_metadata instead. + event_metadata (EventMetadata): Additional metadata in structured format. + structured_input_shapes (List[List[int] | List[List[int]]]): Like ``input_shapes`` + but distinguishes TensorList inputs. Plain tensor inputs are ``List[int]``; + TensorList inputs are ``List[List[int]]`` containing one shape per tensor in the list. + Matches the ``"Input Dims"`` field in the Chrome trace JSON. + structured_input_strides (List[List[int] | List[List[int]]]): Strides of input + tensors in the same format as ``structured_input_shapes`` (requires + record_shapes=True). + + Properties: + cpu_time_total (float): Total CPU time in microseconds. + device_time_total (float): Total device (CUDA/XPU/etc) time in microseconds. + self_cpu_time_total (float): CPU time excluding child operations. + self_device_time_total (float): Device time excluding child operations. + self_cpu_memory_usage (int): CPU memory usage excluding child operations. + self_device_memory_usage (int): Device memory usage excluding child operations. + cpu_time (float): Average CPU time per call. + device_time (float): Average device time per call. + key (str): Key used for grouping events (usually same as name). + + See Also: + - :class:`torch.profiler.profile`: Context manager for profiling + - :class:`EventList`: List container for FunctionEvent objects with helper methods + - :class:`FunctionEventAvg`: Averaged statistics over multiple FunctionEvent objects + """ + + def __init__( + self, + id, + name, + thread, + start_us, + end_us, + overload_name=None, + fwd_thread=None, + input_shapes=None, + stack=None, + scope=0, + use_device=None, + cpu_memory_usage=0, + device_memory_usage=0, + is_async=False, + is_remote=False, + sequence_nr=-1, + node_id=-1, + device_type=DeviceType.CPU, + device_index=0, + device_resource_id=None, + is_legacy=False, + flops=None, + trace_name=None, + concrete_inputs=None, + kwinputs=None, + is_user_annotation=False, + is_python_function=False, + activity_type=None, + metadata_json=None, + flow_id=None, + flow_type=None, + flow_start=None, + external_id=0, + linked_correlation_id=0, + extra_meta=None, + structured_input_shapes=None, + structured_input_strides=None, + input_dtypes=None, + python_id=-1, + python_parent_id=-1, + python_module_id=-1, + ): + self.id: int = id + self.node_id: int = node_id + self.name: str = name + # pyrefly: ignore [bad-assignment] + self.overload_name: str = overload_name + # pyrefly: ignore [bad-assignment] + self.trace_name: str = trace_name + self.time_range: Interval = Interval(start_us, end_us) + self.thread: int = thread + self.fwd_thread: int | None = fwd_thread + self.kernels: list[Kernel] = [] + self.count: int = 1 + self.cpu_children: list[FunctionEvent] = [] + self.cpu_parent: FunctionEvent | None = None + # pyrefly: ignore [bad-assignment] + self.input_shapes: tuple[int, ...] = input_shapes + # pyrefly: ignore [bad-assignment] + self.concrete_inputs: list[Any] = concrete_inputs + # pyrefly: ignore [bad-assignment] + self.kwinputs: dict[str, Any] = kwinputs + # pyrefly: ignore [bad-assignment] + self.stack: list = stack + self.scope: int = scope + self.use_device: str | None = use_device + self.cpu_memory_usage: int = cpu_memory_usage + self.device_memory_usage: int = device_memory_usage + self.is_async: bool = is_async + self.is_remote: bool = is_remote + self.sequence_nr: int = sequence_nr + self.device_type: DeviceType = device_type + self.device_index: int = device_index + self.device_resource_id: int = ( + thread if device_resource_id is None else device_resource_id + ) + self.is_legacy: bool = is_legacy + self.flops: int | None = flops + self.is_user_annotation: bool | None = is_user_annotation + self.is_python_function: bool = is_python_function + self.activity_type: str | None = activity_type + self.self_cpu_percent = -1 + self.total_cpu_percent = -1 + self.total_device_percent = -1 + self._metadata_json = metadata_json + self.flow_id: int | None = flow_id + self.flow_type: int | None = flow_type + self.flow_start: bool | None = flow_start + self.external_id: int = external_id + self.linked_correlation_id: int = linked_correlation_id + self.event_metadata: EventMetadata | None = ( + _build_metadata(extra_meta) if extra_meta else None + ) + # pyrefly: ignore [bad-assignment] + self.structured_input_shapes: list = structured_input_shapes + # pyrefly: ignore [bad-assignment] + self.structured_input_strides: list = structured_input_strides + # pyrefly: ignore [bad-assignment] + self.input_dtypes: list[str] = input_dtypes + self.python_id: int = python_id + self.python_parent_id: int = python_parent_id + self.python_module_id: int = python_module_id + + def append_kernel(self, name, device, duration): + if self.device_type != DeviceType.CPU: + raise AssertionError("Expected device_type to be CPU") + self.kernels.append(Kernel(name, device, duration)) + + def append_cpu_child(self, child): + """Append a CPU child of type FunctionEvent. + + One is supposed to append only direct children to the event to have + correct self cpu time being reported. + """ + if self.device_type != DeviceType.CPU: + raise AssertionError("Expected device_type to be CPU") + if not isinstance(child, FunctionEvent): + raise AssertionError("Expected child to be a FunctionEvent") + if child.device_type != DeviceType.CPU: + raise AssertionError("Expected child device_type to be CPU") + self.cpu_children.append(child) + + def set_cpu_parent(self, parent): + """Set the immediate CPU parent of type FunctionEvent. + + One profiling FunctionEvent should have only one CPU parent such that + the child's range interval is completely inside the parent's. We use + this connection to determine the event is from top-level op or not. + """ + if self.device_type != DeviceType.CPU: + raise AssertionError("Expected device_type to be CPU") + if not isinstance(parent, FunctionEvent): + raise AssertionError("Expected parent to be a FunctionEvent") + if parent.device_type != DeviceType.CPU: + raise AssertionError("Expected parent device_type to be CPU") + self.cpu_parent = parent + + # Note: async events don't have children, are not used when computing 'self' + # metrics of other events, have only total cpu time + @property + def self_cpu_memory_usage(self): + if self.is_async or self.device_type != DeviceType.CPU: + return 0 + return self.cpu_memory_usage - sum( + child.cpu_memory_usage for child in self.cpu_children + ) + + @property + def self_device_memory_usage(self): + if self.is_async or self.device_type != DeviceType.CPU: + return 0 + return self.device_memory_usage - sum( + child.device_memory_usage for child in self.cpu_children + ) + + @property + @deprecated( + "`metadata_json` is deprecated. Use `event_metadata` instead.", + category=FutureWarning, + ) + def metadata_json(self): + return self._metadata_json + + @property + @deprecated( + "`self_cuda_memory_usage` is deprecated. Use `self_device_memory_usage` instead.", + category=FutureWarning, + ) + def self_cuda_memory_usage(self): # To be deprecated + return self.self_device_memory_usage + + @property + def cpu_time_total(self): + if self.device_type == DeviceType.CPU: + return self.time_range.elapsed_us() + else: + return 0 + + @property + def self_cpu_time_total(self): + if self.is_async or self.device_type != DeviceType.CPU: + return 0 + return self.cpu_time_total - sum( + child.cpu_time_total for child in self.cpu_children + ) + + @property + def device_time_total(self): + if self.is_async or not self.use_device: + return 0 + if self.device_type == DeviceType.CPU: + if not self.is_legacy: + # account for the kernels in the children ops + return sum(kinfo.duration for kinfo in self.kernels) + sum( + ch.device_time_total for ch in self.cpu_children + ) + else: + # each legacy cpu events has a single (fake) kernel + return sum(kinfo.duration for kinfo in self.kernels) + else: + if self.device_type not in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.MTIA, + DeviceType.HPU, + DeviceType.XPU, + ]: + raise AssertionError( + f"Expected device_type to be CUDA, PrivateUse1, MTIA, HPU or XPU, but got {self.device_type}" + ) + return self.time_range.elapsed_us() + + @property + @deprecated( + "`cuda_time_total` is deprecated. Use `device_time_total` instead.", + category=FutureWarning, + ) + def cuda_time_total(self): # To be deprecated + return self.device_time_total + + @property + def self_device_time_total(self): + if self.is_async or not self.use_device: + return 0 + if self.device_type == DeviceType.CPU: + return self.device_time_total - sum( + child.device_time_total for child in self.cpu_children + ) + else: + if self.device_type not in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.MTIA, + DeviceType.HPU, + DeviceType.XPU, + ]: + raise AssertionError( + f"Expected device_type to be CUDA, PrivateUse1, MTIA, HPU or XPU, but got {self.device_type}" + ) + return self.device_time_total + + @property + @deprecated( + "`self_cuda_time_total` is deprecated. Use `self_device_time_total` instead.", + category=FutureWarning, + ) + def self_cuda_time_total(self): # To be deprecated + return self.self_device_time_total + + @property + def key(self): + return self.name + + def __repr__(self): + device_name = self.use_device + device_time = self.device_time_str + device_memory_usage = self.device_memory_usage + return ( + f"" + ) + + +class FunctionEventAvg(FormattedTimesMixin): + """Averaged profiling statistics over multiple FunctionEvent objects. + + FunctionEventAvg aggregates statistics from multiple FunctionEvent objects + with the same key (typically same operation name). This is useful for getting + average performance metrics across multiple invocations of the same operation. + + This class is typically created by calling :meth:`EventList.key_averages()` on + a profiler's event list. + + Attributes: + key (str): Grouping key for the events (typically operation name). + count (int): Total number of events aggregated. + node_id (int): Node identifier for distributed profiling (-1 if not applicable). + is_async (bool): Whether the operations are asynchronous. + is_remote (bool): Whether the operations occurred on a remote node. + use_device (str): Device type being profiled ("cuda", "xpu", etc.). + cpu_time_total (int): Accumulated total CPU time in microseconds. + device_time_total (int): Accumulated total device time in microseconds. + self_cpu_time_total (int): Accumulated self CPU time (excluding children) in microseconds. + self_device_time_total (int): Accumulated self device time (excluding children) in microseconds. + input_shapes (List[List[int]]): Input tensor shapes (requires record_shapes=true). + overload_name (str): Operator overload name (requires _ExperimentalConfig(capture_overload_names=True) set). + stack (List[str]): Python stack trace where the operation was called (requires with_stack=true). + scope (int): at::RecordScope identifier (0=forward, 1=backward, etc.). + cpu_memory_usage (int): Accumulated CPU memory usage in bytes. + device_memory_usage (int): Accumulated device memory usage in bytes. + self_cpu_memory_usage (int): Accumulated self CPU memory usage in bytes. + self_device_memory_usage (int): Accumulated self device memory usage in bytes. + cpu_children (List[FunctionEvent]): CPU child events. + cpu_parent (FunctionEvent): CPU parent event. + device_type (DeviceType): Type of device (CPU, CUDA, XPU, PrivateUse1, etc.). + is_legacy (bool): Whether from legacy profiler. + flops (int): Total floating point operations. + is_user_annotation (bool): Whether this is a user-annotated region. + + Properties: + cpu_time (float): Average CPU time per invocation. + device_time (float): Average device time per invocation. + + See Also: + - :class:`EventList.key_averages`: Method that creates FunctionEventAvg objects + - :class:`FunctionEvent`: Individual profiling event + - :class:`EventList`: Container for profiling events + """ + + def __init__(self) -> None: + self.key: str | None = None + self.count: int = 0 + self.node_id: int = 0 + self.is_async: bool = False + self.is_remote: bool = False + self.use_device: str | None = None + self.cpu_time_total: int = 0 + self.device_time_total: int = 0 + self.self_cpu_time_total: int = 0 + self.self_device_time_total: int = 0 + self.input_shapes: list[list[int]] | None = None + self.overload_name: str | None = None + self.stack: list | None = None + self.scope: int | None = None + self.cpu_memory_usage: int = 0 + self.device_memory_usage: int = 0 + self.self_cpu_memory_usage: int = 0 + self.self_device_memory_usage: int = 0 + self.cpu_children: list[FunctionEvent] | None = None + self.cpu_parent: FunctionEvent | None = None + self.device_type: DeviceType = DeviceType.CPU + self.is_legacy: bool = False + self.flops: int = 0 + + def add(self, other): + if self.key is None: + # First function being recorded as part of FunctionEventAvg, propagate + # fields. + self.key = other.key + self.node_id = other.node_id + self.is_async = other.is_async + self.is_remote = other.is_remote + self.cpu_parent = other.cpu_parent + self.cpu_children = other.cpu_children + + self.overload_name = other.overload_name + self.input_shapes = other.input_shapes + self.stack = other.stack + self.scope = other.scope + self.device_type = other.device_type + self.is_legacy = other.is_legacy + self.use_device = other.use_device + self.is_user_annotation = other.is_user_annotation + + if not isinstance(other, (FunctionEvent, FunctionEventAvg)): + raise AssertionError( + "Expected other to be a FunctionEvent or FunctionEventAvg" + ) + if other.key != self.key: + raise AssertionError( + f"Expected keys to match, but got {other.key} vs {self.key}" + ) + + self.cpu_time_total += other.cpu_time_total + self.device_time_total += other.device_time_total + self.self_cpu_time_total += other.self_cpu_time_total + self.self_device_time_total += other.self_device_time_total + self.cpu_memory_usage += other.cpu_memory_usage + self.device_memory_usage += other.device_memory_usage + self.self_cpu_memory_usage += other.self_cpu_memory_usage + self.self_device_memory_usage += other.self_device_memory_usage + self.count += other.count + if self.flops is None: + # pyrefly: ignore [bad-assignment] + self.flops = other.flops + elif other.flops is not None: + self.flops += other.flops + return self + + def __iadd__(self, other): + return self.add(other) + + def __repr__(self): + device_name = "cuda" if not self.use_device else self.use_device + self_device_time = self.self_device_time_total_str + device_time = self.device_time_str + device_memory = self.device_memory_usage + return ( + f"" + ) + + +class StringTable(defaultdict): + def __missing__(self, key): + # manage cases like 't' (demangled to 'unsigned short') separately, + # for now simply check the length to avoid unexpected results for + # the short sequences + self[key] = torch._C._demangle(key) if len(key) > 1 else key + return self[key] + + +class MemRecordsAcc: + """Acceleration structure for accessing mem_records in interval.""" + + def __init__(self, mem_records): + self._mem_records = mem_records + self._start_nses: list[int] = [] + self._indices: list[int] = [] + if len(mem_records) > 0: + tmp = sorted([(r[0].start_ns(), i) for i, r in enumerate(mem_records)]) + self._start_nses, self._indices = zip(*tmp) # type: ignore[assignment] + + def in_interval(self, start_ns, end_ns): + r""" + Return all records in the given interval + """ + start_idx = bisect.bisect_left(self._start_nses, start_ns) + end_idx = bisect.bisect_right(self._start_nses, end_ns) + for i in range(start_idx, end_idx): + yield self._mem_records[self._indices[i]] + + +def _filter_stack_entry(entry): + filtered_entries = [ + ("autograd/__init__", "_make_grads"), + ("autograd/__init__", "backward"), + ("torch/tensor", "backward"), + ("_internal/common_utils", "prof_callable"), + ("_internal/common_utils", "prof_func_call"), + ("_internal/common_utils", "prof_meth_call"), + ] + return all(not (f[0] in entry and f[1] in entry) for f in filtered_entries) + + +MEMORY_EVENT_NAME = "[memory]" +OUT_OF_MEMORY_EVENT_NAME = "[OutOfMemory]" + + +def _filter_name(name): + # ignoring the following utility ops + filtered_out_names = [ + MEMORY_EVENT_NAME, # used only for the top-level memory events + OUT_OF_MEMORY_EVENT_NAME, + "profiler::_record_function_enter", + "profiler::_record_function_enter_new", + "profiler::_record_function_exit", + "aten::is_leaf", + "aten::output_nr", + "aten::_version", + ] + return name in filtered_out_names + + +# Demangles and optionally rewrites the provided event name, +# with_wildcard - whether to replace certain numbered event names +# with a wildcard name to aggregate them together in the profiler table +# output +def _rewrite_name(name, with_wildcard=False): + string_table = StringTable() + name = string_table[name] + if with_wildcard: + if name.startswith("ProfilerStep#"): + name = "ProfilerStep*" + return name + + +def _build_table( + events, + sort_by=None, + header=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + with_flops=False, + profile_memory=False, + top_level_events_only=False, + time_unit=None, +): + """Print a summary of events (which can be a list of FunctionEvent or FunctionEventAvg).""" + if len(events) == 0: + return "" + + has_device_time = any(event.self_device_time_total > 0 for event in events) + has_device_mem = any(event.self_device_memory_usage > 0 for event in events) + use_device = events[0].use_device + # Running on PrivateUse1 device with profiler but not enable + # ProfilerActivity.PrivateUse1 can also catch privateuse1 memory usage. + # Here only need to check has_privateuse1_time if not use_device. + if not use_device and has_device_time: + raise RuntimeError("use_device is None, but there is device performance data.") + + has_input_shapes = any( + (event.input_shapes is not None and len(event.input_shapes) > 0) + for event in events + ) + + has_overload_names = any( + (event.overload_name is not None and len(event.overload_name) > 0) + for event in events + ) + + if sort_by is not None: + events = EventList( + sorted( + events, + key=lambda evt: getattr( + evt, + sort_by.replace("cuda", "device") + .replace("xpu", "device") + .replace("privateuse1", "device"), + ), + reverse=True, + ), + use_device=use_device, + profile_memory=profile_memory, + with_flops=with_flops, + ) + + name_column_width = max(len(evt.key) for evt in events) + 4 + if max_name_column_width is not None: + name_column_width = min(name_column_width, max_name_column_width) + + shapes_column_width = max(len(str(evt.input_shapes)) for evt in events) + 4 + if max_shapes_column_width is not None: + shapes_column_width = min(shapes_column_width, max_shapes_column_width) + + DEFAULT_COLUMN_WIDTH = 12 + flops_column_width = DEFAULT_COLUMN_WIDTH + + src_column_width = None + stacks = [ + evt.stack for evt in events if evt.stack is not None and len(evt.stack) > 0 + ] + has_stack = len(stacks) > 0 + if has_stack: + src_column_width = ( + max(max(len(entry) for entry in stack) for stack in stacks) + 4 + ) + if max_src_column_width is not None: + src_column_width = min(src_column_width, max_src_column_width) + + headers = ["Name"] + if has_overload_names: + headers.append("Overload Name") + headers += [ + "Self CPU %", + "Self CPU", + "CPU total %", + "CPU total", + "CPU time avg", + ] + + device_name = use_device.upper() if use_device is not None else "None" + if has_device_time: + headers.extend( + [ + f"Self {device_name}", + f"Self {device_name} %", + f"{device_name} total", + f"{device_name} time avg", + ] + ) + if profile_memory: + headers.extend( + [ + "CPU Mem", + "Self CPU Mem", + ] + ) + if use_device and has_device_mem: + headers.extend( + [ + f"{device_name} Mem", + f"Self {device_name} Mem", + ] + ) + headers.append("# of Calls") + # Only append Node ID if any event has a valid (>= 0) Node ID + append_node_id = any(evt.node_id != -1 for evt in events) + if append_node_id: + headers.append("Node ID") + + SPACING_SIZE = 2 + row_format_lst = [""] + header_sep_lst = [""] + line_length_lst = [-SPACING_SIZE] + + def add_column(padding, text_dir=">"): + row_format_lst[0] += ( + "{: " + text_dir + str(padding) + "}" + (" " * SPACING_SIZE) + ) + header_sep_lst[0] += "-" * padding + (" " * SPACING_SIZE) + line_length_lst[0] += padding + SPACING_SIZE + + def auto_scale_flops(flops): + flop_headers = [ + "FLOPs", + "KFLOPs", + "MFLOPs", + "GFLOPs", + "TFLOPs", + "PFLOPs", + ] + if flops <= 0: + raise AssertionError(f"Expected flops to be positive, but got {flops}") + # pyrefly: ignore [no-matching-overload] + log_flops = max(0, min(math.log10(flops) / 3, float(len(flop_headers) - 1))) + if not (log_flops >= 0 and log_flops < len(flop_headers)): + raise AssertionError( + f"Expected log_flops to be in range [0, {len(flop_headers)}), but got {log_flops}" + ) + return (pow(10, (math.floor(log_flops) * -3.0)), flop_headers[int(log_flops)]) + + add_column(name_column_width) + if has_overload_names: + add_column(name_column_width) + for _ in headers[1 + has_overload_names :]: + add_column(DEFAULT_COLUMN_WIDTH) + + if has_input_shapes: + headers.append("Input Shapes") + add_column(shapes_column_width) + + if has_stack: + headers.append("Source Location") + add_column(src_column_width, text_dir="<") + + if with_flops: + # Auto-scaling of flops header + raw_flops = [evt.flops for evt in events if evt.flops > 0] + if len(raw_flops) != 0: + (flops_scale, flops_header) = auto_scale_flops(min(raw_flops)) + headers.append(f"Total {flops_header}") + add_column(flops_column_width) + else: + with_flops = False # can't find any valid flops + + row_format = row_format_lst[0] + header_sep = header_sep_lst[0] + line_length = line_length_lst[0] + add_column = None # type: ignore[assignment] + + result = [] + + def append(s): + result.append(s) + result.append("\n") # Yes, newline after the end as well + + sum_self_cpu_time_total = 0 + sum_self_device_time_total = 0 + for evt in events: + sum_self_cpu_time_total += evt.self_cpu_time_total + if evt.device_type == DeviceType.CPU and evt.is_legacy: + # in legacy profiler, kernel info is stored in cpu events + sum_self_device_time_total += evt.self_device_time_total + elif ( + evt.device_type + in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.MTIA, + DeviceType.XPU, + ] + and not evt.is_user_annotation + ): + # in kineto profiler, there're events with the correct device type (e.g. CUDA) + sum_self_device_time_total += evt.self_device_time_total + + # Actual printing + if header is not None: + append("=" * line_length) + append(header) + if top_level_events_only: + append("=" * line_length) + append("This report only display top-level ops statistics") + append(header_sep) + append(row_format.format(*headers)) + + append(header_sep) + + def trim_path(path, src_column_width): + if len(path) > src_column_width: + offset = len(path) - src_column_width + path = path[offset:] + if len(path) > 3: + path = "..." + path[3:] + return path + + def override_time_unit(time_us, default_str, time_unit): + US_IN_SECOND = 1000.0 * 1000.0 + US_IN_MS = 1000.0 + if time_unit == "s": + return f"{time_us / US_IN_SECOND:.3f}s" + elif time_unit == "ms": + return f"{time_us / US_IN_MS:.3f}ms" + elif time_unit == "us": + return f"{time_us:.3f}us" + else: + return default_str + + event_limit = 0 + for evt in events: + if event_limit == row_limit: + break + if top_level_events_only and evt.cpu_parent is not None: + continue + else: + event_limit += 1 + name = evt.key + if max_name_column_width is not None and len(name) >= max_name_column_width - 3: + name = name[: (max_name_column_width - 3)] + "..." + + evt.self_cpu_percent = _format_time_share( + evt.self_cpu_time_total, sum_self_cpu_time_total + ) + evt.total_cpu_percent = ( + _format_time_share(evt.cpu_time_total, sum_self_cpu_time_total) + if not evt.is_async + else 0 + ) + + row_values = [name] + if has_overload_names: + overload_name = evt.overload_name + if ( + max_name_column_width is not None + and len(overload_name) >= max_name_column_width - 3 + ): + overload_name = overload_name[: (max_name_column_width - 3)] + "..." + row_values += [overload_name] + row_values += [ + # Self CPU total %, 0 for async events. + evt.self_cpu_percent, + override_time_unit( + evt.self_cpu_time_total, evt.self_cpu_time_total_str, time_unit + ), # Self CPU total + # CPU total %, 0 for async events. + evt.total_cpu_percent, + override_time_unit( + evt.cpu_time_total, evt.cpu_time_total_str, time_unit + ), # CPU total + override_time_unit( + evt.cpu_time, evt.cpu_time_str, time_unit + ), # CPU time avg + ] + if has_device_time: + evt.total_device_percent = _format_time_share( + evt.self_device_time_total, sum_self_device_time_total + ) + row_values.extend( + [ + override_time_unit( + evt.self_device_time_total, + evt.self_device_time_total_str, + time_unit, + ), + # device time total % + evt.total_device_percent, + override_time_unit( + evt.device_time_total, evt.device_time_total_str, time_unit + ), + override_time_unit( + evt.device_time, evt.device_time_str, time_unit + ), # device time avg + ] + ) + if profile_memory: + row_values.extend( + [ + # CPU Mem Total + _format_memory(evt.cpu_memory_usage), + # Self CPU Mem Total + _format_memory(evt.self_cpu_memory_usage), + ] + ) + if use_device and has_device_mem: + row_values.extend( + [ + # Device Mem Total + _format_memory(evt.device_memory_usage), + # Self Device Mem Total + _format_memory(evt.self_device_memory_usage), + ] + ) + row_values.append( + evt.count, # Number of calls + ) + + if append_node_id: + row_values.append(evt.node_id) + if has_input_shapes: + row_values.append(str(evt.input_shapes)[:shapes_column_width]) + if with_flops: + if evt.flops <= 0: + row_values.append("--") + else: + row_values.append(f"{evt.flops * flops_scale:8.3f}") # type: ignore[possibly-undefined] + if has_stack: + src_field = "" + if len(evt.stack) > 0: + src_field = trim_path(evt.stack[0], src_column_width) + row_values.append(src_field) + append(row_format.format(*row_values)) + + if has_stack: + empty_headers = [""] * (len(headers) - 1) + for entry in evt.stack[1:]: + append( + row_format.format( + *(empty_headers + [trim_path(entry, src_column_width)]) + ) + ) + empty_headers.append("") + append(row_format.format(*empty_headers)) + + append(header_sep) + append( + f"Self CPU time total: {override_time_unit(sum_self_cpu_time_total, _format_time(sum_self_cpu_time_total), time_unit)}" + ) + if has_device_time: + append( + f"Self {use_device.upper() if use_device is not None else 'None'} " + f"time total: {override_time_unit(sum_self_device_time_total, _format_time(sum_self_device_time_total), time_unit)}" + ) + return "".join(result) + + +# Collect all events with stack traces and format them canonically +def _canonicalize_profiler_events(events): + """ + Extract and format all events with stack traces in a canonical way + for deterministic testing. + """ + events_with_traces = [] + + for event in events: + # Extract relevant fields + event_name = event.get("name", "") + node_name = event["args"].get("node_name", "") + stack_trace = event["args"].get("stack_trace", "") + + # Get the last non-empty line of the stack trace that is actual source, + # not a caret-marker line. Python 3.11+ appends "^^^^"/"~~~~" indicator + # lines below the source when a FrameSummary has colno/end_colno set + # (e.g. dynamo-generated stack traces); those lines must be skipped so + # we still surface the source code line in canonicalized output. + lines = [s.strip() for s in stack_trace.split("\n") if s.strip()] + source_lines = [s for s in lines if not set(s).issubset({"^", "~", " "})] + stack_trace = source_lines[-1] if source_lines else "" + + events_with_traces.append( + { + "event_name": event_name[:30], + "node_name": node_name, + "stack_trace": stack_trace, + "start_time": event.get("ts", 0), + } + ) + + # Sort by node_name for deterministic ordering + events_with_traces.sort(key=lambda x: x["start_time"]) + + # Format as a string + lines: list[str] = [] + for evt in events_with_traces: + lines.append( + f"event={evt['event_name']} node={evt['node_name']} stack_trace={evt['stack_trace']}" + ) + + return "\n".join(lines) diff --git a/western_ai/usa/pytorch/torch/autograd/variable.py b/western_ai/usa/pytorch/torch/autograd/variable.py new file mode 100644 index 0000000000000000000000000000000000000000..a81d0af68fd6d692b7f14dd8d0b46e515197131f --- /dev/null +++ b/western_ai/usa/pytorch/torch/autograd/variable.py @@ -0,0 +1,15 @@ +# mypy: allow-untyped-defs +import torch +from torch._C import _ImperativeEngine as ImperativeEngine + + +__all__ = ["VariableMeta", "Variable"] + + +class VariableMeta(type): + def __instancecheck__(cls, other): + return isinstance(other, torch.Tensor) + + +class Variable(torch._C._LegacyVariableBase, metaclass=VariableMeta): # type: ignore[misc] + _execution_engine = ImperativeEngine() diff --git a/western_ai/usa/pytorch/torch/backends/__init__.py b/western_ai/usa/pytorch/torch/backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..753271a8aaffaf19d25eae675ccc1bc1cbce6084 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/__init__.py @@ -0,0 +1,140 @@ +# mypy: allow-untyped-defs +import sys +import types +from contextlib import contextmanager + +import torch + + +# The idea for this parameter is that we forbid bare assignment +# to torch.backends..enabled and friends when running our +# test suite, where it's very easy to forget to undo the change +# later. +__allow_nonbracketed_mutation_flag = True + + +def disable_global_flags(): + global __allow_nonbracketed_mutation_flag + __allow_nonbracketed_mutation_flag = False + + +def flags_frozen(): + return not __allow_nonbracketed_mutation_flag + + +@contextmanager +def __allow_nonbracketed_mutation(): + global __allow_nonbracketed_mutation_flag + old = __allow_nonbracketed_mutation_flag + __allow_nonbracketed_mutation_flag = True + try: + yield + finally: + __allow_nonbracketed_mutation_flag = old + + +class ContextProp: + def __init__(self, getter, setter): + self.getter = getter + self.setter = setter + + def __get__(self, obj, objtype): + return self.getter() + + def __set__(self, obj, val): + if not flags_frozen(): + self.setter(val) + else: + raise RuntimeError( + f"not allowed to set {obj.__name__} flags " + "after disable_global_flags; please use flags() context manager instead" + ) + + +class PropModule(types.ModuleType): + def __init__(self, m, name): + super().__init__(name) + self.m = m + + def __getattr__(self, attr): + return self.m.__getattribute__(attr) + + +class _FP32Precision: + def __init__(self, backend, op): + self.backend = backend + self.op = op + + def __setattr__(self, name, value): + if name == "fp32_precision": + torch._C._set_fp32_precision_setter(self.backend, self.op, value) + elif name in ("backend", "op"): + super().__setattr__(name, value) + else: + raise AttributeError("Unknown attribute " + name) + + def __getattr__(self, name): + if name == "fp32_precision": + return torch._C._get_fp32_precision_getter(self.backend, self.op) + else: + raise AttributeError("Unknown attribute " + name) + + +def set_flags(_fp32_precision="none"): + orig_flags = (torch._C._get_fp32_precision_getter("generic", "all"),) + if _fp32_precision is not None: + torch._C._set_fp32_precision_setter("generic", "all", _fp32_precision) + return orig_flags + + +@contextmanager +def flags(fp32_precision="none"): + with __allow_nonbracketed_mutation(): + orig_flags = set_flags(fp32_precision) + try: + yield + finally: + with __allow_nonbracketed_mutation(): + set_flags(*orig_flags) + + +def _get_fp32_precision_getter(backend, op): + def inner(): + return torch._C._get_fp32_precision_getter(backend, op) + + return inner + + +def _set_fp32_precision_setter(backend, op): + def inner(precision): + return torch._C._set_fp32_precision_setter(backend, op, precision) + + return inner + + +class GenericModule(PropModule): + fp32_precision = ContextProp( + _get_fp32_precision_getter("generic", "all"), + _set_fp32_precision_setter("generic", "all"), + ) + + +sys.modules[__name__] = GenericModule(sys.modules[__name__], __name__) + +from torch.backends import ( + cpu as cpu, + cuda as cuda, + cudnn as cudnn, + cusparselt as cusparselt, + kleidiai as kleidiai, + mha as mha, + miopen as miopen, + mkl as mkl, + mkldnn as mkldnn, + mps as mps, + nnpack as nnpack, + openmp as openmp, + opt_einsum as opt_einsum, + python_native as python_native, + quantized as quantized, +) diff --git a/western_ai/usa/pytorch/torch/backends/_coreml/__init__.py b/western_ai/usa/pytorch/torch/backends/_coreml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/torch/backends/_coreml/preprocess.py b/western_ai/usa/pytorch/torch/backends/_coreml/preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..7edbe976c89e84ff3d9a391546c9d9f6e3cd9a17 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/_coreml/preprocess.py @@ -0,0 +1,154 @@ +# mypy: allow-untyped-defs +import hashlib +import json + +import coremltools as ct # type: ignore[import] +from coremltools.converters.mil.input_types import TensorType # type: ignore[import] +from coremltools.converters.mil.mil import types # type: ignore[import] +from coremltools.models.neural_network import quantization_utils # type: ignore[import] + +import torch + + +CT_METADATA_VERSION = "com.github.apple.coremltools.version" +CT_METADATA_SOURCE = "com.github.apple.coremltools.source" + + +class ScalarType: + Float = 0 + Double = 1 + Int = 2 + Long = 3 + Undefined = 4 + + +# Supported Tensor types in coremltools: +# https://github.com/apple/coremltools/blob/main/coremltools/converters/mil/frontend/torch/converter.py#L28 +torch_to_mil_types = { + ScalarType.Float: types.fp32, + ScalarType.Double: types.fp64, + ScalarType.Int: types.int32, + ScalarType.Long: types.int64, +} + + +class CoreMLComputeUnit: + CPU = "cpuOnly" + CPUAndGPU = "cpuAndGPU" + ALL = "all" + + +class CoreMLQuantizationMode: + LINEAR = "linear" + LINEAR_SYMMETRIC = "linear_symmetric" + NONE = "none" + + +def TensorSpec(shape, dtype=ScalarType.Float): + return (shape, dtype) + + +def CompileSpec( + inputs, + outputs, + backend=CoreMLComputeUnit.CPU, + allow_low_precision=True, + quantization_mode=CoreMLQuantizationMode.NONE, + mlmodel_export_path=None, + convert_to=None, +): + return ( + inputs, + outputs, + backend, + allow_low_precision, + quantization_mode, + mlmodel_export_path, + convert_to, + ) + + +def _check_enumerated_shape(shape): + for s in shape: + if not isinstance(s, (list, tuple)): + return False + return True + + +def _convert_to_mil_type(shape, dtype, name: str): + mil_shape = shape + if _check_enumerated_shape(shape): + mil_shape = ct.EnumeratedShapes(shape) + ml_type = TensorType(shape=mil_shape, dtype=torch_to_mil_types[dtype]) + ml_type.name = name + return ml_type + + +def preprocess(script_module: torch._C.ScriptObject, compile_spec: dict[str, tuple]): + spec = compile_spec["forward"] + ( + input_specs, + output_specs, + backend, + allow_low_precision, + quantization_mode, + mlmodel_export_path, + convert_to, + ) = spec + mil_inputs = [] + inputs = [] + for index, input in enumerate(input_specs): + shape, dtype = input + name = "input_" + str(index) + inputs.append([name, str(dtype), str(shape)]) + ml_type = _convert_to_mil_type(shape, dtype, name) + mil_inputs.append(ml_type) + model = torch.jit.RecursiveScriptModule._construct(script_module, lambda x: None) + mlmodel = ct.convert(model, inputs=mil_inputs, convert_to=convert_to) + + if quantization_mode != CoreMLQuantizationMode.NONE: + quant_model_spec = quantization_utils.quantize_weights( + mlmodel, nbits=8, quantization_mode=quantization_mode + ) + mlmodel = ct.models.MLModel(quant_model_spec) + + spec = mlmodel.get_spec() + if len(spec.description.output) != len(output_specs): # type: ignore[attr-defined] + raise AssertionError( + f"Number of outputs in spec ({len(spec.description.output)}) " # type: ignore[attr-defined] + f"does not match output_specs ({len(output_specs)})" + ) + outputs = [] + for index, output in enumerate(output_specs): + shape, dtype = output + name = spec.description.output[index].name # type: ignore[attr-defined] + outputs.append([name, str(dtype), str(shape)]) + mlmodel = ct.models.model.MLModel(spec) + print(mlmodel) + + if mlmodel_export_path is not None: + print(f"Saving CoreML .mlmodel file to {mlmodel_export_path}") + mlmodel.save(mlmodel_export_path) + + config = { + "spec_ver": str(spec.specificationVersion), # type: ignore[attr-defined] + "backend": backend, + "allow_low_precision": str(allow_low_precision), + } + metadata = { + "coremltool_ver": mlmodel.user_defined_metadata[CT_METADATA_VERSION], + "torch_ver": mlmodel.user_defined_metadata[CT_METADATA_SOURCE], + } + coreml_compile_spec = { + "inputs": inputs, + "outputs": outputs, + "config": config, + "metadata": metadata, + } + mlmodel = spec.SerializeToString() # type: ignore[attr-defined] + + return { + "model": mlmodel, + "hash": str(hashlib.sha256(mlmodel).hexdigest()), + "extra": json.dumps(coreml_compile_spec), + } diff --git a/western_ai/usa/pytorch/torch/backends/_nnapi/__init__.py b/western_ai/usa/pytorch/torch/backends/_nnapi/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/torch/backends/_nnapi/prepare.py b/western_ai/usa/pytorch/torch/backends/_nnapi/prepare.py new file mode 100644 index 0000000000000000000000000000000000000000..12e4eb8458a20d209bb9272f55fb4fe38db93819 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/_nnapi/prepare.py @@ -0,0 +1,208 @@ +# mypy: allow-untyped-decorators +# mypy: allow-untyped-defs + +from typing import Optional + +import torch +from torch.backends._nnapi.serializer import _NnapiSerializer + + +ANEURALNETWORKS_PREFER_LOW_POWER = 0 +ANEURALNETWORKS_PREFER_FAST_SINGLE_ANSWER = 1 +ANEURALNETWORKS_PREFER_SUSTAINED_SPEED = 2 + + +class NnapiModule(torch.nn.Module): + """Torch Module that wraps an NNAPI Compilation. + + This module handles preparing the weights, initializing the + NNAPI TorchBind object, and adjusting the memory formats + of all inputs and outputs. + """ + + # _nnapi.Compilation is defined + comp: Optional[torch.classes._nnapi.Compilation] # type: ignore[name-defined] # noqa: UP045 + weights: list[torch.Tensor] + out_templates: list[torch.Tensor] + + def __init__( + self, + shape_compute_module: torch.nn.Module, + ser_model: torch.Tensor, + weights: list[torch.Tensor], + inp_mem_fmts: list[int], + out_mem_fmts: list[int], + compilation_preference: int, + relax_f32_to_f16: bool, + ): + super().__init__() + self.shape_compute_module = shape_compute_module + self.ser_model = ser_model + self.weights = weights + self.inp_mem_fmts = inp_mem_fmts + self.out_mem_fmts = out_mem_fmts + self.out_templates = [] + self.comp = None + self.compilation_preference = compilation_preference + self.relax_f32_to_f16 = relax_f32_to_f16 + + @torch.jit.export + def init(self, args: list[torch.Tensor]): + if self.comp is not None: + raise AssertionError("comp must be None before initialization") + self.out_templates = self.shape_compute_module.prepare(self.ser_model, args) # type: ignore[operator] + self.weights = [w.contiguous() for w in self.weights] + comp = torch.classes._nnapi.Compilation() + comp.init2( + self.ser_model, + self.weights, + self.compilation_preference, + self.relax_f32_to_f16, + ) + + self.comp = comp + + def forward(self, args: list[torch.Tensor]) -> list[torch.Tensor]: + if self.comp is None: + self.init(args) + comp = self.comp + if comp is None: + raise AssertionError("comp must not be None") + outs = [torch.empty_like(out) for out in self.out_templates] + + if len(args) != len(self.inp_mem_fmts): + raise AssertionError( + f"args length {len(args)} != inp_mem_fmts length {len(self.inp_mem_fmts)}" + ) + fixed_args = [] + for idx in range(len(args)): + fmt = self.inp_mem_fmts[idx] + # These constants match the values in DimOrder in serializer.py + # TODO: See if it's possible to use those directly. + if fmt == 0: + fixed_args.append(args[idx].contiguous()) + elif fmt == 1: + fixed_args.append(args[idx].permute(0, 2, 3, 1).contiguous()) + else: + raise ValueError("Invalid mem_fmt") + comp.run(fixed_args, outs) + if len(outs) != len(self.out_mem_fmts): + raise AssertionError( + f"outs length {len(outs)} != out_mem_fmts length {len(self.out_mem_fmts)}" + ) + for idx in range(len(self.out_templates)): + fmt = self.out_mem_fmts[idx] + # These constants match the values in DimOrder in serializer.py + # TODO: See if it's possible to use those directly. + if fmt in (0, 2): + pass + elif fmt == 1: + outs[idx] = outs[idx].permute(0, 3, 1, 2) + else: + raise ValueError("Invalid mem_fmt") + return outs + + +def convert_model_to_nnapi( + model, + inputs, + serializer=None, + return_shapes=None, + use_int16_for_qint16=False, + compilation_preference=ANEURALNETWORKS_PREFER_SUSTAINED_SPEED, + relax_f32_to_f16=False, +): + ( + shape_compute_module, + ser_model_tensor, + used_weights, + inp_mem_fmts, + out_mem_fmts, + retval_count, + ) = process_for_nnapi( + model, inputs, serializer, return_shapes, use_int16_for_qint16 + ) + + nnapi_model = NnapiModule( + shape_compute_module, + ser_model_tensor, + used_weights, + inp_mem_fmts, + out_mem_fmts, + compilation_preference, + relax_f32_to_f16, + ) + + class NnapiInterfaceWrapper(torch.nn.Module): + """NNAPI list-ifying and de-list-ifying wrapper. + + NNAPI always expects a list of inputs and provides a list of outputs. + This module allows us to accept inputs as separate arguments. + It returns results as either a single tensor or tuple, + matching the original module. + """ + + def __init__(self, mod): + super().__init__() + self.mod = mod + + wrapper_model_py = NnapiInterfaceWrapper(nnapi_model) + wrapper_model = torch.jit.script(wrapper_model_py) + # TODO: Maybe make these names match the original. + arg_list = ", ".join(f"arg_{idx}" for idx in range(len(inputs))) + if retval_count < 0: + ret_expr = "retvals[0]" + else: + ret_expr = "".join(f"retvals[{idx}], " for idx in range(retval_count)) + wrapper_model.define( + f"def forward(self, {arg_list}):\n" + f" retvals = self.mod([{arg_list}])\n" + f" return {ret_expr}\n" + ) + return wrapper_model + + +def process_for_nnapi( + model, inputs, serializer=None, return_shapes=None, use_int16_for_qint16=False +): + model = torch.jit.freeze(model) + + if isinstance(inputs, torch.Tensor): + inputs = [inputs] + + serializer = serializer or _NnapiSerializer( + config=None, use_int16_for_qint16=use_int16_for_qint16 + ) + ( + ser_model, + used_weights, + inp_mem_fmts, + out_mem_fmts, + shape_compute_lines, + retval_count, + ) = serializer.serialize_model(model, inputs, return_shapes) + ser_model_tensor = torch.tensor(ser_model, dtype=torch.int32) + + # We have to create a new class here every time this function is called + # because module.define adds a method to the *class*, not the instance. + class ShapeComputeModule(torch.nn.Module): + """Code-gen-ed module for tensor shape computation. + + module.prepare will mutate ser_model according to the computed operand + shapes, based on the shapes of args. Returns a list of output templates. + """ + + shape_compute_module = torch.jit.script(ShapeComputeModule()) + real_shape_compute_lines = [ + "def prepare(self, ser_model: torch.Tensor, args: List[torch.Tensor]) -> List[torch.Tensor]:\n", + ] + [f" {line}\n" for line in shape_compute_lines] + shape_compute_module.define("".join(real_shape_compute_lines)) + + return ( + shape_compute_module, + ser_model_tensor, + used_weights, + inp_mem_fmts, + out_mem_fmts, + retval_count, + ) diff --git a/western_ai/usa/pytorch/torch/backends/_nnapi/serializer.py b/western_ai/usa/pytorch/torch/backends/_nnapi/serializer.py new file mode 100644 index 0000000000000000000000000000000000000000..716465cee4ea2c9562b6dcbac83a36f24106e632 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/_nnapi/serializer.py @@ -0,0 +1,2646 @@ +# mypy: allow-untyped-defs +import array +import enum +import functools +import logging +import operator +import struct +import sys +from typing import NamedTuple + +import torch + + +# TODO: Add type annotations +# TODO: Check tensor types for ops + + +LOG = logging.getLogger("nnapi_serialize") + + +class NNAPI_OperandCode: + FLOAT32 = 0 + INT32 = 1 + UINT32 = 2 + TENSOR_FLOAT32 = 3 + TENSOR_INT32 = 4 + TENSOR_QUANT8_ASYMM = 5 + BOOL = 6 + TENSOR_QUANT16_SYMM = 7 + TENSOR_FLOAT16 = 8 + TENSOR_BOOL8 = 9 + FLOAT16 = 10 + TENSOR_QUANT8_SYMM_PER_CHANNEL = 11 + TENSOR_QUANT16_ASYMM = 12 + + +class NNAPI_OperationCode: + ADD = 0 + AVERAGE_POOL_2D = 1 + CONCATENATION = 2 + CONV_2D = 3 + DEPTHWISE_CONV_2D = 4 + DEPTH_TO_SPACE = 5 + DEQUANTIZE = 6 + EMBEDDING_LOOKUP = 7 + FLOOR = 8 + FULLY_CONNECTED = 9 + HASHTABLE_LOOKUP = 10 + L2_NORMALIZATION = 11 + L2_POOL_2D = 12 + LOCAL_RESPONSE_NORMALIZATION = 13 + LOGISTIC = 14 + LSH_PROJECTION = 15 + LSTM = 16 + MAX_POOL_2D = 17 + MUL = 18 + RELU = 19 + RELU1 = 20 + RELU6 = 21 + RESHAPE = 22 + RESIZE_BILINEAR = 23 + RNN = 24 + SOFTMAX = 25 + SPACE_TO_DEPTH = 26 + SVDF = 27 + TANH = 28 + BATCH_TO_SPACE_ND = 29 + DIV = 30 + MEAN = 31 + PAD = 32 + SPACE_TO_BATCH_ND = 33 + SQUEEZE = 34 + STRIDED_SLICE = 35 + SUB = 36 + TRANSPOSE = 37 + ABS = 38 + ARGMAX = 39 + ARGMIN = 40 + AXIS_ALIGNED_BBOX_TRANSFORM = 41 + BIDIRECTIONAL_SEQUENCE_LSTM = 42 + BIDIRECTIONAL_SEQUENCE_RNN = 43 + BOX_WITH_NMS_LIMIT = 44 + CAST = 45 + CHANNEL_SHUFFLE = 46 + DETECTION_POSTPROCESSING = 47 + EQUAL = 48 + EXP = 49 + EXPAND_DIMS = 50 + GATHER = 51 + GENERATE_PROPOSALS = 52 + GREATER = 53 + GREATER_EQUAL = 54 + GROUPED_CONV_2D = 55 + HEATMAP_MAX_KEYPOINT = 56 + INSTANCE_NORMALIZATION = 57 + LESS = 58 + LESS_EQUAL = 59 + LOG = 60 + LOGICAL_AND = 61 + LOGICAL_NOT = 62 + LOGICAL_OR = 63 + LOG_SOFTMAX = 64 + MAXIMUM = 65 + MINIMUM = 66 + NEG = 67 + NOT_EQUAL = 68 + PAD_V2 = 69 + POW = 70 + PRELU = 71 + QUANTIZE = 72 + QUANTIZED_16BIT_LSTM = 73 + RANDOM_MULTINOMIAL = 74 + REDUCE_ALL = 75 + REDUCE_ANY = 76 + REDUCE_MAX = 77 + REDUCE_MIN = 78 + REDUCE_PROD = 79 + REDUCE_SUM = 80 + ROI_ALIGN = 81 + ROI_POOLING = 82 + RSQRT = 83 + SELECT = 84 + SIN = 85 + SLICE = 86 + SPLIT = 87 + SQRT = 88 + TILE = 89 + TOPK_V2 = 90 + TRANSPOSE_CONV_2D = 91 + UNIDIRECTIONAL_SEQUENCE_LSTM = 92 + UNIDIRECTIONAL_SEQUENCE_RNN = 93 + RESIZE_NEAREST_NEIGHBOR = 94 + + +class NNAPI_FuseCode: + FUSED_NONE = 0 + FUSED_RELU = 1 + FUSED_RELU1 = 2 + FUSED_RELU6 = 3 + + +class OperandValueSourceType: + IMMEDIATE = 0 + NUMBERED_BUFFER = 2 + NUMBERED_MEMORY = 3 + + +# Scalar types that appear explicitly in models. +# These must be kept in sync with +# AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS. +# TODO: Expose these directly to Python to avoid maintaining this list. +class TorchScalarTypes(enum.Enum): + QUINT8 = 13 + + +def approx_equal(lhs, rhs, tolerance=1e-6): + return abs(lhs - rhs) <= tolerance * min(lhs, rhs) + + +def tensor_size(op_type, dims): + ITEM_SIZES = { + NNAPI_OperandCode.TENSOR_FLOAT32: 4, + NNAPI_OperandCode.TENSOR_INT32: 4, + NNAPI_OperandCode.TENSOR_QUANT8_ASYMM: 1, + NNAPI_OperandCode.TENSOR_QUANT16_SYMM: 2, + NNAPI_OperandCode.TENSOR_QUANT16_ASYMM: 2, + } + size = ITEM_SIZES[op_type] + for d in dims: + size *= d + return size + + +def change_element(tup, index, value): + ls = list(tup) + ls[index] = value + return tuple(ls) + + +class ConvPoolArgs2d(NamedTuple): + """Configuration arguments for a convolution.""" + + kernel_h: int + kernel_w: int + stride_h: int + stride_w: int + pad_t: int + pad_b: int + pad_l: int + pad_r: int + dilation_h: int + dilation_w: int + group: int + + +class DimOrder(enum.Enum): + PRESUMED_CONTIGUOUS = 0 + CHANNELS_LAST = 1 + SCALAR_OR_VECTOR = 2 + UNKNOWN_CONSTANT = 999 + + +class Operand(NamedTuple): + """Representation of an NNAPI operand.""" + + # NNAPI operand type. One of NNAPI_OperandCode. + # TODO: Make this an enum. + op_type: int + + # This is always the PyTorch shape, which is NCHW for feature maps. + # The actual NNAPI operand might have a transposed shape. + # we use 0 for load time dynamic shapes & -1 for runtime dynamic shapes + shape: tuple[int, ...] + + # Specifies how the shape of the operand that we define in NNAPI + # relates to the shape we track above. + # - PRESUMED_CONTIGUOUS: physical NNAPI operand will exactly match + # the shape of the PyTorch tensor. + # - CHANNELS_LAST: The PyTorch tensor is expected to be NCHW, and + # the NNAPI operand will be represented explicitly as NHWC. + dim_order: DimOrder + + # Quantization params + scale: float + zero_point: int + + def use_nchw(self): + if self.dim_order is DimOrder.PRESUMED_CONTIGUOUS: + return True + if self.dim_order is DimOrder.CHANNELS_LAST: + return False + raise Exception("Unknown dim order") # noqa: TRY002 + + +def broadcast_shapes(shape1, shape2): + if len(shape1) <= 0: + raise AssertionError(f"shape1 must have length > 0, got {len(shape1)}") + if len(shape2) <= 0: + raise AssertionError(f"shape2 must have length > 0, got {len(shape2)}") + s1 = list(shape1) + s2 = list(shape2) + # TODO: Support non-equal-rank broadcast where semantics match. + # This can be tricky for NHWC tensors because dimension orders + # don't match between PT and NNAPI, even though semantics match. + if len(s1) > len(s2): + # s2 = [1] * (len(s1) - len(s2)) + s2 + raise Exception( # noqa: TRY002 + "Non-equal-rank broadcast is not supported yet." + ) + if len(s2) > len(s1): + # s3 = [1] * (len(s2) - len(s1)) + s1 + raise Exception( # noqa: TRY002 + "Non-equal-rank broadcast is not supported yet." + ) + ret = [] + for d1, d2 in zip(s1, s2): + if d1 == 1: + ret.append(d2) + elif d2 == 1: + ret.append(d1) + elif d1 == d2: + ret.append(d1) + else: + raise Exception( # noqa: TRY002 + f"Cannot broadcast shapes: {shape1} and {shape2}" + ) + return tuple(ret) + + +def get_conv_pool_shape(image_shape, args, out_ch, transpose): + batch, _in_c, in_h, in_w = image_shape + + # TODO: Handle dilation + if args.dilation_h != 1 or args.dilation_w != 1: + raise Exception("Dilation not supported yet.") # noqa: TRY002 + + if transpose: + out_h = (in_h - 1) * args.stride_h + args.kernel_h - args.pad_t - args.pad_b + out_w = (in_w - 1) * args.stride_w + args.kernel_w - args.pad_l - args.pad_l + else: + out_h = (in_h - args.kernel_h + args.pad_t + args.pad_b) // args.stride_h + 1 + out_w = (in_w - args.kernel_w + args.pad_l + args.pad_r) // args.stride_w + 1 + + # Handle variable-sized tensors. + if in_h == 0: + out_h = 0 + if in_w == 0: + out_w = 0 + + out_shape = (batch, out_ch, out_h, out_w) + return out_shape + + +def fix_shape(shape, dim_order): + # Return the actual shape that an operand should have in NNAPI, + # given a PyTorch shape and dimension order. This is where we + # convert from PyTorch's "always NCHW" shape to explicit NHWC. + if dim_order is DimOrder.PRESUMED_CONTIGUOUS: + return shape + if dim_order is DimOrder.CHANNELS_LAST: + return tuple([shape[0]] + list(shape[2:]) + [shape[1]]) + if dim_order is DimOrder.SCALAR_OR_VECTOR: + if not (len(shape) == 0 or len(shape) == 1): + raise AssertionError( + f"SCALAR_OR_VECTOR requires len(shape) == 0 or 1, got {len(shape)}" + ) + return shape + if dim_order is DimOrder.UNKNOWN_CONSTANT: + # XXX think this through + return shape + raise Exception(f"Bad dim_order: {dim_order!r}.") # noqa: TRY002 + + +def reverse_map_dim(dim_order, d): + # Return the original PyTorch dimension position for a given dimension. + # d should be the dimension that NNAPI will see. + # reverse_map_dim(PRESUMED_CONTIGUOUS, x) == x + # reverse_map_dim(CHANNELS_LAST, 3) == 1 + if dim_order in (DimOrder.PRESUMED_CONTIGUOUS, DimOrder.SCALAR_OR_VECTOR): + return d + if dim_order is DimOrder.CHANNELS_LAST: + return [0, 2, 3, 1][d] + raise AssertionError(f"expected DimOrder.CHANNELS_LAST, got {dim_order}") + + +def flex_name(op_id, dim): + # Return the local variable name for the computed flexible size + # for a given op and dimension. + return f"s_{op_id}_{dim}" + + +class _NnapiSerializer: + def __init__(self, config, use_int16_for_qint16=False): + self.operands = [] + self.values = [] + self.operations = [] + self.value_data = [] + self.operation_args = [] + self.inputs = [] + self.outputs = [] + self.flexible_shape_computation_lines = [] + + self.modules = {} + self.constants = {} + self.tensor_sequences = {} + self.jitval_operand_map = {} + self.cached_immediates = {} + self.used_weights = [] + self.weight_offset = 0 + self.use_int16_for_qint16 = use_int16_for_qint16 + + if config is None: + config = {} + + def get_next_operand_id(self): + return len(self.operands) + + # Add a tensor operand corresponding to a JIT Value. + # Returns the NNAPI operand ID. Can be looked up later with + # get_tensor_operand_by_jitval. + def add_tensor_operand(self, jitval, oper): + if not isinstance(oper, Operand): + raise AssertionError(f"expected Operand, got {type(oper)}") + if jitval in self.jitval_operand_map: + raise Exception(f"Duplicate tensor: {jitval!r}") # noqa: TRY002 + + operand_id = self.get_next_operand_id() + self.operands.append(oper) + self.jitval_operand_map[jitval] = operand_id + return operand_id + + # Add a tensor operand that does not correspond to a JIT Value. + # Useful for cases where multiple NNAPI operands are required + # to implement one JIT IR node. Returns the NNAPI operand ID. + def add_anonymous_tensor_operand(self, oper): + if not isinstance(oper, Operand): + raise AssertionError(f"expected Operand, got {type(oper)}") + operand_id = self.get_next_operand_id() + self.operands.append(oper) + return operand_id + + def torch_tensor_to_operand(self, tensor, dim_order): + dtype = str(tensor.dtype).replace("torch.", "") + scale = 0.0 + zero_point = 0 + if dtype == "float32": + op_type = NNAPI_OperandCode.TENSOR_FLOAT32 + elif dtype == "int32": + op_type = NNAPI_OperandCode.TENSOR_INT32 + elif dtype == "quint8": + op_type = NNAPI_OperandCode.TENSOR_QUANT8_ASYMM + scale = tensor.q_scale() + zero_point = tensor.q_zero_point() + elif dtype == "qint32": + op_type = NNAPI_OperandCode.TENSOR_INT32 + scale = tensor.q_scale() + zero_point = tensor.q_zero_point() + if zero_point != 0: + raise AssertionError(f"qint32 zero_point must be 0, got {zero_point}") + elif dtype == "int16": + if self.use_int16_for_qint16: + nnapi_dtype = getattr(tensor, "nnapi_dtype", None) + op_codes = ( + NNAPI_OperandCode.TENSOR_QUANT16_SYMM, + NNAPI_OperandCode.TENSOR_QUANT16_ASYMM, + ) + if nnapi_dtype in op_codes: + op_type = nnapi_dtype + scale = tensor.nnapi_scale + zero_point = tensor.nnapi_zero_point + else: + raise Exception( # noqa: TRY002 + f"`nnapi_type` needs to be one of {op_codes} for `int16`" + ) + else: + raise Exception( # noqa: TRY002 + "`int16` isn't supported. If you're trying to represent NNAPI" + " qint16 with Pytorch int16, set `use_int16_for_qint16 = True`" + ) + else: + raise Exception( # noqa: TRY002 + f"Can't handle input with dtype '{tensor.dtype}'" + ) + return Operand( + shape=tuple(tensor.shape), + # pyrefly: ignore [bad-argument-type] + op_type=op_type, + dim_order=dim_order, + scale=scale, + zero_point=zero_point, + ) + + def add_tensor_operand_for_input(self, arg_idx, jitval, tensor): + dim_order = ( + DimOrder.CHANNELS_LAST + if getattr(tensor, "nnapi_nhwc", False) + else DimOrder.PRESUMED_CONTIGUOUS + ) + toper = self.torch_tensor_to_operand(tensor, dim_order) + operand_id = self.add_tensor_operand(jitval, toper) + self.inputs.append(operand_id) + for dim, size in enumerate(tensor.shape): + if size == 0: + self.compute_operand_shape( + operand_id, dim, f"args[{arg_idx}].shape[{dim}]" + ) + return operand_id + + def add_tensor_operand_for_weight( + self, tensor, dim_order=DimOrder.UNKNOWN_CONSTANT + ): + toper = self.torch_tensor_to_operand(tensor, dim_order) + operand_id = len(self.operands) + self.operands.append(toper) + tsize = tensor_size(toper.op_type, toper.shape) + self.values.append((operand_id, OperandValueSourceType.NUMBERED_BUFFER)) + buf_num = len(self.used_weights) + offset = 0 + self.value_data.append(struct.pack("iii", buf_num, offset, tsize)) + # For NHWC NNAPI op, lay out data in the same dim order by permuting torch tensor + if dim_order == DimOrder.CHANNELS_LAST: + tensor = tensor.permute(0, 2, 3, 1) + self.used_weights.append(tensor) + return operand_id + + def add_immediate_operand(self, code, value, dims): + if not isinstance(dims, tuple): + raise AssertionError(f"dims must be a tuple, got {type(dims)}") + cache_key = (code, value) + if cache_key not in self.cached_immediates: + operand_id = len(self.operands) + self.operands.append(Operand(code, dims, DimOrder.SCALAR_OR_VECTOR, 0.0, 0)) + self.values.append((operand_id, OperandValueSourceType.IMMEDIATE)) + self.value_data.append(value) + self.cached_immediates[cache_key] = operand_id + return self.cached_immediates[cache_key] + + def add_immediate_int_scalar(self, value): + return self.add_immediate_operand( + NNAPI_OperandCode.INT32, struct.pack("i", value), () + ) + + def add_immediate_float_scalar(self, value): + return self.add_immediate_operand( + NNAPI_OperandCode.FLOAT32, struct.pack("f", value), () + ) + + def add_immediate_bool_scalar(self, value): + return self.add_immediate_operand( + NNAPI_OperandCode.BOOL, b"\x01" if value else b"\x00", () + ) + + def add_immediate_int_vector(self, value): + return self.add_immediate_operand( + NNAPI_OperandCode.TENSOR_INT32, + array.array("i", value).tobytes(), + (len(value),), + ) + + def has_operand_for_jitval(self, jitval): + return jitval in self.jitval_operand_map + + def get_tensor_operand_by_jitval(self, jitval): + operand_id = self.jitval_operand_map[jitval] + return (operand_id, self.operands[operand_id]) + + def get_tensor_operand_by_jitval_fixed_size(self, jitval): + op_id, oper = self.get_tensor_operand_by_jitval(jitval) + for s in oper.shape: + if s == 0: + # TODO: Improve this error message, possibly after converting + # many callsites to support flexible size. + raise Exception( # noqa: TRY002 + "Flexible size is not supported for this operand." + ) + if s < 0: + # runtime flex + LOG.warning("Operand %s has runtime flex shape", oper) + return op_id, oper + + def get_tensor_operand_or_constant( + self, jitval, dim_order=DimOrder.PRESUMED_CONTIGUOUS + ): + operand_id = self.jitval_operand_map.get(jitval) + if operand_id is None: + _, value = self.get_constant_value(jitval, "TensorType") + operand_id = self.add_tensor_operand_for_weight(value, dim_order) + return (operand_id, self.operands[operand_id]) + + def get_tensor_operand_for_weight(self, jitval): + _, value = self.get_constant_value(jitval, "TensorType") + operand_id = self.add_tensor_operand_for_weight(value) + return (operand_id, self.operands[operand_id]) + + def add_operation(self, opcode, inputs, outputs): + self.operations.append((opcode, len(inputs), len(outputs))) + self.operation_args.extend(inputs + outputs) + + def add_tensor_sequence(self, jitval, values): + if jitval in self.tensor_sequences: + raise AssertionError(f"jitval {jitval!r} already in tensor_sequences") + self.tensor_sequences[jitval] = values + + def add_constant_value(self, jitval, ctype, value): + if jitval in self.constants: + raise AssertionError(f"jitval {jitval!r} already in constants") + self.constants[jitval] = (ctype, value) + + def get_constant_value(self, jitval, typekind=None): + record = self.constants.get(jitval) + if record is None: + raise Exception( # noqa: TRY002 + f"Could not find constant value for '{jitval!r}'." + ) + ctype, _ = record + if typekind is not None and ctype.kind() != typekind: + raise Exception( # noqa: TRY002 + f"Expected constant value of type {typekind}, but got {ctype.kind()} for value '{jitval!r}'" + ) + return record + + def operand_to_template_torchscript(self, op_id, oper, shape=None): + """Return a TorchScript expression to build a template for a given operand.""" + if shape is None: + shape = oper.shape + else: + if len(shape) != len(oper.shape): + raise AssertionError( + f"shape length {len(shape)} != oper.shape length {len(oper.shape)}" + ) + + shape_parts = ["("] + for d, s in enumerate(shape): + if s > 0: + # Fixed shape dimension: just add the value. + shape_parts.append(str(s)) + elif s == 0: + # Load time flexible shape dimension: it should have been computed in a variable. + shape_parts.append(flex_name(op_id, d)) + elif s == -1: + # Runtime flexible shape + shape_parts.append("0") + else: + raise Exception( # noqa: TRY002 + "Unknown dim value, dimensions should be >= -1" + ) + shape_parts.append(",") + shape_parts.append(")") + shape_code = "".join(shape_parts) + if oper.op_type == NNAPI_OperandCode.TENSOR_FLOAT32: + return f"torch.zeros({shape_code}, dtype=torch.float32)" + elif oper.op_type == NNAPI_OperandCode.TENSOR_INT32: + return f"torch.zeros({shape_code}, dtype=torch.int32)" + elif oper.op_type == NNAPI_OperandCode.TENSOR_QUANT8_ASYMM: + return ( + f"torch.quantize_per_tensor(" + f"torch.zeros(1), scale={oper.scale}, zero_point={oper.zero_point}, dtype=torch.quint8)" + f".expand({shape_code}).contiguous()" + ) + elif oper.op_type in ( + NNAPI_OperandCode.TENSOR_QUANT16_ASYMM, + NNAPI_OperandCode.TENSOR_QUANT16_SYMM, + ): + if self.use_int16_for_qint16: + return f"torch.zeros({shape_code}, dtype=torch.int16)" + else: + raise Exception( # noqa: TRY002 + "`int16` isn't supported. If you're trying to represent NNAPI" + " qint16 with Pytorch int16, set `use_int16_for_qint16 = True`" + ) + + raise Exception( # noqa: TRY002 + f"Unsupported output operand type: {oper.op_type}" + ) + + def forward_operand_shape(self, out_op_id, out_dim, in_op_id, in_dim): + self.compute_operand_shape(out_op_id, out_dim, flex_name(in_op_id, in_dim)) + + def compute_operand_shape(self, op_id, dim, expr): + self.flexible_shape_computation_lines.append( + f"{flex_name(op_id, dim)} = {expr}" + ) + + def transpose_to_nhwc(self, in_id, oper): + if oper.shape[2:] != (1, 1): + raise Exception( # noqa: TRY002 + "Automatic transpose only supported for H,W == 1,1" + ) + + out_oper = oper._replace(dim_order=DimOrder.CHANNELS_LAST) + + inputs = [None] * 2 + inputs[0] = in_id + inputs[1] = self.add_immediate_int_vector([0, 2, 3, 1]) + + outputs = [None] * 1 + outputs[0] = self.add_anonymous_tensor_operand(out_oper) + + self.add_operation(NNAPI_OperationCode.TRANSPOSE, inputs, outputs) + + return outputs[0], out_oper + + # Transpose inputs as necessary to allow broadcasting. + def transpose_for_broadcast(self, in0_id, in0_oper, in1_id, in1_oper): + if in0_oper.dim_order == in1_oper.dim_order: + return in0_id, in0_oper, in1_id, in1_oper + + # Assume NHWC is preferred if there is a mismatch. + orders = (in0_oper.dim_order, in1_oper.dim_order) + if orders == (DimOrder.PRESUMED_CONTIGUOUS, DimOrder.CHANNELS_LAST): + return self.transpose_to_nhwc(in0_id, in0_oper) + (in1_id, in1_oper) + if orders == (DimOrder.CHANNELS_LAST, DimOrder.PRESUMED_CONTIGUOUS): + return (in0_id, in0_oper) + self.transpose_to_nhwc(in1_id, in1_oper) + + raise Exception( # noqa: TRY002 + f"Automatic transpose not supported for dim_orders: {in0_oper.dim_order!r}, {in1_oper.dim_order!r}" + ) + + def get_size_arg(self, jitval): + ctype, value = self.get_constant_value(jitval) + if ctype.kind() == "ListType": + if ctype.getElementType().kind() != "IntType": + raise AssertionError( + f"expected ListType of IntType, got {ctype.getElementType().kind()}" + ) + return value + raise Exception( # noqa: TRY002 + f"Can't handle size arg of type '{ctype!r}' for '{jitval!r}'" + ) + + def get_conv_pool_args_2d_from_pack(self, kernel_size, packed_config): + pc = [i.item() for i in packed_config] + if pc[0] != 2: + raise AssertionError(f"expected pc[0] == 2, got {pc[0]}") + strides = [pc[1], pc[2]] + paddings = [pc[3], pc[4]] + dilations = [pc[5], pc[6]] + output_padding = [pc[7], pc[8]] + group_num = pc[9] + + if len(pc) != 11: + raise AssertionError(f"expected len(pc) == 11, got {len(pc)}") + if output_padding != [0, 0]: + raise AssertionError( + f"expected output_padding == [0, 0], got {output_padding}" + ) + + return self.get_conv_pool_args_2d_common( + kernel_size, strides, paddings, dilations, group_num + ) + + def get_conv_pool_args_2d_from_jit( + self, kernel_size, stride, padding, dilation=None, group=None + ): + strides = self.get_size_arg(stride) + paddings = self.get_size_arg(padding) + if dilation is None: + dilations = [1, 1] + else: + dilations = self.get_size_arg(dilation) + if group is not None: + _, group_num = self.get_constant_value(group, "IntType") + else: + group_num = None + return self.get_conv_pool_args_2d_common( + kernel_size, strides, paddings, dilations, group_num + ) + + def get_conv_pool_args_2d_common( + self, kernel_size, strides, paddings, dilations, group_num + ): + kernels = list(kernel_size) + + if len(kernels) != 2: + raise AssertionError(f"expected len(kernels) == 2, got {len(kernels)}") + if len(strides) != 2: + raise AssertionError(f"expected len(strides) == 2, got {len(strides)}") + if len(paddings) != 2: + raise AssertionError(f"expected len(paddings) == 2, got {len(paddings)}") + if len(dilations) != 2: + raise AssertionError(f"expected len(dilations) == 2, got {len(dilations)}") + + # NNAPI uses 4 values for padding. + ph, pw = paddings + real_paddings = [ph, ph, pw, pw] + + return ConvPoolArgs2d( + *(kernels + strides + real_paddings + dilations + [group_num]) + ) + + def serialize_model(self, model, inputs, return_shapes=None): + self.add_immediate_bool_scalar(False) + self.add_immediate_bool_scalar(True) + + inp_dim_orders = [] + out_dim_orders = [] + + self_jitval = next(model.graph.inputs()) + self.add_constant_value(self_jitval, self_jitval.type(), model) + + for arg_idx, (input_value, input_tensor) in enumerate( + zip(list(model.graph.inputs())[1:], inputs) + ): + op_id = self.add_tensor_operand_for_input( + arg_idx, input_value, input_tensor + ) + inp_dim_orders.append(self.operands[op_id].dim_order.value) + + for idx, node in enumerate(model.graph.nodes()): + LOG.debug("Processing node #%d: %r", idx, node) + self.add_node(node) + + retn = model.graph.return_node() + if retn.inputsSize() != 1: + raise AssertionError( + f"expected retn.inputsSize() == 1, got {retn.inputsSize()}" + ) + if retn.outputsSize() != 0: + raise AssertionError( + f"expected retn.outputsSize() == 0, got {retn.outputsSize()}" + ) + retn_input = retn.inputsAt(0) + template_return_lines = ["return ["] + if retn_input.type().kind() == "TensorType": + return_values = [retn_input] + retval_count = -1 + elif retn_input.type().kind() == "TupleType": + return_values = self.tensor_sequences[retn_input] + retval_count = len(return_values) + else: + raise Exception( # noqa: TRY002 + f"Unsupported return type: {retn_input.type()}" + ) + + if return_shapes is not None: + if len(return_shapes) != len(return_values): + raise AssertionError( + f"return_shapes length {len(return_shapes)} != return_values length {len(return_values)}" + ) + for i, v in enumerate(return_values): + op_id = self.jitval_operand_map[v] + self.outputs.append(op_id) + out_dim_orders.append(self.operands[op_id].dim_order.value) + shape = return_shapes[i] if return_shapes else None + template_return_lines.append( + self.operand_to_template_torchscript(op_id, self.operands[op_id], shape) + + "," + ) + template_return_lines.append("]") + + model = [] + + version = 1 + header = struct.pack( + "iiiiii", + version, + len(self.operands), + len(self.values), + len(self.operations), + len(self.inputs), + len(self.outputs), + ) + model.append(header) + + serialized_values, serialized_value_data = self.serialize_values() + + model.extend( + struct.pack("iifi", t, len(d), s, z) for (t, d, _m, s, z) in self.operands + ) + model.extend(serialized_values) + model.extend(struct.pack("iii", *x) for x in self.operations) + + # Compact the model so we can get its length so far. + model = [b"".join(model)] + model_offset = len(model[0]) + # Model offset is the index into the model (in 32-bit words, not bytes) + # of the next dimension we're about to serialize. If it's 0, + # generate code to mutate it before passing to NNAPI. + if model_offset % 4 != 0: + raise AssertionError( + f"model_offset must be divisible by 4, got {model_offset}" + ) + model_offset = int(model_offset / 4) + + for op_id, (_, dims, dim_order, _, _) in enumerate(self.operands): + shape = fix_shape(dims, dim_order) + for d, s in enumerate(shape): + if s == 0: + pt_d = reverse_map_dim(dim_order, d) + self.flexible_shape_computation_lines.append( + f"ser_model[{model_offset}] = {flex_name(op_id, pt_d)}" + ) + model_offset += 1 + + # convert runtime flex shape from -1 to 0 + shape = tuple(d if d != -1 else 0 for d in shape) + model.append(self.serialize_ints(shape)) + + model.extend(serialized_value_data) + model.append(self.serialize_ints(self.operation_args)) + model.append(self.serialize_ints(self.inputs)) + model.append(self.serialize_ints(self.outputs)) + + self.flexible_shape_computation_lines.extend(template_return_lines) + + return ( + array.array("i", b"".join(model)), + self.used_weights, + inp_dim_orders, + out_dim_orders, + self.flexible_shape_computation_lines, + retval_count, + ) + + def serialize_values(self): + serialized_values = [] + serialized_value_data = [] + if len(self.values) != len(self.value_data): + raise AssertionError( + f"values length {len(self.values)} != value_data length {len(self.value_data)}" + ) + for (op_index, source_type), data in zip(self.values, self.value_data): + source_length = len(data) + + # Pad with 0 bytes out to a multiple of 4 for alignment. + physical_length = ((source_length - 1) | 0x3) + 1 + padded_data = data + (b"\0" * (physical_length - source_length)) + + serialized_values.append( + struct.pack("iii", op_index, source_type, source_length) + ) + serialized_value_data.append(padded_data) + + return serialized_values, serialized_value_data + + @staticmethod + def serialize_ints(ints): + return array.array("i", ints).tobytes() + + ADDER_MAP = { + "prim::GetAttr": lambda self, node: self.add_getattr(node), + "prim::Constant": lambda self, node: self.add_constant_node(node), + "prim::ListConstruct": lambda self, node: self.add_list_construct(node), + "prim::TupleConstruct": lambda self, node: self.add_tuple_construct(node), + "aten::unsqueeze": lambda self, node: self.add_unsqueeze(node), + "aten::to": lambda self, node: self.add_to(node), + "aten::detach": lambda self, node: self._identity(node), + "aten::reshape": lambda self, node: self.add_reshape(node), + "aten::flatten": lambda self, node: self.add_flatten(node), + "aten::slice": lambda self, node: self.add_slice(node), + "aten::size": lambda self, node: self.add_size(node), + "aten::cat": lambda self, node: self.add_cat(node), + "aten::mean": lambda self, node: self.add_mean(node), + "aten::quantize_per_tensor": lambda self, node: self.add_quantize(node), + "aten::dequantize": lambda self, node: self.add_dequantize(node), + "aten::add": lambda self, node: self.add_add_sub_op( + node, NNAPI_OperationCode.ADD, NNAPI_FuseCode.FUSED_NONE + ), + "aten::sub": lambda self, node: self.add_add_sub_op( + node, NNAPI_OperationCode.SUB, NNAPI_FuseCode.FUSED_NONE + ), + "aten::mul": lambda self, node: self.add_pointwise_simple_binary_broadcast_op( + node, NNAPI_OperationCode.MUL, NNAPI_FuseCode.FUSED_NONE + ), + "aten::div": lambda self, node: self.add_pointwise_simple_binary_broadcast_op( + node, NNAPI_OperationCode.DIV, NNAPI_FuseCode.FUSED_NONE + ), + "aten::relu": lambda self, node: self.add_pointwise_simple_unary_op( + node, NNAPI_OperationCode.RELU + ), + "aten::sigmoid": lambda self, node: self.add_pointwise_simple_unary_op( + node, NNAPI_OperationCode.LOGISTIC + ), + "aten::softmax": lambda self, node: self.add_softmax(node), + "aten::hardtanh": lambda self, node: self.add_hardtanh(node), + "aten::avg_pool2d": lambda self, node: self.add_avg_pool2d(node), + "aten::max_pool2d": lambda self, node: self.add_pool2d_node( + node, NNAPI_OperationCode.MAX_POOL_2D + ), + "aten::adaptive_avg_pool2d": lambda self, node: self.add_adaptive_avg_pool2d( + node + ), + "aten::upsample_nearest2d": lambda self, node: self.add_upsample_nearest2d( + node + ), + "aten::prelu": lambda self, node: self.add_prelu_op(node), + "aten::addmm": lambda self, node: self.add_addmm(node), + "aten::linear": lambda self, node: self.add_linear(node), + "aten::_convolution": lambda self, node: self.add_conv_underscore(node), + "aten::conv2d": lambda self, node: self.add_conv2d(node), + "aten::log_softmax": lambda self, node: self.add_log_softmax(node), + "quantized::linear": lambda self, node: self.add_qlinear(node), + "quantized::conv2d": lambda self, node: self.add_qconv2d( + node, NNAPI_FuseCode.FUSED_NONE + ), + "quantized::conv2d_relu": lambda self, node: self.add_qconv2d( + node, NNAPI_FuseCode.FUSED_RELU + ), + "quantized::conv_transpose2d": lambda self, node: self.add_qconv2d( + node, NNAPI_FuseCode.FUSED_NONE, transpose=True + ), + "quantized::add": lambda self, node: self.add_qadd( + node, NNAPI_OperationCode.ADD, NNAPI_FuseCode.FUSED_NONE + ), + "quantized::add_relu": lambda self, node: self.add_qadd( + node, NNAPI_OperationCode.ADD, NNAPI_FuseCode.FUSED_RELU + ), + "quantized::mul": lambda self, node: self.add_qadd( + node, NNAPI_OperationCode.MUL, NNAPI_FuseCode.FUSED_NONE + ), + } + + def add_node(self, node): + adder = self.ADDER_MAP.get(node.kind()) + if not adder: + raise Exception( # noqa: TRY002 + f"Unsupported node kind ({node.kind()!r}) in node {node!r}" + ) + adder(self, node) + + def _identity(self, node): + in_id, _in_oper = self.get_tensor_operand_by_jitval(node.inputsAt(0)) + jitval = node.outputsAt(0) + self.jitval_operand_map[jitval] = in_id + + def add_getattr(self, node): + if node.inputsSize() != 1: + raise AssertionError( + f"expected node.inputsSize() == 1, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + obj_ctype, obj = self.get_constant_value(node.inputsAt(0)) + if not str(obj_ctype).startswith("__torch__."): + raise AssertionError( + f"expected obj_ctype to start with '__torch__.', got {obj_ctype}" + ) + name = node.s("name") + value = getattr(obj, name) + output = node.outputsAt(0) + ctype = output.type() + self.add_constant_value(output, ctype, value) + + def add_constant_node(self, node): + if node.inputsSize() != 0: + raise AssertionError( + f"expected node.inputsSize() == 0, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + output = node.outputsAt(0) + ctype = output.type() + value = output.toIValue() + self.add_constant_value(output, ctype, value) + + def add_list_construct(self, node): + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + output = node.outputsAt(0) + ctype = output.type() + const_vals: list | None = [] + tensors: list | None = [] + for inp in node.inputs(): + if const_vals is not None and inp in self.constants: + _, val = self.get_constant_value(inp) + const_vals.append(val) + else: + const_vals = None + if tensors is not None and inp.type().kind() == "TensorType": + tensors.append(inp) + else: + tensors = None + + if const_vals is not None: + # NOTE: Now that TorchScript supports list constants, + # this code path might not be used anymore. + self.add_constant_value(output, ctype, const_vals) + if tensors is not None: + self.add_tensor_sequence(output, tensors) + if const_vals is None and tensors is None: + raise Exception( # noqa: TRY002 + f"Unable to handle ListConstruct node. Neither all constants nor all tensors. {node!r}" + ) + + def add_tuple_construct(self, node): + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + output = node.outputsAt(0) + values = list(node.inputs()) + self.add_tensor_sequence(output, values) + + def add_unsqueeze(self, node): + if node.inputsSize() != 2: + raise AssertionError( + f"expected node.inputsSize() == 2, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + in_id, in_oper = self.get_tensor_operand_by_jitval_fixed_size(node.inputsAt(0)) + + _, dim = self.get_constant_value(node.inputsAt(1), "IntType") + if in_oper.dim_order != DimOrder.PRESUMED_CONTIGUOUS: + raise AssertionError( + f"expected dim_order PRESUMED_CONTIGUOUS, got {in_oper.dim_order}" + ) + + real_dim = dim if dim >= 0 else dim + len(in_oper.shape) + 1 + out_shape_list = list(in_oper.shape) + out_shape_list.insert(real_dim, 1) + out_shape = tuple(out_shape_list) + out_oper = in_oper._replace(shape=out_shape) + + inputs = [None] * 2 + inputs[0] = in_id + inputs[1] = self.add_immediate_int_scalar(dim) + + outputs = [None] * 1 + outputs[0] = self.add_tensor_operand(node.outputsAt(0), out_oper) + + self.add_operation(NNAPI_OperationCode.EXPAND_DIMS, inputs, outputs) + + def add_to(self, node): + # Handle to("cpu") / to("gpu") case + self._identity(node) + + def add_reshape(self, node): + if node.inputsSize() != 2: + raise AssertionError( + f"expected node.inputsSize() == 2, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + in_id, in_oper = self.get_tensor_operand_by_jitval_fixed_size(node.inputsAt(0)) + + shape_ctype, shape = self.get_constant_value(node.inputsAt(1)) + if shape_ctype.kind() != "ListType": + raise AssertionError( + f"expected shape_ctype ListType, got {shape_ctype.kind()}" + ) + if shape_ctype.getElementType().kind() != "IntType": + raise AssertionError( + f"expected shape element type IntType, got {shape_ctype.getElementType().kind()}" + ) + is_trivial_reshape = len(shape) == 2 and shape[1] == -1 + + if in_oper.dim_order != DimOrder.PRESUMED_CONTIGUOUS and not is_trivial_reshape: + raise Exception( # noqa: TRY002 + "Currently, reshape is only supported on NHWC tensors if the target size is [X, -1]." + ) + + # Bit of a hack here. Use a real tensor to infer the output shape. + out_shape = torch.zeros(1).expand(in_oper.shape).reshape(shape).shape + out_oper = in_oper._replace( + shape=out_shape, dim_order=DimOrder.PRESUMED_CONTIGUOUS + ) + + inputs = [None] * 2 + inputs[0] = in_id + inputs[1] = self.add_immediate_int_vector(shape) + + outputs = [None] * 1 + outputs[0] = self.add_tensor_operand(node.outputsAt(0), out_oper) + + self.add_operation(NNAPI_OperationCode.RESHAPE, inputs, outputs) + + def add_flatten(self, node): + if node.inputsSize() != 3: + raise AssertionError( + f"expected node.inputsSize() == 3, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + in_id, in_oper = self.get_tensor_operand_by_jitval(node.inputsAt(0)) + + _start_ctype, start_dim = self.get_constant_value(node.inputsAt(1), "IntType") + _end_ctype, end_dim = self.get_constant_value(node.inputsAt(2), "IntType") + + # channels last with channels == 1 or (height & width both 1) + is_trivial_flatten = len(in_oper.shape) == 4 and ( + in_oper.shape[1] == 1 or (in_oper.shape[2] == 1 and in_oper.shape[3] == 1) + ) + if in_oper.dim_order != DimOrder.PRESUMED_CONTIGUOUS and not is_trivial_flatten: + raise Exception( # noqa: TRY002 + "Currently, flatten is not supported on NHWC tensors unless C=1 or H=W=1" + ) + + if start_dim < 0: + start_dim += len(in_oper.shape) + if end_dim < 0: + end_dim += len(in_oper.shape) + + out_shape = ( + in_oper.shape[:start_dim] + + (functools.reduce(operator.mul, in_oper.shape[start_dim : end_dim + 1]),) + + in_oper.shape[end_dim + 1 :] + ) + + if any(dim == 0 for dim in in_oper.shape[start_dim : end_dim + 1]): + raise Exception( # noqa: TRY002 + "Flattening flexible dims is not supported yet" + ) + non_flattened_dims = in_oper.shape[:start_dim] + in_oper.shape[end_dim + 1 :] + if non_flattened_dims.count(0) > 1: + raise Exception("Only 1 dim can be flexible") # noqa: TRY002 + + out_oper = in_oper._replace( + shape=out_shape, dim_order=DimOrder.PRESUMED_CONTIGUOUS + ) + out_id = self.add_tensor_operand(node.outputsAt(0), out_oper) + + for idx, dim in enumerate(out_shape): + if dim == 0: + self.forward_operand_shape(out_id, idx, in_id, in_oper.shape.index(0)) + + inputs_1 = tuple(dim if dim != 0 else -1 for dim in out_shape) + inputs = [None] * 2 + inputs[0] = in_id + inputs[1] = self.add_immediate_int_vector(inputs_1) + + outputs = [None] * 1 + outputs[0] = out_id + + self.add_operation(NNAPI_OperationCode.RESHAPE, inputs, outputs) + + def add_slice(self, node): + if node.inputsSize() != 5: + raise AssertionError( + f"expected node.inputsSize() == 5, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + in_id, in_oper = self.get_tensor_operand_by_jitval(node.inputsAt(0)) + _, dim_value = self.get_constant_value(node.inputsAt(1)) + _, start_value = self.get_constant_value(node.inputsAt(2)) + _, stop_value = self.get_constant_value(node.inputsAt(3)) + _, step_value = self.get_constant_value(node.inputsAt(4)) + + if start_value is None: + start_value = 0 + if stop_value is None: + stop_value = sys.maxsize + + if start_value < 0: + start_value += in_oper.shape[dim_value] + elif start_value == sys.maxsize: + start_value = 0 + + if start_value == 0 and stop_value == sys.maxsize: + self._identity(node) + return + + if in_oper.shape[dim_value] == 0: + raise Exception("Unable to slice with flexible shape") # noqa: TRY002 + + if stop_value < 0: + stop_value += in_oper.shape[dim_value] + elif stop_value == sys.maxsize: + stop_value = in_oper.shape[dim_value] + + if start_value >= stop_value: + raise Exception( # noqa: TRY002 + "Slice start value should be less than stop value" + ) + + out_len = (stop_value - start_value) // step_value + out_shape = tuple( + out_len if i == dim_value else dim for i, dim in enumerate(in_oper.shape) + ) + out_id = self.add_tensor_operand( + node.outputsAt(0), in_oper._replace(shape=out_shape) + ) + + # flex inputs + end_mask = 0 + for idx, dim in enumerate(out_shape): + if dim == 0: + self.forward_operand_shape(out_id, idx, in_id, idx) + end_mask |= 1 << idx + + inputs = [None] * 7 + inputs[0] = in_id + inputs[1] = self.add_immediate_int_vector( + [start_value if i == dim_value else 0 for i in range(len(in_oper.shape))] + ) + inputs[2] = self.add_immediate_int_vector( + [ + stop_value if i == dim_value else dim + for i, dim in enumerate(in_oper.shape) + ] + ) + inputs[3] = self.add_immediate_int_vector( + [step_value if i == dim_value else 1 for i in range(len(in_oper.shape))] + ) + inputs[4] = self.add_immediate_int_scalar(0) # begin mask + inputs[5] = self.add_immediate_int_scalar(end_mask) + inputs[6] = self.add_immediate_int_scalar(0) # shrink axis mas + + outputs = [None] * 1 + outputs[0] = out_id + + self.add_operation(NNAPI_OperationCode.STRIDED_SLICE, inputs, outputs) + + def add_size(self, node): + if node.inputsSize() != 2: + raise AssertionError( + f"expected node.inputsSize() == 2, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + _, in_oper = self.get_tensor_operand_by_jitval_fixed_size(node.inputsAt(0)) + _, value = self.constants[node.inputsAt(1)] + res = in_oper.shape[value] + output = node.outputsAt(0) + self.add_constant_value(output, output.type(), res) + + def add_cat(self, node): + if node.inputsSize() != 2: + raise AssertionError( + f"expected node.inputsSize() == 2, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + tensors = self.tensor_sequences[node.inputsAt(0)] + _, dim = self.get_constant_value(node.inputsAt(1), "IntType") + + if len(tensors) <= 0: + raise AssertionError(f"expected len(tensors) > 0, got {len(tensors)}") + in_ids = [] + out_oper = None + out_dim_size = 0 + for inp in tensors: + in_id, in_oper = self.get_tensor_operand_by_jitval(inp) + if out_oper is None: + out_shape = change_element(in_oper.shape, dim, -1) + out_oper = in_oper._replace(shape=out_shape) + if in_oper.op_type != out_oper.op_type: + raise AssertionError( + f"in_oper.op_type {in_oper.op_type} != out_oper.op_type {out_oper.op_type}" + ) + if in_oper.dim_order != out_oper.dim_order: + raise AssertionError( + f"in_oper.dim_order {in_oper.dim_order} != out_oper.dim_order {out_oper.dim_order}" + ) + if change_element(in_oper.shape, dim, -1) != change_element( + out_oper.shape, dim, -1 + ): + raise AssertionError( + f"shape mismatch: {change_element(in_oper.shape, dim, -1)} != {change_element(out_oper.shape, dim, -1)}" + ) + # TODO: Possibly check scale and zero point. + in_ids.append(in_id) + # TODO: Possibly support variable-sized inputs. + out_dim_size += in_oper.shape[dim] + + if out_oper is None: + raise AssertionError("out_oper must not be None") + out_oper = out_oper._replace( + shape=change_element(out_oper.shape, dim, out_dim_size) + ) + + if in_oper.dim_order == DimOrder.CHANNELS_LAST: # type: ignore[possibly-undefined] + if len(out_oper.shape) != 4: + raise AssertionError( + f"expected len(out_oper.shape) == 4 for CHANNELS_LAST, got {len(out_oper.shape)}" + ) + nnapi_dim = [0, 3, 1, 2][dim] + else: + nnapi_dim = dim + + out_id = self.add_tensor_operand(node.outputsAt(0), out_oper) + for idx, d in enumerate(out_oper.shape): + if d == 0: + if idx == dim: + shape = " + ".join(flex_name(ip_id, dim) for ip_id in in_ids) + self.compute_operand_shape(out_id, idx, shape) + else: + self.forward_operand_shape(out_id, idx, in_ids[0], idx) + + inputs = in_ids + [self.add_immediate_int_scalar(nnapi_dim)] + + outputs = [None] * 1 + outputs[0] = out_id + + self.add_operation(NNAPI_OperationCode.CONCATENATION, inputs, outputs) + + def add_mean(self, node): + if node.inputsSize() != 4: + raise AssertionError( + f"expected node.inputsSize() == 4, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + in_id, in_oper = self.get_tensor_operand_by_jitval_fixed_size(node.inputsAt(0)) + dim_ctype, dim = self.get_constant_value(node.inputsAt(1)) + if dim_ctype.kind() != "ListType": + raise AssertionError(f"expected dim_ctype ListType, got {dim_ctype.kind()}") + if dim_ctype.getElementType().kind() != "IntType": + raise AssertionError( + f"expected dim element type IntType, got {dim_ctype.getElementType().kind()}" + ) + _, keep_dim = self.get_constant_value(node.inputsAt(2), "BoolType") + # Expect None for dtype + self.get_constant_value(node.inputsAt(3), "NoneType") + + if in_oper.dim_order == DimOrder.CHANNELS_LAST: + if len(in_oper.shape) != 4: + raise AssertionError( + f"expected len(in_oper.shape) == 4 for CHANNELS_LAST, got {len(in_oper.shape)}" + ) + nnapi_dim = [[0, 3, 1, 2][d] for d in dim] + else: + nnapi_dim = dim + + collapsed_dims = set() + for d in dim: + if d < 0: + d += len(in_oper.shape) + collapsed_dims.add(d) + + if in_oper.dim_order == DimOrder.CHANNELS_LAST and not keep_dim: + if not collapsed_dims.issuperset({2, 3}): + raise AssertionError( + f"expected collapsed_dims to include {{2, 3}}, got {collapsed_dims}" + ) + out_dim_order = DimOrder.PRESUMED_CONTIGUOUS + else: + out_dim_order = in_oper.dim_order + + out_shape = [] + for i, s in enumerate(in_oper.shape): + if i not in collapsed_dims: + out_shape.append(s) + elif keep_dim: + out_shape.append(1) + + out_oper = in_oper._replace(shape=out_shape, dim_order=out_dim_order) + + inputs = [None] * 3 + inputs[0] = in_id + inputs[1] = self.add_immediate_int_vector(nnapi_dim) + inputs[2] = self.add_immediate_int_scalar(keep_dim) + + outputs = [None] * 1 + outputs[0] = self.add_tensor_operand(node.outputsAt(0), out_oper) + + self.add_operation(NNAPI_OperationCode.MEAN, inputs, outputs) + + def add_quantize(self, node): + if node.inputsSize() != 4: + raise AssertionError( + f"expected node.inputsSize() == 4, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + in_id, in_oper = self.get_tensor_operand_by_jitval_fixed_size(node.inputsAt(0)) + if in_oper.dim_order != DimOrder.CHANNELS_LAST: + raise Exception( # noqa: TRY002 + "Most hardware backends prefer NHWC quantized tensors. " + "Try setting `t.nnapi_nhwc = True` on your tensor inputs. " + ) + _, scale = self.get_constant_value(node.inputsAt(1), "FloatType") + _, zero_point = self.get_constant_value(node.inputsAt(2), "IntType") + _, scalar_type = self.get_constant_value(node.inputsAt(3), "IntType") + if scalar_type != TorchScalarTypes.QUINT8.value: + raise Exception( # noqa: TRY002 + "PyTorch NNAPI export only supports quantized tensors " + "with the quint8 dtype." + ) + op_type = NNAPI_OperandCode.TENSOR_QUANT8_ASYMM + + out_oper = in_oper._replace( + op_type=op_type, + scale=scale, + zero_point=zero_point, + ) + + inputs = [None] * 1 + inputs[0] = in_id + + outputs = [None] * 1 + outputs[0] = self.add_tensor_operand(node.outputsAt(0), out_oper) + + self.add_operation(NNAPI_OperationCode.QUANTIZE, inputs, outputs) + + def add_dequantize(self, node): + if node.inputsSize() != 1: + raise AssertionError( + f"expected node.inputsSize() == 1, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + in_id, in_oper = self.get_tensor_operand_by_jitval_fixed_size(node.inputsAt(0)) + out_oper = in_oper._replace( + op_type=NNAPI_OperandCode.TENSOR_FLOAT32, + scale=0.0, + zero_point=0, + ) + + inputs = [None] * 1 + inputs[0] = in_id + + outputs = [None] * 1 + outputs[0] = self.add_tensor_operand(node.outputsAt(0), out_oper) + + self.add_operation(NNAPI_OperationCode.DEQUANTIZE, inputs, outputs) + + def add_pointwise_simple_unary_op(self, node, opcode): + if node.inputsSize() != 1: + raise AssertionError( + f"expected node.inputsSize() == 1, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + in_id, in_oper = self.get_tensor_operand_by_jitval(node.inputsAt(0)) + + out_oper = in_oper + if opcode == NNAPI_OperationCode.LOGISTIC: + # NNAPI docs: For ANEURALNETWORKS_TENSOR_QUANT8_ASYMM, the scale + # must be 1.f / 256 and the zeroPoint must be 0. + # https://fburl.com/h52stoog + if in_oper.op_type == NNAPI_OperandCode.TENSOR_QUANT8_ASYMM: + out_oper = in_oper._replace(zero_point=0, scale=1.0 / 256) + + out_id = self.add_tensor_operand(node.outputsAt(0), out_oper) + + for idx, dim in enumerate(in_oper.shape): + if dim == 0: + self.forward_operand_shape(out_id, idx, in_id, idx) + + inputs = [None] * 1 + inputs[0] = in_id + + outputs = [None] * 1 + outputs[0] = out_id + + self.add_operation(opcode, inputs, outputs) + + def _do_add_binary(self, node, opcode, fuse_code, *, qparams=None): + """Helper for pointwise binary broadcast ops with superfluous extra args.""" + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + if node.inputsAt(0).type().kind() != "TensorType": + raise AssertionError( + f"expected inputsAt(0) TensorType, got {node.inputsAt(0).type().kind()}" + ) + if node.inputsAt(1).type().kind() != "TensorType": + raise AssertionError( + f"expected inputsAt(1) TensorType, got {node.inputsAt(1).type().kind()}" + ) + + if self.has_operand_for_jitval(node.inputsAt(0)): + in0_id, in0_oper = self.get_tensor_operand_by_jitval(node.inputsAt(0)) + in1_id, in1_oper = self.get_tensor_operand_or_constant( + node.inputsAt(1), in0_oper.dim_order + ) + elif self.has_operand_for_jitval(node.inputsAt(1)): + in1_id, in1_oper = self.get_tensor_operand_by_jitval(node.inputsAt(1)) + in0_id, in0_oper = self.get_tensor_operand_or_constant( + node.inputsAt(0), in1_oper.dim_order + ) + else: + raise Exception( # noqa: TRY002 + f"Can't do a NNAPI binary op: {opcode} on two constants" + ) + + if in0_oper.op_type != in1_oper.op_type: + raise AssertionError( + f"in0_oper.op_type {in0_oper.op_type} != in1_oper.op_type {in1_oper.op_type}" + ) + in0_id, in0_oper, in1_id, in1_oper = self.transpose_for_broadcast( + in0_id, in0_oper, in1_id, in1_oper + ) + # NOTE: PyTorch and NNAPI have the same broadcast semantics. + out_shape = broadcast_shapes(in0_oper.shape, in1_oper.shape) + out_oper = in0_oper._replace(shape=out_shape) + if qparams is not None: + scale, zp = qparams + out_oper = out_oper._replace(scale=scale, zero_point=zp) + + out_id = self.add_tensor_operand(node.outputsAt(0), out_oper) + for idx, (d0, d1) in enumerate(zip(in0_oper.shape, in1_oper.shape)): + if d0 == 1 and d1 == 0: + self.forward_operand_shape(out_id, idx, in1_id, idx) + elif d0 == 0 and d1 == 1: + self.forward_operand_shape(out_id, idx, in0_id, idx) + elif d0 == 0 and d1 == 0: + self.flexible_shape_computation_lines.append( + f"assert {flex_name(in0_id, idx)} == {flex_name(in1_id, idx)}" + ) + self.forward_operand_shape(out_id, idx, in0_id, idx) + + inputs = [None] * 3 + inputs[0] = in0_id + inputs[1] = in1_id + inputs[2] = self.add_immediate_int_scalar(fuse_code) + + outputs = [None] * 1 + outputs[0] = out_id + + self.add_operation(opcode, inputs, outputs) + + def add_pointwise_simple_binary_broadcast_op(self, node, opcode, fuse_code): + if node.inputsSize() != 2: + raise AssertionError( + f"expected node.inputsSize() == 2, got {node.inputsSize()}" + ) + self._do_add_binary(node, opcode, fuse_code) + + def add_add_sub_op(self, node, opcode, fuse_code): + if node.inputsSize() != 3: + raise AssertionError( + f"expected node.inputsSize() == 3, got {node.inputsSize()}" + ) + + _, alpha = self.get_constant_value(node.inputsAt(2), "IntType") + if alpha != 1: + raise Exception( # noqa: TRY002 + "NNAPI does not support add/sub with alpha." + ) + + self._do_add_binary(node, opcode, fuse_code) + + def add_qadd(self, node, opcode, fuse_code): + if node.inputsSize() != 4: + raise AssertionError( + f"expected node.inputsSize() == 4, got {node.inputsSize()}" + ) + + _, scale = self.get_constant_value(node.inputsAt(2), "FloatType") + _, zero_point = self.get_constant_value(node.inputsAt(3), "IntType") + + self._do_add_binary(node, opcode, fuse_code, qparams=(scale, zero_point)) + + def add_softmax(self, node): + if node.inputsSize() != 3: + raise AssertionError( + f"expected node.inputsSize() == 3, got {node.inputsSize()}" + ) + in_id, in_oper = self.get_tensor_operand_by_jitval(node.inputsAt(0)) + + _, softmax_dim = self.get_constant_value(node.inputsAt(1), "IntType") + + out_id = self.add_tensor_operand(node.outputsAt(0), in_oper) + for dim, size in enumerate(in_oper.shape): + if size == 0: + self.forward_operand_shape(out_id, dim, in_id, dim) + + inputs = [None] * 3 + inputs[0] = in_id + inputs[1] = self.add_immediate_float_scalar( + 1.0 + ) # positive scaling factor of exponent, beta + inputs[2] = self.add_immediate_int_scalar(softmax_dim) + + outputs = [None] * 1 + outputs[0] = out_id + + self.add_operation(NNAPI_OperationCode.SOFTMAX, inputs, outputs) + + def add_hardtanh(self, node): + if node.inputsSize() != 3: + raise AssertionError( + f"expected node.inputsSize() == 3, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + in_id, in_oper = self.get_tensor_operand_by_jitval_fixed_size(node.inputsAt(0)) + _, min_val = self.get_constant_value(node.inputsAt(1), "FloatType") + _, max_val = self.get_constant_value(node.inputsAt(2), "FloatType") + + op_map = { + (-1, 1): NNAPI_OperationCode.RELU1, + (0, 6): NNAPI_OperationCode.RELU6, # noqa: E201 + } + + opcode = op_map.get((min_val, max_val)) + if opcode is None: + raise Exception( # noqa: TRY002 + "NNAPI only supports hardtanh with args (-1, 1) or (0, 6)." + ) + + inputs = [None] * 1 + inputs[0] = in_id + + outputs = [None] * 1 + outputs[0] = self.add_tensor_operand(node.outputsAt(0), in_oper) + + self.add_operation(opcode, inputs, outputs) + + def add_prelu_op(self, node): + if node.inputsSize() != 2: + raise AssertionError( + f"expected node.inputsSize() == 2, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + if node.inputsAt(0).type().kind() != "TensorType": + raise AssertionError( + f"expected inputsAt(0) TensorType, got {node.inputsAt(0).type().kind()}" + ) + if node.inputsAt(1).type().kind() != "TensorType": + raise AssertionError( + f"expected inputsAt(1) TensorType, got {node.inputsAt(1).type().kind()}" + ) + + in_id, in_oper = self.get_tensor_operand_by_jitval(node.inputsAt(0)) + w_id, w_oper = self.get_tensor_operand_for_weight(node.inputsAt(1)) + if len(w_oper.shape) != 1: + raise AssertionError( + f"expected len(w_oper.shape) == 1, got {len(w_oper.shape)}" + ) + if w_oper.shape[0] <= 0: + raise AssertionError(f"expected w_oper.shape[0] > 0, got {w_oper.shape[0]}") + if w_oper.shape[0] > 1: + if in_oper.use_nchw(): + # TODO: Support this by adding trailing 1 dims. + raise Exception( # noqa: TRY002 + "Per-channel PReLU only supports channels_last right now." + ) + + out_id = self.add_tensor_operand(node.outputsAt(0), in_oper) + for dim, size in enumerate(in_oper.shape): + if size > 0: + pass + elif dim <= 1: + raise Exception( # noqa: TRY002 + "PReLU requires fixed size for dim 0 and dim 1." + ) + else: + self.forward_operand_shape(out_id, dim, in_id, dim) + + inputs = [None] * 2 + inputs[0] = in_id + inputs[1] = w_id + + outputs = [None] * 1 + outputs[0] = out_id + + self.add_operation(NNAPI_OperationCode.PRELU, inputs, outputs) + + def add_pool2d_node(self, node, opcode): + if node.inputsSize() != 6: + raise AssertionError( + f"expected node.inputsSize() == 6, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + image, kernel, stride, padding, dilation, _ceil_mode = node.inputs() + + stride = stride or kernel + + # TODO: Validate ceil_mode semantics. + + args = self.get_conv_pool_args_2d_from_jit( + self.get_size_arg(kernel), stride, padding, dilation + ) + if args.dilation_h != 1 or args.dilation_w != 1: + raise Exception("NNAPI does not support dilated pooling.") # noqa: TRY002 + + image_id, image_oper = self.get_tensor_operand_by_jitval_fixed_size(image) + if len(image_oper.shape) != 4: + raise AssertionError( + f"expected len(image_oper.shape) == 4, got {len(image_oper.shape)}" + ) + + out_shape = get_conv_pool_shape( + image_oper.shape, args, image_oper.shape[1], False + ) + use_nchw = image_oper.use_nchw() + + inputs = [None] * 11 + inputs[0] = image_id + inputs[1] = self.add_immediate_int_scalar(args.pad_l) + inputs[2] = self.add_immediate_int_scalar(args.pad_r) + inputs[3] = self.add_immediate_int_scalar(args.pad_t) + inputs[4] = self.add_immediate_int_scalar(args.pad_b) + inputs[5] = self.add_immediate_int_scalar(args.stride_w) + inputs[6] = self.add_immediate_int_scalar(args.stride_h) + inputs[7] = self.add_immediate_int_scalar(args.kernel_w) + inputs[8] = self.add_immediate_int_scalar(args.kernel_h) + inputs[9] = self.add_immediate_int_scalar(NNAPI_FuseCode.FUSED_NONE) + inputs[10] = self.add_immediate_bool_scalar(use_nchw) + + outputs = [None] * 1 + outputs[0] = self.add_tensor_operand( + node.outputsAt(0), image_oper._replace(shape=out_shape) + ) + + self.add_operation(opcode, inputs, outputs) + + def add_avg_pool2d(self, node): + if node.inputsSize() != 7: + raise AssertionError( + f"expected node.inputsSize() == 7, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + ( + image, + kernel, + stride, + padding, + _ceil_mode, + count_include_pad, + divisor_override, + ) = node.inputs() + + _, count_include_pad_value = self.get_constant_value(count_include_pad) + _, divisor_override_value = self.get_constant_value(divisor_override) + if not count_include_pad_value or divisor_override_value: + raise Exception( # noqa: TRY002 + "NNAPI doesn't support count_include_pad=False or divisor_override" + ) + + args = self.get_conv_pool_args_2d_from_jit( + self.get_size_arg(kernel), stride, padding + ) + + image_id, image_oper = self.get_tensor_operand_by_jitval(image) + if len(image_oper.shape) != 4: + raise AssertionError( + f"expected len(image_oper.shape) == 4, got {len(image_oper.shape)}" + ) + + out_shape = get_conv_pool_shape( + image_oper.shape, args, image_oper.shape[1], False + ) + use_nchw = image_oper.use_nchw() + + inputs = [None] * 11 + inputs[0] = image_id + inputs[1] = self.add_immediate_int_scalar(args.pad_l) + inputs[2] = self.add_immediate_int_scalar(args.pad_r) + inputs[3] = self.add_immediate_int_scalar(args.pad_t) + inputs[4] = self.add_immediate_int_scalar(args.pad_b) + inputs[5] = self.add_immediate_int_scalar(args.stride_w) + inputs[6] = self.add_immediate_int_scalar(args.stride_h) + inputs[7] = self.add_immediate_int_scalar(args.kernel_w) + inputs[8] = self.add_immediate_int_scalar(args.kernel_h) + inputs[9] = self.add_immediate_int_scalar(NNAPI_FuseCode.FUSED_NONE) + inputs[10] = self.add_immediate_bool_scalar(use_nchw) + + outputs = [None] * 1 + out_id = self.add_tensor_operand( + node.outputsAt(0), image_oper._replace(shape=out_shape) + ) + self._handle_conv_pool_flexible_input(out_id, image, args, False) + outputs[0] = out_id + + self.add_operation(NNAPI_OperationCode.AVERAGE_POOL_2D, inputs, outputs) + + def add_adaptive_avg_pool2d(self, node): + if node.inputsSize() != 2: + raise AssertionError( + f"expected node.inputsSize() == 2, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + image_id, image_oper = self.get_tensor_operand_by_jitval_fixed_size( + node.inputsAt(0) + ) + if len(image_oper.shape) != 4: + raise AssertionError( + f"expected len(image_oper.shape) == 4, got {len(image_oper.shape)}" + ) + + size_ctype, size_arg = self.get_constant_value(node.inputsAt(1)) + if size_ctype.kind() != "ListType": + raise AssertionError( + f"expected size_ctype ListType, got {size_ctype.kind()}" + ) + if size_ctype.getElementType().kind() != "IntType": + raise AssertionError( + f"expected size element type IntType, got {size_ctype.getElementType().kind()}" + ) + if size_arg != [1, 1]: + raise Exception( # noqa: TRY002 + "NNAPI only supports adaptive_avg_pool2d with output size (1, 1)." + ) + + out_shape = image_oper.shape[0:2] + tuple(size_arg) + use_nchw = image_oper.use_nchw() + + inputs = [None] * 11 + inputs[0] = image_id + inputs[1] = self.add_immediate_int_scalar(0) + inputs[2] = self.add_immediate_int_scalar(0) + inputs[3] = self.add_immediate_int_scalar(0) + inputs[4] = self.add_immediate_int_scalar(0) + inputs[5] = self.add_immediate_int_scalar(1) + inputs[6] = self.add_immediate_int_scalar(1) + inputs[7] = self.add_immediate_int_scalar(image_oper.shape[3]) + inputs[8] = self.add_immediate_int_scalar(image_oper.shape[2]) + inputs[9] = self.add_immediate_int_scalar(NNAPI_FuseCode.FUSED_NONE) + inputs[10] = self.add_immediate_bool_scalar(use_nchw) + + outputs = [None] * 1 + outputs[0] = self.add_tensor_operand( + node.outputsAt(0), image_oper._replace(shape=out_shape) + ) + + self.add_operation(NNAPI_OperationCode.AVERAGE_POOL_2D, inputs, outputs) + + def add_upsample_nearest2d(self, node): + if not (node.inputsSize() == 3 or node.inputsSize() == 4): + raise AssertionError( + f"expected node.inputsSize() == 3 or 4, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + if node.inputsSize() == 3: + image, size_jit, scale_jit = node.inputs() + else: + image, size_jit, scale_h_jit, scale_w_jit = node.inputs() + size_ctype, size_arg = self.get_constant_value(size_jit) + + if node.inputsSize() == 3: + scale_ctype, scale_arg = self.get_constant_value(scale_jit) # type: ignore[possibly-undefined] + else: + scale_h_ctype, scale_h_arg = self.get_constant_value(scale_h_jit) # type: ignore[possibly-undefined] + scale_w_ctype, _scale_w_arg = self.get_constant_value(scale_w_jit) # type: ignore[possibly-undefined] + + # The only way for the 4-argument overload of upsample_nearest2d to + # have been added to the graph without error is if the scale_h and + # scale_w arguments are None + if scale_h_ctype.kind() != "NoneType": + raise AssertionError( + f"expected scale_h_ctype NoneType, got {scale_h_ctype.kind()}" + ) + if scale_w_ctype.kind() != "NoneType": + raise AssertionError( + f"expected scale_w_ctype NoneType, got {scale_w_ctype.kind()}" + ) + + scale_ctype = scale_h_ctype + scale_arg = scale_h_arg + + image_id, image_oper = self.get_tensor_operand_by_jitval(image) + if len(image_oper.shape) != 4: + raise AssertionError( + f"expected len(image_oper.shape) == 4, got {len(image_oper.shape)}" + ) + + if size_ctype.kind() != "NoneType" and scale_ctype.kind() != "NoneType": + raise Exception("Size and scale cannot both be non-None.") # noqa: TRY002 + elif size_ctype.kind() != "NoneType": + if size_ctype.kind() != "ListType": + raise AssertionError( + f"expected size_ctype ListType, got {size_ctype.kind()}" + ) + if size_ctype.getElementType().kind() != "IntType": + raise AssertionError( + f"expected size element type IntType, got {size_ctype.getElementType().kind()}" + ) + if scale_ctype.kind() != "NoneType": + raise AssertionError( + f"expected scale_ctype NoneType, got {scale_ctype.kind()}" + ) + if scale_arg is not None: + raise AssertionError(f"expected scale_arg None, got {scale_arg}") + if not isinstance(size_arg, list): + raise AssertionError( + f"expected size_arg to be list, got {type(size_arg)}" + ) + if not size_arg: + raise AssertionError("expected size_arg to be non-empty") + if not all(isinstance(val, int) for val in size_arg): + raise AssertionError("expected all size_arg values to be int") + if len(size_arg) == 1: + size_arg = size_arg * 2 + if len(size_arg) != 2: + raise AssertionError( + f"expected len(size_arg) == 2, got {len(size_arg)}" + ) + out_h = size_arg[0] + out_w = size_arg[1] + arg_h = self.add_immediate_int_scalar(out_h) + arg_w = self.add_immediate_int_scalar(out_w) + elif scale_ctype.kind() != "NoneType": + if scale_ctype.kind() != "ListType": + raise AssertionError( + f"expected scale_ctype ListType, got {scale_ctype.kind()}" + ) + if scale_ctype.getElementType().kind() != "FloatType": + raise AssertionError( + f"expected scale element type FloatType, got {scale_ctype.getElementType().kind()}" + ) + if size_ctype.kind() != "NoneType": + raise AssertionError( + f"expected size_ctype NoneType, got {size_ctype.kind()}" + ) + if size_arg is not None: + raise AssertionError(f"expected size_arg None, got {size_arg}") + if not isinstance(scale_arg, list): + raise AssertionError( + f"expected scale_arg to be list, got {type(scale_arg)}" + ) + if not scale_arg: + raise AssertionError("expected scale_arg to be non-empty") + if not all(isinstance(val, float) for val in scale_arg): + raise AssertionError("expected all scale_arg values to be float") + if len(scale_arg) == 1: + scale_arg = scale_arg * 2 + if len(scale_arg) != 2: + raise AssertionError( + f"expected len(scale_arg) == 2, got {len(scale_arg)}" + ) + out_h = int(scale_arg[0] * image_oper.shape[2]) + out_w = int(scale_arg[1] * image_oper.shape[3]) + arg_h = self.add_immediate_float_scalar(scale_arg[0]) + arg_w = self.add_immediate_float_scalar(scale_arg[1]) + else: + raise Exception("Size and scale cannot both be None.") # noqa: TRY002 + + out_shape = (image_oper.shape[0], image_oper.shape[1], out_h, out_w) + use_nchw = image_oper.use_nchw() + out_id = self.add_tensor_operand( + node.outputsAt(0), image_oper._replace(shape=out_shape) + ) + + if image_oper.shape[0] == 0 or image_oper.shape[1] == 0: + raise Exception("Flexible batch or channels not supported") # noqa: TRY002 + + # Handle variable input size + for dim in (2, 3): # h, w indices + if image_oper.shape[dim] == 0: + if size_ctype.kind() != "NoneType": + # pyrefly: ignore [unsupported-operation] + self.compute_operand_shape(out_id, dim, size_arg[dim - 2]) + elif scale_ctype.kind() != "NoneType": + self.compute_operand_shape( + out_id, + dim, + # pyrefly: ignore [unsupported-operation] + f"int({scale_arg[dim - 2]} * {flex_name(image_id, dim)})", + ) + else: + raise Exception( # noqa: TRY002 + "Size and scale cannot both be None." + ) + + inputs = [None] * 4 + inputs[0] = image_id + inputs[1] = arg_w + inputs[2] = arg_h + inputs[3] = self.add_immediate_bool_scalar(use_nchw) + + outputs = [None] * 1 + outputs[0] = out_id + + self.add_operation(NNAPI_OperationCode.RESIZE_NEAREST_NEIGHBOR, inputs, outputs) + + def add_addmm(self, node): + if node.inputsSize() != 5: + raise AssertionError( + f"expected node.inputsSize() == 5, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + jit_bias, jit_input, jit_weight, jit_beta, jit_alpha = node.inputs() + + for jitval in (jit_beta, jit_alpha): + scale_ctype, scale_value = self.get_constant_value(jitval) + if scale_ctype.kind() not in ("IntType", "FloatType"): + raise AssertionError( + f"expected scale_ctype IntType or FloatType, got {scale_ctype.kind()}" + ) + if scale_value != 1: + raise Exception( # noqa: TRY002 + "NNAPI Fully-Connected does not support alpha and beta." + ) + + self.add_addmm_or_linear(node, True, jit_input, jit_weight, jit_bias) + + def add_linear(self, node): + if node.inputsSize() != 3: + raise AssertionError( + f"expected node.inputsSize() == 3, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + jit_input, jit_weight, jit_bias = node.inputs() + + self.add_addmm_or_linear(node, False, jit_input, jit_weight, jit_bias) + + def add_addmm_or_linear( + self, node, transpose_weight, jit_input, jit_weight, jit_bias + ): + input_id, input_oper = self.get_tensor_operand_by_jitval(jit_input) + bias_id, bias_oper = self.get_tensor_operand_for_weight(jit_bias) + + if len(input_oper.shape) != 2: + raise AssertionError( + f"expected len(input_oper.shape) == 2, got {len(input_oper.shape)}" + ) + if len(bias_oper.shape) != 1: + raise AssertionError( + f"expected len(bias_oper.shape) == 1, got {len(bias_oper.shape)}" + ) + + # TODO: Transform at load time to share weights with CPU model. + _, weight_tensor = self.get_constant_value(jit_weight, "TensorType") + if len(weight_tensor.shape) != 2: + raise AssertionError( + f"expected len(weight_tensor.shape) == 2, got {len(weight_tensor.shape)}" + ) + if transpose_weight: + nnapi_weight_tensor = weight_tensor.t().contiguous() + else: + nnapi_weight_tensor = weight_tensor.contiguous() + weight_id = self.add_tensor_operand_for_weight(nnapi_weight_tensor) + weight_oper = self.operands[weight_id] + + out_shape = (input_oper.shape[0], weight_oper.shape[0]) + out_id = self.add_tensor_operand( + node.outputsAt(0), input_oper._replace(shape=out_shape) + ) + + if input_oper.shape[0] == 0: + self.forward_operand_shape(out_id, 0, input_id, 0) + + inputs = [None] * 4 + inputs[0] = input_id + inputs[1] = weight_id + inputs[2] = bias_id + inputs[3] = self.add_immediate_int_scalar(NNAPI_FuseCode.FUSED_NONE) + + outputs = [None] * 1 + outputs[0] = out_id + + self.add_operation(NNAPI_OperationCode.FULLY_CONNECTED, inputs, outputs) + + def add_qlinear(self, node): + if node.inputsSize() != 4: + raise AssertionError( + f"expected node.inputsSize() == 4, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + ( + jit_input, + jit_packed_weight, + jit_scale, + jit_zero_point, + ) = node.inputs() + + input_id, input_oper = self.get_tensor_operand_by_jitval_fixed_size(jit_input) + # TODO: Support automatic reshape + if len(input_oper.shape) != 2: + raise AssertionError( + f"expected len(input_oper.shape) == 2, got {len(input_oper.shape)}" + ) + + _, out_scale = self.get_constant_value(jit_scale, "FloatType") + _, out_zero_point = self.get_constant_value(jit_zero_point, "IntType") + weight_ctype, packed_weight = self.get_constant_value(jit_packed_weight) + if weight_ctype.name() != "LinearPackedParamsBase": + raise AssertionError( + f"expected weight_ctype LinearPackedParamsBase, got {weight_ctype.name()}" + ) + raw_weight, raw_bias = packed_weight.__getstate__()[0] + if raw_bias is None: + raise AssertionError("raw_bias must not be None") + + if len(raw_weight.shape) != 2: + raise AssertionError( + f"expected len(raw_weight.shape) == 2, got {len(raw_weight.shape)}" + ) + if len(raw_bias.shape) != 1: + raise AssertionError( + f"expected len(raw_bias.shape) == 1, got {len(raw_bias.shape)}" + ) + if raw_bias.shape[0] != raw_weight.shape[0]: + raise AssertionError( + f"raw_bias.shape[0] {raw_bias.shape[0]} != raw_weight.shape[0] {raw_weight.shape[0]}" + ) + if raw_weight.shape[1] != input_oper.shape[1]: + raise AssertionError( + f"raw_weight.shape[1] {raw_weight.shape[1]} != input_oper.shape[1] {input_oper.shape[1]}" + ) + + if raw_weight.qscheme() != torch.per_tensor_affine: + raise AssertionError( + f"expected raw_weight.qscheme() per_tensor_affine, got {raw_weight.qscheme()}" + ) + if raw_weight.dtype == torch.quint8: + unsigned_weight = raw_weight + else: + if raw_weight.dtype != torch.qint8: + raise AssertionError( + f"expected raw_weight.dtype qint8, got {raw_weight.dtype}" + ) + unsigned_weight = torch._make_per_tensor_quantized_tensor( + (raw_weight.int_repr().int() + 128).to(torch.uint8), + scale=raw_weight.q_scale(), + zero_point=raw_weight.q_zero_point() + 128, + ) + weight_scale = unsigned_weight.q_scale() + bias_scale = input_oper.scale * weight_scale + int_bias = torch.quantize_per_tensor(raw_bias, bias_scale, 0, torch.qint32) + bias_id = self.add_tensor_operand_for_weight(int_bias) + + multiplier = input_oper.scale * weight_scale / out_scale + if multiplier <= 0: + raise AssertionError(f"expected multiplier > 0, got {multiplier}") + if multiplier >= 1: + raise Exception( # noqa: TRY002 + "Quantized convolution multiplier is greater than 1. " + "This is supported by NNAPI, but not by most hardware backends. " + "Try training a model without quantization-aware training. " + ) + + # TODO: Transform at load time to share weights with CPU model. + nnapi_weight_tensor = unsigned_weight.contiguous() + weight_id = self.add_tensor_operand_for_weight(nnapi_weight_tensor) + weight_oper = self.operands[weight_id] + + out_shape = (input_oper.shape[0], weight_oper.shape[0]) + out_oper = input_oper._replace( + shape=out_shape, + scale=out_scale, + zero_point=out_zero_point, + ) + + inputs = [None] * 4 + inputs[0] = input_id + inputs[1] = weight_id + inputs[2] = bias_id + inputs[3] = self.add_immediate_int_scalar(NNAPI_FuseCode.FUSED_NONE) + + outputs = [None] * 1 + outputs[0] = self.add_tensor_operand(node.outputsAt(0), out_oper) + + self.add_operation(NNAPI_OperationCode.FULLY_CONNECTED, inputs, outputs) + + def get_optional_bias(self, jit_bias, weight_tensor, transpose=False): + ctype, _value = self.get_constant_value(jit_bias) + if ctype.kind() == "NoneType": + bias_idx = 1 if transpose else 0 + nnapi_bias_tensor = torch.zeros( + weight_tensor.size()[bias_idx], dtype=weight_tensor.dtype + ) + bias_id = self.add_tensor_operand_for_weight(nnapi_bias_tensor) + bias_oper = self.operands[bias_id] + return bias_id, bias_oper + else: + return self.get_tensor_operand_for_weight(jit_bias) + + def add_conv2d(self, node): + if node.inputsSize() != 7: + raise AssertionError( + f"expected node.inputsSize() == 7, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + ( + jit_image, + jit_weight, + jit_bias, + jit_stride, + jit_pad, + jit_dilation, + jit_groups, + ) = node.inputs() + + _, weight_tensor = self.get_constant_value(jit_weight, "TensorType") + bias_id, _bias_oper = self.get_optional_bias(jit_bias, weight_tensor) + args = self.get_conv_pool_args_2d_from_jit( + weight_tensor.shape[2:4], jit_stride, jit_pad, jit_dilation, jit_groups + ) + + return self.add_conv2d_common( + node.outputsAt(0), + 0.0, + 0, + jit_image, + weight_tensor, + bias_id, + args, + False, # transpose + NNAPI_FuseCode.FUSED_NONE, + ) + + def add_conv_underscore(self, node): + if node.inputsSize() != 13: + raise AssertionError( + f"expected node.inputsSize() == 13, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + ( + jit_image, + jit_weight, + jit_bias, + jit_stride, + jit_pad, + jit_dilation, + jit_transpose, + _, + jit_groups, + _, + _, + _, + _, + ) = node.inputs() + + _, weight_tensor = self.get_constant_value(jit_weight, "TensorType") + _, transpose = self.get_constant_value(jit_transpose) + bias_id, _bias_oper = self.get_optional_bias(jit_bias, weight_tensor, transpose) + args = self.get_conv_pool_args_2d_from_jit( + weight_tensor.shape[2:4], jit_stride, jit_pad, jit_dilation, jit_groups + ) + + return self.add_conv2d_common( + node.outputsAt(0), + 0.0, + 0, + jit_image, + weight_tensor, + bias_id, + args, + transpose, + NNAPI_FuseCode.FUSED_NONE, + ) + + def add_log_softmax(self, node): + if node.inputsSize() != 3: + raise AssertionError( + f"expected node.inputsSize() == 3, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + jit_input, jit_dim, _jit_half_to_float = node.inputs() + input_id, input_oper = self.get_tensor_operand_by_jitval_fixed_size(jit_input) + _, dim = self.get_constant_value(jit_dim, "IntType") + + out_shape = input_oper.shape + + inputs = [None] * 3 + inputs[0] = input_id + # specifying 1 as the scaling factor for the exponent, beta + inputs[1] = self.add_immediate_float_scalar(1) + inputs[2] = self.add_immediate_int_scalar(dim) + + outputs = [None] * 1 + outputs[0] = self.add_tensor_operand( + node.outputsAt(0), input_oper._replace(shape=out_shape) + ) + self.add_operation(NNAPI_OperationCode.LOG_SOFTMAX, inputs, outputs) + + def add_qconv2d(self, node, fuse_code, transpose=False): + if node.inputsSize() != 4: + raise AssertionError( + f"expected node.inputsSize() == 4, got {node.inputsSize()}" + ) + if node.outputsSize() != 1: + raise AssertionError( + f"expected node.outputsSize() == 1, got {node.outputsSize()}" + ) + + ( + jit_image, + jit_packed_weight, + jit_scale, + jit_zero_point, + ) = node.inputs() + + _, out_scale = self.get_constant_value(jit_scale, "FloatType") + _, out_zero_point = self.get_constant_value(jit_zero_point, "IntType") + weight_ctype, packed_weight = self.get_constant_value(jit_packed_weight) + if weight_ctype.name() != "Conv2dPackedParamsBase": + raise AssertionError( + f"expected weight_ctype Conv2dPackedParamsBase, got {weight_ctype.name()}" + ) + ( + pack_version, + tensors, + opt_tensors, + ) = packed_weight.__getstate__()[0] + if pack_version != "2": + raise AssertionError(f"expected pack_version '2', got {pack_version!r}") + packed_config, raw_weight = tensors + (raw_bias,) = opt_tensors + if raw_bias is None: + raise AssertionError("raw_bias must not be None") + args = self.get_conv_pool_args_2d_from_pack( + raw_weight.shape[2:4], packed_config + ) + + if raw_weight.qscheme() != torch.per_tensor_affine: + raise AssertionError( + f"expected raw_weight.qscheme() per_tensor_affine, got {raw_weight.qscheme()}" + ) + if raw_weight.dtype == torch.quint8: + unsigned_weight = raw_weight + else: + if raw_weight.dtype != torch.qint8: + raise AssertionError( + f"expected raw_weight.dtype qint8, got {raw_weight.dtype}" + ) + unsigned_weight = torch._make_per_tensor_quantized_tensor( + (raw_weight.int_repr().int() + 128).to(torch.uint8), + scale=raw_weight.q_scale(), + zero_point=raw_weight.q_zero_point() + 128, + ) + weight_scale = unsigned_weight.q_scale() + _, image_oper = self.get_tensor_operand_by_jitval(jit_image) + bias_scale = image_oper.scale * weight_scale + int_bias = torch.quantize_per_tensor(raw_bias, bias_scale, 0, torch.qint32) + bias_id = self.add_tensor_operand_for_weight(int_bias) + + multiplier = image_oper.scale * weight_scale / out_scale + if multiplier <= 0: + raise AssertionError(f"expected multiplier > 0, got {multiplier}") + if multiplier >= 1: + raise Exception( # noqa: TRY002 + "Quantized convolution multiplier is greater than 1. " + "This is supported by NNAPI, but not by most hardware backends. " + "Try training a model without quantization-aware training. " + ) + + return self.add_conv2d_common( + node.outputsAt(0), + out_scale, + out_zero_point, + jit_image, + unsigned_weight, + bias_id, + args, + transpose, + fuse_code, + ) + + def add_conv2d_common( + self, + jit_out, + out_scale, + out_zero_point, + jit_image, + weight_tensor, + bias_id, + args, + transpose, + fuse_code, + ): + image_id, image_oper = self.get_tensor_operand_by_jitval(jit_image) + in_c = image_oper.shape[1] + + if args.group == 1: + # Full convolution + depthwise = False + if transpose: + weight_permutation = (1, 2, 3, 0) + else: + weight_permutation = (0, 2, 3, 1) + elif args.group == in_c: + # Depthwise convolution + depthwise = True + weight_permutation = (1, 2, 3, 0) + else: + raise Exception("Group convolution not supported yet.") # noqa: TRY002 + + # TODO: Transform at load time to share weights with CPU model. + nnapi_weight_tensor = weight_tensor.permute(*weight_permutation).contiguous() + weight_id = self.add_tensor_operand_for_weight(nnapi_weight_tensor) + weight_oper = self.operands[weight_id] + + bias_oper = self.operands[bias_id] + + if image_oper.op_type == NNAPI_OperandCode.TENSOR_FLOAT32: + if weight_oper.op_type != NNAPI_OperandCode.TENSOR_FLOAT32: + raise AssertionError( + f"expected weight_oper TENSOR_FLOAT32, got {weight_oper.op_type}" + ) + if bias_oper.op_type != NNAPI_OperandCode.TENSOR_FLOAT32: + raise AssertionError( + f"expected bias_oper TENSOR_FLOAT32, got {bias_oper.op_type}" + ) + elif image_oper.op_type == NNAPI_OperandCode.TENSOR_QUANT8_ASYMM: + if weight_oper.op_type != NNAPI_OperandCode.TENSOR_QUANT8_ASYMM: + raise AssertionError( + f"expected weight_oper TENSOR_QUANT8_ASYMM, got {weight_oper.op_type}" + ) + if bias_oper.op_type != NNAPI_OperandCode.TENSOR_INT32: + raise AssertionError( + f"expected bias_oper TENSOR_INT32, got {bias_oper.op_type}" + ) + if not approx_equal(image_oper.scale * weight_oper.scale, bias_oper.scale): + raise AssertionError( + f"scale mismatch: image*weight scale {image_oper.scale * weight_oper.scale} != bias scale {bias_oper.scale}" + ) + if bias_oper.zero_point != 0: + raise AssertionError( + f"expected bias_oper.zero_point == 0, got {bias_oper.zero_point}" + ) + else: + raise Exception( # noqa: TRY002 + f"Unsupported input type for conv2d: {image_oper.op_type}" + ) + + if len(image_oper.shape) != 4: + raise AssertionError( + f"expected len(image_oper.shape) == 4, got {len(image_oper.shape)}" + ) + if len(weight_oper.shape) != 4: + raise AssertionError( + f"expected len(weight_oper.shape) == 4, got {len(weight_oper.shape)}" + ) + if len(bias_oper.shape) != 1: + raise AssertionError( + f"expected len(bias_oper.shape) == 1, got {len(bias_oper.shape)}" + ) + + if depthwise: + # Depthwise convolution + one, _kern_h, _kern_w, out_c = weight_oper.shape + if one != 1: + raise AssertionError(f"expected weight_oper.shape[0] == 1, got {one}") + if out_c % in_c != 0: + raise AssertionError(f"out_c {out_c} must be divisible by in_c {in_c}") + channel_multiplier = out_c // in_c + if channel_multiplier != 1: + raise AssertionError( + f"channel_multiplier must be 1, got {channel_multiplier}" + ) + if out_c != in_c: + raise AssertionError(f"out_c {out_c} != in_c {in_c}") + else: + # Full convolution + out_c, _kern_h, _kern_w, kern_d = weight_oper.shape + if kern_d != in_c: + raise AssertionError(f"kern_d {kern_d} != in_c {in_c}") + + if out_c != bias_oper.shape[0]: + raise AssertionError( + f"out_c {out_c} != bias_oper.shape[0] {bias_oper.shape[0]}" + ) + + use_nchw = image_oper.use_nchw() + + if depthwise: + num_args = 12 + opcode = NNAPI_OperationCode.DEPTHWISE_CONV_2D + else: + num_args = 11 + if transpose: + opcode = NNAPI_OperationCode.TRANSPOSE_CONV_2D + else: + opcode = NNAPI_OperationCode.CONV_2D + + inputs = [None] * num_args + inputs[0] = image_id + inputs[1] = weight_id + inputs[2] = bias_id + inputs[3] = self.add_immediate_int_scalar(args.pad_l) + inputs[4] = self.add_immediate_int_scalar(args.pad_r) + inputs[5] = self.add_immediate_int_scalar(args.pad_t) + inputs[6] = self.add_immediate_int_scalar(args.pad_b) + inputs[7] = self.add_immediate_int_scalar(args.stride_w) + inputs[8] = self.add_immediate_int_scalar(args.stride_h) + if depthwise: + inputs[9] = self.add_immediate_int_scalar(1) + inputs[10] = self.add_immediate_int_scalar(fuse_code) + inputs[11] = self.add_immediate_bool_scalar(use_nchw) + else: + inputs[9] = self.add_immediate_int_scalar(fuse_code) + inputs[10] = self.add_immediate_bool_scalar(use_nchw) + + outputs = [None] * 1 + out_shape = get_conv_pool_shape(image_oper.shape, args, out_c, transpose) + out_oper = image_oper._replace( + shape=out_shape, + scale=out_scale, + zero_point=out_zero_point, + ) + out_id = self.add_tensor_operand(jit_out, out_oper) + self._handle_conv_pool_flexible_input(out_id, jit_image, args, transpose) + + outputs[0] = out_id + self.add_operation(opcode, inputs, outputs) + + def _handle_conv_pool_flexible_input(self, out_id, jit_image, args, transpose): + image_id, image_oper = self.get_tensor_operand_by_jitval(jit_image) + batch, in_ch, in_h, in_w = image_oper.shape + + if batch == 0: + self.forward_operand_shape(out_id, 0, image_id, 0) + if in_ch == 0: + raise Exception("Input channels can't be flexible") # noqa: TRY002 + # H & W + if transpose: + if in_h == 0: + self.compute_operand_shape( + out_id, + 2, + f"({flex_name(image_id, 2)} - 1) * {args.stride_h} + {args.kernel_h} - {args.pad_t} - {args.pad_b}", + ) + if in_w == 0: + self.compute_operand_shape( + out_id, + 3, + f"({flex_name(image_id, 3)} - 1) * {args.stride_w} + {args.kernel_w} - {args.pad_l} - {args.pad_r}", + ) + else: + if in_h == 0: + self.compute_operand_shape( + out_id, + 2, + f"({flex_name(image_id, 2)} - {args.kernel_h} + {args.pad_t} + {args.pad_b}) // {args.stride_h} + 1", + ) + if in_w == 0: + self.compute_operand_shape( + out_id, + 3, + f"({flex_name(image_id, 3)} - {args.kernel_w} + {args.pad_l} + {args.pad_r}) // {args.stride_w} + 1", + ) + + +def serialize_model( + module, inputs, *, config=None, return_shapes=None, use_int16_for_qint16=False +): + """Convert to NNAPI and serialize torchscript module. + + Parameters: + module: Torchscript module to convert + inputs: Tensors used to specify input details for NNAPI + config (optional): Optional config to attach to module + return_shapes (optional): Specify shape of outputs if + your module uses runtime flexible shapes to set output + buffer size for NNAPI + use_int16_for_qint16 (optional): Use Pytorch int16 to represent NNAPI qint16 values + """ + return _NnapiSerializer(config, use_int16_for_qint16).serialize_model( + module, inputs, return_shapes + ) diff --git a/western_ai/usa/pytorch/torch/backends/cpu/__init__.py b/western_ai/usa/pytorch/torch/backends/cpu/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fedf31830caffed0b2b1e16de42598e1d3e0f3bc --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/cpu/__init__.py @@ -0,0 +1,21 @@ +import torch + + +__all__ = [ + "get_cpu_capability", +] + + +def get_cpu_capability() -> str: + r"""Return cpu capability as a string value. + + Possible values: + - "DEFAULT" + - "VSX" + - "Z VECTOR" + - "NO AVX" + - "AVX2" + - "AVX512" + - "SVE256" + """ + return torch._C._get_cpu_capability() diff --git a/western_ai/usa/pytorch/torch/backends/cuda/__init__.py b/western_ai/usa/pytorch/torch/backends/cuda/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..db0711a24604950c8d56b11abe37e66a5052c4c4 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/cuda/__init__.py @@ -0,0 +1,711 @@ +# mypy: allow-untyped-defs +import contextlib +from typing import Any +from typing_extensions import deprecated + +import torch + + +__all__ = [ + "is_built", + "cuFFTPlanCacheAttrContextProp", + "cuFFTPlanCache", + "cuFFTPlanCacheManager", + "cuBLASModule", + "preferred_linalg_library", + "preferred_blas_library", + "cublas_workspace_size", + "cublaslt_workspace_size", + "blas_workspace_size", + "preferred_rocm_fa_library", + "is_ck_sdpa_available", + "cufft_plan_cache", + "matmul", + "SDPAParams", + "enable_cudnn_sdp", + "cudnn_sdp_enabled", + "enable_flash_sdp", + "flash_sdp_enabled", + "enable_mem_efficient_sdp", + "mem_efficient_sdp_enabled", + "math_sdp_enabled", + "enable_math_sdp", + "allow_fp16_bf16_reduction_math_sdp", + "fp16_bf16_reduction_math_sdp_allowed", + "is_flash_attention_available", + "can_use_flash_attention", + "can_use_efficient_attention", + "can_use_cudnn_attention", + "sdp_kernel", +] + + +def is_built(): + r""" + Return whether PyTorch is built with CUDA support. + + Note that this doesn't necessarily mean CUDA is available; just that if this PyTorch + binary were run on a machine with working CUDA drivers and devices, we would be able to use it. + """ + return torch._C._has_cuda + + +class cuFFTPlanCacheAttrContextProp: + # Like regular ContextProp, but uses the `.device_index` attribute from the + # calling object as the first argument to the getter and setter. + def __init__(self, getter, setter): + self.getter = getter + self.setter = setter + + def __get__(self, obj, objtype): + return self.getter(obj.device_index) + + def __set__(self, obj, val): + if isinstance(self.setter, str): + raise RuntimeError(self.setter) + self.setter(obj.device_index, val) + + +class cuFFTPlanCache: + r""" + Represent a specific plan cache for a specific `device_index`. + + The attributes `size` and `max_size`, and method `clear`, can fetch and/ or + change properties of the C++ cuFFT plan cache. + """ + + def __init__(self, device_index): + self.device_index = device_index + + size = cuFFTPlanCacheAttrContextProp( + torch._cufft_get_plan_cache_size, + ".size is a read-only property showing the number of plans currently in the " + "cache. To change the cache capacity, set cufft_plan_cache.max_size.", + ) + + max_size = cuFFTPlanCacheAttrContextProp( + torch._cufft_get_plan_cache_max_size, torch._cufft_set_plan_cache_max_size + ) + + def clear(self): + return torch._cufft_clear_plan_cache(self.device_index) + + +class cuFFTPlanCacheManager: + r""" + Represent all cuFFT plan caches, return the cuFFTPlanCache for a given device when indexed. + + Finally, this object, when used directly as a `cuFFTPlanCache` object (e.g., + setting the `.max_size`) attribute, the current device's cuFFT plan cache is + used. + """ + + __initialized = False + + def __init__(self): + self.caches = [] + self.__initialized = True + + def __getitem__(self, device): + index = torch.cuda._utils._get_device_index(device) + if index < 0 or index >= torch.cuda.device_count(): + raise RuntimeError( + f"cufft_plan_cache: expected 0 <= device index < {torch.cuda.device_count()}, but got " + f"device with index {index}" + ) + if len(self.caches) == 0: + self.caches.extend( + cuFFTPlanCache(index) for index in range(torch.cuda.device_count()) + ) + return self.caches[index] + + def __getattr__(self, name): + return getattr(self[torch.cuda.current_device()], name) + + def __setattr__(self, name, value): + if self.__initialized: + return setattr(self[torch.cuda.current_device()], name, value) + else: + return super().__setattr__(name, value) + + +class cuBLASModule: + @staticmethod + def _parse_reduction_setting(value: Any, attr_name: str) -> tuple[bool, bool]: + def _ensure_bool(obj: Any, which: str) -> bool: + if isinstance(obj, bool): + return obj + raise TypeError( + f"{attr_name} expects a bool for {which}, but got {type(obj)!r}" + ) + + if isinstance(value, bool): + return value, True + if isinstance(value, (list, tuple)): + if not value: + raise TypeError(f"{attr_name} expects at least one boolean argument") + if len(value) > 2: + raise TypeError(f"{attr_name} expects at most two boolean arguments") + allow_reduced_precision = _ensure_bool(value[0], "allow_reduced_precision") + if len(value) == 1: + return allow_reduced_precision, True + allow_splitk = _ensure_bool(value[1], "allow_splitk") + return allow_reduced_precision, allow_splitk + raise TypeError( + f"{attr_name} expects a bool or a tuple/list of bools, but got {type(value)!r}" + ) + + def __getattr__(self, name): + if name == "allow_tf32": + return torch._C._get_cublas_allow_tf32() + elif name == "allow_fp16_reduced_precision_reduction": + allow_reduced_precision, _ = ( + torch._C._get_cublas_allow_fp16_reduced_precision_reduction() + ) + return allow_reduced_precision + elif name == "allow_fp16_reduced_precision_reduction_split_k": + _, allow_splitk = ( + torch._C._get_cublas_allow_fp16_reduced_precision_reduction() + ) + return allow_splitk + elif name == "allow_bf16_reduced_precision_reduction": + allow_reduced_precision, _ = ( + torch._C._get_cublas_allow_bf16_reduced_precision_reduction() + ) + return allow_reduced_precision + elif name == "allow_bf16_reduced_precision_reduction_split_k": + _, allow_splitk = ( + torch._C._get_cublas_allow_bf16_reduced_precision_reduction() + ) + return allow_splitk + elif name == "allow_fp16_accumulation": + return torch._C._get_cublas_allow_fp16_accumulation() + elif name == "fp32_precision": + return torch._C._get_fp32_precision_getter("cuda", "matmul") + raise AttributeError("Unknown attribute " + name) + + def __setattr__(self, name, value): + if name == "allow_tf32": + return torch._C._set_cublas_allow_tf32(value) + elif name == "allow_fp16_reduced_precision_reduction": + allow_reduced_precision, allow_splitk = self._parse_reduction_setting( + value, "allow_fp16_reduced_precision_reduction" + ) + return torch._C._set_cublas_allow_fp16_reduced_precision_reduction( + allow_reduced_precision, + allow_splitk, + ) + elif name == "allow_bf16_reduced_precision_reduction": + allow_reduced_precision, allow_splitk = self._parse_reduction_setting( + value, "allow_bf16_reduced_precision_reduction" + ) + return torch._C._set_cublas_allow_bf16_reduced_precision_reduction( + allow_reduced_precision, + allow_splitk, + ) + elif name == "allow_fp16_accumulation": + return torch._C._set_cublas_allow_fp16_accumulation(value) + elif name == "fp32_precision": + return torch._C._set_fp32_precision_setter("cuda", "matmul", value) + raise AttributeError("Unknown attribute " + name) + + +_LinalgBackends = { + "default": torch._C._LinalgBackend.Default, + "cusolver": torch._C._LinalgBackend.Cusolver, + "magma": torch._C._LinalgBackend.Magma, +} +_LinalgBackends_str = ", ".join(_LinalgBackends.keys()) + + +def preferred_linalg_library( + backend: None | str | torch._C._LinalgBackend = None, +) -> torch._C._LinalgBackend: + r""" + Override the heuristic PyTorch uses to choose between cuSOLVER and MAGMA for CUDA linear algebra operations. + + .. warning:: This flag is experimental and subject to change. + + When PyTorch runs a CUDA linear algebra operation it often uses the cuSOLVER or MAGMA libraries, + and if both are available it decides which to use with a heuristic. + This flag (a :class:`str`) allows overriding those heuristics. + + * If `"cusolver"` is set then cuSOLVER will be used wherever possible. + * If `"magma"` is set then MAGMA will be used wherever possible. + * If `"default"` (the default) is set then heuristics will be used to pick between + cuSOLVER and MAGMA if both are available. + * When no input is given, this function returns the currently preferred library. + * User may use the environment variable TORCH_LINALG_PREFER_CUSOLVER=1 to set the preferred library to cuSOLVER + globally. + This flag only sets the initial value of the preferred library and the preferred library + may still be overridden by this function call later in your script. + + Note: When a library is preferred other libraries may still be used if the preferred library + doesn't implement the operation(s) called. + This flag may achieve better performance if PyTorch's heuristic library selection is incorrect + for your application's inputs. + + Currently supported linalg operators: + + * :func:`torch.linalg.inv` + * :func:`torch.linalg.inv_ex` + * :func:`torch.linalg.cholesky` + * :func:`torch.linalg.cholesky_ex` + * :func:`torch.cholesky_solve` + * :func:`torch.cholesky_inverse` + * :func:`torch.linalg.lu_factor` + * :func:`torch.linalg.lu` + * :func:`torch.linalg.lu_solve` + * :func:`torch.linalg.qr` + * :func:`torch.linalg.eigh` + * :func:`torch.linalg.eighvals` + * :func:`torch.linalg.svd` + * :func:`torch.linalg.svdvals` + """ + if backend is None: + pass + elif isinstance(backend, str): + if backend not in _LinalgBackends: + raise RuntimeError( + f"Unknown input value. Choose from: {_LinalgBackends_str}." + ) + torch._C._set_linalg_preferred_backend(_LinalgBackends[backend]) + elif isinstance(backend, torch._C._LinalgBackend): + torch._C._set_linalg_preferred_backend(backend) + else: + raise RuntimeError("Unknown input value type.") + + return torch._C._get_linalg_preferred_backend() + + +_BlasBackends = { + "default": torch._C._BlasBackend.Default, + "cublas": torch._C._BlasBackend.Cublas, + "hipblas": torch._C._BlasBackend.Cublas, # alias + "cublaslt": torch._C._BlasBackend.Cublaslt, + "hipblaslt": torch._C._BlasBackend.Cublaslt, # alias + "ck": torch._C._BlasBackend.Ck, +} +_BlasBackends_str = ", ".join(_BlasBackends.keys()) + + +def preferred_blas_library( + backend: None | str | torch._C._BlasBackend = None, +) -> torch._C._BlasBackend: + r""" + Override the library PyTorch uses for BLAS operations. Choose between cuBLAS, cuBLASLt, and CK [ROCm-only]. + + .. warning:: This flag is experimental and subject to change. + + When PyTorch runs a CUDA BLAS operation it defaults to cuBLAS even if both cuBLAS and cuBLASLt are available. + For PyTorch built for ROCm, hipBLAS, hipBLASLt, and CK may offer different performance. + This flag (a :class:`str`) allows overriding which BLAS library to use. + + * If `"cublas"` is set then cuBLAS will be used wherever possible. + * If `"cublaslt"` is set then cuBLASLt will be used wherever possible. + * If `"ck"` is set then CK will be used wherever possible. + * If `"default"` (the default) is set then heuristics will be used to pick between the other options. + * When no input is given, this function returns the currently preferred library. + * User may use the environment variable TORCH_BLAS_PREFER_CUBLASLT=1 to set the preferred library to cuBLASLt + globally. + This flag only sets the initial value of the preferred library and the preferred library + may still be overridden by this function call later in your script. + + Note: When a library is preferred other libraries may still be used if the preferred library + doesn't implement the operation(s) called. + This flag may achieve better performance if PyTorch's library selection is incorrect + for your application's inputs. + + """ + if backend is None: + pass + elif isinstance(backend, str): + if backend not in _BlasBackends: + raise RuntimeError( + f"Unknown input value. Choose from: {_BlasBackends_str}." + ) + torch._C._set_blas_preferred_backend(_BlasBackends[backend]) + elif isinstance(backend, torch._C._BlasBackend): + torch._C._set_blas_preferred_backend(backend) + else: + raise RuntimeError("Unknown input value type.") + + return torch._C._get_blas_preferred_backend() + + +def cublas_workspace_size(size: None | int = None) -> int: + r"""Query or set the cuBLAS workspace size in bytes. + + When called with no arguments, returns the current workspace size. + When called with a size argument, sets the workspace size and returns the new value. + Setting the workspace size will take precedence over the CUBLAS_WORKSPACE_CONFIG environment variable. + Changes take effect lazily: only handles used after the change get new workspaces. + + Args: + size (int, optional): workspace size in bytes. Must be non-negative. + + Returns: + int: the current (or newly set) workspace size in bytes. + """ + if size is not None: + torch._C._cuda_setCublasWorkspaceSize(size) + return torch._C._cuda_getCublasWorkspaceSize() + + +def cublaslt_workspace_size(size: None | int = None) -> int: + r"""Query or set the cuBLASLt workspace size in bytes. + + When called with no arguments, returns the current workspace size. + When called with a size argument, sets the workspace size and returns the new value. + Setting the workspace size will take precedence over the CUBLASLT_WORKSPACE_SIZE environment variable. + Changes take effect lazily: only handles used after the change get new workspaces. + + Args: + size (int, optional): workspace size in bytes. Must be non-negative. + + Returns: + int: the current (or newly set) workspace size in bytes. + """ + if size is not None: + torch._C._cuda_setCublasLtWorkspaceSize(size) + return torch._C._cuda_getCublasLtWorkspaceSize() + + +def blas_workspace_size( + size: None | int = None, + backend: None | str | torch._C._BlasBackend = None, +) -> int: + r"""Query or set the BLAS workspace size for a given backend. + + Convenience wrapper that dispatches to :func:`cublas_workspace_size` or + :func:`cublaslt_workspace_size` depending on the backend. + + When *backend* is ``None`` the current :func:`preferred_blas_library` is + used. ``Default`` is resolved to the platform's default backend (cuBLAS + on NVIDIA, potentially hipBLASLt on supported ROCm architectures). + + .. note:: + + When ``TORCH_CUBLASLT_UNIFIED_WORKSPACE`` is enabled (the default on + open-source CUDA builds), the cuBLASLt workspace is capped at the + cuBLAS workspace size and physically reuses the same allocation. + Setting a large cuBLASLt workspace via this function will therefore + *not* increase memory beyond the cuBLAS workspace size. + + .. note:: + + Setting the workspace size for the cublas backend will take precedence + over the CUBLAS_WORKSPACE_CONFIG environment variable, and setting the + workspace size for the cublaslt backend will take precedence over the + CUBLASLT_WORKSPACE_SIZE environment variable. + + Args: + size (int, optional): workspace size in bytes. Must be non-negative. + When omitted the current size is returned without modification. + backend (str | torch._C._BlasBackend, optional): which backend's + workspace to query/set. Accepts the same strings as + :func:`preferred_blas_library` (e.g. ``"cublas"``, ``"cublaslt"``). + + Returns: + int: the current (or newly set) workspace size in bytes. + + Raises: + RuntimeError: if the resolved backend is CK (no workspace concept). + """ + if backend is None: + resolved = preferred_blas_library() + elif isinstance(backend, str): + if backend not in _BlasBackends: + raise RuntimeError( + f"Unknown backend string. Choose from: {_BlasBackends_str}." + ) + resolved = _BlasBackends[backend] + elif isinstance(backend, torch._C._BlasBackend): + resolved = backend + else: + raise RuntimeError("Unknown backend type.") + + if resolved == torch._C._BlasBackend.Default: + resolved = torch._C._get_blas_default_backend() + + if resolved == torch._C._BlasBackend.Ck: + raise RuntimeError("CK backend does not use a workspace.") + + if resolved == torch._C._BlasBackend.Cublaslt: + return cublaslt_workspace_size(size) + return cublas_workspace_size(size) + + +_ROCmFABackends = { + "default": torch._C._ROCmFABackend.Default, + "aotriton": torch._C._ROCmFABackend.AOTriton, + "ck": torch._C._ROCmFABackend.Ck, +} +_ROCmFABackends_str = ", ".join(_ROCmFABackends.keys()) + + +from torch._C import _SDPAParams as SDPAParams, _SDPBackend as SDPBackend + + +def preferred_rocm_fa_library( + backend: None | str | torch._C._ROCmFABackend = None, +) -> torch._C._ROCmFABackend: + r""" + [ROCm-only] + Override the backend PyTorch uses in ROCm environments for Flash Attention. Choose between AOTriton and CK + + .. warning:: This flag is experimental and subject to change. + + When Flash Attention is enabled and desired, PyTorch defaults to using AOTriton as the backend. + This flag (a :class:`str`) allows users to override this backend to use composable_kernel + + * If `"default"` is set then the default backend will be used wherever possible. Currently AOTriton. + * If `"aotriton"` is set then AOTriton will be used wherever possible. + * If `"ck"` is set then CK will be used wherever possible. + * When no input is given, this function returns the currently preferred library. + * User may use the environment variable TORCH_ROCM_FA_PREFER_CK=1 to set the preferred library to CK + globally. + + Note: When a library is preferred other libraries may still be used if the preferred library + doesn't implement the operation(s) called. + This flag may achieve better performance if PyTorch's library selection is incorrect + for your application's inputs. + """ + if backend is None: + pass + elif isinstance(backend, str): + if backend not in _ROCmFABackends: + raise RuntimeError( + f"Unknown input value. Choose from: {_ROCmFABackends_str}." + ) + torch._C._set_rocm_fa_preferred_backend(_ROCmFABackends[backend]) + elif isinstance(backend, torch._C._ROCmFABackend): + torch._C._set_rocm_fa_preferred_backend(backend) + else: + raise ValueError(f"Unknown input value. Choose from: {_ROCmFABackends_str}.") + + return torch._C._get_rocm_fa_preferred_backend() + + +# Set the __module__ attribute +SDPAParams.__module__ = "torch.backends.cuda" +SDPAParams.__name__ = "SDPAParams" + + +def is_ck_sdpa_available() -> bool: + r""" + .. warning:: This flag is beta and subject to change. + + Returns whether composable_kernel may be used as the backend for + scaled-dot-product-attention. + """ + # pyrefly: ignore [missing-attribute] + return torch._C._is_ck_sdpa_available() + + +def flash_sdp_enabled(): + r""" + .. warning:: This flag is beta and subject to change. + + Returns whether flash scaled dot product attention is enabled or not. + """ + return torch._C._get_flash_sdp_enabled() + + +def enable_flash_sdp(enabled: bool): + r""" + .. warning:: This flag is beta and subject to change. + + Enables or disables flash scaled dot product attention. + """ + torch._C._set_sdp_use_flash(enabled) + + +def mem_efficient_sdp_enabled(): + r""" + .. warning:: This flag is beta and subject to change. + + Returns whether memory efficient scaled dot product attention is enabled or not. + """ + return torch._C._get_mem_efficient_sdp_enabled() + + +def enable_mem_efficient_sdp(enabled: bool): + r""" + .. warning:: This flag is beta and subject to change. + + Enables or disables memory efficient scaled dot product attention. + """ + torch._C._set_sdp_use_mem_efficient(enabled) + + +def math_sdp_enabled(): + r""" + .. warning:: This flag is beta and subject to change. + + Returns whether math scaled dot product attention is enabled or not. + """ + return torch._C._get_math_sdp_enabled() + + +def enable_math_sdp(enabled: bool): + r""" + .. warning:: This flag is beta and subject to change. + + Enables or disables math scaled dot product attention. + """ + torch._C._set_sdp_use_math(enabled) + + +def allow_fp16_bf16_reduction_math_sdp(enabled: bool): + r""" + .. warning:: This flag is beta and subject to change. + + Enables or disables fp16/bf16 reduction in math scaled dot product attention. + """ + torch._C._set_math_sdp_allow_fp16_bf16_reduction(enabled) + + +def fp16_bf16_reduction_math_sdp_allowed(): + r""" + .. warning:: This flag is beta and subject to change. + + Returns whether fp16/bf16 reduction in math scaled dot product attention is enabled or not. + """ + return torch._C._get_math_sdp_allow_fp16_bf16_reduction() + + +def is_flash_attention_available() -> bool: + r"""Check if PyTorch was built with FlashAttention for scaled_dot_product_attention. + + Returns: + True if FlashAttention is built and available; otherwise, False. + + Note: + This function is dependent on a CUDA-enabled build of PyTorch. It will return False + in non-CUDA environments. + """ + return torch._C._is_flash_attention_available() + + +def can_use_flash_attention(params: SDPAParams, debug: bool = False) -> bool: + r"""Check if FlashAttention can be utilized in scaled_dot_product_attention. + + Args: + params: An instance of SDPAParams containing the tensors for query, + key, value, an optional attention mask, dropout rate, and + a flag indicating if the attention is causal. + debug: Whether to logging.warn debug information as to why FlashAttention could not be run. + Defaults to False. + + Returns: + True if FlashAttention can be used with the given parameters; otherwise, False. + + Note: + This function is dependent on a CUDA-enabled build of PyTorch. It will return False + in non-CUDA environments. + """ + return torch._C._can_use_flash_attention(params, debug) + + +def can_use_efficient_attention(params: SDPAParams, debug: bool = False) -> bool: + r"""Check if efficient_attention can be utilized in scaled_dot_product_attention. + + Args: + params: An instance of SDPAParams containing the tensors for query, + key, value, an optional attention mask, dropout rate, and + a flag indicating if the attention is causal. + debug: Whether to logging.warn with information as to why efficient_attention could not be run. + Defaults to False. + + Returns: + True if efficient_attention can be used with the given parameters; otherwise, False. + + Note: + This function is dependent on a CUDA-enabled build of PyTorch. It will return False + in non-CUDA environments. + """ + return torch._C._can_use_mem_efficient_attention(params, debug) + + +def can_use_cudnn_attention(params: SDPAParams, debug: bool = False) -> bool: + r"""Check if cudnn_attention can be utilized in scaled_dot_product_attention. + + Args: + params: An instance of SDPAParams containing the tensors for query, + key, value, an optional attention mask, dropout rate, and + a flag indicating if the attention is causal. + debug: Whether to logging.warn with information as to why cuDNN attention could not be run. + Defaults to False. + + Returns: + True if cuDNN can be used with the given parameters; otherwise, False. + + Note: + This function is dependent on a CUDA-enabled build of PyTorch. It will return False + in non-CUDA environments. + """ + return torch._C._can_use_cudnn_attention(params, debug) + + +def cudnn_sdp_enabled(): + r""" + .. warning:: This flag is beta and subject to change. + + Returns whether cuDNN scaled dot product attention is enabled or not. + """ + return torch._C._get_cudnn_sdp_enabled() + + +def enable_cudnn_sdp(enabled: bool): + r""" + .. warning:: This flag is beta and subject to change. + + Enables or disables cuDNN scaled dot product attention. + """ + torch._C._set_sdp_use_cudnn(enabled) + + +@contextlib.contextmanager +@deprecated( + ( + "`torch.backends.cuda.sdp_kernel()` is deprecated. " + "In the future, this context manager will be removed. " + "Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, " + "with updated signature." + ), + category=FutureWarning, +) +def sdp_kernel( + enable_flash: bool = True, + enable_math: bool = True, + enable_mem_efficient: bool = True, + enable_cudnn: bool = True, +): + r""" + .. warning:: This flag is beta and subject to change. + + This context manager can be used to temporarily enable or disable any of the three backends for scaled dot product attention. + Upon exiting the context manager, the previous state of the flags will be restored. + """ + from torch.nn.attention import sdpa_kernel + + backend_list = [] + if enable_flash: + backend_list.append(SDPBackend.FLASH_ATTENTION) + if enable_mem_efficient: + backend_list.append(SDPBackend.EFFICIENT_ATTENTION) + if enable_math: + backend_list.append(SDPBackend.MATH) + if enable_cudnn: + backend_list.append(SDPBackend.CUDNN_ATTENTION) + + with sdpa_kernel(backend_list) as context: + try: + yield context + finally: + pass + + +cufft_plan_cache = cuFFTPlanCacheManager() +matmul = cuBLASModule() diff --git a/western_ai/usa/pytorch/torch/backends/cudnn/__init__.py b/western_ai/usa/pytorch/torch/backends/cudnn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7a341e1a63e4b858258706644dd92781459b0b1a --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/cudnn/__init__.py @@ -0,0 +1,261 @@ +# mypy: allow-untyped-defs +import os +import sys +import warnings +from contextlib import contextmanager +from typing import Optional + +import torch +from torch.backends import ( + __allow_nonbracketed_mutation, + _FP32Precision, + _get_fp32_precision_getter, + _set_fp32_precision_setter, + ContextProp, + PropModule, +) + +from . import rnn + + +try: + from torch._C import _cudnn +except ImportError: + _cudnn = None # type: ignore[assignment] + +# Write: +# +# torch.backends.cudnn.enabled = False +# +# to globally disable CuDNN/MIOpen + +__cudnn_version: int | None = None + +if _cudnn is not None: + + def _init(): + global __cudnn_version + if __cudnn_version is None: + # pyrefly: ignore [missing-attribute] + __cudnn_version = _cudnn.getVersionInt() + # pyrefly: ignore [missing-attribute] + runtime_version = _cudnn.getRuntimeVersion() + # pyrefly: ignore [missing-attribute] + compile_version = _cudnn.getCompileVersion() + runtime_major, runtime_minor, _ = runtime_version + compile_major, compile_minor, _ = compile_version + # Different major versions are always incompatible + # Starting with cuDNN 7, minor versions are backwards-compatible + # Not sure about MIOpen (ROCm), so always do a strict check + if runtime_major != compile_major: + cudnn_compatible = False + # pyrefly: ignore [missing-attribute] + elif runtime_major < 7 or not _cudnn.is_cuda: + cudnn_compatible = runtime_minor == compile_minor + else: + cudnn_compatible = runtime_minor >= compile_minor + if not cudnn_compatible: + if os.environ.get("PYTORCH_SKIP_CUDNN_COMPATIBILITY_CHECK", "0") == "1": + return True + base_error_msg = ( + f"cuDNN version incompatibility: " + f"PyTorch was compiled against {compile_version} " + f"but found runtime version {runtime_version}. " + f"PyTorch already comes bundled with cuDNN. " + f"One option to resolving this error is to ensure PyTorch " + f"can find the bundled cuDNN. " + ) + + if "LD_LIBRARY_PATH" in os.environ: + ld_library_path = os.environ.get("LD_LIBRARY_PATH", "") + if any( + substring in ld_library_path for substring in ["cuda", "cudnn"] + ): + raise RuntimeError( + f"{base_error_msg}" + f"Looks like your LD_LIBRARY_PATH contains incompatible version of cudnn. " + f"Please either remove it from the path or install cudnn {compile_version}" + ) + else: + raise RuntimeError( + f"{base_error_msg}" + f"one possibility is that there is a " + f"conflicting cuDNN in LD_LIBRARY_PATH." + ) + else: + raise RuntimeError(base_error_msg) + # Check if cuDNN version is compatible with available CUDA devices + if torch.cuda.is_available() and not torch.version.hip: + min_cc = min( + [ + torch.cuda.get_device_capability(i) + for i in range(torch.cuda.device_count()) + ] + ) + if __cudnn_version >= 91100 and min_cc < (7, 5): + raise RuntimeError( + f"cuDNN version {__cudnn_version} is not compatible with devices with SM < 7.5. " + f"Please install a version of PyTorch with a compatible cuDNN version. " + f"https://github.com/pytorch/pytorch/blob/main/RELEASE.md#release-compatibility-matrix" + ) + + return True + +else: + + def _init(): + return False + + +def version(): + """Return the version of cuDNN.""" + if not _init(): + return None + return __cudnn_version + + +CUDNN_TENSOR_DTYPES = { + torch.half, + torch.float, + torch.double, +} + + +def is_available(): + r"""Return a bool indicating if CUDNN is currently available.""" + return torch._C._has_cudnn + + +def is_acceptable(tensor): + if not torch._C._get_cudnn_enabled(): + return False + if tensor.device.type != "cuda" or tensor.dtype not in CUDNN_TENSOR_DTYPES: + return False + if not is_available(): + warnings.warn( + "PyTorch was compiled without cuDNN/MIOpen support. To use cuDNN/MIOpen, rebuild " + "PyTorch making sure the library is visible to the build system.", + stacklevel=2, + ) + return False + if not _init(): + warnings.warn( + "cuDNN/MIOpen library not found. Check your {libpath}".format( + libpath={"darwin": "DYLD_LIBRARY_PATH", "win32": "PATH"}.get( + sys.platform, "LD_LIBRARY_PATH" + ) + ), + stacklevel=2, + ) + return False + return True + + +def set_flags( + _enabled=None, + _benchmark=None, + _benchmark_limit=None, + _deterministic=None, + _allow_tf32=None, + _fp32_precision="none", + _depthwise_kernel=None, +): + orig_flags = ( + torch._C._get_cudnn_enabled(), + torch._C._get_cudnn_benchmark(), + None if not is_available() else torch._C._cuda_get_cudnn_benchmark_limit(), + torch._C._get_cudnn_deterministic(), + torch._C._get_cudnn_allow_tf32(), + torch._C._get_fp32_precision_getter("cuda", "all"), + torch._C._get_cudnn_depthwise_kernel(), + ) + if _enabled is not None: + torch._C._set_cudnn_enabled(_enabled) + if _benchmark is not None: + torch._C._set_cudnn_benchmark(_benchmark) + if _benchmark_limit is not None and is_available(): + torch._C._cuda_set_cudnn_benchmark_limit(_benchmark_limit) + if _deterministic is not None: + torch._C._set_cudnn_deterministic(_deterministic) + if _allow_tf32 is not None: + torch._C._set_cudnn_allow_tf32(_allow_tf32) + if _fp32_precision is not None: + torch._C._set_fp32_precision_setter("cuda", "all", _fp32_precision) + if _depthwise_kernel is not None: + torch._C._set_cudnn_depthwise_kernel(_depthwise_kernel) + return orig_flags + + +@contextmanager +def flags( + enabled=False, + benchmark=False, + benchmark_limit=10, + deterministic=False, + allow_tf32=True, + fp32_precision="none", + depthwise_kernel="auto", +): + with __allow_nonbracketed_mutation(): + orig_flags = set_flags( + enabled, + benchmark, + benchmark_limit, + deterministic, + allow_tf32, + fp32_precision, + depthwise_kernel, + ) + try: + yield + finally: + # recover the previous values + with __allow_nonbracketed_mutation(): + set_flags(*orig_flags) + + +# The magic here is to allow us to intercept code like this: +# +# torch.backends..enabled = True + + +class CudnnModule(PropModule): + enabled = ContextProp(torch._C._get_cudnn_enabled, torch._C._set_cudnn_enabled) + deterministic = ContextProp( + torch._C._get_cudnn_deterministic, torch._C._set_cudnn_deterministic + ) + benchmark = ContextProp( + torch._C._get_cudnn_benchmark, torch._C._set_cudnn_benchmark + ) + benchmark_limit = None + if is_available(): + benchmark_limit = ContextProp( + torch._C._cuda_get_cudnn_benchmark_limit, + torch._C._cuda_set_cudnn_benchmark_limit, + ) + allow_tf32 = ContextProp( + torch._C._get_cudnn_allow_tf32, torch._C._set_cudnn_allow_tf32 + ) + conv = _FP32Precision("cuda", "conv") + fp32_precision = ContextProp( + _get_fp32_precision_getter("cuda", "all"), + _set_fp32_precision_setter("cuda", "all"), + ) + depthwise_kernel = ContextProp( + torch._C._get_cudnn_depthwise_kernel, + torch._C._set_cudnn_depthwise_kernel, + ) + + +# This is the sys.modules replacement trick, see +# https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273 +sys.modules[__name__] = CudnnModule(sys.modules[__name__], __name__) + +# Add type annotation for the replaced module +enabled: bool +deterministic: bool +benchmark: bool +allow_tf32: bool +fp32_precision: str +benchmark_limit: int +depthwise_kernel: str diff --git a/western_ai/usa/pytorch/torch/backends/cudnn/rnn.py b/western_ai/usa/pytorch/torch/backends/cudnn/rnn.py new file mode 100644 index 0000000000000000000000000000000000000000..c62968c43c9e97c7d5663b27abe36c119dd7f3c3 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/cudnn/rnn.py @@ -0,0 +1,128 @@ +# mypy: allow-untyped-defs +import sys + +import torch._C +import torch.cuda +from torch.backends import ( + _get_fp32_precision_getter, + _set_fp32_precision_setter, + PropModule, +) + + +try: + from torch._C import _cudnn +except ImportError: + # Uses of all the functions below should be guarded by torch.backends.cudnn.is_available(), + # so it's safe to not emit any checks here. + _cudnn = None # type: ignore[assignment] + + +def get_cudnn_mode(mode): + if mode == "RNN_RELU": + # pyrefly: ignore [missing-attribute] + return int(_cudnn.RNNMode.rnn_relu) + elif mode == "RNN_TANH": + # pyrefly: ignore [missing-attribute] + return int(_cudnn.RNNMode.rnn_tanh) + elif mode == "LSTM": + # pyrefly: ignore [missing-attribute] + return int(_cudnn.RNNMode.lstm) + elif mode == "GRU": + # pyrefly: ignore [missing-attribute] + return int(_cudnn.RNNMode.gru) + else: + raise ValueError(f"Unknown mode: {mode}") + + +# NB: We don't actually need this class anymore (in fact, we could serialize the +# dropout state for even better reproducibility), but it is kept for backwards +# compatibility for old models. +class Unserializable: + def __init__(self, inner): + self.inner = inner + + def get(self): + return self.inner + + def __getstate__(self): + return "" + + def __setstate__(self, state): + self.inner = None + + +# we would like to use ContextProp from backends here but the +# frozen flags appears to be overzealous +class ContextProp: + def __init__(self, getter, setter): + self.getter = getter + self.setter = setter + + def __get__(self, obj, objtype): + return self.getter() + + def __set__(self, obj, val): + self.setter(val) + + +def init_dropout_state(dropout, train, dropout_seed, dropout_state): + dropout_desc_name = "desc_" + str(torch.cuda.current_device()) + dropout_p = dropout if train else 0 + if (dropout_desc_name not in dropout_state) or ( + dropout_state[dropout_desc_name].get() is None + ): + if dropout_p == 0: + dropout_state[dropout_desc_name] = Unserializable(None) + else: + dropout_state[dropout_desc_name] = Unserializable( + torch._cudnn_init_dropout_state( # type: ignore[call-arg] + dropout_p, + train, + dropout_seed, + # pyrefly: ignore [unexpected-keyword] + self_ty=torch.uint8, + device=torch.device("cuda"), + ) + ) + dropout_ts = dropout_state[dropout_desc_name].get() + return dropout_ts + + +class CudnnRNNModule(PropModule): + def __init__(self, m, name): + super().__init__(m, name) + self.m.Unserializable = Unserializable + self.m.get_cudnn_mode = get_cudnn_mode + self.m.init_dropout_state = init_dropout_state + + @staticmethod + def init_dropout_state(dropout, train, dropout_seed, dropout_state): + dropout_desc_name = "desc_" + str(torch.cuda.current_device()) + dropout_p = dropout if train else 0 + if (dropout_desc_name not in dropout_state) or ( + dropout_state[dropout_desc_name].get() is None + ): + if dropout_p == 0: + dropout_state[dropout_desc_name] = Unserializable(None) + else: + dropout_state[dropout_desc_name] = Unserializable( + torch._cudnn_init_dropout_state( # type: ignore[call-arg] + dropout_p, + train, + dropout_seed, + # pyrefly: ignore [unexpected-keyword] + self_ty=torch.uint8, + device=torch.device("cuda"), + ) + ) + dropout_ts = dropout_state[dropout_desc_name].get() + return dropout_ts + + fp32_precision = ContextProp( + _get_fp32_precision_getter("cuda", "rnn"), + _set_fp32_precision_setter("cuda", "rnn"), + ) + + +sys.modules[__name__] = CudnnRNNModule(sys.modules[__name__], __name__) diff --git a/western_ai/usa/pytorch/torch/backends/cusparselt/__init__.py b/western_ai/usa/pytorch/torch/backends/cusparselt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..757ab460f2a58755ed7561d15f294d33921eda83 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/cusparselt/__init__.py @@ -0,0 +1,57 @@ +from typing import Optional + +import torch + + +__all__ = [ + "version", + "is_available", + "get_max_alg_id", +] + +try: + from torch._C import _cusparselt +except ImportError: + _cusparselt = None # type: ignore[assignment] + +__cusparselt_version: int | None = None +__MAX_ALG_ID: int | None = None + +if _cusparselt is not None: + + def _init() -> bool: + global __cusparselt_version + global __MAX_ALG_ID + if __cusparselt_version is None: + # pyrefly: ignore [missing-attribute] + __cusparselt_version = _cusparselt.getVersionInt() + if __cusparselt_version == 400: + __MAX_ALG_ID = 4 + elif __cusparselt_version == 502: + __MAX_ALG_ID = 5 + elif __cusparselt_version == 602: + __MAX_ALG_ID = 37 + return True + +else: + + def _init() -> bool: + return False + + +def version() -> int | None: + """Return the version of cuSPARSELt""" + if not _init(): + return None + return __cusparselt_version + + +def is_available() -> bool: + r"""Return a bool indicating if cuSPARSELt is currently available.""" + return torch._C._has_cusparselt + + +def get_max_alg_id() -> int | None: + if not _init(): + return None + return __MAX_ALG_ID diff --git a/western_ai/usa/pytorch/torch/backends/kleidiai/__init__.py b/western_ai/usa/pytorch/torch/backends/kleidiai/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4ba82b8f269eafc30d8964b68e84e28cb957523f --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/kleidiai/__init__.py @@ -0,0 +1,7 @@ +# mypy: allow-untyped-defs +import torch + + +def is_available(): + r"""Return whether PyTorch is built with KleidiAI support.""" + return torch._C._has_kleidiai diff --git a/western_ai/usa/pytorch/torch/backends/mha/__init__.py b/western_ai/usa/pytorch/torch/backends/mha/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..54693e7dd54720c74a1157cc413bbcd6beab6592 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/mha/__init__.py @@ -0,0 +1,25 @@ +# Config options to enable/disable C++ kernel for nn.functional.MHA +# and nn.TransformerEncoder +import torch + + +_is_fastpath_enabled: bool = True + + +def get_fastpath_enabled() -> bool: + """Returns whether fast path for TransformerEncoder and MultiHeadAttention + is enabled, or ``True`` if jit is scripting. + + .. note:: + The fastpath might not be run even if ``get_fastpath_enabled`` returns + ``True`` unless all conditions on inputs are met. + """ + if not torch.jit.is_scripting(): + return _is_fastpath_enabled + return True + + +def set_fastpath_enabled(value: bool) -> None: + """Sets whether fast path is enabled""" + global _is_fastpath_enabled + _is_fastpath_enabled = value diff --git a/western_ai/usa/pytorch/torch/backends/miopen/__init__.py b/western_ai/usa/pytorch/torch/backends/miopen/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4fb7256ea46407b854dfe288c4dc0260ae94acc0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/miopen/__init__.py @@ -0,0 +1,50 @@ +# mypy: allow-untyped-defs +import sys +from contextlib import contextmanager + +import torch +from torch.backends import __allow_nonbracketed_mutation, ContextProp, PropModule + + +def set_flags( + _immediate=None, +): + orig_flags = (torch._C._get_miopen_immediate(),) + if _immediate is not None: + torch._C._set_miopen_immediate(_immediate) + return orig_flags + + +@contextmanager +def flags( + immediate=False, +): + with __allow_nonbracketed_mutation(): + orig_flags = set_flags( + immediate, + ) + try: + yield + finally: + # recover the previous values + with __allow_nonbracketed_mutation(): + set_flags(*orig_flags) + + +# The magic here is to allow us to intercept code like this: +# +# torch.backends..immediate = True + + +class MiopenModule(PropModule): + immediate = ContextProp( + torch._C._get_miopen_immediate, torch._C._set_miopen_immediate + ) + + +# This is the sys.modules replacement trick, see +# https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273 +sys.modules[__name__] = MiopenModule(sys.modules[__name__], __name__) + +# Add type annotation for the replaced module +immediate: bool diff --git a/western_ai/usa/pytorch/torch/backends/mkl/__init__.py b/western_ai/usa/pytorch/torch/backends/mkl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6c7b20e9dacbda66e4c5e7d82de4cef730aa68a5 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/mkl/__init__.py @@ -0,0 +1,59 @@ +# mypy: allow-untyped-defs +import torch + + +def is_available(): + r"""Return whether PyTorch is built with MKL support.""" + return torch._C.has_mkl + + +VERBOSE_OFF = 0 +VERBOSE_ON = 1 + + +class verbose: + """ + On-demand oneMKL verbosing functionality. + + To make it easier to debug performance issues, oneMKL can dump verbose + messages containing execution information like duration while executing + the kernel. The verbosing functionality can be invoked via an environment + variable named `MKL_VERBOSE`. However, this methodology dumps messages in + all steps. Those are a large amount of verbose messages. Moreover, for + investigating the performance issues, generally taking verbose messages + for one single iteration is enough. This on-demand verbosing functionality + makes it possible to control scope for verbose message dumping. In the + following example, verbose messages will be dumped out for the second + inference only. + + .. highlight:: python + .. code-block:: python + + import torch + + model(data) + with torch.backends.mkl.verbose(torch.backends.mkl.VERBOSE_ON): + model(data) + + Args: + level: Verbose level + - ``VERBOSE_OFF``: Disable verbosing + - ``VERBOSE_ON``: Enable verbosing + """ + + def __init__(self, enable): + self.enable = enable + + def __enter__(self): + if self.enable == VERBOSE_OFF: + return + st = torch._C._verbose.mkl_set_verbose(self.enable) + if not st: + raise AssertionError( + "Failed to set MKL into verbose mode. Please consider to disable this verbose scope." + ) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + torch._C._verbose.mkl_set_verbose(VERBOSE_OFF) + return False diff --git a/western_ai/usa/pytorch/torch/backends/mkldnn/__init__.py b/western_ai/usa/pytorch/torch/backends/mkldnn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..97e858b173b0dbe316f94e76e8c87bea72a16887 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/mkldnn/__init__.py @@ -0,0 +1,139 @@ +# mypy: allow-untyped-defs +import sys +from contextlib import contextmanager +from typing import TYPE_CHECKING + +import torch +from torch.backends import ( + __allow_nonbracketed_mutation, + _FP32Precision, + _get_fp32_precision_getter, + _set_fp32_precision_setter, + ContextProp, + PropModule, +) + + +def is_available(): + r"""Return whether PyTorch is built with MKL-DNN support.""" + return torch._C._has_mkldnn + + +def is_acl_available(): + r"""Return whether PyTorch is built with MKL-DNN + ACL support.""" + # pyrefly: ignore [missing-attribute] + return torch._C._has_mkldnn_acl + + +VERBOSE_OFF = 0 +VERBOSE_ON = 1 +VERBOSE_ON_CREATION = 2 + + +class verbose: + """ + On-demand oneDNN (former MKL-DNN) verbosing functionality. + + To make it easier to debug performance issues, oneDNN can dump verbose + messages containing information like kernel size, input data size and + execution duration while executing the kernel. The verbosing functionality + can be invoked via an environment variable named `DNNL_VERBOSE`. However, + this methodology dumps messages in all steps. Those are a large amount of + verbose messages. Moreover, for investigating the performance issues, + generally taking verbose messages for one single iteration is enough. + This on-demand verbosing functionality makes it possible to control scope + for verbose message dumping. In the following example, verbose messages + will be dumped out for the second inference only. + + .. highlight:: python + .. code-block:: python + + import torch + + model(data) + with torch.backends.mkldnn.verbose(torch.backends.mkldnn.VERBOSE_ON): + model(data) + + Args: + level: Verbose level + - ``VERBOSE_OFF``: Disable verbosing + - ``VERBOSE_ON``: Enable verbosing + - ``VERBOSE_ON_CREATION``: Enable verbosing, including oneDNN kernel creation + """ + + def __init__(self, level): + self.level = level + + def __enter__(self): + if self.level == VERBOSE_OFF: + return + st = torch._C._verbose.mkldnn_set_verbose(self.level) + if not st: + raise AssertionError( + "Failed to set MKLDNN into verbose mode. Please consider to disable this verbose scope." + ) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + torch._C._verbose.mkldnn_set_verbose(VERBOSE_OFF) + return False + + +def set_flags( + _enabled=None, _deterministic=None, _allow_tf32=None, _fp32_precision="none" +): + orig_flags = ( + torch._C._get_mkldnn_enabled(), + torch._C._get_mkldnn_deterministic(), + torch._C._get_onednn_allow_tf32(), + torch._C._get_fp32_precision_getter("mkldnn", "all"), + ) + if _enabled is not None: + torch._C._set_mkldnn_enabled(_enabled) + if _deterministic is not None: + torch._C._set_mkldnn_deterministic(_deterministic) + if _allow_tf32 is not None: + torch._C._set_onednn_allow_tf32(_allow_tf32) + if _fp32_precision is not None: + torch._C._set_fp32_precision_setter("mkldnn", "all", _fp32_precision) + return orig_flags + + +@contextmanager +def flags(enabled=False, deterministic=False, allow_tf32=True, fp32_precision="none"): + with __allow_nonbracketed_mutation(): + orig_flags = set_flags(enabled, deterministic, allow_tf32, fp32_precision) + try: + yield + finally: + with __allow_nonbracketed_mutation(): + set_flags(*orig_flags) + + +class MkldnnModule(PropModule): + def is_available(self): + return is_available() + + enabled = ContextProp(torch._C._get_mkldnn_enabled, torch._C._set_mkldnn_enabled) + deterministic = ContextProp( + torch._C._get_mkldnn_deterministic, torch._C._set_mkldnn_deterministic + ) + allow_tf32 = ContextProp( + torch._C._get_onednn_allow_tf32, torch._C._set_onednn_allow_tf32 + ) + matmul = _FP32Precision("mkldnn", "matmul") + conv = _FP32Precision("mkldnn", "conv") + rnn = _FP32Precision("mkldnn", "rnn") + fp32_precision = ContextProp( + _get_fp32_precision_getter("mkldnn", "all"), + _set_fp32_precision_setter("generic", "all"), + ) + + +if TYPE_CHECKING: + enabled: ContextProp + deterministic: ContextProp + allow_tf32: ContextProp + fp32_precision: str + +sys.modules[__name__] = MkldnnModule(sys.modules[__name__], __name__) diff --git a/western_ai/usa/pytorch/torch/backends/mps/__init__.py b/western_ai/usa/pytorch/torch/backends/mps/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..db2af7e8874179fd35b9379ce68e3a9c6bb17264 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/mps/__init__.py @@ -0,0 +1,60 @@ +from functools import lru_cache as _lru_cache +from typing import Optional, TYPE_CHECKING + +import torch +from torch.library import Library as _Library + + +__all__ = [ + "get_core_count", + "get_name", + "is_built", + "is_available", + "is_macos13_or_newer", + "is_macos_or_newer", +] + + +def is_built() -> bool: + r"""Return whether PyTorch is built with MPS support. + + Note that this doesn't necessarily mean MPS is available; just that + if this PyTorch binary were run a machine with working MPS drivers + and devices, we would be able to use it. + """ + return torch._C._has_mps + + +@_lru_cache +def is_available() -> bool: + r"""Return a bool indicating if MPS is currently available.""" + return torch._C._mps_is_available() + + +@_lru_cache +def is_macos_or_newer(major: int, minor: int) -> bool: + r"""Return a bool indicating whether MPS is running on given MacOS or newer.""" + return torch._C._mps_is_on_macos_or_newer(major, minor) + + +@_lru_cache +def is_macos13_or_newer(minor: int = 0) -> bool: + r"""Return a bool indicating whether MPS is running on MacOS 13 or newer.""" + return torch._C._mps_is_on_macos_or_newer(13, minor) + + +@_lru_cache +def get_name() -> str: + r"""Return Metal device name""" + return torch._C._mps_get_name() + + +@_lru_cache +def get_core_count() -> int: + r"""Return GPU core count. + + According to the documentation, one core is comprised of 16 Execution Units. + One execution Unit has 8 ALUs. + And one ALU can run 24 threads, i.e. one core is capable of executing 3072 threads concurrently. + """ + return torch._C._mps_get_core_count() diff --git a/western_ai/usa/pytorch/torch/backends/nnpack/__init__.py b/western_ai/usa/pytorch/torch/backends/nnpack/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cc9f01a16cd4d873100e3a8d234d9e2752305a8e --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/nnpack/__init__.py @@ -0,0 +1,32 @@ +# mypy: allow-untyped-defs +from contextlib import contextmanager + +import torch +from torch.backends import __allow_nonbracketed_mutation, ContextProp, PropModule + + +__all__ = ["is_available", "flags", "set_flags"] + + +def is_available(): + r"""Return whether PyTorch is built with NNPACK support.""" + return torch._nnpack_available() + + +def set_flags(_enabled): + r"""Set if nnpack is enabled globally""" + orig_flags = (torch._C._get_nnpack_enabled(),) + torch._C._set_nnpack_enabled(_enabled) + return orig_flags + + +@contextmanager +def flags(enabled=False): + r"""Context manager for setting if nnpack is enabled globally""" + with __allow_nonbracketed_mutation(): + orig_flags = set_flags(enabled) + try: + yield + finally: + with __allow_nonbracketed_mutation(): + set_flags(orig_flags[0]) diff --git a/western_ai/usa/pytorch/torch/backends/openmp/__init__.py b/western_ai/usa/pytorch/torch/backends/openmp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..62b181dff7241084824598098f9430b243f99ad5 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/openmp/__init__.py @@ -0,0 +1,7 @@ +# mypy: allow-untyped-defs +import torch + + +def is_available(): + r"""Return whether PyTorch is built with OpenMP support.""" + return torch._C.has_openmp diff --git a/western_ai/usa/pytorch/torch/backends/opt_einsum/__init__.py b/western_ai/usa/pytorch/torch/backends/opt_einsum/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..304b9bcf73b685fcd0aceb4469e27014f67c85b3 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/opt_einsum/__init__.py @@ -0,0 +1,117 @@ +# mypy: allow-untyped-defs +import sys +import warnings +from contextlib import contextmanager +from functools import lru_cache as _lru_cache +from typing import Any + +from torch.backends import __allow_nonbracketed_mutation, ContextProp, PropModule + + +try: + import opt_einsum as _opt_einsum # type: ignore[import] +except ImportError: + _opt_einsum = None + + +@_lru_cache +def is_available() -> bool: + r"""Return a bool indicating if opt_einsum is currently available. + + You must install opt-einsum in order for torch to automatically optimize einsum. To + make opt-einsum available, you can install it along with torch: ``pip install torch[opt-einsum]`` + or by itself: ``pip install opt-einsum``. If the package is installed, torch will import + it automatically and use it accordingly. Use this function to check whether opt-einsum + was installed and properly imported by torch. + """ + return _opt_einsum is not None + + +def get_opt_einsum() -> Any: + r"""Return the opt_einsum package if opt_einsum is currently available, else None.""" + return _opt_einsum + + +def _set_enabled(_enabled: bool) -> None: + if not is_available() and _enabled: + raise ValueError( + f"opt_einsum is not available, so setting `enabled` to {_enabled} will not reap " + "the benefits of calculating an optimal path for einsum. torch.einsum will " + "fall back to contracting from left to right. To enable this optimal path " + "calculation, please install opt-einsum." + ) + global enabled + enabled = _enabled + + +def _get_enabled() -> bool: + return enabled + + +def _set_strategy(_strategy: str) -> None: + if not is_available(): + raise ValueError( + f"opt_einsum is not available, so setting `strategy` to {_strategy} will not be meaningful. " + "torch.einsum will bypass path calculation and simply contract from left to right. " + "Please install opt_einsum or unset `strategy`." + ) + if not enabled: + raise ValueError( + f"opt_einsum is not enabled, so setting a `strategy` to {_strategy} will not be meaningful. " + "torch.einsum will bypass path calculation and simply contract from left to right. " + "Please set `enabled` to `True` as well or unset `strategy`." + ) + if _strategy not in ["auto", "greedy", "optimal"]: + raise ValueError( + f"`strategy` must be one of the following: [auto, greedy, optimal] but is {_strategy}" + ) + global strategy + strategy = _strategy + + +def _get_strategy() -> str: + # pyrefly: ignore [bad-return] + return strategy + + +def set_flags(_enabled=None, _strategy=None): + orig_flags = (enabled, None if not is_available() else strategy) + if _enabled is not None: + _set_enabled(_enabled) + if _strategy is not None: + _set_strategy(_strategy) + return orig_flags + + +@contextmanager +def flags(enabled=None, strategy=None): + with __allow_nonbracketed_mutation(): + orig_flags = set_flags(enabled, strategy) + try: + yield + finally: + # recover the previous values + with __allow_nonbracketed_mutation(): + set_flags(*orig_flags) + + +# The magic here is to allow us to intercept code like this: +# +# torch.backends.opt_einsum.enabled = True + + +class OptEinsumModule(PropModule): + global enabled + enabled = ContextProp(_get_enabled, _set_enabled) + global strategy + strategy = None + if is_available(): + strategy = ContextProp(_get_strategy, _set_strategy) + + +# This is the sys.modules replacement trick, see +# https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273 +sys.modules[__name__] = OptEinsumModule(sys.modules[__name__], __name__) + +enabled = bool(is_available()) +strategy = "auto" if is_available() else None diff --git a/western_ai/usa/pytorch/torch/backends/python_native/__init__.py b/western_ai/usa/pytorch/torch/backends/python_native/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1e0e9858eafa43f122792fcdf490ce9ace676a0d --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/python_native/__init__.py @@ -0,0 +1,387 @@ +""" +User-facing API for controlling DSL operation overrides. + +The torch.backends.python_native module provides control over DSL (Domain Specific Language) +operation overrides defined in torch._native. This allows users to selectively enable or disable +high-performance implementations from various DSLs like Triton and CuteDSL. + +The module supports both coarse-grained control (entire DSLs) and fine-grained control +(individual operations or dispatch keys). All control operations support context managers +for temporary state changes. + +Example usage:: + + import torch.backends.python_native as pn + + # DSL-level control + pn.triton.enabled = False # Disable all triton ops + pn.cutedsl.enabled = True # Enable all cutedsl ops + + # Individual operation control + pn.disable_operations("scaled_mm") # Disable specific op across all DSLs + pn.enable_operations("scaled_mm") # Re-enable specific op + + # Context manager support + with pn.triton.disabled(): + result = some_computation() # Triton ops disabled here + + # Query capabilities + print(pn.available_dsls) # ['triton', 'cutedsl'] + print(pn.get_dsl_operations("triton")) # Operations for triton +""" + +import functools +import sys +import types +from contextlib import contextmanager + +from torch.backends import ContextProp, flags_frozen, PropModule + + +@contextmanager +def _preserve_filter_state(): + """Context manager to save and restore registry filter state.""" + filter_state = _get_filter_state() + + # Save original state + original_state = ( + set(filter_state._dsl_names), + set(filter_state._op_symbols), + set(filter_state._dispatch_keys), + ) + + try: + yield filter_state + finally: + # Restore original state + filter_state._dsl_names.clear() + filter_state._op_symbols.clear() + filter_state._dispatch_keys.clear() + + filter_state._dsl_names.update(original_state[0]) + filter_state._op_symbols.update(original_state[1]) + filter_state._dispatch_keys.update(original_state[2]) + + +def _get_dsl_registry(): + """Lazy import to avoid circular imports.""" + from torch._native.dsl_registry import dsl_registry + + return dsl_registry + + +def _get_registry_functions(): + """Lazy import of registry functions.""" + from torch._native.registry import ( + _filter_state, + _graphs, + deregister_op_overrides, + reenable_op_overrides, + ) + + return deregister_op_overrides, reenable_op_overrides, _graphs, _filter_state + + +def _get_filter_state(): + """Direct access to filter state.""" + return _get_registry_functions()[3] + + +def _get_dsl_module(dsl_name: str): + """Get the registered DSL module for direct control. + + Uses the DSL registry to dynamically look up DSL modules instead of + hard-coding the mapping. This makes the function automatically extensible + for new DSLs without code changes. + + Args: + dsl_name (str): Name of the DSL to retrieve. + + Returns: + DSLModuleProtocol: The registered DSL module. + + Raises: + ValueError: If the DSL is not registered. + """ + registry = _get_dsl_registry() + + # Use the public API to get the DSL module + dsl_module = registry.get_dsl_module(dsl_name) + if dsl_module is not None: + return dsl_module + else: + raise ValueError( + f"Unknown DSL: {dsl_name}. Available DSLs: {registry.list_all_dsls()}" + ) + + +class DSLController: + """Controller for a specific DSL.""" + + def __init__(self, dsl_name: str): + self._dsl_name = dsl_name + + @property + def name(self) -> str: + return self._dsl_name + + @property + def available(self) -> bool: + """Check if DSL runtime is available.""" + registry = _get_dsl_registry() + return registry.is_dsl_available(self._dsl_name) + + @property + def version(self): + """Get DSL version.""" + registry = _get_dsl_registry() + return registry.get_dsl_version(self._dsl_name) + + @property + def enabled(self) -> bool: + """Check if DSL is currently enabled.""" + filter_state = _get_filter_state() + return self._dsl_name not in filter_state._dsl_names + + @enabled.setter + def enabled(self, value: bool): + """Enable or disable the DSL.""" + if flags_frozen(): + raise RuntimeError( + f"not allowed to set {self._dsl_name} DSL flags " + "after disable_global_flags; please use flags() context manager instead" + ) + if value: + self.enable() + else: + self.disable() + + def disable(self): + """Disable all operations for this DSL.""" + dsl_module = _get_dsl_module(self._dsl_name) + dsl_module.deregister_op_overrides() + + def enable(self): + """Re-enable all operations for this DSL.""" + reenable_op_overrides = _get_registry_functions()[1] + reenable_op_overrides(enable_dsl_names=self._dsl_name) + + @contextmanager + def disabled(self): + """Context manager to temporarily disable DSL.""" + original_state = self.enabled + try: + self.disable() + yield + finally: + if original_state: + self.enable() + + def __repr__(self): + status = "available" if self.available else "unavailable" + enabled_status = "enabled" if self.enabled else "disabled" + return f"DSLController({self._dsl_name}, {status}, {enabled_status})" + + +class PythonNativeModule(PropModule): + """Main module for python_native DSL control.""" + + def __init__(self, original_module): + super().__init__(original_module, original_module.__name__) + + @property + def available_dsls(self) -> list[str]: + """Get list of available DSLs.""" + registry = _get_dsl_registry() + result = registry.list_available_dsls() + return list(result) if not isinstance(result, list) else result + + @property + def all_dsls(self) -> list[str]: + """Get list of all registered DSLs.""" + registry = _get_dsl_registry() + result = registry.list_all_dsls() + return list(result) if not isinstance(result, list) else result + + def get_dsl_operations(self, dsl_name: str) -> list[str]: + """Get list of operations registered by a specific DSL. + + Args: + dsl_name (str): Name of the DSL to query (e.g., 'triton', 'cutedsl'). + + Returns: + list[str]: Sorted list of operation names registered by the DSL. + + Example:: + + ops = torch.backends.python_native.get_dsl_operations("triton") + print(ops) # ['triton_to_mxfp8_dim0', ...] + """ + from torch._native.registry import get_dsl_operations + + return get_dsl_operations(dsl_name) + + def disable_operations(self, *op_symbols: str): + """Disable specific operations across all DSLs. + + Args: + *op_symbols (str): Names of operations to disable. + + Example:: + + # Disable scaled matrix multiply across all DSLs + torch.backends.python_native.disable_operations("scaled_mm") + + # Disable multiple operations + torch.backends.python_native.disable_operations( + "scaled_mm", "flash_attention" + ) + """ + deregister_op_overrides = _get_registry_functions()[0] + deregister_op_overrides(disable_op_symbols=list(op_symbols)) + + def enable_operations(self, *op_symbols: str): + """Re-enable specific operations across all DSLs. + + Args: + *op_symbols (str): Names of operations to re-enable. + + Example:: + + # Re-enable previously disabled operations + torch.backends.python_native.enable_operations( + "scaled_mm", "flash_attention" + ) + """ + reenable_op_overrides = _get_registry_functions()[1] + reenable_op_overrides(enable_op_symbols=list(op_symbols)) + + def disable_dispatch_keys(self, *dispatch_keys: str): + """Disable operations at specific dispatch keys. + + Args: + *dispatch_keys (str): Dispatch keys to disable (e.g., 'CUDA', 'CPU'). + + Example:: + + # Disable all native operations on CUDA + torch.backends.python_native.disable_dispatch_keys("CUDA") + """ + deregister_op_overrides = _get_registry_functions()[0] + deregister_op_overrides(disable_dispatch_keys=list(dispatch_keys)) + + def enable_dispatch_keys(self, *dispatch_keys: str): + """Re-enable operations at specific dispatch keys. + + Args: + *dispatch_keys (str): Dispatch keys to re-enable (e.g., 'CUDA', 'CPU'). + + Example:: + + # Re-enable native operations on CUDA + torch.backends.python_native.enable_dispatch_keys("CUDA") + """ + reenable_op_overrides = _get_registry_functions()[1] + reenable_op_overrides(enable_dispatch_keys=list(dispatch_keys)) + + @contextmanager + def operations_disabled(self, *op_symbols: str): + """Context manager to temporarily disable operations. + + Args: + *op_symbols (str): Names of operations to temporarily disable. + + Example:: + + with torch.backends.python_native.operations_disabled("scaled_mm"): + # scaled_mm is disabled across all DSLs + result = model(input) + # scaled_mm is automatically re-enabled here + """ + filter_state = _get_filter_state() + previously_disabled_ops = { + op for op in op_symbols if op in filter_state._op_symbols + } + + self.disable_operations(*op_symbols) + try: + yield + finally: + # Only re-enable operations that weren't already disabled + ops_to_reenable = [ + op for op in op_symbols if op not in previously_disabled_ops + ] + if ops_to_reenable: + self.enable_operations(*ops_to_reenable) + + @functools.lru_cache(maxsize=16) # noqa: B019 + def _get_dsl_controller(self, name: str) -> "DSLController": + """Get or create a DSL controller (cached).""" + return DSLController(name) + + def _get_registry_functions(self): + """Expose registry functions for testing.""" + return _get_registry_functions() + + def is_operation_disabled(self, op_symbol: str) -> bool: + """Check if an operation is currently disabled.""" + filter_state = _get_filter_state() + return op_symbol in filter_state._op_symbols + + def is_dsl_disabled(self, dsl_name: str) -> bool: + """Check if a DSL is currently disabled.""" + filter_state = _get_filter_state() + return dsl_name in filter_state._dsl_names + + def __getattr__(self, name: str): + """Dynamic attribute access for DSL controllers.""" + # Skip dunder attributes to avoid triggering DSL registry lookups + # during torch initialization. inspect.getmodule() calls + # hasattr(module, '__file__') which would otherwise cause a circular + # import through _get_dsl_registry() while torch is still loading. + if name.startswith("__") and name.endswith("__"): + raise AttributeError(f"module '{self.__name__}' has no attribute '{name}'") + + if name in self.all_dsls: + return self._get_dsl_controller(name) + + # Expose private functions for testing + if name == "_get_dsl_module": + return _get_dsl_module + if name == "_get_registry_functions": + return self._get_registry_functions + if name == "_get_filter_state": + return _get_filter_state + + raise AttributeError(f"module '{self.__name__}' has no attribute '{name}'") + + def __dir__(self): + """Return available attributes including DSL names.""" + attrs = set(super().__dir__()) + attrs.update( + { + "available_dsls", + "all_dsls", + "get_dsl_operations", + "disable_operations", + "enable_operations", + "disable_dispatch_keys", + "enable_dispatch_keys", + "operations_disabled", + "is_operation_disabled", + "is_dsl_disabled", + } + ) + + # Add DSL names + try: + attrs.update(self.all_dsls) + except Exception: + # If registry not available yet, skip DSL names + pass + + return sorted(attrs) + + +# Replace the current module with our enhanced version +sys.modules[__name__] = PythonNativeModule(sys.modules[__name__]) diff --git a/western_ai/usa/pytorch/torch/backends/quantized/__init__.py b/western_ai/usa/pytorch/torch/backends/quantized/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..464ba861c73d2247277e5c081c87fc6b1dbcd77c --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/quantized/__init__.py @@ -0,0 +1,65 @@ +# mypy: allow-untyped-defs +import sys +import types + +import torch + + +# This function should correspond to the enums present in c10/core/QEngine.h +def _get_qengine_id(qengine: str) -> int: + if qengine == "none" or qengine == "" or qengine is None: + ret = 0 + elif qengine == "fbgemm": + ret = 1 + elif qengine == "qnnpack": + ret = 2 + elif qengine == "onednn": + ret = 3 + elif qengine == "x86": + ret = 4 + else: + ret = -1 + raise RuntimeError(f"{qengine} is not a valid value for quantized engine") + return ret + + +# This function should correspond to the enums present in c10/core/QEngine.h +def _get_qengine_str(qengine: int) -> str: + all_engines = {0: "none", 1: "fbgemm", 2: "qnnpack", 3: "onednn", 4: "x86"} + return all_engines.get(qengine, "*undefined") + + +class _QEngineProp: + def __get__(self, obj, objtype) -> str: + return _get_qengine_str(torch._C._get_qengine()) + + def __set__(self, obj, val: str) -> None: + torch._C._set_qengine(_get_qengine_id(val)) + + +class _SupportedQEnginesProp: + def __get__(self, obj, objtype) -> list[str]: + qengines = torch._C._supported_qengines() + return [_get_qengine_str(qe) for qe in qengines] + + def __set__(self, obj, val) -> None: + raise RuntimeError("Assignment not supported") + + +class QuantizedEngine(types.ModuleType): + def __init__(self, m, name): + super().__init__(name) + self.m = m + + def __getattr__(self, attr): + return self.m.__getattribute__(attr) + + engine = _QEngineProp() + supported_engines = _SupportedQEnginesProp() + + +# This is the sys.modules replacement trick, see +# https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273 +sys.modules[__name__] = QuantizedEngine(sys.modules[__name__], __name__) +engine: str +supported_engines: list[str] diff --git a/western_ai/usa/pytorch/torch/backends/xeon/__init__.py b/western_ai/usa/pytorch/torch/backends/xeon/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/torch/backends/xeon/run_cpu.py b/western_ai/usa/pytorch/torch/backends/xeon/run_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..ceb079d0854df9cbe2cd1ac6bcbcaa831cd84e47 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/xeon/run_cpu.py @@ -0,0 +1,957 @@ +# mypy: allow-untyped-defs +""" +This is a script for launching PyTorch inference on Intel(R) Xeon(R) Scalable Processors with optimal configurations. + +Single instance inference, multi-instance inference are enabled. + +Note: term "instance" here doesn't refer to a cloud instance. This script is executed as a single process. It invokes +multiple "instances" which are formed from multiple threads for each. "instance" is kind of group of threads in this +context. + +Illustrated as below: + +:: + + +-----------------------------+----------------------+-------+ + | process | thread | core | + +=============================+======================+=======+ + | torch.backends.xeon.run_cpu | instance 0: thread 0 | 0 | + | | thread 1 | 1 | + | +----------------------+-------+ + | | instance 1: thread 0 | 2 | + | | thread 1 | 3 | + | +----------------------+-------+ + | | ... | ... | + | +----------------------+-------+ + | | instance N: thread 0 | M | + | | thread 1 | M+1 | + +-----------------------------+----------------------+-------+ + +To get the peak performance on Intel(R) Xeon(R) Scalable Processors, the script optimizes the configuration of thread and memory +management. For thread management, the script configures thread affinity and the preload of Intel OMP library. +For memory management, it configures NUMA binding and preload optimized memory allocation library (e.g. tcmalloc, jemalloc). + +Environment variables that will be set by this script: + ++------------------+-------------------------------------------------------------------------------------------------+ +| Environ Variable | Value | ++==================+=================================================================================================+ +| LD_PRELOAD | Depending on knobs you set, /libiomp5.so, /libjemalloc.so, /libtcmalloc.so might | +| | be appended to LD_PRELOAD. | ++------------------+-------------------------------------------------------------------------------------------------+ +| KMP_AFFINITY | If libiomp5.so is preloaded, KMP_AFFINITY could be set to "granularity=fine,compact,1,0". | ++------------------+-------------------------------------------------------------------------------------------------+ +| KMP_BLOCKTIME | If libiomp5.so is preloaded, KMP_BLOCKTIME is set to "1". | ++------------------+-------------------------------------------------------------------------------------------------+ +| OMP_NUM_THREADS | value of ncores_per_instance | ++------------------+-------------------------------------------------------------------------------------------------+ +| MALLOC_CONF | If libjemalloc.so is preloaded, MALLOC_CONF will be set to | +| | "oversize_threshold:1,background_thread:true,metadata_thp:auto". | ++------------------+-------------------------------------------------------------------------------------------------+ + +*Note*: This script respects environment variables set preliminarily. I.e. If you set the environment variables +mentioned above before running the script, the script will not overwrite the values in the script. + +How to use this module: +~~~~~~~~~~~~~~~~~~~~~~~ + +Single instance inference +------------------------- + +1. Run single-instance inference on a single node with all CPU nodes. + +:: + + python -m torch.backends.xeon.run_cpu --throughput-mode script.py args + +2. Run single-instance inference on a single CPU node. + +:: + + python -m torch.backends.xeon.run_cpu --node-id 1 script.py args + +Multi-instance inference +------------------------ + +1. Multi-instance + By default this tool runs one process per node. If you want to set the instance numbers and core per instance, + --ninstances and --ncores-per-instance should be set. + +:: + + python -m torch.backends.xeon.run_cpu -- python_script args + + eg: on an Intel(R) Xeon(R) Scalable Processor with 14 instance, 4 cores per instance + +:: + + python -m torch.backends.xeon.run_cpu --ninstances 14 --ncores-per-instance 4 python_script args + +2. Run single-instance inference among multiple instances. + By default, runs all ninstances. If you want to independently run a single instance among ninstances, specify rank. + + eg: run 0th instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance (i.e., numactl -C 0-27) + +:: + + python -m torch.backends.xeon.run_cpu --ninstances 2 --rank 0 python_script args + + eg: run 1st instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance (i.e., numactl -C 28-55) + +:: + + python -m torch.backends.xeon.run_cpu --ninstances 2 --rank 1 python_script args + + eg: run 0th instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance, 2 cores per instance, + first four cores (i.e., numactl -C 0-1) + +:: + + python -m torch.backends.xeon.run_cpu --core-list "0, 1, 2, 3" --ninstances 2 --ncores-per-instance 2 + --rank 0 python_script args + +3. To look up what optional arguments this module offers: + +:: + + python -m torch.backends.xeon.run_cpu --help + +Memory allocator +---------------- + +"--enable-tcmalloc" and "--enable-jemalloc" can be used to enable different memory allocator. + +""" + +import glob +import logging +import os +import platform +import re +import subprocess +import sys +from argparse import ArgumentParser, RawTextHelpFormatter, REMAINDER +from os.path import expanduser + +from torch.distributed.elastic.multiprocessing import ( + DefaultLogsSpecs as _DefaultLogsSpecs, + start_processes, + Std, +) + + +format_str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" +logging.basicConfig(level=logging.INFO, format=format_str) +logger = logging.getLogger(__name__) + + +class _CPUinfo: + """Get CPU information, such as cores list and NUMA information.""" + + def __init__(self, test_input=""): + self.cpuinfo = [] + if platform.system() in ["Windows", "Darwin"]: + raise RuntimeError(f"{platform.system()} is not supported!!!") + elif platform.system() == "Linux": + # Sample output of: `lscpu --parse=CPU,Core,Socket,Node` + # + # # The following is the parsable format, which can be fed to other + # # programs. Each different item in every column has a unique ID + # # starting from zero. + # # CPU,Core,Socket,Node + # 0,0,0,0 + # 1,1,0,0 + # ... + if test_input == "": + lscpu_cmd = ["lscpu", "--parse=CPU,Core,Socket,Node"] + lscpu_info = subprocess.check_output( + lscpu_cmd, universal_newlines=True + ).split("\n") + else: + lscpu_info = test_input.split("\n") + + # Get information about cpu, core, socket and node + for line in lscpu_info: + pattern = r"^([\d]+,[\d]+,[\d]+,[\d]?)" + regex_out = re.search(pattern, line) + if regex_out: + self.cpuinfo.append(regex_out.group(1).strip().split(",")) + + # physical cores := core column in lscpu output + # logical cores := cPU column in lscpu output + self.node_nums = int(max(line[3] for line in self.cpuinfo)) + 1 + self.node_physical_cores: list[list[int]] = [] # node_id is index + self.node_logical_cores: list[list[int]] = [] # node_id is index + self.physical_core_node_map = {} # physical core to numa node id + self.logical_core_node_map = {} # logical core to numa node id + + for node_id in range(self.node_nums): + cur_node_physical_core = [] + cur_node_logical_core = [] + for cpuinfo in self.cpuinfo: + nid = cpuinfo[3] if cpuinfo[3] != "" else "0" + if node_id == int(nid): + if int(cpuinfo[1]) not in cur_node_physical_core: + cur_node_physical_core.append(int(cpuinfo[1])) + self.physical_core_node_map[int(cpuinfo[1])] = int(node_id) + cur_node_logical_core.append(int(cpuinfo[0])) + self.logical_core_node_map[int(cpuinfo[0])] = int(node_id) + self.node_physical_cores.append(cur_node_physical_core) + self.node_logical_cores.append(cur_node_logical_core) + + def _physical_core_nums(self): + return len(self.node_physical_cores) * len(self.node_physical_cores[0]) + + def _logical_core_nums(self): + return len(self.node_logical_cores) * len(self.node_logical_cores[0]) + + def get_node_physical_cores(self, node_id): + if node_id < 0 or node_id > self.node_nums - 1: + raise ValueError( + f"Invalid node id: {node_id}. Valid node ids: {list(range(len(self.node_physical_cores)))}" + ) + return self.node_physical_cores[node_id] + + def get_node_logical_cores(self, node_id): + if node_id < 0 or node_id > self.node_nums - 1: + raise ValueError( + f"Invalid node id: {node_id}. Valid node ids: {list(range(len(self.node_physical_cores)))}" + ) + return self.node_logical_cores[node_id] + + def get_all_physical_cores(self): + all_cores = [] + for cores in self.node_physical_cores: + all_cores.extend(cores) + return all_cores + + def get_all_logical_cores(self): + all_cores = [] + for cores in self.node_logical_cores: + all_cores.extend(cores) + return all_cores + + def numa_aware_check(self, core_list): + """ + Check whether all cores in core_list are in the same NUMA node. + + Cross NUMA will reduce performance. + We strongly advice to not use cores on different nodes. + """ + cores_numa_map = self.logical_core_node_map + numa_ids = [] + for core in core_list: + numa_id = cores_numa_map[core] + if numa_id not in numa_ids: + numa_ids.append(numa_id) + if len(numa_ids) > 1: + logger.warning( + "Numa Aware: cores:%s on different NUMA nodes:%s. To avoid \ +this behavior, please use --ncores-per-instance knob to make sure number of cores is divisible by --ncores-per-\ +instance. Alternatively, please use --skip-cross-node-cores knob.", + core_list, + numa_ids, + ) + if len(numa_ids) == 0: + raise RuntimeError( + "invalid number of NUMA nodes; please make sure numa_ids >= 1" + ) + return numa_ids + + +class _Launcher: + r"""Class for launcher.""" + + msg_lib_notfound = ( + f"Unable to find the {{0}} library file lib{{1}}.so in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib \ +or /.local/lib/ or /usr/local/lib/ or /usr/local/lib64/ or /usr/lib or /usr/lib64 or \ +{expanduser('~')}/.local/lib/ so the LD_PRELOAD environment variable will not be set." + ) + + def __init__(self) -> None: + self.cpuinfo = _CPUinfo() + + def add_lib_preload(self, lib_type): + """Enable TCMalloc/JeMalloc/intel OpenMP.""" + library_paths = [] + if "CONDA_PREFIX" in os.environ: + library_paths.append(f"{os.environ['CONDA_PREFIX']}/lib") + if "VIRTUAL_ENV" in os.environ: + library_paths.append(f"{os.environ['VIRTUAL_ENV']}/lib") + + library_paths += [ + f"{expanduser('~')}/.local/lib", + "/usr/local/lib", + "/usr/local/lib64", + "/usr/lib", + "/usr/lib64", + ] + + lib_find = False + lib_set = False + for item in os.getenv("LD_PRELOAD", "").split(":"): + if item.endswith(f"lib{lib_type}.so"): + lib_set = True + break + if not lib_set: + for lib_path in library_paths: + # pyrefly: ignore [unbound-name] + library_file = os.path.join(lib_path, f"lib{lib_type}.so") + matches = glob.glob(library_file) + if len(matches) > 0: + # pyrefly: ignore [unbound-name] + ld_preloads = [f"{matches[0]}", os.getenv("LD_PRELOAD", "")] + # pyrefly: ignore [unbound-name] + os.environ["LD_PRELOAD"] = os.pathsep.join( + # pyrefly: ignore [unbound-name] + [p.strip(os.pathsep) for p in ld_preloads if p] + ) + lib_find = True + break + return lib_set or lib_find + + def is_numactl_available(self): + numactl_available = False + try: + cmd = ["numactl", "-C", "0", "-m", "0", "hostname"] + r = subprocess.run( + cmd, + env=os.environ, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if r.returncode == 0: + numactl_available = True + except Exception: + pass + return numactl_available + + def set_memory_allocator( + self, enable_tcmalloc=True, enable_jemalloc=False, use_default_allocator=False + ): + """ + Enable TCMalloc/JeMalloc with LD_PRELOAD and set configuration for JeMalloc. + + By default, PTMalloc will be used for PyTorch, but TCMalloc and JeMalloc can get better + memory reuse and reduce page fault to improve performance. + """ + if enable_tcmalloc and enable_jemalloc: + raise RuntimeError( + "Unable to enable TCMalloc and JEMalloc at the same time." + ) + + if enable_tcmalloc: + find_tc = self.add_lib_preload(lib_type="tcmalloc") + if not find_tc: + msg = f'{self.msg_lib_notfound} you can use "conda install -c conda-forge gperftools" to install {{0}}' + logger.warning(msg.format("TCmalloc", "tcmalloc")) + else: + logger.info("Use TCMalloc memory allocator") + + elif enable_jemalloc: + find_je = self.add_lib_preload(lib_type="jemalloc") + if not find_je: + msg = f'{self.msg_lib_notfound} you can use "conda install -c conda-forge jemalloc" to install {{0}}' + logger.warning(msg.format("Jemalloc", "jemalloc")) + else: + logger.info("Use JeMalloc memory allocator") + self.set_env( + "MALLOC_CONF", + "oversize_threshold:1,background_thread:true,metadata_thp:auto", + ) + + elif use_default_allocator: + pass + + else: + find_tc = self.add_lib_preload(lib_type="tcmalloc") + if find_tc: + logger.info("Use TCMalloc memory allocator") + return + find_je = self.add_lib_preload(lib_type="jemalloc") + if find_je: + logger.info("Use JeMalloc memory allocator") + return + logger.warning( + """Neither TCMalloc nor JeMalloc is found in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib + or /.local/lib/ or /usr/local/lib/ or /usr/local/lib64/ or /usr/lib or /usr/lib64 or + %s/.local/lib/ so the LD_PRELOAD environment variable will not be set. + This may drop the performance""", + expanduser("~"), + ) + + def log_env_var(self, env_var_name=""): + if env_var_name in os.environ: + logger.info("%s=%s", env_var_name, os.environ[env_var_name]) + + def set_env(self, env_name, env_value): + if not env_value: + logger.warning("%s is None", env_name) + if env_name not in os.environ: + os.environ[env_name] = env_value + elif os.environ[env_name] != env_value: + logger.warning( + "Overriding value with the one set in environment variable: %s. \ +Value applied: %s. Value ignored: %s", + env_name, + os.environ[env_name], + env_value, + ) + self.log_env_var(env_name) + + # set_kmp_affinity is used to control whether to set KMP_AFFINITY or not. + # In scenario that use all cores on all nodes, including logical cores, setting KMP_AFFINITY disables logical cores. + # In this case, KMP_AFFINITY should not be set. + def set_multi_thread_and_allocator( + self, + ncores_per_instance, + disable_iomp=False, + set_kmp_affinity=True, + enable_tcmalloc=True, + enable_jemalloc=False, + use_default_allocator=False, + ): + """ + Set multi-thread configuration and enable Intel openMP and TCMalloc/JeMalloc. + + By default, GNU openMP and PTMalloc are used in PyTorch. but Intel openMP and TCMalloc/JeMalloc are better alternatives + to get performance benefit. + """ + self.set_memory_allocator( + enable_tcmalloc, enable_jemalloc, use_default_allocator + ) + self.set_env("OMP_NUM_THREADS", str(ncores_per_instance)) + if not disable_iomp: + find_iomp = self.add_lib_preload(lib_type="iomp5") + if not find_iomp: + msg = f'{self.msg_lib_notfound} you can use "conda install mkl" to install {{0}}' + logger.warning(msg.format("iomp", "iomp5")) + else: + logger.info("Using Intel OpenMP") + if set_kmp_affinity: + self.set_env("KMP_AFFINITY", "granularity=fine,compact,1,0") + self.set_env("KMP_BLOCKTIME", "1") + self.log_env_var("LD_PRELOAD") + + r""" + Launcher for single instance and multi-instance + """ + + def launch(self, args): + cores = [] + set_kmp_affinity = True + enable_taskset = False + if args.core_list: # user specify what cores will be used by params + cores = [int(x) for x in args.core_list.split(",")] + if args.ncores_per_instance == -1: + raise RuntimeError( + 'please specify the "--ncores-per-instance" if you have pass the --core-list params' + ) + elif ( + args.ninstances > 1 + and args.ncores_per_instance * args.ninstances < len(cores) + ): + logger.warning( + "only first %s cores will be used, \ +but you specify %s cores in core_list", + args.ncores_per_instance * args.ninstances, + len(cores), + ) + else: + args.ninstances = len(cores) // args.ncores_per_instance + + else: + if args.use_logical_core: + if args.node_id != -1: + cores = self.cpuinfo.get_node_logical_cores(args.node_id) + else: + cores = self.cpuinfo.get_all_logical_cores() + # When using all cores on all nodes, including logical cores, + # setting KMP_AFFINITY disables logical cores. Thus, KMP_AFFINITY should not be set. + set_kmp_affinity = False + else: + if args.node_id != -1: + cores = self.cpuinfo.get_node_physical_cores(args.node_id) + else: + cores = self.cpuinfo.get_all_physical_cores() + if ( + not args.multi_instance + and args.ninstances == -1 + and args.ncores_per_instance == -1 + ): + args.ninstances = 1 + args.ncores_per_instance = len(cores) + elif ( + args.multi_instance + and args.ninstances == -1 + and args.ncores_per_instance == -1 + ): + args.throughput_mode = True + elif args.ncores_per_instance == -1 and args.ninstances != -1: + if args.ninstances > len(cores): + raise RuntimeError( + f"there are {len(cores)} total cores but you specify {args.ninstances} ninstances; \ +please make sure ninstances <= total_cores)" + ) + else: + args.ncores_per_instance = len(cores) // args.ninstances + elif args.ncores_per_instance != -1 and args.ninstances == -1: + if not args.skip_cross_node_cores: + args.ninstances = len(cores) // args.ncores_per_instance + else: + ncore_per_node = len(self.cpuinfo.node_physical_cores[0]) + num_leftover_cores = ncore_per_node % args.ncores_per_instance + if args.ncores_per_instance > ncore_per_node: + # too many ncores_per_instance to skip cross-node cores + logger.warning( + "there are %s core(s) per socket, but you specify %s ncores_per_instance and \ +skip_cross_node_cores. Please make sure --ncores-per-instance < core(s) per \ +socket", + ncore_per_node, + args.ncores_per_instance, + ) + sys.exit(-1) + elif num_leftover_cores == 0: + # aren't any cross-node cores + logger.info( + "--skip-cross-node-cores is set, but there are no cross-node cores." + ) + args.ninstances = len(cores) // args.ncores_per_instance + else: + # skip cross-node cores + if args.ninstances != -1: + logger.warning( + "--skip-cross-node-cores is exclusive to --ninstances. --ninstances \ +won't take effect even if it is set explicitly." + ) + + i = 1 + leftover_cores = set() + while ncore_per_node * i <= len(cores): + leftover_cores.update( + cores[ + ncore_per_node * i + - num_leftover_cores : ncore_per_node * i + ] + ) + i += 1 + cores = list(set(cores) - leftover_cores) + if len(cores) % args.ncores_per_instance != 0: + raise AssertionError( + f"Number of cores ({len(cores)}) must be divisible by " + f"ncores_per_instance ({args.ncores_per_instance})" + ) + args.ninstances = len(cores) // args.ncores_per_instance + else: + if args.ninstances * args.ncores_per_instance > len(cores): + raise RuntimeError( + "Please make sure ninstances * ncores_per_instance <= total_cores" + ) + if args.latency_mode: + logger.warning( + "--latency-mode is exclusive to --ninstances, --ncores-per-instance, --node-id and \ +--use-logical-core. They won't take effect even they are set explicitly." + ) + args.ncores_per_instance = 4 + cores = self.cpuinfo.get_all_physical_cores() + args.ninstances = len(cores) // args.ncores_per_instance + + if args.throughput_mode: + logger.warning( + "--throughput-mode is exclusive to --ninstances, --ncores-per-instance, --node-id and \ +--use-logical-core. They won't take effect even they are set explicitly." + ) + args.ninstances = self.cpuinfo.node_nums + cores = self.cpuinfo.get_all_physical_cores() + args.ncores_per_instance = len(cores) // args.ninstances + + if args.ninstances > 1 and args.rank != -1: + logger.info( + "assigning %s cores for instance %s", + args.ncores_per_instance, + args.rank, + ) + + if not args.disable_numactl: + numactl_available = self.is_numactl_available() + if not numactl_available: + if not args.disable_taskset: + logger.warning( + "Core binding with numactl is not available. Disabling numactl and using taskset instead. \ + This may affect performance in multi-socket system; please use numactl if memory binding is needed." + ) + args.disable_numactl = True + enable_taskset = True + else: + logger.warning( + "Core binding with numactl is not available, and --disable_taskset is set. \ + Please unset --disable_taskset to use taskset instead of numactl." + ) + sys.exit(-1) + + if not args.disable_taskset: + enable_taskset = True + + self.set_multi_thread_and_allocator( + args.ncores_per_instance, + args.disable_iomp, + set_kmp_affinity, + args.enable_tcmalloc, + args.enable_jemalloc, + args.use_default_allocator, + ) + entrypoint = "" + launch_args = {} + launch_envs: dict[int, dict] = {} + launch_tee = {} + # check whether is launched from torchrun with --nproc-per-node + local_size = int(os.environ.get("LOCAL_WORLD_SIZE", 1)) + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + for i in range(args.ninstances): + cmd = [] + cur_process_cores = "" + if not args.disable_numactl or enable_taskset: + if not args.disable_numactl: + cmd = ["numactl"] + elif enable_taskset: + cmd = ["taskset"] + cores = sorted(cores) + if ( + args.rank == -1 + ): # sequentially assign ncores_per_instance to ninstances + core_list = cores[ + i * args.ncores_per_instance : (i + 1) + * args.ncores_per_instance + ] + else: # assign ncores_per_instance from rank + core_list = cores[ + args.rank * args.ncores_per_instance : (args.rank + 1) + * args.ncores_per_instance + ] + + core_ranges: list[dict] = [] + if local_size > 1: + total_num_cores = len(core_list) + cores_per_rank = total_num_cores // local_size + if cores_per_rank < 1: + raise AssertionError( + f"At least one core needs to be assigned to each rank, " + f"got {total_num_cores} cores for {local_size} ranks" + ) + core_list = core_list[ + cores_per_rank * local_rank : cores_per_rank * (local_rank + 1) + ] + for core in core_list: + if len(core_ranges) == 0: + range_elem = {"start": core, "end": core} + core_ranges.append(range_elem) + else: + if core - core_ranges[-1]["end"] == 1: + core_ranges[-1]["end"] = core + else: + range_elem = {"start": core, "end": core} + core_ranges.append(range_elem) + for r in core_ranges: + cur_process_cores = f"{cur_process_cores}{r['start']}-{r['end']}," + cur_process_cores = cur_process_cores[:-1] + if not args.disable_numactl: + numa_params = f"-C {cur_process_cores} " + numa_ids = ",".join( + [ + str(numa_id) + for numa_id in self.cpuinfo.numa_aware_check(core_list) + ] + ) + numa_params += f"-m {numa_ids}" + cmd.extend(numa_params.split()) + elif enable_taskset: + taskset_params = f"-c {cur_process_cores} " + cmd.extend(taskset_params.split()) + with_python = not args.no_python + if with_python: + cmd.append(sys.executable) + cmd.append("-u") + if args.module: + cmd.append("-m") + cmd.append(args.program) + cmd.extend(args.program_args) + cmd_s = " ".join(cmd) + logger.info(cmd_s) + if entrypoint == "": + entrypoint = cmd[0] + del cmd[0] + launch_args[i] = tuple(cmd) + launch_envs[i] = {} + launch_tee[i] = Std.ALL + + if args.rank != -1: # launches single instance, rank, only + break + + ctx = start_processes( + name=args.log_file_prefix, + entrypoint=entrypoint, + args=launch_args, + envs=launch_envs, + logs_specs=_DefaultLogsSpecs(log_dir=args.log_path, tee=launch_tee), + ) + ctx.wait() + + +def _add_memory_allocator_params(parser): + group = parser.add_argument_group("Memory Allocator Parameters") + # allocator control + group.add_argument( + "--enable-tcmalloc", + "--enable_tcmalloc", + action="store_true", + default=False, + help="Enable tcmalloc allocator", + ) + group.add_argument( + "--enable-jemalloc", + "--enable_jemalloc", + action="store_true", + default=False, + help="Enable jemalloc allocator", + ) + group.add_argument( + "--use-default-allocator", + "--use_default_allocator", + action="store_true", + default=False, + help="Use default memory allocator", + ) + + +def _add_multi_instance_params(parser): + group = parser.add_argument_group("Multi-instance Parameters") + # multi-instance control + group.add_argument( + "--ncores-per-instance", + "--ncores_per_instance", + metavar="\b", + default=-1, + type=int, + help="Cores per instance", + ) + group.add_argument( + "--ninstances", + metavar="\b", + default=-1, + type=int, + help="For multi-instance, you should give the cores number you used for per instance.", + ) + group.add_argument( + "--skip-cross-node-cores", + "--skip_cross_node_cores", + action="store_true", + default=False, + help="If specified --ncores-per-instance, skips cross-node cores.", + ) + group.add_argument( + "--rank", + metavar="\b", + default="-1", + type=int, + help="Specify instance index to assign ncores_per_instance for rank; \ +otherwise ncores_per_instance will be assigned sequentially to ninstances. Please refer to \ +https://github.com/intel/intel-extension-for-pytorch/blob/master/docs/tutorials/performance_tuning/launch_script.md", + ) + group.add_argument( + "--latency-mode", + "--latency_mode", + action="store_true", + default=False, + help="By default 4 core per instance and use all physical cores", + ) + group.add_argument( + "--throughput-mode", + "--throughput_mode", + action="store_true", + default=False, + help="By default one instance per node and use all physical cores", + ) + group.add_argument( + "--node-id", + "--node_id", + metavar="\b", + default=-1, + type=int, + help="node id for multi-instance, by default all nodes will be used", + ) + group.add_argument( + "--use-logical-core", + "--use_logical_core", + action="store_true", + default=False, + help="Whether only use physical cores", + ) + group.add_argument( + "--disable-numactl", + "--disable_numactl", + action="store_true", + default=False, + help="Disable numactl", + ) + group.add_argument( + "--disable-taskset", + "--disable_taskset", + action="store_true", + default=False, + help="Disable taskset", + ) + group.add_argument( + "--core-list", + "--core_list", + metavar="\b", + default=None, + type=str, + help='Specify the core list as "core_id, core_id, ....", otherwise, all the cores will be used.', + ) + group.add_argument( + "--log-path", + "--log_path", + metavar="\b", + default="", + type=str, + help="The log file directory. Default path is " + ", which means disable logging to files.", + ) + group.add_argument( + "--log-file-prefix", + "--log_file_prefix", + metavar="\b", + default="run", + type=str, + help="log file prefix", + ) + + +def _add_kmp_iomp_params(parser): + group = parser.add_argument_group("IOMP Parameters") + group.add_argument( + "--disable-iomp", + "--disable_iomp", + action="store_true", + default=False, + help="By default, we use Intel OpenMP and libiomp5.so will be add to LD_PRELOAD", + ) + + +def create_args(parser=None): + """ + Parse the command line options. + + @retval ArgumentParser + """ + # pyrefly: ignore [missing-attribute] + parser.add_argument( + "--multi-instance", + "--multi_instance", + action="store_true", + default=False, + help="Enable multi-instance, by default one instance per node", + ) + + # pyrefly: ignore [missing-attribute] + parser.add_argument( + "-m", + "--module", + default=False, + action="store_true", + help="Changes each process to interpret the launch script " + "as a python module, executing with the same behavior as" + '"python -m".', + ) + + # pyrefly: ignore [missing-attribute] + parser.add_argument( + "--no-python", + "--no_python", + default=False, + action="store_true", + help='Do not prepend the --program script with "python" - just exec ' + "it directly. Useful when the script is not a Python script.", + ) + + _add_memory_allocator_params(parser) + _add_kmp_iomp_params(parser) + + _add_multi_instance_params(parser) + # positional + # pyrefly: ignore [missing-attribute] + parser.add_argument( + "program", + type=str, + help="The full path to the program/script to be launched. " + "followed by all the arguments for the script", + ) + + # rest from the training program + # pyrefly: ignore [missing-attribute] + parser.add_argument("program_args", nargs=REMAINDER) + + +def main(args): + env_before = set(os.environ.keys()) + if platform.system() in ["Windows", "Darwin"]: + raise RuntimeError(f"{platform.system()} is not supported!!!") + + if args.log_path: + os.makedirs(args.log_path, exist_ok=True) + else: + args.log_path = os.devnull + + if args.latency_mode and args.throughput_mode: + raise RuntimeError( + "Either args.latency_mode or args.throughput_mode should be set" + ) + + if not args.no_python and not args.program.endswith(".py"): + raise RuntimeError( + 'For non Python script, you should use "--no-python" parameter.' + ) + + # Verify LD_PRELOAD + if "LD_PRELOAD" in os.environ: + lst_valid = [] + tmp_ldpreload = os.environ["LD_PRELOAD"] + for item in tmp_ldpreload.split(":"): + matches = glob.glob(item) + if len(matches) > 0: + lst_valid.append(item) + else: + logger.warning("%s doesn't exist. Removing it from LD_PRELOAD.", item) + if len(lst_valid) > 0: + os.environ["LD_PRELOAD"] = ":".join(lst_valid) + else: + os.environ["LD_PRELOAD"] = "" + + launcher = _Launcher() + launcher.launch(args) + for x in sorted(set(os.environ.keys()) - env_before): + logger.debug("%s=%s", x, os.environ[x]) + + +if __name__ == "__main__": + parser = ArgumentParser( + description="This is a script for launching PyTorch inference on Intel(R) Xeon(R) Scalable " + "Processors with optimal configurations. Single instance inference, " + "multi-instance inference are enable. To get the peak performance on Intel(R) " + "Xeon(R) Scalable Processors, the script optimizes the configuration " + "of thread and memory management. For thread management, the script configures thread " + "affinity and the preload of Intel OMP library. For memory management, it configures " + "NUMA binding and preload optimized memory allocation library (e.g. tcmalloc, jemalloc) " + "\n################################# Basic usage ############################# \n" + "\n 1. single instance\n" + "\n >>> python -m torch.backends.xeon.run_cpu python_script args \n" + "\n2. multi-instance \n" + "\n >>> python -m torch.backends.xeon.run_cpu --ninstances xxx " + "--ncores-per-instance xx python_script args\n" + "\n############################################################################# \n", + formatter_class=RawTextHelpFormatter, + ) + create_args(parser) + args = parser.parse_args() + main(args) diff --git a/western_ai/usa/pytorch/torch/backends/xnnpack/__init__.py b/western_ai/usa/pytorch/torch/backends/xnnpack/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a8f40f78be9d9f6683c0cf9daad5d6d3ede98271 --- /dev/null +++ b/western_ai/usa/pytorch/torch/backends/xnnpack/__init__.py @@ -0,0 +1,29 @@ +# mypy: allow-untyped-defs +import sys +import types + +import torch + + +class _XNNPACKEnabled: + def __get__(self, obj, objtype): + return torch._C._is_xnnpack_enabled() + + def __set__(self, obj, val): + raise RuntimeError("Assignment not supported") + + +class XNNPACKEngine(types.ModuleType): + def __init__(self, m, name): + super().__init__(name) + self.m = m + + def __getattr__(self, attr): + return self.m.__getattribute__(attr) + + enabled = _XNNPACKEnabled() + + +# This is the sys.modules replacement trick, see +# https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273 +sys.modules[__name__] = XNNPACKEngine(sys.modules[__name__], __name__) diff --git a/western_ai/usa/pytorch/torch/compiler/__init__.py b/western_ai/usa/pytorch/torch/compiler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cec395ddb8535195e972bdef857848ea4a2ec2be --- /dev/null +++ b/western_ai/usa/pytorch/torch/compiler/__init__.py @@ -0,0 +1,984 @@ +# mypy: allow-untyped-defs +import contextlib +import io +from collections.abc import Callable +from typing import Any, TYPE_CHECKING, TypeVar +from typing_extensions import ParamSpec + +import torch +from torch._higher_order_ops.invoke_subgraph import NestedCompileRegionOptions + +from . import config +from ._cache import CacheInfo + + +if TYPE_CHECKING: + from torch._dynamo.eval_frame import StanceStr + + +__all__ = [ + "compile", + "config", + "assume_constant_result", + "reset", + "nonstrict_trace", + "allow_in_graph", + "substitute_in_graph", + "list_backends", + "disable", + "set_default_backend", + "get_default_backend", + "set_stance", + "set_enable_guard_collectives", + "cudagraph_mark_step_begin", + "load_compiled_function", + "wrap_numpy", + "is_compiling", + "is_dynamo_compiling", + "is_exporting", + "save_cache_artifacts", + "load_cache_artifacts", + "keep_portable_guards_unsafe", + "skip_guard_on_inbuilt_nn_modules_unsafe", + "skip_guard_on_all_nn_modules_unsafe", + "keep_tensor_guards_unsafe", + "skip_guard_on_globals_unsafe", + "skip_all_guards_unsafe", + "nested_compile_region", +] + + +_P = ParamSpec("_P") +_R = TypeVar("_R") +FuncType = Callable[..., Any] +F = TypeVar("F", bound=FuncType) + + +def compile(*args, **kwargs): + """ + See :func:`torch.compile` for details on the arguments for this function. + """ + # pyrefly: ignore [not-iterable] + return torch.compile(*args, **kwargs) + + +def reset() -> None: + """ + Reset the in-process compiler state. + + This function clears Dynamo's in-memory compilation caches and related + process-local state used by :func:`torch.compile`. It does not delete + filesystem caches, such as Inductor's disk cache. + """ + import torch._dynamo + + torch._dynamo.reset() + + +def allow_in_graph(fn): + """ + Tells the compiler frontend (Dynamo) to skip symbolic introspection of the function + and instead directly write it to the graph when encountered. + + If you are using :func:`torch.compile` (with backend="inductor" (the default)), or + :func:`torch.export.export`, and trying to black-box a Python function throughout + all tracing, do not use this API. + Instead, please create a custom operator (see `PyTorch Custom Operators Landing Page + `_) + + .. warning:: + + If you're a typical torch.compile user (e.g. you're applying torch.compile to + a model to make it run faster), you probably don't want to use this function. + :func:`allow_in_graph` is a footgun because it skips the compiler frontend + (Dynamo) that is responsible for doing safety checks (graph breaks, handling + closures, etc). Incorrect usage will lead to difficult-to-debug silent + incorrectness issues. + + Given a Python function with no allow_in_graph decorator, regular execution + of torch.compile traces through the function. :func:`allow_in_graph` changes + it so that the frontend does not trace inside the function, but the compiler + backend still traces through it. Compare this to custom operators, which + treats a function as a black box throughout the torch.compile stack. The following + table compares these mechanisms. + + +------------------------+-----------------------+--------------------------------+ + | Mechanism | Frontend (Dynamo) | Backend (AOTAutograd+Inductor) | + +========================+=======================+================================+ + | no decorator | trace inside | trace inside | + +------------------------+-----------------------+--------------------------------+ + | allow_in_graph | opaque callable | trace inside | + +------------------------+-----------------------+--------------------------------+ + | custom op | opaque callable | opaque callable | + +------------------------+-----------------------+--------------------------------+ + + One common use case for :func:`allow_in_graph()` is as an escape hatch for the compiler + frontend: if you know the function works w.r.t. to the downstream components of the + compilation stack (AOTAutograd and Inductor) but there is a Dynamo bug that prevents it from + symbolically introspecting the function properly (or if your code is in C/C++ and + therefore cannot be introspected with Dynamo), then one can decorate said function + with :func:`allow_in_graph` to bypass Dynamo. + + We require that ``fn`` adhere to the following restrictions. Failure to adhere + results in undefined behavior: + + - The inputs to ``fn`` must be Proxy-able types in the FX graph. Valid types include: + Tensor/int/bool/float/None/List[Tensor?]/List[int?]/List[float?] + Tuple[Tensor?, ...]/Tuple[int?, ...]/Tuple[float?, ...]/torch.dtype/torch.device + - The outputs to ``fn`` must be Proxy-able types in the FX graph (see previous bullet) + - all Tensors used inside of ``fn`` must be passed directly as inputs to ``fn`` + (as opposed to being captured variables). + + See also :func:`nonstrict_trace()`, which has slightly fewer restrictions on the inputs. + + Args: + fn: A callable representing the function to be included in the graph. + If ``fn`` is a list or tuple of callables it recursively applies + :func:`allow_in_graph()` to each function and returns a new list or + tuple containing the modified functions. + + Example:: + + torch.compiler.allow_in_graph(my_custom_function) + + + @torch.compile(...) + def fn(x): + x = torch.add(x, 1) + x = my_custom_function(x) + x = torch.add(x, 1) + return x + + + fn(...) + + Will capture a single graph containing ``my_custom_function()``. + + """ + import torch._dynamo + + return torch._dynamo.allow_in_graph(fn) + + +def nonstrict_trace(traceable_fn: Callable[_P, _R]) -> Callable[_P, _R]: + """ + Decorator to mark a function as nonstrict-traceable for dynamo. + + A nonstrict-traced function appears as an opaque call in the dynamo graph. + Dynamo does not trace into the function body (hence the "nonstrict"), but + aot_autograd will trace into it. + + This is similar to ``allow_in_graph`` but with enhanced support for: + - User-defined classes as inputs (must be registered with pytree) + - ``nn.Module`` as input arguments (parameters and buffers are tracked for autograd) + - Global/captured tensors treated as constants (assumed not updated during execution) + + Note: + - With ``backend="eager"``, the original Python function runs directly. + With ``backend="aot_eager"``, the graph traced by aot_autograd runs. + With ``backend="inductor"``, the traced graph is compiled with inductor. + + - Training is supported: you can call ``.backward()`` on outputs and gradients + will flow through the nonstrict-traced function. + + Dangerous patterns (may cause silent incorrectness): + - Side effects between nonstric_trace'd fn and compiled region: The function should + not depend on variables mutated by other code inside the compiled function, and code + after the call should not depend on mutations made by it. + + - Implicit inputs (closures/globals): Tensors captured from enclosing scopes + are treated as constants. Gradients will NOT flow back to them. Pass tensors + as explicit arguments if gradients are needed. + + Restrictions: + - Both inputs and outputs must use pytree-compatible types. User-defined classes + must be registered via :func:`torch.utils._pytree.register_pytree_node`, + :func:`torch.utils._pytree.register_dataclass`, or + :func:`torch.utils._pytree.register_constant`. Tensors, Python primitives (int, float, bool, str), + symbolic types (SymInt, SymFloat, SymBool), and built-in containers (list, + tuple, dict) are already handled by default. + - Primitive values and container structure are specialized per call site: + each call site expects the same primitives and structure on every execution. + + Example:: + + >>> import torch + >>> @torch.compiler.nonstrict_trace + ... def traced_forward(model, x): + ... # It's OK to have dynamo graph break within nonstrict_trace region + ... torch._dynamo.graph_break() + ... return model(x) + x + ... + >>> class MyModule(torch.nn.Module): + ... def __init__(self): + ... super().__init__() + ... self.inner = torch.nn.Linear(10, 10) + ... + ... def forward(self, x): + ... return traced_forward(self.inner, x) + ... + >>> # Compile and run + >>> model = MyModule() + >>> opt_model = torch.compile(model, backend="aot_eager", fullgraph=True) + >>> out = opt_model(torch.randn(10, 10)) + >>> out.sum().backward() # Gradients flow through traced_forward + + """ + import torch._dynamo + + return torch._dynamo.nonstrict_trace(traceable_fn) + + +def substitute_in_graph( + original_fn: Callable[_P, _R], + *, + can_constant_fold_through: bool = False, + skip_signature_check: bool = False, +) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]: + """ + Register a polyfill handler for a function, usually a C function from the C extension, to be + used in place of the original function when inlining the original function in the graph. + + .. note:: + + The polyfill handler is only used when inlining the original function. It is not used when + the original function is called directly. In the eager mode, the decorated function calls + the performant C function rather than the polyfill handler. + + The polyfill handler is a function that will be called in place of the original function when + inlining the original function. The polyfill handler should have the same signature and the same + behavior as the original function. + + Args: + original_fn (callable): The original function, usually a C function, to register a polyfill + handler for. + can_constant_fold_through (bool, optional): Whether the polyfill handler can be constant + folded through. That is, if the polyfill handler is a pure function and its arguments + are constant, the result of the polyfill handler can be constant folded during the + compilation. Defaults to ``False``. + skip_signature_check (bool, optional): Whether to skip the signature check between the + original function and the polyfill handler. Defaults to ``False``. + + Returns: + A decorator that registers the polyfill handler for the original function. + + Example:: + + >>> import operator + >>> operator.indexOf([1, 2, 3, 4, 5], 3) + 2 + >>> torch.compile(operator.indexOf, fullgraph=True)([1, 2, 3, 4, 5], 3) + ... # xdoctest: +SKIP("Long tracebacks") + Traceback (most recent call last): + ... + torch._dynamo.exc.Unsupported: ... + + >>> @torch.compiler.substitute_in_graph(operator.indexOf) + ... def indexOf(a, b, /): + ... for i, item in enumerate(a): + ... if item is b or item == b: + ... return i + ... raise ValueError("sequence.index(x): x not in sequence") + >>> + >>> torch.compile(operator.indexOf, fullgraph=True)([1, 2, 3, 4, 5], 3) + 2 + """ + import torch._dynamo + + return torch._dynamo.substitute_in_graph( + original_fn, + can_constant_fold_through=can_constant_fold_through, + skip_signature_check=skip_signature_check, + ) + + +def list_backends(exclude_tags=("debug", "experimental")) -> list[str]: + """ + Return valid strings that can be passed to `torch.compile(..., backend="name")`. + + Args: + exclude_tags(optional): A tuple of strings representing tags to exclude. + """ + import torch._dynamo + + return torch._dynamo.list_backends(exclude_tags) + + +def assume_constant_result(fn): + """ + This function is used to mark a function `fn` as having a constant result. + This allows the compiler to optimize away your function. + Returns The same function `fn` + + Args: + fn: The function to be marked as having a constant result. + + .. warning:: + `assume_constant_result` can, if invalid, cause safety and soundness issues, :func:`torch.compile` + will not attempt to validate whether the constant assumption is true or not + + """ + import torch._dynamo + + return torch._dynamo.assume_constant_result(fn) + + +def disable(fn=None, recursive=True, *, reason=None): + """ + This function provides a decorator to disable compilation on a function. + It also provides the option of recursively disabling called functions. + + Args: + fn (optional): The function to disable + recursive (optional): A boolean value indicating whether the disabling should be recursive. + reason (optional): A string value indicating the reason for disabling the function. + """ + import torch._dynamo + + return torch._dynamo.disable(fn, recursive, reason=reason) + + +def set_default_backend(backend: str | Callable[..., Any] | None) -> None: + """Set the default backend for ``torch.compile`` when no ``backend`` argument is specified. + + Passing ``None`` resets the default back to ``"inductor"``. + + Args: + backend: A backend name (string), a callable backend, or ``None``. + + Example:: + + >>> torch.compiler.set_default_backend("eager") + >>> torch.compiler.get_default_backend() + 'eager' + >>> torch.compiler.set_default_backend(None) # reset + >>> torch.compiler.get_default_backend() + 'inductor' + """ + from torch._dynamo.backends.registry import set_default_backend + + set_default_backend(backend) + + +def get_default_backend() -> str | Callable[..., Any]: + """Return the current default backend for ``torch.compile``. + + Returns: + The current default backend (string or callable). Initially ``"inductor"``. + """ + from torch._dynamo.backends.registry import get_default_backend + + return get_default_backend() + + +def set_stance( + stance: "StanceStr" = "default", + *, + skip_guard_eval_unsafe: bool = False, + force_backend: str | Callable[..., Any] | None = None, +): + """ + Set the current stance of the compiler. + Can be used as a function, context manager, or decorator. + Do not use this function inside a `torch.compile` region - an error will be raised otherwise. + + .. code-block:: python + + @torch.compile + def foo(x): ... + + + @torch.compiler.set_stance("force_eager") + def bar(): + # will not be compiled + foo(...) + + + bar() + + with torch.compiler.set_stance("force_eager"): + # will also not be compiled + foo(...) + + torch.compiler.set_stance("force_eager") + # will also not be compiled + foo(...) + torch.compiler.set_stance("default") + + # will be compiled + foo(...) + + Args: + stance: The stance to set the compiler to. Valid values are: + + - "default": The default stance, used for normal compilation. + - "force_eager": Ignore all `torch.compile` directives. + - "eager_on_recompile": Run code eagerly when a recompile is necessary. + If there is cached compiled code valid for the input, it will still be used. + - "fail_on_recompile": Raise an error when recompiling a function. + - "eager_then_compile": Run the first invocation in eager mode, then compile on + subsequent calls. This is beneficial for dynamic shapes as it allows inferring + dynamism from the first two invocations instead of wasting a static compile on + the first invocation. + - "aot_eager_then_compile": Run the first invocation with AOT eager to get memory + benefits from activation checkpointing, then compile on subsequent calls. Like + eager_then_compile, this improves handling of dynamic shapes by avoiding an + initial static compile. + + + skip_guard_eval_unsafe: A flag to run only differentiating guards. + CAUTION - This flag is unsafe and should only be used if your setup + meets the following conditions. + + torch.compile uses a guard system to support recompilations and + choose which compiled artifact to run at runtime. These guards, + though efficient, add some overhead, which may impact performance in + scenarios where you need to optimize for minimal guard processing + time. This API enables you to disable guard evaluation, assuming + that you have warmed up the compiled model with a sufficient variety + of inputs. This assumption means that, after the warmup phase, no + further recompilations will be necessary. If this assumption fails, + there is a risk of silently producing incorrect results (hence the + term "unsafe" in the API name). + + force_backend: If `stance` is "default", this argument can be used to force `torch.compile` + to use a specific backend. Otherwise, an error is raised. + """ + import torch._dynamo + + return torch._dynamo.set_stance( + stance, + skip_guard_eval_unsafe=skip_guard_eval_unsafe, + force_backend=force_backend, + ) + + +# forbid in graph +set_stance._dynamo_forbidden = True # type: ignore[attr-defined] + + +def set_enable_guard_collectives(enabled: bool): + """ + Enables use of collectives *during* guard evaluation to synchronize behavior + across ranks. This is expensive: we have to issue a collective every time + we enter a compiled code region, even if no rank actually would need to + compile. This can help prevent NCCL hangs by ensuring that we never have a + situation where one rank starts recompiling while other ranks don't compile; + it is especially useful in conjunction with enable_compiler_collectives + where such a situation would immediately cause a hang (as it is necessary + for all ranks to compile at the same time to run compiler collectives). Like + compiler collectives, you can only run this on SPMD programs; you will hang + otherwise. Note that a guard collective is only issued if there is any + compiled code to guard on; if this is the first time we encounter a frame or + the frame is skipped, we don't issue collectives. + + Returns the previous setting of enabled. + """ + from torch._C._dynamo.eval_frame import set_guard_complete_hook + from torch._dynamo.eval_frame import guard_collectives_hook + + if enabled: + return set_guard_complete_hook(guard_collectives_hook) is not None # type: ignore[arg-type] + else: + return set_guard_complete_hook(None) is not None + + +set_enable_guard_collectives._dynamo_forbidden = True # type: ignore[attr-defined] + + +def cudagraph_mark_step_begin(): + """ + Indicates that a new iteration of inference or training is about to begin. + + CUDA Graphs will free tensors of a prior iteration. A new iteration is started on each invocation of + torch.compile, so long as there is not a pending backward that has not been called. + + If that heuristic is wrong, such as in the following example, manually mark it with this api. + + .. code-block:: python + + @torch.compile(mode="reduce-overhead") + def rand_foo(): + return torch.rand([4], device="cuda") + + + for _ in range(5): + torch.compiler.cudagraph_mark_step_begin() + rand_foo() + rand_foo() + + For more details, see `torch.compiler_cudagraph_trees `__ # noqa: B950 + """ + from torch._inductor import cudagraph_trees + + cudagraph_trees.mark_step_begin() + + +def wrap_numpy(fn): + r"""Decorator that turns a function from ``np.ndarray``\ s to ``np.ndarray``\ s into a function + from ``torch.Tensor``\ s to ``torch.Tensor``\ s. + + It is designed to be used with :func:`torch.compile` with ``fullgraph=True``. It allows you to + compile a NumPy function as if it were a PyTorch function. This allows you to run NumPy code + on CUDA or compute its gradients. + + .. note:: + + This decorator does not work without :func:`torch.compile`. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> # Compile a NumPy function as a Tensor -> Tensor function + >>> @torch.compile(fullgraph=True) + >>> @torch.compiler.wrap_numpy + >>> def fn(a: np.ndarray): + >>> return np.sum(a * a) + >>> # Execute the NumPy function using Tensors on CUDA and compute the gradients + >>> x = torch.arange(6, dtype=torch.float32, device="cuda", requires_grad=True) + >>> out = fn(x) + >>> out.backward() + >>> print(x.grad) + tensor([ 0., 2., 4., 6., 8., 10.], device='cuda:0') + """ + from torch._dynamo.external_utils import wrap_numpy as wrap + + return wrap(fn) + + +_is_compiling_flag: bool = False +_is_exporting_flag: bool = False +_is_non_strict_tracing_flag: bool = False + + +def is_compiling() -> bool: + """ + Indicates whether a graph is executed/traced as part of torch.compile() or torch.export(). + + Note that there are 2 other related flags that should deprecated eventually: + * torch._dynamo.external_utils.is_compiling() + * torch._utils.is_compiling() + + Example:: + + >>> def forward(self, x): + >>> if not torch.compiler.is_compiling(): + >>> pass # ...logic that is not needed in a compiled/traced graph... + >>> + >>> # ...rest of the function... + """ + if torch.jit.is_scripting(): + return False + else: + return _is_compiling_flag + + +def _is_non_strict_tracing() -> bool: + """ + Indicates whether we are inside a non-strict make_fx-based tracing session. + """ + return _is_non_strict_tracing_flag + + +@contextlib.contextmanager +def _non_strict_tracing_context(): + """Context manager that sets the non-strict tracing flag.""" + global _is_non_strict_tracing_flag + old = _is_non_strict_tracing_flag + try: + _is_non_strict_tracing_flag = True + yield + finally: + _is_non_strict_tracing_flag = old + + +@contextlib.contextmanager +def _compile_session_context(): + """Context manager that sets _is_compiling_flag for the duration of a + torch.compile session. + """ + global _is_compiling_flag + old = _is_compiling_flag + try: + _is_compiling_flag = True + yield + finally: + _is_compiling_flag = old + + +@contextlib.contextmanager +def _patch_autograd_grad(): + """Patch autograd.grad for non-strict make_fx tracing. + + This patch installs autograd hooks so traced backward nodes preserve + stack trace, seq_nr, and autograd_backward metadata before delegating to + the real torch.autograd.grad. + """ + import functools + + import torch.autograd + from torch._dynamo.utils import warn_once + from torch._functorch._aot_autograd.logging_utils import ( + setup_stacktrace_preservation_hooks_from_tensors, + ) + + warn_once( + "torch.compiler._patch_autograd_grad() is deprecated; " + "use torch.compiler._patch_engine_backward() instead." + ) + # TODO: Remove this helper once Titan no longer depends on it. + + _orig_grad = torch.autograd.grad + + @functools.wraps(_orig_grad) + def _patched_grad(outputs, inputs, *args, **kwargs): + if not _is_non_strict_tracing(): + raise AssertionError( + "_patch_autograd_grad() must be used under " + "_non_strict_tracing_context()" + ) + + setup_stacktrace_preservation_hooks_from_tensors(outputs) + return _orig_grad(outputs, inputs, *args, **kwargs) + + torch.autograd.grad = _patched_grad + try: + yield + finally: + torch.autograd.grad = _orig_grad + + +@contextlib.contextmanager +def _patch_engine_backward(): + """Patch _engine_run_backward for non-strict make_fx tracing. + + This patch installs autograd hooks so traced backward nodes preserve + stack trace, seq_nr, and autograd_backward metadata before delegating to + the real autograd engine entrypoint used by backward(). + """ + import functools + + import torch.autograd + import torch.autograd.graph + from torch._functorch._aot_autograd.logging_utils import ( + setup_stacktrace_preservation_hooks_from_tensors, + ) + + _orig_engine_run_backward = torch.autograd.graph._engine_run_backward + + @functools.wraps(_orig_engine_run_backward) + def _patched_engine_backward(outputs, *args, **kwargs): + if not _is_non_strict_tracing(): + raise AssertionError( + "_patch_engine_backward() must be used under " + "_non_strict_tracing_context()" + ) + + setup_stacktrace_preservation_hooks_from_tensors(outputs) + return _orig_engine_run_backward(outputs, *args, **kwargs) + + torch.autograd.graph._engine_run_backward = _patched_engine_backward + torch.autograd._engine_run_backward = _patched_engine_backward + try: + yield + finally: + torch.autograd.graph._engine_run_backward = _orig_engine_run_backward + torch.autograd._engine_run_backward = _orig_engine_run_backward + + +def is_dynamo_compiling() -> bool: + """ + Indicates whether a graph is traced via TorchDynamo. + + It's stricter than is_compiling() flag, as it would only be set to True when + TorchDynamo is used. + + Example:: + + >>> def forward(self, x): + >>> if not torch.compiler.is_dynamo_compiling(): + >>> pass # ...logic that is not needed in a TorchDynamo-traced graph... + >>> + >>> # ...rest of the function... + """ + return False + + +def is_exporting() -> bool: + """ + Indicates whether we're under exporting. + + It's stricter than is_compiling() flag, as it would only be set to True when + torch.export is used. + + Example:: + + >>> def forward(self, x): + >>> if not torch.compiler.is_exporting(): + >>> pass # ...logic that is not needed in export... + >>> + >>> # ...rest of the function... + """ + return _is_exporting_flag + + +def save_cache_artifacts() -> tuple[bytes, CacheInfo] | None: + """ + Serializes all the cache artifacts that were created during the compilation + + Example: + + - Execute torch.compile + - Call torch.compiler.save_cache_artifacts() + """ + from ._cache import CacheArtifactManager + + if torch._dynamo.config.caching_precompile: + from torch._dynamo.precompile_context import PrecompileContext + + PrecompileContext.save_to_dynamo_cache() + + return CacheArtifactManager.serialize() + + +def load_cache_artifacts(serialized_artifacts: bytes) -> CacheInfo | None: + """ + Hot loads cache artifacts that were previously serialized via + save_cache_artifacts + + Example: + + # From a previous invocation + artifacts = torch.compiler.save_cache_artifacts() + + torch.compiler.load_cache_artifacts(artifacts[0]) + """ + from ._cache import CacheArtifactManager, CacheInfo + + artifacts = CacheArtifactManager.deserialize(serialized_artifacts) + if artifacts is not None: + return CacheArtifactManager.populate_caches(artifacts) + return None + + +def keep_portable_guards_unsafe(guard_entries): + """ + A common function to only keep guards that can be used in both Python and non-Python environments. + This includes: + - Tensor metadata and dynamic shape information. + - Global contexts state (e.g. autocast, no_grad, etc.) + + This is unsafe to use by default. + To use this API, use guard_filter_fn argument while calling torch.compile + + >> opt_mod = torch.compile( + >> mod, + >> options={"guard_filter_fn": torch.compiler.keep_global_context_and_tensor_guards_unsafe}, + >> ) + """ + return [ + ( + g.guard_type in ("GLOBAL_STATE", "SHAPE_ENV") + or (g.guard_type == "TENSOR_MATCH" and not g.is_global) + ) + for g in guard_entries + ] + + +def skip_guard_on_inbuilt_nn_modules_unsafe(guard_entries): + """ + A common function to skip guards on the inbuilt nn modules like + torch.nn.Linear. This is unsafe to use by default. But for majority of + torch.compile users, the model code does not modify the inbuilt nn module + attributes. They can benefit from reduction in guard latency overhead using + this API. + + To use this API, use guard_filter_fn argument while calling torch.compile + + >> opt_mod = torch.compile( + >> mod, + >> options={"guard_filter_fn": torch.compiler.skip_guard_on_all_nn_modules_unsafe}, + >> ) + """ + return [ + not entry.orig_guard.source.is_unspecialized_builtin_nn_module() + for entry in guard_entries + ] + + +def skip_guard_on_all_nn_modules_unsafe(guard_entries): + """ + A common function to skip guards on all nn modules, both user defined as + well inbuilt nn modules (like torch.nn.Linear). This is unsafe to use by + default. But for majority of torch.compile users, the model code does not + modify the nn module attributes. They can benefit from reduction in guard + latency overhead using this API. + + To use this API, use guard_filter_fn argument while calling torch.compile + + >> opt_mod = torch.compile( + >> mod, + >> options={"guard_filter_fn": torch.compiler.skip_guard_on_all_nn_modules_unsafe}, + >> ) + """ + + return [ + not entry.orig_guard.source.is_unspecialized_nn_module() + for entry in guard_entries + ] + + +def keep_tensor_guards_unsafe(guard_entries, keep_parameters=False): + """ + A common function to keep tensor guards on all tensors. This is unsafe to + use by default. But if you don't expect any changes in the model code, you + can just keep the tensor guards. + + + >> opt_mod = torch.compile( + >> mod, + >> options={"guard_filter_fn": torch.compiler.keep_tensor_guards}, + >> ) + """ + + keep_flags = [] + for entry in guard_entries: + if entry.guard_type == "TENSOR_MATCH": + if not isinstance(entry.value, torch.nn.Parameter): + keep_flags.append(True) + elif keep_parameters: + keep_flags.append(True) + else: + keep_flags.append(False) + else: + keep_flags.append(False) + return keep_flags + + +def skip_guard_on_globals_unsafe(guard_entries): + """ + A common function to skip guards on all globals. This is unsafe to use by + default. But if you don't expect any changes in the globals, you can just + keep the tensor guards. + + >> opt_mod = torch.compile( + >> mod, + >> options={"guard_filter_fn": torch.compiler.skip_guard_on_globals}, + >> ) + """ + + return [not entry.is_global for entry in guard_entries] + + +def skip_all_guards_unsafe(guard_entries): + """ + A function for skipping all guards on a compiled function. + + WARNING: This function will drop all the safety guarantees from Dynamo + compiled function. Use this with caution. + + To use this API, use guard_filter_fn argument while calling torch.compile + + >> opt_mod = torch.compile( + >> mod, + >> options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe}, + >> ) + """ + return [False for entry in guard_entries] + + +def nested_compile_region( + fn=None, + *, + options: NestedCompileRegionOptions | None = None, + max_reuse_entries: int = 8, + reuse_hash_fn=None, +): + """ + Tells **``torch.compile``** that the marked set of operations forms a nested + compile region (which is often repeated in the full model) whose code can be + compiled once and safely reused. ``nested_compile_region`` can also be used + as a decorator. + + During **``torch.compile``** tracing, the compiler applies *hierarchical + compilation* with ``nested_compile_region``: it emits optimized code for the + marked region the first time it is encountered and re-emits (or "stamps + out") the previously compiled code on every subsequent invocation. This can + substantially reduce overall compile time for deeply-stacked, + structurally-identical components such as the transformer layers of a + large-language-model (LLM). + + Outside a ``torch.compile`` context—i.e., in standard eager execution—the + call is a no-op, so existing workflows remain unaffected. + + Note that ``nested_compile_region`` **does not** promise that a region will + be compiled exactly once. If the compiler detects that new input conditions + (shape, dtype, device, stride, globals etc.) make the cached version invalid + to reuse, it will transparently re-compile the region. Using it is + therefore *safe*: correctness is always preserved, and you pay the extra + compilation cost only when required. + + Args: + fn: The function to wrap + options: Optional backend to use for compiling the subgraph. + Warning: this is an experimental feature under development and + not ready for use yet. + max_reuse_entries: Maximum number of reuse cache entries per function + before raising an error. If this limit is hit, guards keep failing + across invocations and hierarchical compilation is not effective. + reuse_hash_fn: Optional callable that takes the same ``*args, **kwargs`` + as the wrapped function and returns an integer hash key. When + provided, Dynamo traces this function to obtain a constant integer + and uses it as the cache key for subgraph reuse, bypassing the + automatic fingerprint/guard machinery. Two calls that produce the + same hash key reuse the same cached subgraph. The hash function + must be fully traceable (no graph breaks) and must return a + constant integer. + """ + + if options is not None: + from torch._dynamo import config as dynamo_config + + if not dynamo_config.enable_invoke_subgraph_regional_compile: + raise RuntimeError( + "nested_compile_region config is an experimental feature for testing only." + ) + + from torch._higher_order_ops.invoke_subgraph import ( + mark_compile_region as _mark_compile_region, + ) + + return _mark_compile_region( + fn, + options=options, + max_reuse_entries=max_reuse_entries, + reuse_hash_fn=reuse_hash_fn, + ) + + +def load_compiled_function( + file: io.IOBase, + *, + f_globals: dict[str, object] | None = None, + external_data: dict[str, Any] | None = None, +) -> Callable[..., Any]: + """ + Load an aot-compiled function from a file. + + .. warning:: + + This API is currently experimental and subject to change. + + Args: + file: A file-like object containing the serialized compiled function. + f_globals: Optional global scope enclosing the compiled function. + external_data: Optional data to be loaded into the runtime environment + of the compiled function. This should contain the same + data as AOTCompileResult.external_data returned from save_compiled_function() call. + + Returns: + A torch-compiled function with compilation preloaded from disk. + """ + from torch._dynamo.aot_compile import AOTCompiledFunction + + data = file.read() + return AOTCompiledFunction.deserialize(data, f_globals, external_data) diff --git a/western_ai/usa/pytorch/torch/compiler/_cache.py b/western_ai/usa/pytorch/torch/compiler/_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..0e7231c738c9146196f4376c927bc10138cef478 --- /dev/null +++ b/western_ai/usa/pytorch/torch/compiler/_cache.py @@ -0,0 +1,342 @@ +import copy +import dataclasses +import logging +from abc import ABC, abstractmethod +from collections import defaultdict +from collections.abc import Generator +from contextlib import contextmanager +from itertools import chain +from typing import Any + +from torch.utils._appending_byte_serializer import ( + AppendingByteSerializer, + BytesReader, + BytesWriter, +) +from torch.utils._ordered_set import OrderedSet + + +log = logging.getLogger(__name__) + + +@dataclasses.dataclass(frozen=True) +class CacheArtifact(ABC): + """ + Data for each cache artifact that will be serialized and deserialized + """ + + key: str + content: bytes = dataclasses.field(repr=False) # Do not display potential binary + + @staticmethod + def serialize(writer: BytesWriter, cls: "CacheArtifact") -> None: + writer.write_str(cls.key) + writer.write_bytes(cls.content) + + @staticmethod + def deserialize(artifact_type: str, reader: BytesReader) -> "CacheArtifact": + key = reader.read_str() + content = reader.read_bytes() + return CacheArtifactFactory.create(artifact_type, key, content) + + @staticmethod + def encode(content: Any) -> bytes: + if not isinstance(content, bytes): + raise AssertionError(f"Expected bytes, got {type(content)}") + return content + + @abstractmethod + def populate_cache(self) -> None: + pass + + @staticmethod + def type() -> str: + """ + Returns the type of the artifact. Must be unique across all CacheArtifact classes. + + CacheArtifactFactory.register will add property method to CacheInfo based on this (def {type}_artifacts) + that returns all artifacts for specific cache. + """ + raise RuntimeError("CacheArtifact is an abstract class, please use a subclass") + + +class CacheArtifactFactory: + """ + Factory for creating CacheArtifact objects based on their type + """ + + _artifact_types: dict[str, type[CacheArtifact]] = {} + + @classmethod + def register(cls, artifact_cls: type[CacheArtifact]) -> type[CacheArtifact]: + artifact_type_key = artifact_cls.type() + if artifact_cls.type() in cls._artifact_types: + raise AssertionError( + f"Artifact of type={artifact_type_key} already registered in mega-cache artifact factory" + ) + cls._artifact_types[artifact_type_key] = artifact_cls + setattr( + CacheInfo, + f"{artifact_type_key}_artifacts", + property(lambda self: self.artifacts[artifact_type_key]), + ) + return artifact_cls + + @classmethod + def _get_artifact_type(cls, artifact_type_key: str) -> type[CacheArtifact]: + if artifact_type_key not in cls._artifact_types: + raise AssertionError( + f"Artifact of type={artifact_type_key} not registered in mega-cache artifact factory" + ) + return cls._artifact_types[artifact_type_key] + + @classmethod + def create(cls, artifact_type_key: str, key: str, content: bytes) -> CacheArtifact: + artifact_cls = cls._get_artifact_type(artifact_type_key) + # pyrefly: ignore [bad-instantiation] + return artifact_cls(key, content) + + @classmethod + def encode_create( + cls, artifact_type_key: str, key: str, content: Any + ) -> CacheArtifact: + artifact_cls = cls._get_artifact_type(artifact_type_key) + # pyrefly: ignore [bad-instantiation] + return artifact_cls(key, artifact_cls.encode(content)) + + +@dataclasses.dataclass +class CacheInfo: + """ + Return value of serialization and deserialization for the purpose of + instrumentation + """ + + artifacts: defaultdict[str, list[str]] = dataclasses.field( + default_factory=lambda: defaultdict(list) + ) + + # Methods set by CacheArtifactFactory.register based on CacheArtifact.type() + @property + def inductor_artifacts(self) -> list[str]: # type: ignore[empty-body] + ... + + @property + def autotune_artifacts(self) -> list[str]: # type: ignore[empty-body] + ... + + @property + def aot_autograd_artifacts(self) -> list[str]: # type: ignore[empty-body] + ... + + @property + def pgo_artifacts(self) -> list[str]: # type: ignore[empty-body] + ... + + @property + def precompile_artifacts(self) -> list[str]: # type: ignore[empty-body] + ... + + def add(self, artifact: CacheArtifact) -> None: + self.artifacts[artifact.type()].append(artifact.key) + + def clear(self) -> None: + self.artifacts.clear() + + def empty(self) -> bool: + return not self.artifacts + + +def _serialize_single_cache( + writer: BytesWriter, cls: "tuple[str, list[CacheArtifact]]" +) -> None: + writer.write_str(cls[0]) + writer.write_uint64(len(cls[1])) + for artifact in cls[1]: + CacheArtifact.serialize(writer, artifact) + + +def _deserialize_single_cache( + reader: BytesReader, +) -> "tuple[str, list[CacheArtifact]]": + artifacts = [] + artifact_type_key = reader.read_str() + num_artifacts = reader.read_uint64() + for _ in range(num_artifacts): + artifacts.append(CacheArtifact.deserialize(artifact_type_key, reader)) + + return artifact_type_key, artifacts + + +CacheArtifactsResult = dict[str, list[CacheArtifact]] + + +@dataclasses.dataclass(frozen=True) +class CacheArtifactRecorder: + """ + Shared entry point for cache implementations to record artifacts. + """ + + artifact_type: str + key: str + + def record(self, content: Any) -> None: + CacheArtifactManager.record_artifact(self.artifact_type, self.key, content) + + def record_if_present(self, content: Any | None) -> None: + if content is not None: + self.record(content) + + +class CacheArtifactManager: + """ + Lightweight manager class for collecting and processing cache artifacts for + hot loading + + Intended Lifecycle: + - Execute code via torch.compile, this will call + CacheArtifactRecorder.record on each cache artifact + - Call CacheArtifactManager.serialize to convert all the cache artifacts + to portable format + - Call CacheArtifactManager.deserialize to hot load the cache artifacts on + a potentially different process + + NOTE: There's no FB/FC guarantees, results of cache artifacts will not be + used unless code version matches. + """ + + # Protected by the compile_lock + _new_cache_artifacts: CacheArtifactsResult = defaultdict(list) + # Keep a separate seen artifacts list to make avoid unnecessary duplicates + # This list will not be cleared between serialize() calls + _seen_artifacts: OrderedSet[CacheArtifact] = OrderedSet() + # When serialize() is called, artifacts are transferred from _cache_artifacts to + # internal data structure of the _serializer + # This allows us to only pay the cost of serialization if serialize() is called + _serializer: AppendingByteSerializer[tuple[str, list[CacheArtifact]]] = ( + AppendingByteSerializer(serialize_fn=_serialize_single_cache) + ) + _cache_info: CacheInfo = CacheInfo() + + @classmethod + def clear(cls) -> None: + cls._new_cache_artifacts.clear() + cls._seen_artifacts.clear() + cls._serializer.clear() + cls._cache_info.clear() + + @classmethod + @contextmanager + def with_fresh_cache(cls) -> Generator[None, None, None]: + original_new_cache_artifacts = cls._new_cache_artifacts + original_seen_artifacts = cls._seen_artifacts + original_serializer = cls._serializer + original_cache_info = cls._cache_info + + cls._new_cache_artifacts = defaultdict(list) + cls._seen_artifacts = OrderedSet() + cls._serializer = AppendingByteSerializer(serialize_fn=_serialize_single_cache) + cls._cache_info = cls._cache_info.__class__() + try: + yield + finally: + cls._new_cache_artifacts = original_new_cache_artifacts + cls._seen_artifacts = original_seen_artifacts + cls._serializer = original_serializer + cls._cache_info = original_cache_info + + @classmethod + def record_artifact( + cls, + artifact_type: str, + key: str, + content: Any, + ) -> None: + """ + Called from each caching operation to record the artifact in this + "mega" list + """ + artifact = CacheArtifactFactory.encode_create(artifact_type, key, content) + if artifact in cls._seen_artifacts: + return + log.debug("Recording %s", artifact) + cls._new_cache_artifacts[artifact_type].append(artifact) + cls._seen_artifacts.add(artifact) + + @classmethod + def need_serialize(cls) -> bool: + """ + Have we seen new artifacts since last serialize call? + """ + return len(cls._new_cache_artifacts) != 0 + + @classmethod + def serialize(cls) -> tuple[bytes, CacheInfo] | None: + """ + Converts the "mega" list into portable format + """ + for artifact in chain(*cls._new_cache_artifacts.values()): + log.debug("saving: %s", artifact) + cls._cache_info.add(artifact) + + if cls._cache_info.empty(): + # If there are no artifacts, don't just return bytes with + # version. + return None + + try: + # We deep copy cls._cache_info since later compilations + # can keep adding to cache_info + info = copy.deepcopy(cls._cache_info) + cls._serializer.extend(cls._new_cache_artifacts.items()) + artifact_bytes = cls._serializer.to_bytes() + cls._new_cache_artifacts.clear() + return artifact_bytes, info + except Exception: + log.warning("Failed to pickle cache artifacts", exc_info=True) + return None + + @staticmethod + def deserialize(serialized_artifacts: bytes) -> CacheArtifactsResult | None: + """ + Converts the portable format back into CacheArtifacts + """ + try: + CacheArtifactManager._ensure_cache_artifacts_registered() + artifacts = dict( + AppendingByteSerializer.to_list( + serialized_artifacts, + deserialize_fn=_deserialize_single_cache, + ) + ) + except Exception: + log.warning("Failed to un-pickle cache artifacts", exc_info=True) + return None + + return artifacts + + @staticmethod + def populate_caches(artifacts: CacheArtifactsResult) -> CacheInfo: + info = CacheInfo() + for artifact in chain(*artifacts.values()): + log.debug("writing: %s", artifact) + info.add(artifact) + artifact.populate_cache() + + return info + + @classmethod + def _ensure_cache_artifacts_registered(cls) -> None: + """When deserializing caches in fresh process, we need to ensure that all + cache artifacts are registered in the cache registry. This is done by + simply importing all the cache artifacts already wrapped with register call. + """ + from torch._dynamo.package import PrecompileCacheArtifact # noqa: F401 + from torch._dynamo.pgo import PGOCacheArtifact # noqa: F401 + from torch._functorch._aot_autograd.autograd_cache import ( # noqa: F401 + AOTAutogradCacheArtifact, + ) + from torch._inductor.codecache import InductorCacheArtifact # noqa: F401 + from torch._inductor.runtime.autotune_cache import ( # noqa: F401 + AutotuneCacheArtifact, + ) diff --git a/western_ai/usa/pytorch/torch/compiler/config.py b/western_ai/usa/pytorch/torch/compiler/config.py new file mode 100644 index 0000000000000000000000000000000000000000..42ebc5b27ef33a09ae4b73b52af82e799ffa0650 --- /dev/null +++ b/western_ai/usa/pytorch/torch/compiler/config.py @@ -0,0 +1,344 @@ +""" +This is the top-level configuration module for the compiler, containing +cross-cutting configuration options that affect all parts of the compiler +stack. + +You may also be interested in the per-component configuration modules, which +contain configuration options that affect only a specific part of the compiler: + +* :mod:`torch._dynamo.config` +* :mod:`torch._inductor.config` +* :mod:`torch._functorch.config` +* :mod:`torch.fx.experimental.config` +""" + +import sys + +from torch.utils._config_module import Config, install_config_module + + +__all__ = [ + "job_id", + "dynamic_shapes", + "assume_static_by_default", + "automatic_dynamic_shapes", + "recompile_limit", + "accumulated_recompile_limit", + "verbose", + "capture_scalar_outputs", + "capture_dynamic_output_shape_ops", + "log_file_name", + "fail_on_recompile_limit_hit", + "allow_unspec_int_on_nn_module", + "skip_tensor_guards_with_matching_dict_tags", + "enable_cpp_symbolic_shape_guards", + "wrap_top_frame", + "reorderable_logging_functions", + "force_disable_caches", + "compile_on_one_rank", +] + + +# NB: Docblocks go UNDER variable definitions! Use spacing to make the +# grouping clear. + +# FB-internal note: you do NOT have to specify this explicitly specify this if +# you run on MAST, we will automatically default this to +# mast:MAST_JOB_NAME:MAST_JOB_VERSION. +job_id: str | None = Config( + env_name_default=["TORCH_COMPILE_JOB_ID", "TORCH_COMPILE_STICKY_PGO_KEY"], + default=None, +) +""" +Semantically, this should be an identifier that uniquely identifies, e.g., a +training job. You might have multiple attempts of the same job, e.g., if it was +preempted or needed to be restarted, but each attempt should be running +substantially the same workload with the same distributed topology. You can +set this by environment variable with :envvar:`TORCH_COMPILE_JOB_ID`. + +Operationally, this controls the effect of profile-guided optimization related +persistent state. PGO state can affect how we perform compilation across +multiple invocations of PyTorch, e.g., the first time you run your program we +may compile twice as we discover what inputs are dynamic, and then PGO will +save this state so subsequent invocations only need to compile once, because +they remember it is dynamic. This profile information, however, is sensitive +to what workload you are running, so we require you to tell us that two jobs +are *related* (i.e., are the same workload) before we are willing to reuse +this information. Notably, PGO does nothing (even if explicitly enabled) +unless a valid ``job_id`` is available. In some situations, PyTorch can +configured to automatically compute a ``job_id`` based on the environment it +is running in. + +Profiles are always collected on a per rank basis, so different ranks may have +different profiles. If you know your workload is truly SPMD, you can run with +:data:`torch._dynamo.config.enable_compiler_collectives` to ensure nodes get +consistent profiles across all ranks. +""" + +pgo_extra_read_key: str | None = Config( + env_name_default="TORCH_COMPILE_STICKY_PGO_READ", default=None +) +pgo_extra_write_key: str | None = Config( + env_name_default="TORCH_COMPILE_STICKY_PGO_WRITE", default=None +) +""" +Additional read/write keys for PGO. +Write key: Besides writing to the default local/remote PGO state, this also writes to the specified key. +Read key: Besides reading from the default state, this also reads from the specified key (if written to before) +and merges it with the default state. +""" + + +cache_key_tag: str = Config(env_name_default="TORCH_COMPILE_CACHE_KEY_TAG", default="") +""" +Tag to be included in the cache key generation for all torch compile caching. +A common use case for such a tag is to break caches. +""" + +force_disable_caches: bool = Config( + justknob="pytorch/remote_cache:force_disable_caches", + env_name_force=[ + "TORCHINDUCTOR_FORCE_DISABLE_CACHES", + "TORCH_COMPILE_FORCE_DISABLE_CACHES", + ], + default=False, +) +""" +Force disables all caching -- This will take precedence over and override any other caching flag +""" + +compile_on_one_rank: bool = Config( + default=False, + env_name_default=[ + "TORCH_COMPILE_ON_ONE_RANK", + "TORCH_DISTRIBUTED_COMPILE_ON_ONE_RANK", + ], +) +""" +When enabled, device- and rank-specific values (devices, process groups) are computed at +runtime via custom ops rather than baked in at compile time, so a graph can be compiled on +one rank and run on all ranks. Read across the stack: make_fx, inductor, and distributed. +""" + +dynamic_sources: str = Config( + env_name_default="TORCH_COMPILE_DYNAMIC_SOURCES", default="" +) +r""" +Comma delimited list of sources that should be marked as dynamic. Primarily useful for large +models with graph breaks where you need intermediate tensors and ints to be marked dynamic. + +Each entry is either an exact source name or a Python regex (regex applies only to the +source name, not to the dim) matched with ``re.match``. An optional ``:N`` suffix restricts +the entry to dim ``N`` of the matched tensor; without the suffix all dims are marked dynamic. +``N`` must be a literal non-negative integer. To target multiple dims of the same source, +list it multiple times. Examples:: + + L['x'] # all dims of x dynamic (current behavior) + L['x']:0 # only dim 0 of x dynamic + L['x']:0, L['x']:2 # dims 0 and 2 of x dynamic + +Regex examples (``re.match`` anchors at the start, not the end):: + + L\['x.*'\] # all sources whose name starts with L['x (e.g. L['x'], L['x1']) + L\['x.*'\]:0 # dim 0 dynamic for every source matching L['x.* + .* # mark every source dynamic (all dims) + .*:0 # mark dim 0 dynamic for every tensor source + +The ``:N`` suffix is ignored for non-tensor (int/scalar) sources, which have no dim. + +Entries in this list are dominant over all other flags dynamic=False, force_nn_module_property_static_shapes +and force_parameter_static_shapes. +""" + +dynamic_values: str = Config( + env_name_default="TORCH_COMPILE_DYNAMIC_VALUES", default="" +) +""" +Comma delimited list of integer values that should cause any matching int source or +tensor dim to be marked dynamic. + +This is useful for JIT workflows with graph breaks where you don't know up-front which +sources need to be dynamic, but you can warm up the model with a distinctive "sentinel" +value (e.g. an odd batch size like 1111) and have any int or tensor dim that equals one +of those values be marked dynamic automatically. This is needed because with graph +breaks we do not propagate dynamic properties through eager code, so an int or tensor +derived in eager between two compiled regions cannot inherit the dynamic-ness of its +source — matching on the sentinel value lets us recover it on the next compiled region. + +Example:: + + TORCH_COMPILE_DYNAMIC_VALUES="1111,2222" + +Pick sentinel values unlikely to occur as legitimate shapes/ints in your +workload (e.g. 1111, 2222) — matching is purely by value, so any int source +or tensor dim equal to a listed value anywhere in the program will be +marked dynamic. + +unlike ``dynamic_sources``, ``dynamic_values`` does NOT override +``force_parameter_static_shapes`` or ``force_nn_module_property_static_shapes``. +""" + +unbacked_sources: str = Config( + env_name_default="TORCH_COMPILE_UNBACKED_SOURCES", default="" +) +""" +Comma delimited list of sources that should be marked as unbacked. Primarily useful for large +models with graph breaks where you need intermediate tensors marked unbacked. + +Supports the same ``:N`` per-dim suffix syntax as ``dynamic_sources``; see that config's +docstring for details. + +Entries in this list are dominant over all other flags dynamic=False, force_nn_module_property_static_shapes +and force_parameter_static_shapes. +""" + +# force a python GC before recording cudagraphs +force_cudagraph_gc: bool = Config(env_name_default="TORCH_CUDAGRAPH_GC", default=False) +""" +If True (the backward-compatible behavior) then gc.collect() before recording +any cudagraph. +""" + + +# Cross-cutting configuration options that affect the entire compilation pipeline + +dynamic_shapes: bool = Config(alias="torch._dynamo.config.dynamic_shapes") +""" +Controls whether the compilation pipeline supports dynamic tensor shapes. +When enabled, the compiler can handle tensors with varying dimensions across +different invocations. This is a cross-cutting setting that affects shape +inference, guard generation, and code generation across the entire compilation +stack. +""" + +assume_static_by_default: bool = Config( + alias="torch._dynamo.config.assume_static_by_default" +) +""" +When enabled, all tensor dimensions are assumed to be static unless explicitly +marked as dynamic or detected as changing. This compilation-wide behavior affects +how the entire stack handles shape specialization and can improve performance +for static workloads. +""" + +automatic_dynamic_shapes: bool = Config( + alias="torch._dynamo.config.automatic_dynamic_shapes" +) +""" +Enables automatic detection and handling of dynamic shapes. When a tensor's +shape changes between compilations, the system automatically marks those +dimensions as dynamic rather than requiring manual specification. This +cross-cutting optimization improves the user experience by reducing recompilations. +""" + +recompile_limit: int = Config(alias="torch._dynamo.config.recompile_limit") +""" +Maximum number of recompilations allowed for a single function before falling +back to eager execution. This compilation performance control prevents excessive +recompilation overhead that can degrade overall performance. +""" + +accumulated_recompile_limit: int = Config( + alias="torch._dynamo.config.accumulated_recompile_limit" +) +""" +Global limit on total recompilations across all compiled functions to prevent +runaway recompilation scenarios. This safeguard protects against compilation +performance issues that could affect the entire program. +""" + +verbose: bool = Config(alias="torch._dynamo.config.verbose") +""" +Enables verbose debugging output for Dynamo. When enabled, provides detailed +information about Dynamo's compilation decisions, optimizations, and potential +issues. +""" + + +# TorchDynamo-specific configuration options + +capture_scalar_outputs: bool = Config( + alias="torch._dynamo.config.capture_scalar_outputs" +) +""" +Controls whether TorchDynamo captures operations that return scalar values (like .item()) +into the FX graph. When disabled, these operations cause graph breaks. This is a +TorchDynamo-specific tracing behavior that affects how the tracer handles +scalar-returning operations. +""" + +capture_dynamic_output_shape_ops: bool = Config( + alias="torch._dynamo.config.capture_dynamic_output_shape_ops" +) +""" +Controls whether TorchDynamo captures operations with dynamic output shapes (like +nonzero, unique) into the FX graph. When disabled, these operations cause graph breaks. +This is a TorchDynamo-specific setting for handling operations with unpredictable +output shapes during tracing. +""" + +log_file_name: str | None = Config(alias="torch._dynamo.config.log_file_name") +""" +Specifies a file path for TorchDynamo-specific logging output. When set, internal +TorchDynamo debug information is written to this file rather than stdout. This is +useful for debugging TorchDynamo's internal tracing behavior. +""" + +fail_on_recompile_limit_hit: bool = Config( + alias="torch._dynamo.config.fail_on_recompile_limit_hit" +) +""" +Raises a hard error when recompile limits are exceeded instead of falling back +to eager execution. This is useful for detecting excessive recompilation in +performance-critical deployments where you want to ensure compilation overhead +is kept under control. +""" + +allow_unspec_int_on_nn_module: bool = Config( + alias="torch._dynamo.config.allow_unspec_int_on_nn_module" +) +""" +Allows integer attributes of nn.Module instances to be unspecialized through +the dynamic shape mechanism. By default, TorchDynamo specializes on all integer +module attributes, but this can cause excessive recompilation when integers +like step counters change frequently. +""" + +skip_tensor_guards_with_matching_dict_tags: bool = Config( + alias="torch._dynamo.config.skip_tensor_guards_with_matching_dict_tags" +) +""" +Optimizes guard generation by treating tensors as immutable when they are +dictionary values with consistent dictionary tags across invocations. This +reduces guard overhead for tensors stored in persistent data structures. +""" + +enable_cpp_symbolic_shape_guards: bool = Config( + alias="torch._dynamo.config.enable_cpp_symbolic_shape_guards" +) +""" +Uses C++ implementation for symbolic shape guard evaluation to improve performance. +The C++ guard manager can significantly speed up guard checking for symbolic shapes +in shape-polymorphic compilations. +""" + +wrap_top_frame: bool = Config(alias="torch._dynamo.config.wrap_top_frame") +""" +Wraps the top-level decorated function/module in a frame wrapper to ensure +nn.Module hooks are compiled within the same frame as the main function. This +improves compilation coverage for models that rely on hooks. +""" + +reorderable_logging_functions: set = Config( + alias="torch._dynamo.config.reorderable_logging_functions" +) +""" +A set of logging functions that can be reordered to execute after the compiled +portion of the graph, allowing larger graphs to be captured. Functions in this +set will have their execution deferred to avoid graph breaks, though this may +affect the timing of log output. In particular, mutated values will not be logged +at the right time, leading to incorrect logging. +""" + + +install_config_module(sys.modules[__name__]) diff --git a/western_ai/usa/pytorch/torch/contrib/__init__.py b/western_ai/usa/pytorch/torch/contrib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/western_ai/usa/pytorch/torch/contrib/_tensorboard_vis.py b/western_ai/usa/pytorch/torch/contrib/_tensorboard_vis.py new file mode 100644 index 0000000000000000000000000000000000000000..77babfd8eac47f8c41572a5b6493b96f554bc1f9 --- /dev/null +++ b/western_ai/usa/pytorch/torch/contrib/_tensorboard_vis.py @@ -0,0 +1,152 @@ +# mypy: allow-untyped-defs +import time +from collections import defaultdict +from functools import partial + +import torch + + +# Unfortunately it doesn't seem as if there was any way to get TensorBoard to do +# anything without having TF installed, and so this file has a hard dependency on it +# as well. It really is a debugging tool, so it doesn't matter. +try: + from tensorflow.core.framework import graph_pb2 + from tensorflow.core.util import event_pb2 + from tensorflow.python.summary.writer.writer import FileWriter +except ImportError: + raise ImportError( + "TensorBoard visualization of GraphExecutors requires having " + "TensorFlow installed" + ) from None + + +def dump_tensorboard_summary(graph_executor, logdir): + with FileWriter(logdir) as w: + pb_graph = visualize(graph_executor) + evt = event_pb2.Event( + wall_time=time.time(), graph_def=pb_graph.SerializeToString() + ) + w.add_event(evt) + + +def visualize(graph, name_prefix="", pb_graph=None, executors_it=None): + """Visualizes an independent graph, or a graph executor.""" + value_map = {} + pb_graph = pb_graph or graph_pb2.GraphDef() + + if isinstance(graph, torch._C.GraphExecutorState): + visualize_graph_executor( + graph, name_prefix, pb_graph, partial(visualize, pb_graph=pb_graph) + ) + return pb_graph + + # Set up an input node + pb_graph.node.add(op="input", name=name_prefix + "input") + for i, value in enumerate(graph.param_node().outputs()): + value_map[value.unique()] = name_prefix + "input:" + str(i) + + visualize_rec(graph, value_map, name_prefix, pb_graph, executors_it) + + # Gather all outputs + return_node = pb_graph.node.add(op="output", name=name_prefix + "output") + for value in graph.return_node().inputs(): + return_node.input.append(value_map[value.unique()]) + + return pb_graph + + +def visualize_graph_executor(state, name_prefix, pb_graph, inline_graph): + """Append the state of a given GraphExecutor to the graph protobuf. + + Args: + state (GraphExecutor or GraphExecutorState): GraphExecutor to display. + name_prefix (str): Name prefix of the containing subgraph. + pb_graph (GraphDef): graph to append to. + inline_graph (Callable): a function that handles setting up a value_map, + so that some graphs in here can be inlined. This is necessary, because + this will simply be `visualize` for the top-level GraphExecutor, + or `inline_graph` for all nested ones. + + The signature should look like (Graph, name_prefix) -> (). + It will be called exactly once. + + The strategy is to embed all different configurations as independent subgraphs, + while inlining the original graph as the one that actually produces the values. + """ + if state.autograd_fallback_graph is not None: + visualize( + graph=state.autograd_fallback_graph, + name_prefix=name_prefix + "autograd_fallback/", + pb_graph=pb_graph, + executors_it=iter(state.autograd_fallback.executors()), + ) + + for i, (arg_spec, plan) in enumerate(state.execution_plans.items()): + subgraph_name = name_prefix + f"plan{i}/" + + # Create a disconnected node that will keep information regarding the input + # types of this trace. This is unfortunately a bit too verbose to be included + # in the subgraph name. + input_kinds = pb_graph.node.add(op="INPUT_KIND", name=subgraph_name) + input_kinds.attr["inputs"].s = repr(arg_spec).encode("ascii") + + visualize(plan.graph, subgraph_name, pb_graph, iter(plan.code.executors())) + + # Show gradient as an independent subgraph of this plan + if plan.grad_executor is not None: + grad_subgraph_name = subgraph_name + "grad/" + visualize(plan.grad_executor, grad_subgraph_name, pb_graph) + + return inline_graph(state.graph, name_prefix + "original/") + + +def visualize_rec(graph, value_map, name_prefix, pb_graph, executors_it=None): + """Recursive part of visualize (basically skips setting up the input and output nodes).""" + + def inline_graph(subgraph, name, node): + rec_value_map = { + inp.unique(): value_map[val.unique()] + for inp, val in zip(subgraph.inputs(), node.inputs()) + } + visualize_rec( + graph=subgraph, value_map=rec_value_map, name_prefix=name, pb_graph=pb_graph + ) + for out, val in zip(subgraph.outputs(), node.outputs()): + value_map[val.unique()] = rec_value_map[out.unique()] + + op_id_counter: defaultdict[str, int] = defaultdict(int) + + def name_for(node): + kind = node.kind()[node.kind().index("::") + 2 :] + op_id_counter[kind] += 1 + return kind, name_prefix + kind + "_" + str(op_id_counter[kind]) + + def add_fusion_group(node): + op, name = name_for(node) + inline_graph(node.g("Subgraph"), name + "/", node) + + def add_graph_executor(node): + op, name = name_for(node) + if executors_it is None: + add_node(node) + else: + ge = next(executors_it) + visualize_graph_executor( + ge, name + "/", pb_graph, partial(inline_graph, node=node) + ) + + def add_node(node): + if node.kind() == "prim::FusionGroup": + return add_fusion_group(node) + elif node.kind() == "prim::GraphExecutor": + return add_graph_executor(node) + op, name = name_for(node) + pb_node = pb_graph.node.add(op=op, name=name) + for value in node.inputs(): + pb_node.input.append(value_map[value.unique()]) + # TODO: handle attrs + for i, value in enumerate(node.outputs()): + value_map[value.unique()] = name + ":" + str(i) + + for node in graph.nodes(): + add_node(node) diff --git a/western_ai/usa/pytorch/torch/cpu/__init__.py b/western_ai/usa/pytorch/torch/cpu/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..268b76cb4cbcb6491efc7295787d7c602024418a --- /dev/null +++ b/western_ai/usa/pytorch/torch/cpu/__init__.py @@ -0,0 +1,252 @@ +# mypy: allow-untyped-defs +r""" +This package implements abstractions found in ``torch.cuda`` +to facilitate writing device-agnostic code. +""" + +from collections.abc import Mapping +from contextlib import AbstractContextManager +from functools import lru_cache +from types import MappingProxyType +from typing import Any + +import torch + +from .. import device as _device +from . import amp + + +__all__ = [ + "is_available", + "is_initialized", + "synchronize", + "current_device", + "current_stream", + "stream", + "set_device", + "device_count", + "Stream", + "StreamContext", + "Event", + "get_capabilities", +] + + +@lru_cache(None) +def get_capabilities() -> Mapping[str, Any]: + """ + Returns an immutable mapping of CPU capabilities detected at runtime. + + This function queries the CPU for supported instruction sets and features + using cpuinfo. The result is cached after the first call for efficiency. + + The returned mapping contains architecture-specific capabilities: + + For x86/x86_64: + - SSE family: sse, sse2, sse3, ssse3, sse4_1, sse4_2, sse4a + - AVX family: avx, avx2, avx_vnni + - AVX-512 family: avx512_f, avx512_cd, avx512_dq, avx512_bw, avx512_vl, + avx512_ifma, avx512_vbmi, avx512_vbmi2, avx512_bitalg, avx512_vpopcntdq, + avx512_vnni, avx512_bf16, avx512_fp16, avx512_vp2intersect, + avx512_4vnniw, avx512_4fmaps + - AVX10 family: avx10_1, avx10_2 + - AVX-VNNI-INT: avx_vnni_int8, avx_vnni_int16, avx_ne_convert + - AMX: amx_bf16, amx_tile, amx_int8, amx_fp16 + - FMA: fma3, fma4 + - Other: f16c, bmi, bmi2, popcnt, lzcnt, aes, sha, clflush, clflushopt, clwb + + For ARM64: + - SIMD: neon, fp16_arith, bf16, i8mm, dot + - SVE: sve, sve2, sve_bf16, sve_max_length (when supported) + - SME: sme, sme2, sme_max_length (when supported) + - Other: atomics, fhm, rdm, crc32, aes, sha1, sha2, pmull + + Common to all architectures: + - architecture: string identifying the CPU architecture + + Returns: + MappingProxyType: An immutable mapping where keys are capability names + (e.g., 'avx2', 'sve') and values are booleans indicating + support, or integers for properties like vector lengths. + + Example: + >>> caps = torch.cpu.get_capabilities() + >>> if caps.get("avx2", False): + ... print("AVX2 is supported") + >>> print(f"Architecture: {caps['architecture']}") + """ + return MappingProxyType(torch._C._cpu._get_cpu_capability()) + + +def _is_avx2_supported() -> bool: + r"""Returns a bool indicating if CPU supports AVX2.""" + return get_capabilities().get("avx2", False) + + +def _is_avx512_supported() -> bool: + r"""Returns a bool indicating if CPU supports AVX512.""" + return get_capabilities().get("avx512_f", False) + + +def _is_avx512_bf16_supported() -> bool: + r"""Returns a bool indicating if CPU supports AVX512_BF16.""" + return get_capabilities().get("avx512_bf16", False) + + +def _is_vnni_supported() -> bool: + r"""Returns a bool indicating if CPU supports VNNI.""" + # Note: Currently, it only checks avx512_vnni, will add the support of avx2_vnni later. + return get_capabilities().get("avx512_vnni", False) + + +def _is_amx_tile_supported() -> bool: + r"""Returns a bool indicating if CPU supports AMX_TILE.""" + return get_capabilities().get("amx_tile", False) + + +def _is_amx_fp16_supported() -> bool: + r"""Returns a bool indicating if CPU supports AMX FP16.""" + return get_capabilities().get("amx_fp16", False) + + +def _init_amx() -> bool: + r"""Initializes AMX instructions.""" + return torch._C._cpu._init_amx() + + +def is_available() -> bool: + r"""Returns a bool indicating if CPU is currently available. + + N.B. This function only exists to facilitate device-agnostic code + + """ + return True + + +def synchronize(device: torch.types.Device = None) -> None: + r"""Waits for all kernels in all streams on the CPU device to complete. + + Args: + device (torch.device or int, optional): ignored, there's only one CPU device. + + N.B. This function only exists to facilitate device-agnostic code. + """ + + +class Stream: + """ + N.B. This class only exists to facilitate device-agnostic code + """ + + def __init__(self, priority: int = -1) -> None: + pass + + def wait_stream(self, stream) -> None: + pass + + def record_event(self) -> None: + pass + + def wait_event(self, event) -> None: + pass + + +class Event: + def query(self) -> bool: + return True + + def record(self, stream=None) -> None: + pass + + def synchronize(self) -> None: + pass + + def wait(self, stream=None) -> None: + pass + + +_default_cpu_stream = Stream() +_current_stream = _default_cpu_stream + + +def current_stream(device: torch.types.Device = None) -> Stream: + r"""Returns the currently selected :class:`Stream` for a given device. + + Args: + device (torch.device or int, optional): Ignored. + + N.B. This function only exists to facilitate device-agnostic code + + """ + return _current_stream + + +class StreamContext(AbstractContextManager): + r"""Context-manager that selects a given stream. + + N.B. This class only exists to facilitate device-agnostic code + + """ + + cur_stream: Stream | None + + def __init__(self, stream): + self.stream = stream + self.prev_stream = _default_cpu_stream + + def __enter__(self): + cur_stream = self.stream + if cur_stream is None: + return + + global _current_stream + self.prev_stream = _current_stream + _current_stream = cur_stream + + def __exit__(self, type: Any, value: Any, traceback: Any) -> None: + cur_stream = self.stream + if cur_stream is None: + return + + global _current_stream + _current_stream = self.prev_stream + + +def stream(stream: Stream) -> AbstractContextManager: + r"""Wrapper around the Context-manager StreamContext that + selects a given stream. + + N.B. This function only exists to facilitate device-agnostic code + """ + return StreamContext(stream) + + +def device_count() -> int: + r"""Returns number of CPU devices (not cores). Always 1. + + N.B. This function only exists to facilitate device-agnostic code + """ + return 1 + + +def set_device(device: torch.types.Device) -> None: + r"""Sets the current device, in CPU we do nothing. + + N.B. This function only exists to facilitate device-agnostic code + """ + + +def current_device() -> str: + r"""Returns current device for cpu. Always 'cpu'. + + N.B. This function only exists to facilitate device-agnostic code + """ + return "cpu" + + +def is_initialized() -> bool: + r"""Returns True if the CPU is initialized. Always True. + + N.B. This function only exists to facilitate device-agnostic code + """ + return True diff --git a/western_ai/usa/pytorch/torch/cpu/amp/__init__.py b/western_ai/usa/pytorch/torch/cpu/amp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b5c7fc57623feb1d2841b111f97fc29db573c4a1 --- /dev/null +++ b/western_ai/usa/pytorch/torch/cpu/amp/__init__.py @@ -0,0 +1,3 @@ +# pyrefly: ignore [deprecated] +from .autocast_mode import autocast +from .grad_scaler import GradScaler diff --git a/western_ai/usa/pytorch/torch/cpu/amp/autocast_mode.py b/western_ai/usa/pytorch/torch/cpu/amp/autocast_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..8e04185ae70ce5ab97fe4822a5f67a9cba8abf57 --- /dev/null +++ b/western_ai/usa/pytorch/torch/cpu/amp/autocast_mode.py @@ -0,0 +1,71 @@ +# mypy: allow-untyped-defs +import sys +from typing import Any +from typing_extensions import deprecated + +import torch + + +__all__ = ["autocast"] + + +@deprecated( + "`torch.cpu.amp.autocast(args...)` is deprecated. " + "Please use `torch.amp.autocast('cpu', args...)` instead.", + category=FutureWarning, +) +class autocast(torch.amp.autocast_mode.autocast): + r""" + See :class:`torch.autocast`. + ``torch.cpu.amp.autocast(args...)`` is deprecated. Please use ``torch.amp.autocast("cpu", args...)`` instead. + """ + + # TODO: remove this conditional once we stop supporting Python < 3.13 + # Prior to Python 3.13, inspect.signature could not retrieve the correct + # signature information for classes decorated with @deprecated (unless + # the __new__ static method was explicitly defined); + # + # However, this issue has been fixed in Python 3.13 and later versions. + if sys.version_info < (3, 13): + + def __new__( + cls, + enabled: bool = True, + dtype: torch.dtype = torch.bfloat16, + cache_enabled: bool = True, + ): + return super().__new__(cls) + + def __init_subclass__(cls): + pass + + def __init__( + self, + enabled: bool = True, + dtype: torch.dtype = torch.bfloat16, + cache_enabled: bool = True, + ): + if torch._jit_internal.is_scripting(): + self._enabled = enabled + self.device = "cpu" + self.fast_dtype = dtype + return + super().__init__( + "cpu", enabled=enabled, dtype=dtype, cache_enabled=cache_enabled + ) + + def __enter__(self): + if torch._jit_internal.is_scripting(): + return self + return super().__enter__() + + # TODO: discuss a unified TorchScript-friendly API for autocast + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any): # type: ignore[override] + if torch._jit_internal.is_scripting(): + return + return super().__exit__(exc_type, exc_val, exc_tb) + + def __call__(self, func): + if torch._jit_internal.is_scripting(): + return func + return super().__call__(func) diff --git a/western_ai/usa/pytorch/torch/cpu/amp/grad_scaler.py b/western_ai/usa/pytorch/torch/cpu/amp/grad_scaler.py new file mode 100644 index 0000000000000000000000000000000000000000..3e61c0b41ec440a20f35217ecba53a5b22eb1c93 --- /dev/null +++ b/western_ai/usa/pytorch/torch/cpu/amp/grad_scaler.py @@ -0,0 +1,35 @@ +from typing_extensions import deprecated + +import torch + + +__all__ = ["GradScaler"] + + +class GradScaler(torch.amp.GradScaler): + r""" + See :class:`torch.amp.GradScaler`. + ``torch.cpu.amp.GradScaler(args...)`` is deprecated. Please use ``torch.amp.GradScaler("cpu", args...)`` instead. + """ + + @deprecated( + "`torch.cpu.amp.GradScaler(args...)` is deprecated. " + "Please use `torch.amp.GradScaler('cpu', args...)` instead.", + category=FutureWarning, + ) + def __init__( + self, + init_scale: float = 2.0**16, + growth_factor: float = 2.0, + backoff_factor: float = 0.5, + growth_interval: int = 2000, + enabled: bool = True, + ) -> None: + super().__init__( + "cpu", + init_scale=init_scale, + growth_factor=growth_factor, + backoff_factor=backoff_factor, + growth_interval=growth_interval, + enabled=enabled, + ) diff --git a/western_ai/usa/pytorch/torch/csrc/CudaIPCTypes.cpp b/western_ai/usa/pytorch/torch/csrc/CudaIPCTypes.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a02af4601c383953ce198268d8ef4d5262d4d8d1 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/CudaIPCTypes.cpp @@ -0,0 +1,273 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace torch { + +namespace { + +void warnProducerTerminatedBeforeSharedTensorsReleased() { + static bool warned = false; + if (!warned) { + LOG(WARNING) + << "Producer process has been terminated before all shared CUDA tensors released. See Note [Sharing CUDA tensors]"; + warned = true; + } +} + +struct CudaIPCGlobalEntities { + // This class is used as a singleton (see cuda_ipc_global_entities) + // This variable is used to track its lifetime to avoid accessing it + // after it was destroyed which would lead to segmentation faults + // Note that a trivial type is used which doesn't suffer from construction + // and destruction order issues + static bool alive; + + std::mutex ref_counters_mutex_; + std::atomic sync_events_used_{0}; + std::map> + ref_counters_files_; + std::shared_ptr next_available_ref_counters_file_; + CudaIPCSentDataLimbo CudaIPCSentDataLimbo_; + CudaIPCGlobalEntities() { + alive = true; + } + CudaIPCGlobalEntities(const CudaIPCGlobalEntities&) = delete; + CudaIPCGlobalEntities(CudaIPCGlobalEntities&&) = delete; + CudaIPCGlobalEntities& operator=(const CudaIPCGlobalEntities&) = delete; + CudaIPCGlobalEntities& operator=(CudaIPCGlobalEntities&&) = delete; + ~CudaIPCGlobalEntities() { + alive = false; + CudaIPCSentDataLimbo_.collect(); + safe_clean_current_file(); + if (next_available_ref_counters_file_) { + warnProducerTerminatedBeforeSharedTensorsReleased(); + } + } + void safe_clean_current_file() { + std::lock_guard lock(ref_counters_mutex_); + if (next_available_ref_counters_file_ && + next_available_ref_counters_file_->offsets_in_use() == 0) { + ref_counters_files_.erase(next_available_ref_counters_file_->handle()); + next_available_ref_counters_file_.reset(); + } + } +}; + +bool CudaIPCGlobalEntities::alive = false; +CudaIPCGlobalEntities cuda_ipc_global_entities; + +CudaIPCSentDataLimbo::~CudaIPCSentDataLimbo() { + collect(); + if (size() > 0) { + warnProducerTerminatedBeforeSharedTensorsReleased(); + } +} + +bool CudaIPCSentDataLimbo::collect() { + bool freed_memory = false; + std::vector> reset_blocks; + { // Begin critical section to modify shared blocks + std::lock_guard lock(limbo_mutex_); + std::vector> kept_blocks; + for (auto& sd : shared_blocks_) { + if (sd->counter_value() > 0) { + kept_blocks.push_back(std::move(sd)); + } else { + freed_memory = true; + reset_blocks.push_back(std::move(sd)); + } + } + shared_blocks_ = std::move(kept_blocks); + } + // Need to reset blocks out of the critical section here, otherwise it + // deadlocks. + for (auto& sd : reset_blocks) { + sd.reset(); + } + return freed_memory; +} + +void CudaIPCSentDataLimbo::add(std::unique_ptr shared_block) { + std::lock_guard lock(limbo_mutex_); + static bool warned = false; + if (shared_blocks_.size() > CUDA_IPC_WARN_AFTER_X_BLOCKS_IN_LIMBO && + !warned) { + LOG(WARNING) + << "Producer process tried to deallocate over " + << CUDA_IPC_WARN_AFTER_X_BLOCKS_IN_LIMBO + << " memory blocks referred by consumer processes. Deallocation might be significantly slowed down. " + << "We assume it will never going to be the case, but if it is, please file but to https://github.com/pytorch/pytorch"; + warned = true; + } + shared_blocks_.push_back(std::move(shared_block)); +} + +uint64_t CudaIPCSentDataLimbo::size() { + std::lock_guard lock(limbo_mutex_); + return shared_blocks_.size(); +} + +void CudaIPCSentDataDelete(void* ptr) { + std::unique_ptr sent_data( + static_cast(ptr)); + if (!CudaIPCGlobalEntities::alive) { + return; + } + if (sent_data->counter_value() > 0) { + cuda_ipc_global_entities.CudaIPCSentDataLimbo_.add(std::move(sent_data)); + } + cuda_ipc_global_entities.CudaIPCSentDataLimbo_.collect(); +} + +void ReturnRefCounter(const std::string& handle, uint64_t offset /* unused */) { + if (!CudaIPCGlobalEntities::alive) { + return; + } + std::lock_guard lock( + cuda_ipc_global_entities.ref_counters_mutex_); + auto& map = cuda_ipc_global_entities.ref_counters_files_; + auto it = map.find(handle); + if (it != map.end()) { + it->second->return_offset(offset); + if (it->second->offsets_in_use() == 0 && !it->second->have_offsets()) { + map.erase(handle); + } + } +} + +} // namespace + +CudaIPCSentData::CudaIPCSentData( + std::string handle, + uint64_t offset, + uint64_t* counter_ptr, + at::Device device) + : handle_(std::move(handle)), + offset_(offset), + counter_ptr_(counter_ptr), + device_(device) { +#if !defined(USE_ROCM) + // CUDA have the unofficial limit on the number of recorded blocking + // interprocess events, to prevent using of all events, we are switching to + // StreamSync before limit reached. + // + // ```python + // import torch + // a = [ torch.cuda.Event( + // enable_timing=False, blocking=True, interprocess=True) for i in + // range(30000) ] + // [i.record() for i in a] + // ``` + // + if (cuda_ipc_global_entities.sync_events_used_.load() < + CUDA_IPC_MAXIMUM_EVENTS_TO_USE) { + // TODO: More efficient would be to create event inside of main thread (at + // the moment of the queue.put). The reason this is more efficient is + // because the main thread may have queued extra work on the stream, which + // this event will consequently wait for (uselessly). + cuda_ipc_global_entities.sync_events_used_++; + C10_CUDA_CHECK(cudaEventCreateWithFlags( + &event_, + cudaEventDisableTiming | cudaEventInterprocess | + cudaEventBlockingSync)); + C10_CUDA_CHECK(cudaEventRecord( + event_, c10::cuda::getCurrentCUDAStream(device.index()))); + event_sync_required_ = true; + } else { + auto stream = c10::cuda::getCurrentCUDAStream(device.index()); + at::cuda::stream_synchronize(stream); + event_ = nullptr; + event_sync_required_ = false; + } +#else + // cuIpcGetEventHandle with HIP is not supported, so we have to sync + // stream instead of passing event + auto stream = c10::cuda::getCurrentCUDAStream(device.index()); + at::cuda::stream_synchronize(stream); + event_sync_required_ = false; +#endif +} + +CudaIPCSentData::~CudaIPCSentData() { + if (!CudaIPCGlobalEntities::alive) { + original_ptr_.release_context(); + } + ReturnRefCounter(handle_, offset_); +#if !defined(USE_ROCM) + try { + if (event_sync_required_) { + at::cuda::CUDAGuard device_guard(device_.index()); + C10_CUDA_CHECK(cudaEventDestroy(event_)); + if (!CudaIPCGlobalEntities::alive) { + return; + } + cuda_ipc_global_entities.sync_events_used_--; + } + // NOLINTNEXTLINE(bugprone-empty-catch) + } catch (...) { /* No throw */ + } +#endif +} + +uint64_t CudaIPCSentData::counter_value() { + return *counter_ptr_; +} + +at::DataPtr GetNewRefCountedSentData(void* data, at::Device device) { + { + std::lock_guard lock( + cuda_ipc_global_entities.ref_counters_mutex_); + if (!cuda_ipc_global_entities.next_available_ref_counters_file_) { + std::string ref_counter_handle = at::NewProcessWideShmHandle(); + + int flags = + at::ALLOCATOR_MAPPED_SHAREDMEM | at::ALLOCATOR_MAPPED_EXCLUSIVE; + at::DataPtr sptr = at::RefcountedMapAllocator::makeDataPtr( + ref_counter_handle.c_str(), + flags, + sizeof(int64_t) * CUDA_IPC_REF_COUNTER_FILE_SIZE, + nullptr); + auto rc = std::make_shared( + ref_counter_handle, CUDA_IPC_REF_COUNTER_FILE_SIZE, std::move(sptr)); + cuda_ipc_global_entities.ref_counters_files_[ref_counter_handle] = rc; + cuda_ipc_global_entities.next_available_ref_counters_file_ = rc; + } + } + cuda_ipc_global_entities.next_available_ref_counters_file_->set_counter(1); + auto sent_data = new CudaIPCSentData( + cuda_ipc_global_entities.next_available_ref_counters_file_->handle(), + cuda_ipc_global_entities.next_available_ref_counters_file_->get_offset(), + cuda_ipc_global_entities.next_available_ref_counters_file_->counter_ptr(), + device); + + cuda_ipc_global_entities.next_available_ref_counters_file_->rotate_offset(); + if (!cuda_ipc_global_entities.next_available_ref_counters_file_ + ->have_offsets()) { + cuda_ipc_global_entities.next_available_ref_counters_file_.reset(); + } + return at::DataPtr(data, sent_data, CudaIPCSentDataDelete, device); +} + +bool CudaIPCCollect() { + if (!CudaIPCGlobalEntities::alive) { + return true; + } + bool freed_memory = cuda_ipc_global_entities.CudaIPCSentDataLimbo_.collect(); + if (cuda_ipc_global_entities.CudaIPCSentDataLimbo_.size() == 0) { + cuda_ipc_global_entities.safe_clean_current_file(); + } + return freed_memory; +} + +} // namespace torch + +namespace c10 { +namespace { +REGISTER_FREE_MEMORY_CALLBACK("cuda_ipc_collect", CudaIPCCollectCallback) +} +} // namespace c10 diff --git a/western_ai/usa/pytorch/torch/csrc/CudaIPCTypes.h b/western_ai/usa/pytorch/torch/csrc/CudaIPCTypes.h new file mode 100644 index 0000000000000000000000000000000000000000..45930e5327aaa556eb7e6e34e1fa64d91639308c --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/CudaIPCTypes.h @@ -0,0 +1,143 @@ +#pragma once +#ifdef USE_CUDA +#include +#include +#include +#include +#include +#include +#include +namespace torch { + +TORCH_CUDA_CU_API bool CudaIPCCollect(); + +struct CudaIPCReceivedData final { + CudaIPCReceivedData() = default; + explicit CudaIPCReceivedData(std::shared_ptr shared_ptr) + : shared_ptr_(std::move(shared_ptr)) {} + std::shared_ptr shared_ptr_; +}; + +struct CudaIPCSentData final { + std::string handle_; + uint64_t offset_; + uint64_t* counter_ptr_; // Reference counter shared memory block + at::DataPtr original_ptr_; // Original mem allocation + cudaEvent_t event_; // Sync cuEventDestroy + bool event_sync_required_; + at::Device device_; + + CudaIPCSentData( + std::string handle, + uint64_t offset, + uint64_t* counter_ptr, + at::Device device); + ~CudaIPCSentData(); + + uint64_t counter_value(); + std::string handle() { + return handle_; + } + uint64_t offset() { + return offset_; + } + void set_original_ptr(at::DataPtr data_ptr) { + original_ptr_ = std::move(data_ptr); + } +}; + +TORCH_CUDA_CU_API at::DataPtr GetNewRefCountedSentData( + void* data, + at::Device device); + +namespace { + +inline constexpr int64_t CUDA_IPC_REF_COUNTER_FILE_SIZE = 10000; +inline constexpr int64_t CUDA_IPC_WARN_AFTER_X_BLOCKS_IN_LIMBO = 1000; +// This was determined empirically that CUDA (v10.1 and below) have the limit +// on the number of recorded blocking interprocess events. It is around ~22,000. +// And to give us leeway, we picked 1000 as it gives us enough events to share +// tensors effectively. +inline constexpr int64_t CUDA_IPC_MAXIMUM_EVENTS_TO_USE = 1000; + +// All to be deleted data blocks with non zero reference counter goes there +struct CudaIPCSentDataLimbo final { + ~CudaIPCSentDataLimbo(); + bool collect(); + void add(std::unique_ptr shared_block); + uint64_t size(); + + private: + // TODO: Can be changed to FIFO in order to avoid full traverse on every + // collect() + std::vector> shared_blocks_; + std::mutex limbo_mutex_; +}; + +struct CudaIPCRefCountersFile final { + CudaIPCRefCountersFile( + std::string handle, + uint64_t size, + at::DataPtr data_ptr) + : size_(size), + + handle_(std::move(handle)), + refcounted_shared_mem_(std::move(data_ptr)) {} + + uint64_t* counter_ptr() { + return static_cast(refcounted_shared_mem_.get()) + next_offset_; + } + + void set_counter(uint64_t value) { + *counter_ptr() = value; + } + + bool have_offsets() { + return next_offset_ < size_; + } + + bool offsets_in_use() { + return used_slots_; + } + + uint64_t get_offset() { + return next_offset_; + } + + void rotate_offset() { + next_offset_++; + used_slots_++; + } + + void return_offset(uint64_t offset /* unused */) { + used_slots_--; + } + + std::string handle() { + return handle_; + } + + private: + uint64_t next_offset_{0}; + uint64_t size_; + uint64_t used_slots_{0}; + std::string handle_; + at::DataPtr refcounted_shared_mem_; +}; + +} // namespace +} // namespace torch + +namespace c10 { +namespace { +class CudaIPCCollectCallback : public FreeMemoryCallback { + public: + bool Execute() override { + return torch::CudaIPCCollect(); + } +}; +} // namespace + +} // namespace c10 + +#endif diff --git a/western_ai/usa/pytorch/torch/csrc/DataLoader.cpp b/western_ai/usa/pytorch/torch/csrc/DataLoader.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cd88ec966da25473cde21a4f9d085929a06d33d9 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/DataLoader.cpp @@ -0,0 +1,265 @@ +#include + +// Together with `torch/utils/data/_utils/signal_handling.py`, the following +// is an effort to do our best to provide some error message to users when a +// worker dies due to error / critical signals. +// +// See NOTE [ Signal handling in multiprocessing data loading ] for more +// details. + +// TODO: The following don't work on Windows. Specifically, sigaction, waitid +// calls, and SIGCHLD handler. Currently, dummy implementations are provided +// for Windows. + +#ifndef _WIN32 + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +using namespace torch; + +// Critical signal handlers should be registered on worker processes before +// doing work. +// The handler will raise default handler so that the kill information will be +// retrieved from main process. +// Python handle is _set_worker_signal_handlers(). +#define SIGNAL_HANDLER(SIGNAL, HANDLER_NAME, ERROR_MSG) \ + static void HANDLER_NAME(int sig, siginfo_t* info, void* ctx) { \ + auto _w = \ + write(STDERR_FILENO, ERROR_MSG, sizeof(ERROR_MSG) / sizeof(char)); \ + (void)_w; \ + struct sigaction sa{}; \ + sa.sa_handler = SIG_DFL; \ + sa.sa_flags = 0; \ + if (sigemptyset(&sa.sa_mask) != 0 || \ + sigaction(SIGNAL, &sa, nullptr) != 0) { \ + _exit(EXIT_FAILURE); \ + } else { \ + raise(SIGNAL); \ + } \ + } + +// signal(2) is really not portable. So use sigaction. +// http://man7.org/linux/man-pages/man2/signal.2.html +static void setSignalHandler( + int signal, + void (*handler)(int, siginfo_t*, void*), + struct sigaction* old_sa_ptr) { + struct sigaction sa{}; + sa.sa_sigaction = handler; + sa.sa_flags = SA_RESTART | SA_SIGINFO | SA_NOCLDSTOP | SA_NODEFER; + if (sigemptyset(&sa.sa_mask) != 0 || + sigaction(signal, &sa, old_sa_ptr) != 0) { + std::ostringstream oss; + oss << "An error occurred while setting handler for " << strsignal(signal) + << '.'; + TORCH_CHECK(false, oss.str()); + } +} + +SIGNAL_HANDLER( + SIGBUS, + handler_SIGBUS, + "ERROR: Unexpected bus error encountered in worker. " + "This might be caused by insufficient shared memory (shm).\n") +SIGNAL_HANDLER( + SIGSEGV, + handler_SIGSEGV, + "ERROR: Unexpected segmentation fault encountered in worker.\n") +SIGNAL_HANDLER( + SIGFPE, + handler_SIGFPE, + "ERROR: Unexpected floating-point exception encountered in worker.\n") + +// When an error happened in DataLoader methods and Python starts to exit, the +// error trace will keep the loader alive, and Python may kill the children +// processes first before deleting the loader object. Then the cleaning up +// methods in DataLoader.__del__ are not yet called, and SIGCHILD will print an +// error saying a worker is killed by SIGTERM. So we suppress SIGTERM from main +// loader process here to avoid this by _exit(EXIT_SUCCESS). Note that if we +// exit with nonzero code, the loader SIGCHLD handler may report RuntimeError +// again, and then it defeats the whole purpose. +static void handler_SIGTERM(int sig, siginfo_t* info, void* ctx) { + if (info->si_pid == getppid()) { + _exit(EXIT_SUCCESS); + } + struct sigaction sa{}; + sa.sa_handler = SIG_DFL; + sa.sa_flags = 0; + if (sigemptyset(&sa.sa_mask) != 0 || sigaction(SIGTERM, &sa, nullptr) != 0) { + _exit(EXIT_FAILURE); + } else { + raise(SIGTERM); + } +} + +// NOLINTNEXTLINE(misc-use-internal-linkage) +__attribute__((weak)) void setDataLoaderSignalHandlers() {} + +static PyObject* THPModule_setWorkerSignalHandlers( + PyObject* module, + PyObject* arg) { + HANDLE_TH_ERRORS + setSignalHandler(SIGBUS, &handler_SIGBUS, nullptr); + setSignalHandler(SIGSEGV, &handler_SIGSEGV, nullptr); + setSignalHandler(SIGTERM, &handler_SIGTERM, nullptr); + setSignalHandler(SIGFPE, &handler_SIGFPE, nullptr); + setDataLoaderSignalHandlers(); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static std::map> worker_pids = {}; + +static PyObject* THPModule_errorIfAnyWorkerFails( + PyObject* module, + PyObject* noargs) { + HANDLE_TH_ERRORS + + // Only check the pids we care about + for (auto& w : worker_pids) { + auto& pid_set = w.second; + for (auto worker_pid : pid_set) { + // Use waitid rather than waitpid so that we can set NOWAIT, and that + // Python and other handlers can get whatever info they want about the + // child. + siginfo_t infop{}; + infop.si_pid = 0; + auto error = + waitid(P_PID, worker_pid, &infop, WEXITED | WNOHANG | WNOWAIT); + // ignore errors and case with no waitable child + if (error < 0 || infop.si_pid == 0) + continue; + if (infop.si_code == CLD_EXITED && + infop.si_status != EXIT_SUCCESS) { // exit with error + auto error_msg = fmt::format( + "DataLoader worker (pid {}) exited unexpectedly with exit code {}. " + "Details are lost due to multiprocessing. Rerunning with " + "num_workers=0 may give better error trace.", + worker_pid, + infop.si_status); + // This is necessary. Otherwise, the runtime error will kill the other + // workers, and trigger this again. + pid_set.clear(); + TORCH_CHECK(false, error_msg); + } else if ( + infop.si_code == CLD_KILLED || + infop.si_code == CLD_DUMPED) { // killed by signal + auto error_msg = fmt::format( + "DataLoader worker (pid {}) is killed by signal: {}. ", + worker_pid, + strsignal(infop.si_status)); + if (infop.si_status == SIGBUS) { + error_msg += + "It is possible that dataloader's workers are out of shared memory. " + "Please try to raise your shared memory limit."; + } + // This is necessary. Otherwise, the runtime error will kill the other + // workers, and trigger this again. + pid_set.clear(); + TORCH_CHECK(false, error_msg); + } + } + } + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +// We don't want to exit on any SIGCHLD from any child. child_pids is a tuple +// of pids we are interested in. +static PyObject* THPModule_setWorkerPIDs(PyObject* module, PyObject* args) { + HANDLE_TH_ERRORS + TORCH_CHECK_TYPE( + PyTuple_GET_SIZE(args) == 2, + "_set_worker_pids expects exactly 2 arguments."); + int64_t key = THPUtils_unpackLong(PyTuple_GET_ITEM(args, 0)); + TORCH_CHECK_VALUE( + worker_pids.find(key) == worker_pids.end(), + "_set_worker_pids should be called only once for each _BaseDataLoaderIter."); + PyObject* child_pids = PyTuple_GET_ITEM(args, 1); + TORCH_CHECK_TYPE( + PyTuple_Check(child_pids), + "_set_worker_pids expects a tuple for child_pids, but got ", + Py_TYPE(child_pids)->tp_name, + "."); + std::set pids_set = {}; + auto size = PyTuple_GET_SIZE(child_pids); + for (const auto idx : c10::irange(size)) { + PyObject* obj = PyTuple_GET_ITEM(child_pids, idx); + pids_set.insert(static_cast(THPUtils_unpackLong(obj))); + } + + worker_pids[key] = pids_set; + + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* THPModule_removeWorkerPIDs( + PyObject* module, + PyObject* loader_id) { + HANDLE_TH_ERRORS + + int64_t key = THPUtils_unpackLong(loader_id); + auto it = worker_pids.find(key); + TORCH_CHECK_VALUE( + it != worker_pids.end(), + "Cannot find worker information for _BaseDataLoaderIter with id ", + key); + worker_pids.erase(it); + + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +#undef SIGNAL_HANDLER + +#else +// dummy implementations for windows + +static PyObject* THPModule_setWorkerSignalHandlers( + PyObject* module, + PyObject* _ignored) { + Py_RETURN_NONE; +} + +static PyObject* THPModule_setWorkerPIDs(PyObject* module, PyObject* _ignored) { + Py_RETURN_NONE; +} + +static PyObject* THPModule_removeWorkerPIDs( + PyObject* module, + PyObject* _ignored) { + Py_RETURN_NONE; +} + +static PyObject* THPModule_errorIfAnyWorkerFails( + PyObject* module, + PyObject* _ignored) { + Py_RETURN_NONE; +} + +#endif + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,cppcoreguidelines-avoid-non-const-global-variables,modernize-avoid-c-arrays) +PyMethodDef DataLoaderMethods[] = { + {"_set_worker_signal_handlers", + THPModule_setWorkerSignalHandlers, + METH_NOARGS, + nullptr}, + {"_set_worker_pids", THPModule_setWorkerPIDs, METH_VARARGS, nullptr}, + {"_remove_worker_pids", THPModule_removeWorkerPIDs, METH_O, nullptr}, + {"_error_if_any_worker_fails", + THPModule_errorIfAnyWorkerFails, + METH_NOARGS, + nullptr}, + {nullptr, nullptr, 0, nullptr}}; diff --git a/western_ai/usa/pytorch/torch/csrc/DataLoader.h b/western_ai/usa/pytorch/torch/csrc/DataLoader.h new file mode 100644 index 0000000000000000000000000000000000000000..405c2620b62e7663a6a847515e4a019819d2be11 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/DataLoader.h @@ -0,0 +1,6 @@ +#pragma once + +#include + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,cppcoreguidelines-avoid-non-const-global-variables,modernize-avoid-c-arrays) +extern PyMethodDef DataLoaderMethods[]; diff --git a/western_ai/usa/pytorch/torch/csrc/Device.cpp b/western_ai/usa/pytorch/torch/csrc/Device.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1e8d46a495e6476cb132470eea00a124715e86ac --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/Device.cpp @@ -0,0 +1,290 @@ +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +static PyObject* THPUpperModuleOfDevice = nullptr; + +PyObject* THPDevice_New(const at::Device& device) { + auto type = &THPDeviceType; + auto self = THPObjectPtr{type->tp_alloc(type, 0)}; + if (!self) + throw python_error(); + auto self_ = reinterpret_cast(self.get()); + self_->device = device; + return self.release(); +} + +static PyObject* THPDevice_repr(THPDevice* self) { + std::ostringstream oss; + oss << "device(type=\'" << self->device.type() << '\''; + if (self->device.has_index()) { + // `self->device.index()` returns uint8_t which is treated as ascii while + // printing, hence casting it to uint16_t. + // https://stackoverflow.com/questions/19562103/uint8-t-cant-be-printed-with-cout + oss << ", index=" << static_cast(self->device.index()); + } + oss << ')'; + return THPUtils_packString(oss.str().c_str()); +} + +static PyObject* THPDevice_str(THPDevice* self) { + std::ostringstream oss; + oss << self->device; + return THPUtils_packString(oss.str().c_str()); +} + +static PyObject* THPDevice_pynew( + PyTypeObject* type, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static torch::PythonArgParser parser( + {"device(Device device)", + "device(std::string_view type, int64_t? index=-1)"}); + torch::ParsedArgs<2> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + if (r.has_torch_function()) { + return handle_torch_function( + r, nullptr, args, kwargs, THPUpperModuleOfDevice, "torch"); + } + if (r.idx == 0) { + auto device = r.device(0); + return THPDevice_New(device); + } else if (r.idx == 1) { + auto as_device = r.device(0); // this works, because device can take strings + if (as_device.has_index()) { + auto device_type = r.string(0); + TORCH_CHECK( + false, + "type (string) must not include an index because index " + "was passed explicitly: " + + device_type); + } + int64_t device_index = -1; + if (!r.isNone(1)) { + device_index = r.toInt64(1); + // -1 is allowed in ATen/C++, to mean the default device, but not in + // Python. + TORCH_CHECK(device_index >= 0, "Device index must not be negative"); + } + at::Device device( + as_device.type(), static_cast(device_index)); + return THPDevice_New(device); + } + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDevice_type(THPDevice* self, PyObject* noargs) { + HANDLE_TH_ERRORS + std::ostringstream oss; + oss << self->device.type(); + return THPUtils_packString(oss.str().c_str()); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDevice_index(THPDevice* self, PyObject* noargs) { + HANDLE_TH_ERRORS + if (self->device.has_index()) { + return THPUtils_packInt64(self->device.index()); + } else { + Py_RETURN_NONE; + } + END_HANDLE_TH_ERRORS +} + +static Py_ssize_t THPDevice_hash(THPDevice* self) { + HANDLE_TH_ERRORS + return static_cast( + std::hash{}(self->device) % + std::numeric_limits::max()); + END_HANDLE_TH_ERRORS_RET(-1) +} + +static PyObject* THPDevice_rc(PyObject* a, PyObject* b, int op) { + HANDLE_TH_ERRORS + if (!THPDevice_Check(a) || !THPDevice_Check(b)) { + Py_RETURN_NOTIMPLEMENTED; + } + THPDevice* da = reinterpret_cast(a); + THPDevice* db = reinterpret_cast(b); + + switch (op) { + case Py_EQ: + if (da->device == db->device) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + case Py_NE: + if (da->device == db->device) { + Py_RETURN_FALSE; + } else { + Py_RETURN_TRUE; + } + case Py_LT: + case Py_LE: + case Py_GT: + case Py_GE: + TORCH_CHECK_TYPE(false, "comparison not implemented"); + default: + TORCH_CHECK_TYPE(false, "unexpected comparison op"); + } + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDevice_reduce(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS + auto self = reinterpret_cast(_self); + auto ret = THPObjectPtr{PyTuple_New(2)}; + if (!ret) + throw python_error(); + + py::object torch_module = py::module::import("torch"); + py::object torch_device = torch_module.attr("device"); + PyTuple_SET_ITEM(ret.get(), 0, torch_device.release().ptr()); + + THPObjectPtr args; + std::ostringstream oss; + oss << self->device.type(); + if (self->device.has_index()) { + args = THPObjectPtr{Py_BuildValue( + "(si)", oss.str().c_str(), static_cast(self->device.index()))}; + } else { + args = THPObjectPtr{Py_BuildValue("(s)", oss.str().c_str())}; + } + if (!args) + throw python_error(); + PyTuple_SET_ITEM(ret.get(), 1, args.release()); + + return ret.release(); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDevice_enter(PyObject* self, PyObject* noargs) { + HANDLE_TH_ERRORS + py::object mode = py::module::import("torch.utils._device") + .attr("DeviceContext")(py::handle(self)); + at::impl::PythonTorchFunctionTLS::push_onto_stack( + std::make_shared( + mode.release().ptr(), getPyInterpreter())); + // So that with torch.device('cuda') as dev: works + return Py_NewRef(self); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDevice_exit(PyObject* self, PyObject* unused) { + HANDLE_TH_ERRORS + at::impl::PythonTorchFunctionTLS::pop_stack(); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +/* +// TODO: We're not sure if this is a good idea or not, because making +// torch.device callable means that it will start returning true +// for callable() queries, and that is unexpected. We can always add +// this later, so for now, don't actually implement this. +static PyObject* THPDevice_call( + PyObject* self, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + py::object deco = + py::module::import("torch.utils._device").attr("device_decorator"); + return deco(py::handle(self), *py::handle(args), **py::handle(kwargs)) + .release() + .ptr(); + END_HANDLE_TH_ERRORS +} +*/ + +typedef PyObject* (*getter)(PyObject*, void*); + +// NB: If you edit these properties/methods, update torch/_C/__init__.pyi.in + +static const std::initializer_list THPDevice_properties = { + {"type", + reinterpret_cast(THPDevice_type), + nullptr, + nullptr, + nullptr}, + {"index", + reinterpret_cast(THPDevice_index), + nullptr, + nullptr, + nullptr}, + {nullptr}}; + +static const std::initializer_list THPDevice_methods = { + {"__reduce__", THPDevice_reduce, METH_NOARGS, nullptr}, + {"__enter__", THPDevice_enter, METH_NOARGS, nullptr}, + {"__exit__", THPDevice_exit, METH_VARARGS, nullptr}, + {nullptr} /* Sentinel */ +}; + +PyTypeObject THPDeviceType = { + PyVarObject_HEAD_INIT(nullptr, 0) + "torch.device", /* tp_name */ + sizeof(THPDevice), /* tp_basicsize */ + 0, /* tp_itemsize */ + nullptr, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + nullptr, /* tp_getattr */ + nullptr, /* tp_setattr */ + nullptr, /* tp_reserved */ + reinterpret_cast(THPDevice_repr), /* tp_repr */ + nullptr, /* tp_as_number */ + nullptr, /* tp_as_sequence */ + nullptr, /* tp_as_mapping */ + reinterpret_cast(THPDevice_hash), /* tp_hash */ + // TODO: We're not sure if this is a good idea or not, because making + // torch.device callable means that it will start returning true + // for callable() queries, and that is unexpected. We can always add + // this later, so for now, don't actually implement this + // THPDevice_call, /* tp_call */ + nullptr, /* tp_call */ + reinterpret_cast(THPDevice_str), /* tp_str */ + nullptr, /* tp_getattro */ + nullptr, /* tp_setattro */ + nullptr, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + nullptr, /* tp_doc */ + nullptr, /* tp_traverse */ + nullptr, /* tp_clear */ + static_cast(THPDevice_rc), /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + nullptr, /* tp_iter */ + nullptr, /* tp_iternext */ + // NOLINTNEXTLINE(*const-cast) + const_cast(std::data(THPDevice_methods)), /* tp_methods */ + nullptr, /* tp_members */ + // NOLINTNEXTLINE(*const-cast) + const_cast(std::data(THPDevice_properties)), /* tp_getset */ + nullptr, /* tp_base */ + nullptr, /* tp_dict */ + nullptr, /* tp_descr_get */ + nullptr, /* tp_descr_set */ + 0, /* tp_dictoffset */ + nullptr, /* tp_init */ + nullptr, /* tp_alloc */ + THPDevice_pynew, /* tp_new */ +}; + +void THPDevice_init(PyObject* module) { + THPUpperModuleOfDevice = module; + if (PyModule_AddType(module, &THPDeviceType) < 0) { + throw python_error(); + } +} diff --git a/western_ai/usa/pytorch/torch/csrc/Device.h b/western_ai/usa/pytorch/torch/csrc/Device.h new file mode 100644 index 0000000000000000000000000000000000000000..d4d304df6d15ac66ddfbd73daa5591e9c3d048f9 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/Device.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include + +// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) +struct TORCH_PYTHON_API THPDevice { + PyObject_HEAD + at::Device device; +}; + +TORCH_PYTHON_API extern PyTypeObject THPDeviceType; + +inline bool THPDevice_Check(PyObject* obj) { + return Py_TYPE(obj) == &THPDeviceType; +} + +TORCH_PYTHON_API PyObject* THPDevice_New(const at::Device& device); + +TORCH_PYTHON_API void THPDevice_init(PyObject* module); diff --git a/western_ai/usa/pytorch/torch/csrc/DeviceAccelerator.cpp b/western_ai/usa/pytorch/torch/csrc/DeviceAccelerator.cpp new file mode 100644 index 0000000000000000000000000000000000000000..04701e939828a6ac32ac9c96bb74477d6efcb74d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/DeviceAccelerator.cpp @@ -0,0 +1,251 @@ +#include +#include +#include +#include + +namespace torch::accelerator { + +void initModule(PyObject* module) { + auto m = py::handle(module).cast(); + + m.def("_accelerator_getAccelerator", []() -> std::optional { + // If no accelerator was available at compile time, return None. + auto acc = at::getAccelerator(false); + if (acc.has_value()) { + return acc.value(); + } else { + return std::nullopt; + } + }); + + m.def("_accelerator_setDeviceIndex", [](c10::DeviceIndex device_index) { + // If device index is negative, no-op + if (device_index < 0) { + return; + } + const auto device_type = at::accelerator::getAccelerator(true).value(); + torch::utils::maybe_initialize_device(device_type); + at::accelerator::setDeviceIndex(device_index); + }); + + m.def("_accelerator_getDeviceIndex", []() { + const auto device_type = at::accelerator::getAccelerator(true).value(); + torch::utils::maybe_initialize_device(device_type); + return at::accelerator::getDeviceIndex(); + }); + + m.def("_accelerator_getDeviceCapability", [](c10::DeviceIndex device_index) { + const auto device_type = at::accelerator::getAccelerator(true).value(); + torch::utils::maybe_initialize_device(device_type); + auto caps = at::accelerator::getDeviceCapability(device_index); + + py::dict dict; + + py::set dtype_set; + caps.forEachSupportedScalarType([&](c10::ScalarType dtype) { + THPDtype* thp_dtype = torch::getTHPDtype(dtype); + py::object dtype_obj = + py::reinterpret_borrow((PyObject*)thp_dtype); + dtype_set.add(dtype_obj); + }); + + dict["supported_dtypes"] = dtype_set; + return dict; + }); + + m.def("_accelerator_setStream", [](c10::Stream stream) { + const auto device_type = at::accelerator::getAccelerator(true).value(); + torch::utils::maybe_initialize_device(device_type); + // Set the current device to the device of stream + if (at::accelerator::getDeviceIndex() != stream.device_index()) { + at::accelerator::setDeviceIndex(stream.device_index()); + } + at::accelerator::setCurrentStream(stream); + }); + + m.def("_accelerator_getStream", [](c10::DeviceIndex device_index) { + const auto device_type = at::accelerator::getAccelerator(true).value(); + torch::utils::maybe_initialize_device(device_type); + return at::accelerator::getCurrentStream(device_index); + }); + + m.def("_accelerator_synchronizeDevice", [](c10::DeviceIndex device_index) { + const auto device_type = at::accelerator::getAccelerator(true).value(); + if (torch::utils::is_device_lazy_init_supported(device_type) && + !torch::utils::is_device_initialized(device_type)) { + return; + } + torch::utils::maybe_initialize_device(device_type); + { + py::gil_scoped_release no_gil; + at::accelerator::synchronizeDevice(device_index); + } + }); + + m.def("_accelerator_exchangeDevice", [](c10::DeviceIndex device_index) { + const auto device_type = at::accelerator::getAccelerator(true).value(); + torch::utils::maybe_initialize_device(device_type); + return at::accelerator::exchangeDevice(device_index); + }); + + m.def("_accelerator_maybeExchangeDevice", [](c10::DeviceIndex device_index) { + const auto device_type = at::accelerator::getAccelerator(true).value(); + torch::utils::maybe_initialize_device(device_type); + return at::accelerator::maybeExchangeDevice(device_index); + }); + + m.def("_accelerator_isAllocatorInitialized", []() { + const auto device_type = at::accelerator::getAccelerator(true).value(); + return at::getDeviceAllocator(device_type)->initialized(); + }); + + m.def("_accelerator_emptyCache", []() { at::accelerator::emptyCache(); }); + + m.def("_accelerator_emptyHostCache", []() { + const auto device_type = at::accelerator::getAccelerator(true).value(); + if (torch::utils::is_device_lazy_init_supported(device_type) && + !torch::utils::is_device_initialized(device_type)) { + return; + } + at::accelerator::emptyHostCache(); + }); + + m.def("_accelerator_getDeviceStats", [](c10::DeviceIndex device_index) { + using c10::CachingAllocator::Stat; + using c10::CachingAllocator::StatArray; + using c10::CachingAllocator::StatType; + using c10::CachingDeviceAllocator::DeviceStats; + + const auto stats = at::accelerator::getDeviceStats(device_index); + const auto stat_to_dict = [](const Stat& stat) -> py::dict { + py::dict dict; + dict["current"] = stat.current; + dict["peak"] = stat.peak; + dict["allocated"] = stat.allocated; + dict["freed"] = stat.freed; + return dict; + }; + + const auto stat_array_to_dict = [=](const StatArray& stats) -> py::dict { + const std::array(StatType::NUM_TYPES)> + kStatTypeNames = {"all", "small_pool", "large_pool"}; + py::dict dict; + for (const auto i : c10::irange(kStatTypeNames.size())) { + dict[kStatTypeNames[i]] = stat_to_dict(stats[i]); + } + return dict; + }; + + py::dict result; + result["num_alloc_retries"] = stats.num_alloc_retries; + result["num_ooms"] = stats.num_ooms; + result["max_split_size"] = stats.max_split_size; + result["num_sync_all_streams"] = stats.num_sync_all_streams; + result["num_device_alloc"] = stats.num_device_alloc; + result["num_device_free"] = stats.num_device_free; + result["allocated_bytes"] = stat_array_to_dict(stats.allocated_bytes); + result["reserved_bytes"] = stat_array_to_dict(stats.reserved_bytes); + result["active_bytes"] = stat_array_to_dict(stats.active_bytes); + result["requested_bytes"] = stat_array_to_dict(stats.requested_bytes); + result["allocation"] = stat_array_to_dict(stats.allocation); + result["segment"] = stat_array_to_dict(stats.segment); + result["active"] = stat_array_to_dict(stats.active); + result["inactive_split"] = stat_array_to_dict(stats.inactive_split); + result["inactive_split_bytes"] = + stat_array_to_dict(stats.inactive_split_bytes); + result["oversize_allocations"] = stat_to_dict(stats.oversize_allocations); + result["oversize_segments"] = stat_to_dict(stats.oversize_segments); + return result; + }); + + m.def( + "_accelerator_resetAccumulatedStats", [](c10::DeviceIndex device_index) { + at::accelerator::resetAccumulatedStats(device_index); + }); + + m.def("_accelerator_resetPeakStats", [](c10::DeviceIndex device_index) { + at::accelerator::resetPeakStats(device_index); + }); + + m.def("_accelerator_getMemoryInfo", [](c10::DeviceIndex device_index) { + const auto device_type = at::accelerator::getAccelerator(true).value(); + torch::utils::maybe_initialize_device(device_type); + py::gil_scoped_release no_gil; + return at::accelerator::getMemoryInfo(device_index); + }); + + m.def("_accelerator_getAllocatorSettings", []() { + return c10::CachingAllocator::getAllocatorSettings(); + }); + + m.def("_accelerator_setAllocatorSettings", [](std::string env) { + c10::CachingAllocator::setAllocatorSettings(env); + }); + + m.def("_accelerator_getDefaultGenerator", [](c10::DeviceIndex device_index) { + const auto device_type = at::accelerator::getAccelerator(true).value(); + torch::utils::maybe_initialize_device(device_type); + return at::accelerator::getDefaultGenerator(device_index); + }); + + // Accelerator Graph class binding + py::class_>( + m, "_acceleratorGraph") + .def(py::init(), py::arg("keep_graph") = false) + .def( + "capture_begin", + [](at::accelerator::Graph& self, + std::optional pool_opt, + const std::string& capture_error_mode) { + c10::MempoolId_t pool = pool_opt.has_value() + ? pool_opt.value() + : c10::MempoolId_t{0, 0}; + at::GraphCaptureMode capture_mode = at::GraphCaptureMode::Default; + if (capture_error_mode == "default") { + capture_mode = at::GraphCaptureMode::Default; + } else if (capture_error_mode == "global") { + capture_mode = at::GraphCaptureMode::Global; + } else if (capture_error_mode == "thread_local") { + capture_mode = at::GraphCaptureMode::ThreadLocal; + } else if (capture_error_mode == "relaxed") { + capture_mode = at::GraphCaptureMode::Relaxed; + } else { + TORCH_CHECK( + false, + "Unknown capture error mode. Expected `default`, `global`, `thread_local`, or `relaxed`, got ", + capture_error_mode); + } + return self.capture_begin(pool, capture_mode); + }, + py::arg("pool") = std::nullopt, + py::arg("capture_error_mode") = "default", + py::call_guard()) + .def( + "capture_end", + torch::wrap_pybind_function_no_gil( + &at::accelerator::Graph::capture_end)) + .def( + "instantiate", + torch::wrap_pybind_function_no_gil( + &at::accelerator::Graph::instantiate)) + .def( + "replay", + torch::wrap_pybind_function_no_gil(&at::accelerator::Graph::replay)) + .def( + "reset", + torch::wrap_pybind_function_no_gil(&at::accelerator::Graph::reset)) + .def( + "pool", + torch::wrap_pybind_function_no_gil(&at::accelerator::Graph::pool)) + .def( + "enable_debug_mode", + torch::wrap_pybind_function_no_gil( + &::at::accelerator::Graph::enable_debug_mode)) + .def( + "debug_dump", + torch::wrap_pybind_function_no_gil( + &::at::accelerator::Graph::debug_dump), + py::arg("path")); +} + +} // namespace torch::accelerator diff --git a/western_ai/usa/pytorch/torch/csrc/DeviceAccelerator.h b/western_ai/usa/pytorch/torch/csrc/DeviceAccelerator.h new file mode 100644 index 0000000000000000000000000000000000000000..33865e2d6cc39c1347d7de592a73202ec308452b --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/DeviceAccelerator.h @@ -0,0 +1,8 @@ +#include +#include + +namespace torch::accelerator { + +void initModule(PyObject* module); + +} // namespace torch::accelerator diff --git a/western_ai/usa/pytorch/torch/csrc/Dtype.cpp b/western_ai/usa/pytorch/torch/csrc/Dtype.cpp new file mode 100644 index 0000000000000000000000000000000000000000..354de13c625c96a8fdc8b951efb5c6106a31fe9d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/Dtype.cpp @@ -0,0 +1,199 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +PyObject* THPDtype_New(at::ScalarType scalar_type, const std::string& name) { + AT_ASSERT(name.length() < DTYPE_NAME_LEN); + auto type = &THPDtypeType; + auto self = THPObjectPtr{type->tp_alloc(type, 0)}; + if (!self) + throw python_error(); + auto self_ = reinterpret_cast(self.get()); + self_->scalar_type = scalar_type; + std::strncpy(self_->name, name.c_str(), DTYPE_NAME_LEN); + return self.release(); +} + +static PyObject* THPDtype_is_floating_point(THPDtype* self, PyObject* noargs) { + HANDLE_TH_ERRORS + if (at::isFloatingType(self->scalar_type)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDtype_itemsize(THPDtype* self, PyObject* noargs) { + HANDLE_TH_ERRORS + return THPUtils_packUInt64( + scalarTypeToTypeMeta(self->scalar_type).itemsize()); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDtype_is_complex(THPDtype* self, PyObject* noargs) { + HANDLE_TH_ERRORS + if (at::isComplexType(self->scalar_type)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDtype_is_signed(THPDtype* self, PyObject* noargs) { + HANDLE_TH_ERRORS + if (at::isSignedType(self->scalar_type)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDtype_abbr(THPDtype* self, PyObject* noargs) { + HANDLE_TH_ERRORS + auto abbr = c10::getScalarTypeAbbr(self->scalar_type); + return PyUnicode_FromStringAndSize( + abbr.data(), static_cast(abbr.size())); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDtype_reduce(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS + /* + * For singletons, a string is returned. The string should be interpreted + * as the name of a global variable. + */ + auto self = reinterpret_cast(_self); + return THPUtils_packString(self->name); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDtype_to_real(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS + auto* self = reinterpret_cast(_self); + auto scalar_type = self->scalar_type; + if (!at::isFloatingType(self->scalar_type)) { + scalar_type = at::toRealValueType(self->scalar_type); + } + return Py_NewRef(torch::getTHPDtype(scalar_type)); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPDtype_to_complex(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS + auto* self = reinterpret_cast(_self); + auto scalar_type = self->scalar_type; + if (!at::isComplexType(self->scalar_type)) { + scalar_type = at::toComplexType(self->scalar_type); + } + return Py_NewRef(torch::getTHPDtype(scalar_type)); + END_HANDLE_TH_ERRORS +} + +typedef PyObject* (*getter)(PyObject*, void*); + +static const std::initializer_list THPDtype_properties = { + {"is_floating_point", + reinterpret_cast(THPDtype_is_floating_point), + nullptr, + nullptr, + nullptr}, + {"is_complex", + reinterpret_cast(THPDtype_is_complex), + nullptr, + nullptr, + nullptr}, + {"is_signed", + reinterpret_cast(THPDtype_is_signed), + nullptr, + nullptr, + nullptr}, + {"itemsize", + reinterpret_cast(THPDtype_itemsize), + nullptr, + nullptr, + nullptr}, + {"abbr", + reinterpret_cast(THPDtype_abbr), + nullptr, + nullptr, + nullptr}, + {nullptr}}; + +static const std::initializer_list THPDtype_methods = { + {"__reduce__", THPDtype_reduce, METH_NOARGS, nullptr}, + {"to_real", THPDtype_to_real, METH_NOARGS, nullptr}, + {"to_complex", THPDtype_to_complex, METH_NOARGS, nullptr}, + {nullptr} /* Sentinel */ +}; + +static PyObject* THPDtype_repr(THPDtype* self) { + return THPUtils_packString(std::string("torch.") + self->name); +} + +PyTypeObject THPDtypeType = { + PyVarObject_HEAD_INIT(nullptr, 0) + "torch.dtype", /* tp_name */ + sizeof(THPDtype), /* tp_basicsize */ + 0, /* tp_itemsize */ + nullptr, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + nullptr, /* tp_getattr */ + nullptr, /* tp_setattr */ + nullptr, /* tp_reserved */ + reinterpret_cast(THPDtype_repr), /* tp_repr */ + nullptr, /* tp_as_number */ + nullptr, /* tp_as_sequence */ + nullptr, /* tp_as_mapping */ + nullptr, /* tp_hash */ + nullptr, /* tp_call */ + nullptr, /* tp_str */ + nullptr, /* tp_getattro */ + nullptr, /* tp_setattro */ + nullptr, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + nullptr, /* tp_doc */ + nullptr, /* tp_traverse */ + nullptr, /* tp_clear */ + nullptr, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + nullptr, /* tp_iter */ + nullptr, /* tp_iternext */ + // NOLINTNEXTLINE(*const-cast) + const_cast(std::data(THPDtype_methods)), /* tp_methods */ + nullptr, /* tp_members */ + // NOLINTNEXTLINE(*const-cast) + const_cast(std::data(THPDtype_properties)), /* tp_getset */ + nullptr, /* tp_base */ + nullptr, /* tp_dict */ + nullptr, /* tp_descr_get */ + nullptr, /* tp_descr_set */ + 0, /* tp_dictoffset */ + nullptr, /* tp_init */ + nullptr, /* tp_alloc */ + nullptr, /* tp_new */ +}; + +void THPDtype_init(PyObject* module) { + // Set __module__ = "torch" so pickle can find dtype instances without + // scanning sys.modules. See https://github.com/pytorch/pytorch/issues/65077 + if (PyModule_AddType(module, &THPDtypeType) < 0) { + throw python_error(); + } + auto torch_name = THPUtils_packString("torch"); + if (!torch_name) + throw python_error(); + if (PyDict_SetItemString(THPDtypeType.tp_dict, "__module__", torch_name) < + 0) { + throw python_error(); + } + PyType_Modified(&THPDtypeType); +} diff --git a/western_ai/usa/pytorch/torch/csrc/Dtype.h b/western_ai/usa/pytorch/torch/csrc/Dtype.h new file mode 100644 index 0000000000000000000000000000000000000000..1f196eab736a4ba3565ed13f73c874427ae472c2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/Dtype.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +constexpr int DTYPE_NAME_LEN = 64; + +struct TORCH_PYTHON_API THPDtype { + PyObject_HEAD + at::ScalarType scalar_type; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) + char name[DTYPE_NAME_LEN + 1]; +}; + +TORCH_PYTHON_API extern PyTypeObject THPDtypeType; + +inline bool THPDtype_Check(PyObject* obj) { + return Py_TYPE(obj) == &THPDtypeType; +} + +inline bool THPPythonScalarType_Check(PyObject* obj) { + return obj == (PyObject*)(&PyFloat_Type) || + obj == (PyObject*)(&PyComplex_Type) || obj == (PyObject*)(&PyBool_Type) || + obj == (PyObject*)(&PyLong_Type); +} + +TORCH_PYTHON_API PyObject* THPDtype_New( + at::ScalarType scalar_type, + const std::string& name); + +TORCH_PYTHON_API void THPDtype_init(PyObject* module); diff --git a/western_ai/usa/pytorch/torch/csrc/DynamicTypes.cpp b/western_ai/usa/pytorch/torch/csrc/DynamicTypes.cpp new file mode 100644 index 0000000000000000000000000000000000000000..391b27e358d1f18d79877e688f39c1b5cf5946ab --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/DynamicTypes.cpp @@ -0,0 +1,123 @@ + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace torch { +namespace { +std::array(at::ScalarType::NumOptions)> + dtype_registry = {}; + +std::array(at::Layout::NumOptions)> + layout_registry = {}; + +} // namespace + +void registerDtypeObject(THPDtype* dtype, at::ScalarType scalarType) { + dtype_registry[static_cast(scalarType)] = dtype; +} + +void registerLayoutObject(THPLayout* thp_layout, at::Layout layout) { + layout_registry[static_cast(layout)] = thp_layout; +} + +THPDtype* getTHPDtype(at::ScalarType scalarType) { + auto dtype = dtype_registry[static_cast(scalarType)]; + if (!dtype) { + throw std::invalid_argument("unsupported scalarType"); + } + return dtype; +} + +THPLayout* getTHPLayout(at::Layout layout) { + auto thp_layout = layout_registry[static_cast(layout)]; + if (!thp_layout) { + throw std::invalid_argument("unsupported at::Layout"); + } + return thp_layout; +} + +PyObject* createPyObject(const at::Storage& storage) { + // Note [Invalid Python Storages] + // When a user creates a python tensor wrapper subclass, the subclass + // is a tensor object that has a nullptr storage. + // We still allow users to call `my_subclass.untyped_storage()`, and get back + // a valid storage object (this can be useful for detecting aliasing + // information about storages from python). However, any accesses to the + // data_ptr is not allowed, through methods like + // x.untyped_storage().data_ptr() + PyObject* obj = THPStorage_Wrap(storage); + if (!obj) + throw python_error(); + return obj; +} + +static PyTypeObject* loadTypedStorageTypeObject() { + THPObjectPtr storage_module(PyImport_ImportModule("torch.storage")); + TORCH_INTERNAL_ASSERT(storage_module && PyModule_Check(storage_module)); + + PyObject* typed_storage_obj = + PyObject_GetAttrString(storage_module, "TypedStorage"); + TORCH_INTERNAL_ASSERT(typed_storage_obj && PyType_Check(typed_storage_obj)); + return reinterpret_cast(typed_storage_obj); +} + +static PyTypeObject* getTypedStorageTypeObject() { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) + static PyTypeObject* typed_storage_type_obj = loadTypedStorageTypeObject(); + return typed_storage_type_obj; +} + +bool isStorage(PyObject* obj) { + if (PyObject_TypeCheck(obj, getTypedStorageTypeObject())) { + return true; + } + return THPStorage_Check(obj); +} + +std::tuple createStorageGetType( + PyObject* obj) { + at::ScalarType scalar_type = at::ScalarType::Undefined; + bool is_typed_storage = PyObject_TypeCheck(obj, getTypedStorageTypeObject()); + PyObject* untyped_storage_obj = nullptr; + + if (is_typed_storage) { + // NOTE: `PyObject_GetAttrString` increments the refcounts to `dtype` and + // `_untyped_storage`, so we must decrement them. The refcounts will still + // stay nonzero since the `TypedStorage` maintains a reference. + PyObject* dtype_obj = PyObject_GetAttrString(obj, "dtype"); + TORCH_INTERNAL_ASSERT(dtype_obj); + TORCH_INTERNAL_ASSERT(THPDtype_Check(dtype_obj)); + scalar_type = reinterpret_cast(dtype_obj)->scalar_type; + Py_DECREF(dtype_obj); + + untyped_storage_obj = PyObject_GetAttrString(obj, "_untyped_storage"); + TORCH_INTERNAL_ASSERT(untyped_storage_obj); + Py_DECREF(untyped_storage_obj); + + } else { + scalar_type = at::kByte; + untyped_storage_obj = obj; + } + + TORCH_CHECK( + THPStorage_Check(untyped_storage_obj), + "not a storage '", + Py_TYPE(obj)->tp_name, + "'"); + + auto storage = THPStorage_Unpack(untyped_storage_obj); + return std::make_tuple(std::move(storage), scalar_type, is_typed_storage); +} + +at::Storage createStorage(PyObject* obj) { + return std::get<0>(createStorageGetType(obj)); +} + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/DynamicTypes.h b/western_ai/usa/pytorch/torch/csrc/DynamicTypes.h new file mode 100644 index 0000000000000000000000000000000000000000..f7d3e0855f30d0984945c70dabe43e5f0f3a7651 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/DynamicTypes.h @@ -0,0 +1,37 @@ +#pragma once + +// Provides conversions between Python tensor objects and at::Tensor. + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +struct THPDtype; +struct THPLayout; + +namespace c10 { +struct Storage; +} + +namespace torch { +void registerDtypeObject(THPDtype* dtype, at::ScalarType scalarType); +void registerLayoutObject(THPLayout* thp_layout, at::Layout layout); + +TORCH_PYTHON_API PyObject* createPyObject(const at::Storage& storage); +TORCH_PYTHON_API at::Storage createStorage(PyObject* obj); +TORCH_PYTHON_API std::tuple +createStorageGetType(PyObject* obj); +TORCH_PYTHON_API bool isStorage(PyObject* obj); + +// Both methods below return a borrowed reference! +TORCH_PYTHON_API THPDtype* getTHPDtype(at::ScalarType scalarType); +TORCH_PYTHON_API THPLayout* getTHPLayout(at::Layout layout); +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/Event.cpp b/western_ai/usa/pytorch/torch/csrc/Event.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2178360d6d87acdbf8843796ef5b849c9d877f35 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/Event.cpp @@ -0,0 +1,341 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +PyTypeObject* THPEventClass = nullptr; + +static PyObject* THPEvent_pynew( + PyTypeObject* type, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + + unsigned char enable_timing = 0; + unsigned char blocking = 0; + unsigned char interprocess = 0; + + static torch::PythonArgParser parser({ + "Event(Device device=None, *, bool enable_timing=False, bool blocking=False, bool interprocess=False)", + }); + + torch::ParsedArgs<4> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + + auto device = r.deviceOptional(0); + + if (!device.has_value()) { + device = at::Device(at::getAccelerator(false).value_or(at::kCPU)); + } + enable_timing = r.toBoolWithDefault(1, false); + blocking = r.toBoolWithDefault(2, false); + interprocess = r.toBoolWithDefault(3, false); + + THPObjectPtr ptr(type->tp_alloc(type, 0)); + if (!ptr) { + TORCH_CHECK(ptr, "Failed to allocate memory for Event"); + } + + THPEvent* self = reinterpret_cast(ptr.get()); + self->weakreflist = nullptr; + + // TODO: blocking and interprocess are not supported yet. To support them, the + // flag system of c10::Event needs to be refactored. c10::Event should also + // provide a generic constructor to support blocking and interprocess events. + (void)blocking; + (void)interprocess; + + new (&self->event) c10::Event( + device->type(), + // See note [Flags defining the behavior of events] + // BACKEND_DEFAULT is a enable-timing flag, and + // PYTORCH_DEFAULT is a disable-timing flag. + (enable_timing ? c10::EventFlag::BACKEND_DEFAULT + : c10::EventFlag::PYTORCH_DEFAULT)); + + return static_cast(ptr.release()); + END_HANDLE_TH_ERRORS +} + +PyObject* THPEvent_new(c10::DeviceType device_type, c10::EventFlag flag) { + auto type = &THPEventType; + auto self = THPObjectPtr{type->tp_alloc(type, 0)}; + TORCH_CHECK(self, "Failed to allocate memory for Event"); + auto self_ = reinterpret_cast(self.get()); + self_->weakreflist = nullptr; + new (&self_->event) c10::Event(device_type, flag); + return self.release(); +} + +void THPEvent_dealloc_common(THPEvent* self) { + PyObject_ClearWeakRefs((PyObject*)self); + Py_TYPE(self)->tp_free(reinterpret_cast(self)); +} + +static void THPEvent_dealloc(THPEvent* self) { + { + pybind11::gil_scoped_release no_gil{}; + self->event.~Event(); + } + THPEvent_dealloc_common(self); +} + +static PyObject* THPEvent_get_device(THPEvent* self, void* unused) { + HANDLE_TH_ERRORS + return THPDevice_New(self->event.device()); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPEvent_record( + PyObject* _self, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + auto self = reinterpret_cast(_self); + PyObject* _stream = Py_None; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) + constexpr const char* accepted_args[] = {"stream", nullptr}; + if (!PyArg_ParseTupleAndKeywords( + args, + kwargs, + "|O", + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + const_cast(accepted_args), + &_stream)) { + TORCH_WARN("Parsing THPEvent_record arg fails"); + return nullptr; + } + if (!Py_IsNone(_stream)) { + auto stream = reinterpret_cast(_stream); + self->event.record(c10::Stream::unpack3( + stream->stream_id, + static_cast(stream->device_index), + static_cast(stream->device_type))); + } else { + c10::impl::VirtualGuardImpl impl{ + static_cast(self->event.device_type())}; + self->event.record(impl.getStream(impl.getDevice())); + } + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* THPEvent_from_ipc_handle( + PyObject* _type, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + auto type = reinterpret_cast(_type); + + static torch::PythonArgParser parser({ + "from_ipc_handle(Device device, std::string ipc_handle)", + }); + torch::ParsedArgs<2> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + + at::Device device = r.device(0); + TORCH_CHECK_NOT_IMPLEMENTED( + false, + "torch.Event ipc is not supported yet, please open an issue if you need this!"); + THPObjectPtr ptr(type->tp_alloc(type, 0)); + if (!ptr) { + return nullptr; + } + THPEvent* self = reinterpret_cast(ptr.get()); + + // TODO: for constructing event from ipc handle, the c10::Event needs to have + // more general constructor to achieve that. + new (&self->event) c10::Event(device.type(), c10::EventFlag::PYTORCH_DEFAULT); + + return static_cast(ptr.release()); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPEvent_ipc_handle( + PyObject* _self [[maybe_unused]], + PyObject* noargs) { + HANDLE_TH_ERRORS + TORCH_CHECK_NOT_IMPLEMENTED( + false, + "torch.Event ipc is not supported yet, please open an issue if you need this!"); + constexpr const char* handle = "0"; + return PyBytes_FromStringAndSize( + handle, std::char_traits::length(handle)); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPEvent_wait( + PyObject* _self, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS { + auto self = reinterpret_cast(_self); + PyObject* _stream = Py_None; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) + constexpr const char* accepted_args[] = {"stream", nullptr}; + if (!PyArg_ParseTupleAndKeywords( + args, + kwargs, + "|O", + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + const_cast(accepted_args), + &_stream)) { + TORCH_WARN("Parsing THPEvent_wait arg fails"); + return nullptr; + } + if (!Py_IsNone(_stream)) { + auto stream = reinterpret_cast(_stream); + self->event.block(c10::Stream::unpack3( + stream->stream_id, + static_cast(stream->device_index), + static_cast(stream->device_type))); + } else { + c10::impl::VirtualGuardImpl impl{ + static_cast(self->event.device_type())}; + self->event.block(impl.getStream(impl.getDevice())); + } + } + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* THPEvent_query(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS + auto self = reinterpret_cast(_self); + return PyBool_FromLong(self->event.query()); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPEvent_elapsed_time(PyObject* _self, PyObject* _other) { + HANDLE_TH_ERRORS + auto self = reinterpret_cast(_self); + // We expect it to be an explicit torch.Event instance. + TORCH_CHECK( + Py_TYPE(_other) == THPEventClass, + "expected other to be a torch.Event object"); + auto other = reinterpret_cast(_other); + return PyFloat_FromDouble(self->event.elapsedTime(other->event)); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPEvent_synchronize(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS { + pybind11::gil_scoped_release no_gil{}; + auto self = reinterpret_cast(_self); + self->event.synchronize(); + } + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* THPEvent_evend_id(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS + auto self = reinterpret_cast(_self); + return PyLong_FromVoidPtr(self->event.eventId()); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPEvent_repr(THPEvent* self) { + HANDLE_TH_ERRORS + return THPUtils_packString( + "torch.Event device_type=" + + c10::DeviceTypeName( + static_cast(self->event.device_type()), true) + + ", device_index=" + std::to_string(self->event.device_index()) + + ", event_flag=" + + std::to_string(static_cast(self->event.flag())) + ", event_id=" + + std::to_string(reinterpret_cast(self->event.eventId()))); + END_HANDLE_TH_ERRORS +} + +// NOLINTNEXTLINE(*c-arrays*, *global-variables) +static struct PyGetSetDef THPEvent_properties[] = { + {"device", + reinterpret_cast(THPEvent_get_device), + nullptr, + nullptr, + nullptr}, + {"event_id", + reinterpret_cast(THPEvent_evend_id), + nullptr, + nullptr, + nullptr}, + {nullptr}}; + +// NOLINTNEXTLINE(*c-arrays*, *global-variables) +static PyMethodDef THPEvent_methods[] = { + {"from_ipc_handle", + castPyCFunctionWithKeywords(THPEvent_from_ipc_handle), + METH_CLASS | METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"record", + castPyCFunctionWithKeywords(THPEvent_record), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"wait", + castPyCFunctionWithKeywords(THPEvent_wait), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"query", THPEvent_query, METH_NOARGS, nullptr}, + {"elapsed_time", THPEvent_elapsed_time, METH_O, nullptr}, + {"synchronize", THPEvent_synchronize, METH_NOARGS, nullptr}, + {"ipc_handle", THPEvent_ipc_handle, METH_NOARGS, nullptr}, + {nullptr}}; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +PyTypeObject THPEventType = { + PyVarObject_HEAD_INIT(nullptr, 0) + "torch.Event", /* tp_name */ + sizeof(THPEvent), /* tp_basicsize */ + 0, /* tp_itemsize */ + reinterpret_cast(THPEvent_dealloc), /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + nullptr, /* tp_getattr */ + nullptr, /* tp_setattr */ + nullptr, /* tp_reserved */ + reinterpret_cast(THPEvent_repr), /* tp_repr */ + nullptr, /* tp_as_number */ + nullptr, /* tp_as_sequence */ + nullptr, /* tp_as_mapping */ + nullptr, /* tp_hash */ + nullptr, /* tp_call */ + nullptr, /* tp_str */ + nullptr, /* tp_getattro */ + nullptr, /* tp_setattro */ + nullptr, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + nullptr, /* tp_doc */ + nullptr, /* tp_traverse */ + nullptr, /* tp_clear */ + nullptr, /* tp_richcompare */ + offsetof(THPEvent, weakreflist), /* tp_weaklistoffset */ + nullptr, /* tp_iter */ + nullptr, /* tp_iternext */ + THPEvent_methods, /* tp_methods */ + nullptr, /* tp_members */ + THPEvent_properties, /* tp_getset */ + nullptr, /* tp_base */ + nullptr, /* tp_dict */ + nullptr, /* tp_descr_get */ + nullptr, /* tp_descr_set */ + 0, /* tp_dictoffset */ + nullptr, /* tp_init */ + nullptr, /* tp_alloc */ + THPEvent_pynew, /* tp_new */ +}; +#pragma GCC diagnostic pop + +void THPEvent_init(PyObject* module) { + THPEventClass = &THPEventType; + if (PyModule_AddType(module, &THPEventType) < 0) { + throw python_error(); + } +} diff --git a/western_ai/usa/pytorch/torch/csrc/Event.h b/western_ai/usa/pytorch/torch/csrc/Event.h new file mode 100644 index 0000000000000000000000000000000000000000..ab9eb119aac33cc0d0696e8a723a7fa626e97848 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/Event.h @@ -0,0 +1,25 @@ +#ifndef THP_EVENT_INC +#define THP_EVENT_INC + +#include +#include +#include + +struct TORCH_PYTHON_API THPEvent { + PyObject_HEAD + c10::Event event; + PyObject* weakreflist; +}; +TORCH_PYTHON_API extern PyTypeObject* THPEventClass; +TORCH_PYTHON_API extern PyTypeObject THPEventType; + +TORCH_PYTHON_API void THPEvent_init(PyObject* module); +TORCH_PYTHON_API void THPEvent_dealloc_common(THPEvent* self); +TORCH_PYTHON_API PyObject* THPEvent_new( + c10::DeviceType device_type, + c10::EventFlag flag); +inline bool THPEvent_Check(PyObject* obj) { + return THPEventClass && PyObject_IsInstance(obj, (PyObject*)THPEventClass); +} + +#endif // THP_EVENT_INC diff --git a/western_ai/usa/pytorch/torch/csrc/Exceptions.cpp b/western_ai/usa/pytorch/torch/csrc/Exceptions.cpp new file mode 100644 index 0000000000000000000000000000000000000000..08658d76321936744b45cfda38f0908e99f62b4e --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/Exceptions.cpp @@ -0,0 +1,309 @@ +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +PyObject *THPException_FatalError, *THPException_LinAlgError, + *THPException_OutOfMemoryError, *THPException_DistError, + *THPException_DistBackendError, *THPException_DistNetworkError, + *THPException_DistStoreError, *THPException_DistQueueEmptyError, + *THPException_AcceleratorError; + +#define ASSERT_TRUE(cond) \ + if (!(cond)) \ + return false +bool THPException_init(PyObject* module) { + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + ASSERT_TRUE( + THPException_FatalError = + PyErr_NewException("torch.FatalError", nullptr, nullptr)); + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + ASSERT_TRUE( + PyModule_AddObject(module, "FatalError", THPException_FatalError) == 0); + + // Set the doc string here since _add_docstr throws malloc errors if tp_doc is + // modified for an error class. + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + ASSERT_TRUE( + THPException_LinAlgError = PyErr_NewExceptionWithDoc( + "torch._C._LinAlgError", + "Error raised by torch.linalg function when the cause of error is a numerical inconsistency in the data.\n \ +For example, you can the torch.linalg.inv function will raise torch.linalg.LinAlgError when it finds that \ +a matrix is not invertible.\n \ +\n\ +Example:\n \ +>>> # xdoctest: +REQUIRES(env:TORCH_DOCKTEST_LAPACK)\n \ +>>> matrix = torch.eye(3, 3)\n \ +>>> matrix[-1, -1] = 0\n \ +>>> matrix\n \ + tensor([[1., 0., 0.],\n \ + [0., 1., 0.],\n \ + [0., 0., 0.]])\n \ +>>> torch.linalg.inv(matrix)\n \ +Traceback (most recent call last):\n \ +File \"\", line 1, in \n \ +torch._C._LinAlgError: torch.linalg.inv: The diagonal element 3 is zero, the inversion\n \ +could not be completed because the input matrix is singular.", + PyExc_RuntimeError, + nullptr)); + ASSERT_TRUE( + PyModule_AddObject(module, "_LinAlgError", THPException_LinAlgError) == + 0); + + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + ASSERT_TRUE( + THPException_OutOfMemoryError = PyErr_NewExceptionWithDoc( + "torch.OutOfMemoryError", + "Exception raised when device is out of memory", + PyExc_RuntimeError, + nullptr)); + ASSERT_TRUE( + PyModule_AddObject( + module, "OutOfMemoryError", THPException_OutOfMemoryError) == 0); + + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + ASSERT_TRUE( + THPException_DistError = PyErr_NewExceptionWithDoc( + "torch.distributed.DistError", + "Exception raised when an error occurs in the distributed library", + PyExc_RuntimeError, + nullptr)); + ASSERT_TRUE( + PyModule_AddObject(module, "_DistError", THPException_DistError) == 0); + + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + ASSERT_TRUE( + THPException_DistBackendError = PyErr_NewExceptionWithDoc( + "torch.distributed.DistBackendError", + "Exception raised when a backend error occurs in distributed", + THPException_DistError, + nullptr)); + ASSERT_TRUE( + PyModule_AddObject( + module, "_DistBackendError", THPException_DistBackendError) == 0); + + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + ASSERT_TRUE( + THPException_DistNetworkError = PyErr_NewExceptionWithDoc( + "torch.distributed.DistNetworkError", + "Exception raised when a network error occurs in distributed", + THPException_DistError, + nullptr)); + ASSERT_TRUE( + PyModule_AddObject( + module, "_DistNetworkError", THPException_DistNetworkError) == 0); + + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + ASSERT_TRUE( + THPException_DistStoreError = PyErr_NewExceptionWithDoc( + "torch.distributed.DistStoreError", + "Exception raised when an error occurs in the distributed store", + THPException_DistError, + nullptr)); + ASSERT_TRUE( + PyModule_AddObject( + module, "_DistStoreError", THPException_DistStoreError) == 0); + + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + ASSERT_TRUE( + THPException_DistQueueEmptyError = PyErr_NewExceptionWithDoc( + "torch.distributed.QueueEmptyError", + "Exception raised when an error occurs in the distributed store", + THPException_DistStoreError, + nullptr)); + ASSERT_TRUE( + PyModule_AddObject( + module, "_DistQueueEmptyError", THPException_DistQueueEmptyError) == + 0); + + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + ASSERT_TRUE( + THPException_AcceleratorError = PyErr_NewExceptionWithDoc( + "torch.AcceleratorError", + "Exception raised while executing on device", + PyExc_RuntimeError, + nullptr)); + ASSERT_TRUE( + PyModule_AddObject( + module, "AcceleratorError", THPException_AcceleratorError) == 0); + + return true; +} + +namespace torch { + +static void processErrorMsgInplace(std::string& str) { + // Translate Aten types to their respective pytorch ones + static constexpr auto changes = + std::to_array>({ + {"SparseCUDAByteType", "torch.cuda.sparse.ByteTensor"}, + {"SparseCUDACharType", "torch.cuda.sparse.CharTensor"}, + {"SparseCUDADoubleType", "torch.cuda.sparse.DoubleTensor"}, + {"SparseCUDAFloatType", "torch.cuda.sparse.FloatTensor"}, + {"SparseCUDAIntType", "torch.cuda.sparse.IntTensor"}, + {"SparseCUDALongType", "torch.cuda.sparse.LongTensor"}, + {"SparseCUDAShortType", "torch.cuda.sparse.ShortTensor"}, + {"SparseCUDAHalfType", "torch.cuda.sparse.HalfTensor"}, + {"SparseCPUByteType", "torch.sparse.ByteTensor"}, + {"SparseCPUCharType", "torch.sparse.CharTensor"}, + {"SparseCPUDoubleType", "torch.sparse.DoubleTensor"}, + {"SparseCPUFloatType", "torch.sparse.FloatTensor"}, + {"SparseCPUIntType", "torch.sparse.IntTensor"}, + {"SparseCPULongType", "torch.sparse.LongTensor"}, + {"SparseCPUShortType", "torch.sparse.ShortTensor"}, + {"SparseCPUHalfType", "torch.sparse.HalfTensor"}, + {"CUDAByteType", "torch.cuda.ByteTensor"}, + {"CUDACharType", "torch.cuda.CharTensor"}, + {"CUDADoubleType", "torch.cuda.DoubleTensor"}, + {"CUDAFloatType", "torch.cuda.FloatTensor"}, + {"CUDAIntType", "torch.cuda.IntTensor"}, + {"CUDALongType", "torch.cuda.LongTensor"}, + {"CUDAShortType", "torch.cuda.ShortTensor"}, + {"CUDAHalfType", "torch.cuda.HalfTensor"}, + {"CPUByteType", "torch.ByteTensor"}, + {"CPUCharType", "torch.CharTensor"}, + {"CPUDoubleType", "torch.DoubleTensor"}, + {"CPUFloatType", "torch.FloatTensor"}, + {"CPUIntType", "torch.IntTensor"}, + {"CPULongType", "torch.LongTensor"}, + {"CPUShortType", "torch.ShortTensor"}, + {"CPUHalfType", "torch.HalfTensor"}, + }); + + // Avoid doing any work if no types need translated + if (str.find("Type") == str.npos) { + return; + } + for (const auto& it : changes) { + c10::ReplaceAll(str, it.first, it.second); + } +} + +std::string processErrorMsg(std::string str) { + processErrorMsgInplace(str); + return str; +} + +void translate_exception_to_python(const std::exception_ptr& e_ptr) { + try { + TORCH_INTERNAL_ASSERT( + e_ptr, + "translate_exception_to_python " + "called with invalid exception pointer"); + std::rethrow_exception(e_ptr); + } + CATCH_ALL_ERRORS(return) +} + +void PyWarningHandler::InternalHandler::process(const c10::Warning& warning) { + warning_buffer_.push_back(warning); +} + +PyWarningHandler::PyWarningHandler() noexcept(true) + : prev_handler_(c10::WarningUtils::get_warning_handler()) { + c10::WarningUtils::set_warning_handler(&internal_handler_); +} + +// Get the Python warning type for a warning +static PyObject* map_warning_to_python_type(const c10::Warning& warning) { + struct Visitor { + PyObject* operator()(const c10::UserWarning& /*unused*/) const { + return PyExc_UserWarning; + } + PyObject* operator()(const c10::DeprecationWarning& /*unused*/) const { + return PyExc_DeprecationWarning; + } + }; + return std::visit(Visitor(), warning.type()); +} + +/// See NOTE [ Conversion Cpp Python Warning ] for noexcept justification +/// NOLINTNEXTLINE(bugprone-exception-escape) +PyWarningHandler::~PyWarningHandler() noexcept(false) { + c10::WarningUtils::set_warning_handler(prev_handler_); + auto& warning_buffer = internal_handler_.warning_buffer_; + + if (!warning_buffer.empty()) { + PyObject *type = nullptr, *value = nullptr, *traceback = nullptr; + pybind11::gil_scoped_acquire gil; + auto result = 0; + if (in_exception_) { + // This (combined with PyErr_Restore below) also works when no python + // error has been set yet + PyErr_Fetch(&type, &value, &traceback); + } + for (const auto& warning : warning_buffer) { + auto source_location = warning.source_location(); + auto msg = warning.msg(); + processErrorMsgInplace(msg); + if (source_location.file == nullptr) { + result = + PyErr_WarnEx(map_warning_to_python_type(warning), msg.c_str(), 1); + } else if (warning.verbatim()) { + // Sets the source location from the warning + // Note: PyErr_WarnExplicit will disregard Python's warning filter + // and always appear. This is in contrast to PyErr_WarnEx, + // which respects the warning filter. + result = PyErr_WarnExplicit( + /*category=*/map_warning_to_python_type(warning), + /*message=*/msg.c_str(), + /*filename=*/source_location.file, + /*lineno=*/static_cast(source_location.line), + /*module=*/nullptr, + /*registry=*/nullptr); + } else { + // Lets Python set the source location and puts the C++ warning + // location into the message. + auto buf = fmt::format( + "{} (Triggered internally at {}:{}.)", + msg, + source_location.file, + source_location.line); + result = + PyErr_WarnEx(map_warning_to_python_type(warning), buf.c_str(), 1); + } + if (result < 0) { + if (in_exception_) { + // PyErr_Print prints the traceback to sys.stderr and + // clears the error indicator + PyErr_Print(); + } else { + break; + } + } + } + warning_buffer.clear(); + if ((result < 0) && (!in_exception_)) { + /// A warning raised an error, we need to force the parent + /// function to return an error code. + throw python_error(); + } + if (in_exception_) { + PyErr_Restore(type, value, traceback); + } + } +} + +namespace detail { +PyObject* _new_accelerator_error_object(const c10::AcceleratorError& e) { + auto msg = torch::get_cpp_stacktraces_enabled() ? e.what() + : e.what_without_backtrace(); + + auto py_msg = PyUnicode_FromString(msg); + auto rc = PyObject_CallOneArg(THPException_AcceleratorError, py_msg); + auto error_code = THPUtils_packUInt32(e.get_error_code()); + PyObject_SetAttrString(rc, "error_code", error_code); + Py_XDECREF(py_msg); + Py_XDECREF(error_code); + return rc; +} +} // namespace detail +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/Exceptions.h b/western_ai/usa/pytorch/torch/csrc/Exceptions.h new file mode 100644 index 0000000000000000000000000000000000000000..c4fda2005ce767c66efe21b969057bc7281cf654 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/Exceptions.h @@ -0,0 +1,400 @@ +// @allow-raw-throw +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(USE_DISTRIBUTED) +#include +#endif + +inline void PyErr_SetString(PyObject* type, const std::string& message) { + PyErr_SetString(type, message.c_str()); +} +/// NOTE [ Conversion Cpp Python Warning ] +/// The warning handler cannot set python warnings immediately +/// as it requires acquiring the GIL (potential deadlock) +/// and would need to cleanly exit if the warning raised a +/// python error. To solve this, we buffer the warnings and +/// process them when we go back to python. +/// This requires the two try/catch blocks below to handle the +/// following cases: +/// - If there is no Error raised in the inner try/catch, the +/// buffered warnings are processed as python warnings. +/// - If they don't raise an error, the function process with the +/// original return code. +/// - If any of them raise an error, the error is set (PyErr_*) and +/// the destructor will raise a cpp exception python_error() that +/// will be caught by the outer try/catch that will be able to change +/// the return value of the function to reflect the error. +/// - If an Error was raised in the inner try/catch, the inner try/catch +/// must set the python error. The buffered warnings are then +/// processed as cpp warnings as we cannot predict before hand +/// whether a python warning will raise an error or not and we +/// cannot handle two errors at the same time. +/// This advanced handler will only be used in the current thread. +/// If any other thread is used, warnings will be processed as +/// cpp warnings. +#define HANDLE_TH_ERRORS \ + try { \ + torch::PyWarningHandler __enforce_warning_buffer; \ + try { +#define _CATCH_GENERIC_ERROR(ErrorType, PythonErrorType, retstmnt) \ + catch (const c10::ErrorType& e) { \ + auto msg = torch::get_cpp_stacktraces_enabled() \ + ? e.what() \ + : e.what_without_backtrace(); \ + PyErr_SetString(PythonErrorType, torch::processErrorMsg(msg)); \ + retstmnt; \ + } + +// Only catch torch-specific exceptions +#define CATCH_CORE_ERRORS(retstmnt) \ + catch (python_error & e) { \ + e.restore(); \ + retstmnt; \ + } \ + catch (py::error_already_set & e) { \ + e.restore(); \ + retstmnt; \ + } \ + _CATCH_GENERIC_ERROR(IndexError, PyExc_IndexError, retstmnt) \ + _CATCH_GENERIC_ERROR(ValueError, PyExc_ValueError, retstmnt) \ + _CATCH_GENERIC_ERROR(TypeError, PyExc_TypeError, retstmnt) \ + _CATCH_GENERIC_ERROR( \ + NotImplementedError, PyExc_NotImplementedError, retstmnt) \ + _CATCH_GENERIC_ERROR(BufferError, PyExc_BufferError, retstmnt) \ + _CATCH_GENERIC_ERROR(SyntaxError, PyExc_SyntaxError, retstmnt) \ + _CATCH_GENERIC_ERROR(LinAlgError, THPException_LinAlgError, retstmnt) \ + _CATCH_GENERIC_ERROR( \ + OutOfMemoryError, THPException_OutOfMemoryError, retstmnt) \ + _CATCH_GENERIC_ERROR( \ + DistBackendError, THPException_DistBackendError, retstmnt) \ + _CATCH_GENERIC_ERROR( \ + DistNetworkError, THPException_DistNetworkError, retstmnt) \ + _CATCH_GENERIC_ERROR( \ + DistQueueEmptyError, THPException_DistQueueEmptyError, retstmnt) \ + _CATCH_GENERIC_ERROR(DistStoreError, THPException_DistStoreError, retstmnt) \ + _CATCH_GENERIC_ERROR(DistError, THPException_DistError, retstmnt) \ + catch (c10::AcceleratorError & e) { \ + auto exc = torch::detail::_new_accelerator_error_object(e); \ + PyErr_SetObject(THPException_AcceleratorError, exc); \ + Py_XDECREF(exc); \ + retstmnt; \ + } \ + _CATCH_GENERIC_ERROR(Error, PyExc_RuntimeError, retstmnt) \ + catch (torch::PyTorchError & e) { \ + auto msg = torch::processErrorMsg(e.what()); \ + PyErr_SetString(e.python_type(), msg); \ + retstmnt; \ + } + +#define CATCH_TH_ERRORS(retstmnt) CATCH_CORE_ERRORS(retstmnt) + +#define CATCH_ALL_ERRORS(retstmnt) \ + CATCH_TH_ERRORS(retstmnt) \ + catch (const std::exception& e) { \ + auto msg = torch::processErrorMsg(e.what()); \ + PyErr_SetString(PyExc_RuntimeError, msg); \ + retstmnt; \ + } + +#define END_HANDLE_TH_ERRORS_PYBIND \ + } \ + catch (...) { \ + __enforce_warning_buffer.set_in_exception(); \ + throw; \ + } \ + } \ + catch (py::error_already_set&) { \ + throw; \ + } \ + catch (py::builtin_exception&) { \ + throw; \ + } \ + catch (torch::jit::JITException&) { \ + throw; \ + } \ + catch (const std::exception&) { \ + torch::translate_exception_to_python(std::current_exception()); \ + throw py::error_already_set(); \ + } + +#define END_HANDLE_TH_ERRORS_RET(retval) \ + } \ + catch (...) { \ + __enforce_warning_buffer.set_in_exception(); \ + throw; \ + } \ + } \ + catch (const std::exception&) { \ + torch::translate_exception_to_python(std::current_exception()); \ + return retval; \ + } + +#define END_HANDLE_TH_ERRORS END_HANDLE_TH_ERRORS_RET(nullptr) + +extern PyObject *THPException_FatalError, *THPException_LinAlgError, + *THPException_OutOfMemoryError, *THPException_DistError, + *THPException_DistBackendError, *THPException_DistNetworkError, + *THPException_DistStoreError, *THPException_DistQueueEmptyError, + *THPException_AcceleratorError; + +// Throwing this exception means that the python error flags have been already +// set and control should be immediately returned to the interpreter. +struct python_error : public std::exception { + python_error() = default; + + python_error(const python_error& other) + : type(other.type), + value(other.value), + traceback(other.traceback), + message(other.message) { + pybind11::gil_scoped_acquire gil; + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(traceback); + } + + python_error(python_error&& other) noexcept + : type(other.type), + value(other.value), + traceback(other.traceback), + message(std::move(other.message)) { + other.type = nullptr; + other.value = nullptr; + other.traceback = nullptr; + } + + python_error& operator=(const python_error& other) = delete; + python_error& operator=(python_error&& other) = delete; + + // NOLINTNEXTLINE(bugprone-exception-escape) + ~python_error() override { + if (type || value || traceback) { + pybind11::gil_scoped_acquire gil; + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); + } + } + + const char* what() const noexcept override { + return message.c_str(); + } + + void build_message() { + // Ensure we have the GIL. + pybind11::gil_scoped_acquire gil; + + // No errors should be set when we enter the function since PyErr_Fetch + // clears the error indicator. + TORCH_INTERNAL_ASSERT(!PyErr_Occurred()); + + // Default message. + message = "python_error"; + + // Try to retrieve the error message from the value. + if (value != nullptr) { + // Reference count should not be zero. + TORCH_INTERNAL_ASSERT(Py_REFCNT(value) > 0); + + PyObject* pyStr = PyObject_Str(value); + if (pyStr != nullptr) { + PyObject* encodedString = + PyUnicode_AsEncodedString(pyStr, "utf-8", "strict"); + if (encodedString != nullptr) { + char* bytes = PyBytes_AS_STRING(encodedString); + if (bytes != nullptr) { + // Set the message. + message = std::string(bytes); + } + Py_XDECREF(encodedString); + } + Py_XDECREF(pyStr); + } + } + + // Clear any errors since we don't want to propagate errors for functions + // that are trying to build a string for the error message. + PyErr_Clear(); + } + + /** Saves the exception so that it can be re-thrown on a different thread */ + inline void persist() { + if (type) + return; // Don't overwrite exceptions + // PyErr_Fetch overwrites the pointers + pybind11::gil_scoped_acquire gil; + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); + PyErr_Fetch(&type, &value, &traceback); + build_message(); + } + + /** Sets the current Python error from this exception */ + inline void restore() { + if (!type) + return; + // PyErr_Restore steals references + pybind11::gil_scoped_acquire gil; + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(traceback); + PyErr_Restore(type, value, traceback); + } + + PyObject* type{nullptr}; + PyObject* value{nullptr}; + PyObject* traceback{nullptr}; + + // Message to return to the user when 'what()' is invoked. + std::string message; +}; + +bool THPException_init(PyObject* module); + +namespace torch { + +// Set python current exception from a C++ exception +TORCH_PYTHON_API void translate_exception_to_python( + const std::exception_ptr& /*e_ptr*/); + +TORCH_PYTHON_API std::string processErrorMsg(std::string str); + +// Abstract base class for exceptions which translate to specific Python types +struct PyTorchError : public std::exception { + PyTorchError() = default; + PyTorchError(std::string msg_) : msg(std::move(msg_)) {} + virtual PyObject* python_type() = 0; + const char* what() const noexcept override { + return msg.c_str(); + } + std::string msg; +}; + +// Translates to Python TypeError +struct TypeError : public PyTorchError { + TORCH_PYTHON_API TypeError() = default; + TORCH_PYTHON_API TypeError(std::string msg_) + : PyTorchError(std::move(msg_)) {} + using PyTorchError::PyTorchError; + PyObject* python_type() override { + return PyExc_TypeError; + } +}; + +// Translates to Python AttributeError +struct AttributeError : public PyTorchError { + using PyTorchError::PyTorchError; + PyObject* python_type() override { + return PyExc_AttributeError; + } +}; + +// ATen warning handler for Python +struct PyWarningHandler { + // Move actual handler into a separate class with a noexcept + // destructor. Otherwise, we need to force all WarningHandler + // subclasses to have a noexcept(false) destructor. + struct InternalHandler : at::WarningHandler { + ~InternalHandler() override = default; + void process(const c10::Warning& warning) override; + + std::vector warning_buffer_; + }; + + public: + /// See NOTE [ Conversion Cpp Python Warning ] for noexcept justification + TORCH_PYTHON_API PyWarningHandler() noexcept(true); + // NOLINTNEXTLINE(bugprone-exception-escape) + TORCH_PYTHON_API ~PyWarningHandler() noexcept(false); + + /** Call if an exception has been thrown + + * Necessary to determine if it is safe to throw from the destructor since + * std::uncaught_exception is buggy on some platforms and generally + * unreliable across dynamic library calls. + */ + void set_in_exception() { + in_exception_ = true; + } + + private: + InternalHandler internal_handler_; + at::WarningHandler* prev_handler_; + bool in_exception_{false}; +}; + +namespace detail { + +struct noop_gil_scoped_release { + // user-defined constructor (i.e. not defaulted) to avoid + // unused-variable warnings at usage sites of this class + // NOLINTNEXTLINE(modernize-use-equals-default) + noop_gil_scoped_release() {} +}; + +template +using conditional_gil_scoped_release = std::conditional_t< + release_gil, + pybind11::gil_scoped_release, + noop_gil_scoped_release>; + +template +using Arg = typename invoke_traits::template arg::type; + +template +auto wrap_pybind_function_impl_( + Func&& f, + std::index_sequence /*unused*/, + std::bool_constant /*unused*/) { + namespace py = pybind11; + + // f=f is needed to handle function references on older compilers + return [f = std::forward(f)](Arg... args) { + HANDLE_TH_ERRORS + conditional_gil_scoped_release no_gil; + return std::invoke(f, std::forward>(args)...); + END_HANDLE_TH_ERRORS_PYBIND + }; +} + +PyObject* _new_accelerator_error_object(const c10::AcceleratorError& /*e*/); +} // namespace detail + +// Wrap a function with TH error and warning handling. +// Returns a function object suitable for registering with pybind11. +template +auto wrap_pybind_function(Func&& f) { + using traits = invoke_traits; + return torch::detail::wrap_pybind_function_impl_( + std::forward(f), + std::make_index_sequence{}, + std::false_type{}); +} + +// Wrap a function with TH error, warning handling and releases the GIL. +// Returns a function object suitable for registering with pybind11. +template +auto wrap_pybind_function_no_gil(Func&& f) { + using traits = invoke_traits; + return torch::detail::wrap_pybind_function_impl_( + std::forward(f), + std::make_index_sequence{}, + std::true_type{}); +} + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/Export.h b/western_ai/usa/pytorch/torch/csrc/Export.h new file mode 100644 index 0000000000000000000000000000000000000000..b441b88375254ebffd50179e6dfe73f681ca6a80 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/Export.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +#ifdef THP_BUILD_MAIN_LIB +#define TORCH_PYTHON_API C10_EXPORT +#else +#define TORCH_PYTHON_API C10_IMPORT +#endif diff --git a/western_ai/usa/pytorch/torch/csrc/Generator.cpp b/western_ai/usa/pytorch/torch/csrc/Generator.cpp new file mode 100644 index 0000000000000000000000000000000000000000..10ec07da2f3f3831f1fc0b131c0ab64dd73d1834 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/Generator.cpp @@ -0,0 +1,442 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +using namespace at; +using namespace torch; + +PyObject* THPGeneratorClass = nullptr; +static bool generatorMetaclassSet = false; + +PyObject* THPGenerator_initDefaultGenerator(const at::Generator& cdata) { + auto type = reinterpret_cast(THPGeneratorClass); + auto self = THPObjectPtr{type->tp_alloc(type, 0)}; + if (!self) + throw python_error(); // @allow-raw-throw + auto self_ = reinterpret_cast(self.get()); + self_->cdata = cdata; + self_->weakreflist = nullptr; + return self.release(); +} + +static void THPGenerator_dealloc(PyObject* _self) { + auto self = reinterpret_cast(_self); + PyObject_ClearWeakRefs(_self); + if (self->cdata.defined()) { + self->cdata.set_pyobj(nullptr); + self->cdata.~Generator(); + } + Py_TYPE(_self)->tp_free(_self); +} + +static PyObject* THPGenerator_pynew( + PyTypeObject* type, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + TORCH_CHECK( + generatorMetaclassSet, + "torch.Generator() cannot be called before the opaque metaclass is set. " + "This is a bug — torch._prims.rng_prims must be imported first."); + static torch::PythonArgParser parser({"Generator(Device device=None)"}); + torch::ParsedArgs<1> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + auto device = r.deviceWithDefault(0, at::Device(at::kCPU)); + + THPGeneratorPtr self( + reinterpret_cast(type->tp_alloc(type, 0))); + + c10::DeviceType device_type = device.type(); + if (device_type == at::kCPU) { + self->cdata = make_generator(); + } else { + self->cdata = globalContext() + .getAcceleratorHooksInterface(device_type) + .getNewGenerator(device.index()); + } + + return reinterpret_cast(self.release()); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_getState(PyObject* _self, PyObject* noargs) { + using namespace torch::autograd; + HANDLE_TH_ERRORS + auto& gen = (reinterpret_cast(_self))->cdata; + + // See Note [Acquire lock when using random generators] + std::scoped_lock lock(gen.mutex()); + auto state_tensor = gen.get_state(); + + return THPVariable_Wrap(state_tensor); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_setState(PyObject* _self, PyObject* _new_state) { + using namespace torch::autograd; + + HANDLE_TH_ERRORS + if (!THPVariable_Check(_new_state)) { + TORCH_CHECK_TYPE( + false, + fmt::format( + "expected a torch.ByteTensor, but got {}", + Py_TYPE(_new_state)->tp_name)); + } + auto self = reinterpret_cast(_self); + auto& gen = self->cdata; + const auto& new_state_tensor = THPVariable_Unpack(_new_state); + + // See Note [Acquire lock when using random generators] + std::scoped_lock lock(gen.mutex()); + gen.set_state(new_state_tensor); + + return Py_NewRef(self); + END_HANDLE_TH_ERRORS +} + +static uint64_t unpack_uint64(PyObject* pyobj) { + uint64_t unsigned_obj = 0; + try { + // First try to interpret as unsigned long + unsigned_obj = THPUtils_unpackUInt64(pyobj); + } catch (...) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + // If an overflow happened, then the pyobj could be negative, + // so try to interpret it as signed long + PyErr_Clear(); + int64_t obj = THPUtils_unpackLong(pyobj); + unsigned_obj = *(reinterpret_cast(&obj)); + } else { + // If any other type of exception happened, rethrow it + throw; // @allow-raw-throw + } + } + return unsigned_obj; +} + +static PyObject* THPGenerator_graphSafeGetState( + PyObject* _self, + PyObject* noargs) { + HANDLE_TH_ERRORS + auto& gen = (reinterpret_cast(_self))->cdata; + + // See Note [Acquire lock when using random generators] + std::scoped_lock lock(gen.mutex()); + + return THPGenerator_Wrap(gen.graphsafe_get_state()); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_graphSafeSetState( + PyObject* _self, + PyObject* _state) { + HANDLE_TH_ERRORS + auto self = reinterpret_cast(_self); + auto& gen = self->cdata; + + // See Note [Acquire lock when using random generators] + std::scoped_lock lock(gen.mutex()); + gen.graphsafe_set_state(THPGenerator_Unwrap(_state)); + + return Py_NewRef(self); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_cloneState(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS + auto& gen = (reinterpret_cast(_self))->cdata; + + // See Note [Acquire lock when using random generators] + std::scoped_lock lock(gen.mutex()); + + return THPGenerator_Wrap(gen.clone()); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_manualSeed(PyObject* _self, PyObject* seed) { + HANDLE_TH_ERRORS + auto self = reinterpret_cast(_self); + auto generator = self->cdata; + TORCH_CHECK( + THPUtils_checkLong(seed), + "manual_seed expected a long, " + "but got ", + THPUtils_typename(seed)); + uint64_t unsigned_seed = unpack_uint64(seed); + // See Note [Acquire lock when using random generators] + std::scoped_lock lock(generator.mutex()); + generator.set_current_seed(unsigned_seed); + return Py_NewRef(self); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_setOffset(PyObject* _self, PyObject* offset) { + HANDLE_TH_ERRORS + auto self = reinterpret_cast(_self); + auto generator = self->cdata; + TORCH_CHECK( + THPUtils_checkLong(offset), + "manual_offset expected a long, " + "but got ", + THPUtils_typename(offset)); + uint64_t unsigned_offset = unpack_uint64(offset); + // See Note [Acquire lock when using random generators] + std::scoped_lock lock(generator.mutex()); + generator.set_offset(unsigned_offset); + return Py_NewRef(self); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_seed(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS + // See Note [Acquire lock when using random generators] + auto self = reinterpret_cast(_self); + std::scoped_lock lock(self->cdata.mutex()); + uint64_t seed_val = self->cdata.seed(); + return THPUtils_packUInt64(seed_val); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_initialSeed(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS + auto self = reinterpret_cast(_self); + return THPUtils_packUInt64(self->cdata.current_seed()); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_getOffset(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS + auto self = reinterpret_cast(_self); + return THPUtils_packUInt64(self->cdata.get_offset()); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_get_device(THPGenerator* self, void* unused) { + HANDLE_TH_ERRORS + return THPDevice_New(self->cdata.device()); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_reduce(PyObject* _self, PyObject* noargs) { + HANDLE_TH_ERRORS + auto self = reinterpret_cast(_self); + auto& gen = self->cdata; + + auto ret = THPObjectPtr{PyTuple_New(3)}; + if (!ret) + throw python_error(); // @allow-raw-throw + + py::object torch_module = py::module::import("torch"); + py::object torch_generator = torch_module.attr("Generator"); + PyTuple_SET_ITEM(ret.get(), 0, torch_generator.release().ptr()); + + auto args = THPObjectPtr{PyTuple_New(1)}; + if (!args) + throw python_error(); // @allow-raw-throw + + PyTuple_SET_ITEM(args.get(), 0, THPGenerator_get_device(self, nullptr)); + PyTuple_SET_ITEM(ret.get(), 1, args.release()); + + auto state = THPObjectPtr{PyTuple_New(3)}; + if (!state) + throw python_error(); // @allow-raw-throw + + c10::DeviceType device_type = gen.device().type(); + PyTuple_SET_ITEM(state.get(), 0, THPGenerator_initialSeed(_self, nullptr)); + PyTuple_SET_ITEM( + state.get(), + 1, + device_type != at::kCPU ? THPGenerator_getOffset(_self, nullptr) + : Py_NewRef(Py_None)); + PyTuple_SET_ITEM(state.get(), 2, THPGenerator_getState(_self, nullptr)); + PyTuple_SET_ITEM(ret.get(), 2, state.release()); + + return ret.release(); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPGenerator_pickleSetState(PyObject* _self, PyObject* state) { + HANDLE_TH_ERRORS + THPGenerator_manualSeed(_self, PyTuple_GET_ITEM(state, 0)); + auto& offset = PyTuple_GET_ITEM(state, 1); + if (!Py_IsNone(offset)) { + THPGenerator_setOffset(_self, offset); + } + THPGenerator_setState(_self, PyTuple_GET_ITEM(state, 2)); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-avoid-non-const-global-variables) +static struct PyGetSetDef THPGenerator_properties[] = { + {"device", + reinterpret_cast(THPGenerator_get_device), + nullptr, + nullptr, + nullptr}, + {nullptr}}; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-avoid-non-const-global-variables) +static PyMethodDef THPGenerator_methods[] = { + {"__reduce__", THPGenerator_reduce, METH_NOARGS, nullptr}, + {"__setstate__", THPGenerator_pickleSetState, METH_O, nullptr}, + {"get_state", THPGenerator_getState, METH_NOARGS, nullptr}, + {"set_state", THPGenerator_setState, METH_O, nullptr}, + {"clone_state", THPGenerator_cloneState, METH_NOARGS, nullptr}, + {"graphsafe_get_state", + THPGenerator_graphSafeGetState, + METH_NOARGS, + nullptr}, + {"graphsafe_set_state", THPGenerator_graphSafeSetState, METH_O, nullptr}, + {"set_offset", THPGenerator_setOffset, METH_O, nullptr}, + {"manual_seed", THPGenerator_manualSeed, METH_O, nullptr}, + {"seed", THPGenerator_seed, METH_NOARGS, nullptr}, + {"initial_seed", THPGenerator_initialSeed, METH_NOARGS, nullptr}, + {"get_offset", THPGenerator_getOffset, METH_NOARGS, nullptr}, + {nullptr}}; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-avoid-non-const-global-variables) +static struct PyMemberDef THPGenerator_members[] = { + {"_cdata", T_ULONGLONG, offsetof(THPGenerator, cdata), READONLY, nullptr}, + {nullptr}}; + +static PyTypeObject THPGeneratorType = { + PyVarObject_HEAD_INIT(nullptr, 0) + "torch._C.Generator", /* tp_name */ + sizeof(THPGenerator), /* tp_basicsize */ + 0, /* tp_itemsize */ + THPGenerator_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + nullptr, /* tp_getattr */ + nullptr, /* tp_setattr */ + nullptr, /* tp_reserved */ + nullptr, /* tp_repr */ + nullptr, /* tp_as_number */ + nullptr, /* tp_as_sequence */ + nullptr, /* tp_as_mapping */ + nullptr, /* tp_hash */ + nullptr, /* tp_call */ + nullptr, /* tp_str */ + nullptr, /* tp_getattro */ + nullptr, /* tp_setattro */ + nullptr, /* tp_as_buffer */ + // NOLINTNEXTLINE(misc-redundant-expression) + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + nullptr, /* tp_doc */ + nullptr, /* tp_traverse */ + nullptr, /* tp_clear */ + nullptr, /* tp_richcompare */ + offsetof(THPGenerator, weakreflist), /* tp_weaklistoffset */ + nullptr, /* tp_iter */ + nullptr, /* tp_iternext */ + THPGenerator_methods, /* tp_methods */ + THPGenerator_members, /* tp_members */ + THPGenerator_properties, /* tp_getset */ + nullptr, /* tp_base */ + nullptr, /* tp_dict */ + nullptr, /* tp_descr_get */ + nullptr, /* tp_descr_set */ + 0, /* tp_dictoffset */ + nullptr, /* tp_init */ + nullptr, /* tp_alloc */ + THPGenerator_pynew, /* tp_new */ +}; + +static PyObject* THPGenerator_pySetMetaclass( + PyObject* /* unused */, + PyObject* metaclass) { + if (generatorMetaclassSet) { + Py_RETURN_NONE; + } + if (!PyType_Check(metaclass)) { + PyErr_SetString(PyExc_TypeError, "metaclass must be a type"); + return nullptr; + } + Py_INCREF(metaclass); + Py_SET_TYPE(&THPGeneratorType, (PyTypeObject*)metaclass); + generatorMetaclassSet = true; + // Swapping the metaclass changes __module__ (Python resolves it via the + // metaclass). Reset it so pickle can find the class at torch._C.Generator. + auto module_name = THPObjectPtr(PyUnicode_FromString("torch._C")); + if (!module_name) + return nullptr; + if (PyDict_SetItemString( + THPGeneratorType.tp_dict, "__module__", module_name) < 0) + return nullptr; + Py_RETURN_NONE; +} + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) +static PyMethodDef THPGenerator_moduleMethods[] = { + {"_set_generator_metaclass", THPGenerator_pySetMetaclass, METH_O, nullptr}, + {nullptr, nullptr, 0, nullptr}}; + +bool THPGenerator_init(PyObject* module) { + THPGeneratorClass = reinterpret_cast(&THPGeneratorType); + if (PyModule_AddType(module, &THPGeneratorType) < 0) + return false; + // Register _set_generator_metaclass on torch._C so Python code can + // late-bind CustomClassBaseMeta as Generator's metaclass (see rng_prims.py). + if (PyModule_AddFunctions(module, THPGenerator_moduleMethods) < 0) + return false; + return true; +} + +static void set_pyobj(const Generator& self, PyObject* pyobj) { + TORCH_CHECK(self.defined(), "cannot call set_pyobj() on undefined generator"); + self.set_pyobj(pyobj); +} + +static PyObject* pyobj(const Generator& self) { + TORCH_CHECK(self.defined(), "cannot call pyobj() on undefined generator"); + return self.pyobj(); +} + +PyObject* THPGenerator_Wrap(const Generator& gen) { + if (!gen.defined()) { + Py_RETURN_NONE; + } + + if (auto obj = pyobj(gen)) { + return Py_NewRef(obj); + } + + return THPGenerator_NewWithVar( + reinterpret_cast(THPGeneratorClass), gen); +} + +at::Generator THPGenerator_Unwrap(PyObject* state) { + if (!Py_IS_TYPE(state, &THPGeneratorType)) { + TORCH_CHECK_TYPE( + false, + fmt::format( + "expected a Generator, but got {}", Py_TYPE(state)->tp_name)); + } + return reinterpret_cast(state)->cdata; +} + +// Creates a new Python object for a Generator. The Generator must not already +// have a PyObject* associated with it. +PyObject* THPGenerator_NewWithVar(PyTypeObject* type, Generator gen) { + PyObject* obj = type->tp_alloc(type, 0); + if (obj) { + auto g = reinterpret_cast(obj); + new (&g->cdata) Generator(std::move(gen)); + set_pyobj(g->cdata, obj); + } + return obj; +} diff --git a/western_ai/usa/pytorch/torch/csrc/Generator.h b/western_ai/usa/pytorch/torch/csrc/Generator.h new file mode 100644 index 0000000000000000000000000000000000000000..0e2bffa69a11c32750344b195af38d50372168a3 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/Generator.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) +struct THPGenerator { + PyObject_HEAD + at::Generator cdata; + PyObject* weakreflist; +}; + +// Creates a new Python object wrapping the default at::Generator. The reference +// is borrowed. The caller should ensure that the at::Generator object lifetime +// last at least as long as the Python wrapper. +TORCH_PYTHON_API PyObject* THPGenerator_initDefaultGenerator( + const at::Generator& cdata); + +#define THPGenerator_Check(obj) PyObject_IsInstance(obj, THPGeneratorClass) + +TORCH_PYTHON_API extern PyObject* THPGeneratorClass; + +bool THPGenerator_init(PyObject* module); + +TORCH_PYTHON_API PyObject* THPGenerator_Wrap(const at::Generator& gen); + +TORCH_PYTHON_API at::Generator THPGenerator_Unwrap(PyObject* state); + +// Creates a new Python object for a Generator. The Generator must not already +// have a PyObject* associated with it. +PyObject* THPGenerator_NewWithVar(PyTypeObject* type, at::Generator gen); diff --git a/western_ai/usa/pytorch/torch/csrc/acc/Module.cpp b/western_ai/usa/pytorch/torch/csrc/acc/Module.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6b90eae320325a7f01f2a0fa19dec03a23c7b871 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/acc/Module.cpp @@ -0,0 +1,197 @@ +#include + +#include +#include + +#include +#include +#include +#include + +namespace py = pybind11; + +namespace torch::acc { + +// python hook interface +struct PythonHooks final : public at::PrivateUse1HooksInterface { + using at::PrivateUse1HooksInterface::PrivateUse1HooksInterface; + bool hasPrimaryContext(c10::DeviceIndex device_index) const override { + PYBIND11_OVERRIDE_PURE_NAME( + bool, + at::PrivateUse1HooksInterface, + "has_primary_context", + hasPrimaryContext, + device_index); + } + + bool isBuilt() const override { + PYBIND11_OVERRIDE_PURE_NAME( + bool, at::PrivateUse1HooksInterface, "is_built", isBuilt, ); + } + + bool isAvailable() const override { + PYBIND11_OVERRIDE_PURE_NAME( + bool, at::PrivateUse1HooksInterface, "is_available", isBuilt, ); + } + + // TODO(qihqi): these is not supported from python yet + const at::Generator& getDefaultGenerator( + c10::DeviceIndex device_index) const override { + return at::PrivateUse1HooksInterface::getDefaultGenerator(device_index); + } + + at::Generator getNewGenerator( + c10::DeviceIndex device_index = -1) const override { + return at::PrivateUse1HooksInterface::getNewGenerator(device_index); + } + + at::Device getDeviceFromPtr(void* data) const override { + return at::PrivateUse1HooksInterface::getDeviceFromPtr(data); + } + + bool isPinnedPtr(const void* data) const override { + return at::PrivateUse1HooksInterface::isPinnedPtr(data); + } + + at::Allocator* getPinnedMemoryAllocator() const override { + return at::PrivateUse1HooksInterface::getPinnedMemoryAllocator(); + } +}; + +struct PythonDeviceGuard final : public c10::impl::DeviceGuardImplInterface { + using c10::impl::DeviceGuardImplInterface::DeviceGuardImplInterface; + + c10::DeviceType type() const override { + PYBIND11_OVERRIDE_PURE_NAME( + c10::DeviceType, c10::impl::DeviceGuardImplInterface, "type_", type, ); + } + + // TODO(qihqi): figure out if those are even useful + // to python or not + c10::Device exchangeDevice(c10::Device device) const override { + return getDevice(); + } + c10::Device getDevice() const override { + return c10::Device(type(), 0); + } + void setDevice(c10::Device device) const override {} + void uncheckedSetDevice(c10::Device device) const noexcept override {} + c10::Stream getStream(c10::Device /*unused*/) const noexcept override { + // no-op + return c10::Stream(c10::Stream::DEFAULT, getDevice()); + } + + c10::Stream getNewStream(c10::Device /*unused*/, int priority = 0) + const override { + // no-op + (void)priority; + return c10::Stream(c10::Stream::DEFAULT, getDevice()); + } + + c10::Stream exchangeStream(c10::Stream /*unused*/) const noexcept override { + // no-op + return c10::Stream(c10::Stream::DEFAULT, getDevice()); + } + c10::DeviceIndex deviceCount() const noexcept override { + return 1; + } + + // TODO(qihqi): support Event-related functions + void record( + void** /*event*/, + const c10::Stream& /*stream*/, + const c10::DeviceIndex /*device_index*/, + const c10::EventFlag /*flag*/) const override {} + void block(void* /*event*/, const c10::Stream& /*stream*/) const override {} + bool queryEvent(void* /*event*/) const override { + return true; + } + void destroyEvent(void* /*event*/, const c10::DeviceIndex /*device_index*/) + const noexcept override {} + + // Stream-related functions + bool queryStream(const c10::Stream& /*stream*/) const override { + return true; + } + void synchronizeStream(const c10::Stream& /*stream*/) const override {} +}; + +namespace { + +bool registerPythonPrivateUse1Hook(const py::object& hook) { + if (at::isPrivateUse1HooksRegistered()) { + return false; + } + hook.inc_ref(); + at::RegisterPrivateUse1HooksInterface( + hook.cast()); + return true; +} + +bool registerPythonPrivateUse1DeviceGuard(const py::object& guard) { + if (c10::impl::hasDeviceGuardImpl(c10::DeviceType::PrivateUse1)) { + return false; + } + guard.inc_ref(); + c10::impl::registerDeviceGuard( + c10::DeviceType::PrivateUse1, + guard.cast()); + return true; +} + +at::Tensor createEmptyTensor( + const std::vector& shape, + c10::ScalarType dtype) { + c10::Storage storage{ + c10::Storage::use_byte_size_t{}, + 0, + c10::GetAllocator(c10::kMeta), + true, + }; + + c10::Device device(c10::DeviceType::PrivateUse1, 0); + storage.set_data_ptr_noswap(at::DataPtr{nullptr, device}); + c10::DispatchKeySet key_set({c10::DispatchKey::PrivateUse1}); + at::Tensor tensor = at::detail::make_tensor( + std::move(storage), key_set, c10::scalarTypeToTypeMeta(dtype)); + + std::vector strides(shape.size()); + int64_t size = 1; + for (auto i = strides.size(); i > 0; --i) { + strides[i - 1] = size; + size *= shape[i - 1]; + } + + tensor.unsafeGetTensorImpl()->set_sizes_and_strides(shape, strides, 0); + return tensor; +} +} // namespace + +void initModule(PyObject* module) { + auto py_module = py::reinterpret_borrow(module); + auto _acc = + py_module.def_submodule("_acc", "classes related to custom accelerators"); + + py::class_( + _acc.ptr(), "PrivateUse1Hooks") + .def(py::init<>()) + .def( + "has_primary_context", + &at::PrivateUse1HooksInterface::hasPrimaryContext) + .def("is_built", &at::PrivateUse1HooksInterface::isBuilt) + .def("is_available", &at::PrivateUse1HooksInterface::isAvailable); + + py::class_( + _acc.ptr(), "DeviceGuard") + .def(py::init<>()) + .def("type_", &c10::impl::DeviceGuardImplInterface::type); + + _acc.def( + "register_python_privateuseone_hook", ®isterPythonPrivateUse1Hook); + _acc.def( + "register_python_privateuseone_device_guard", + ®isterPythonPrivateUse1DeviceGuard); + _acc.def("create_empty_tensor", &createEmptyTensor); +} + +} // namespace torch::acc diff --git a/western_ai/usa/pytorch/torch/csrc/acc/Module.h b/western_ai/usa/pytorch/torch/csrc/acc/Module.h new file mode 100644 index 0000000000000000000000000000000000000000..6465d824c285fbc4b0e290c4600605193ec9a9d2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/acc/Module.h @@ -0,0 +1,8 @@ +#pragma once +#include + +namespace torch::acc { +// PyMethodDef* python_functions(); +void initModule(PyObject* module); + +} // namespace torch::acc diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/all.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/all.h new file mode 100644 index 0000000000000000000000000000000000000000..519875ea6ed14c0d07ffdceb60de5369cd036d99 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/all.h @@ -0,0 +1,24 @@ +#pragma once + +#if !defined(_MSC_VER) && __cplusplus < 201703L +#error C++17 or later compatible compiler is required to use PyTorch. +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/arg.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/arg.h new file mode 100644 index 0000000000000000000000000000000000000000..70c07e30aece4294c58c53a80b241a33b9c6425f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/arg.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +#define TORCH_ARG(T, name) \ + public: \ + inline auto name(const T& new_##name) -> decltype(*this) { /* NOLINT */ \ + this->name##_ = new_##name; \ + return *this; \ + } \ + inline auto name(T&& new_##name) -> decltype(*this) { /* NOLINT */ \ + this->name##_ = std::move(new_##name); \ + return *this; \ + } \ + inline const T& name() const noexcept { /* NOLINT */ \ + return this->name##_; \ + } \ + inline T& name() noexcept { /* NOLINT */ \ + return this->name##_; \ + } \ + \ + private: \ + T name##_ /* NOLINT */ diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/autograd.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/autograd.h new file mode 100644 index 0000000000000000000000000000000000000000..4f6ae577c0b7ebcce2c56c177d590dd4f5426919 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/autograd.h @@ -0,0 +1,5 @@ +#pragma once + +#include +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/cuda.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/cuda.h new file mode 100644 index 0000000000000000000000000000000000000000..f860660d13f0c5aeaafd08d1e6b05e3d8de67d40 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/cuda.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +#include + +namespace torch::cuda { + +/// Returns the number of CUDA devices available. +c10::DeviceIndex TORCH_API device_count(); + +/// Returns true if at least one CUDA device is available. +bool TORCH_API is_available(); + +/// Returns true if CUDA is available, and CuDNN is available. +bool TORCH_API cudnn_is_available(); + +/// Sets the seed for the current GPU. +void TORCH_API manual_seed(uint64_t seed); + +/// Sets the seed for all available GPUs. +void TORCH_API manual_seed_all(uint64_t seed); + +/// Waits for all kernels in all streams on a CUDA device to complete. +void TORCH_API synchronize(int64_t device_index = -1); + +} // namespace torch::cuda diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data.h new file mode 100644 index 0000000000000000000000000000000000000000..e9c1198a7b086647022d961a07b4a9e4f1110c77 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data.h @@ -0,0 +1,13 @@ +#pragma once + +#include +#include +#include +#include + +// Some "exports". + +namespace torch::data { +using datasets::BatchDataset; // NOLINT +using datasets::Dataset; // NOLINT +} // namespace torch::data diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader.h new file mode 100644 index 0000000000000000000000000000000000000000..16aea86ec73bf7914597c9d3158204c695eac37a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include + +#include + +#include + +#include +#include +#include +#include + +namespace torch::data { + +/// Creates a `DataLoader` instance for a stateless `dataset`, a `sampler` and +/// some `options`. +template +std::enable_if_t< + !Dataset::is_stateful, + std::unique_ptr>> +make_data_loader(Dataset dataset, Sampler sampler, DataLoaderOptions options) { + return std::make_unique>( + std::move(dataset), std::move(sampler), options); +} + +/// Creates a `DataLoader` instance for a stateless `dataset` and some +/// `options`. A sampler (by default a `RandomSampler`) will be constructed from +/// the size of the dataset. +template +std::enable_if_t< + !Dataset::is_stateful && std::is_constructible_v, + std::unique_ptr>> +make_data_loader( + Dataset dataset, + DataLoaderOptions options = DataLoaderOptions()) { + const std::optional size = dataset.size(); + TORCH_CHECK( + size.has_value(), + "Expected the dataset to be sized in " + "order to construct the Sampler"); + return make_data_loader(std::move(dataset), Sampler(*size), options); +} + +/// Creates a `DataLoader` for a stateful `dataset` and some `options`. +template > +std::unique_ptr> make_data_loader( + Dataset dataset, + DataLoaderOptions options = DataLoaderOptions()) { + return std::make_unique>( + std::move(dataset), options); +} +} // namespace torch::data diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader/base.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader/base.h new file mode 100644 index 0000000000000000000000000000000000000000..0e67df98f071dde76e1cdd3ae65023e079d31270 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader/base.h @@ -0,0 +1,254 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace torch::data { +template +class DataLoaderBase { + public: + using BatchType = Batch; + using BatchRequestType = BatchRequest; + + /// Constructs a new DataLoader from a `dataset` to sample from, `options` + /// to configure the DataLoader with, and a `sampler` that specifies the + /// sampling strategy. + DataLoaderBase( + DataLoaderOptions options, + std::unique_ptr main_thread_dataset = nullptr) + : options_(options), + main_thread_dataset_(std::move(main_thread_dataset)), + sequencer_(new_sequencer()) {} + + DataLoaderBase(const DataLoaderBase&) = delete; + DataLoaderBase(DataLoaderBase&&) = delete; + DataLoaderBase& operator=(const DataLoaderBase&) = delete; + DataLoaderBase& operator=(DataLoaderBase&&) = delete; + // NOLINTNEXTLINE(bugprone-exception-escape) + virtual ~DataLoaderBase() { + join(); + } + + /// Returns an iterator into the DataLoader. The lifetime of the iterator is + /// bound to the DataLoader. In C++ standards language, the category of the + /// iterator is `OutputIterator`. See + /// https://en.cppreference.com/w/cpp/named_req/OutputIterator for what this + /// means. In short: you may increment the iterator and dereference it, but + /// cannot go back, or step forward more than one position at a time. When the + /// DataLoader is exhausted, it will compare equal with the special + /// "sentinel" iterator returned by `DataLoader::end()`. Most of the time, you + /// should only use range-for loops to loop over the DataLoader, but + /// standard algorithms like `std::copy(dataloader.begin(), dataloader.end(), + /// output_iterator)` are supported too. + Iterator begin() { + TORCH_CHECK( + shuttle_.in_flight_jobs() == 0, + "Attempted to get a new DataLoader iterator " + "while another iterator is not yet exhausted"); + reset(); + return Iterator(std::make_unique>( + [this] { return this->next(); })); + } + + /// Returns a special "sentinel" iterator that compares equal with a + /// non-sentinel iterator once the DataLoader is exhausted. + Iterator end() { + return Iterator(std::make_unique>()); + } + + /// Joins the DataLoader's worker threads and drains internal queues. + /// This function may only be invoked from the main thread (in which the + /// DataLoader lives). + void join() { + if (joined_) { + return; + } + shuttle_.drain(); + // Send one 'quit' message per worker. Since a worker dies (exits its + // thread) after receiving this message, each `QuitWorker()` message will be + // read by exactly one worker. + for ([[maybe_unused]] const auto w : c10::irange(options_.workers)) { + push_job(QuitWorker()); + } + for (auto& worker : workers_) { + worker.join(); + } + joined_ = true; + } + + /// Returns the options with which the DataLoader was configured. + const FullDataLoaderOptions& options() const noexcept { + return options_; + } + + protected: + /// Simple mix-in to give something a sequence number. + struct Sequenced { + Sequenced() = default; + Sequenced(size_t sqn) : sequence_number(sqn) {} + size_t sequence_number; + }; + + struct QuitWorker {}; + + /// A `Job` is either a `BatchRequest` (new indices to fetch data at) or a + /// `QuitWorker` object, to indicate the worker should shut down. + struct Job : Sequenced { + Job() = default; + Job(QuitWorker q, size_t sqn) : Sequenced(sqn), quit(q) {} + Job(BatchRequest&& i, size_t sqn) + : Sequenced(sqn), batch_request(std::move(i)) {} + std::optional quit; + std::optional batch_request; + }; + + /// The finished result of a job. + struct Result : Sequenced { + Result() = default; + Result(std::optional&& b, size_t sqn) + : Sequenced(sqn), batch(std::move(b)) {} + Result(std::exception_ptr exception, size_t sqn) + : Sequenced(sqn), exception(std::move(exception)) {} + std::optional batch; + std::exception_ptr exception; + }; + + /// Subclass hook for getting the next batch request. The stateless case will + /// ask the sampler for a new batch request (e.g. a vector of indices), while + /// the stateful one will simply return the batch size. + virtual std::optional get_batch_request() = 0; + + /// Resets the internal state of the DataLoader, optionally pre-fetching + /// new jobs. + virtual void reset() { + shuttle_.drain(); + sequence_number_ = 0; + sequencer_ = new_sequencer(); + prefetch(); + } + + /// Schedules `requested_jobs` many new batches to be fetched. The actual + /// number of jobs scheduled may be less if the DataLoader exhausts. + void prefetch(size_t requested_jobs) { + for ([[maybe_unused]] const auto r : c10::irange(requested_jobs)) { + if (auto batch_request = get_batch_request()) { + this->push_job(std::move(*batch_request)); + } else { + break; + } + } + } + + /// Schedules the maximum number of jobs (based on the `max_jobs` option). + void prefetch() { + prefetch(options_.max_jobs); + } + + /// Returns the next batch of data, or an empty `optional` if the DataLoader + /// is exhausted. This operation will block until a batch is available if one + /// is still expected. + std::optional next() { + if (options_.workers > 0) { + while (std::optional result = this->pop_result()) { + if (result->exception) { + throw WorkerException(result->exception); + } else if (result->batch) { + prefetch(1); + return std::move(result->batch); + } + } + } else if (auto batch_request = get_batch_request()) { + return this->main_thread_dataset_->get_batch(std::move(*batch_request)); + } + return std::nullopt; + } + + /// The function that worker threads run. + void worker_thread(Dataset& dataset) { + while (true) { + auto job = shuttle_.pop_job(); + if (job.quit) { + break; + } + try { + auto batch = dataset.get_batch(std::move(*job.batch_request)); + shuttle_.push_result({std::move(batch), job.sequence_number}); + } catch (...) { + shuttle_.push_result({std::current_exception(), job.sequence_number}); + } + } + } + + /// Convenience method that calls `shuttle_.push_job()` with the next sequence + /// number. + template + void push_job(T value) { + shuttle_.push_job({std::move(value), sequence_number_++}); + } + + /// Convenience method that gets the next result from the sequencer. + std::optional pop_result() { + return sequencer_->next( + [this] { return this->shuttle_.pop_result(this->options_.timeout); }); + } + + /// Convenience method that creates a new sequencer based on the + /// `enforce_ordering` option. + std::unique_ptr> new_sequencer() { + if (options_.enforce_ordering) { + return std::make_unique>( + options_.max_jobs); + } + return std::make_unique>(); + } + + /// The options the DataLoader was configured with. + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const FullDataLoaderOptions options_; + + /// The dataset for the main thread, only has a value if the number of + /// worker threads was configured as zero, meaning the main thread has to do + /// all the work (synchronously). NOTE: Really want this to be on the heap + /// when empty, therefore `unique_ptr` and not `optional`. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::unique_ptr main_thread_dataset_; + + /// The sequence number for the *next* batch to be retrieved from the + /// dataset. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + size_t sequence_number_ = 0; + + /// The worker threads, running the `worker_thread()` method. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::vector workers_; + + /// The `DataShuttle` which takes care of the life cycle of a job. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + detail::DataShuttle shuttle_; + + /// The `Sequencer`, which handles optional ordering of batches. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::unique_ptr> sequencer_; + + /// True if the DataLoader has joined its worker threads. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + bool joined_ = false; +}; +} // namespace torch::data diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader/stateful.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader/stateful.h new file mode 100644 index 0000000000000000000000000000000000000000..e525a1c09f90f323cc9ffdf0f3fb2bade98ae761 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader/stateful.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace torch::data { + +/// A dataloader for stateful datasets. +/// +/// A dataloader for stateful datasets differs from one for stateless +/// datasets one in that the dataset is shared among worker threads, and that +/// this dataset is itself responsible for producing batches rather than +/// depending on a sampler. The statefulness here actually refers to the +/// dataset. The StatefulDataLoader simply alters the data loading algorithm to +/// accommodate the stateful, shared nature of the dataset. Note that the +/// dataset must be thread safe if more than one worker thread is used. +/// +/// A stateful dataloader is created by calling `make_data_loader` with a +/// stateful dataset. +template +class StatefulDataLoader : public DataLoaderBase< + Dataset, + typename Dataset::BatchType::value_type, + typename Dataset::BatchRequestType> { + public: + using super = DataLoaderBase< + Dataset, + typename Dataset::BatchType::value_type, + typename Dataset::BatchRequestType>; + using typename super::BatchRequestType; + + /// Constructs the `StatefulDataLoader` from a `dataset` and some `options`. + StatefulDataLoader(Dataset dataset, DataLoaderOptions options) + : super(options, std::make_unique(std::move(dataset))) { + for ([[maybe_unused]] const auto _ : c10::irange(this->options_.workers)) { + // As opposed to the stateless case, here all worker threads access the + // same underlying dataset. + this->workers_.emplace_back( + [this] { this->worker_thread(*this->main_thread_dataset_); }); + } + } + + private: + /// Resets the internal state of the dataloader and the dataset. + void reset() override { + this->main_thread_dataset_->reset(); + // Call the base class method last because it calls `prefetch()` + super::reset(); + } + + /// For stateful datasets, the batch request is always the batch size. The + /// dataset is responsible for determining what goes into the batch next. + std::optional get_batch_request() override { + return this->options_.batch_size; + } +}; +} // namespace torch::data diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader/stateless.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader/stateless.h new file mode 100644 index 0000000000000000000000000000000000000000..1ccc9157087362af08b1feeb54b17c62a066cfb8 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader/stateless.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include + +#include +#include + +#include +#include +#include + +namespace torch::data { + +/// A dataloader for stateless datasets. +/// +/// This dataloader follows the traditional PyTorch dataloader design, whereby a +/// (possibly) stateful sampler produces *batch requests* for a stateless +/// dataset, which acts as a simple batch request to batch mapping. The batch +/// request will often be an array of indices, and if the dataset is a simple +/// image dataset, the dataset would produce the images at those indices. +template +class StatelessDataLoader : public DataLoaderBase< + Dataset, + typename Dataset::BatchType, + typename Sampler::BatchRequestType> { + public: + using super = DataLoaderBase< + Dataset, + typename Dataset::BatchType, + typename Sampler::BatchRequestType>; + using typename super::BatchRequestType; + + /// Constructs the `StatelessDataLoader` from a `dataset`, a `sampler` and + /// some `options`. + StatelessDataLoader( + Dataset dataset, + Sampler sampler, + DataLoaderOptions options) + : super(options), sampler_(std::move(sampler)) { + for (const auto w : c10::irange(this->options_.workers)) { + // Here we copy the dataset into the worker thread closure. Each worker + // has its own copy of the dataset. This means the dataset must be + // trivially copiable, or else we don't expect more than one worker to + // be in use. + (void)w; // Suppress unused variable warning + this->workers_.emplace_back( + [this, dataset]() mutable { this->worker_thread(dataset); }); + } + if (this->options_.workers == 0) { + this->main_thread_dataset_ = + std::make_unique(std::move(dataset)); + } + } + + private: + /// Resets the internal state of the dataloader and the sampler. + void reset() override { + sampler_.reset(); + // Call the base class method last because it calls `prefetch()` + super::reset(); + } + + /// Queries the sampler for the next batch request (possibly progressing its + /// internal state). + std::optional get_batch_request() override { + auto indices = sampler_.next(this->options_.batch_size); + if (!indices || + (indices->size() < this->options_.batch_size && + this->options_.drop_last)) { + return std::nullopt; + } + AT_ASSERT(indices->size() > 0); + return indices; + } + + /// The `Sampler` used to produce batch requests. + Sampler sampler_; +}; +} // namespace torch::data diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader_options.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader_options.h new file mode 100644 index 0000000000000000000000000000000000000000..ae7eadf44d2c97b559abe73cc3ec2f9e1453cfe5 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/dataloader_options.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include + +#include +#include + +namespace torch::data { + +/// Options to configure a `DataLoader`. +struct DataLoaderOptions { + DataLoaderOptions() = default; + /* implicit */ DataLoaderOptions(size_t batch_size) + : batch_size_(batch_size) {} + + /// The size of each batch to fetch. + TORCH_ARG(size_t, batch_size) = 1; + + /// The number of worker threads to launch. If zero, the main thread will + /// synchronously perform the data loading. + TORCH_ARG(size_t, workers) = 0; + + /// The maximum number of jobs to enqueue for fetching by worker threads. + /// Defaults to two times the number of worker threads. + TORCH_ARG(std::optional, max_jobs); + + /// An optional limit on the time to wait for the next batch. + TORCH_ARG(std::optional, timeout); + + /// Whether to enforce ordering of batches when multiple are loaded + /// asynchronously by worker threads. Set to `false` for better performance if + /// you do not care about determinism. + TORCH_ARG(bool, enforce_ordering) = true; + + /// Whether to omit the last batch if it contains less than `batch_size` + /// examples. + TORCH_ARG(bool, drop_last) = false; +}; + +/// Like `DataLoaderOptions`, but without any unconfigured state. +/// `DataLoaderOptions` has some options that depend on other options +/// (`max_jobs` => `2 * workers`). In the spirit of properly using the C++ type +/// system, `DataLoaderOptions` allows only setting values. To access values, +/// you must create a `FullDataLoaderOptions` from a `DataLoaderOptions` +/// instance, which will do any necessary coalescing. +struct FullDataLoaderOptions { + explicit FullDataLoaderOptions(DataLoaderOptions options) + : batch_size(options.batch_size()), + workers(options.workers()), + max_jobs(options.max_jobs().value_or(2 * workers)), + timeout(options.timeout()), + enforce_ordering(options.enforce_ordering()), + drop_last(options.drop_last()) {} + + size_t batch_size; + size_t workers; + size_t max_jobs; + std::optional timeout; + bool enforce_ordering; + bool drop_last; +}; +} // namespace torch::data diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets.h new file mode 100644 index 0000000000000000000000000000000000000000..b9b55ddddbb15df9865b4f6470af9ee076eb256e --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets.h @@ -0,0 +1,9 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/base.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/base.h new file mode 100644 index 0000000000000000000000000000000000000000..9c44abe04dbb28003ac27680421f86139e75dc3d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/base.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace torch::data::datasets { +template +class MapDataset; +template +MapDataset map(D, T); // NOLINT +} // namespace torch::data::datasets + +namespace torch::data::datasets { +namespace detail { +template +struct is_optional : std::false_type {}; +template +struct is_optional> : std::true_type {}; +} // namespace detail + +/// A dataset that can yield data only in batches. +template < + typename Self, + typename Batch = std::vector>, + typename BatchRequest = ArrayRef> +class BatchDataset { + public: + using SelfType = Self; + using BatchType = Batch; + using BatchRequestType = BatchRequest; + constexpr static bool is_stateful = detail::is_optional::value; + + virtual ~BatchDataset() = default; + + /// Returns a batch of data given an index. + virtual Batch get_batch(BatchRequest request) = 0; + + /// Returns the size of the dataset, or an empty std::optional if it is + /// unsized. + virtual std::optional size() const = 0; + + /// Creates a `MapDataset` that applies the given `transform` to this dataset. + template + MapDataset map(TransformType transform) & { + return datasets::map(static_cast(*this), std::move(transform)); + } + + /// Creates a `MapDataset` that applies the given `transform` to this dataset. + template + MapDataset map(TransformType transform) && { + return datasets::map( + std::move(static_cast(*this)), std::move(transform)); + } +}; + +/// A dataset that can yield data in batches, or as individual examples. +/// +/// A `Dataset` is a `BatchDataset`, because it supports random access and +/// therefore batched access is implemented (by default) by calling the random +/// access indexing function for each index in the requested batch of indices. +/// This can be customized. +template > +class Dataset : public BatchDataset> { + public: + using ExampleType = SingleExample; + + /// Returns the example at the given index. + virtual ExampleType get(size_t index) = 0; + + /// Returns a batch of data. + /// The default implementation calls `get()` for every requested index + /// in the batch. + std::vector get_batch(ArrayRef indices) override { + std::vector batch; + batch.reserve(indices.size()); + for (const auto i : indices) { + batch.push_back(get(i)); + } + return batch; + } +}; + +/// A `StreamDataset` represents a dataset that is a potentially infinite +/// stream. It takes as batch index only a number, which is the batch size, and +/// yields that many elements from the stream. +template >> +using StreamDataset = BatchDataset; +} // namespace torch::data::datasets diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/chunk.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/chunk.h new file mode 100644 index 0000000000000000000000000000000000000000..17293f90d073b6f92e9f00d19b99f379c7455635 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/chunk.h @@ -0,0 +1,527 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace torch::data::datasets { + +/// Interface for chunk reader, which performs data chunking and reading of +/// entire chunks. +/// +/// A chunk could be an entire file, such as an audio data file or an image, +/// or part of a file in the case of a large text-file split based on seek +/// positions. +template < + typename ExampleType_, + typename ChunkType_ = std::vector> +class ChunkDataReader { + public: + virtual ~ChunkDataReader() = default; + + using ChunkType = ChunkType_; + using ExampleType = ExampleType_; + + /// Read an entire chunk. + virtual ChunkType read_chunk(size_t chunk_index) = 0; + + /// Returns the number of chunks available in this reader. + virtual size_t chunk_count() = 0; + + /// This will clear any internal state associate with this reader. + virtual void reset() = 0; +}; + +namespace detail { +/// BatchDataBuffer manages a queue of UnwrappedBatchData. After a new chunk is +/// loaded, BatchDataBuffer splits it into small batches and push them into the +/// queue. When get_batch is called from data loader, it pops cached batches and +/// return. If the cache is empty, it either waits to load more chunks or return +/// null if all chunks are loaded. +template < + typename UnwrappedBatch, + typename ExampleSampler = samplers::RandomSampler> +class BatchDataBuffer { + public: + using UnwrappedBatchType = UnwrappedBatch; + using BatchType = std::optional; + using BatchRequestType = typename ExampleSampler::BatchRequestType; + + BatchDataBuffer( + size_t batch_size, + ExampleSampler& example_sampler, + size_t queue_capacity) + : batch_size_(batch_size), + example_sampler_(example_sampler), + queue_capacity_(queue_capacity) {} + + /// Return batch data from the queue. Called from the ChunkDataset main + /// thread. + BatchType get_batch() { + std::unique_lock lock(queue_mutex_); + cv_read_.wait(lock, [this] { + // wait till there is available data in the queue or if all chunks are + // loaded (i.e. the dataset is exhausted for this epoch) + return ( + this->total_example_count_in_queue_ >= batch_size_ || this->stop_); + }); + if (batch_queue_.empty()) { + AT_ASSERT(stop_); + // All batches have been retrieved. Return an empty batch. + return std::nullopt; + } + + UnwrappedBatchData batch = std::move(batch_queue_.front()); + batch_queue_.pop(); + if (batch.exception) { + throw WorkerException(batch.exception); + } + + total_example_count_in_queue_ -= batch.batch_data.size(); + lock.unlock(); + cv_write_.notify_all(); + + return batch.batch_data; + } + + /// Push preloaded chunks to batch queue. Called from the ChunkDataset worker + /// threads. + void add_chunk_data(UnwrappedBatchType data) { + std::unique_lock lock(queue_mutex_); + cv_write_.wait(lock, [this] { + // stop loading if we have preloaded enough data. + return this->total_example_count_in_queue_ < this->queue_capacity_ || + this->stop_; + }); + if (stop_) { + // When stop_ is true, it means no further chunk loading is necessary. + // Return without any further processing. + return; + } + + auto data_size = data.size(); + auto remaining_size = data_size; + example_sampler_.reset(data_size); + + auto fill_batch = [&](size_t example_count, UnwrappedBatchType& batch) { + auto batch_example_indices = this->example_sampler_.next(example_count); + AT_ASSERT( + batch_example_indices && + batch_example_indices.value().size() == example_count); + BatchRequestType& indices = batch_example_indices.value(); + for (size_t i : indices) { + TORCH_CHECK(i < data_size, "Index out of range"); + batch.emplace_back(std::move(data[i])); + } + remaining_size -= example_count; + }; + + if (!batch_queue_.empty()) { + // if the queue has existing data, and the last batch doesn't have enough + // examples to fill a batch_size batch, add more example to this batch + // first. + auto& batch = batch_queue_.back(); + size_t current_count = batch.batch_data.size(); + if (current_count < batch_size_) { + auto example_count = + std::min(remaining_size, batch_size_ - current_count); + fill_batch(example_count, batch.batch_data); + } + } + + // If we still have data remaining after filling the last pushed batch, add + // them to the queue too. + while (remaining_size > 0) { + UnwrappedBatchType current_batch; + + // Allocate the batch memory ahead of time. + current_batch.reserve(batch_size_); + + auto example_count = std::min(remaining_size, batch_size_); + fill_batch(example_count, current_batch); + batch_queue_.emplace(std::move(current_batch)); + } + total_example_count_in_queue_ += data_size; + lock.unlock(); + cv_read_.notify_all(); + } + + /// Push exceptions thrown during preloading into batch queue. Called from + /// the ChunkDataset worker threads. + void add_chunk_data(std::exception_ptr e_ptr) { + std::unique_lock lock(queue_mutex_); + cv_write_.wait(lock, [this] { + // stop loading if we have preloaded enough data. + return ( + this->total_example_count_in_queue_ < this->queue_capacity_ || + this->stop_); + }); + if (stop_) { + // When stop_ is true, it means this current thread needs to be tore down, + // the batch buffer will be discarded, so no need to enqueue any new + // exceptions. + return; + } + + batch_queue_.emplace(e_ptr); + lock.unlock(); + cv_read_.notify_all(); + } + + void stop() { + { + // Hold the lock before changing stop_ to prevent a race condition which + // can cause a deadlock. To be more specific, conditional variable + // cv_write_ waits on predicate stop_ in add_chunk_data(). The wait + // happens in two steps: 1) while still holding the lock, check if + // predicate is true; 2) if it is true, proceeds, otherwise, release the + // lock and wait until notified. Without holding a lock, cv_write_'s + // notification can happen in between step 1) and 2). In that case, as + // cv_write_ is not in waiting status yet, so the notification is lost and + // cv_write_ will sleep forever. By taking a lock before changing + // predicate stop_, it is ensured updating and evaluating stop_ always + // happen in a synchronized way + std::lock_guard lock(queue_mutex_); + stop_ = true; + } + + // notify all writers, wake them from wait to exit current method. + cv_write_.notify_all(); + // notify all readers too. + cv_read_.notify_all(); + } + /// The batch size is needed to create batches from the chunk data. Similar to + /// regular dataloader where the batches are created with prefetches, + /// BatchDataBuffer perform the batch creation using the provided batch size. + size_t batch_size_ = 0; + + /// count of total example stored in the queue + size_t total_example_count_in_queue_ = 0; + + /// struct that contains a raw unwrapped batch unit. An unwrapped batch unit + /// is the raw data without 'optional' wrapper. It can be a collection of + /// images, utterances, e.t.c. + struct UnwrappedBatchData { + explicit UnwrappedBatchData(UnwrappedBatchType data) + : batch_data(std::move(data)) {} + + explicit UnwrappedBatchData(std::exception_ptr e) + : exception(std::move(e)) {} + + /// batch data to return + UnwrappedBatchType batch_data; + + /// exception pointer which captures any abnormal exceptions while creating + /// the batch. + std::exception_ptr exception; + }; + + /// local cache to store example batches from loaded chunk + std::queue batch_queue_; + + // sync batch_queue_ update. + std::mutex queue_mutex_; + + std::condition_variable cv_read_; + std::condition_variable cv_write_; + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + ExampleSampler& example_sampler_; + + // configurable maximum number of elements the queue can hold at one time. + size_t queue_capacity_; + + // When set to true, it wakes the writer threads from the wait and exit + // current function call. This is needed when ChunkDataSet.Reset is called + // while the previous epoch is not exhausted yet. When ChunkDataset is waiting + // its preloader to finish previous work before tearing down the thread, the + // preloader could be still waiting for the conditional variable, thus cause + // the program to hang. This boolean is used to break this waiting condition. + bool stop_ = false; +}; +} // namespace detail + +/// Options to configure a `ChunkDataset`. +struct ChunkDatasetOptions { + ChunkDatasetOptions() = delete; + ChunkDatasetOptions( + size_t preloader_count, + size_t batch_size, + size_t cache_size = 2048, + size_t cross_chunk_shuffle_count = 1) + : preloader_count_(preloader_count), + batch_size_(batch_size), + cache_size_(cache_size), + cross_chunk_shuffle_count_(cross_chunk_shuffle_count) { + TORCH_CHECK( + preloader_count_ > 0, + "Preloader count is 0. At least one preloader needs to be specified."); + TORCH_CHECK( + batch_size_ > 0, + "Batch size is 0. A positive batch size needs to be specified."); + TORCH_CHECK( + cache_size_ > 0, + "Cache size is 0. A positive cache size needs to be specified."); + TORCH_CHECK( + cache_size_ >= batch_size_, + "Cache size is less than batch size. Cache needs to be large enough to " + "hold at least one batch."); + TORCH_CHECK( + cross_chunk_shuffle_count_ > 0, + "cross_chunk_shuffle_count needs to be greater than 0."); + } + + /// The number of worker thread to preload chunk data. + TORCH_ARG(size_t, preloader_count); + + /// The size of each batch. + TORCH_ARG(size_t, batch_size); + + /// The capacity of the queue for batch caching. + TORCH_ARG(size_t, cache_size) = 2048; + + // The number of chunks to perform cross-chunk shuffling. Default to 1 meaning + // no cross-chunk shuffling. When it is equal to n (n > 1), n random + // chunks will be loaded at once and example shuffling will be performed + // across all those n chunks. + // Note: Usually the default config (1 chunk shuffle + example shuffle) is + // good enough to generate random distributed data. Use this parameter only if + // you know cross-shuffle is needed in your case. Also there is a performance + // penalty when this value is greater than 1, as we need to do extra merge + // between multiple chunks before performing example sampling. + TORCH_ARG(size_t, cross_chunk_shuffle_count) = 1; +}; + +/// A stateful dataset that support hierarchical sampling and prefetching of +/// entre chunks. +/// +/// Unlike regular dataset, chunk dataset require two samplers to operate and +/// keeps an internal state. `ChunkSampler` selects, which chunk to load next, +/// while the `ExampleSampler` determines the order of Examples that are +/// returned in each `get_batch` call. The hierarchical sampling approach used +/// here is inspired by this paper +/// http://martin.zinkevich.org/publications/nips2010.pdf +template < + typename ChunkReader, + typename ChunkSampler = samplers::RandomSampler, + typename ExampleSampler = samplers::RandomSampler> +class ChunkDataset final + : public StatefulDataset< + ChunkDataset, + typename ChunkReader::BatchType, + size_t> { + public: + using BatchType = std::optional; + using UnwrappedBatchType = typename ChunkReader::BatchType; + using BatchRequestType = size_t; + using ChunkSamplerType = ChunkSampler; + using ExampleSamplerType = ExampleSampler; + + ChunkDataset( + ChunkReader chunk_reader, + ChunkSampler chunk_sampler, + ExampleSampler example_sampler, + ChunkDatasetOptions options, + std::function preprocessing_policy = + std::function()) + : chunk_reader_(std::move(chunk_reader)), + chunk_sampler_(std::move(chunk_sampler)), + example_sampler_(std::move(example_sampler)), + options_(options), + preprocessing_policy_(std::move(preprocessing_policy)), + quit_worker_(false), + running_preloaders_(0) {} + + ~ChunkDataset() override { + // stop batch buffer first. + if (batch_buffer_) { + batch_buffer_->stop(); + } + free_workers(); + } + + /// Default get_batch method of BatchDataset. This method returns + /// Example batches created from the preloaded chunks. The implementation + /// is dataset agnostic and does not need overriding in different chunk + /// datasets. + BatchType get_batch(size_t batch_size) override { + TORCH_CHECK( + batch_buffer_ != nullptr, + "Dataset needs to call reset() before calling get_batch()."); + + TORCH_CHECK( + batch_size == options_.batch_size(), + "The requested batch size does not match with the initialized batch size.\n" + " The requested batch size is ", + batch_size, + ", while the dataset is created with batch size equal to ", + options_.batch_size()); + return batch_buffer_->get_batch(); + } + + /// Helper method around get_batch as `batch_size` is not strictly necessary + BatchType get_batch() { + return get_batch(options_.batch_size()); + } + + /// This will clear any internal state and starts the internal prefetching + /// mechanism for the chunk dataset. + void reset() override { + // We need this to support partial data reads via dataloader iterator. + if (batch_buffer_) { + batch_buffer_->stop(); + } + // free workers from previous reset if there is any. + free_workers(); + preload_threads_.clear(); + + if (!load_checkpoint_) { + chunk_reader_.reset(); + chunk_sampler_.reset(chunk_reader_.chunk_count()); + load_checkpoint_ = false; + } + + // Throw out any existing cached batch in the buffer and re-creates a new + // chunk buffer. + batch_buffer_ = std::make_unique< + detail::BatchDataBuffer>( + options_.batch_size(), example_sampler_, options_.cache_size()); + + // create new workers for this new epoch. + quit_worker_ = false; + + AT_ASSERT(running_preloaders_ == 0); + running_preloaders_ = options_.preloader_count(); + for (const auto i : c10::irange(options_.preloader_count())) { + preload_threads_.emplace_back([this, i]() { this->preloader(i); }); + } + } + + /// size is not used for chunk dataset. + std::optional size() const override { + return std::nullopt; + } + + // provide a references to chunk sampler. Used mainly in distributed data + // loading to set the epoch number for the sampler. + ChunkSamplerType& chunk_sampler() { + return chunk_sampler_; + } + + void save(serialize::OutputArchive& archive) const override { + std::lock_guard lock(chunk_index_guard_); + chunk_sampler_.save(archive); + } + + void load(serialize::InputArchive& archive) override { + std::lock_guard lock(chunk_index_guard_); + chunk_sampler_.load(archive); + load_checkpoint_ = true; + } + + private: + /// running on worker thread to preload chunk data. + void preloader(size_t id) { + while (!quit_worker_.load()) { + try { + std::vector chunk_idx; + { + std::lock_guard lock(chunk_index_guard_); + if (auto chunk_sampler_result = chunk_sampler_.next( + this->options_.cross_chunk_shuffle_count())) { + chunk_idx = chunk_sampler_result.value(); + } else { + break; + } + } + UnwrappedBatchType data = chunk_reader_.read_chunk(chunk_idx[0]); + for (const auto i : c10::irange(1, chunk_idx.size())) { + auto chunk_data = chunk_reader_.read_chunk(chunk_idx[i]); + std::move( + chunk_data.begin(), chunk_data.end(), std::back_inserter(data)); + } + if (preprocessing_policy_) { + preprocessing_policy_(data); + } + if (!data.empty()) { // skip empty chunks. + batch_buffer_->add_chunk_data(std::move(data)); + } + } catch (...) { + batch_buffer_->add_chunk_data(std::current_exception()); + } + } + AT_ASSERT(running_preloaders_.load() > 0); + --running_preloaders_; + if (running_preloaders_.load() == 0) { + // all preloaders are completed, so we can notify the batch_buffer. + batch_buffer_->stop(); + } + } + + /// Block the current thread until the workers finish execution and exit. + void free_workers() { + if (!quit_worker_.load()) { + quit_worker_ = true; + for (auto& worker_thread : preload_threads_) { + worker_thread.join(); + } + } + } + + private: + // Templated class that defines what is a chunk and how to read chunk data. + // When a chunk is returned by chunk_reader_, ChunkDataset split it into + // batches and caches them in batch_buffer_. + ChunkReader chunk_reader_; + + // chunk sampler to shuffle different chunks + ChunkSamplerType chunk_sampler_; + + // example sampler to shuffle examples in a specific chunk + ExampleSamplerType example_sampler_; + + // batch data buffer which holds chunk data from preloading thread. + std::shared_ptr< + detail::BatchDataBuffer> + batch_buffer_; + + // worker thread pool + std::vector preload_threads_; + + /// The options the Dataset was configured with. + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const ChunkDatasetOptions options_; + + // function pointer wrapper to apply custom processing over chunk data. This + // is considered an advanced parameter for developers who want to apply a + // pre-process to the chunk data before sampling into minibatch. + // Different than the collate function, this policy is applied on the chunk + // level, instead of minibatch level. When a chunk of data is loaded (multiple + // chunks if cross_chunk_shuffle_count_ is greater than 1), this policy is + // applied to the full loaded data. It is useful if developers want to + // perform pre-processing (like bucketing) to the chunk data before + // example sampler samples the data. By default it's an empty pointer and no + // action will be taken. + std::function preprocessing_policy_; + + // indicate whether the worker thread can be teared down + std::atomic quit_worker_; + + // keep track of running preloaders to notify batch buffer. A value 0 + // indicates that the chunk loading is completed. + std::atomic running_preloaders_; + + // mutex to synchronize chunk sampler next() call. + mutable std::mutex chunk_index_guard_; + + // boolean value to indicate whether we need to load the checkpoint for + // chunk_sampler_. + bool load_checkpoint_{false}; +}; +} // namespace torch::data::datasets diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/map.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/map.h new file mode 100644 index 0000000000000000000000000000000000000000..b21404dd5dd0203392023439053e9aff17c3857d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/map.h @@ -0,0 +1,114 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include + +namespace torch::data::datasets { +namespace detail { +template +using optional_if_t = std::conditional_t, T>; +} // namespace detail + +/// A `MapDataset` is a dataset that applies a transform to a source dataset. +template +class MapDataset : public BatchDataset< + MapDataset, + detail::optional_if_t< + SourceDataset::is_stateful, + typename AppliedTransform::OutputBatchType>, + typename SourceDataset::BatchRequestType> { + public: + using DatasetType = SourceDataset; + using TransformType = AppliedTransform; + using BatchRequestType = typename SourceDataset::BatchRequestType; + using OutputBatchType = detail::optional_if_t< + SourceDataset::is_stateful, + typename AppliedTransform::OutputBatchType>; + + MapDataset(DatasetType dataset, TransformType transform) + : dataset_(std::move(dataset)), transform_(std::move(transform)) {} + + /// Gets a batch from the source dataset and applies the transform to it, + /// returning the result. + OutputBatchType get_batch(BatchRequestType indices) override { + return get_batch_impl(std::move(indices)); + } + + /// Returns the size of the source dataset. + // NOLINTNEXTLINE(bugprone-exception-escape) + std::optional size() const noexcept override { + return dataset_.size(); + } + + /// Calls `reset()` on the underlying dataset. + /// NOTE: Stateless datasets do not have a reset() method, so a call to this + /// method will only compile for stateful datasets (which have a reset() + /// method). + void reset() { + dataset_.reset(); + } + + /// Returns the underlying dataset. + const SourceDataset& dataset() noexcept { + return dataset_; + } + + /// Returns the transform being applied. + const AppliedTransform& transform() noexcept { + return transform_; + } + + private: + /// The implementation of `get_batch()` for the stateless case, which simply + /// applies the transform to the output of `get_batch()` from the dataset. + template < + typename D = SourceDataset, + typename = std::enable_if_t> + OutputBatchType get_batch_impl(BatchRequestType indices) { + return transform_.apply_batch(dataset_.get_batch(std::move(indices))); + } + + /// The implementation of `get_batch()` for the stateful case. Here, we follow + /// the semantics of `Optional.map()` in many functional languages, which + /// applies a transformation to the optional's content when the optional + /// contains a value, and returns a new optional (of a different type) if the + /// original optional returned by `get_batch()` was empty. + template + std::enable_if_t get_batch_impl( + BatchRequestType indices) { + if (auto batch = dataset_.get_batch(std::move(indices))) { + return transform_.apply_batch(std::move(*batch)); + } + return std::nullopt; + } + + /// The underlying dataset being transformed. + SourceDataset dataset_; + + // The transformation that is applied to batches received from the dataset. + AppliedTransform transform_; +}; + +/// Creates a `MapDataset` with the given dataset and transform. +template +MapDataset map( + DatasetType dataset, + TransformType transform) { + static_assert( + std::is_same_v< + std::conditional_t< + DatasetType::is_stateful, + typename DatasetType::BatchType::value_type, + typename DatasetType::BatchType>, + typename TransformType::InputBatchType>, + "BatchType type of dataset does not match input type of transform"); + return {std::move(dataset), std::move(transform)}; +} + +} // namespace torch::data::datasets diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/mnist.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/mnist.h new file mode 100644 index 0000000000000000000000000000000000000000..aa8a3fc8f46ea7f029549bf3c7959a74f2e7677a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/mnist.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include + +namespace torch::data::datasets { +/// The MNIST dataset. +class TORCH_API MNIST : public Dataset { + public: + /// The mode in which the dataset is loaded. + enum class Mode { kTrain, kTest }; + + /// Loads the MNIST dataset from the `root` path. + /// + /// The supplied `root` path should contain the *content* of the unzipped + /// MNIST dataset, available from http://yann.lecun.com/exdb/mnist. + explicit MNIST(const std::string& root, Mode mode = Mode::kTrain); + + /// Returns the `Example` at the given `index`. + Example<> get(size_t index) override; + + /// Returns the size of the dataset. + std::optional size() const override; + + /// Returns true if this is the training subset of MNIST. + // NOLINTNEXTLINE(bugprone-exception-escape) + bool is_train() const noexcept; + + /// Returns all images stacked into a single tensor. + const Tensor& images() const; + + /// Returns all targets stacked into a single tensor. + const Tensor& targets() const; + + private: + Tensor images_, targets_; +}; +} // namespace torch::data::datasets diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/shared.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/shared.h new file mode 100644 index 0000000000000000000000000000000000000000..0699b8e757d0b9de3c4f2c02ce1475b57fdf36e7 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/shared.h @@ -0,0 +1,79 @@ +#pragma once + +#include + +#include +#include + +namespace torch::data::datasets { + +/// A dataset that wraps another dataset in a shared pointer and implements the +/// `BatchDataset` API, delegating all calls to the shared instance. This is +/// useful when you want all worker threads in the dataloader to access the same +/// dataset instance. The dataset must take care of synchronization and +/// thread-safe access itself. +/// +/// Use `torch::data::datasets::make_shared_dataset()` to create a new +/// `SharedBatchDataset` like you would a `std::shared_ptr`. +template +class SharedBatchDataset : public BatchDataset< + SharedBatchDataset, + typename UnderlyingDataset::BatchType, + typename UnderlyingDataset::BatchRequestType> { + public: + using BatchType = typename UnderlyingDataset::BatchType; + using BatchRequestType = typename UnderlyingDataset::BatchRequestType; + + /// Constructs a new `SharedBatchDataset` from a `shared_ptr` to the + /// `UnderlyingDataset`. + /* implicit */ SharedBatchDataset( + std::shared_ptr shared_dataset) + : dataset_(std::move(shared_dataset)) {} + + /// Calls `get_batch` on the underlying dataset. + BatchType get_batch(BatchRequestType request) override { + return dataset_->get_batch(std::move(request)); + } + + /// Returns the `size` from the underlying dataset. + std::optional size() const override { + return dataset_->size(); + } + + /// Accesses the underlying dataset. + UnderlyingDataset& operator*() { + return *dataset_; + } + + /// Accesses the underlying dataset. + const UnderlyingDataset& operator*() const { + return *dataset_; + } + + /// Accesses the underlying dataset. + UnderlyingDataset* operator->() { + return dataset_.get(); + } + + /// Accesses the underlying dataset. + const UnderlyingDataset* operator->() const { + return dataset_.get(); + } + + /// Calls `reset()` on the underlying dataset. + void reset() { + dataset_->reset(); + } + + private: + std::shared_ptr dataset_; +}; + +/// Constructs a new `SharedBatchDataset` by creating a +/// `shared_ptr`. All arguments are forwarded to +/// `make_shared`. +template +SharedBatchDataset make_shared_dataset(Args&&... args) { + return std::make_shared(std::forward(args)...); +} +} // namespace torch::data::datasets diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/stateful.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/stateful.h new file mode 100644 index 0000000000000000000000000000000000000000..4961852d19399c43135db4a6bcdb8f84adec352b --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/stateful.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include + +#include +#include + +namespace torch::serialize { +class OutputArchive; +class InputArchive; +} // namespace torch::serialize + +namespace torch::data::datasets { + +/// A stateful dataset is a dataset that maintains some internal state, which +/// will be `reset()` at the beginning of each epoch. Subclasses can override +/// the `reset()` method to configure this behavior. Further, the return type of +/// a stateful dataset's `get_batch()` method is always an `optional`. When the +/// stateful dataset wants to indicate to the dataloader that its epoch has +/// ended, it should return an empty optional. The dataloader knows to modify +/// its implementation based on whether the dataset is stateless or stateful. +/// +/// Note that when subclassing a from `StatefulDataset`, the return +/// type of `get_batch()`, which the subclass must override, will be +/// `optional` (i.e. the type specified in the `StatefulDataset` +/// specialization is automatically boxed into an `optional` for the dataset's +/// `BatchType`). +template < + typename Self, + typename Batch = std::vector>, + typename BatchRequest = size_t> +class StatefulDataset + : public BatchDataset, BatchRequest> { + public: + /// Resets internal state of the dataset. + virtual void reset() = 0; + + /// Saves the statefulDataset's state to OutputArchive. + virtual void save(serialize::OutputArchive& archive) const = 0; + + /// Deserializes the statefulDataset's state from the `archive`. + virtual void load(serialize::InputArchive& archive) = 0; +}; + +/// Serializes a statefulDataset to `OutputArchive`. +template +serialize::OutputArchive& operator<<( + serialize::OutputArchive& archive, + const StatefulDataset& statefulDataset) { + statefulDataset.save(archive); + return archive; +} + +/// Deserializes a statefulDataset from an `InputArchive`. +template +serialize::InputArchive& operator>>( + serialize::InputArchive& archive, + StatefulDataset& statefulDataset) { + statefulDataset.load(archive); + return archive; +} + +} // namespace torch::data::datasets diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/tensor.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/tensor.h new file mode 100644 index 0000000000000000000000000000000000000000..0eb188baa345e24ce36d12690225cbcfa1c9c9b2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/datasets/tensor.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace torch::data::datasets { + +/// A dataset of tensors. +/// Stores a single tensor internally, which is then indexed inside `get()`. +struct TensorDataset : public Dataset { + /// Creates a `TensorDataset` from a vector of tensors. + explicit TensorDataset(const std::vector& tensors) + : TensorDataset(torch::stack(tensors)) {} + + explicit TensorDataset(torch::Tensor tensor) : tensor(std::move(tensor)) {} + + /// Returns a single `TensorExample`. + TensorExample get(size_t index) override { + return tensor[static_cast(index)]; + } + + /// Returns the number of tensors in the dataset. + std::optional size() const override { + return tensor.size(0); + } + + Tensor tensor; +}; + +} // namespace torch::data::datasets diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/detail/data_shuttle.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/detail/data_shuttle.h new file mode 100644 index 0000000000000000000000000000000000000000..3ca3c8c0d31dd131e8ea7e8e2ed5811bd8b404b8 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/detail/data_shuttle.h @@ -0,0 +1,83 @@ +#pragma once + +#include +#include + +#include +#include + +#include +#include + +namespace torch::data::detail { + +/// Encapsulates the full life cycle of DataLoader jobs. +/// +/// When a new job is enqueued to the `DataShuttle`, a counter for in-flight +/// jobs is bumped. This job is said to be "in-flight" until its result is +/// popped. Worker threads dequeue jobs as soon as they are available. When a +/// worker finishes a job, it enqueues the result. Only when the main thread +/// dequeues a result is the count of in-flight jobs decremented. When the main +/// thread attempts to dequeue a job but no jobs are in-flight, that means the +/// epoch is complete and `pop_result` returns an empty optional. +template +class DataShuttle { + public: + /// Pushes a new job. Called by the main thread. + void push_job(Job job) { + new_jobs_.push(std::move(job)); + ++in_flight_jobs_; + } + + /// Pushes the result of a job. Called by worker threads. + void push_result(Result result) { + results_.push(std::move(result)); + } + + /// Returns the next job, blocking until there is one available. Called by + /// worker threads. + Job pop_job() { + return new_jobs_.pop(); + } + + /// Returns the result of a job, or nullopt if all jobs were exhausted. Called + /// by the main thread. + std::optional pop_result( + std::optional timeout = std::nullopt) { + if (in_flight_jobs_ > 0) { + auto result = results_.pop(timeout); + --in_flight_jobs_; + return result; + } + return std::nullopt; + } + + /// Discards any jobs that are not yet in flight, and waits for all in-flight + /// jobs to finish, discarding their result. + void drain() { + // Clear all inputs so that no further jobs are scheduled. + auto number_cleared = new_jobs_.clear(); + in_flight_jobs_ -= number_cleared; + // Remove any outstanding results. + while (in_flight_jobs_ > 0) { + pop_result(); + } + } + + /// Returns the number of jobs that are still in progress. + /// When this number is zero, an epoch is finished. + size_t in_flight_jobs() const noexcept { + return in_flight_jobs_; + } + + private: + /// The queue for jobs that are not yet in flight. + Queue new_jobs_; + /// The number of in-flight jobs. + /// NOTE: Not atomic because only manipulated by the main thread. + size_t in_flight_jobs_ = 0; + /// The queue for results of finished jobs. + Queue results_; +}; + +} // namespace torch::data::detail diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/detail/queue.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/detail/queue.h new file mode 100644 index 0000000000000000000000000000000000000000..c332fca058b6ef9b9ea5e8b29e709c7a38eb7a23 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/detail/queue.h @@ -0,0 +1,80 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include + +namespace torch::data::detail { + +/// A basic locked, blocking MPMC queue. +/// +/// Every `push` and `pop` is guarded by a mutex. A condition variable is used +/// to communicate insertion of new elements, such that waiting threads will be +/// woken up if they are currently waiting inside a call to `pop()`. +/// +/// Note that this data structure is written specifically for use with the +/// `DataLoader`. Its behavior is tailored to this use case and may not be +/// applicable to more general uses. +template +class Queue { + public: + /// Pushes a new value to the back of the `Queue` and notifies one thread on + /// the waiting side about this event. + void push(T value) { + { + std::lock_guard lock(mutex_); + queue_.push(std::move(value)); + } + cv_.notify_one(); + } + + /// Blocks until at least one element is ready to be popped from the front of + /// the queue. An optional `timeout` in seconds can be used to limit the time + /// spent waiting for an element. If the wait times out, an exception is + /// raised. + T pop(std::optional timeout = std::nullopt) { + std::unique_lock lock(mutex_); + if (timeout) { + if (!cv_.wait_for( + lock, *timeout, [this] { return !this->queue_.empty(); })) { + // clang-format off + TORCH_CHECK(false, + "Timeout in DataLoader queue while waiting for next batch" + " (timeout was ", timeout->count(), " ms)"); + // clang-format on + } + } else { + cv_.wait(lock, [this] { return !this->queue_.empty(); }); + } + AT_ASSERT(!queue_.empty()); + T value = queue_.front(); + queue_.pop(); + lock.unlock(); + return value; + } + + /// Empties the queue and returns the number of elements that were present at + /// the start of the function. No threads are notified about this event as it + /// is assumed to be used to drain the queue during shutdown of a + /// `DataLoader`. + size_t clear() { + std::lock_guard lock(this->mutex_); + const auto size = queue_.size(); + while (!queue_.empty()) { + queue_.pop(); + } + return size; + } + + private: + std::queue queue_; + std::mutex mutex_; + std::condition_variable cv_; +}; +} // namespace torch::data::detail diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/detail/sequencers.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/detail/sequencers.h new file mode 100644 index 0000000000000000000000000000000000000000..85a048976e30b225fa441275c93c46601cea2b8a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/detail/sequencers.h @@ -0,0 +1,107 @@ +#pragma once + +#include + +#include +#include +#include + +namespace torch::data::detail::sequencers { +namespace detail { +template +bool buffer_contains_result(const std::vector>& buffer) { + return std::any_of( + buffer.begin(), buffer.end(), [](const std::optional& result) { + return result.has_value(); + }); +} +} // namespace detail + +/// A `Sequencer` accepts a function that yields the next result of a +/// `DataLoader` and then has the opportunity to influence the order in which +/// these results are returned. The `NoSequencer` does not enforce any +/// sequencing and returns any result directly. The `OrderedSequencer` instead +/// buffers results internally to return them in order of their sequence number. +template +struct Sequencer { + using ResultProducer = std::function()>; + virtual ~Sequencer() = default; + virtual std::optional next(ResultProducer next_result) = 0; +}; + +/// A `Sequencer` that does not enforce any ordering. It is effectively the +/// identity function. +template +struct NoSequencer final : public Sequencer { + using typename Sequencer::ResultProducer; + std::optional next(ResultProducer next_result) override { + return next_result(); + } +}; + +/// A `Sequencer` that buffers results and returns them in order of their +/// sequence number. The `OrderedSequencer` maintains an internal, monotonically +/// incrementing counter for the next sequence number it expects. If it receives +/// a result with a higher sequence number, it will buffer it for later (when +/// the sequence number reaches that of this result). Otherwise, if the sequence +/// numbers match, the result is returned. +/// +/// Implementation note: The `OrderedSequencer` is implemented with a fixed-size +/// buffer. Let `m` be the maximum number of jobs in the data loader's queue and +/// `s` be the current sequence number. Assume `m` jobs are scheduled in the +/// `DataLoader`. Any new result is stored at index `job.sqn mod m` in the +/// `OrderedSequencer`. Why are we sure sequence numbers of new jobs will not +/// collide with sequence numbers of buffered jobs? The `OrderedSequencer` will +/// not return from `next()` until it receives the result with sqn `s`. This +/// means no new jobs can be scheduled in the `DataLoader` in the meantime, +/// which enforces that as long as sqn `s` has not been received, `s + m` (which +/// would cause a collision in the fixed-size buffer) will not yet be scheduled. +template +struct OrderedSequencer : public Sequencer { + using typename Sequencer::ResultProducer; + + /// Constructs the `OrderedSequencer` with the maximum number of results it + /// will ever hold at one point in time. + explicit OrderedSequencer(size_t max_jobs) : buffer_(max_jobs) {} + + /// Buffers results until the next one in the expected order is received. + std::optional next(ResultProducer next_result) override { + // If we already have the result for the next sqn, return it. + if (auto& maybe_result = buffer(next_sequence_number_)) { + auto result = std::move(*maybe_result); + buffer(next_sequence_number_++).reset(); + return result; + } + // Otherwise wait for the next result. + while (true) { + auto result = next_result(); + if (!result) { + AT_ASSERT(!detail::buffer_contains_result(buffer_)); + break; + } + // If it was not nullopt and the sequence numbers match, return it + // directly and bump the sequence number. + if (result->sequence_number == next_sequence_number_) { + ++next_sequence_number_; + return result; + } + // Stash the result for later. + AT_ASSERT(!buffer(result->sequence_number).has_value()); + buffer(result->sequence_number) = std::move(result); + } + // The result was an empty optional, so we are done with this epoch. + return std::nullopt; + } + + /// Accesses the buffer at the `index` modulo the buffer size. + std::optional& buffer(size_t index) { + return buffer_.at(index % buffer_.size()); + } + + /// The monotonically increasing sequence number we expect. + size_t next_sequence_number_ = 0; + + /// A fixed-size buffer (after construction). + std::vector> buffer_; +}; +} // namespace torch::data::detail::sequencers diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/example.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/example.h new file mode 100644 index 0000000000000000000000000000000000000000..e08e301af1694eb51211d5452264976ba521bf9f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/example.h @@ -0,0 +1,53 @@ +#pragma once + +#include + +namespace torch::data { + +/// An `Example` from a dataset. +/// +/// A dataset consists of data and an associated target (label). +template +struct Example { + using DataType = Data; + using TargetType = Target; + + Example() = default; + Example(Data data, Target target) + : data(std::move(data)), target(std::move(target)) {} + + Data data; + Target target; +}; + +namespace example { +using NoTarget = void; +} // namespace example + +/// A specialization for `Example` that does not have a target. +/// +/// This class exists so that code can be written for a templated `Example` +/// type, and work both for labeled and unlabeled datasets. +template +struct Example { + using DataType = Data; + using TargetType = example::NoTarget; + + Example() = default; + /* implicit */ Example(Data data) : data(std::move(data)) {} + + // When a DataLoader returns an Example like this, that example should be + // implicitly convertible to the underlying data type. + + operator Data&() { + return data; + } + operator const Data&() const { + return data; + } + + Data data; +}; + +using TensorExample = Example; +} // namespace torch::data diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/iterator.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/iterator.h new file mode 100644 index 0000000000000000000000000000000000000000..263e1bbbddb1ca00fbb802f2d22876893a5ed6a5 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/iterator.h @@ -0,0 +1,178 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace torch::data { +namespace detail { +// For increased safety and more separated logic, this implementation of +// `Iterator` consists of a `ValidIterator` and a `SentinelIterator`. A +// `ValidIterator` yields new batches until the `DataLoader` is exhausted. While +// the `DataLoader` is not exhausted, `ValidIterator`s compare equal if they are +// the same object. When the `ValidIterator` becomes exhausted, it compares +// equal to the `SentinelIterator`, but not before. Half the code here is to +// implement double dispatch for the comparison. Got damnit, C++. + +template +struct ValidIterator; + +template +struct SentinelIterator; + +/// Base class for the `ValidIterator` and `SentinelIterator` +template +struct IteratorImpl { + virtual ~IteratorImpl() = default; + virtual void next() = 0; + virtual Batch& get() = 0; + virtual bool operator==(const IteratorImpl& other) const = 0; + virtual bool operator==(const ValidIterator& other) const = 0; + virtual bool operator==(const SentinelIterator& other) const = 0; +}; + +template +struct ValidIterator : public IteratorImpl { + using BatchProducer = std::function()>; + + explicit ValidIterator(BatchProducer next_batch) + : next_batch_(std::move(next_batch)) {} + + /// Fetches the next batch. + void next() override { + // If we didn't get the very first batch yet, get it now. + lazy_initialize(); + TORCH_CHECK( + batch_.has_value(), "Attempted to increment iterator past the end"); + // Increment to the next batch. + batch_ = next_batch_(); + } + + /// Returns the current batch. The precondition for this operation to not + /// throw an exception is that it has been compared to the `SentinelIterator` + /// and did not compare equal. + Batch& get() override { + // If we didn't get the very first batch yet, get it now. + lazy_initialize(); + TORCH_CHECK( + batch_.has_value(), + "Attempted to dereference iterator that was past the end"); + return batch_.value(); + } + + /// Does double dispatch. + bool operator==(const IteratorImpl& other) const override { + return other == *this; + } + + /// A `ValidIterator` is equal to the `SentinelIterator` iff. the + /// `ValidIterator` has reached the end of the dataloader. + bool operator==(const SentinelIterator& /* unused */) const override { + lazy_initialize(); + return !batch_; + } + + /// Returns true if the memory address of `other` equals that of `this`. + bool operator==(const ValidIterator& other) const override { + return &other == this; + } + + /// Gets the very first batch if it has not yet been fetched. + void lazy_initialize() const { + if (!initialized_) { + batch_ = next_batch_(); + initialized_ = true; + } + } + + BatchProducer next_batch_; + mutable std::optional batch_; + mutable bool initialized_ = false; +}; + +template +struct SentinelIterator : public IteratorImpl { + void next() override { + TORCH_CHECK( + false, + "Incrementing the DataLoader's past-the-end iterator is not allowed"); + } + + Batch& get() override { + TORCH_CHECK( + false, + "Dereferencing the DataLoader's past-the-end iterator is not allowed"); + } + + /// Does double dispatch. + bool operator==(const IteratorImpl& other) const override { + return other == *this; + } + + /// Calls the comparison operator between `ValidIterator` and + /// `SentinelIterator`. + bool operator==(const ValidIterator& other) const override { + return other == *this; + } + + /// Sentinel iterators always compare equal. + bool operator==(const SentinelIterator& other) const override { + return true; + } +}; +} // namespace detail + +template +class Iterator { + public: + // Type aliases to make the class recognized as a proper iterator. + using difference_type = std::ptrdiff_t; + using value_type = Batch; + using pointer = Batch*; + using reference = Batch&; + using iterator_category = std::input_iterator_tag; + + explicit Iterator(std::unique_ptr> impl) + : impl_(std::move(impl)) {} + + /// Increments the iterator. + /// Only permitted for valid iterators (not past the end). + Iterator& operator++() { + impl_->next(); + return *this; + } + + /// Returns the current batch. + /// Only permitted for valid iterators (not past the end). + Batch& operator*() { + return impl_->get(); + } + + /// Returns a pointer to the current batch. + /// Only permitted for valid iterators (not past the end). + Batch* operator->() { + return &impl_->get(); + } + + /// Compares two iterators for equality. + bool operator==(const Iterator& other) const { + return *impl_ == *other.impl_; + } + + /// Compares two iterators for inequality. + bool operator!=(const Iterator& other) const { + return !(*this == other); + } + + private: + /// Points either to a `ValidIterator` or to a `SentinelIterator`. + std::shared_ptr> impl_; +}; +} // namespace torch::data diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers.h new file mode 100644 index 0000000000000000000000000000000000000000..6159a454950d276451fd45d5d001fa30b40db918 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers.h @@ -0,0 +1,9 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/base.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/base.h new file mode 100644 index 0000000000000000000000000000000000000000..cab4166ce23e93f0d0e969be473aad21c47138c9 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/base.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace torch::serialize { +class OutputArchive; +class InputArchive; +} // namespace torch::serialize + +namespace torch::data::samplers { +/// A `Sampler` is an object that yields an index with which to access a +/// dataset. +template > +class Sampler { + public: + using BatchRequestType = BatchRequest; + + virtual ~Sampler() = default; + + /// Resets the `Sampler`'s internal state. + /// Typically called before a new epoch. + /// Optionally, accepts a new size when resetting the sampler. + virtual void reset(std::optional new_size) = 0; + + /// Returns the next index if possible, or an empty optional if the + /// sampler is exhausted for this epoch. + virtual std::optional next(size_t batch_size) = 0; + + /// Serializes the `Sampler` to the `archive`. + virtual void save(serialize::OutputArchive& archive) const = 0; + + /// Deserializes the `Sampler` from the `archive`. + virtual void load(serialize::InputArchive& archive) = 0; +}; + +} // namespace torch::data::samplers diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/custom_batch_request.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/custom_batch_request.h new file mode 100644 index 0000000000000000000000000000000000000000..efefb24e83b8fbedc430002c9d5910e16acbf5c6 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/custom_batch_request.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +namespace torch::data::samplers { +/// A base class for custom index types. +struct TORCH_API CustomBatchRequest { + CustomBatchRequest() = default; + CustomBatchRequest(const CustomBatchRequest&) = default; + CustomBatchRequest(CustomBatchRequest&&) noexcept = default; + virtual ~CustomBatchRequest() = default; + + /// The number of elements accessed by this index. + virtual size_t size() const = 0; +}; +} // namespace torch::data::samplers diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/distributed.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/distributed.h new file mode 100644 index 0000000000000000000000000000000000000000..aa098aebc3d62bddcf1da68bb1bff5edb76e7fa1 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/distributed.h @@ -0,0 +1,133 @@ +#pragma once + +#include +#include + +#include +#include + +namespace torch::serialize { +class OutputArchive; +class InputArchive; +} // namespace torch::serialize + +namespace torch::data::samplers { + +/// A `Sampler` that selects a subset of indices to sample from and defines a +/// sampling behavior. In a distributed setting, this selects a subset of the +/// indices depending on the provided num_replicas and rank parameters. The +/// `Sampler` performs a rounding operation based on the `allow_duplicates` +/// parameter to decide the local sample count. +template > +class DistributedSampler : public Sampler { + public: + DistributedSampler( + size_t size, + size_t num_replicas = 1, + size_t rank = 0, + bool allow_duplicates = true) + : size_(size), + num_replicas_(num_replicas), + rank_(rank), + + allow_duplicates_(allow_duplicates) {} + + /// Set the epoch for the current enumeration. This can be used to alter the + /// sample selection and shuffling behavior. + void set_epoch(size_t epoch) { + epoch_ = epoch; + } + + size_t epoch() const { + return epoch_; + } + + protected: + size_t local_sample_count() { + if (allow_duplicates_) { + return (size_ + num_replicas_ - 1) / num_replicas_; + } else { + return size_ / num_replicas_; + } + } + + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + size_t size_; + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + size_t num_replicas_; + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + size_t rank_; + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + size_t epoch_{0}; + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + bool allow_duplicates_; +}; + +/// Select samples randomly. The sampling order is shuffled at each `reset()` +/// call. +class TORCH_API DistributedRandomSampler : public DistributedSampler<> { + public: + DistributedRandomSampler( + size_t size, + size_t num_replicas = 1, + size_t rank = 0, + bool allow_duplicates = true); + + /// Resets the `DistributedRandomSampler` to a new set of indices. + void reset(std::optional new_size = std::nullopt) override; + + /// Returns the next batch of indices. + std::optional> next(size_t batch_size) override; + + /// Serializes the `DistributedRandomSampler` to the `archive`. + void save(serialize::OutputArchive& archive) const override; + + /// Deserializes the `DistributedRandomSampler` from the `archive`. + void load(serialize::InputArchive& archive) override; + + /// Returns the current index of the `DistributedRandomSampler`. + size_t index() const noexcept; + + private: + void populate_indices(); + + size_t begin_index_{0}; + size_t end_index_{0}; + size_t sample_index_{0}; + std::vector all_indices_; +}; + +/// Select samples sequentially. +class TORCH_API DistributedSequentialSampler : public DistributedSampler<> { + public: + DistributedSequentialSampler( + size_t size, + size_t num_replicas = 1, + size_t rank = 0, + bool allow_duplicates = true); + + /// Resets the `DistributedSequentialSampler` to a new set of indices. + void reset(std::optional new_size = std::nullopt) override; + + /// Returns the next batch of indices. + std::optional> next(size_t batch_size) override; + + /// Serializes the `DistributedSequentialSampler` to the `archive`. + void save(serialize::OutputArchive& archive) const override; + + /// Deserializes the `DistributedSequentialSampler` from the `archive`. + void load(serialize::InputArchive& archive) override; + + /// Returns the current index of the `DistributedSequentialSampler`. + size_t index() const noexcept; + + private: + void populate_indices(); + + size_t begin_index_{0}; + size_t end_index_{0}; + size_t sample_index_{0}; + std::vector all_indices_; +}; + +} // namespace torch::data::samplers diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/random.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/random.h new file mode 100644 index 0000000000000000000000000000000000000000..a8e749db55a762079ab30dd236a196b8680fd034 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/random.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace torch::serialize { +class OutputArchive; +class InputArchive; +} // namespace torch::serialize + +namespace torch::data::samplers { + +/// A `Sampler` that returns random indices. +class TORCH_API RandomSampler : public Sampler<> { + public: + /// Constructs a `RandomSampler` with a size and dtype for the stored indices. + /// + /// The constructor will eagerly allocate all required indices, which is the + /// sequence `0 ... size - 1`. `index_dtype` is the data type of the stored + /// indices. You can change it to influence memory usage. + explicit RandomSampler(int64_t size, Dtype index_dtype = torch::kInt64); + + ~RandomSampler() override; + + /// Resets the `RandomSampler` to a new set of indices. + void reset(std::optional new_size = std::nullopt) override; + + /// Returns the next batch of indices. + std::optional> next(size_t batch_size) override; + + /// Serializes the `RandomSampler` to the `archive`. + void save(serialize::OutputArchive& archive) const override; + + /// Deserializes the `RandomSampler` from the `archive`. + void load(serialize::InputArchive& archive) override; + + /// Returns the current index of the `RandomSampler`. + size_t index() const noexcept; + + private: + at::Tensor indices_; + int64_t index_ = 0; +}; +} // namespace torch::data::samplers diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/sequential.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/sequential.h new file mode 100644 index 0000000000000000000000000000000000000000..c29e285e0566b402408368107bcbabc2fac1d32f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/sequential.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace torch::serialize { +class OutputArchive; +class InputArchive; +} // namespace torch::serialize + +namespace torch::data::samplers { + +/// A `Sampler` that returns indices sequentially. +class TORCH_API SequentialSampler : public Sampler<> { + public: + /// Creates a `SequentialSampler` that will return indices in the range + /// `0...size - 1`. + explicit SequentialSampler(size_t size); + + /// Resets the `SequentialSampler` to zero. + void reset(std::optional new_size = std::nullopt) override; + + /// Returns the next batch of indices. + std::optional> next(size_t batch_size) override; + + /// Serializes the `SequentialSampler` to the `archive`. + void save(serialize::OutputArchive& archive) const override; + + /// Deserializes the `SequentialSampler` from the `archive`. + void load(serialize::InputArchive& archive) override; + + /// Returns the current index of the `SequentialSampler`. + size_t index() const noexcept; + + private: + size_t size_; + size_t index_{0}; +}; + +} // namespace torch::data::samplers diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/serialize.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/serialize.h new file mode 100644 index 0000000000000000000000000000000000000000..46fc7f916de8fe9c43ea898209df59ba22408c25 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/serialize.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +namespace torch::data::samplers { +/// Serializes a `Sampler` into an `OutputArchive`. +template +serialize::OutputArchive& operator<<( + serialize::OutputArchive& archive, + const Sampler& sampler) { + sampler.save(archive); + return archive; +} + +/// Deserializes a `Sampler` from an `InputArchive`. +template +serialize::InputArchive& operator>>( + serialize::InputArchive& archive, + Sampler& sampler) { + sampler.load(archive); + return archive; +} +} // namespace torch::data::samplers diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/stream.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/stream.h new file mode 100644 index 0000000000000000000000000000000000000000..e651c19d0b929a44850b7e80b0b3821e79b237eb --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/samplers/stream.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace torch::serialize { +class InputArchive; +class OutputArchive; +} // namespace torch::serialize + +namespace torch::data::samplers { + +/// A wrapper around a batch size value, which implements the +/// `CustomBatchRequest` interface. +struct TORCH_API BatchSize : public CustomBatchRequest { + explicit BatchSize(size_t size); + size_t size() const noexcept override; + operator size_t() const noexcept; + size_t size_; +}; + +/// A sampler for (potentially infinite) streams of data. +/// +/// The major feature of the `StreamSampler` is that it does not return +/// particular indices, but instead only the number of elements to fetch from +/// the dataset. The dataset has to decide how to produce those elements. +class TORCH_API StreamSampler : public Sampler { + public: + /// Constructs the `StreamSampler` with the number of individual examples that + /// should be fetched until the sampler is exhausted. + explicit StreamSampler(size_t epoch_size); + + /// Resets the internal state of the sampler. + void reset(std::optional new_size = std::nullopt) override; + + /// Returns a `BatchSize` object with the number of elements to fetch in the + /// next batch. This number is the minimum of the supplied `batch_size` and + /// the difference between the `epoch_size` and the current index. If the + /// `epoch_size` has been reached, returns an empty optional. + std::optional next(size_t batch_size) override; + + /// Serializes the `StreamSampler` to the `archive`. + void save(serialize::OutputArchive& archive) const override; + + /// Deserializes the `StreamSampler` from the `archive`. + void load(serialize::InputArchive& archive) override; + + private: + size_t examples_retrieved_so_far_ = 0; + size_t epoch_size_; +}; + +} // namespace torch::data::samplers diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms.h new file mode 100644 index 0000000000000000000000000000000000000000..5d532dc82ca26f56ad157c07a991b654d59a4e25 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms.h @@ -0,0 +1,7 @@ +#pragma once + +#include +#include +#include +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/base.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/base.h new file mode 100644 index 0000000000000000000000000000000000000000..76798ce5b948a13d2019f28b646a893b5ac45995 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/base.h @@ -0,0 +1,49 @@ +#pragma once + +#include + +#include +#include + +namespace torch::data::transforms { + +/// A transformation of a batch to a new batch. +template +class BatchTransform { + public: + using InputBatchType = InputBatch; + using OutputBatchType = OutputBatch; + + virtual ~BatchTransform() = default; + + /// Applies the transformation to the given `input_batch`. + virtual OutputBatch apply_batch(InputBatch input_batch) = 0; +}; + +/// A transformation of individual input examples to individual output examples. +/// +/// Just like a `Dataset` is a `BatchDataset`, a `Transform` is a +/// `BatchTransform` that can operate on the level of individual examples rather +/// than entire batches. The batch-level transform is implemented (by default) +/// in terms of the example-level transform, though this can be customized. +template +class Transform + : public BatchTransform, std::vector> { + public: + using InputType = Input; + using OutputType = Output; + + /// Applies the transformation to the given `input`. + virtual OutputType apply(InputType input) = 0; + + /// Applies the `transformation` over the entire `input_batch`. + std::vector apply_batch(std::vector input_batch) override { + std::vector output_batch; + output_batch.reserve(input_batch.size()); + for (auto&& input : input_batch) { + output_batch.push_back(apply(std::move(input))); + } + return output_batch; + } +}; +} // namespace torch::data::transforms diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/collate.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/collate.h new file mode 100644 index 0000000000000000000000000000000000000000..252e64496e584bc63bae582103698630405cc7cb --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/collate.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + +#include + +namespace torch::data::transforms { + +/// A `Collation` is a transform that reduces a batch into a single value. +/// The result is a `BatchDataset` that has the type of the single value as its +/// `BatchType`. +template > +using Collation = BatchTransform; + +/// A `Collate` allows passing a custom function to reduce/collate a batch +/// into a single value. It's effectively the lambda version of `Collation`, +/// which you could subclass and override `operator()` to achieve the same. +/// +/// \rst +/// .. code-block:: cpp +/// using namespace torch::data; +/// +/// auto dataset = datasets::MNIST("path/to/mnist") +/// .map(transforms::Collate>([](std::vector> e) { +/// return std::move(e.front()); +/// })); +/// \endrst +template > +using Collate = BatchLambda; +} // namespace torch::data::transforms diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/lambda.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/lambda.h new file mode 100644 index 0000000000000000000000000000000000000000..861fb48cd6fc8ca241fb6b450aad75223f7ac0fe --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/lambda.h @@ -0,0 +1,52 @@ +#pragma once + +#include + +#include +#include +#include + +namespace torch::data::transforms { + +/// A `BatchTransform` that applies a user-provided functor to a batch. +template +class BatchLambda : public BatchTransform { + public: + using typename BatchTransform::InputBatchType; + using typename BatchTransform::OutputBatchType; + using FunctionType = std::function; + + /// Constructs the `BatchLambda` from the given `function` object. + explicit BatchLambda(FunctionType function) + : function_(std::move(function)) {} + + /// Applies the user-provided function object to the `input_batch`. + OutputBatchType apply_batch(InputBatchType input_batch) override { + return function_(std::move(input_batch)); + } + + private: + FunctionType function_; +}; + +// A `Transform` that applies a user-provided functor to individual examples. +template +class Lambda : public Transform { + public: + using typename Transform::InputType; + using typename Transform::OutputType; + using FunctionType = std::function; + + /// Constructs the `Lambda` from the given `function` object. + explicit Lambda(FunctionType function) : function_(std::move(function)) {} + + /// Applies the user-provided function object to the `input`. + OutputType apply(InputType input) override { + return function_(std::move(input)); + } + + private: + FunctionType function_; +}; + +} // namespace torch::data::transforms diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/stack.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/stack.h new file mode 100644 index 0000000000000000000000000000000000000000..0751d73cc2186fda2dd8d67a43f77f78928ee7d8 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/stack.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace torch::data::transforms { + +template > +struct Stack; + +/// A `Collation` for `Example` types that stacks all data +/// tensors into one tensor, and all target (label) tensors into one tensor. +template <> +struct Stack> : public Collation> { + Example<> apply_batch(std::vector> examples) override { + std::vector data, targets; + data.reserve(examples.size()); + targets.reserve(examples.size()); + for (auto& example : examples) { + data.push_back(std::move(example.data)); + targets.push_back(std::move(example.target)); + } + return {torch::stack(data), torch::stack(targets)}; + } +}; + +/// A `Collation` for `Example` types that stacks all data +/// tensors into one tensor. +template <> +struct Stack + : public Collation> { + TensorExample apply_batch(std::vector examples) override { + std::vector data; + data.reserve(examples.size()); + for (auto& example : examples) { + data.push_back(std::move(example.data)); + } + return torch::stack(data); + } +}; +} // namespace torch::data::transforms diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/tensor.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/tensor.h new file mode 100644 index 0000000000000000000000000000000000000000..6a965c5dbffb924310c89d6f92bddb05a18fc797 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/transforms/tensor.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace torch::data::transforms { + +/// A `Transform` that is specialized for the typical `Example` +/// combination. It exposes a single `operator()` interface hook (for +/// subclasses), and calls this function on input `Example` objects. +template +class TensorTransform + : public Transform, Example> { + public: + using E = Example; + using typename Transform::InputType; + using typename Transform::OutputType; + + /// Transforms a single input tensor to an output tensor. + virtual Tensor operator()(Tensor input) = 0; + + /// Implementation of `Transform::apply` that calls `operator()`. + OutputType apply(InputType input) override { + input.data = (*this)(std::move(input.data)); + return input; + } +}; + +/// A `Lambda` specialized for the typical `Example` input type. +template +class TensorLambda : public TensorTransform { + public: + using FunctionType = std::function; + + /// Creates a `TensorLambda` from the given `function`. + explicit TensorLambda(FunctionType function) + : function_(std::move(function)) {} + + /// Applies the user-provided functor to the input tensor. + Tensor operator()(Tensor input) override { + return function_(std::move(input)); + } + + private: + FunctionType function_; +}; + +/// Normalizes input tensors by subtracting the supplied mean and dividing by +/// the given standard deviation. +template +struct Normalize : public TensorTransform { + /// Constructs a `Normalize` transform. The mean and standard deviation can be + /// anything that is broadcastable over the input tensors (like single + /// scalars). + Normalize(ArrayRef mean, ArrayRef stddev) + : mean(torch::tensor(mean, torch::kFloat32) + .unsqueeze(/*dim=*/1) + .unsqueeze(/*dim=*/2)), + stddev(torch::tensor(stddev, torch::kFloat32) + .unsqueeze(/*dim=*/1) + .unsqueeze(/*dim=*/2)) {} + + torch::Tensor operator()(Tensor input) override { + return input.sub(mean).div(stddev); + } + + torch::Tensor mean, stddev; +}; +} // namespace torch::data::transforms diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/worker_exception.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/worker_exception.h new file mode 100644 index 0000000000000000000000000000000000000000..fe65680c15733f94a1cd83882c6b30e8be5acc63 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/data/worker_exception.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +namespace torch::data { + +/// An exception thrown when a DataLoader's worker thread throws an exception, +/// which is caught. A `WorkerException` stores an `exception_ptr` to the +/// original exception thrown in the worker thread. +struct WorkerException : public std::exception { + /// Constructs a `WorkerException` from an `exception_ptr`. + explicit WorkerException(std::exception_ptr original) + // NOLINTNEXTLINE(bugprone-throw-keyword-missing) + : original_exception(std::move(original)), + message("Caught exception in DataLoader worker thread.") { + try { + std::rethrow_exception(original_exception); + } catch (std::exception& e) { + message += " Original message: "; + message += e.what(); + } + } + + const char* what() const noexcept override { + return message.c_str(); + } + + /// The original exception thrown in the worker thread. + std::exception_ptr original_exception; + + /// This exception's message (not the original exception's message). + std::string message; +}; + +} // namespace torch::data diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/detail/TensorDataContainer.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/detail/TensorDataContainer.h new file mode 100644 index 0000000000000000000000000000000000000000..04afd40f02faa9ca5adf0beb8b929441b6510aea --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/detail/TensorDataContainer.h @@ -0,0 +1,349 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#ifndef AT_PER_OPERATOR_HEADERS +#include +#else +#include +#include +#endif + +#include + +namespace torch::detail { + +enum class TensorDataContainerType { Scalar, InitList, Tensor }; + +struct TensorDataContainer; + +inline std::ostream& operator<<( + std::ostream& stream, + const TensorDataContainer& tensor_data_container); + +inline c10::ScalarType compute_desired_dtype(c10::ScalarType scalar_type) { + if (scalar_type == at::kInt || scalar_type == at::kLong) { + // C++ `torch::tensor` with an integer type or an `at::ArrayRef` / + // `std::vector` / (nested) braced-init-list of integer types always + // produces a tensor of dtype `at::kLong` (aka. int64_t), matching Python + // `torch.tensor` behavior. + return at::kLong; + } else if (scalar_type == at::kFloat || scalar_type == at::kDouble) { + // C++ `torch::tensor` with a floating-point type or an `at::ArrayRef` / + // `std::vector` / (nested) braced-init-list of floating-point types always + // produces a tensor of dtype `torch::get_default_dtype()`, matching Python + // `torch.tensor` behavior. + return at::typeMetaToScalarType(at::get_default_dtype()); + } else { + return scalar_type; + } +} + +// We use `TensorDataContainer` to support converting the following data +// container types into the equivalent Tensor: +// +// 1. Arbitrarily nested braced-init-list (e.g. `{{1, 2}, {3, 4}}`). +// 2. `at::ArrayRef` of supported tensor data types. +// 3. `std::vector` of supported tensor data types. +// +// At any time, a `TensorDataContainer` object represents one of the following: +// +// 1. A scalar with value `scalar()` and type `scalar_type()`. +// 2. A Tensor represented in `std::initializer_list` form, +// with value `init_list()`, Tensor scalar type `scalar_type()`, and Tensor +// sizes `sizes()`. +// 3. A Tensor represented in `at::Tensor` form, with value `tensor()`, scalar +// type `scalar_type()`, +// and Tensor sizes `sizes()`. +// +// All the infrastructure here is mostly to support converting an arbitrarily +// nested braced-init-list to the equivalent Tensor successfully. Consider the +// following example: +// +// `torch::tensor({{1}, {2}})` +// +// this will call into the `torch::tensor` function: +// +// `at::Tensor tensor(detail::TensorDataContainer tensor_data_container, const +// at::TensorOptions& options = {})` +// +// the compiler will first try to convert `{{1}, {2}}` to `TensorDataContainer` +// type: +// +// `TensorDataContainer({{1}, {2}})` +// +// which matches to the +// `TensorDataContainer(std::initializer_list)` +// constructor, and in an attempt to convert `{1}` and `{2}` to +// `TensorDataContainer`, it calls the following: +// +// `TensorDataContainer({1})` (same call path happens for `{2}`, and we'll just +// focus on `{1}` here) +// +// At this point, theoretically there are two plausible ways for `{1}` to be +// matched to one of the constructors of `TensorDataContainer`: +// +// 1. It can be a list-initialization of a scalar value, thus matching +// `TensorDataContainer(int value)`. +// 2. It can be converted to `std::initializer_list`, thus +// matching +// `TensorDataContainer(std::initializer_list)`. +// +// How does the compiler decide which one to choose? According to +// `https://en.cppreference.com/w/cpp/language/list_initialization`, +// braced-init-list always prefers the constructor that takes +// `std::initializer_list`. Hence we happily move forward with constructor #2, +// and it calls the following: +// +// `TensorDataContainer(1)` +// +// Now it matches `TensorDataContainer(int value)`, which stores `1` as a scalar +// value. All is good. +struct TensorDataContainer { + // NOTE: For tensors with zero-size dimensions (e.g. `torch::tensor({{}, + // {}})`), the innermost empty braced-init-list `{}` matches the default + // constructor of the innermost `TensorDataContainer`. + TensorDataContainer() + : sizes_({0}), + // NOTE: In Python, the dtype of tensors with zero-size dimensions (e.g. + // `torch.tensor([[], []])`) depends on the value of + // `torch.get_default_dtype()`, and we should do the same for the C++ + // equivalent. + scalar_type_(at::typeMetaToScalarType(at::get_default_dtype())), + type_(TensorDataContainerType::InitList) {} +#define TENSOR(T, S) \ + TensorDataContainer(T value) \ + : scalar_type_(at::k##S), \ + type_(TensorDataContainerType::Scalar), \ + scalar_(value) {} + AT_FORALL_SCALAR_TYPES_AND3(Bool, Half, BFloat16, TENSOR) + AT_FORALL_COMPLEX_TYPES(TENSOR) +#undef TENSOR + TensorDataContainer(std::initializer_list init_list) + : scalar_type_(init_list.begin()->scalar_type()), + type_(TensorDataContainerType::InitList), + init_list_(init_list) { + const TensorDataContainer& first_elem = *(init_list.begin()); + for (const auto& elem : init_list) { + TORCH_CHECK( + elem.sizes() == first_elem.sizes(), + "Expected all sub-lists to have sizes: ", + first_elem.sizes(), + " (e.g. ", + first_elem, + "), ", + "but got sub-list ", + elem, + " with sizes: ", + elem.sizes()); + TORCH_CHECK( + elem.scalar_type() == first_elem.scalar_type(), + "Expected all elements of the tensor to have the same scalar type: ", + first_elem.scalar_type(), + ", but got element of scalar type: ", + elem.scalar_type()); + } + sizes_.reserve(first_elem.sizes().size() + 1); + sizes_.push_back(static_cast(init_list.size())); + sizes_.insert( + sizes_.end(), first_elem.sizes().begin(), first_elem.sizes().end()); + } + +#define TENSOR(T, S) \ + TensorDataContainer(at::ArrayRef values) \ + : sizes_({(int64_t)values.size()}), \ + scalar_type_(at::k##S), \ + type_(TensorDataContainerType::Tensor) { \ + at::AutoDispatchBelowAutograd mode; \ + if (scalar_type_ == at::kBool) { \ + tensor_ = at::tensor(values, at::TensorOptions().device(at::kCPU)); \ + } else { \ + tensor_ = at::tensor(values, at::dtype(scalar_type_).device(at::kCPU)); \ + } \ + } + AT_FORALL_SCALAR_TYPES_AND3(Bool, Half, BFloat16, TENSOR) + AT_FORALL_COMPLEX_TYPES(TENSOR) +#undef TENSOR + + // NOTE: We need to handle `std::vector` explicitly instead of relying on an + // implicit conversion to `at::ArrayRef`, otherwise the following error can be + // thrown when calling `torch::tensor(std::vector({1, 2}))`: + // ``` + // error: no matching function for call to 'tensor(const std::vector&)' + // no known conversion for argument 1 from 'const std::vector' to + // 'torch::detail::TensorDataContainer' + // ``` + // + // NOTE: `torch::tensor(std::vector)` is not supported for now, because + // ArrayRef cannot be constructed from a std::vector bitfield. +#define TENSOR(T, S) \ + TensorDataContainer(const std::vector& values) \ + : TensorDataContainer(at::ArrayRef(values)) {} + AT_FORALL_SCALAR_TYPES_AND2(Half, BFloat16, TENSOR) + AT_FORALL_COMPLEX_TYPES(TENSOR) +#undef TENSOR + + bool is_scalar() const { + return type_ == TensorDataContainerType::Scalar; + } + + const c10::Scalar& scalar() const { + TORCH_CHECK( + is_scalar(), + "Can only call `scalar()` on a TensorDataContainer that has `is_scalar() == true`"); + return scalar_; + } + + bool is_init_list() const { + return type_ == TensorDataContainerType::InitList; + } + + const std::initializer_list& init_list() const { + TORCH_CHECK( + is_init_list(), + "Can only call `init_list()` on a TensorDataContainer that has `is_init_list() == true`"); + return init_list_; + } + + bool is_tensor() const { + return type_ == TensorDataContainerType::Tensor; + } + + const at::Tensor& tensor() const { + TORCH_CHECK( + is_tensor(), + "Can only call `tensor()` on a TensorDataContainer that has `is_tensor() == true`"); + return tensor_; + } + + const std::vector& sizes() const { + return sizes_; + } + + const c10::ScalarType& scalar_type() const { + return scalar_type_; + } + + at::Tensor convert_to_tensor(at::TensorOptions options) const { + if (!options.has_dtype()) { + options = options.dtype(compute_desired_dtype(scalar_type_)); + } + + if (is_scalar()) { + at::AutoDispatchBelowAutograd mode; + return at::scalar_tensor(scalar_, options); + } else if (is_init_list()) { + // NOTE: Here we explicitly choose to initialize the tensor on CPU first, + // fill each element of the tensor, and then move the tensor to the + // desired device. For CUDA device, this approach only involves 1 CUDA + // kernel launch, and is much faster than initializing the tensor on CUDA + // first and then filling each element of it (which involves `N` CUDA + // kernel launches where `N` is the number of the elements in the tensor). + at::Tensor tensor = ([&]() { + at::AutoDispatchBelowAutograd mode; + return at::empty(sizes_, options.device(at::kCPU)); + })(); + fill_tensor(tensor); + return tensor.to(options.device()); + } else if (is_tensor()) { + auto output = tensor_.to(options); + TORCH_CHECK( + !tensor_.is_complex() || output.is_complex(), + "can not do torch::tensor(complex, dtype=non-complex) because complex can not be casted to real number without loss of information"); + return output; + } else { + TORCH_INTERNAL_ASSERT(false, "Invalid TensorDataContainer type"); + } + } + + void pretty_print_recursive(std::ostream& stream) const { + if (is_scalar()) { + AT_DISPATCH_ALL_TYPES_AND3( + at::kBool, + at::kHalf, + at::kBFloat16, + scalar_type_, + "TensorDataContainer_pretty_print_scalar", + [&] { stream << scalar_.to(); }); + } else if (is_init_list()) { + stream << '{'; + for (const TensorDataContainer* it = init_list_.begin(); + it != init_list_.end(); + it++) { + stream << *it; + if (std::next(it) != init_list_.end()) + stream << ", "; + } + stream << '}'; + } else if (is_tensor()) { + stream << '{'; + for (const auto i : c10::irange(tensor_.sizes()[0])) { + AT_DISPATCH_ALL_TYPES_AND3( + at::kBool, + at::kHalf, + at::kBFloat16, + scalar_type_, + "TensorDataContainer_pretty_print_tensor_item", + [&] { stream << tensor_[i].item(); }); + if (i != tensor_.sizes()[0] - 1) + stream << ", "; + } + stream << '}'; + } else { + TORCH_INTERNAL_ASSERT(false, "Invalid TensorDataContainer type"); + } + } + + private: + void fill_tensor(at::Tensor& tensor) const { + if (is_scalar()) { + TORCH_INTERNAL_ASSERT( + tensor.dim() == 0, + "Expected a 0-dim Tensor, but got Tensor with dimensions: ", + tensor.dim()); + at::NoGradGuard guard; + tensor.fill_(scalar_); + } else if (is_init_list()) { + TORCH_INTERNAL_ASSERT( + tensor.sizes()[0] == (int64_t)init_list_.size(), + "Expected a Tensor with size ", + init_list_.size(), + " in its first dimension, but got Tensor with size ", + tensor.sizes()[0], + " in its first dimension"); + int64_t index = 0; + for (const auto& elem : init_list_) { + at::Tensor slice = tensor[index]; + elem.fill_tensor(slice); + index++; + } + } else if (is_tensor()) { + TORCH_INTERNAL_ASSERT( + false, + "TensorDataContainer is already a Tensor type, `fill_tensor` should not be called"); + } else { + TORCH_INTERNAL_ASSERT(false, "Invalid TensorDataContainer type"); + } + } + + std::vector sizes_; + c10::ScalarType scalar_type_; + TensorDataContainerType type_; + c10::Scalar scalar_; + std::initializer_list init_list_; + at::Tensor tensor_; +}; + +inline std::ostream& operator<<( + std::ostream& stream, + const TensorDataContainer& tensor_data_container) { + tensor_data_container.pretty_print_recursive(stream); + return stream; +} + +} // namespace torch::detail diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/detail/static.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/detail/static.h new file mode 100644 index 0000000000000000000000000000000000000000..f269cbc06eff210c45dcddeaa6c02600030cdc80 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/detail/static.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include + +#include +#include + +namespace torch::nn { +class Module; +} // namespace torch::nn + +namespace torch::detail { +/// Detects if a type T has a forward() method. +template +struct has_forward { + // Declare two types with differing size. + using yes = int8_t; + using no = int16_t; + + // Here we declare two functions. The first is only enabled if `&U::forward` + // is well-formed and returns the `yes` type. In C++, the ellipsis parameter + // type (`...`) always puts the function at the bottom of overload resolution. + // This is specified in the standard as: 1) A standard conversion sequence is + // always better than a user-defined conversion sequence or an ellipsis + // conversion sequence. 2) A user-defined conversion sequence is always better + // than an ellipsis conversion sequence This means that if the first overload + // is viable, it will be preferred over the second as long as we pass any + // convertible type. The type of `&U::forward` is a pointer type, so we can + // pass e.g. 0. + template + static yes test(decltype(&U::forward)); + template + static no test(...); + + // Finally we test statically whether the size of the type returned by the + // selected overload is the size of the `yes` type. + static constexpr bool value = (sizeof(test(nullptr)) == sizeof(yes)); +}; + +template +constexpr bool check_not_lvalue_references() { + return (!std::is_lvalue_reference_v || + std::is_const_v>) && + check_not_lvalue_references(); +} + +template <> +inline constexpr bool check_not_lvalue_references() { + return true; +} + +/// A type trait whose `value` member is true if `M` derives from `Module`. +template +using is_module = std::is_base_of>; + +template +using enable_if_module_t = std::enable_if_t::value, T>; +} // namespace torch::detail diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/enum.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/enum.h new file mode 100644 index 0000000000000000000000000000000000000000..d9a564d16afe2ce8b84dccddccce063887f6dd6f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/enum.h @@ -0,0 +1,210 @@ +#pragma once + +#include +#include + +#include +#include +#include + +#define TORCH_ENUM_DECLARE(name) \ + namespace torch { \ + namespace enumtype { \ + /* \ + NOTE: We need to provide the default constructor for each struct, \ + otherwise Clang 3.8 would complain: \ + ``` \ + error: default initialization of an object of const type 'const \ + enumtype::Enum1' without a user-provided default constructor \ + ``` \ + */ \ + struct k##name { \ + k##name() {} \ + }; \ + } \ + TORCH_API extern const enumtype::k##name k##name; \ + } + +#define TORCH_ENUM_DEFINE(name) \ + namespace torch { \ + const enumtype::k##name k##name; \ + } + +#define TORCH_ENUM_PRETTY_PRINT(name) \ + std::string operator()(const enumtype::k##name& v [[maybe_unused]]) const { \ + std::string k("k"); \ + return k + #name; \ + } + +// NOTE: Backstory on why we need the following two macros: +// +// Consider the following options class: +// +// ``` +// struct TORCH_API SomeOptions { +// typedef std::variant +// reduction_t; SomeOptions(reduction_t reduction = torch::kMean) : +// reduction_(reduction) {} +// +// TORCH_ARG(reduction_t, reduction); +// }; +// ``` +// +// and the functional that uses it: +// +// ``` +// Tensor some_functional( +// const Tensor& input, +// SomeOptions options = {}) { +// ... +// } +// ``` +// +// Normally, we would expect this to work: +// +// `F::some_functional(input, torch::kNone)` +// +// However, it throws the following error instead: +// +// ``` +// error: could not convert `torch::kNone` from `const torch::enumtype::kNone` +// to `torch::nn::SomeOptions` +// ``` +// +// To get around this problem, we explicitly provide the following constructors +// for `SomeOptions`: +// +// ``` +// SomeOptions(torch::enumtype::kNone reduction) : reduction_(torch::kNone) {} +// SomeOptions(torch::enumtype::kMean reduction) : reduction_(torch::kMean) {} +// SomeOptions(torch::enumtype::kSum reduction) : reduction_(torch::kSum) {} +// ``` +// +// so that the conversion from `torch::kNone` to `SomeOptions` would work. +// +// Note that we also provide the default constructor `SomeOptions() {}`, so that +// `SomeOptions options = {}` can work. +#define TORCH_OPTIONS_CTOR_VARIANT_ARG3( \ + OPTIONS_NAME, ARG_NAME, TYPE1, TYPE2, TYPE3) \ + OPTIONS_NAME() = default; \ + OPTIONS_NAME(torch::enumtype::TYPE1 ARG_NAME) : ARG_NAME##_(torch::TYPE1) {} \ + OPTIONS_NAME(torch::enumtype::TYPE2 ARG_NAME) : ARG_NAME##_(torch::TYPE2) {} \ + OPTIONS_NAME(torch::enumtype::TYPE3 ARG_NAME) : ARG_NAME##_(torch::TYPE3) {} + +#define TORCH_OPTIONS_CTOR_VARIANT_ARG4( \ + OPTIONS_NAME, ARG_NAME, TYPE1, TYPE2, TYPE3, TYPE4) \ + OPTIONS_NAME() = default; \ + OPTIONS_NAME(torch::enumtype::TYPE1 ARG_NAME) : ARG_NAME##_(torch::TYPE1) {} \ + OPTIONS_NAME(torch::enumtype::TYPE2 ARG_NAME) : ARG_NAME##_(torch::TYPE2) {} \ + OPTIONS_NAME(torch::enumtype::TYPE3 ARG_NAME) : ARG_NAME##_(torch::TYPE3) {} \ + OPTIONS_NAME(torch::enumtype::TYPE4 ARG_NAME) : ARG_NAME##_(torch::TYPE4) {} + +TORCH_ENUM_DECLARE(Linear) +TORCH_ENUM_DECLARE(Conv1D) +TORCH_ENUM_DECLARE(Conv2D) +TORCH_ENUM_DECLARE(Conv3D) +TORCH_ENUM_DECLARE(ConvTranspose1D) +TORCH_ENUM_DECLARE(ConvTranspose2D) +TORCH_ENUM_DECLARE(ConvTranspose3D) +TORCH_ENUM_DECLARE(Sigmoid) +TORCH_ENUM_DECLARE(Tanh) +TORCH_ENUM_DECLARE(ReLU) +TORCH_ENUM_DECLARE(GELU) +TORCH_ENUM_DECLARE(SiLU) +TORCH_ENUM_DECLARE(Mish) +TORCH_ENUM_DECLARE(LeakyReLU) +TORCH_ENUM_DECLARE(FanIn) +TORCH_ENUM_DECLARE(FanOut) +TORCH_ENUM_DECLARE(Constant) +TORCH_ENUM_DECLARE(Reflect) +TORCH_ENUM_DECLARE(Replicate) +TORCH_ENUM_DECLARE(Circular) +TORCH_ENUM_DECLARE(Nearest) +TORCH_ENUM_DECLARE(Bilinear) +TORCH_ENUM_DECLARE(Bicubic) +TORCH_ENUM_DECLARE(Trilinear) +TORCH_ENUM_DECLARE(Area) +TORCH_ENUM_DECLARE(NearestExact) +TORCH_ENUM_DECLARE(Sum) +TORCH_ENUM_DECLARE(Mean) +TORCH_ENUM_DECLARE(Max) +TORCH_ENUM_DECLARE(None) +TORCH_ENUM_DECLARE(BatchMean) +TORCH_ENUM_DECLARE(Zeros) +TORCH_ENUM_DECLARE(Border) +TORCH_ENUM_DECLARE(Reflection) +TORCH_ENUM_DECLARE(RNN_TANH) +TORCH_ENUM_DECLARE(RNN_RELU) +TORCH_ENUM_DECLARE(LSTM) +TORCH_ENUM_DECLARE(GRU) +TORCH_ENUM_DECLARE(Valid) +TORCH_ENUM_DECLARE(Same) + +namespace torch::enumtype { + +struct _compute_enum_name { + TORCH_ENUM_PRETTY_PRINT(Linear) + TORCH_ENUM_PRETTY_PRINT(Conv1D) + TORCH_ENUM_PRETTY_PRINT(Conv2D) + TORCH_ENUM_PRETTY_PRINT(Conv3D) + TORCH_ENUM_PRETTY_PRINT(ConvTranspose1D) + TORCH_ENUM_PRETTY_PRINT(ConvTranspose2D) + TORCH_ENUM_PRETTY_PRINT(ConvTranspose3D) + TORCH_ENUM_PRETTY_PRINT(Sigmoid) + TORCH_ENUM_PRETTY_PRINT(Tanh) + TORCH_ENUM_PRETTY_PRINT(ReLU) + TORCH_ENUM_PRETTY_PRINT(GELU) + TORCH_ENUM_PRETTY_PRINT(SiLU) + TORCH_ENUM_PRETTY_PRINT(Mish) + TORCH_ENUM_PRETTY_PRINT(LeakyReLU) + TORCH_ENUM_PRETTY_PRINT(FanIn) + TORCH_ENUM_PRETTY_PRINT(FanOut) + TORCH_ENUM_PRETTY_PRINT(Constant) + TORCH_ENUM_PRETTY_PRINT(Reflect) + TORCH_ENUM_PRETTY_PRINT(Replicate) + TORCH_ENUM_PRETTY_PRINT(Circular) + TORCH_ENUM_PRETTY_PRINT(Nearest) + TORCH_ENUM_PRETTY_PRINT(Bilinear) + TORCH_ENUM_PRETTY_PRINT(Bicubic) + TORCH_ENUM_PRETTY_PRINT(Trilinear) + TORCH_ENUM_PRETTY_PRINT(Area) + TORCH_ENUM_PRETTY_PRINT(NearestExact) + TORCH_ENUM_PRETTY_PRINT(Sum) + TORCH_ENUM_PRETTY_PRINT(Mean) + TORCH_ENUM_PRETTY_PRINT(Max) + TORCH_ENUM_PRETTY_PRINT(None) + TORCH_ENUM_PRETTY_PRINT(BatchMean) + TORCH_ENUM_PRETTY_PRINT(Zeros) + TORCH_ENUM_PRETTY_PRINT(Border) + TORCH_ENUM_PRETTY_PRINT(Reflection) + TORCH_ENUM_PRETTY_PRINT(RNN_TANH) + TORCH_ENUM_PRETTY_PRINT(RNN_RELU) + TORCH_ENUM_PRETTY_PRINT(LSTM) + TORCH_ENUM_PRETTY_PRINT(GRU) + TORCH_ENUM_PRETTY_PRINT(Valid) + TORCH_ENUM_PRETTY_PRINT(Same) +}; + +template +std::string get_enum_name(V variant_enum) { + return std::visit(enumtype::_compute_enum_name{}, variant_enum); +} + +template +at::Reduction::Reduction reduction_get_enum(V variant_enum) { + if (std::holds_alternative(variant_enum)) { + return at::Reduction::None; + } else if (std::holds_alternative(variant_enum)) { + return at::Reduction::Mean; + } else if (std::holds_alternative(variant_enum)) { + return at::Reduction::Sum; + } else { + TORCH_CHECK( + false, + get_enum_name(variant_enum), + " is not a valid value for reduction"); + return at::Reduction::END; + } +} + +} // namespace torch::enumtype diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/expanding_array.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/expanding_array.h new file mode 100644 index 0000000000000000000000000000000000000000..2b99f32317845d38d18072de9690e2aca9ea31ff --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/expanding_array.h @@ -0,0 +1,182 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace torch { + +/// A utility class that accepts either a container of `D`-many values, or a +/// single value, which is internally repeated `D` times. This is useful to +/// represent parameters that are multidimensional, but often equally sized in +/// all dimensions. For example, the kernel size of a 2D convolution has an `x` +/// and `y` length, but `x` and `y` are often equal. In such a case you could +/// just pass `3` to an `ExpandingArray<2>` and it would "expand" to `{3, 3}`. +template +class ExpandingArray { + public: + /// Constructs an `ExpandingArray` from an `initializer_list`. The extent of + /// the length is checked against the `ExpandingArray`'s extent parameter `D` + /// at runtime. + /*implicit*/ ExpandingArray(std::initializer_list list) + : ExpandingArray(c10::ArrayRef(list)) {} + + /// Constructs an `ExpandingArray` from an `std::vector`. The extent of + /// the length is checked against the `ExpandingArray`'s extent parameter `D` + /// at runtime. + /*implicit*/ ExpandingArray(std::vector vec) + : ExpandingArray(c10::ArrayRef(vec)) {} + + /// Constructs an `ExpandingArray` from an `c10::ArrayRef`. The extent of + /// the length is checked against the `ExpandingArray`'s extent parameter `D` + /// at runtime. + /*implicit*/ ExpandingArray(c10::ArrayRef values) { + // clang-format off + TORCH_CHECK( + values.size() == D, + "Expected ", D, " values, but instead got ", values.size()); + // clang-format on + std::copy(values.begin(), values.end(), values_.begin()); + } + + /// Constructs an `ExpandingArray` from a single value, which is repeated `D` + /// times (where `D` is the extent parameter of the `ExpandingArray`). + /*implicit*/ ExpandingArray(T single_size) { + values_.fill(single_size); + } + + /// Constructs an `ExpandingArray` from a correctly sized `std::array`. + /*implicit*/ ExpandingArray(const std::array& values) + : values_(values) {} + + /// Accesses the underlying `std::array`. + std::array& operator*() { + return values_; + } + + /// Accesses the underlying `std::array`. + const std::array& operator*() const { + return values_; + } + + /// Accesses the underlying `std::array`. + std::array* operator->() { + return &values_; + } + + /// Accesses the underlying `std::array`. + const std::array* operator->() const { + return &values_; + } + + /// Returns an `ArrayRef` to the underlying `std::array`. + operator c10::ArrayRef() const { + return values_; + } + + /// Returns the extent of the `ExpandingArray`. + size_t size() const noexcept { + return D; + } + + protected: + /// The backing array. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::array values_; +}; + +template +std::ostream& operator<<( + std::ostream& stream, + const ExpandingArray& expanding_array) { + if (expanding_array.size() == 1) { + return stream << expanding_array->at(0); + } + return stream << static_cast>(expanding_array); +} + +/// A utility class that accepts either a container of `D`-many +/// `std::optional` values, or a single `std::optional` value, which is +/// internally repeated `D` times. It has the additional ability to accept +/// containers of the underlying type `T` and convert them to a container of +/// `std::optional`. +template +class ExpandingArrayWithOptionalElem + : public ExpandingArray> { + public: + using ExpandingArray>::ExpandingArray; + + /// Constructs an `ExpandingArrayWithOptionalElem` from an `initializer_list` + /// of the underlying type `T`. The extent of the length is checked against + /// the `ExpandingArrayWithOptionalElem`'s extent parameter `D` at runtime. + /*implicit*/ ExpandingArrayWithOptionalElem(std::initializer_list list) + : ExpandingArrayWithOptionalElem(c10::ArrayRef(list)) {} + + /// Constructs an `ExpandingArrayWithOptionalElem` from an `std::vector` of + /// the underlying type `T`. The extent of the length is checked against the + /// `ExpandingArrayWithOptionalElem`'s extent parameter `D` at runtime. + /*implicit*/ ExpandingArrayWithOptionalElem(std::vector vec) + : ExpandingArrayWithOptionalElem(c10::ArrayRef(vec)) {} + + /// Constructs an `ExpandingArrayWithOptionalElem` from an `c10::ArrayRef` of + /// the underlying type `T`. The extent of the length is checked against the + /// `ExpandingArrayWithOptionalElem`'s extent parameter `D` at runtime. + /*implicit*/ ExpandingArrayWithOptionalElem(c10::ArrayRef values) + : ExpandingArray>(0) { + // clang-format off + TORCH_CHECK( + values.size() == D, + "Expected ", D, " values, but instead got ", values.size()); + // clang-format on + for (const auto i : c10::irange(this->values_.size())) { + this->values_[i] = values[i]; + } + } + + /// Constructs an `ExpandingArrayWithOptionalElem` from a single value of the + /// underlying type `T`, which is repeated `D` times (where `D` is the extent + /// parameter of the `ExpandingArrayWithOptionalElem`). + /*implicit*/ ExpandingArrayWithOptionalElem(T single_size) + : ExpandingArray>(0) { + for (const auto i : c10::irange(this->values_.size())) { + this->values_[i] = single_size; + } + } + + /// Constructs an `ExpandingArrayWithOptionalElem` from a correctly sized + /// `std::array` of the underlying type `T`. + /*implicit*/ ExpandingArrayWithOptionalElem(const std::array& values) + : ExpandingArray>(0) { + for (const auto i : c10::irange(this->values_.size())) { + this->values_[i] = values[i]; + } + } +}; + +template +std::ostream& operator<<( + std::ostream& stream, + const ExpandingArrayWithOptionalElem& expanding_array_with_opt_elem) { + if (expanding_array_with_opt_elem.size() == 1) { + const auto& elem = expanding_array_with_opt_elem->at(0); + stream << (elem.has_value() ? c10::str(elem.value()) : "None"); + } else { + std::vector str_array; + for (const auto& elem : *expanding_array_with_opt_elem) { + str_array.emplace_back( + elem.has_value() ? c10::str(elem.value()) : "None"); + } + stream << c10::ArrayRef(str_array); + } + return stream; +} + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/fft.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/fft.h new file mode 100644 index 0000000000000000000000000000000000000000..bc68957f576d931650c3f6d65ec6aeeae2a996f6 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/fft.h @@ -0,0 +1,390 @@ +#pragma once + +#include +#include + +#include + +namespace torch::fft { + +/// Computes the 1 dimensional fast Fourier transform over a given dimension. +/// See https://pytorch.org/docs/main/fft.html#torch.fft.fft. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kComplexDouble); +/// torch::fft::fft(t); +/// ``` +inline Tensor fft( + const Tensor& self, + std::optional n = std::nullopt, + int64_t dim = -1, + std::optional norm = std::nullopt) { + return torch::fft_fft_symint(self, std::move(n), dim, norm); +} + +/// Computes the 1 dimensional inverse Fourier transform over a given dimension. +/// See https://pytorch.org/docs/main/fft.html#torch.fft.ifft. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kComplexDouble); +/// torch::fft::ifft(t); +/// ``` +inline Tensor ifft( + const Tensor& self, + std::optional n = std::nullopt, + int64_t dim = -1, + std::optional norm = std::nullopt) { + return torch::fft_ifft_symint(self, std::move(n), dim, norm); +} + +/// Computes the 2-dimensional fast Fourier transform over the given dimensions. +/// See https://pytorch.org/docs/main/fft.html#torch.fft.fft2. +/// +/// Example: +/// ``` +/// auto t = torch::randn({128, 128}, dtype=kComplexDouble); +/// torch::fft::fft2(t); +/// ``` +inline Tensor fft2( + const Tensor& self, + OptionalIntArrayRef s = std::nullopt, + IntArrayRef dim = {-2, -1}, + std::optional norm = std::nullopt) { + return torch::fft_fft2(self, s, dim, norm); +} + +/// Computes the inverse of torch.fft.fft2 +/// See https://pytorch.org/docs/main/fft.html#torch.fft.ifft2. +/// +/// Example: +/// ``` +/// auto t = torch::randn({128, 128}, dtype=kComplexDouble); +/// torch::fft::ifft2(t); +/// ``` +inline Tensor ifft2( + const Tensor& self, + at::OptionalIntArrayRef s = std::nullopt, + IntArrayRef dim = {-2, -1}, + std::optional norm = std::nullopt) { + return torch::fft_ifft2(self, s, dim, norm); +} + +/// Computes the N dimensional fast Fourier transform over given dimensions. +/// See https://pytorch.org/docs/main/fft.html#torch.fft.fftn. +/// +/// Example: +/// ``` +/// auto t = torch::randn({128, 128}, dtype=kComplexDouble); +/// torch::fft::fftn(t); +/// ``` +inline Tensor fftn( + const Tensor& self, + at::OptionalIntArrayRef s = std::nullopt, + at::OptionalIntArrayRef dim = std::nullopt, + std::optional norm = std::nullopt) { + return torch::fft_fftn(self, s, dim, norm); +} + +/// Computes the N dimensional fast Fourier transform over given dimensions. +/// See https://pytorch.org/docs/main/fft.html#torch.fft.ifftn. +/// +/// Example: +/// ``` +/// auto t = torch::randn({128, 128}, dtype=kComplexDouble); +/// torch::fft::ifftn(t); +/// ``` +inline Tensor ifftn( + const Tensor& self, + at::OptionalIntArrayRef s = std::nullopt, + at::OptionalIntArrayRef dim = std::nullopt, + std::optional norm = std::nullopt) { + return torch::fft_ifftn(self, s, dim, norm); +} + +/// Computes the 1 dimensional FFT of real input with onesided Hermitian output. +/// See https://pytorch.org/docs/main/fft.html#torch.fft.rfft. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128); +/// auto T = torch::fft::rfft(t); +/// assert(T.is_complex() && T.numel() == 128 / 2 + 1); +/// ``` +inline Tensor rfft( + const Tensor& self, + std::optional n = std::nullopt, + int64_t dim = -1, + std::optional norm = std::nullopt) { + return torch::fft_rfft_symint(self, std::move(n), dim, norm); +} + +/// Computes the inverse of torch.fft.rfft +/// +/// The input is a onesided Hermitian Fourier domain signal, with real-valued +/// output. See https://pytorch.org/docs/main/fft.html#torch.fft.irfft +/// +/// Example: +/// ``` +/// auto T = torch::randn(128 / 2 + 1, torch::kComplexDouble); +/// auto t = torch::fft::irfft(t, /*n=*/128); +/// assert(t.is_floating_point() && T.numel() == 128); +/// ``` +inline Tensor irfft( + const Tensor& self, + std::optional n = std::nullopt, + int64_t dim = -1, + std::optional norm = std::nullopt) { + return torch::fft_irfft_symint(self, std::move(n), dim, norm); +} + +/// Computes the 2-dimensional FFT of real input. Returns a onesided Hermitian +/// output. See https://pytorch.org/docs/main/fft.html#torch.fft.rfft2 +/// +/// Example: +/// ``` +/// auto t = torch::randn({128, 128}, dtype=kDouble); +/// torch::fft::rfft2(t); +/// ``` +inline Tensor rfft2( + const Tensor& self, + at::OptionalIntArrayRef s = std::nullopt, + IntArrayRef dim = {-2, -1}, + std::optional norm = std::nullopt) { + return torch::fft_rfft2(self, s, dim, norm); +} + +/// Computes the inverse of torch.fft.rfft2. +/// See https://pytorch.org/docs/main/fft.html#torch.fft.irfft2. +/// +/// Example: +/// ``` +/// auto t = torch::randn({128, 128}, dtype=kComplexDouble); +/// torch::fft::irfft2(t); +/// ``` +inline Tensor irfft2( + const Tensor& self, + at::OptionalIntArrayRef s = std::nullopt, + IntArrayRef dim = {-2, -1}, + std::optional norm = std::nullopt) { + return torch::fft_irfft2(self, s, dim, norm); +} + +/// Computes the N dimensional FFT of real input with onesided Hermitian output. +/// See https://pytorch.org/docs/main/fft.html#torch.fft.rfftn +/// +/// Example: +/// ``` +/// auto t = torch::randn({128, 128}, dtype=kDouble); +/// torch::fft::rfftn(t); +/// ``` +inline Tensor rfftn( + const Tensor& self, + at::OptionalIntArrayRef s = std::nullopt, + at::OptionalIntArrayRef dim = std::nullopt, + std::optional norm = std::nullopt) { + return torch::fft_rfftn(self, s, dim, norm); +} + +/// Computes the inverse of torch.fft.rfftn. +/// See https://pytorch.org/docs/main/fft.html#torch.fft.irfftn. +/// +/// Example: +/// ``` +/// auto t = torch::randn({128, 128}, dtype=kComplexDouble); +/// torch::fft::irfftn(t); +/// ``` +inline Tensor irfftn( + const Tensor& self, + at::OptionalIntArrayRef s = std::nullopt, + at::OptionalIntArrayRef dim = std::nullopt, + std::optional norm = std::nullopt) { + return torch::fft_irfftn(self, s, dim, norm); +} + +/// Computes the 1 dimensional FFT of a onesided Hermitian signal +/// +/// The input represents a Hermitian symmetric time domain signal. The returned +/// Fourier domain representation of such a signal is a real-valued. See +/// https://pytorch.org/docs/main/fft.html#torch.fft.hfft +/// +/// Example: +/// ``` +/// auto t = torch::randn(128 / 2 + 1, torch::kComplexDouble); +/// auto T = torch::fft::hfft(t, /*n=*/128); +/// assert(T.is_floating_point() && T.numel() == 128); +/// ``` +inline Tensor hfft( + const Tensor& self, + std::optional n = std::nullopt, + int64_t dim = -1, + std::optional norm = std::nullopt) { + return torch::fft_hfft_symint(self, std::move(n), dim, norm); +} + +/// Computes the inverse FFT of a real-valued Fourier domain signal. +/// +/// The output is a onesided representation of the Hermitian symmetric time +/// domain signal. See https://pytorch.org/docs/main/fft.html#torch.fft.ihfft. +/// +/// Example: +/// ``` +/// auto T = torch::randn(128, torch::kDouble); +/// auto t = torch::fft::ihfft(T); +/// assert(t.is_complex() && T.numel() == 128 / 2 + 1); +/// ``` +inline Tensor ihfft( + const Tensor& self, + std::optional n = std::nullopt, + int64_t dim = -1, + std::optional norm = std::nullopt) { + return torch::fft_ihfft_symint(self, std::move(n), dim, norm); +} + +/// Computes the 2-dimensional FFT of a Hermitian symmetric input signal. +/// +/// The input is a onesided representation of the Hermitian symmetric time +/// domain signal. See https://pytorch.org/docs/main/fft.html#torch.fft.hfft2. +/// +/// Example: +/// ``` +/// auto t = torch::randn({128, 65}, torch::kComplexDouble); +/// auto T = torch::fft::hfft2(t, /*s=*/{128, 128}); +/// assert(T.is_floating_point() && T.numel() == 128 * 128); +/// ``` +inline Tensor hfft2( + const Tensor& self, + at::OptionalIntArrayRef s = std::nullopt, + IntArrayRef dim = {-2, -1}, + std::optional norm = std::nullopt) { + return torch::fft_hfft2(self, s, dim, norm); +} + +/// Computes the 2-dimensional IFFT of a real input signal. +/// +/// The output is a onesided representation of the Hermitian symmetric time +/// domain signal. See +/// https://pytorch.org/docs/main/fft.html#torch.fft.ihfft2. +/// +/// Example: +/// ``` +/// auto T = torch::randn({128, 128}, torch::kDouble); +/// auto t = torch::fft::hfft2(T); +/// assert(t.is_complex() && t.size(1) == 65); +/// ``` +inline Tensor ihfft2( + const Tensor& self, + at::OptionalIntArrayRef s = std::nullopt, + IntArrayRef dim = {-2, -1}, + std::optional norm = std::nullopt) { + return torch::fft_ihfft2(self, s, dim, norm); +} + +/// Computes the N-dimensional FFT of a Hermitian symmetric input signal. +/// +/// The input is a onesided representation of the Hermitian symmetric time +/// domain signal. See https://pytorch.org/docs/main/fft.html#torch.fft.hfftn. +/// +/// Example: +/// ``` +/// auto t = torch::randn({128, 65}, torch::kComplexDouble); +/// auto T = torch::fft::hfftn(t, /*s=*/{128, 128}); +/// assert(T.is_floating_point() && T.numel() == 128 * 128); +/// ``` +inline Tensor hfftn( + const Tensor& self, + at::OptionalIntArrayRef s = std::nullopt, + IntArrayRef dim = {-2, -1}, + std::optional norm = std::nullopt) { + return torch::fft_hfftn(self, s, dim, norm); +} + +/// Computes the N-dimensional IFFT of a real input signal. +/// +/// The output is a onesided representation of the Hermitian symmetric time +/// domain signal. See +/// https://pytorch.org/docs/main/fft.html#torch.fft.ihfftn. +/// +/// Example: +/// ``` +/// auto T = torch::randn({128, 128}, torch::kDouble); +/// auto t = torch::fft::hfft2(T); +/// assert(t.is_complex() && t.size(1) == 65); +/// ``` +inline Tensor ihfftn( + const Tensor& self, + at::OptionalIntArrayRef s = std::nullopt, + IntArrayRef dim = {-2, -1}, + std::optional norm = std::nullopt) { + return torch::fft_ihfftn(self, s, dim, norm); +} + +/// Computes the discrete Fourier Transform sample frequencies for a signal of +/// size n. +/// +/// See https://pytorch.org/docs/main/fft.html#torch.fft.fftfreq +/// +/// Example: +/// ``` +/// auto frequencies = torch::fft::fftfreq(128, torch::kDouble); +/// ``` +inline Tensor fftfreq(int64_t n, double d, const TensorOptions& options = {}) { + return torch::fft_fftfreq(n, d, options); +} + +inline Tensor fftfreq(int64_t n, const TensorOptions& options = {}) { + return torch::fft_fftfreq(n, /*d=*/1.0, options); +} + +/// Computes the sample frequencies for torch.fft.rfft with a signal of size n. +/// +/// Like torch.fft.rfft, only the positive frequencies are included. +/// See https://pytorch.org/docs/main/fft.html#torch.fft.rfftfreq +/// +/// Example: +/// ``` +/// auto frequencies = torch::fft::rfftfreq(128, torch::kDouble); +/// ``` +inline Tensor rfftfreq(int64_t n, double d, const TensorOptions& options) { + return torch::fft_rfftfreq(n, d, options); +} + +inline Tensor rfftfreq(int64_t n, const TensorOptions& options) { + return torch::fft_rfftfreq(n, /*d=*/1.0, options); +} + +/// Reorders n-dimensional FFT output to have negative frequency terms first, by +/// a torch.roll operation. +/// +/// See https://pytorch.org/docs/main/fft.html#torch.fft.fftshift +/// +/// Example: +/// ``` +/// auto x = torch::randn({127, 4}); +/// auto centred_fft = torch::fft::fftshift(torch::fft::fftn(x)); +/// ``` +inline Tensor fftshift( + const Tensor& x, + at::OptionalIntArrayRef dim = std::nullopt) { + return torch::fft_fftshift(x, dim); +} + +/// Inverse of torch.fft.fftshift +/// +/// See https://pytorch.org/docs/main/fft.html#torch.fft.ifftshift +/// +/// Example: +/// ``` +/// auto x = torch::randn({127, 4}); +/// auto shift = torch::fft::fftshift(x) +/// auto unshift = torch::fft::ifftshift(shift); +/// assert(torch::allclose(x, unshift)); +/// ``` +inline Tensor ifftshift( + const Tensor& x, + at::OptionalIntArrayRef dim = std::nullopt) { + return torch::fft_ifftshift(x, dim); +} + +} // namespace torch::fft diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/imethod.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/imethod.h new file mode 100644 index 0000000000000000000000000000000000000000..2c7f520b157402bccb47cb136a5efe227e2c1a6a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/imethod.h @@ -0,0 +1,53 @@ +#pragma once +#include +#include + +namespace torch { + +class TORCH_API IMethod { + /* + IMethod provides a portable interface for torch methods, whether + they are backed by torchscript or python/deploy. + + This is helpful since torchscript methods provide additional information + (e.g. FunctionSchema, Graph) which aren't available in pure python methods. + + Higher level APIs should prefer depending on this interface rather + than a specific implementation of it, to promote portability and reuse, and + avoid unintentional dependencies on e.g. script methods. + + Note: This API is experimental, and may evolve. + */ + public: + using IValueList = std::vector; + using IValueMap = std::unordered_map; + + IMethod() = default; + IMethod(const IMethod&) = default; + IMethod& operator=(const IMethod&) = default; + IMethod(IMethod&&) noexcept = default; + IMethod& operator=(IMethod&&) noexcept = default; + virtual ~IMethod() = default; + + virtual c10::IValue operator()( + std::vector args, + const IValueMap& kwargs = IValueMap()) const = 0; + + virtual const std::string& name() const = 0; + + // Returns an ordered list of argument names, possible in both + // script and python methods. This is a more portable dependency + // than a ScriptMethod FunctionSchema, which has more information + // than can be generally expected from a python method. + const std::vector& getArgumentNames() const; + + protected: + virtual void setArgumentNames( + std::vector& argumentNames) const = 0; + + private: + mutable bool isArgumentNamesInitialized_{false}; + mutable std::vector argumentNames_; +}; + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/jit.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/jit.h new file mode 100644 index 0000000000000000000000000000000000000000..58e8e8b3a49054c03e904acff2513fc6c26ac2ce --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/jit.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +#include +#include + +namespace torch::jit { + +/// Compiles script code into an executable graph. +/// +/// Takes a string containing functions in script syntax and compiles them into +/// a module (graph). The returned module provides a `run_method` function +/// that may be used to invoke the compiled functions. +/// +/// For example: +/// \rst +/// .. code-block:: cpp +/// +/// auto module = torch::jit::compile(R"JIT( +/// def relu_script(a, b): +/// return torch.relu(a + b) +/// def test_while(a, i): +/// while i < 10: +/// a += a +/// i += 1 +/// return a +/// )JIT"); +/// IValue output = module->run_method("relu_script", a, b); +/// \endrst +TORCH_API std::shared_ptr compile(const std::string& source); + +} // namespace torch::jit diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/mps.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/mps.h new file mode 100644 index 0000000000000000000000000000000000000000..90b8ed8598eddcb5f38289b76e8b96048b04b02c --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/mps.h @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include +#include + +#ifdef __OBJC__ +#include +#include +using MTLCommandBuffer_t = id; +using DispatchQueue_t = dispatch_queue_t; +#else +using MTLCommandBuffer_t = void*; +using DispatchQueue_t = void*; +#endif + +namespace torch::mps { + +/// Returns true if MPS device is available. +bool TORCH_API is_available(); + +/// Sets the RNG seed for the MPS device. +void TORCH_API manual_seed(uint64_t seed); + +/// Waits for all streams on the MPS device to complete. +/// This blocks the calling CPU thread by using the 'waitUntilCompleted()' +/// method to wait for Metal command buffers finish executing all the +/// encoded GPU operations before returning. +void TORCH_API synchronize(); + +/// Submits the currently active command buffer to run on the MPS device. +void TORCH_API commit(); + +/// Get the current command buffer to encode the Metal commands. +MTLCommandBuffer_t TORCH_API get_command_buffer(); + +/// Get the dispatch_queue_t to synchronize encoding the custom kernels +/// with the PyTorch MPS backend. +DispatchQueue_t TORCH_API get_dispatch_queue(); + +} // namespace torch::mps diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nativert/ModelRunnerHandle.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nativert/ModelRunnerHandle.h new file mode 100644 index 0000000000000000000000000000000000000000..4d6d161991292412c9a8c61879a1b12ac083154c --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nativert/ModelRunnerHandle.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace torch::nativert { + +// We don't want to forward declare in general but including ModelRunner will +// pollute the public API namespace too much. Therefore, we just use pimpl an +// incomplete ModelRunner here. +class ModelRunner; + +class TORCH_API ModelRunnerHandle { + public: + ModelRunnerHandle( + const std::string& packagePath, + const std::string& modelName); + + ModelRunnerHandle(ModelRunnerHandle&&) = default; + ModelRunnerHandle& operator=(ModelRunnerHandle&&) = default; + ModelRunnerHandle(const ModelRunnerHandle&) = delete; + ModelRunnerHandle& operator=(const ModelRunnerHandle&) = delete; + ~ModelRunnerHandle(); + + c10::IValue run( + const std::vector& args, + const std::unordered_map& kwargs); + + /** + * A low level API which expects user to always pass in flattened inputs. + * The ownership of the entire input list must be transferred to the + * executor via std::move or in-place construction. + */ + std::vector runWithFlatInputsAndOutputs( + std::vector flatInputs); + + private: + std::unique_ptr impl_; +}; + +} // namespace torch::nativert diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nested.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nested.h new file mode 100644 index 0000000000000000000000000000000000000000..764e93db61acfd3c3e707b86a57560c98ffe069e --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nested.h @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nested { + +/// Nested tensor +/// +/// See +/// https://pytorch.org/docs/main/nested.html#torch.nested.nested_tensor +/// +/// ``` +// implemented on python object to allow torch.nested.nested_tensor to be +// constructed with arbitrarily nested python objects - for now, only arbitrary +// python lists and lists of Tensors +// See torch/csrc/autograd/python_nested_functions_manual.cpp for Python +// implementation +// See here for C++ implementation +inline at::Tensor nested_tensor( + at::TensorList nested_tensor_data, + const at::TensorOptions& options = {}) { + auto out = at::_nested_tensor_from_tensor_list( + nested_tensor_data, + c10::typeMetaToScalarType(options.dtype()), + std::nullopt, + options.device(), + options.pinned_memory()); + if (options.has_requires_grad() && options.requires_grad()) { + out.requires_grad_(true); + } + return out; +} + +inline at::Tensor nested_tensor( + at::ArrayRef nested_tensor_data, + const at::TensorOptions& options = {}) { + for (const auto& tdc : nested_tensor_data) { + TORCH_CHECK( + tdc.is_init_list(), + "nested_tensor() not implemented for these parameters"); + } + // Construct a TensorList using nested_tensor_data + std::vector tensor_list(nested_tensor_data.size()); + std::transform( + nested_tensor_data.begin(), + nested_tensor_data.end(), + tensor_list.begin(), + [&](const detail::TensorDataContainer& tdc) { + return tdc.convert_to_tensor(options); + }); + auto out = at::_nested_tensor_from_tensor_list( + tensor_list, + c10::typeMetaToScalarType(options.dtype()), + std::nullopt, + options.device(), + options.pinned_memory()); + if (options.has_requires_grad() && options.requires_grad()) { + out.requires_grad_(true); + } + return out; +} + +/// As Nested Tensor +/// +/// See +/// https://pytorch.org/docs/main/nested.html#torch.nested.as_nested_tensor +/// +/// ``` +inline at::Tensor as_nested_tensor( + at::TensorList list, + std::optional dtype = std::nullopt, + std::optional device = std::nullopt) { + return at::_nested_tensor_from_tensor_list( + list, dtype, std::nullopt, device, std::nullopt); +} + +/// Nested to padded tensor +/// +/// See +/// https://pytorch.org/docs/main/nested.html#torch.nested.to_padded_tensor +/// +/// ``` +inline at::Tensor to_padded_tensor( + const at::Tensor& self, + double padding, + at::OptionalIntArrayRef output_size = std::nullopt) { + return at::nested_to_padded_tensor(self, padding, output_size); +} + +} // namespace torch::nested diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn.h new file mode 100644 index 0000000000000000000000000000000000000000..2a0d26cd949f8c354364c2b8a793bb060f6c28d2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/cloneable.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/cloneable.h new file mode 100644 index 0000000000000000000000000000000000000000..6e9011827faa103a1f5b4be7f07a0dbdb0993a8d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/cloneable.h @@ -0,0 +1,94 @@ +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include + +namespace torch::nn { +/// The `clone()` method in the base `Module` class does not have knowledge of +/// the concrete runtime type of its subclasses. Therefore, `clone()` must +/// either be called from within the subclass, or from a base class that has +/// knowledge of the concrete type. `Cloneable` uses the CRTP to gain +/// knowledge of the subclass' static type and provide an implementation of the +/// `clone()` method. We do not want to use this pattern in the base class, +/// because then storing a module would always require templatizing it. +template +// NOLINTNEXTLINE(bugprone-exception-escape) +class Cloneable : public Module { + public: + using Module::Module; + + /// `reset()` must perform initialization of all members with reference + /// semantics, most importantly parameters, buffers and submodules. + virtual void reset() = 0; + + /// Performs a recursive "deep copy" of the `Module`, such that all parameters + /// and submodules in the cloned module are different from those in the + /// original module. + std::shared_ptr clone( + const std::optional& device = std::nullopt) const override { + NoGradGuard no_grad; + + const auto& self = static_cast(*this); + auto copy = std::make_shared(self); + copy->parameters_.clear(); + copy->buffers_.clear(); + copy->children_.clear(); + copy->reset(); + TORCH_CHECK( + copy->parameters_.size() == parameters_.size(), + "The cloned module does not have the same number of " + "parameters as the original module after calling reset(). " + "Are you sure you called register_parameter() inside reset() " + "and not the constructor?"); + for (const auto& parameter : named_parameters(/*recurse=*/false)) { + auto& tensor = *parameter; + auto data = device && tensor.device() != *device ? tensor.to(*device) + : tensor.clone(); + copy->parameters_[parameter.key()].set_data(data); + } + TORCH_CHECK( + copy->buffers_.size() == buffers_.size(), + "The cloned module does not have the same number of " + "buffers as the original module after calling reset(). " + "Are you sure you called register_buffer() inside reset() " + "and not the constructor?"); + for (const auto& buffer : named_buffers(/*recurse=*/false)) { + auto& tensor = *buffer; + auto data = device && tensor.device() != *device ? tensor.to(*device) + : tensor.clone(); + copy->buffers_[buffer.key()].set_data(data); + } + TORCH_CHECK( + copy->children_.size() == children_.size(), + "The cloned module does not have the same number of " + "child modules as the original module after calling reset(). " + "Are you sure you called register_module() inside reset() " + "and not the constructor?"); + for (const auto& child : children_) { + copy->children_[child.key()]->clone_(*child.value(), device); + } + return copy; + } + + private: + void clone_(Module& other, const std::optional& device) final { + // Here we are *pretty* certain that `other's` type is `Derived` (because it + // was registered under the same name as `this`), but you never know what + // crazy things `reset()` does, so `dynamic_cast` just to be safe. + auto clone = std::dynamic_pointer_cast(other.clone(device)); + TORCH_CHECK( + clone != nullptr, + "Attempted to clone submodule, but it is of a " + "different type than the submodule it was to be cloned into"); + static_cast(*this) = *clone; + } +}; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional.h new file mode 100644 index 0000000000000000000000000000000000000000..46c705adf8def92827e91b71632c4e61dd5db168 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/activation.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/activation.h new file mode 100644 index 0000000000000000000000000000000000000000..63051cafa2791709e8239cb84169f2b0cd00e19a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/activation.h @@ -0,0 +1,962 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor elu(Tensor input, double alpha, bool inplace) { + if (inplace) { + return torch::elu_(input, alpha); + } else { + return torch::elu(input, alpha); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.elu +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::ELUFuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::elu(x, F::ELUFuncOptions().alpha(0.42).inplace(true)); +/// ``` +inline Tensor elu(Tensor input, const ELUFuncOptions& options = {}) { + return detail::elu(std::move(input), options.alpha(), options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor selu(Tensor input, bool inplace) { + if (inplace) { + return torch::selu_(input); + } else { + return torch::selu(input); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.selu +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::SELUFuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::selu(input, F::SELUFuncOptions(false)); +/// ``` +inline Tensor selu(Tensor input, const SELUFuncOptions& options = {}) { + return detail::selu(std::move(input), options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor hardshrink(const Tensor& input, double lambda) { + return torch::hardshrink(input, lambda); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.hardshrink +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::HardshrinkFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::hardshrink(x, F::HardshrinkFuncOptions().lambda(0.42)); +/// ``` +inline Tensor hardshrink( + const Tensor& input, + const HardshrinkFuncOptions& options = {}) { + return detail::hardshrink(input, options.lambda()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor hardtanh( + Tensor input, + double min_val, + double max_val, + bool inplace) { + if (inplace) { + return torch::hardtanh_(input, min_val, max_val); + } else { + return torch::hardtanh(input, min_val, max_val); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.hardtanh +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::HardtanhFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::hardtanh(x, +/// F::HardtanhFuncOptions().min_val(-1.0).max_val(1.0).inplace(true)); +/// ``` +inline Tensor hardtanh(Tensor input, const HardtanhFuncOptions& options = {}) { + return detail::hardtanh( + std::move(input), + options.min_val(), + options.max_val(), + options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor leaky_relu(Tensor input, double negative_slope, bool inplace) { + if (inplace) { + return torch::leaky_relu_(input, negative_slope); + } else { + return torch::leaky_relu(input, negative_slope); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.leaky_relu +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::LeakyReLUFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::leaky_relu(x, +/// F::LeakyReLUFuncOptions().negative_slope(0.42).inplace(true)); +/// ``` +inline Tensor leaky_relu( + Tensor input, + const LeakyReLUFuncOptions& options = {}) { + return detail::leaky_relu( + std::move(input), options.negative_slope(), options.inplace()); +} + +// ============================================================================ + +inline Tensor logsigmoid(const Tensor& input) { + return torch::log_sigmoid(input); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor gumbel_softmax( + const Tensor& logits, + double tau, + bool hard, + int dim) { + auto gumbels = + -torch::empty_like(logits).exponential_().log(); // ~Gumbel(0,1) + gumbels = (logits + gumbels) / tau; // ~Gumbel(logits, tau) + auto y_soft = gumbels.softmax(dim); + + torch::Tensor ret; + if (hard) { + // Straight through. + auto index = std::get<1>(y_soft.max(dim, /*keepdim=*/true)); + auto y_hard = torch::zeros_like(logits).scatter_(dim, index, 1.0); + ret = y_hard - y_soft.detach() + y_soft; + } else { + ret = y_soft; + } + return ret; +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.gumbel_softmax +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::GumbelSoftmaxFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::gumbel_softmax(logits, F::GumbelSoftmaxFuncOptions().hard(true).dim(-1)); +/// ``` +inline Tensor gumbel_softmax( + const Tensor& logits, + const GumbelSoftmaxFuncOptions& options = {}) { + return detail::gumbel_softmax( + logits, options.tau(), options.hard(), options.dim()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor softmax( + const Tensor& input, + int64_t dim, + std::optional dtype) { + Tensor ret; + + if (dtype == std::nullopt) { + ret = input.softmax(dim); + } else { + ret = input.softmax(dim, dtype); + } + + return ret; +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.softmax +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::SoftmaxFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::softmax(input, F::SoftmaxFuncOptions(1)); +/// ``` +inline Tensor softmax(const Tensor& input, const SoftmaxFuncOptions& options) { + return detail::softmax(input, options.dim(), options.dtype()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor softmin( + const Tensor& input, + int64_t dim, + std::optional dtype) { + Tensor ret; + + if (dtype == std::nullopt) { + ret = (-input).softmax(dim); + } else { + ret = (-input).softmax(dim, dtype); + } + + return ret; +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.softmin +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::SoftminFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::softmin(input, F::SoftminFuncOptions(1)); +/// ``` +inline Tensor softmin(const Tensor& input, const SoftminFuncOptions& options) { + return detail::softmin(input, options.dim(), options.dtype()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor log_softmax( + const Tensor& input, + int64_t dim, + std::optional dtype) { + Tensor ret; + + if (dtype == std::nullopt) { + ret = input.log_softmax(dim); + } else { + ret = input.log_softmax(dim, dtype); + } + + return ret; +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.log_softmax +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::LogSoftmaxFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::log_softmax(input, LogSoftmaxFuncOptions(1)); +/// ``` +inline Tensor log_softmax( + const Tensor& input, + const LogSoftmaxFuncOptions& options) { + return detail::log_softmax(input, options.dim(), options.dtype()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor glu(const Tensor& input, int64_t dim) { + TORCH_CHECK( + input.dim() != 0, + "glu does not support scalars because halving size must be even"); + return torch::glu(input, dim); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.glu +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::GLUFuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::glu(input, GLUFuncOptions(1)); +/// ``` +inline Tensor glu(const Tensor& input, const GLUFuncOptions& options = {}) { + return detail::glu(input, options.dim()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor gelu(const Tensor& input, const std::string& approximate) { + return torch::gelu(input, approximate); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +inline Tensor gelu(const Tensor& input, const GELUFuncOptions& options = {}) { + return detail::gelu(input, options.approximate()); +} + +// ============================================================================ + +inline Tensor silu(const Tensor& input) { + return torch::silu(input); +} + +// ============================================================================ + +inline Tensor mish(const Tensor& input) { + return torch::mish(input); +} + +// ============================================================================ + +inline Tensor prelu(const Tensor& input, const Tensor& weight) { + return torch::prelu(input, weight); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor relu(Tensor input, bool inplace) { + if (inplace) { + return torch::relu_(input); + } else { + return torch::relu(input); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.relu +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::ReLUFuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::relu(x, F::ReLUFuncOptions().inplace(true)); +/// ``` +inline Tensor relu(Tensor input, const ReLUFuncOptions& options = {}) { + return detail::relu(std::move(input), options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor relu6(Tensor input, bool inplace) { + if (inplace) { + return torch::relu6_(input); + } else { + return torch::relu6(input); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.relu6 +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::ReLU6FuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::relu6(x, F::ReLU6FuncOptions().inplace(true)); +/// ``` +inline Tensor relu6(Tensor input, const ReLU6FuncOptions& options = {}) { + return detail::relu6(std::move(input), options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor rrelu( + Tensor input, + double lower, + double upper, + bool training, + bool inplace) { + if (inplace) { + return torch::rrelu_(input, lower, upper, training); + } else { + return torch::rrelu(input, lower, upper, training); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.rrelu +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::RReLUFuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::rrelu(x, F::RReLUFuncOptions().lower(0.1).upper(0.4).inplace(true)); +/// ``` +inline Tensor rrelu(Tensor input, const RReLUFuncOptions& options = {}) { + return detail::rrelu( + std::move(input), + options.lower(), + options.upper(), + options.training(), + options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor celu(Tensor input, double alpha, bool inplace) { + if (inplace) { + return torch::celu_(input, alpha); + } else { + return torch::celu(input, alpha); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.celu +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::CELUFuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::celu(x, F::CELUFuncOptions().alpha(0.42).inplace(true)); +/// ``` +inline Tensor celu(Tensor input, const CELUFuncOptions& options = {}) { + return detail::celu(std::move(input), options.alpha(), options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor softplus(const Tensor& input, double beta, double threshold) { + return torch::softplus(input, beta, threshold); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.softplus +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::SoftplusFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::softplus(x, F::SoftplusFuncOptions().beta(0.5).threshold(3.0)); +/// ``` +inline Tensor softplus( + const Tensor& input, + const SoftplusFuncOptions& options = {}) { + return detail::softplus(input, options.beta(), options.threshold()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor softshrink(const Tensor& input, double lambda) { + return torch::softshrink(input, lambda); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.softshrink +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::SoftshrinkFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::softshrink(x, F::SoftshrinkFuncOptions(0.42)); +/// ``` +inline Tensor softshrink( + const Tensor& input, + const SoftshrinkFuncOptions& options = {}) { + return detail::softshrink(input, options.lambda()); +} + +// ============================================================================ + +inline Tensor softsign(const Tensor& input) { + return input / (input.abs() + 1); +} + +// ============================================================================ + +inline Tensor tanhshrink(const Tensor& input) { + return input - input.tanh(); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor threshold( + Tensor input, + double threshold, + double value, + bool inplace) { + if (inplace) { + return torch::threshold_(input, threshold, value); + } else { + return torch::threshold(input, threshold, value); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.threshold +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::ThresholdFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::threshold(x, F::ThresholdFuncOptions(0.5, 0.5).inplace(true)); +/// ``` +inline Tensor threshold(Tensor input, const ThresholdFuncOptions& options) { + return detail::threshold( + std::move(input), + options.threshold(), + options.value(), + options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline std::tuple multi_head_attention_forward( + const Tensor& query, + const Tensor& key, + const Tensor& value, + int64_t embed_dim_to_check, + int64_t num_heads, + const Tensor& in_proj_weight, + const Tensor& in_proj_bias, + const Tensor& bias_k, + const Tensor& bias_v, + bool add_zero_attn, + double dropout_p, + const Tensor& out_proj_weight, + const Tensor& out_proj_bias, + bool training = true, + const Tensor& key_padding_mask = {}, + bool need_weights = true, + const Tensor& attn_mask = {}, + bool use_separate_proj_weight = false, + const Tensor& q_proj_weight = {}, + const Tensor& k_proj_weight = {}, + const Tensor& v_proj_weight = {}, + const Tensor& static_k = {}, + const Tensor& static_v = {}, + bool average_attn_weights = true) { + namespace F = torch::nn::functional; + + const auto query_sizes = query.sizes(); + const auto& tgt_len = query_sizes[0]; + const auto& bsz = query_sizes[1]; + const auto& embed_dim = query_sizes[2]; + TORCH_INTERNAL_ASSERT(embed_dim == embed_dim_to_check); + TORCH_INTERNAL_ASSERT(key.sizes() == value.sizes()); + + const auto head_dim = embed_dim / num_heads; + TORCH_CHECK( + head_dim * num_heads == embed_dim, + "embed_dim must be divisible by num_heads"); + const auto scaling = 1 / std::sqrt(head_dim); + + Tensor q, k, v; + if (!use_separate_proj_weight) { + if (torch::equal(query, key) && torch::equal(key, value)) { + // self-attention + const auto chunks = + F::linear(query, in_proj_weight, in_proj_bias).chunk(3, /*dim=*/-1); + q = chunks[0]; + k = chunks[1]; + v = chunks[2]; + } else if (torch::equal(key, value)) { + // encoder-decoder attention + // This is inline in_proj function with in_proj_weight and in_proj_bias + auto _b = in_proj_bias; + int64_t _start = 0; + auto _end = embed_dim; + auto _w = in_proj_weight.slice(/*dim=*/0, _start, _end); + if (_b.defined()) { + _b = _b.slice(/*dim=*/0, _start, _end); + } + q = F::linear(query, _w, _b); + + if (!key.defined()) { + TORCH_INTERNAL_ASSERT(!value.defined()); + k.reset(); + v.reset(); + } else { + // This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias; + _start = embed_dim; + _w = in_proj_weight.slice(/*dim=*/0, _start); + if (_b.defined()) { + _b = _b.slice(/*dim=*/0, _start); + } + const auto chunks = F::linear(key, _w, _b).chunk(2, /*dim=*/-1); + k = chunks[0]; + v = chunks[1]; + } + } else { + // This is inline in_proj function with in_proj_weight and in_proj_bias + auto _b = in_proj_bias; + int64_t _start = 0; + auto _end = embed_dim; + auto _w = in_proj_weight.slice(/*dim=*/0, _start, _end); + if (_b.defined()) { + _b = _b.slice(/*dim=*/0, _start, _end); + } + q = F::linear(query, _w, _b); + + // This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias; + _start = embed_dim; + _end = embed_dim * 2; + _w = in_proj_weight.slice(/*dim=*/0, _start, _end); + if (_b.defined()) { + _b = _b.slice(/*dim=*/0, _start, _end); + } + k = F::linear(key, _w, _b); + + // This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias; + _start = embed_dim * 2; + _w = in_proj_weight.slice(/*dim=*/0, _start); + if (_b.defined()) { + _b = _b.slice(0, _start); + } + v = F::linear(value, _w, _b); + } + } else { + const auto& q_proj_weight_non_opt = q_proj_weight; + { + const auto sizes = q_proj_weight_non_opt.sizes(); + const auto len1 = sizes[0]; + const auto len2 = sizes[1]; + TORCH_CHECK(len1 == embed_dim && len2 == query.size(-1)); + } + + const auto& k_proj_weight_non_opt = k_proj_weight; + { + const auto sizes = k_proj_weight_non_opt.sizes(); + const auto len1 = sizes[0]; + const auto len2 = sizes[1]; + TORCH_CHECK(len1 == embed_dim && len2 == key.size(-1)); + } + + const auto& v_proj_weight_non_opt = v_proj_weight; + { + const auto sizes = v_proj_weight_non_opt.sizes(); + const auto len1 = sizes[0]; + const auto len2 = sizes[1]; + TORCH_CHECK(len1 == embed_dim && len2 == value.size(-1)); + } + + if (in_proj_bias.defined()) { + q = F::linear( + query, + q_proj_weight_non_opt, + in_proj_bias.slice(/*dim=*/0, 0, embed_dim)); + k = F::linear( + key, + k_proj_weight_non_opt, + in_proj_bias.slice(/*dim=*/0, embed_dim, (embed_dim * 2))); + v = F::linear( + value, + v_proj_weight_non_opt, + in_proj_bias.slice(/*dim=*/0, (embed_dim * 2))); + } else { + q = F::linear(query, q_proj_weight_non_opt, in_proj_bias); + k = F::linear(key, k_proj_weight_non_opt, in_proj_bias); + v = F::linear(value, v_proj_weight_non_opt, in_proj_bias); + } + } + q = q * scaling; + Tensor attn_mask_ = attn_mask; + Tensor key_padding_mask_ = key_padding_mask; + if (bias_k.defined() && bias_v.defined()) { + if (!static_k.defined() && !static_v.defined()) { + k = torch::cat({k, bias_k.repeat({1, bsz, 1})}); + v = torch::cat({v, bias_v.repeat({1, bsz, 1})}); + if (attn_mask_.defined()) { + attn_mask_ = torch::cat( + {attn_mask_, + torch::zeros( + {attn_mask_.size(0), 1}, + at::TensorOptions(attn_mask_.dtype()) + .device(attn_mask_.device()))}, + /*dim=*/1); + } + if (key_padding_mask_.defined()) { + key_padding_mask_ = torch::cat( + {key_padding_mask_, + torch::zeros( + {key_padding_mask_.size(0), 1}, + at::TensorOptions(key_padding_mask_.dtype()) + .device(key_padding_mask_.device()))}, + /*dim=*/1); + } + } else { + TORCH_CHECK(!static_k.defined(), "bias cannot be added to static key."); + TORCH_CHECK(!static_v.defined(), "bias cannot be added to static value."); + } + } else { + TORCH_CHECK(!bias_k.defined()); + TORCH_CHECK(!bias_v.defined()); + } + q = q.contiguous().view({tgt_len, bsz * num_heads, head_dim}).transpose(0, 1); + if (k.defined()) { + k = k.contiguous().view({-1, bsz * num_heads, head_dim}).transpose(0, 1); + } + if (v.defined()) { + v = v.contiguous().view({-1, bsz * num_heads, head_dim}).transpose(0, 1); + } + if (static_k.defined()) { + TORCH_CHECK(static_k.size(0) == bsz * num_heads); + TORCH_CHECK(static_k.size(2) == head_dim); + k = static_k; + } + if (static_v.defined()) { + TORCH_CHECK(static_v.size(0) == bsz * num_heads); + TORCH_CHECK(static_v.size(2) == head_dim); + v = static_v; + } + auto src_len = k.size(1); + if (key_padding_mask_.defined()) { + TORCH_CHECK(key_padding_mask_.size(0) == bsz); + TORCH_CHECK(key_padding_mask_.size(1) == src_len); + } + if (add_zero_attn) { + src_len += 1; + auto k_sizes = k.sizes().vec(); + k_sizes[1] = 1; + k = torch::cat( + {k, + torch::zeros( + k_sizes, at::TensorOptions(k.dtype()).device(k.device()))}, + /*dim=*/1); + auto v_sizes = v.sizes().vec(); + v_sizes[1] = 1; + v = torch::cat( + {v, + torch::zeros( + v_sizes, at::TensorOptions(v.dtype()).device(v.device()))}, + /*dim=*/1); + if (attn_mask_.defined()) { + attn_mask_ = torch::cat( + {attn_mask_, + torch::zeros( + {attn_mask_.size(0), 1}, + at::TensorOptions(attn_mask_.dtype()) + .device(attn_mask_.device()))}, + /*dim=*/1); + } + if (key_padding_mask_.defined()) { + key_padding_mask_ = torch::cat( + {key_padding_mask_, + torch::zeros( + {key_padding_mask_.size(0), 1}, + at::TensorOptions(key_padding_mask_.dtype()) + .device(key_padding_mask_.device()))}, + /*dim=*/1); + } + } + auto attn_output_weights = torch::bmm(q, k.transpose(1, 2)); + TORCH_CHECK( + attn_output_weights.sizes() == + IntArrayRef({bsz * num_heads, tgt_len, src_len})); + if (attn_mask_.defined()) { + attn_mask_ = attn_mask_.unsqueeze(0); + attn_output_weights += attn_mask_; + } + if (key_padding_mask_.defined()) { + attn_output_weights = + attn_output_weights.view({bsz, num_heads, tgt_len, src_len}); + attn_output_weights = AT_DISPATCH_FLOATING_TYPES( + attn_output_weights.scalar_type(), + "attn_output_weights.masked_fill", + [&]() { + return attn_output_weights.masked_fill( + key_padding_mask_.unsqueeze(1).unsqueeze(2), + -std::numeric_limits::infinity()); + }); + attn_output_weights = + attn_output_weights.view({bsz * num_heads, tgt_len, src_len}); + } + attn_output_weights = F::softmax(attn_output_weights, /*options=*/-1); + attn_output_weights = F::dropout( + attn_output_weights, + F::DropoutFuncOptions().p(dropout_p).training(training)); + auto attn_output = torch::bmm(attn_output_weights, v); + TORCH_CHECK( + attn_output.sizes() == IntArrayRef({bsz * num_heads, tgt_len, head_dim})); + attn_output = + attn_output.transpose(0, 1).contiguous().view({tgt_len, bsz, embed_dim}); + attn_output = F::linear(attn_output, out_proj_weight, out_proj_bias); + if (need_weights) { + attn_output_weights = + attn_output_weights.view({bsz, num_heads, tgt_len, src_len}); + if (average_attn_weights) { + // average attention weights over heads + attn_output_weights = attn_output_weights.sum(/*dim=*/1) / num_heads; + } + return std::make_tuple( + std::move(attn_output), std::move(attn_output_weights)); + } else { + return std::make_tuple(std::move(attn_output), Tensor()); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +inline std::tuple multi_head_attention_forward( + const Tensor& query, + const Tensor& key, + const Tensor& value, + const MultiheadAttentionForwardFuncOptions& options) { + return detail::multi_head_attention_forward( + query, + key, + value, + options.embed_dim_to_check(), + options.num_heads(), + options.in_proj_weight(), + options.in_proj_bias(), + options.bias_k(), + options.bias_v(), + options.add_zero_attn(), + options.dropout_p(), + options.out_proj_weight(), + options.out_proj_bias(), + options.training(), + options.key_padding_mask(), + options.need_weights(), + options.attn_mask(), + options.use_separate_proj_weight(), + options.q_proj_weight(), + options.k_proj_weight(), + options.v_proj_weight(), + options.static_k(), + options.static_v(), + options.average_attn_weights()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/batchnorm.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/batchnorm.h new file mode 100644 index 0000000000000000000000000000000000000000..4c9d3f7a698de0a1f59634ab4c5d912eb6200488 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/batchnorm.h @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor batch_norm( + const Tensor& input, + const Tensor& running_mean, + const Tensor& running_var, + Tensor weight, + Tensor bias, + bool training, + double momentum, + double eps) { + TORCH_CHECK( + input.dim() >= 2, + "Expected at least 2 input dimensions, but got ", + input.dim()); + if (training) { + auto size = input.sizes(); + int64_t size_prods = size[0]; + for (const auto i : c10::irange(size.size() - 2)) { + size_prods *= size[i + 2]; + } + TORCH_CHECK( + size_prods != 1, + "Expected more than 1 value per channel when training, got input size ", + size); + } + + return torch::batch_norm( + input, + weight, + bias, + running_mean, + running_var, + training, + momentum, + eps, + at::globalContext().userEnabledCuDNN()); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.batch_norm +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::BatchNormFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::batch_norm(input, mean, variance, +/// F::BatchNormFuncOptions().weight(weight).bias(bias).momentum(0.1).eps(1e-05).training(false)); +/// ``` +inline Tensor batch_norm( + const Tensor& input, + const Tensor& running_mean, + const Tensor& running_var, + const BatchNormFuncOptions& options = {}) { + return detail::batch_norm( + input, + running_mean, + running_var, + options.weight(), + options.bias(), + options.training(), + options.momentum(), + options.eps()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/conv.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/conv.h new file mode 100644 index 0000000000000000000000000000000000000000..70b5cb0aa7e1392d23094a0f8f2926344e774d6b --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/conv.h @@ -0,0 +1,297 @@ +#pragma once + +#include +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { + +inline std::string padding_unwrap(enumtype::kValid /*unused*/) { + return "valid"; +} + +inline std::string padding_unwrap(enumtype::kSame /*unused*/) { + return "same"; +} + +template +IntArrayRef padding_unwrap(const ExpandingArray& array) { + return array; +} + +inline Tensor conv1d( + const Tensor& input, + const Tensor& weight, + const Tensor& bias, + ExpandingArray<1> stride, + const Conv1dFuncOptions::padding_t& padding, + ExpandingArray<1> dilation, + int64_t groups) { + return std::visit( + [&](const auto& pad) { + return torch::conv1d( + input, weight, bias, stride, padding_unwrap(pad), dilation, groups); + }, + padding); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.conv1d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::Conv1dFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv1d(x, weight, F::Conv1dFuncOptions().stride(1)); +/// ``` +inline Tensor conv1d( + const Tensor& input, + const Tensor& weight, + const Conv1dFuncOptions& options = {}) { + return detail::conv1d( + input, + weight, + options.bias(), + options.stride(), + options.padding(), + options.dilation(), + options.groups()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor conv2d( + const Tensor& input, + const Tensor& weight, + const Tensor& bias, + ExpandingArray<2> stride, + const Conv2dFuncOptions::padding_t& padding, + ExpandingArray<2> dilation, + int64_t groups) { + return std::visit( + [&](const auto& pad) { + return torch::conv2d( + input, weight, bias, stride, padding_unwrap(pad), dilation, groups); + }, + padding); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.conv2d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::Conv2dFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv2d(x, weight, F::Conv2dFuncOptions().stride(1)); +/// ``` +inline Tensor conv2d( + const Tensor& input, + const Tensor& weight, + const Conv2dFuncOptions& options = {}) { + return detail::conv2d( + input, + weight, + options.bias(), + options.stride(), + options.padding(), + options.dilation(), + options.groups()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor conv3d( + const Tensor& input, + const Tensor& weight, + const Tensor& bias, + ExpandingArray<3> stride, + const Conv3dFuncOptions::padding_t& padding, + ExpandingArray<3> dilation, + int64_t groups) { + return std::visit( + [&](const auto& pad) { + return torch::conv3d( + input, weight, bias, stride, padding_unwrap(pad), dilation, groups); + }, + padding); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.conv3d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::Conv3dFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv3d(x, weight, F::Conv3dFuncOptions().stride(1)); +/// ``` +inline Tensor conv3d( + const Tensor& input, + const Tensor& weight, + const Conv3dFuncOptions& options = {}) { + return detail::conv3d( + input, + weight, + options.bias(), + options.stride(), + options.padding(), + options.dilation(), + options.groups()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor conv_transpose1d( + const Tensor& input, + const Tensor& weight, + const Tensor& bias, + IntArrayRef stride, + IntArrayRef padding, + IntArrayRef output_padding, + int64_t groups, + IntArrayRef dilation) { + return torch::conv_transpose1d( + input, weight, bias, stride, padding, output_padding, groups, dilation); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.conv_transpose1d +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::ConvTranspose1dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv_transpose1d(x, weight, F::ConvTranspose1dFuncOptions().stride(1)); +/// ``` +inline Tensor conv_transpose1d( + const Tensor& input, + const Tensor& weight, + const ConvTranspose1dFuncOptions& options = {}) { + return detail::conv_transpose1d( + input, + weight, + options.bias(), + options.stride(), + options.padding(), + options.output_padding(), + options.groups(), + options.dilation()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor conv_transpose2d( + const Tensor& input, + const Tensor& weight, + const Tensor& bias, + IntArrayRef stride, + IntArrayRef padding, + IntArrayRef output_padding, + int64_t groups, + IntArrayRef dilation) { + return torch::conv_transpose2d( + input, weight, bias, stride, padding, output_padding, groups, dilation); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.conv_transpose2d +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::ConvTranspose2dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv_transpose2d(x, weight, F::ConvTranspose2dFuncOptions().stride(1)); +/// ``` +inline Tensor conv_transpose2d( + const Tensor& input, + const Tensor& weight, + const ConvTranspose2dFuncOptions& options = {}) { + return detail::conv_transpose2d( + input, + weight, + options.bias(), + options.stride(), + options.padding(), + options.output_padding(), + options.groups(), + options.dilation()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor conv_transpose3d( + const Tensor& input, + const Tensor& weight, + const Tensor& bias, + IntArrayRef stride, + IntArrayRef padding, + IntArrayRef output_padding, + int64_t groups, + IntArrayRef dilation) { + return torch::conv_transpose3d( + input, weight, bias, stride, padding, output_padding, groups, dilation); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.conv_transpose3d +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::ConvTranspose3dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv_transpose3d(x, weight, F::ConvTranspose3dFuncOptions().stride(1)); +/// ``` +inline Tensor conv_transpose3d( + const Tensor& input, + const Tensor& weight, + const ConvTranspose3dFuncOptions& options = {}) { + return detail::conv_transpose3d( + input, + weight, + options.bias(), + options.stride(), + options.padding(), + options.output_padding(), + options.groups(), + options.dilation()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/distance.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/distance.h new file mode 100644 index 0000000000000000000000000000000000000000..a0fdc3ccc91e6a971d5a11c5a7503f625496e067 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/distance.h @@ -0,0 +1,84 @@ +#pragma once + +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor cosine_similarity( + const Tensor& x1, + const Tensor& x2, + int64_t dim, + double eps) { + return torch::cosine_similarity(x1, x2, dim, eps); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.cosine_similarity +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::CosineSimilarityFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::cosine_similarity(input1, input2, +/// F::CosineSimilarityFuncOptions().dim(1)); +/// ``` +inline Tensor cosine_similarity( + const Tensor& x1, + const Tensor& x2, + const CosineSimilarityFuncOptions& options = {}) { + return detail::cosine_similarity(x1, x2, options.dim(), options.eps()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor pairwise_distance( + const Tensor& x1, + const Tensor& x2, + double p, + double eps, + bool keepdim) { + return torch::pairwise_distance(x1, x2, p, eps, keepdim); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.pairwise_distance +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::PairwiseDistanceFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::pairwise_distance(input1, input2, F::PairwiseDistanceFuncOptions().p(1)); +/// ``` +inline Tensor pairwise_distance( + const Tensor& x1, + const Tensor& x2, + const PairwiseDistanceFuncOptions& options = {}) { + return detail::pairwise_distance( + x1, x2, options.p(), options.eps(), options.keepdim()); +} + +// ============================================================================ + +/// Computes the p-norm distance between every pair of row vectors in the input. +/// This function will be faster if the rows are contiguous. +inline Tensor pdist(const Tensor& input, double p = 2.0) { + return torch::pdist(input, p); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/dropout.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/dropout.h new file mode 100644 index 0000000000000000000000000000000000000000..0c3893241da81f63a7f518152f46b79b8b491af5 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/dropout.h @@ -0,0 +1,230 @@ +#pragma once + +#include + +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { + +inline Tensor dropout(Tensor input, double p, bool training, bool inplace) { + TORCH_CHECK( + p >= 0. && p <= 1., + "dropout probability has to be between 0 and 1, but got ", + p); + if (inplace) { + return torch::dropout_(input, p, training); + } else { + return torch::dropout(input, p, training); + } +} + +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.dropout +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::DropoutFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::dropout(input, F::DropoutFuncOptions().p(0.5)); +/// ``` +inline Tensor dropout(Tensor input, const DropoutFuncOptions& options = {}) { + return detail::dropout( + std::move(input), options.p(), options.training(), options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { + +template +inline Tensor _dropoutNd_helper( + Tensor input, + double p, + bool training, + bool inplace, + const char* fn_name) { + TORCH_CHECK( + p >= 0. && p <= 1., + "dropout probability has to be between 0 and 1, but got ", + p); + + auto inp_dim = input.dim(); + auto is_batched = inp_dim == batched_dim; + if (!is_batched) { + if (inplace) { + input = input.unsqueeze_(0); + } else { + input = input.unsqueeze(0); + } + } + + Tensor result; + if (inplace) { + result = torch::feature_dropout_(input, p, training); + } else { + result = torch::feature_dropout(input, p, training); + } + + if (!is_batched) { + if (inplace) { + result = result.squeeze_(0); + } else { + result = result.squeeze(0); + } + } + return result; +} + +inline Tensor dropout2d(Tensor input, double p, bool training, bool inplace) { + return _dropoutNd_helper<3, 4>( + std::move(input), p, training, inplace, "dropout2d"); +} + +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.dropout2d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::Dropout2dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::dropout2d(input, F::Dropout2dFuncOptions().p(0.5)); +/// ``` +inline Tensor dropout2d( + Tensor input, + const Dropout2dFuncOptions& options = {}) { + return detail::dropout2d( + std::move(input), options.p(), options.training(), options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { + +inline Tensor dropout3d(Tensor input, double p, bool training, bool inplace) { + return _dropoutNd_helper<4, 5>( + std::move(input), p, training, inplace, "dropout3d"); +} + +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.dropout3d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::Dropout3dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::dropout3d(input, F::Dropout3dFuncOptions().p(0.5)); +/// ``` +inline Tensor dropout3d( + Tensor input, + const Dropout3dFuncOptions& options = {}) { + return detail::dropout3d( + std::move(input), options.p(), options.training(), options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { + +inline Tensor alpha_dropout( + Tensor input, + double p, + bool training, + bool inplace) { + if (p < 0. || p > 1.) { + TORCH_CHECK( + false, "dropout probability has to be between 0 and 1, but got ", p); + } + return inplace ? torch::alpha_dropout_(input, p, training) + : torch::alpha_dropout(input, p, training); +} + +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.alpha_dropout +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::AlphaDropoutFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::alpha_dropout(input, +/// F::AlphaDropoutFuncOptions().p(0.5).training(false)); +/// ``` +inline Tensor alpha_dropout( + Tensor input, + const AlphaDropoutFuncOptions& options = {}) { + return detail::alpha_dropout( + std::move(input), options.p(), options.training(), options.inplace()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { + +inline Tensor feature_alpha_dropout( + Tensor input, + double p, + bool training, + bool inplace) { + if (p < 0. || p > 1.) { + TORCH_CHECK( + false, "dropout probability has to be between 0 and 1, but got ", p); + } + return inplace ? torch::feature_alpha_dropout_(input, p, training) + : torch::feature_alpha_dropout(input, p, training); +} + +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.feature_alpha_dropout +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::FeatureAlphaDropoutFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::feature_alpha_dropout(input, +/// F::FeatureAlphaDropoutFuncOptions().p(0.5).training(false)); +/// ``` +inline Tensor feature_alpha_dropout( + Tensor input, + const FeatureAlphaDropoutFuncOptions& options = {}) { + return detail::feature_alpha_dropout( + std::move(input), options.p(), options.training(), options.inplace()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/embedding.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/embedding.h new file mode 100644 index 0000000000000000000000000000000000000000..db97ec422107ab0eb3d8a78cd19ccd99707d97c3 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/embedding.h @@ -0,0 +1,206 @@ +#pragma once + +#include + +namespace torch::nn::functional { + +inline Tensor one_hot(const Tensor& tensor, int64_t num_classes = -1) { + return torch::one_hot(tensor, num_classes); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline void _no_grad_embedding_renorm_( + Tensor weight, + const Tensor& input, + float max_norm, + float norm_type) { + torch::NoGradGuard no_grad; + torch::embedding_renorm_(weight, input, max_norm, norm_type); +} + +inline Tensor embedding( + const Tensor& input, + const Tensor& weight, + std::optional padding_idx, + std::optional max_norm, + double norm_type, + bool scale_grad_by_freq, + bool sparse) { + auto input_ = input; + + if (padding_idx != std::nullopt) { + if (*padding_idx > 0) { + TORCH_CHECK( + *padding_idx < weight.size(0), + "Padding_idx must be within num_embeddings"); + } else if (*padding_idx < 0) { + TORCH_CHECK( + *padding_idx >= -weight.size(0), + "Padding_idx must be within num_embedding"); + padding_idx = weight.size(0) + *padding_idx; + } + } else { + padding_idx = -1; + } + + if (max_norm != std::nullopt) { + input_ = input_.contiguous(); + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + _no_grad_embedding_renorm_(weight, input_, *max_norm, norm_type); + } + return torch::embedding( + weight, input_, *padding_idx, scale_grad_by_freq, sparse); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.embedding +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::EmbeddingFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::embedding(input, weight, +/// F::EmbeddingFuncOptions().norm_type(2.5).scale_grad_by_freq(true).sparse(true)); +/// ``` +inline Tensor embedding( + const Tensor& input, + const Tensor& weight, + const EmbeddingFuncOptions& options = {}) { + return detail::embedding( + input, + weight, + options.padding_idx(), + options.max_norm(), + options.norm_type(), + options.scale_grad_by_freq(), + options.sparse()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor embedding_bag( + const Tensor& input, + const Tensor& weight, + const Tensor& offsets, + std::optional max_norm, + double norm_type, + bool scale_grad_by_freq, + EmbeddingBagMode mode, + bool sparse, + const Tensor& per_sample_weights, + bool include_last_offset, + std::optional padding_idx) { + auto input_ = input; + auto offsets_ = offsets; + auto per_sample_weights_ = per_sample_weights; + TORCH_CHECK( + !per_sample_weights_.defined() || + input_.sizes() == per_sample_weights_.sizes(), + "embedding_bag: If per_sample_weights (", + per_sample_weights_.sizes(), + ") is not null, then it must have the same shape as the input (", + input_.sizes(), + ")"); + if (input_.dim() == 2) { + TORCH_CHECK( + !offsets_.defined(), + "If input is 2D, then offsets has to be null, as input is treated is a mini-batch of fixed length sequences. However, found offsets of type Tensor"); + offsets_ = torch::arange( + 0, + input_.numel(), + input_.size(1), + torch::TensorOptions().dtype(torch::kLong).device(input_.device())); + input_ = input_.reshape(-1); + if (per_sample_weights_.defined()) { + per_sample_weights_ = per_sample_weights_.reshape(-1); + } + } else if (input_.dim() == 1) { + TORCH_CHECK( + offsets_.defined(), "offsets has to be a 1D Tensor but got null"); + TORCH_CHECK(offsets_.dim() == 1, "offsets has to be a 1D Tensor"); + } else { + TORCH_CHECK( + false, + "input has to be 1D or 2D Tensor, but got Tensor of dimension ", + input_.dim()); + } + + int mode_enum = 0; + if (std::holds_alternative(mode)) { + mode_enum = 0; + } else if (std::holds_alternative(mode)) { + mode_enum = 1; + } else if (std::holds_alternative(mode)) { + mode_enum = 2; + TORCH_CHECK( + !scale_grad_by_freq, + "max mode does not support scaling the gradient by the frequency"); + TORCH_CHECK(!sparse, "max mode does not support sparse weights"); + } else { + TORCH_CHECK(false, "mode has to be one of sum, mean or max"); + } + + if (max_norm != std::nullopt) { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) + _no_grad_embedding_renorm_(weight, input_, *max_norm, norm_type); + } + + TORCH_CHECK( + !per_sample_weights_.defined() || std::get_if(&mode), + "embedding_bag: per_sample_weights was not null. ", + "per_sample_weights is only supported for mode='kSum' (got mode='", + torch::enumtype::get_enum_name(mode), + "').Please open a feature request on GitHub."); + + return std::get<0>(torch::embedding_bag( + weight, + input_, + offsets_, + scale_grad_by_freq, + mode_enum, + sparse, + per_sample_weights_, + include_last_offset, + padding_idx)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.embedding_bag +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::EmbeddingBagFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::embedding_bag(input, weight, +/// F::EmbeddingBagFuncOptions().mode(torch::kSum).offsets(offsets)); +/// ``` +inline Tensor embedding_bag( + const Tensor& input, + const Tensor& weight, + const EmbeddingBagFuncOptions& options = {}) { + return detail::embedding_bag( + input, + weight, + options.offsets(), + options.max_norm(), + options.norm_type(), + options.scale_grad_by_freq(), + options.mode(), + options.sparse(), + options.per_sample_weights(), + options.include_last_offset(), + options.padding_idx()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/fold.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/fold.h new file mode 100644 index 0000000000000000000000000000000000000000..227924eecdb7b65315e2cd597b2858e6c0184e40 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/fold.h @@ -0,0 +1,98 @@ +#pragma once + +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor fold( + const Tensor& input, + ExpandingArray<2> output_size, + ExpandingArray<2> kernel_size, + ExpandingArray<2> dilation, + ExpandingArray<2> padding, + ExpandingArray<2> stride) { + if (input.dim() == 3 || input.dim() == 2) { + return torch::col2im( + input, output_size, kernel_size, dilation, padding, stride); + } else { + TORCH_CHECK( + false, + "Input Error: Only unbatched (2D) or batched (3D) input Tensors are supported " + "(got ", + input.dim(), + "D)"); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.fold +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::FoldFuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::fold(input, F::FoldFuncOptions({3, 2}, {2, 2})); +/// ``` +inline Tensor fold(const Tensor& input, const FoldFuncOptions& options) { + return detail::fold( + input, + options.output_size(), + options.kernel_size(), + options.dilation(), + options.padding(), + options.stride()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor unfold( + const Tensor& input, + ExpandingArray<2> kernel_size, + ExpandingArray<2> dilation, + ExpandingArray<2> padding, + ExpandingArray<2> stride) { + if (input.dim() == 4) { + return torch::im2col(input, kernel_size, dilation, padding, stride); + } else { + TORCH_CHECK( + false, + "Input Error: Only 4D input Tensors are supported " + "(got ", + input.dim(), + "D)"); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.unfold +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::UnfoldFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::unfold(input, F::UnfoldFuncOptions({2, 2}).padding(1).stride(2)); +/// ``` +inline Tensor unfold(const Tensor& input, const UnfoldFuncOptions& options) { + return detail::unfold( + input, + options.kernel_size(), + options.dilation(), + options.padding(), + options.stride()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/instancenorm.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/instancenorm.h new file mode 100644 index 0000000000000000000000000000000000000000..8142c0a9e339a9ed5c271ae84c6645ce8a4d2e1a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/instancenorm.h @@ -0,0 +1,59 @@ +#pragma once + +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor instance_norm( + const Tensor& input, + const Tensor& running_mean, + const Tensor& running_var, + const Tensor& weight, + const Tensor& bias, + bool use_input_stats, + double momentum, + double eps) { + return torch::instance_norm( + input, + weight, + bias, + running_mean, + running_var, + use_input_stats, + momentum, + eps, + at::globalContext().userEnabledCuDNN()); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.instance_norm +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::InstanceNormFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::instance_norm(input, +/// F::InstanceNormFuncOptions().running_mean(mean).running_var(variance).weight(weight).bias(bias).momentum(0.1).eps(1e-5)); +/// ``` +inline Tensor instance_norm( + const Tensor& input, + const InstanceNormFuncOptions& options = {}) { + return detail::instance_norm( + input, + options.running_mean(), + options.running_var(), + options.weight(), + options.bias(), + options.use_input_stats(), + options.momentum(), + options.eps()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/linear.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/linear.h new file mode 100644 index 0000000000000000000000000000000000000000..22afd656f1b2103bcce84204aa09108449390461 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/linear.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +namespace torch::nn::functional { + +inline Tensor bilinear( + const Tensor& input1, + const Tensor& input2, + const Tensor& weight, + const Tensor& bias = Tensor()) { + return torch::bilinear(input1, input2, weight, bias); +} + +// ============================================================================ + +inline Tensor linear( + const Tensor& input, + const Tensor& weight, + const Tensor& bias = {}) { + if (input.dim() == 2 && bias.defined()) { + // fused op is marginally faster + return torch::addmm(bias, input, weight.t()); + } else { + auto output = input.matmul(weight.t()); + if (bias.defined()) { + output += bias; + } + return output; + } +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/loss.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/loss.h new file mode 100644 index 0000000000000000000000000000000000000000..6ab5374def51c44225568c2e022959555e4d7f12 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/loss.h @@ -0,0 +1,1039 @@ +#pragma once + +#include +#include +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor l1_loss( + const Tensor& input, + const Tensor& target, + L1LossFuncOptions::reduction_t reduction) { + return torch::l1_loss(input, target, enumtype::reduction_get_enum(reduction)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.l1_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::L1LossFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::l1_loss(input, target, F::L1LossFuncOptions(torch::kNone)); +/// ``` +inline Tensor l1_loss( + const Tensor& input, + const Tensor& target, + const L1LossFuncOptions& options = {}) { + return detail::l1_loss(input, target, options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor kl_div( + const Tensor& input, + const Tensor& target, + KLDivFuncOptions::reduction_t reduction, + bool log_target = false) { + torch::Reduction::Reduction reduction_enum{}; + + if (std::holds_alternative(reduction)) { + TORCH_WARN( + "reduction: 'mean' divides the total loss by both the batch size and the support size." + "'batchmean' divides only by the batch size, and aligns with the KL div math definition." + "'mean' will be changed to behave the same as 'batchmean' in the next major release."); + } + + // special case for batchmean + if (std::holds_alternative(reduction)) { + reduction_enum = torch::Reduction::Sum; + } else { + reduction_enum = enumtype::reduction_get_enum(reduction); + } + + auto reduced = torch::kl_div(input, target, reduction_enum, log_target); + + if (std::holds_alternative(reduction) && + input.dim() != 0) { + reduced = reduced / input.sizes()[0]; + } + + return reduced; +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.kl_div +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::KLDivFuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::kl_div(input, target, +/// F::KLDivFuncOptions.reduction(torch::kNone).log_target(false)); +/// ``` +inline Tensor kl_div( + const Tensor& input, + const Tensor& target, + const KLDivFuncOptions& options = {}) { + return detail::kl_div( + input, target, options.reduction(), options.log_target()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor mse_loss( + const Tensor& input, + const Tensor& target, + MSELossFuncOptions::reduction_t reduction) { + if (!(target.sizes() == input.sizes())) { + TORCH_WARN( + "Using a target size (", + target.sizes(), + ") that is different to the input size (", + input.sizes(), + "). ", + "This will likely lead to incorrect results due to broadcasting. ", + "Please ensure they have the same size."); + } + std::vector broadcast_tensors = + torch::broadcast_tensors({input, target}); + auto expanded_input = broadcast_tensors[0]; + auto expanded_target = broadcast_tensors[1]; + return torch::mse_loss( + expanded_input, expanded_target, enumtype::reduction_get_enum(reduction)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.mse_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::MSELossFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::mse_loss(input, target, F::MSELossFuncOptions(torch::kNone)); +/// ``` +inline Tensor mse_loss( + const Tensor& input, + const Tensor& target, + const MSELossFuncOptions& options = {}) { + return detail::mse_loss(input, target, options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor binary_cross_entropy( + const Tensor& input, + const Tensor& target, + const Tensor& weight, + BinaryCrossEntropyFuncOptions::reduction_t reduction) { + auto reduction_enum = enumtype::reduction_get_enum(reduction); + + if (target.sizes() != input.sizes()) { + TORCH_CHECK( + false, + "Using a target size (", + target.sizes(), + ") ", + "that is different to the input size (", + input.sizes(), + ") is deprecated. ", + "Please ensure they have the same size."); + } + + auto weight_ = weight; + if (weight_.defined()) { + auto new_size = at::infer_size(target.sizes(), weight_.sizes()); + weight_ = weight_.expand(new_size); + } + + return torch::binary_cross_entropy(input, target, weight_, reduction_enum); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.binary_cross_entropy +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::BinaryCrossEntropyFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::binary_cross_entropy(input, target, +/// F::BinaryCrossEntropyFuncOptions().weight(weight)); +/// ``` +inline Tensor binary_cross_entropy( + const Tensor& input, + const Tensor& target, + const BinaryCrossEntropyFuncOptions& options = {}) { + return detail::binary_cross_entropy( + input, target, options.weight(), options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor hinge_embedding_loss( + const Tensor& input, + const Tensor& target, + double margin, + HingeEmbeddingLossFuncOptions::reduction_t reduction) { + return torch::hinge_embedding_loss( + input, target, margin, enumtype::reduction_get_enum(reduction)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.hinge_embedding_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::HingeEmbeddingLossFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::hinge_embedding_loss(input, target, +/// F::HingeEmbeddingLossFuncOptions().margin(2)); +/// ``` +inline Tensor hinge_embedding_loss( + const Tensor& input, + const Tensor& target, + const HingeEmbeddingLossFuncOptions& options = {}) { + return detail::hinge_embedding_loss( + input, target, options.margin(), options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor multi_margin_loss( + const Tensor& input, + const Tensor& target, + int64_t p, + double margin, + const Tensor& weight, + MultiMarginLossFuncOptions::reduction_t reduction) { + TORCH_CHECK(p == 1 || p == 2, "only p == 1 and p == 2 supported"); + if (weight.defined()) { + TORCH_CHECK(weight.dim() == 1, "weight must be one-dimensional"); + } + + return torch::multi_margin_loss( + input, + target, + p, + margin, + weight, + enumtype::reduction_get_enum(reduction)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.multi_margin_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::MultiMarginLossFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::multi_margin_loss(input, target, +/// F::MultiMarginLossFuncOptions().margin(2).weight(weight)); +/// ``` +inline Tensor multi_margin_loss( + const Tensor& input, + const Tensor& target, + const MultiMarginLossFuncOptions& options = {}) { + return detail::multi_margin_loss( + input, + target, + options.p(), + options.margin(), + options.weight(), + options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor cosine_embedding_loss( + const Tensor& input1, + const Tensor& input2, + const Tensor& target, + double margin, + CosineEmbeddingLossFuncOptions::reduction_t reduction) { + return torch::cosine_embedding_loss( + input1, input2, target, margin, enumtype::reduction_get_enum(reduction)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.cosine_embedding_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::CosineEmbeddingLossFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::cosine_embedding_loss(input1, input2, target, +/// F::CosineEmbeddingLossFuncOptions().margin(0.5)); +/// ``` +inline Tensor cosine_embedding_loss( + const Tensor& input1, + const Tensor& input2, + const Tensor& target, + const CosineEmbeddingLossFuncOptions& options = {}) { + return detail::cosine_embedding_loss( + input1, input2, target, options.margin(), options.reduction()); +} + +// ============================================================================ + +inline Tensor _smooth_l1_loss( + const Tensor& input, + const Tensor& target, + double beta = 1.) { + auto t = torch::abs(input - target); + return torch::where(t < beta, 0.5 * torch::pow(t, 2) / beta, t - 0.5 * beta); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor smooth_l1_loss( + const Tensor& input, + const Tensor& target, + SmoothL1LossFuncOptions::reduction_t reduction, + std::optional beta_opt = std::nullopt) { + if (target.sizes() != input.sizes()) { + TORCH_WARN( + "Using a target size (", + target.sizes(), + ") that is different to the input size (", + input.sizes(), + "). ", + "This will likely lead to incorrect results due to broadcasting. ", + "Please ensure they have the same size."); + } + double beta = beta_opt.value_or(1.0); + + std::vector expanded_tensors = + torch::broadcast_tensors({input, target}); + return torch::smooth_l1_loss( + expanded_tensors[0], + expanded_tensors[1], + enumtype::reduction_get_enum(reduction), + beta); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.smooth_l1_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::SmoothL1LossFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::smooth_l1_loss(input, target, F::SmoothL1LossFuncOptions(torch::kNone)); +/// ``` +inline Tensor smooth_l1_loss( + const Tensor& input, + const Tensor& target, + const SmoothL1LossFuncOptions& options = {}) { + return detail::smooth_l1_loss( + input, target, options.reduction(), options.beta()); +} + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.smooth_l1_loss +/// about the exact behavior of this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::smooth_l1_loss(input, target, /*options=*/torch::kNone, /*beta=*/0.5); +/// ``` +inline Tensor smooth_l1_loss( + const Tensor& input, + const Tensor& target, + const SmoothL1LossFuncOptions& options, + double beta) { + TORCH_CHECK( + !options.beta().has_value(), + "expected beta not to be provided in 'options', but got ", + options.beta()); + return detail::smooth_l1_loss(input, target, options.reduction(), beta); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor huber_loss( + const Tensor& input, + const Tensor& target, + HuberLossFuncOptions::reduction_t reduction, + double delta = 1.) { + if (target.sizes() != input.sizes()) { + TORCH_WARN( + "Using a target size (", + target.sizes(), + ") that is different to the input size (", + input.sizes(), + "). ", + "This will likely lead to incorrect results due to broadcasting. ", + "Please ensure they have the same size."); + } + + std::vector expanded_tensors = + torch::broadcast_tensors({input, target}); + return torch::huber_loss( + expanded_tensors[0], + expanded_tensors[1], + enumtype::reduction_get_enum(reduction), + delta); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.huber_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::HuberLossFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::huber_loss(input, target, +/// F::HuberLossFuncOptions().reduction(torch::kNone).delta(0.5)); +/// ``` +inline Tensor huber_loss( + const Tensor& input, + const Tensor& target, + const HuberLossFuncOptions& options = {}) { + return detail::huber_loss( + input, target, options.reduction(), options.delta()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor multilabel_margin_loss( + const Tensor& input, + const Tensor& target, + MultilabelMarginLossFuncOptions::reduction_t reduction) { + return torch::multilabel_margin_loss( + input, target, enumtype::reduction_get_enum(reduction)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.multilabel_margin_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::MultilabelMarginLossFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::multilabel_margin_loss(input, target, +/// F::MultilabelMarginLossFuncOptions(torch::kNone)); +/// ``` +inline Tensor multilabel_margin_loss( + const Tensor& input, + const Tensor& target, + const MultilabelMarginLossFuncOptions& options = {}) { + return detail::multilabel_margin_loss(input, target, options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor soft_margin_loss( + const Tensor& input, + const Tensor& target, + SoftMarginLossFuncOptions::reduction_t reduction) { + return torch::soft_margin_loss( + input, target, enumtype::reduction_get_enum(reduction)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.soft_margin_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::SoftMarginLossFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::soft_margin_loss(input, target, +/// F::SoftMarginLossFuncOptions(torch::kNone)); +/// ``` +inline Tensor soft_margin_loss( + const Tensor& input, + const Tensor& target, + const SoftMarginLossFuncOptions& options = {}) { + return detail::soft_margin_loss(input, target, options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor multilabel_soft_margin_loss( + const Tensor& input, + const Tensor& target, + const Tensor& weight, + MultilabelSoftMarginLossFuncOptions::reduction_t reduction) { + auto loss = + -(target * torch::log_sigmoid(input) + + (1 - target) * torch::log_sigmoid(-input)); + if (weight.defined()) { + loss = loss * weight; + } + + auto class_dim = input.dim() - 1; + auto C = input.size(class_dim); + loss = loss.sum(class_dim) / C; // only return N loss values + + Tensor ret; + + if (std::holds_alternative(reduction)) { + ret = loss; + } else if (std::holds_alternative(reduction)) { + ret = loss.mean(); + } else if (std::holds_alternative(reduction)) { + ret = loss.sum(); + } else { + ret = input; + TORCH_INTERNAL_ASSERT( + false, enumtype::get_enum_name(reduction), " is not valid"); + } + return ret; +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.multilabel_soft_margin_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::MultilabelSoftMarginLossFuncOptions` class to learn +/// what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::multilabel_soft_margin_loss(input, target, +/// F::MultilabelSoftMarginLossFuncOptions().reduction(torch::kNone).weight(weight)); +/// ``` +inline Tensor multilabel_soft_margin_loss( + const Tensor& input, + const Tensor& target, + const MultilabelSoftMarginLossFuncOptions& options = {}) { + return detail::multilabel_soft_margin_loss( + input, target, options.weight(), options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor triplet_margin_loss( + const Tensor& anchor, + const Tensor& positive, + const Tensor& negative, + double margin, + double p, + double eps, + bool swap, + TripletMarginLossFuncOptions::reduction_t reduction) { + return torch::triplet_margin_loss( + anchor, + positive, + negative, + margin, + p, + eps, + swap, + enumtype::reduction_get_enum(reduction)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.triplet_margin_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::TripletMarginLossFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::triplet_margin_loss(anchor, positive, negative, +/// F::TripletMarginLossFuncOptions().margin(1.0)); +/// ``` +inline Tensor triplet_margin_loss( + const Tensor& anchor, + const Tensor& positive, + const Tensor& negative, + const TripletMarginLossFuncOptions& options = {}) { + return detail::triplet_margin_loss( + anchor, + positive, + negative, + options.margin(), + options.p(), + options.eps(), + options.swap(), + options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor triplet_margin_with_distance_loss( + const Tensor& anchor, + const Tensor& positive, + const Tensor& negative, + std::optional + distance_function, + double margin, + bool swap, + TripletMarginWithDistanceLossFuncOptions::reduction_t reduction) { + Tensor dist_pos, dist_neg; + if (distance_function.has_value()) { + auto distance_function_impl = distance_function.value(); + dist_pos = distance_function_impl(anchor, positive); + dist_neg = distance_function_impl(anchor, negative); + } else { + dist_pos = pairwise_distance(anchor, positive); + dist_neg = pairwise_distance(anchor, negative); + } + + if (swap) { + Tensor dist_swap; + if (distance_function.has_value()) { + dist_swap = distance_function.value()(positive, negative); + } else { + dist_swap = pairwise_distance(positive, negative); + } + dist_neg = torch::min(dist_neg, dist_swap); + } + + auto loss = torch::clamp_min(dist_pos - dist_neg + margin, 0); + + Tensor ret; + if (std::holds_alternative(reduction)) { + ret = loss; + } else if (std::holds_alternative(reduction)) { + ret = loss.mean(); + } else if (std::holds_alternative(reduction)) { + ret = loss.sum(); + } else { + ret = anchor; + TORCH_INTERNAL_ASSERT( + false, enumtype::get_enum_name(reduction), " is not valid"); + } + return ret; +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.triplet_margin_with_distance_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::TripletMarginWithDistanceLossFuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::triplet_margin_with_distance_loss(anchor, positive, negative, +/// F::TripletMarginWithDistanceLossFuncOptions().margin(1.0)); +/// ``` +inline Tensor triplet_margin_with_distance_loss( + const Tensor& anchor, + const Tensor& positive, + const Tensor& negative, + const TripletMarginWithDistanceLossFuncOptions& options = {}) { + return detail::triplet_margin_with_distance_loss( + anchor, + positive, + negative, + options.distance_function(), + options.margin(), + options.swap(), + options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor ctc_loss( + const Tensor& log_probs, + const Tensor& targets, + const Tensor& input_lengths, + const Tensor& target_lengths, + int64_t blank, + CTCLossFuncOptions::reduction_t reduction, + bool zero_infinity) { + return torch::ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + enumtype::reduction_get_enum(reduction), + zero_infinity); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.ctc_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::CTCLossFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::ctc_loss(log_probs, targets, input_lengths, target_lengths, +/// F::CTCLossFuncOptions().reduction(torch::kNone)); +/// ``` +inline Tensor ctc_loss( + const Tensor& log_probs, + const Tensor& targets, + const Tensor& input_lengths, + const Tensor& target_lengths, + const CTCLossFuncOptions& options = {}) { + return detail::ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + options.blank(), + options.reduction(), + options.zero_infinity()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor poisson_nll_loss( + const Tensor& input, + const Tensor& target, + bool log_input, + bool full, + double eps, + PoissonNLLLossFuncOptions::reduction_t reduction) { + return torch::poisson_nll_loss( + input, + target, + log_input, + full, + eps, + enumtype::reduction_get_enum(reduction)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.poisson_nll_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::PoissonNLLLossFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::poisson_nll_loss(input, target, +/// F::PoissonNLLLossFuncOptions().reduction(torch::kNone)); +/// ``` +inline Tensor poisson_nll_loss( + const Tensor& input, + const Tensor& target, + const PoissonNLLLossFuncOptions& options = {}) { + return detail::poisson_nll_loss( + input, + target, + options.log_input(), + options.full(), + options.eps(), + options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor margin_ranking_loss( + const Tensor& input1, + const Tensor& input2, + const Tensor& target, + double margin, + MarginRankingLossFuncOptions::reduction_t reduction) { + TORCH_CHECK( + input1.dim() == input2.dim() && input1.dim() == target.dim(), + "margin_ranking_loss : All input tensors should have same dimension but got sizes: " + "input1: ", + input1.sizes(), + ", input2: ", + input2.sizes(), + ", target: ", + target.sizes()); + return torch::margin_ranking_loss( + input1, input2, target, margin, enumtype::reduction_get_enum(reduction)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.margin_ranking_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::MarginRankingLossFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::margin_ranking_loss(input1, input2, target, +/// F::MarginRankingLossFuncOptions().margin(0.5).reduction(torch::kSum)); +/// ``` +inline Tensor margin_ranking_loss( + const Tensor& input1, + const Tensor& input2, + const Tensor& target, + const MarginRankingLossFuncOptions& options = {}) { + return detail::margin_ranking_loss( + input1, input2, target, options.margin(), options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor nll_loss( + const Tensor& input, + const Tensor& target, + const Tensor& weight, + int64_t ignore_index, + const NLLLossFuncOptions::reduction_t& reduction) { + if (input.dim() < 2) { + TORCH_CHECK(false, "Expected 2 or more dimensions (got ", input.dim(), ")"); + } + + if (input.sizes()[0] != target.sizes()[0]) { + TORCH_CHECK( + false, + "Expected input batch_size (", + input.sizes()[0], + ") to match target batch_size (", + target.sizes()[0], + ")."); + } + + return torch::nll_loss_nd( + input, + target, + weight, + enumtype::reduction_get_enum(reduction), + ignore_index); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.nll_loss +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::NLLLossFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::nll_loss(input, target, +/// F::NLLLossFuncOptions().ignore_index(-100).reduction(torch::kMean)); +/// ``` +inline Tensor nll_loss( + const Tensor& input, + const Tensor& target, + const NLLLossFuncOptions& options = {}) { + return detail::nll_loss( + input, + target, + options.weight(), + options.ignore_index(), + options.reduction()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor cross_entropy( + const Tensor& input, + const Tensor& target, + const Tensor& weight, + int64_t ignore_index, + CrossEntropyFuncOptions::reduction_t reduction, + double label_smoothing) { + return torch::cross_entropy_loss( + input, + target, + weight, + enumtype::reduction_get_enum(reduction), + ignore_index, + label_smoothing); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.cross_entropy +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::CrossEntropyFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::cross_entropy(input, target, +/// F::CrossEntropyFuncOptions().ignore_index(-100).reduction(torch::kMean)); +/// ``` +inline Tensor cross_entropy( + const Tensor& input, + const Tensor& target, + const CrossEntropyFuncOptions& options = {}) { + return detail::cross_entropy( + input, + target, + options.weight(), + options.ignore_index(), + options.reduction(), + options.label_smoothing()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor binary_cross_entropy_with_logits( + const Tensor& input, + const Tensor& target, + const Tensor& weight, + BinaryCrossEntropyWithLogitsFuncOptions::reduction_t reduction, + const Tensor& pos_weight) { + TORCH_CHECK( + target.sizes() == input.sizes(), + "Target size (", + target.sizes(), + ") must be the same as input size (", + input.sizes(), + ")"); + + return torch::binary_cross_entropy_with_logits( + input, + target, + weight, + pos_weight, + enumtype::reduction_get_enum(reduction)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.binary_cross_entropy_with_logits +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::BinaryCrossEntropyWithLogitsFuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::binary_cross_entropy_with_logits(input, target, +/// F::BinaryCrossEntropyWithLogitsFuncOptions().pos_weight(pos_weight).reduction(torch::kSum)); +/// ``` +inline Tensor binary_cross_entropy_with_logits( + const Tensor& input, + const Tensor& target, + const BinaryCrossEntropyWithLogitsFuncOptions& options = {}) { + return detail::binary_cross_entropy_with_logits( + input, + target, + options.weight(), + options.reduction(), + options.pos_weight()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/normalization.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/normalization.h new file mode 100644 index 0000000000000000000000000000000000000000..50095822fae5277819e03934482ab0de0db86556 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/normalization.h @@ -0,0 +1,207 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor normalize( + const Tensor& input, + double p, + int64_t dim, + double eps, + std::optional out) { + if (out == std::nullopt) { + auto denom = input.norm(p, dim, true).clamp_min(eps).expand_as(input); + return input / denom; + } else { + auto denom = input.norm(p, dim, true).clamp_min(eps).expand_as(input); + return torch::div_out(*out, input, denom); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.normalize +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::NormalizeFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::normalize(input, F::NormalizeFuncOptions().p(1).dim(-1)); +/// ``` +inline Tensor normalize( + const Tensor& input, + NormalizeFuncOptions options = {}) { + return detail::normalize( + input, options.p(), options.dim(), options.eps(), options.out()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor layer_norm( + const Tensor& input, + const std::vector& normalized_shape, + const Tensor& weight, + const Tensor& bias, + double eps) { + return torch::layer_norm(input, normalized_shape, weight, bias, eps); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.layer_norm +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::LayerNormFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::layer_norm(input, F::LayerNormFuncOptions({2, 2}).eps(2e-5)); +/// ``` +inline Tensor layer_norm( + const Tensor& input, + const LayerNormFuncOptions& options) { + return detail::layer_norm( + input, + options.normalized_shape(), + options.weight(), + options.bias(), + options.eps()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor local_response_norm( + const Tensor& input, + int64_t size, + double alpha, + double beta, + double k) { + auto dim = input.dim(); + TORCH_CHECK( + dim >= 3, + "Expected 3D or higher dimensionality input (got ", + dim, + " dimensions)"); + auto div = input.mul(input).unsqueeze(1); + if (dim == 3) { + div = detail::pad( + div, + /*pad=*/{0, 0, size / 2, (size - 1) / 2}, + /*mode=*/torch::kConstant, + /*value=*/0); + div = detail::avg_pool2d( + div, + /*kernel_size=*/{size, 1}, + /*stride=*/1, + /*padding=*/0, + /*ceil_mode=*/false, + /*count_include_pad=*/true, + /*divisor_override=*/std::nullopt) + .squeeze(1); + } else { + auto sizes = input.sizes(); + div = div.view({sizes[0], 1, sizes[1], sizes[2], -1}); + div = detail::pad( + div, + /*pad=*/{0, 0, 0, 0, size / 2, (size - 1) / 2}, + /*mode=*/torch::kConstant, + /*value=*/0); + div = detail::avg_pool3d( + div, + /*kernel_size=*/{size, 1, 1}, + /*stride=*/1, + /*padding=*/0, + /*ceil_mode=*/false, + /*count_include_pad=*/true, + /*divisor_override=*/std::nullopt) + .squeeze(1); + div = div.view(sizes); + } + div = div.mul(alpha).add(k).pow(beta); + return input / div; +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.local_response_norm +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::LocalResponseNormFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::local_response_norm(x, F::LocalResponseNormFuncOptions(2)); +/// ``` +inline Tensor local_response_norm( + const Tensor& input, + const LocalResponseNormFuncOptions& options) { + return detail::local_response_norm( + input, options.size(), options.alpha(), options.beta(), options.k()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor group_norm( + const Tensor& input, + int64_t num_groups, + const Tensor& weight, + const Tensor& bias, + double eps) { + return torch::group_norm( + input, + num_groups, + weight, + bias, + eps, + at::globalContext().userEnabledCuDNN()); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.group_norm +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::GroupNormFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::group_norm(input, F::GroupNormFuncOptions(2).eps(2e-5)); +/// ``` +inline Tensor group_norm( + const Tensor& input, + const GroupNormFuncOptions& options) { + return detail::group_norm( + input, + options.num_groups(), + options.weight(), + options.bias(), + options.eps()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/padding.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/padding.h new file mode 100644 index 0000000000000000000000000000000000000000..9a0fe71d9c9159c63c3af4b0e17e5c5b33fbb277 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/padding.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor pad( + const Tensor& input, + IntArrayRef pad, + PadFuncOptions::mode_t mode, + double value) { + const auto mode_enum = [&] { + if (std::holds_alternative(mode)) { + return at::padding_mode::constant; + } else if (std::holds_alternative(mode)) { + return at::padding_mode::reflect; + } else if (std::holds_alternative(mode)) { + return at::padding_mode::replicate; + } else if (std::holds_alternative(mode)) { + return at::padding_mode::circular; + } + TORCH_CHECK(false, "Unrecognised padding mode"); + }(); + + std::optional fill_value; + if (value != 0.0) { + fill_value = value; + } + return at::_pad_enum(input, pad, static_cast(mode_enum), fill_value); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.pad +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::PadFuncOptions` class to +/// learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::pad(input, F::PadFuncOptions({1, 2, 2, 1, 1, +/// 2}).mode(torch::kReplicate)); +/// ``` +inline Tensor pad(const Tensor& input, const PadFuncOptions& options) { + return detail::pad(input, options.pad(), options.mode(), options.value()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/pixelshuffle.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/pixelshuffle.h new file mode 100644 index 0000000000000000000000000000000000000000..0c709d47f50f090835e41c64858c1fe00b806936 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/pixelshuffle.h @@ -0,0 +1,43 @@ +#pragma once + +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor pixel_shuffle(const Tensor& input, int64_t upscale_factor) { + return torch::pixel_shuffle(input, upscale_factor); +} + +inline Tensor pixel_unshuffle(const Tensor& input, int64_t downscale_factor) { + return torch::pixel_unshuffle(input, downscale_factor); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.pixel_shuffle +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::PixelShuffleFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::pixel_shuffle(x, F::PixelShuffleFuncOptions(2)); +/// ``` +inline Tensor pixel_shuffle( + const Tensor& input, + const PixelShuffleFuncOptions& options) { + return detail::pixel_shuffle(input, options.upscale_factor()); +} + +inline Tensor pixel_unshuffle( + const Tensor& input, + const PixelUnshuffleFuncOptions& options) { + return detail::pixel_unshuffle(input, options.downscale_factor()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/pooling.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/pooling.h new file mode 100644 index 0000000000000000000000000000000000000000..3894f55e46b78c00ff26b83ee81fdd4a5d4c918c --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/pooling.h @@ -0,0 +1,1149 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn::functional { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor avg_pool1d( + const Tensor& input, + ExpandingArray<1> kernel_size, + ExpandingArray<1> stride, + ExpandingArray<1> padding, + bool ceil_mode, + bool count_include_pad) { + return torch::avg_pool1d( + input, kernel_size, stride, padding, ceil_mode, count_include_pad); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.avg_pool1d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::AvgPool1dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::avg_pool1d(x, F::AvgPool1dFuncOptions(3).stride(2)); +/// ``` +inline Tensor avg_pool1d( + const Tensor& input, + const AvgPool1dFuncOptions& options) { + return avg_pool1d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.ceil_mode(), + options.count_include_pad()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor avg_pool2d( + const Tensor& input, + ExpandingArray<2> kernel_size, + ExpandingArray<2> stride, + ExpandingArray<2> padding, + bool ceil_mode, + bool count_include_pad, + std::optional divisor_override) { + return torch::avg_pool2d( + input, + kernel_size, + stride, + padding, + ceil_mode, + count_include_pad, + divisor_override); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.avg_pool2d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::AvgPool2dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::avg_pool2d(x, F::AvgPool2dFuncOptions(3).stride(2)); +/// ``` +inline Tensor avg_pool2d( + const Tensor& input, + const AvgPool2dFuncOptions& options) { + return detail::avg_pool2d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.ceil_mode(), + options.count_include_pad(), + options.divisor_override()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor avg_pool3d( + const Tensor& input, + ExpandingArray<3> kernel_size, + ExpandingArray<3> stride, + ExpandingArray<3> padding, + bool ceil_mode, + bool count_include_pad, + std::optional divisor_override) { + return torch::avg_pool3d( + input, + kernel_size, + stride, + padding, + ceil_mode, + count_include_pad, + divisor_override); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.avg_pool3d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::AvgPool3dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::avg_pool3d(x, F::AvgPool3dFuncOptions(3).stride(2)); +/// ``` +inline Tensor avg_pool3d( + const Tensor& input, + const AvgPool3dFuncOptions& options) { + return detail::avg_pool3d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.ceil_mode(), + options.count_include_pad(), + options.divisor_override()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor max_pool1d( + const Tensor& input, + ExpandingArray<1> kernel_size, + ExpandingArray<1> stride, + ExpandingArray<1> padding, + ExpandingArray<1> dilation, + bool ceil_mode) { + return torch::max_pool1d( + input, kernel_size, stride, padding, dilation, ceil_mode); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.max_pool1d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::MaxPool1dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_pool1d(x, F::MaxPool1dFuncOptions(3).stride(2)); +/// ``` +inline Tensor max_pool1d( + const Tensor& input, + const MaxPool1dFuncOptions& options) { + return detail::max_pool1d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline std::tuple max_pool1d_with_indices( + const Tensor& input, + ExpandingArray<1> kernel_size, + ExpandingArray<1> stride, + ExpandingArray<1> padding, + ExpandingArray<1> dilation, + bool ceil_mode) { + return torch::max_pool1d_with_indices( + input, kernel_size, stride, padding, dilation, ceil_mode); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See the documentation for `torch::nn::functional::MaxPool1dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_pool1d_with_indices(x, F::MaxPool1dFuncOptions(3).stride(2)); +/// ``` +inline std::tuple max_pool1d_with_indices( + const Tensor& input, + const MaxPool1dFuncOptions& options) { + return detail::max_pool1d_with_indices( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor max_pool2d( + const Tensor& input, + ExpandingArray<2> kernel_size, + ExpandingArray<2> stride, + ExpandingArray<2> padding, + ExpandingArray<2> dilation, + bool ceil_mode) { + return torch::max_pool2d( + input, kernel_size, stride, padding, dilation, ceil_mode); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.max_pool2d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::MaxPool2dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_pool2d(x, F::MaxPool2dFuncOptions(3).stride(2)); +/// ``` +inline Tensor max_pool2d( + const Tensor& input, + const MaxPool2dFuncOptions& options) { + return detail::max_pool2d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline std::tuple max_pool2d_with_indices( + const Tensor& input, + ExpandingArray<2> kernel_size, + ExpandingArray<2> stride, + ExpandingArray<2> padding, + ExpandingArray<2> dilation, + bool ceil_mode) { + return torch::max_pool2d_with_indices( + input, kernel_size, stride, padding, dilation, ceil_mode); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See the documentation for `torch::nn::functional::MaxPool2dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_pool2d_with_indices(x, F::MaxPool2dFuncOptions(3).stride(2)); +/// ``` +inline std::tuple max_pool2d_with_indices( + const Tensor& input, + const MaxPool2dFuncOptions& options) { + return detail::max_pool2d_with_indices( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor max_pool3d( + const Tensor& input, + ExpandingArray<3> kernel_size, + ExpandingArray<3> stride, + ExpandingArray<3> padding, + ExpandingArray<3> dilation, + bool ceil_mode) { + return torch::max_pool3d( + input, kernel_size, stride, padding, dilation, ceil_mode); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.max_pool3d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::MaxPool3dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_pool3d(x, F::MaxPool3dFuncOptions(3).stride(2)); +/// ``` +inline Tensor max_pool3d( + const Tensor& input, + const MaxPool3dFuncOptions& options) { + return detail::max_pool3d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline std::tuple max_pool3d_with_indices( + const Tensor& input, + ExpandingArray<3> kernel_size, + ExpandingArray<3> stride, + ExpandingArray<3> padding, + ExpandingArray<3> dilation, + bool ceil_mode) { + return torch::max_pool3d_with_indices( + input, kernel_size, stride, padding, dilation, ceil_mode); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See the documentation for `torch::nn::functional::MaxPool3dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_pool3d_with_indices(x, F::MaxPool3dFuncOptions(3).stride(2)); +/// ``` +inline std::tuple max_pool3d_with_indices( + const Tensor& input, + const MaxPool3dFuncOptions& options) { + return detail::max_pool3d_with_indices( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline std::tuple adaptive_max_pool1d_with_indices( + const Tensor& input, + ExpandingArray<1> output_size) { + return torch::adaptive_max_pool1d(input, output_size); +} +} // namespace detail + +/// See the documentation for +/// `torch::nn::functional::AdaptiveMaxPool1dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_max_pool1d_with_indices(x, F::AdaptiveMaxPool1dFuncOptions(3)); +/// ``` +inline std::tuple adaptive_max_pool1d_with_indices( + const Tensor& input, + const AdaptiveMaxPool1dFuncOptions& options) { + return detail::adaptive_max_pool1d_with_indices(input, options.output_size()); +} + +namespace detail { +inline Tensor adaptive_max_pool1d( + const Tensor& input, + ExpandingArray<1> output_size) { + return std::get<0>(adaptive_max_pool1d_with_indices(input, output_size)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.adaptive_max_pool1d +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::AdaptiveMaxPool1dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_max_pool1d(x, F::AdaptiveMaxPool1dFuncOptions(3)); +/// ``` +inline Tensor adaptive_max_pool1d( + const Tensor& input, + const AdaptiveMaxPool1dFuncOptions& options) { + return detail::adaptive_max_pool1d(input, options.output_size()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline std::tuple adaptive_max_pool2d_with_indices( + const Tensor& input, + ExpandingArrayWithOptionalElem<2> output_size) { + auto output_size_ = + torch::nn::modules::utils::_list_with_default(output_size, input.sizes()); + return torch::adaptive_max_pool2d(input, output_size_); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See the documentation for +/// `torch::nn::functional::AdaptiveMaxPool2dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_max_pool2d_with_indices(x, F::AdaptiveMaxPool2dFuncOptions(3)); +/// ``` +inline std::tuple adaptive_max_pool2d_with_indices( + const Tensor& input, + const AdaptiveMaxPool2dFuncOptions& options) { + return detail::adaptive_max_pool2d_with_indices(input, options.output_size()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor adaptive_max_pool2d( + const Tensor& input, + ExpandingArrayWithOptionalElem<2> output_size) { + return std::get<0>(adaptive_max_pool2d_with_indices(input, output_size)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.adaptive_max_pool2d +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::AdaptiveMaxPool2dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_max_pool2d(x, F::AdaptiveMaxPool2dFuncOptions(3)); +/// ``` +inline Tensor adaptive_max_pool2d( + const Tensor& input, + const AdaptiveMaxPool2dFuncOptions& options) { + return detail::adaptive_max_pool2d(input, options.output_size()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline std::tuple adaptive_max_pool3d_with_indices( + const Tensor& input, + ExpandingArrayWithOptionalElem<3> output_size) { + auto output_size_ = + torch::nn::modules::utils::_list_with_default(output_size, input.sizes()); + return torch::adaptive_max_pool3d(input, output_size_); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See the documentation for +/// `torch::nn::functional::AdaptiveMaxPool3dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_max_pool3d_with_indices(x, F::AdaptiveMaxPool3dFuncOptions(3)); +/// ``` +inline std::tuple adaptive_max_pool3d_with_indices( + const Tensor& input, + const AdaptiveMaxPool3dFuncOptions& options) { + return detail::adaptive_max_pool3d_with_indices(input, options.output_size()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor adaptive_max_pool3d( + const Tensor& input, + ExpandingArrayWithOptionalElem<3> output_size) { + return std::get<0>(adaptive_max_pool3d_with_indices(input, output_size)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.adaptive_max_pool3d +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::AdaptiveMaxPool3dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_max_pool3d(x, F::AdaptiveMaxPool3dFuncOptions(3)); +/// ``` +inline Tensor adaptive_max_pool3d( + const Tensor& input, + const AdaptiveMaxPool3dFuncOptions& options) { + return detail::adaptive_max_pool3d(input, options.output_size()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor adaptive_avg_pool1d( + const Tensor& input, + ExpandingArray<1> output_size) { + return torch::adaptive_avg_pool1d(input, output_size); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.adaptive_avg_pool1d +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::AdaptiveAvgPool1dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_avg_pool1d(x, F::AdaptiveAvgPool1dFuncOptions(3)); +/// ``` +inline Tensor adaptive_avg_pool1d( + const Tensor& input, + const AdaptiveAvgPool1dFuncOptions& options) { + return detail::adaptive_avg_pool1d(input, options.output_size()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor adaptive_avg_pool2d( + const Tensor& input, + ExpandingArrayWithOptionalElem<2> output_size) { + auto output_size_ = + torch::nn::modules::utils::_list_with_default(output_size, input.sizes()); + return torch::adaptive_avg_pool2d(input, output_size_); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.adaptive_avg_pool2d +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::AdaptiveAvgPool2dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_avg_pool2d(x, F::AdaptiveAvgPool2dFuncOptions(3)); +/// ``` +inline Tensor adaptive_avg_pool2d( + const Tensor& input, + const AdaptiveAvgPool2dFuncOptions& options) { + return detail::adaptive_avg_pool2d(input, options.output_size()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor adaptive_avg_pool3d( + const Tensor& input, + ExpandingArrayWithOptionalElem<3> output_size) { + auto output_size_ = + torch::nn::modules::utils::_list_with_default(output_size, input.sizes()); + return torch::adaptive_avg_pool3d(input, output_size_); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.adaptive_avg_pool3d +/// about the exact behavior of this functional. +/// +/// See the documentation for +/// `torch::nn::functional::AdaptiveAvgPool3dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_avg_pool3d(x, F::AdaptiveAvgPool3dFuncOptions(3)); +/// ``` +inline Tensor adaptive_avg_pool3d( + const Tensor& input, + const AdaptiveAvgPool3dFuncOptions& options) { + return detail::adaptive_avg_pool3d(input, options.output_size()); +} + +// ============================================================================ + +inline std::vector _unpool_output_size( + const Tensor& input, + const IntArrayRef& kernel_size, + const IntArrayRef& stride, + const IntArrayRef& padding, + const std::optional>& output_size) { + auto input_size = input.sizes(); + std::vector default_size; + for (const auto d : c10::irange(kernel_size.size())) { + default_size.push_back( + (input_size[input_size.size() - kernel_size.size() + d] - 1) * + stride[d] + + kernel_size[d] - 2 * padding[d]); + } + if (!output_size) { + return default_size; + } else { + std::vector output_size_; + if (output_size->size() == kernel_size.size() + 2) { + output_size_ = IntArrayRef(*output_size).slice(2).vec(); + } + if (output_size_.size() != kernel_size.size()) { + TORCH_CHECK( + false, + "output_size should be a sequence containing ", + kernel_size.size(), + " or ", + kernel_size.size() + 2, + " elements, but it has a length of '", + output_size_.size(), + "'"); + } + for (const auto d : c10::irange(kernel_size.size())) { + const auto min_size = default_size[d] - stride[d]; + const auto max_size = default_size[d] + stride[d]; + if (!(min_size <= output_size_[d] && output_size_[d] <= max_size)) { + TORCH_CHECK( + false, + "invalid output_size ", + output_size_, + " (dim ", + d, + " must be between ", + min_size, + " and ", + max_size, + ")"); + } + } + return output_size_; + } +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor max_unpool1d( + const Tensor& input, + const Tensor& indices, + ExpandingArray<1> kernel_size, + ExpandingArray<1> stride, + ExpandingArray<1> padding, + const std::optional>& output_size) { + auto output_size_ = + _unpool_output_size(input, kernel_size, stride, padding, output_size); + output_size_.push_back(1); + return torch::max_unpool2d( + input.unsqueeze(-1), indices.unsqueeze(-1), output_size_) + .squeeze(-1); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.max_unpool1d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::MaxUnpool1dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_unpool1d(x, indices, +/// F::MaxUnpool1dFuncOptions(3).stride(2).padding(1)); +/// ``` +inline Tensor max_unpool1d( + const Tensor& input, + const Tensor& indices, + const MaxUnpool1dFuncOptions& options) { + return detail::max_unpool1d( + input, + indices, + options.kernel_size(), + options.stride(), + options.padding(), + options.output_size()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor max_unpool2d( + const Tensor& input, + const Tensor& indices, + ExpandingArray<2> kernel_size, + ExpandingArray<2> stride, + ExpandingArray<2> padding, + const std::optional>& output_size) { + auto output_size_ = + _unpool_output_size(input, kernel_size, stride, padding, output_size); + + return torch::max_unpool2d(input, indices, output_size_); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.max_unpool2d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::MaxUnpool2dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_unpool2d(x, indices, +/// F::MaxUnpool2dFuncOptions(3).stride(2).padding(1)); +/// ``` +inline Tensor max_unpool2d( + const Tensor& input, + const Tensor& indices, + const MaxUnpool2dFuncOptions& options) { + return detail::max_unpool2d( + input, + indices, + options.kernel_size(), + options.stride(), + options.padding(), + options.output_size()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor max_unpool3d( + const Tensor& input, + const Tensor& indices, + ExpandingArray<3> kernel_size, + ExpandingArray<3> stride, + ExpandingArray<3> padding, + const std::optional>& output_size) { + auto output_size_ = + _unpool_output_size(input, kernel_size, stride, padding, output_size); + + return torch::max_unpool3d(input, indices, output_size_, stride, padding); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.max_unpool3d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::MaxUnpool3dFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_unpool3d(x, indices, F::MaxUnpool3dFuncOptions(3)); +/// ``` +inline Tensor max_unpool3d( + const Tensor& input, + const Tensor& indices, + const MaxUnpool3dFuncOptions& options) { + return detail::max_unpool3d( + input, + indices, + options.kernel_size(), + options.stride(), + options.padding(), + options.output_size()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline std::tuple fractional_max_pool2d_with_indices( + const Tensor& input, + const ExpandingArray<2>& kernel_size, + const std::optional>& output_size, + const std::optional>& output_ratio, + const Tensor& _random_samples) { + if (output_size == std::nullopt && output_ratio == std::nullopt) { + TORCH_CHECK( + false, + "fractional_max_pool2d requires specifying either ", + "an output_size or an output_ratio"); + } + std::optional> output_size_ = output_size; + if (output_size_ == std::nullopt) { + TORCH_INTERNAL_ASSERT(output_ratio != std::nullopt); + output_size_ = { + (int64_t)(static_cast(input.size(-2)) * + (*output_ratio.value())[0]), + (int64_t)(static_cast(input.size(-1)) * + (*output_ratio.value())[1])}; + } + + Tensor _random_samples_ = _random_samples; + if (!_random_samples_.defined()) { + auto n_batch = input.dim() == 3 ? 1 : input.size(0); + _random_samples_ = torch::rand( + {n_batch, input.size(-3), 2}, + torch::TensorOptions().dtype(input.dtype()).device(input.device())); + } + return torch::fractional_max_pool2d( + input, kernel_size, *output_size_, _random_samples_); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See the documentation for +/// `torch::nn::functional::FractionalMaxPool2dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::fractional_max_pool2d_with_indices(x, +/// F::FractionalMaxPool2dFuncOptions(3).output_size(2)); +/// ``` +inline std::tuple fractional_max_pool2d_with_indices( + const Tensor& input, + const FractionalMaxPool2dFuncOptions& options) { + return detail::fractional_max_pool2d_with_indices( + input, + options.kernel_size(), + options.output_size(), + options.output_ratio(), + options._random_samples()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor fractional_max_pool2d( + const Tensor& input, + ExpandingArray<2> kernel_size, + std::optional> output_size, + std::optional> output_ratio, + const Tensor& _random_samples) { + return std::get<0>(fractional_max_pool2d_with_indices( + input, kernel_size, output_size, output_ratio, _random_samples)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See the documentation for +/// `torch::nn::functional::FractionalMaxPool2dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::fractional_max_pool2d(x, +/// F::FractionalMaxPool2dFuncOptions(3).output_size(2)); +/// ``` +inline Tensor fractional_max_pool2d( + const Tensor& input, + const FractionalMaxPool2dFuncOptions& options) { + return detail::fractional_max_pool2d( + input, + options.kernel_size(), + options.output_size(), + options.output_ratio(), + options._random_samples()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline std::tuple fractional_max_pool3d_with_indices( + const Tensor& input, + const ExpandingArray<3>& kernel_size, + const std::optional>& output_size, + const std::optional>& output_ratio, + const Tensor& _random_samples) { + if (output_size == std::nullopt && output_ratio == std::nullopt) { + TORCH_CHECK( + false, + "fractional_max_pool3d requires specifying either ", + "an output_size or an output_ratio"); + } + + std::optional> output_size_ = output_size; + if (output_size_ == std::nullopt) { + TORCH_INTERNAL_ASSERT(output_ratio != std::nullopt); + output_size_ = { + (int64_t)(static_cast(input.size(-3)) * + (*output_ratio.value())[0]), + (int64_t)(static_cast(input.size(-2)) * + (*output_ratio.value())[1]), + (int64_t)(static_cast(input.size(-1)) * + (*output_ratio.value())[2])}; + } + + Tensor _random_samples_ = _random_samples; + if (!_random_samples_.defined()) { + auto n_batch = input.dim() == 4 ? 1 : input.size(0); + _random_samples_ = torch::rand( + {n_batch, input.size(-4), 3}, + torch::TensorOptions().dtype(input.dtype()).device(input.device())); + } + return torch::fractional_max_pool3d( + input, kernel_size, *output_size_, _random_samples_); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See the documentation for +/// `torch::nn::functional::FractionalMaxPool3dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::fractional_max_pool3d_with_indices(x, +/// F::FractionalMaxPool3dFuncOptions(3).output_size(2)); +/// ``` +inline std::tuple fractional_max_pool3d_with_indices( + const Tensor& input, + const FractionalMaxPool3dFuncOptions& options) { + return detail::fractional_max_pool3d_with_indices( + input, + options.kernel_size(), + options.output_size(), + options.output_ratio(), + options._random_samples()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor fractional_max_pool3d( + const Tensor& input, + ExpandingArray<3> kernel_size, + std::optional> output_size, + std::optional> output_ratio, + const Tensor& _random_samples) { + return std::get<0>(fractional_max_pool3d_with_indices( + input, kernel_size, output_size, output_ratio, _random_samples)); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See the documentation for +/// `torch::nn::functional::FractionalMaxPool3dFuncOptions` class to learn what +/// optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::fractional_max_pool3d(x, +/// F::FractionalMaxPool3dFuncOptions(3).output_size(2)); +/// ``` +inline Tensor fractional_max_pool3d( + const Tensor& input, + const FractionalMaxPool3dFuncOptions& options) { + return detail::fractional_max_pool3d( + input, + options.kernel_size(), + options.output_size(), + options.output_ratio(), + options._random_samples()); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor lp_pool1d( + const Tensor& input, + double norm_type, + ExpandingArray<1> kernel_size, + ExpandingArray<1> stride, + bool ceil_mode) { + Tensor out = detail::avg_pool1d( + input.pow(norm_type), + kernel_size, + stride, + /*padding=*/0, + ceil_mode, + /*count_include_pad=*/true); + + return (torch::sign(out) * relu(torch::abs(out))) + .mul((*kernel_size)[0]) + .pow(1. / norm_type); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.lp_pool1d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::LPPool1dFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::lp_pool1d(x, F::LPPool1dFuncOptions(2, 3).stride(2)); +/// ``` +inline Tensor lp_pool1d( + const Tensor& input, + const LPPool1dFuncOptions& options) { + return detail::lp_pool1d( + input, + options.norm_type(), + options.kernel_size(), + options.stride(), + options.ceil_mode()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor lp_pool2d( + const Tensor& input, + double norm_type, + ExpandingArray<2> kernel_size, + ExpandingArray<2> stride, + bool ceil_mode) { + auto kw = (*kernel_size)[0]; + auto kh = (*kernel_size)[1]; + Tensor out = detail::avg_pool2d( + input.pow(norm_type), + kernel_size, + stride, + /*padding=*/0, + ceil_mode, + /*count_include_pad=*/true, + /*divisor_override=*/std::nullopt); + + return (torch::sign(out) * relu(torch::abs(out))) + .mul(kw * kh) + .pow(1. / norm_type); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.lp_pool2d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::LPPool2dFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::lp_pool2d(x, F::LPPool2dFuncOptions(2, {2, 3}).stride(2)); +/// ``` +inline Tensor lp_pool2d( + const Tensor& input, + const LPPool2dFuncOptions& options) { + return detail::lp_pool2d( + input, + options.norm_type(), + options.kernel_size(), + options.stride(), + options.ceil_mode()); +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor lp_pool3d( + const Tensor& input, + double norm_type, + ExpandingArray<3> kernel_size, + ExpandingArray<3> stride, + bool ceil_mode) { + auto kd = (*kernel_size)[0]; + auto kw = (*kernel_size)[1]; + auto kh = (*kernel_size)[2]; + Tensor out = detail::avg_pool3d( + input.pow(norm_type), + kernel_size, + stride, + /*padding=*/0, + ceil_mode, + /*count_include_pad=*/true, + /*divisor_override=*/std::nullopt); + + return (torch::sign(out) * relu(torch::abs(out))) + .mul(kd * kw * kh) + .pow(1. / norm_type); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.lp_pool3d +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::LPPool3dFuncOptions` class +/// to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::lp_pool3d(x, F::LPPool3dFuncOptions(3, {3, 3, 5}).stride(3)); +/// ``` +inline Tensor lp_pool3d( + const Tensor& input, + const LPPool3dFuncOptions& options) { + return detail::lp_pool3d( + input, + options.norm_type(), + options.kernel_size(), + options.stride(), + options.ceil_mode()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/upsampling.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/upsampling.h new file mode 100644 index 0000000000000000000000000000000000000000..4d6c14817805a5228de26b68ce0242a79fbac8d0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/upsampling.h @@ -0,0 +1,286 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace torch::nn::functional { + +inline std::vector _interp_output_size( + int64_t dim, + std::tuple< + Tensor, + std::optional>, + std::optional>, + std::optional> closed_over_args) { + auto [input, size, scale_factor, recompute_scale_factor] = + std::move(closed_over_args); + if (size == std::nullopt && scale_factor == std::nullopt) { + TORCH_CHECK(false, "either size or scale_factor should be defined"); + } + if (size != std::nullopt && scale_factor != std::nullopt) { + TORCH_CHECK(false, "only one of size or scale_factor should be defined"); + } + if (scale_factor != std::nullopt) { + if (static_cast(scale_factor.value().size()) != dim) { + TORCH_CHECK( + false, + "scale_factor shape must match input shape. ", + "Input is ", + dim, + "D, scale_factor size is ", + torch::ArrayRef(*scale_factor)); + } + } + if (size != std::nullopt) { + return *size; + } + + TORCH_INTERNAL_ASSERT(scale_factor != std::nullopt); + auto scale_factors = *scale_factor; + + if (recompute_scale_factor == std::nullopt) { + // only warn when the scales have floating values since + // the result for ints is the same with/without recompute_scale_factor + bool is_float_scale_factor = false; + for (double scale : scale_factors) { + is_float_scale_factor = floor(scale) != scale; + if (is_float_scale_factor) { + break; + } + } + if (is_float_scale_factor) { + TORCH_WARN( + "The default behavior for interpolate/upsample with float scale_factor changed " + "in 1.6.0 to align with other frameworks/libraries, and uses scale_factor directly, " + "instead of relying on the computed output size. " + "If you wish to keep the old behavior, please set recompute_scale_factor=True. " + "See the documentation of nn.Upsample for details. "); + } + } + + std::vector ret; + for (const auto i : c10::irange(dim)) { + ret.emplace_back(static_cast( + floor(static_cast(input.size(i + 2)) * scale_factors[i]))); + } + return ret; +} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor interpolate( + const Tensor& input, + const std::optional>& size, + const std::optional>& scale_factor, + InterpolateFuncOptions::mode_t mode, + std::optional align_corners, + std::optional recompute_scale_factor, + bool antialias) { + if (std::holds_alternative(mode) || + std::get_if(&mode)) { + if (align_corners != std::nullopt) { + TORCH_CHECK( + false, + "align_corners option can only be set with the " + "interpolating modes: linear | bilinear | bicubic | trilinear"); + } + } else { + if (align_corners == std::nullopt) { + TORCH_WARN( + "Default upsampling behavior when mode=", + enumtype::get_enum_name(mode), + " is changed " + "to align_corners=False since 0.4.0. Please specify " + "align_corners=True if the old behavior is desired. " + "See the documentation of nn.Upsample for details."); + align_corners = false; + } + } + + TORCH_CHECK( + input.dim() >= 3 && input.dim() <= 5, + "Input Error: Only 3D, 4D and 5D input Tensors supported " + "(got ", + input.dim(), + "D) for the modes: nearest | linear | bilinear | bicubic | trilinear " + "(got ", + enumtype::get_enum_name(mode), + ")"); + + auto scale_factor_len = input.dim() - 2; + std::vector> scale_factor_list( + scale_factor_len, std::nullopt); + if (scale_factor != std::nullopt && !recompute_scale_factor.value_or(false)) { + auto _scale_factor_repeated = *scale_factor; + scale_factor_list = {}; + for (const auto& elem : _scale_factor_repeated) { + scale_factor_list.emplace_back(elem); + } + } + + if (antialias && + !(input.dim() == 4 && + (std::get_if(&mode) || + std::get_if(&mode)))) { + TORCH_CHECK( + false, + "Anti-alias option is only supported for bilinear and bicubic modes"); + } + + auto closed_over_args = + std::make_tuple(input, size, scale_factor, recompute_scale_factor); + if (input.dim() == 3 && std::get_if(&mode)) { + return torch::upsample_nearest1d( + input, + _interp_output_size(1, std::move(closed_over_args)), + scale_factor_list.at(0)); + } else if (input.dim() == 4 && std::get_if(&mode)) { + return torch::upsample_nearest2d( + input, + _interp_output_size(2, std::move(closed_over_args)), + scale_factor_list.at(0), + scale_factor_list.at(1)); + } else if (input.dim() == 5 && std::get_if(&mode)) { + return torch::upsample_nearest3d( + input, + _interp_output_size(3, std::move(closed_over_args)), + scale_factor_list.at(0), + scale_factor_list.at(1), + scale_factor_list.at(2)); + } else if (input.dim() == 3 && std::get_if(&mode)) { + return torch::_upsample_nearest_exact1d( + input, + _interp_output_size(1, std::move(closed_over_args)), + scale_factor_list.at(0)); + } else if (input.dim() == 4 && std::get_if(&mode)) { + return torch::_upsample_nearest_exact2d( + input, + _interp_output_size(2, std::move(closed_over_args)), + scale_factor_list.at(0), + scale_factor_list.at(1)); + } else if (input.dim() == 5 && std::get_if(&mode)) { + return torch::_upsample_nearest_exact3d( + input, + _interp_output_size(3, std::move(closed_over_args)), + scale_factor_list.at(0), + scale_factor_list.at(1), + scale_factor_list.at(2)); + } else if (input.dim() == 3 && std::get_if(&mode)) { + return detail::adaptive_avg_pool1d( + input, _interp_output_size(1, std::move(closed_over_args))); + } else if (input.dim() == 4 && std::get_if(&mode)) { + return detail::adaptive_avg_pool2d( + input, _interp_output_size(2, std::move(closed_over_args))); + } else if (input.dim() == 5 && std::get_if(&mode)) { + return detail::adaptive_avg_pool3d( + input, _interp_output_size(3, std::move(closed_over_args))); + } else if (input.dim() == 3 && std::get_if(&mode)) { + TORCH_CHECK( + align_corners != std::nullopt, "align_corners should be specified."); + return torch::upsample_linear1d( + input, + _interp_output_size(1, std::move(closed_over_args)), + *align_corners, + scale_factor_list.at(0)); + } else if (input.dim() == 3 && std::get_if(&mode)) { + TORCH_CHECK(false, "Got 3D input, but bilinear mode needs 4D input"); + } else if (input.dim() == 3 && std::get_if(&mode)) { + TORCH_CHECK(false, "Got 3D input, but trilinear mode needs 5D input"); + } else if (input.dim() == 4 && std::get_if(&mode)) { + TORCH_CHECK(false, "Got 4D input, but linear mode needs 3D input"); + } else if (input.dim() == 4 && std::get_if(&mode)) { + TORCH_CHECK( + align_corners != std::nullopt, "align_corners should be specified."); + if (antialias) { + return torch::_upsample_bilinear2d_aa( + input, + _interp_output_size(2, std::move(closed_over_args)), + *align_corners, + scale_factor_list.at(0), + scale_factor_list.at(1)); + } + return torch::upsample_bilinear2d( + input, + _interp_output_size(2, std::move(closed_over_args)), + *align_corners, + scale_factor_list.at(0), + scale_factor_list.at(1)); + } else if (input.dim() == 4 && std::get_if(&mode)) { + TORCH_CHECK(false, "Got 4D input, but trilinear mode needs 5D input"); + } else if (input.dim() == 5 && std::get_if(&mode)) { + TORCH_CHECK(false, "Got 5D input, but linear mode needs 3D input"); + } else if (input.dim() == 5 && std::get_if(&mode)) { + TORCH_CHECK(false, "Got 5D input, but bilinear mode needs 4D input"); + } else if (input.dim() == 5 && std::get_if(&mode)) { + TORCH_CHECK( + align_corners != std::nullopt, "align_corners should be specified."); + return torch::upsample_trilinear3d( + input, + _interp_output_size(3, std::move(closed_over_args)), + *align_corners, + scale_factor_list.at(0), + scale_factor_list.at(1), + scale_factor_list.at(2)); + } else if (input.dim() == 4 && std::get_if(&mode)) { + TORCH_CHECK( + align_corners != std::nullopt, "align_corners should be specified."); + if (antialias) { + return torch::_upsample_bicubic2d_aa( + input, + _interp_output_size(2, std::move(closed_over_args)), + *align_corners, + scale_factor_list.at(0), + scale_factor_list.at(1)); + } + return torch::upsample_bicubic2d( + input, + _interp_output_size(2, std::move(closed_over_args)), + *align_corners, + scale_factor_list.at(0), + scale_factor_list.at(1)); + } else { + TORCH_CHECK( + false, + "Input Error: Only 3D, 4D and 5D input Tensors supported " + "(got ", + input.dim(), + "D) for the modes: nearest | linear | bilinear | bicubic | trilinear " + "(got ", + enumtype::get_enum_name(mode), + ")"); + } +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.interpolate +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::InterpolateFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::interpolate(input, +/// F::InterpolateFuncOptions().size({4}).mode(torch::kNearest)); +/// ``` +inline Tensor interpolate( + const Tensor& input, + const InterpolateFuncOptions& options = {}) { + return detail::interpolate( + input, + options.size(), + options.scale_factor(), + options.mode(), + options.align_corners(), + options.recompute_scale_factor(), + options.antialias()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/vision.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/vision.h new file mode 100644 index 0000000000000000000000000000000000000000..2805a330ed5997c19b8931eb9c76b1dbcaf504da --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/functional/vision.h @@ -0,0 +1,119 @@ +#pragma once + +#include +#include + +namespace torch::nn::functional { + +inline Tensor affine_grid( + const Tensor& theta, + const IntArrayRef& size, + bool align_corners = false) { + // enforce floating point dtype on theta + TORCH_CHECK( + theta.is_floating_point(), + "Expected theta to have floating point type, but got ", + theta.dtype()); + + // check that shapes and sizes match + if (size.size() == 4) { + TORCH_CHECK( + theta.dim() == 3 && theta.size(-2) == 2 && theta.size(-1) == 3, + "Expected a batch of 2D affine matrices of shape Nx2x3 for size ", + size, + ". Got ", + theta.sizes(), + "."); + } else if (size.size() == 5) { + TORCH_CHECK( + theta.dim() == 3 && theta.size(-2) == 3 && theta.size(-1) == 4, + "Expected a batch of 3D affine matrices of shape Nx3x4 for size ", + size, + ". Got ", + theta.sizes(), + "."); + } else { + TORCH_CHECK( + false, + "affine_grid only supports 4D and 5D sizes, ", + "for 2D and 3D affine transforms, respectively. ", + "Got size ", + size); + } + + if (*std::min_element(size.begin(), size.end()) <= 0) { + TORCH_CHECK(false, "Expected non-zero, positive output size. Got ", size); + } + + return torch::affine_grid_generator(theta, size, align_corners); +} + +// ============================================================================ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace detail { +inline Tensor grid_sample( + const Tensor& input, + const Tensor& grid, + GridSampleFuncOptions::mode_t mode, + GridSampleFuncOptions::padding_mode_t padding_mode, + std::optional align_corners) { + int64_t mode_enum = 0, padding_mode_enum = 0; + + if (std::holds_alternative(mode)) { + mode_enum = 0; + } else if (std::holds_alternative(mode)) { + mode_enum = 1; + } else { /// mode == 'bicubic' + mode_enum = 2; + } + + if (std::holds_alternative(padding_mode)) { + padding_mode_enum = 0; + } else if (std::holds_alternative(padding_mode)) { + padding_mode_enum = 1; + } else { /// padding_mode == 'reflection' + padding_mode_enum = 2; + } + + if (!align_corners.has_value()) { + TORCH_WARN( + "Default grid_sample and affine_grid behavior has changed ", + "to align_corners=False since 1.3.0. Please specify ", + "align_corners=True if the old behavior is desired. ", + "See the documentation of grid_sample for details."); + align_corners = false; + } + + return torch::grid_sampler( + input, grid, mode_enum, padding_mode_enum, align_corners.value()); +} +} // namespace detail +#endif /* DOXYGEN_SHOULD_SKIP_THIS */ + +/// See +/// https://pytorch.org/docs/main/nn.functional.html#torch.nn.functional.grid_sample +/// about the exact behavior of this functional. +/// +/// See the documentation for `torch::nn::functional::GridSampleFuncOptions` +/// class to learn what optional arguments are supported for this functional. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::grid_sample(input, grid, +/// F::GridSampleFuncOptions().mode(torch::kBilinear).padding_mode(torch::kZeros).align_corners(true)); +/// ``` +inline Tensor grid_sample( + const Tensor& input, + const Tensor& grid, + const GridSampleFuncOptions& options = {}) { + return detail::grid_sample( + input, + grid, + options.mode(), + options.padding_mode(), + options.align_corners()); +} + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/init.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/init.h new file mode 100644 index 0000000000000000000000000000000000000000..a19bc4afe1d834399e28de553a08bba05e8de077 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/init.h @@ -0,0 +1,122 @@ +#pragma once + +#include +#include +#include + +namespace torch { + +namespace nn::init { + +using NonlinearityType = std::variant< + enumtype::kLinear, + enumtype::kConv1D, + enumtype::kConv2D, + enumtype::kConv3D, + enumtype::kConvTranspose1D, + enumtype::kConvTranspose2D, + enumtype::kConvTranspose3D, + enumtype::kSigmoid, + enumtype::kTanh, + enumtype::kReLU, + enumtype::kLeakyReLU>; + +using FanModeType = std::variant; + +} // namespace nn::init + +namespace nn::init { + +/// Return the recommended gain value for the given nonlinearity function. +TORCH_API double calculate_gain( + NonlinearityType nonlinearity, + double param = 0.01); + +/// Fills the given `tensor` with the provided `value` in-place, and returns it. +/// No gradient will be recorded for this operation. +TORCH_API Tensor constant_(Tensor tensor, Scalar value); + +/// Fills the given `tensor` with the Dirac delta function in-place, and returns +/// it. No gradient will be recorded for this operation. +TORCH_API Tensor dirac_(Tensor tensor); + +/// Fills the given 2-dimensional `matrix` with an identity matrix. +/// No gradient will be recorded for this operation. +TORCH_API Tensor eye_(Tensor matrix); + +/// Fills the given 2-dimensional `matrix` with values drawn from a normal +/// distribution parameterized by `mean` and `std`. +/// No gradient will be recorded for this operation. +TORCH_API Tensor normal_(Tensor tensor, double mean = 0, double std = 1); + +/// Fills the given `tensor` with ones. +/// No gradient will be recorded for this operation. +TORCH_API Tensor ones_(Tensor tensor); + +/// Fills the input `Tensor` with a (semi) orthogonal matrix, as described in +/// "Exact solutions to the nonlinear dynamics of learning in deep linear neural +/// networks" - Saxe, A. et al. (2013). The input tensor must have at least 2 +/// dimensions, and for tensors with more than 2 dimensions the trailing +/// dimensions are flattened. +/// No gradient will be recorded for this operation. +TORCH_API Tensor orthogonal_(Tensor tensor, double gain = 1.0); + +/// Fills the 2D input `Tensor` as a sparse matrix, where the +/// non-zero elements will be drawn from a centered normal distribution +/// with the given standard deviation `std`, as described in "Deep learning via +/// Hessian-free optimization" - Martens, J. (2010). The `sparsity` is a real +/// value between 0 and 1 that controls the fraction of elements in each column +/// to be set to zero. +/// No gradient will be recorded for this operation. +TORCH_API Tensor sparse_(Tensor tensor, double sparsity, double std = 0.01); + +/// Fills the given 2-dimensional `matrix` with values drawn from a uniform +/// distribution parameterized by `low` and `high`. +/// No gradient will be recorded for this operation. +TORCH_API Tensor uniform_(Tensor tensor, double low = 0, double high = 1); + +/// Fills the input `Tensor` with values according to the method +/// described in "Delving deep into rectifiers: Surpassing human-level +/// performance on ImageNet classification" - He, K. et al. (2015), using a +/// normal distribution. Also known as He initialization. +/// No gradient will be recorded for this operation. +TORCH_API Tensor kaiming_normal_( + Tensor tensor, + double a = 0, + FanModeType mode = torch::kFanIn, + NonlinearityType nonlinearity = torch::kLeakyReLU); + +/// Fills the input `Tensor` with values according to the method +/// described in "Delving deep into rectifiers: Surpassing human-level +/// performance on ImageNet classification" - He, K. et al. (2015), using a +/// uniform distribution. Also known as He initialization. +/// No gradient will be recorded for this operation. +TORCH_API Tensor kaiming_uniform_( + Tensor tensor, + double a = 0, + FanModeType mode = torch::kFanIn, + NonlinearityType nonlinearity = torch::kLeakyReLU); + +/// Fills the input `Tensor` with values according to the method +/// described in "Understanding the difficulty of training deep feedforward +/// neural networks" - Glorot, X. & Bengio, Y. (2010). Values are scaled by the +/// `gain` parameter. No gradient will be recorded for this operation. +TORCH_API Tensor xavier_normal_(Tensor tensor, double gain = 1.0); + +/// Fills the input `Tensor` with values according to the method +/// described in "Understanding the difficulty of training deep feedforward +/// neural networks" - Glorot, X. & Bengio, Y. (2010), using a uniform +/// distribution. Values are scaled by the `gain` parameter +/// No gradient will be recorded for this operation. +TORCH_API Tensor xavier_uniform_(Tensor tensor, double gain = 1.0); + +/// Fills the given `tensor` with zeros. +/// No gradient will be recorded for this operation. +TORCH_API Tensor zeros_(Tensor tensor); + +TORCH_API std::tuple _calculate_fan_in_and_fan_out( + const Tensor& tensor); + +} // namespace nn::init + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/module.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/module.h new file mode 100644 index 0000000000000000000000000000000000000000..d36140320e78589865784cab8489cf1e9939a5bd --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/module.h @@ -0,0 +1,719 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace torch::nn { + +/// The base class for all modules in PyTorch. +/// +/// \rst +/// .. note:: +/// The design and implementation of this class is largely based on the Python +/// API. You may want to consult the python documentation for +/// :py:class:`pytorch:torch.nn.Module` for further clarification on certain +/// methods or behavior. +/// \endrst +/// +/// A `Module` is an abstraction over the implementation of some function or +/// algorithm, possibly associated with some persistent data. A `Module` may +/// contain further `Module`s ("submodules"), each with their own +/// implementation, persistent data and further submodules. `Module`s can thus +/// be said to form a recursive tree structure. A `Module` is registered as a +/// submodule to another `Module` by calling `register_module()`, typically from +/// within a parent module's constructor. +/// +/// A distinction is made between three kinds of persistent data that may be +/// associated with a `Module`: +/// +/// 1. *Parameters*: tensors that record gradients, typically weights updated +/// during the backward step (e.g. the `weight` of a `Linear` module), +/// 2. *Buffers*: tensors that do not record gradients, typically updated during +/// the forward step, such as running statistics (e.g. `mean` and `variance` +/// in the `BatchNorm` module), +/// 3. Any additional state, not necessarily tensors, required for the +/// implementation or configuration of a `Module`. +/// +/// The first two kinds of state are special in that they may be registered +/// with the `Module` system to allow convenient access and batch configuration. +/// For example, registered parameters in any `Module` may be iterated over via +/// the `parameters()` accessor. Further, changing the data type of a `Module`'s +/// registered parameters can be done conveniently via `Module::to()`, e.g. +/// `module->to(torch::kCUDA)` to move all parameters to GPU memory. Lastly, +/// registered parameters and buffers are handled specially during a `clone()` +/// operation, which performs a deepcopy of a cloneable `Module` hierarchy. +/// +/// Parameters are registered with a `Module` via `register_parameter`. Buffers +/// are registered separately via `register_buffer`. These methods are part of +/// the public API of `Module` and are typically invoked from within a +/// concrete `Module`s constructor. +class TORCH_API Module : public std::enable_shared_from_this { + public: + using ModuleApplyFunction = std::function; + using ConstModuleApplyFunction = std::function; + using NamedModuleApplyFunction = + std::function; + using ConstNamedModuleApplyFunction = + std::function; + using ModulePointerApplyFunction = + std::function&)>; + using NamedModulePointerApplyFunction = + std::function&)>; + + /// Tells the base `Module` about the name of the submodule. + explicit Module(std::string name); + + /// Constructs the module without immediate knowledge of the submodule's name. + /// The name of the submodule is inferred via RTTI (if possible) the first + /// time `.name()` is invoked. + Module(); + Module(const Module&) = default; + Module& operator=(const Module&) = default; + Module(Module&&) noexcept = default; + Module& operator=(Module&&) noexcept = default; + + virtual ~Module() = default; + + /// Returns the name of the `Module`. + /// + /// A `Module` has an associated `name`, which is a string representation of + /// the kind of concrete `Module` it represents, such as `"Linear"` for the + /// `Linear` module. Under most circumstances, this name is automatically + /// inferred via runtime type information (RTTI). In the unusual circumstance + /// that you have this feature disabled, you may want to manually name your + /// `Module`s by passing the string name to the `Module` base class' + /// constructor. + const std::string& name() const noexcept; + + /// Performs a recursive deep copy of the module and all its registered + /// parameters, buffers and submodules. + /// + /// Optionally, this method sets the current device + /// to the one supplied before cloning. If no device is given, each + /// parameter and buffer will be moved to the device of its source. + /// + /// \rst + /// .. attention:: + /// Attempting to call the `clone()` method inherited from the base `Module` + /// class (the one documented here) will fail. To inherit an actual + /// implementation of `clone()`, you must subclass `Cloneable`. `Cloneable` + /// is templatized on the concrete module type, and can thus properly copy a + /// `Module`. This method is provided on the base class' API solely for an + /// easier-to-use polymorphic interface. + /// \endrst + virtual std::shared_ptr clone( + const std::optional& device = std::nullopt) const; + + /// Applies the `function` to the `Module` and recursively to every submodule. + /// The function must accept a `Module&`. + /// + /// \rst + /// .. code-block:: cpp + /// + /// MyModule module; + /// module->apply([](nn::Module& module) { + /// std::cout << module.name() << std::endl; + /// }); + /// + /// \endrst + void apply(const ModuleApplyFunction& function); + + /// Applies the `function` to the `Module` and recursively to every submodule. + /// The function must accept a `const Module&`. + /// + /// \rst + /// .. code-block:: cpp + /// + /// MyModule module; + /// module->apply([](const nn::Module& module) { + /// std::cout << module.name() << std::endl; + /// }); + /// + /// \endrst + void apply(const ConstModuleApplyFunction& function) const; + + /// Applies the `function` to the `Module` and recursively to every submodule. + /// The function must accept a `const std::string&` for the key of the module, + /// and a `Module&`. The key of the module itself is the empty string. If + /// `name_prefix` is given, it is prepended to every key as + /// `.` (and just `name_prefix` for the module itself). + /// + /// \rst + /// .. code-block:: cpp + /// + /// MyModule module; + /// module->apply([](const std::string& key, nn::Module& module) { + /// std::cout << key << ": " << module.name() << std::endl; + /// }); + /// + /// \endrst + void apply( + const NamedModuleApplyFunction& function, + const std::string& name_prefix = std::string()); + + /// Applies the `function` to the `Module` and recursively to every submodule. + /// The function must accept a `const std::string&` for the key of the module, + /// and a `const Module&`. The key of the module itself is the empty string. + /// If `name_prefix` is given, it is prepended to every key as + /// `.` (and just `name_prefix` for the module itself). + /// + /// \rst + /// .. code-block:: cpp + /// + /// MyModule module; + /// module->apply([](const std::string& key, const nn::Module& module) { + /// std::cout << key << ": " << module.name() << std::endl; + /// }); + /// + /// \endrst + void apply( + const ConstNamedModuleApplyFunction& function, + const std::string& name_prefix = std::string()) const; + + /// Applies the `function` to the `Module` and recursively to every submodule. + /// The function must accept a `const std::shared_ptr&`. + /// + /// \rst + /// .. code-block:: cpp + /// + /// MyModule module; + /// module->apply([](const std::shared_ptr& module) { + /// std::cout << module->name() << std::endl; + /// }); + /// + /// \endrst + void apply(const ModulePointerApplyFunction& function) const; + + /// Applies the `function` to the `Module` and recursively to every submodule. + /// The function must accept a `const std::string&` for the key of the module, + /// and a `const std::shared_ptr&`. The key of the module itself is + /// the empty string. If `name_prefix` is given, it is prepended to every key + /// as + /// `.` (and just `name_prefix` for the module itself). + /// + /// \rst + /// .. code-block:: cpp + /// + /// MyModule module; + /// module->apply([](const std::string& key, + /// const std::shared_ptr& module) { + /// std::cout << key << ": " << module->name() << std::endl; + /// }); + /// + /// \endrst + void apply( + const NamedModulePointerApplyFunction& function, + const std::string& name_prefix = std::string()) const; + + /// Returns the parameters of this `Module` and if `recurse` is true, also + /// recursively of every submodule. + std::vector parameters(bool recurse = true) const; + + /// Returns an `OrderedDict` with the parameters of this `Module` along with + /// their keys, and if `recurse` is true also recursively of every submodule. + OrderedDict named_parameters(bool recurse = true) const; + + /// Returns the buffers of this `Module` and if `recurse` is true, also + /// recursively of every submodule. + std::vector buffers(bool recurse = true) const; + + /// Returns an `OrderedDict` with the buffers of this `Module` along with + /// their keys, and if `recurse` is true also recursively of every submodule. + OrderedDict named_buffers(bool recurse = true) const; + + /// Returns the submodules of this `Module` (the entire submodule hierarchy) + /// and if `include_self` is true, also inserts a `shared_ptr` to this module + /// in the first position. + /// + /// \rst + /// .. warning:: + /// Only pass `include_self` as `true` if this `Module` is stored in a + /// `shared_ptr`! Otherwise an exception will be thrown. You may still call + /// this method with `include_self` set to false if your `Module` is not + /// stored in a `shared_ptr`. + /// \endrst + std::vector> modules(bool include_self = true) const; + + /// Returns an `OrderedDict` of the submodules of this `Module` (the entire + /// submodule hierarchy) and their keys, and if `include_self` is true, also + /// inserts a `shared_ptr` to this module in the first position. If + /// `name_prefix` is given, it is prepended to every key as + /// `.` (and just `name_prefix` for the module itself). + /// + /// \rst + /// .. warning:: + /// Only pass `include_self` as `true` if this `Module` is stored in a + /// `shared_ptr`! Otherwise an exception will be thrown. You may still call + /// this method with `include_self` set to false if your `Module` is not + /// stored in a `shared_ptr`. + /// \endrst + OrderedDict> named_modules( + const std::string& name_prefix = std::string(), + bool include_self = true) const; + + /// Returns the direct submodules of this `Module`. + std::vector> children() const; + + /// Returns an `OrderedDict` of the direct submodules of this `Module` and + /// their keys. + OrderedDict> named_children() const; + + /// Enables "training" mode. + virtual void train(bool on = true); + + /// Calls train(false) to enable "eval" mode. + /// Do not override this method, override `train()` instead. + void eval(); + + /// True if the module is in training mode. + /// + /// Every `Module` has a boolean associated with it that determines whether + /// the `Module` is currently in *training* mode (set via `.train()`) or in + /// *evaluation* (inference) mode (set via `.eval()`). This property is + /// exposed via `is_training()`, and may be used by the implementation of a + /// concrete module to modify its runtime behavior. See the `BatchNorm` or + /// `Dropout` modules for examples of `Module`s that use different code paths + /// depending on this property. + virtual bool is_training() const noexcept; + + /// Recursively casts all parameters to the given `dtype` and `device`. + /// + /// If `non_blocking` is true and the source is in pinned memory and + /// destination is on the GPU or vice versa, the copy is performed + /// asynchronously with respect to the host. Otherwise, the argument has no + /// effect. + virtual void to( + torch::Device device, + torch::Dtype dtype, + bool non_blocking = false); + + /// Recursively casts all parameters to the given dtype. + /// + /// If `non_blocking` is true and the source is in pinned memory and + /// destination is on the GPU or vice versa, the copy is performed + /// asynchronously with respect to the host. Otherwise, the argument has no + /// effect. + virtual void to(torch::Dtype dtype, bool non_blocking = false); + + /// Recursively moves all parameters to the given device. + /// + /// If `non_blocking` is true and the source is in pinned memory and + /// destination is on the GPU or vice versa, the copy is performed + /// asynchronously with respect to the host. Otherwise, the argument has no + /// effect. + virtual void to(torch::Device device, bool non_blocking = false); + + /// Recursively zeros out the `grad` value of each registered parameter. + virtual void zero_grad(bool set_to_none = true); + + /// Attempts to cast this `Module` to the given `ModuleType`. + /// + /// This method is useful when calling `apply()`. + /// + /// \rst + /// .. code-block:: cpp + /// + /// void initialize_weights(nn::Module& module) { + /// torch::NoGradGuard no_grad; + /// if (auto* linear = module.as()) { + /// linear->weight.normal_(0.0, 0.02); + /// } + /// } + /// + /// MyModule module; + /// module->apply(initialize_weights); + /// + /// \endrst + template + typename ModuleType::ContainedType* as() noexcept; + + /// Attempts to cast this `Module` to the given `ModuleType`. + /// + /// This method is useful when calling `apply()`. + /// + /// \rst + /// .. code-block:: cpp + /// + /// void initialize_weights(nn::Module& module) { + /// torch::NoGradGuard no_grad; + /// if (auto* linear = module.as()) { + /// linear->weight.normal_(0.0, 0.02); + /// } + /// } + /// + /// MyModule module; + /// module->apply(initialize_weights); + /// + /// \endrst + template + const typename ModuleType::ContainedType* as() const noexcept; + + /// Attempts to cast this `Module` to the given `ModuleType`. + /// + /// This method is useful when calling `apply()`. + /// + /// \rst + /// .. code-block:: cpp + /// + /// void initialize_weights(nn::Module& module) { + /// torch::NoGradGuard no_grad; + /// if (auto* linear = module.as()) { + /// linear->weight.normal_(0.0, 0.02); + /// } + /// } + /// + /// MyModule module; + /// module.apply(initialize_weights); + /// + /// \endrst + template < + typename ModuleType, + typename = torch::detail::disable_if_module_holder_t> + ModuleType* as() noexcept; + + /// Attempts to cast this `Module` to the given `ModuleType`. + /// + /// This method is useful when calling `apply()`. + /// + /// \rst + /// .. code-block:: cpp + /// + /// void initialize_weights(nn::Module& module) { + /// torch::NoGradGuard no_grad; + /// if (auto* linear = module.as()) { + /// linear->weight.normal_(0.0, 0.02); + /// } + /// } + /// + /// MyModule module; + /// module.apply(initialize_weights); + /// + /// \endrst + template < + typename ModuleType, + typename = torch::detail::disable_if_module_holder_t> + const ModuleType* as() const noexcept; + + /// Serializes the `Module` into the given `OutputArchive`. + /// + /// If the `Module` contains unserializable submodules (e.g. + /// `nn::Functional`), those submodules are skipped when serializing. + virtual void save(serialize::OutputArchive& archive) const; + + /// Deserializes the `Module` from the given `InputArchive`. + /// + /// If the `Module` contains unserializable submodules (e.g. + /// `nn::Functional`), we don't check the existence of those submodules in the + /// `InputArchive` when deserializing. + virtual void load(serialize::InputArchive& archive); + + /// Streams a pretty representation of the `Module` into the given `stream`. + /// By default, this representation will be the name of the module (taken from + /// `name()`), followed by a recursive pretty print of all of the `Module`'s + /// submodules. + /// + /// Override this method to change the pretty print. The input + /// `stream` should be returned from the method, to allow easy chaining. + virtual void pretty_print(std::ostream& stream) const; + + /// Returns whether the `Module` is serializable. + virtual bool is_serializable() const; + + /// Registers a parameter with this `Module`. + /// + /// A parameter should be any gradient-recording tensor used in the + /// implementation of your `Module`. Registering it makes it available to + /// methods such as `parameters()`, `clone()` or `to().` + /// + /// Note that registering an undefined Tensor (e.g. + /// `module.register_parameter("param", Tensor())`) is allowed, and is + /// equivalent to `module.register_parameter("param", None)` in Python API. + /// + /// \rst + /// .. code-block:: cpp + /// + /// MyModule::MyModule() { + /// weight_ = register_parameter("weight", torch::randn({A, B})); + /// } + /// \endrst + Tensor& register_parameter( + std::string name, + Tensor tensor, + bool requires_grad = true); + + /// Registers a buffer with this `Module`. + /// + /// A buffer is intended to be state in your module that does not record + /// gradients, such as running statistics. Registering it makes it available + /// to methods such as `buffers()`, `clone()` or `to()`. + /// + /// \rst + /// .. code-block:: cpp + /// + /// MyModule::MyModule() { + /// mean_ = register_buffer("mean", torch::empty({num_features_})); + /// } + /// \endrst + Tensor& register_buffer(std::string name, Tensor tensor); + + /// Registers a submodule with this `Module`. + /// + /// Registering a module makes it available to methods such as `modules()`, + /// `clone()` or `to()`. + /// + /// \rst + /// .. code-block:: cpp + /// + /// MyModule::MyModule() { + /// submodule_ = register_module("linear", torch::nn::Linear(3, 4)); + /// } + /// \endrst + template + std::shared_ptr register_module( + std::string name, + std::shared_ptr module); + + /// Registers a submodule with this `Module`. + /// + /// This method deals with `ModuleHolder`s. + /// + /// Registering a module makes it available to methods such as `modules()`, + /// `clone()` or `to()`. + /// + /// \rst + /// .. code-block:: cpp + /// + /// MyModule::MyModule() { + /// submodule_ = register_module("linear", torch::nn::Linear(3, 4)); + /// } + /// \endrst + template + std::shared_ptr register_module( + std::string name, + ModuleHolder module_holder); + + /// Replaces a registered submodule with this `Module`. + /// + /// This takes care of the registration, if you used submodule members, you + /// should assign the submodule as well, i.e. use as + /// module->submodule_ = module->replace_module("linear", + /// torch::nn::Linear(3, 4)); + /// It only works when a module of the name is already registered. + /// + /// This is useful for replacing a module after initialization, e.g. + /// for finetuning. + template + std::shared_ptr replace_module( + const std::string& name, + std::shared_ptr module); + + /// Replaces a registered submodule with this `Module`. + /// This method deals with `ModuleHolder`s. + /// + /// This takes care of the registration, if you used submodule members, you + /// should assign the submodule as well, i.e. use as + /// module->submodule_ = module->replace_module("linear", linear_holder); + /// It only works when a module of the name is already registered. + /// + /// This is useful for replacing a module after initialization, e.g. + /// for finetuning. + template + std::shared_ptr replace_module( + const std::string& name, + ModuleHolder module_holder); + + /// Unregisters a submodule from this `Module`. If there is no such module + /// with `name` an exception is thrown. + void unregister_module(const std::string& name); + + protected: + /// The following three functions allow a module with default arguments in its + /// forward method to be used in a Sequential module. + /// You should NEVER override these functions manually. Instead, you should + /// use the `FORWARD_HAS_DEFAULT_ARGS` macro. + virtual bool _forward_has_default_args() { + return false; + } + + virtual unsigned int _forward_num_required_args() { + TORCH_CHECK( + false, + "torch::nn::Module subclass that has default arguments in `forward` method ", + "must override `_forward_num_required_args` method. Please use ", + "`FORWARD_HAS_DEFAULT_ARGS` macro to do so."); + } + + virtual std::vector _forward_populate_default_args( + std::vector&& arguments) { + TORCH_CHECK( + false, + "torch::nn::Module subclass that has default arguments in `forward` method ", + "must override `_forward_populate_default_args` method. Please use ", + "`FORWARD_HAS_DEFAULT_ARGS` macro to do so."); + } + + /// The registered parameters of this `Module`. + /// Inorder to access parameters_ in ParameterDict and ParameterList + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + OrderedDict parameters_; + + private: + // Friend classes. + + template + friend class Cloneable; + + template + friend struct AnyModuleHolder; + + /// Pretty prints the given `Module` into the `ostream`. + TORCH_API friend std::ostream& operator<<( + std::ostream& stream, + const nn::Module& module); + + // data parallel using this method to configure gradient edges during the + // replicate step. + template + friend void replicate_grad_edges( + const std::shared_ptr& module, + const std::vector>& replicas, + const std::vector& devices); + + // Private methods. + + /// Used in the implementation of `Cloneable`. + virtual void clone_(Module& other, const std::optional& device); + + /// The implementation of the various `to()` methods. + template + void to_impl(Ts&&... ts); + + /// Implements pretty printing the module hierarchy. + void pretty_print_recursive( + std::ostream& stream, + const std::string& indentation) const; + + /// Applies the `function` to every submodule recursively, starting at this + /// `Module`'s children (thus not including the module itself). + void apply_to_submodules( + const NamedModulePointerApplyFunction& function, + const std::string& name_prefix = std::string()) const; + + /// Returns a shared_ptr to `this` in a safe (checked) way. + std::shared_ptr shared_from_this_checked() const; + + /// The registered buffers of this `Module`. + OrderedDict buffers_; + + /// The registered (direct) submodules of this `Module`. + OrderedDict> children_; + + /// The module's name (e.g. "LSTM"). + mutable std::optional name_; + + /// Whether the module is in training mode. + bool is_training_{true}; +}; + +/// Serialize a `Module` pointer into an `OutputArchive`. +TORCH_API serialize::OutputArchive& operator<<( + serialize::OutputArchive& archive, + const std::shared_ptr& module); + +/// Deserializes a `Module` from an `InputArchive`. +TORCH_API serialize::InputArchive& operator>>( + serialize::InputArchive& archive, + const std::shared_ptr& module); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ nn::Module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +template +typename ModuleType::ContainedType* Module::as() noexcept { + // Use the contained type of the `ModuleHolder`, e.g. `LinearImpl` for + // `Linear`, since `LinearImpl` inherits `nn::Module`. + return as(); +} + +template +const typename ModuleType::ContainedType* Module::as() const noexcept { + // Use the contained type of the `ModuleHolder`, e.g. `LinearImpl` for + // `Linear`, since `LinearImpl` inherits `nn::Module`. + return as(); +} + +template +ModuleType* Module::as() noexcept { + return dynamic_cast(this); +} + +template +const ModuleType* Module::as() const noexcept { + return dynamic_cast(this); +} + +template +std::shared_ptr Module::register_module( + std::string name, + std::shared_ptr module) { + TORCH_CHECK(!name.empty(), "Submodule name must not be empty"); + TORCH_CHECK( + name.find('.') == std::string::npos, + "Submodule name must not contain a dot (got '", + name, + "')"); + auto& base_module = children_.insert(std::move(name), std::move(module)); + return std::dynamic_pointer_cast(base_module); +} + +template +std::shared_ptr Module::register_module( + std::string name, + ModuleHolder module_holder) { + return register_module(std::move(name), module_holder.ptr()); +} + +template +std::shared_ptr Module::replace_module( + const std::string& name, + std::shared_ptr module) { + auto& base_module = (children_[name] = std::move(module)); + return std::dynamic_pointer_cast(base_module); +} + +template +std::shared_ptr Module::replace_module( + const std::string& name, + ModuleHolder module_holder) { + return replace_module(name, module_holder.ptr()); +} + +template +void Module::to_impl(Ts&&... ts) { + /// First call `to()` on every child module. + for (auto& child : children_) { + child.value()->to(ts...); + } + /// Then move every parameter to the new dtype/device. + for (auto& parameter : named_parameters(/*recurse=*/false)) { + parameter->set_data(parameter->to(ts...)); + } + /// Then move every buffer to the new dtype/device. + for (auto& buffer : named_buffers(/*recurse=*/false)) { + buffer->set_data(buffer->to(ts...)); + } +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules.h new file mode 100644 index 0000000000000000000000000000000000000000..79241fc5e9a8665c723ec8fff72f7d55f7bb82a4 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules.h @@ -0,0 +1,36 @@ +#pragma once + +// Common +#include + +// Containers +#include +#include +#include +#include +#include +#include +#include +#include + +// Layers +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/_functions.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/_functions.h new file mode 100644 index 0000000000000000000000000000000000000000..7ab0e9d8603cb48c6c610473407515fe3f83c3ba --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/_functions.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn::functions { + +class CrossMapLRN2d : public torch::autograd::Function { + public: + static torch::autograd::Variable forward( + torch::autograd::AutogradContext* ctx, + const torch::autograd::Variable& input, + const CrossMapLRN2dOptions& options); + + static torch::autograd::variable_list backward( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_output); +}; + +} // namespace torch::nn::functions diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/activation.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/activation.h new file mode 100644 index 0000000000000000000000000000000000000000..3a086fd1de1943d8bb8020a269bab7f514a17671 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/activation.h @@ -0,0 +1,873 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace torch::nn { + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ELU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies elu over a given input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ELU to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ELUOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ELU model(ELUOptions().alpha(42.42).inplace(true)); +/// ``` +class TORCH_API ELUImpl : public torch::nn::Cloneable { + public: + explicit ELUImpl(const ELUOptions& options_ = {}); + + Tensor forward(Tensor input); + + void reset() override; + + /// Pretty prints the `ELU` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + ELUOptions options; +}; + +/// A `ModuleHolder` subclass for `ELUImpl`. +/// See the documentation for `ELUImpl` class to learn what methods it +/// provides, and examples of how to use `ELU` with `torch::nn::ELUOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(ELU); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SELU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the selu function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.SELU to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::SELUOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// SELU model(SELUOptions().inplace(true)); +/// ``` +class TORCH_API SELUImpl : public torch::nn::Cloneable { + public: + explicit SELUImpl(const SELUOptions& options_ = {}); + + Tensor forward(Tensor input); + + void reset() override; + + /// Pretty prints the `SELU` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + SELUOptions options; +}; + +/// A `ModuleHolder` subclass for `SELUImpl`. +/// See the documentation for `SELUImpl` class to learn what methods it +/// provides, and examples of how to use `SELU` with `torch::nn::SELUOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(SELU); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Hardshrink ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the hard shrinkage function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Hardshrink to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::HardshrinkOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Hardshrink model(HardshrinkOptions().lambda(42.42)); +/// ``` +class TORCH_API HardshrinkImpl : public torch::nn::Cloneable { + public: + explicit HardshrinkImpl(const HardshrinkOptions& options_ = {}); + + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `Hardshrink` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + HardshrinkOptions options; +}; + +/// A `ModuleHolder` subclass for `HardshrinkImpl`. +/// See the documentation for `HardshrinkImpl` class to learn what methods it +/// provides, and examples of how to use `Hardshrink` with +/// `torch::nn::HardshrinkOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Hardshrink); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Hardtanh ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the HardTanh function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Hardtanh to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::HardtanhOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Hardtanh +/// model(HardtanhOptions().min_val(-42.42).max_val(0.42).inplace(true)); +/// ``` +class TORCH_API HardtanhImpl : public torch::nn::Cloneable { + public: + explicit HardtanhImpl(const HardtanhOptions& options_ = {}); + + Tensor forward(Tensor input); + + void reset() override; + + /// Pretty prints the `Hardtanh` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + HardtanhOptions options; +}; + +/// A `ModuleHolder` subclass for `HardtanhImpl`. +/// See the documentation for `HardtanhImpl` class to learn what methods it +/// provides, and examples of how to use `Hardtanh` with +/// `torch::nn::HardtanhOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Hardtanh); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LeakyReLU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the LeakyReLU function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.LeakyReLU to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::LeakyReLUOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// LeakyReLU model(LeakyReLUOptions().negative_slope(0.42).inplace(true)); +/// ``` +class TORCH_API LeakyReLUImpl : public torch::nn::Cloneable { + public: + explicit LeakyReLUImpl(const LeakyReLUOptions& options_ = {}); + + Tensor forward(Tensor input); + + void reset() override; + + /// Pretty prints the `LeakyReLU` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + LeakyReLUOptions options; +}; + +/// A `ModuleHolder` subclass for `LeakyReLUImpl`. +/// See the documentation for `LeakyReLUImpl` class to learn what methods it +/// provides, and examples of how to use `LeakyReLU` with +/// `torch::nn::LeakyReLUOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(LeakyReLU); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LogSigmoid ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the LogSigmoid function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.LogSigmoid to learn +/// about the exact behavior of this module. +class TORCH_API LogSigmoidImpl : public torch::nn::Cloneable { + public: + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `LogSigmoid` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `LogSigmoidImpl`. +/// See the documentation for `LogSigmoidImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(LogSigmoid); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Softmax ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the Softmax function. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Softmax to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::SoftmaxOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Softmax model(SoftmaxOptions(1)); +/// ``` +class TORCH_API SoftmaxImpl : public torch::nn::Cloneable { + public: + explicit SoftmaxImpl(int64_t dim) : SoftmaxImpl(SoftmaxOptions(dim)) {} + explicit SoftmaxImpl(const SoftmaxOptions& options_); + + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `Softmax` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + SoftmaxOptions options; +}; + +/// A `ModuleHolder` subclass for `SoftmaxImpl`. +/// See the documentation for `SoftmaxImpl` class to learn what methods it +/// provides, and examples of how to use `Softmax` with +/// `torch::nn::SoftmaxOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Softmax); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Softmin ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the Softmin function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Softmin to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::SoftminOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Softmin model(SoftminOptions(1)); +/// ``` +class TORCH_API SoftminImpl : public torch::nn::Cloneable { + public: + explicit SoftminImpl(int64_t dim) : SoftminImpl(SoftminOptions(dim)) {} + explicit SoftminImpl(const SoftminOptions& options_); + + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `Softmin` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + SoftminOptions options; +}; + +/// A `ModuleHolder` subclass for `SoftminImpl`. +/// See the documentation for `SoftminImpl` class to learn what methods it +/// provides, and examples of how to use `Softmin` with +/// `torch::nn::SoftminOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Softmin); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LogSoftmax ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the LogSoftmax function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.LogSoftmax to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::LogSoftmaxOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// LogSoftmax model(LogSoftmaxOptions(1)); +/// ``` +class TORCH_API LogSoftmaxImpl : public torch::nn::Cloneable { + public: + explicit LogSoftmaxImpl(int64_t dim) + : LogSoftmaxImpl(LogSoftmaxOptions(dim)) {} + explicit LogSoftmaxImpl(const LogSoftmaxOptions& options_); + + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `LogSoftmax` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + LogSoftmaxOptions options; +}; + +/// A `ModuleHolder` subclass for `LogSoftmaxImpl`. +/// See the documentation for `LogSoftmaxImpl` class to learn what methods it +/// provides, and examples of how to use `LogSoftmax` with +/// `torch::nn::LogSoftmaxOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(LogSoftmax); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Softmax2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the Softmax2d function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Softmax2d to learn +/// about the exact behavior of this module. +class TORCH_API Softmax2dImpl : public torch::nn::Cloneable { + public: + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `Softmax2d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `Softmax2dImpl`. +/// See the documentation for `Softmax2dImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(Softmax2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PReLU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the PReLU function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.PReLU to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::PReLUOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// PReLU model(PReLUOptions().num_parameters(42)); +/// ``` +class TORCH_API PReLUImpl : public torch::nn::Cloneable { + public: + explicit PReLUImpl(const PReLUOptions& options_ = {}); + + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `PReLU` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + PReLUOptions options; + + /// The learned weight. + Tensor weight; +}; + +/// A `ModuleHolder` subclass for `PReLUImpl`. +/// See the documentation for `PReLUImpl` class to learn what methods it +/// provides, and examples of how to use `PReLU` with `torch::nn::PReLUOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(PReLU); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ReLU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the ReLU function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ReLU to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ReLUOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ReLU model(ReLUOptions().inplace(true)); +/// ``` +class TORCH_API ReLUImpl : public torch::nn::Cloneable { + public: + explicit ReLUImpl(const ReLUOptions& options_ = {}); + + Tensor forward(Tensor input); + + void reset() override; + + /// Pretty prints the `ReLU` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + ReLUOptions options; +}; + +/// A `ModuleHolder` subclass for `ReLUImpl`. +/// See the documentation for `ReLUImpl` class to learn what methods it +/// provides, and examples of how to use `ReLU` with `torch::nn::ReLUOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(ReLU); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ReLU6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the ReLU6 function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ReLU6 to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ReLU6Options` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ReLU6 model(ReLU6Options().inplace(true)); +/// ``` +class TORCH_API ReLU6Impl : public torch::nn::Cloneable { + public: + explicit ReLU6Impl(const ReLU6Options& options_ = {}); + + Tensor forward(Tensor input); + + void reset() override; + + /// Pretty prints the `ReLU6` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + ReLU6Options options; +}; + +/// A `ModuleHolder` subclass for `ReLU6Impl`. +/// See the documentation for `ReLU6Impl` class to learn what methods it +/// provides, and examples of how to use `ReLU6` with `torch::nn::ReLU6Options`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(ReLU6); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RReLU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the RReLU function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.RReLU to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::RReLUOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// RReLU model(RReLUOptions().lower(0.24).upper(0.42).inplace(true)); +/// ``` +class TORCH_API RReLUImpl : public torch::nn::Cloneable { + public: + explicit RReLUImpl(const RReLUOptions& options_ = {}); + + Tensor forward(Tensor input); + + void reset() override; + + /// Pretty prints the `RReLU` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + RReLUOptions options; +}; + +/// A `ModuleHolder` subclass for `RReLUImpl`. +/// See the documentation for `RReLUImpl` class to learn what methods it +/// provides, and examples of how to use `RReLU` with `torch::nn::RReLUOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(RReLU); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CELU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies celu over a given input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.CELU to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::CELUOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// CELU model(CELUOptions().alpha(42.42).inplace(true)); +/// ``` +class TORCH_API CELUImpl : public torch::nn::Cloneable { + public: + explicit CELUImpl(const CELUOptions& options_ = {}); + + Tensor forward(Tensor input); + + void reset() override; + + /// Pretty prints the `CELU` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + CELUOptions options; +}; + +/// A `ModuleHolder` subclass for `CELUImpl`. +/// See the documentation for `CELUImpl` class to learn what methods it +/// provides, and examples of how to use `CELU` with `torch::nn::CELUOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(CELU); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GLU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies glu over a given input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.GLU to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::GLUOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// GLU model(GLUOptions(1)); +/// ``` +class TORCH_API GLUImpl : public torch::nn::Cloneable { + public: + explicit GLUImpl(const GLUOptions& options_ = {}); + + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `GLU` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + GLUOptions options; +}; + +/// A `ModuleHolder` subclass for `GLUImpl`. +/// See the documentation for `GLUImpl` class to learn what methods it +/// provides, and examples of how to use `GLU` with `torch::nn::GLUOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(GLU); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GELU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies gelu over a given input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.GELU to learn +/// about the exact behavior of this module. +class TORCH_API GELUImpl : public torch::nn::Cloneable { + public: + explicit GELUImpl(GELUOptions options_ = {}); + + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `GELU` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + GELUOptions options; +}; + +/// A `ModuleHolder` subclass for `GELUImpl`. +/// See the documentation for `GELUImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(GELU); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SiLU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies silu over a given input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.SiLU to learn +/// about the exact behavior of this module. +class TORCH_API SiLUImpl : public torch::nn::Cloneable { + public: + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `SiLU` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `SiLUImpl`. +/// See the documentation for `SiLUImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(SiLU); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Mish ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies mish over a given input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Mish to learn +/// about the exact behavior of this module. +class TORCH_API MishImpl : public torch::nn::Cloneable { + public: + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `Mish` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `MishImpl`. +/// See the documentation for `MishImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(Mish); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Sigmoid ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies sigmoid over a given input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Sigmoid to learn +/// about the exact behavior of this module. +class TORCH_API SigmoidImpl : public torch::nn::Cloneable { + public: + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `Sigmoid` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `SigmoidImpl`. +/// See the documentation for `SigmoidImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(Sigmoid); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Softplus ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies softplus over a given input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Softplus to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::SoftplusOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Softplus model(SoftplusOptions().beta(0.24).threshold(42.42)); +/// ``` +class TORCH_API SoftplusImpl : public torch::nn::Cloneable { + public: + explicit SoftplusImpl(const SoftplusOptions& options_ = {}); + + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `Softplus` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + SoftplusOptions options; +}; + +/// A `ModuleHolder` subclass for `SoftplusImpl`. +/// See the documentation for `SoftplusImpl` class to learn what methods it +/// provides, and examples of how to use `Softplus` with +/// `torch::nn::SoftplusOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Softplus); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Softshrink ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the soft shrinkage function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Softshrink to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::SoftshrinkOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Softshrink model(SoftshrinkOptions(42.42)); +/// ``` +class TORCH_API SoftshrinkImpl : public torch::nn::Cloneable { + public: + explicit SoftshrinkImpl(const SoftshrinkOptions& options_ = {}); + + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `Softshrink` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + SoftshrinkOptions options; +}; + +/// A `ModuleHolder` subclass for `SoftshrinkImpl`. +/// See the documentation for `SoftshrinkImpl` class to learn what methods it +/// provides, and examples of how to use `Softshrink` with +/// `torch::nn::SoftshrinkOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Softshrink); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Softsign ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies Softsign over a given input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Softsign to learn +/// about the exact behavior of this module. +class TORCH_API SoftsignImpl : public torch::nn::Cloneable { + public: + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `Softsign` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `SoftsignImpl`. +/// See the documentation for `SoftsignImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(Softsign); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Tanh ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies Tanh over a given input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Tanh to learn +/// about the exact behavior of this module. +class TORCH_API TanhImpl : public torch::nn::Cloneable { + public: + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `Tanh` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `TanhImpl`. +/// See the documentation for `TanhImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(Tanh); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Tanhshrink ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies Tanhshrink over a given input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Tanhshrink to learn +/// about the exact behavior of this module. +class TORCH_API TanhshrinkImpl : public torch::nn::Cloneable { + public: + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `Tanhshrink` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `TanhshrinkImpl`. +/// See the documentation for `TanhshrinkImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(Tanhshrink); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Threshold ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the Threshold function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Threshold to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ThresholdOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Threshold model(ThresholdOptions(42.42, 24.24).inplace(true)); +/// ``` +class TORCH_API ThresholdImpl : public torch::nn::Cloneable { + public: + ThresholdImpl(double threshold, double value) + : ThresholdImpl(ThresholdOptions(threshold, value)) {} + explicit ThresholdImpl(const ThresholdOptions& options_); + + Tensor forward(Tensor input); + + void reset() override; + + /// Pretty prints the `Threshold` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + ThresholdOptions options; +}; + +/// A `ModuleHolder` subclass for `ThresholdImpl`. +/// See the documentation for `ThresholdImpl` class to learn what methods it +/// provides, and examples of how to use `Threshold` with +/// `torch::nn::ThresholdOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Threshold); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MultiheadAttention ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the MultiheadAttention function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.MultiheadAttention +/// to learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MultiheadAttentionOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MultiheadAttention model(MultiheadAttentionOptions(20, 10).bias(false)); +/// ``` +class TORCH_API MultiheadAttentionImpl + : public torch::nn::Cloneable { + public: + MultiheadAttentionImpl(int64_t embed_dim, int64_t num_heads) + : MultiheadAttentionImpl( + MultiheadAttentionOptions(embed_dim, num_heads)) {} + explicit MultiheadAttentionImpl(const MultiheadAttentionOptions& options_); + + std::tuple forward( + const Tensor& query, + const Tensor& key, + const Tensor& value, + const Tensor& key_padding_mask = {}, + bool need_weights = true, + const Tensor& attn_mask = {}, + bool average_attn_weights = true); + + protected: + FORWARD_HAS_DEFAULT_ARGS( + {3, AnyValue(Tensor())}, + {4, AnyValue(true)}, + {5, AnyValue(Tensor())}, + {6, AnyValue(true)}) + + public: + void reset() override; + + void _reset_parameters(); + + /// The options with which this `Module` was constructed. + MultiheadAttentionOptions options; + + bool _qkv_same_embed_dim{}; + Tensor in_proj_weight; + Tensor in_proj_bias; + Tensor bias_k; + Tensor bias_v; + Linear out_proj = nullptr; + Tensor q_proj_weight; + Tensor k_proj_weight; + Tensor v_proj_weight; + int64_t head_dim{}; +}; + +/// A `ModuleHolder` subclass for `MultiheadAttentionImpl`. +/// See the documentation for `MultiheadAttentionImpl` class to learn what +/// methods it provides, and examples of how to use `MultiheadAttention` with +/// `torch::nn::MultiheadAttentionOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(MultiheadAttention); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/adaptive.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/adaptive.h new file mode 100644 index 0000000000000000000000000000000000000000..3cb8bc210a5d2ed3e81976784635bb8238b2178d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/adaptive.h @@ -0,0 +1,109 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace torch::nn { + +/// The output of a single invocation of an AdaptiveLogSoftmaxWithLoss +/// module's `forward()` method. +struct TORCH_API ASMoutput { + ASMoutput(Tensor output_, double loss_); + + /// Tensor containing computed target log probabilities for each example + Tensor output; + + /// Scalar representing the computed negative log likelihood loss + double loss; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ AdaptiveLogSoftmaxWithLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Efficient softmax approximation as described in +/// `Efficient softmax approximation for GPUs`_ by Edouard Grave, Armand Joulin, +/// Moustapha Cissé, David Grangier, and Hervé Jégou. +/// See +/// https://pytorch.org/docs/main/nn.html#torch.nn.AdaptiveLogSoftmaxWithLoss +/// to learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::AdaptiveLogSoftmaxWithLossOptions` +/// class to learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// AdaptiveLogSoftmaxWithLoss model(AdaptiveLogSoftmaxWithLossOptions(8, 10, +/// {4, 8}).div_value(2.).head_bias(true)); +/// ``` +class TORCH_API AdaptiveLogSoftmaxWithLossImpl + : public Cloneable { + public: + AdaptiveLogSoftmaxWithLossImpl( + int64_t in_features, + int64_t n_classes, + std::vector cutoffs) + : AdaptiveLogSoftmaxWithLossImpl(AdaptiveLogSoftmaxWithLossOptions( + in_features, + n_classes, + std::move(cutoffs))) {} + + explicit AdaptiveLogSoftmaxWithLossImpl( + AdaptiveLogSoftmaxWithLossOptions options_); + + ASMoutput forward(const Tensor& input, const Tensor& target); + + void reset() override; + + void reset_parameters(); + + /// Pretty prints the `AdaptiveLogSoftmaxWithLoss` module into the given + /// `stream`. + void pretty_print(std::ostream& stream) const override; + + /// Given input tensor, and output of `head`, computes the log of the full + /// distribution + Tensor _get_full_log_prob(const Tensor& input, const Tensor& head_output); + + /// Computes log probabilities for all n_classes + Tensor log_prob(const Tensor& input); + + /// This is equivalent to `log_pob(input).argmax(1)` but is more efficient in + /// some cases + Tensor predict(const Tensor& input); + + /// The options with which this `Module` was constructed + AdaptiveLogSoftmaxWithLossOptions options; + + /// Cutoffs used to assign targets to their buckets. It should be an ordered + /// Sequence of integers sorted in the increasing order + std::vector cutoffs; + + int64_t shortlist_size; + + /// Number of clusters + int64_t n_clusters; + + /// Output size of head classifier + int64_t head_size; + + Linear head = nullptr; + + ModuleList tail; +}; + +/// A `ModuleHolder` subclass for `AdaptiveLogSoftmaxWithLossImpl`. +/// See the documentation for `AdaptiveLogSoftmaxWithLossImpl` class to learn +/// what methods it provides, and examples of how to use +/// `AdaptiveLogSoftmaxWithLoss` with +/// `torch::nn::AdaptiveLogSoftmaxWithLossOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(AdaptiveLogSoftmaxWithLoss); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/batchnorm.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/batchnorm.h new file mode 100644 index 0000000000000000000000000000000000000000..92ecbc4547ef09760257548729985724fddff5d1 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/batchnorm.h @@ -0,0 +1,242 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace torch::nn { + +/// Base class for all (dimension-specialized) batchnorm and instancenorm +/// modules. +template +class NormImplBase : public torch::nn::Cloneable { + protected: + virtual void _check_input_dim(const Tensor& input) = 0; + + public: + NormImplBase(const DerivedOptions& options_) : options(options_) { + NormImplBase::reset(); + } + + void reset() override { + if (options.affine()) { + weight = this->register_parameter( + "weight", torch::empty({options.num_features()})); + bias = this->register_parameter( + "bias", torch::empty({options.num_features()})); + } else { + weight = + this->register_parameter("weight", Tensor(), /*requires_grad=*/false); + bias = + this->register_parameter("bias", Tensor(), /*requires_grad=*/false); + } + if (options.track_running_stats()) { + running_mean = this->register_buffer( + "running_mean", torch::zeros({options.num_features()})); + running_var = this->register_buffer( + "running_var", torch::ones({options.num_features()})); + num_batches_tracked = this->register_buffer( + "num_batches_tracked", torch::tensor(0, torch::dtype(torch::kLong))); + } else { + running_mean = this->register_buffer("running_mean", Tensor()); + running_var = this->register_buffer("running_var", Tensor()); + num_batches_tracked = + this->register_buffer("num_batches_tracked", Tensor()); + } + reset_parameters(); + } + + void reset_running_stats() { + if (options.track_running_stats()) { + running_mean.zero_(); + running_var.fill_(1); + num_batches_tracked.zero_(); + } + } + + void reset_parameters() { + reset_running_stats(); + if (options.affine()) { + torch::nn::init::ones_(weight); + torch::nn::init::zeros_(bias); + } + } + + /// The options with which this module was constructed. + DerivedOptions options; + + /// The learned weight. + /// Only defined if the `affine` option was `true` upon construction. + Tensor weight; + + /// The learned bias. + /// Only defined if the `affine` option was `true` upon construction. + Tensor bias; + + /// The running mean. + /// Only defined if the `track_running_stats` option was `true` upon + /// construction. + Tensor running_mean; + + /// The running variance. + /// Only defined if the `track_running_stats` option was `true` upon + /// construction. + Tensor running_var; + + /// The number of the forward call. + /// Only defined if the `track_running_stats` option was `true` upon + /// construction. + Tensor num_batches_tracked; +}; + +/// Base class for all (dimension-specialized) batchnorm modules. +template +class BatchNormImplBase : public NormImplBase { + public: + using NormImplBase::NormImplBase; + + Tensor forward(const Tensor& input) { + this->_check_input_dim(input); + double exponential_average_factor = 0.0; + if (this->options.momentum().has_value()) { + exponential_average_factor = this->options.momentum().value(); + } + + if (this->is_training() && this->options.track_running_stats()) { + if (this->num_batches_tracked.defined()) { + this->num_batches_tracked += 1; + if (this->options.momentum() == + std::nullopt) { // use cumulative moving average + exponential_average_factor = + 1.0 / this->num_batches_tracked.template item(); + } else { // use exponential moving average + exponential_average_factor = this->options.momentum().value(); + } + } + } + + return torch::nn::functional::detail::batch_norm( + input, + this->running_mean, + this->running_var, + this->weight, + this->bias, + this->is_training() || !this->options.track_running_stats(), + /*momentum=*/exponential_average_factor, + this->options.eps()); + } + + /// Pretty prints the `BatchNorm{1,2,3}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override { + stream << std::boolalpha << "torch::nn::BatchNorm" << D << "d(" + << this->options.num_features() << ", " + << "eps=" << this->options.eps() << ", " + << "momentum="; + + if (this->options.momentum().has_value()) { + stream << this->options.momentum().value(); + } else { + stream << "None"; + } + + stream << ", " + << "affine=" << this->options.affine() << ", " + << "track_running_stats=" << this->options.track_running_stats() + << ')'; + } +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BatchNorm1d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the BatchNorm1d function. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.BatchNorm1d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::BatchNorm1dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// BatchNorm1d +/// model(BatchNorm1dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +class TORCH_API BatchNorm1dImpl : public BatchNormImplBase<1, BatchNorm1dImpl> { + protected: + void _check_input_dim(const Tensor& input) override; + + public: + using BatchNormImplBase<1, BatchNorm1dImpl>::BatchNormImplBase; +}; + +/// A `ModuleHolder` subclass for `BatchNorm1dImpl`. +/// See the documentation for `BatchNorm1dImpl` class to learn what methods it +/// provides, and examples of how to use `BatchNorm1d` with +/// `torch::nn::BatchNorm1dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(BatchNorm1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BatchNorm2d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the BatchNorm2d function. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.BatchNorm2d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::BatchNorm2dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// BatchNorm2d +/// model(BatchNorm2dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +class TORCH_API BatchNorm2dImpl : public BatchNormImplBase<2, BatchNorm2dImpl> { + protected: + void _check_input_dim(const Tensor& input) override; + + public: + using BatchNormImplBase<2, BatchNorm2dImpl>::BatchNormImplBase; +}; + +/// A `ModuleHolder` subclass for `BatchNorm2dImpl`. +/// See the documentation for `BatchNorm2dImpl` class to learn what methods it +/// provides, and examples of how to use `BatchNorm2d` with +/// `torch::nn::BatchNorm2dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(BatchNorm2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BatchNorm3d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the BatchNorm3d function. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.BatchNorm3d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::BatchNorm3dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// BatchNorm3d +/// model(BatchNorm3dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +class TORCH_API BatchNorm3dImpl : public BatchNormImplBase<3, BatchNorm3dImpl> { + protected: + void _check_input_dim(const Tensor& input) override; + + public: + using BatchNormImplBase<3, BatchNorm3dImpl>::BatchNormImplBase; +}; + +/// A `ModuleHolder` subclass for `BatchNorm3dImpl`. +/// See the documentation for `BatchNorm3dImpl` class to learn what methods it +/// provides, and examples of how to use `BatchNorm3d` with +/// `torch::nn::BatchNorm3dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(BatchNorm3d); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/common.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/common.h new file mode 100644 index 0000000000000000000000000000000000000000..557bc611d2c4a2a1cf09d753b63dd076f9baee83 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/common.h @@ -0,0 +1,99 @@ +#pragma once + +/// This macro enables a module with default arguments in its forward method +/// to be used in a Sequential module. +/// +/// Example usage: +/// +/// Let's say we have a module declared like this: +/// ``` +/// struct MImpl : torch::nn::Module { +/// public: +/// explicit MImpl(int value_) : value(value_) {} +/// torch::Tensor forward(int a, int b = 2, double c = 3.0) { +/// return torch::tensor(a + b + c); +/// } +/// private: +/// int value; +/// }; +/// TORCH_MODULE(M); +/// ``` +/// +/// If we try to use it in a Sequential module and run forward: +/// ``` +/// torch::nn::Sequential seq(M(1)); +/// seq->forward(1); +/// ``` +/// +/// We will receive the following error message: +/// ``` +/// MImpl's forward() method expects 3 argument(s), but received 1. +/// If MImpl's forward() method has default arguments, please make sure +/// the forward() method is declared with a corresponding +/// `FORWARD_HAS_DEFAULT_ARGS` macro. +/// ``` +/// +/// The right way to fix this error is to use the `FORWARD_HAS_DEFAULT_ARGS` +/// macro when declaring the module: +/// ``` +/// struct MImpl : torch::nn::Module { +/// public: +/// explicit MImpl(int value_) : value(value_) {} +/// torch::Tensor forward(int a, int b = 2, double c = 3.0) { +/// return torch::tensor(a + b + c); +/// } +/// protected: +/// /* +/// NOTE: looking at the argument list of `forward`: +/// `forward(int a, int b = 2, double c = 3.0)` +/// we saw the following default arguments: +/// ---------------------------------------------------------------- +/// 0-based index of default | Default value of arg +/// arg in forward arg list | (wrapped by `torch::nn::AnyValue()`) +/// ---------------------------------------------------------------- +/// 1 | torch::nn::AnyValue(2) +/// 2 | torch::nn::AnyValue(3.0) +/// ---------------------------------------------------------------- +/// Thus we pass the following arguments to the `FORWARD_HAS_DEFAULT_ARGS` +/// macro: +/// */ +/// FORWARD_HAS_DEFAULT_ARGS({1, torch::nn::AnyValue(2)}, {2, +/// torch::nn::AnyValue(3.0)}) +/// private: +/// int value; +/// }; +/// TORCH_MODULE(M); +/// ``` +/// Now, running the following would work: +/// ``` +/// torch::nn::Sequential seq(M(1)); +/// seq->forward(1); // This correctly populates the default arguments for +/// `MImpl::forward` +/// ``` +#define FORWARD_HAS_DEFAULT_ARGS(...) \ + template \ + friend struct torch::nn::AnyModuleHolder; \ + bool _forward_has_default_args() override { \ + return true; \ + } \ + unsigned int _forward_num_required_args() override { \ + std::vector> args_info{ \ + __VA_ARGS__}; \ + return std::begin(args_info)->first; \ + } \ + std::vector _forward_populate_default_args( \ + std::vector&& arguments) override { \ + std::vector> args_info{ \ + __VA_ARGS__}; \ + unsigned int num_all_args = std::rbegin(args_info)->first + 1; \ + TORCH_INTERNAL_ASSERT( \ + arguments.size() >= _forward_num_required_args() && \ + arguments.size() <= num_all_args); \ + std::vector ret = std::move(arguments); \ + ret.reserve(num_all_args); \ + for (auto& arg_info : args_info) { \ + if (arg_info.first > ret.size() - 1) \ + ret.emplace_back(std::move(arg_info.second)); \ + } \ + return ret; \ + } diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/any.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/any.h new file mode 100644 index 0000000000000000000000000000000000000000..fc57391df49bbcbfe0a00ef2db989ae91d80d58c --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/any.h @@ -0,0 +1,363 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +namespace torch::nn { + +/// Stores a type erased `Module`. +/// +/// The PyTorch C++ API does not impose an interface on the signature of +/// `forward()` in `Module` subclasses. This gives you complete freedom to +/// design your `forward()` methods to your liking. However, this also means +/// there is no unified base type you could store in order to call `forward()` +/// polymorphically for any module. This is where the `AnyModule` comes in. +/// Instead of inheritance, it relies on type erasure for polymorphism. +/// +/// An `AnyModule` can store any `nn::Module` subclass that provides a +/// `forward()` method. This `forward()` may accept any types and return any +/// type. Once stored in an `AnyModule`, you can invoke the underlying module's +/// `forward()` by calling `AnyModule::forward()` with the arguments you would +/// supply to the stored module (though see one important limitation below). +/// Example: +/// +/// \rst +/// .. code-block:: cpp +/// +/// struct GenericTrainer { +/// torch::nn::AnyModule module; +/// +/// void train(torch::Tensor input) { +/// module.forward(input); +/// } +/// }; +/// +/// GenericTrainer trainer1{torch::nn::Linear(3, 4)}; +/// GenericTrainer trainer2{torch::nn::Conv2d(3, 4, 2)}; +/// \endrst +/// +/// As `AnyModule` erases the static type of the stored module (and its +/// `forward()` method) to achieve polymorphism, type checking of arguments is +/// moved to runtime. That is, passing an argument with an incorrect type to an +/// `AnyModule` will compile, but throw an exception at runtime: +/// +/// \rst +/// .. code-block:: cpp +/// +/// torch::nn::AnyModule module(torch::nn::Linear(3, 4)); +/// // Linear takes a tensor as input, but we are passing an integer. +/// // This will compile, but throw a `torch::Error` exception at runtime. +/// module.forward(123); +/// \endrst +/// +/// \rst +/// .. attention:: +/// One noteworthy limitation of `AnyModule` is that its `forward()` method +/// does not support implicit conversion of argument types. For example, if +/// the stored module's `forward()` method accepts a `float` and you call +/// `any_module.forward(3.4)` (where `3.4` is a `double`), this will throw +/// an exception. +/// \endrst +/// +/// The return type of the `AnyModule`'s `forward()` method is controlled via +/// the first template argument to `AnyModule::forward()`. It defaults to +/// `torch::Tensor`. To change it, you can write `any_module.forward()`, +/// for example. +/// +/// \rst +/// .. code-block:: cpp +/// +/// torch::nn::AnyModule module(torch::nn::Linear(3, 4)); +/// auto output = module.forward(torch::ones({2, 3})); +/// +/// struct IntModule { +/// int forward(int x) { return x; } +/// }; +/// torch::nn::AnyModule module(IntModule{}); +/// int output = module.forward(5); +/// \endrst +/// +/// The only other method an `AnyModule` provides access to on the stored +/// module is `clone()`. However, you may acquire a handle on the module via +/// `.ptr()`, which returns a `shared_ptr`. Further, if you know +/// the concrete type of the stored module, you can get a concrete handle to it +/// using `.get()` where `T` is the concrete module type. +/// +/// \rst +/// .. code-block:: cpp +/// +/// torch::nn::AnyModule module(torch::nn::Linear(3, 4)); +/// std::shared_ptr ptr = module.ptr(); +/// torch::nn::Linear linear(module.get()); +/// \endrst +class AnyModule { + public: + /// A default-constructed `AnyModule` is in an empty state. + AnyModule() = default; + + /// Constructs an `AnyModule` from a `shared_ptr` to concrete module object. + template + explicit AnyModule(std::shared_ptr module); + + /// Constructs an `AnyModule` from a concrete module object. + template < + typename ModuleType, + typename = torch::detail::enable_if_module_t> + explicit AnyModule(ModuleType&& module); + + /// Constructs an `AnyModule` from a module holder. + template + explicit AnyModule(const ModuleHolder& module_holder); + + /// Move construction and assignment is allowed, and follows the default + /// behavior of move for `std::unique_ptr`. + AnyModule(AnyModule&&) = default; + AnyModule& operator=(AnyModule&&) = default; + + /// Creates a shallow copy of an `AnyModule`. + AnyModule(const AnyModule& other); + AnyModule& operator=(const AnyModule& other); + + /// Creates a deep copy of an `AnyModule` if it contains a module, else an + /// empty `AnyModule` if it is empty. + AnyModule clone(std::optional device = std::nullopt) const; + + /// Assigns a module to the `AnyModule` (to circumvent the explicit + /// constructor). + template + AnyModule& operator=(std::shared_ptr module); + + /// Invokes `forward()` on the contained module with the given arguments, and + /// returns the return value as an `AnyValue`. Use this method when chaining + /// `AnyModule`s in a loop. + template + AnyValue any_forward(ArgumentTypes&&... arguments); + + /// Invokes `forward()` on the contained module with the given arguments, and + /// casts the returned `AnyValue` to the supplied `ReturnType` (which defaults + /// to `torch::Tensor`). + template + ReturnType forward(ArgumentTypes&&... arguments); + + /// Attempts to cast the underlying module to the given module type. Throws an + /// exception if the types do not match. + template > + T& get(); + + /// Attempts to cast the underlying module to the given module type. Throws an + /// exception if the types do not match. + template > + const T& get() const; + + /// Returns the contained module in a `nn::ModuleHolder` subclass if possible + /// (i.e. if `T` has a constructor for the underlying module type). + template + T get() const; + + /// Returns a `std::shared_ptr` whose dynamic type is that of the underlying + /// module. + std::shared_ptr ptr() const; + + /// Like `ptr()`, but casts the pointer to the given type. + template > + std::shared_ptr ptr() const; + + /// Returns the `type_info` object of the contained value. + const std::type_info& type_info() const; + + /// Returns true if the `AnyModule` does not contain a module. + bool is_empty() const noexcept; + + private: + /// Creates a `unique_ptr` pointing to a + /// `AnyModuleHolder` of the correct type. This method is used to deduce the + /// arguments of the module's `forward()` method. + template < + typename ModuleType, + typename Class, + typename ReturnType, + typename... ArgumentTypes> + std::unique_ptr make_holder( + std::shared_ptr&& module, + ReturnType (Class::* /*unused*/)(ArgumentTypes...)); + + /// Helper method invoked by const and non-const `get()`. + template + ModuleType& get_( + ReturnType (ModuleType::* /*unused*/)(ArgumentTypes...)) const; + + /// Helper method invoked by const and non-const `get()`. + template + ModuleType& get_() const; + + /// The type erased module. + std::unique_ptr content_; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ AnyModule ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +template +AnyModule::AnyModule(std::shared_ptr module) + : content_(make_holder( + std::move(module), + &std::remove_reference_t::forward)) { + // `AnyModule` can only store an `nn::Module` subclass object that provides + // a `forward()` method that has a non-templatized return type. + // (e.g. `AnyModule` cannot store `nn::Sequential`, because `nn::Sequential`'s + // `forward()` method has a templatized return type.) + static_assert( + torch::detail::is_module::value, + "Can only store object derived from nn::Module into AnyModule"); + static_assert( + torch::detail::has_forward::value, + "Can only store module with a forward() method that has a non-templatized" + " argument type and return type into AnyModule (e.g. we cannot store nn::Sequential" + "into AnyModule, because its forward() method's argument type and return type are templatized." + " If you need to use nn::Sequentials inside each other you can subclass " + "nn::Sequential and write a non-templatized forward function for it. You can checkout " + "https://github.com/pytorch/vision/blob/2f46070f3cb1ea894d82578f3dc5677f82f34958/torchvision/csrc/models/mnasnet.cpp#L59 " + "for an example on how to do this.)."); +} + +template +AnyModule::AnyModule(ModuleType&& module) + : AnyModule( + std::make_shared(std::forward(module))) {} + +template +AnyModule::AnyModule(const ModuleHolder& module_holder) + : AnyModule(module_holder.ptr()) {} + +inline AnyModule::AnyModule(const AnyModule& other) + : content_(other.content_ ? other.content_->copy() : nullptr) {} + +inline AnyModule& AnyModule::operator=(const AnyModule& other) { + if (this != &other) { + content_ = other.content_ ? other.content_->copy() : nullptr; + } + return *this; +} + +inline AnyModule AnyModule::clone(std::optional device) const { + AnyModule clone; + clone.content_ = content_ ? content_->clone_module(device) : nullptr; + return clone; +} + +template +AnyModule& AnyModule::operator=(std::shared_ptr module) { + *this = AnyModule(std::move(module)); + return *this; +} + +template +AnyValue AnyModule::any_forward(ArgumentTypes&&... arguments) { + TORCH_CHECK(!is_empty(), "Cannot call forward() on an empty AnyModule"); + std::vector values; + values.reserve(sizeof...(ArgumentTypes)); + torch::apply( + [&values](AnyValue&& value) { values.push_back(std::move(value)); }, + AnyValue(std::forward(arguments))...); + return content_->forward(std::move(values)); +} + +template +ReturnType AnyModule::forward(ArgumentTypes&&... arguments) { + return any_forward(std::forward(arguments)...) + .template get(); +} + +template +T& AnyModule::get() { + TORCH_CHECK(!is_empty(), "Cannot call get() on an empty AnyModule"); + return get_(); +} + +template +const T& AnyModule::get() const { + TORCH_CHECK(!is_empty(), "Cannot call get() on an empty AnyModule"); + return get_(); +} + +template +T AnyModule::get() const { + return T(ptr()); +} + +inline std::shared_ptr AnyModule::ptr() const { + TORCH_CHECK(!is_empty(), "Cannot call ptr() on an empty AnyModule"); + return content_->ptr(); +} + +template +std::shared_ptr AnyModule::ptr() const { + TORCH_CHECK(!is_empty(), "Cannot call ptr() on an empty AnyModule"); + // Call get() but discard the value, just to do the type checking. + get_(); + return std::dynamic_pointer_cast(ptr()); +} + +inline const std::type_info& AnyModule::type_info() const { + TORCH_CHECK(!is_empty(), "Cannot call type_info() on an empty AnyModule"); + return content_->type_info; +} + +inline bool AnyModule::is_empty() const noexcept { + return content_ == nullptr; +} + +// Private Methods + +template < + typename ModuleType, + typename Class, + typename ReturnType, + typename... ArgumentTypes> +std::unique_ptr AnyModule::make_holder( + std::shared_ptr&& module, + ReturnType (Class::* /*unused*/)(ArgumentTypes...)) { + static_assert( + torch::detail::check_not_lvalue_references(), + "Modules stored inside AnyModule must not take references. " + "Use pointers instead."); + static_assert( + !std::is_void_v, + "AnyModule cannot store modules that return void " + "(you can return a dummy value)."); + return std::make_unique< + AnyModuleHolder, ArgumentTypes...>>( + std::move(module)); +} + +template +ModuleType& AnyModule::get_() const { + using M = std::remove_reference_t; + static_assert( + torch::detail::has_forward::value, + "Can only call AnyModule::get with a type T that has a forward method"); + return get_(&M::forward); +} + +template +ModuleType& AnyModule::get_( + ReturnType (ModuleType::* /*unused*/)(ArgumentTypes...)) const { + if (typeid(ModuleType).hash_code() == type_info().hash_code()) { + return *static_cast&>( + *content_) + .module; + } + TORCH_CHECK( + false, + "Attempted to cast module of type ", + c10::demangle(type_info().name()), + " to type ", + c10::demangle(typeid(ModuleType).name())); +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/any_module_holder.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/any_module_holder.h new file mode 100644 index 0000000000000000000000000000000000000000..6d5b557d34f1e7e17246fc74aa41aafd37f9e5dd --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/any_module_holder.h @@ -0,0 +1,135 @@ +#pragma once + +#include +#include + +namespace torch::nn { + +class Module; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~ AnyModulePlaceholder ~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// The static type of the object we store in the `AnyModule`, which erases +/// the actual type, but allows us to call `forward()` on the underlying +/// module. +struct AnyModulePlaceholder : public AnyValue::Placeholder { + using AnyValue::Placeholder::Placeholder; + + /// The "erased" `forward()` method. + virtual AnyValue forward(std::vector&& arguments) = 0; + + /// Returns std::shared_ptr pointing to the erased module. + virtual std::shared_ptr ptr() = 0; + + /// Returns a `AnyModulePlaceholder` with a shallow copy of this `AnyModule`. + virtual std::unique_ptr copy() const = 0; + + /// Returns a `AnyModulePlaceholder` with a deep copy of this `AnyModule`. + virtual std::unique_ptr clone_module( + std::optional device) const = 0; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ AnyModuleHolder ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// The dynamic type of the object stored in the `AnyModule`. It contains the +/// concrete instance to which all calls are forwarded. It is parameterized +/// over the concrete type of the module, and the types of the arguments the +/// module takes in its `forward()` method. +template +struct AnyModuleHolder : public AnyModulePlaceholder { + /// \internal + struct CheckedGetter { + template + std::decay_t&& operator()(size_t index) { + AT_ASSERT(index < arguments_.size()); + auto& value = arguments_[index]; + if (auto* maybe_value = value.template try_get>()) { + return std::move(*maybe_value); + } + TORCH_CHECK( + false, + "Expected argument #", + index, + " to be of type ", + c10::demangle(typeid(T).name()), + ", but received value of type ", + c10::demangle(value.type_info().name())); + } + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + std::vector& arguments_; + }; + + /// \internal + struct InvokeForward { + template + AnyValue operator()(Ts&&... ts) { + return AnyValue(module_->forward(std::forward(ts)...)); + } + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + std::shared_ptr& module_; + }; + + /// Constructs the `AnyModuleHolder` from a concrete module. + explicit AnyModuleHolder(std::shared_ptr&& module_) + : AnyModulePlaceholder(typeid(ModuleType)), module(std::move(module_)) {} + + /// Calls `forward()` on the underlying module, casting each `AnyValue` in the + /// argument vector to a concrete value. + AnyValue forward(std::vector&& arguments) override { + if (module->_forward_has_default_args()) { + TORCH_CHECK( + arguments.size() >= module->_forward_num_required_args() && + arguments.size() <= sizeof...(ArgumentTypes), + c10::demangle(type_info.name()), + "'s forward() method expects at least ", + module->_forward_num_required_args(), + " argument(s) and at most ", + sizeof...(ArgumentTypes), + " argument(s), but received ", + arguments.size(), + "."); + arguments = std::move( + module->_forward_populate_default_args(std::move(arguments))); + } else { + std::string use_default_args_macro_prompt = " If " + + c10::demangle(type_info.name()) + + "'s forward() method has default arguments, " + + "please make sure the forward() method is declared with a corresponding `FORWARD_HAS_DEFAULT_ARGS` macro."; + TORCH_CHECK( + arguments.size() == sizeof...(ArgumentTypes), + c10::demangle(type_info.name()), + "'s forward() method expects ", + sizeof...(ArgumentTypes), + " argument(s), but received ", + arguments.size(), + ".", + (arguments.size() < sizeof...(ArgumentTypes)) + ? use_default_args_macro_prompt + : ""); + } + + // FYI: During invocation of a module's `forward()` method, the values live + // in the `arguments` vector inside this function. + return torch::unpack( + InvokeForward{module}, CheckedGetter{arguments}); + } + + std::shared_ptr ptr() override { + return module; + } + + std::unique_ptr copy() const override { + return std::make_unique(*this); + } + + std::unique_ptr clone_module( + std::optional device) const override { + return std::make_unique( + std::dynamic_pointer_cast(module->clone(device))); + } + + /// The actual concrete module instance. + std::shared_ptr module; +}; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/any_value.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/any_value.h new file mode 100644 index 0000000000000000000000000000000000000000..4612e75517bb3319c95efadbcd0eafd5b5fae92b --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/any_value.h @@ -0,0 +1,124 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace torch::nn { + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ AnyValue ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// An implementation of `std::any` which stores +/// a type erased object, whose concrete value can be retrieved at runtime by +/// checking if the `typeid()` of a requested type matches the `typeid()` of +/// the object stored. +class AnyValue { + public: + /// Move construction and assignment is allowed, and follows the default + /// behavior of move for `std::unique_ptr`. + AnyValue(AnyValue&&) = default; + AnyValue& operator=(AnyValue&&) = default; + ~AnyValue() = default; + + /// Copy construction and assignment is allowed. + AnyValue(const AnyValue& other) : content_(other.content_->clone()) {} + AnyValue& operator=(const AnyValue& other) { + content_ = other.content_->clone(); + return *this; + } + + /// Constructs the `AnyValue` from value type. + template < + typename T, + typename = std::enable_if_t>> + explicit AnyValue(T&& value) + : content_( + std::make_unique>>(std::forward(value))) { + } + + /// Returns a pointer to the value contained in the `AnyValue` if the type + /// passed as template parameter matches the type of the value stored, and + /// returns a null pointer otherwise. + template + T* try_get() { + static_assert( + !std::is_reference_v, + "AnyValue stores decayed types, you cannot cast it to a reference type"); + static_assert( + !std::is_array_v, + "AnyValue stores decayed types, you must cast it to T* instead of T[]"); + if (typeid(T).hash_code() == type_info().hash_code()) { + return &static_cast&>(*content_).value; + } + return nullptr; + } + + /// Returns the value contained in the `AnyValue` if the type passed as + /// template parameter matches the type of the value stored, and throws an + /// exception otherwise. + template + T get() { + if (auto* maybe_value = try_get()) { + return *maybe_value; + } + TORCH_CHECK( + false, + "Attempted to cast AnyValue to ", + c10::demangle(typeid(T).name()), + ", but its actual type is ", + c10::demangle(type_info().name())); + } + + /// Returns the `type_info` object of the contained value. + const std::type_info& type_info() const noexcept { + return content_->type_info; + } + + private: + friend struct AnyModulePlaceholder; + friend struct TestAnyValue; + + /// \internal + /// The static type of the object we store in the `AnyValue`, which erases the + /// actual object's type, allowing us only to check the `type_info` of the + /// type stored in the dynamic type. + struct Placeholder { + explicit Placeholder(const std::type_info& type_info_) noexcept + : type_info(type_info_) {} + Placeholder(const Placeholder&) = default; + Placeholder(Placeholder&&) = default; + Placeholder& operator=(const Placeholder&) = delete; + Placeholder& operator=(Placeholder&&) = delete; + virtual ~Placeholder() = default; + virtual std::unique_ptr clone() const { + TORCH_CHECK(false, "clone() should only be called on `AnyValue::Holder`"); + } + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const std::type_info& type_info; + }; + + /// \internal + /// The dynamic type of the object we store in the `AnyValue`, which hides the + /// actual object we have erased in this `AnyValue`. + template + struct Holder : public Placeholder { + /// A template because T&& would not be universal reference here. + template < + typename U, + typename = std::enable_if_t>> + explicit Holder(U&& value_) noexcept + : Placeholder(typeid(T)), value(std::forward(value_)) {} + std::unique_ptr clone() const override { + return std::make_unique>(value); + } + T value; + }; + + /// The type erased object. + std::unique_ptr content_; +}; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/functional.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/functional.h new file mode 100644 index 0000000000000000000000000000000000000000..0d0cdd8861b62277aca5615e2991b53475eaaa66 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/functional.h @@ -0,0 +1,101 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace torch::nn { + +/// Wraps a function in a `Module`. +/// +/// The `Functional` module allows wrapping an arbitrary function or function +/// object in an `nn::Module`. This is primarily handy for usage in +/// `Sequential`. +/// +/// \rst +/// .. code-block:: cpp +/// +/// Sequential sequential( +/// Linear(3, 4), +/// Functional(torch::relu), +/// BatchNorm1d(3), +/// Functional(torch::elu, /*alpha=*/1)); +/// \endrst +/// +/// While a `Functional` module only accepts a single `Tensor` as input, it is +/// possible for the wrapped function to accept further arguments. However, +/// these have to be bound *at construction time*. For example, if +/// you want to wrap `torch::leaky_relu`, which accepts a `slope` scalar as its +/// second argument, with a particular value for its `slope` in a `Functional` +/// module, you could write +/// +/// \rst +/// .. code-block:: cpp +/// +/// Functional(torch::leaky_relu, /*slope=*/0.5) +/// \endrst +/// +/// The value of `0.5` is then stored within the `Functional` object and +/// supplied to the function call at invocation time. Note that such bound +/// values are evaluated eagerly and stored a single time. See the documentation +/// of [std::bind](https://en.cppreference.com/w/cpp/utility/functional/bind) +/// for more information on the semantics of argument binding. +/// +/// \rst +/// .. attention:: +/// After passing any bound arguments, the function must accept a single +/// tensor and return a single tensor. +/// \endrst +/// +/// Note that `Functional` overloads the call operator (`operator()`) such that +/// you can invoke it with `my_func(...)`. +class TORCH_API FunctionalImpl : public torch::nn::Cloneable { + public: + using Function = std::function; + + /// Constructs a `Functional` from a function object. + explicit FunctionalImpl(Function function); + + template < + typename SomeFunction, + typename... Args, + typename = std::enable_if_t<(sizeof...(Args) > 0)>> + explicit FunctionalImpl(SomeFunction original_function, Args&&... args) + // NOLINTNEXTLINE(modernize-avoid-bind) + : function_(std::bind( + original_function, + /*input=*/std::placeholders::_1, + std::forward(args)...)) { + // std::bind is normally evil, but (1) gcc is broken w.r.t. handling + // parameter pack expansion in lambdas and (2) moving parameter packs into + // a lambda only works with C++14, so std::bind is the more move-aware + // solution here. + } + + void reset() override; + + /// Pretty prints the `Functional` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// Forwards the `input` tensor to the underlying (bound) function object. + Tensor forward(Tensor input); + + /// Calls forward(input). + Tensor operator()(Tensor input); + + bool is_serializable() const override; + + private: + Function function_; +}; + +/// A `ModuleHolder` subclass for `FunctionalImpl`. +/// See the documentation for `FunctionalImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(Functional); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/moduledict.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/moduledict.h new file mode 100644 index 0000000000000000000000000000000000000000..fad78ed836b427da4a693b861fb3719c09491ad4 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/moduledict.h @@ -0,0 +1,260 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn { + +/// An OrderedDict of `Module`s that registers its elements by their `key`s. +/// +/// \rst +/// .. code-block:: cpp +/// +/// torch::OrderedDict> ordereddict = { +/// {"linear", Linear(10, 3).ptr()}, +/// {"conv", Conv2d(1, 2, 3).ptr()}, +/// {"dropout", Dropout(0.5).ptr()}, +/// }; +/// torch::nn::ModuleDict dict1(ordereddict); +/// +/// for (const auto &module : *dict1) { +/// module->pretty_print(std::cout); +/// } +/// +/// std::vector>> list = { +/// {"linear", Linear(10, 3).ptr()}, +/// {"conv", Conv2d(1, 2, 3).ptr()}, +/// {"dropout", Dropout(0.5).ptr()}, +/// }; +/// torch::nn::ModuleDict dict2(list); +/// +/// for (const auto &module : *dict2) { +/// module->pretty_print(std::cout); +/// } +/// +/// \endrst +/// +/// Why should you use `ModuleDict` instead of a simple `map` or `OrderedDict`? +/// The value a `ModuleDict` provides over manually calling an ordered map of +/// modules is that it allows treating the whole container *as a single module*, +/// such that performing a transformation on the `ModuleDict` applies to each of +/// the modules it stores (which are each a registered submodule of the +/// `ModuleDict`). For example, calling `.to(torch::kCUDA)` on a `ModuleDict` +/// will move each module in the map to CUDA memory. For example: +/// +/// \rst +/// .. code-block:: cpp +/// +/// torch::OrderedDict> ordereddict = { +/// {"linear", Linear(10, 3).ptr()}, +/// {"conv", Conv2d(1, 2, 3).ptr()}, +/// {"dropout", Dropout(0.5).ptr()}, +/// }; +/// torch::nn::ModuleDict dict(ordereddict); +/// +/// // Convert all modules to CUDA. +/// dict->to(torch::kCUDA); +/// +/// \endrst +/// +/// Finally, `ModuleDict` provides a lightweight container API, such as allowing +/// iteration over submodules, positional access, adding new modules from a +/// vector of key-module pairs or an `OrderedDict` or another `ModuleDict` after +/// construction via `update`. +class ModuleDictImpl : public Cloneable { + public: + using Iterator = + torch::OrderedDict>::Iterator; + using ConstIterator = + torch::OrderedDict>::ConstIterator; + + ModuleDictImpl() = default; + + /// Constructs the `ModuleDict` from a list of string-Module pairs. + explicit ModuleDictImpl( + const std::vector>>& + modules) { + update(modules); + } + + /// Constructs the `ModuleDict` from an `OrderedDict`. + explicit ModuleDictImpl( + const torch::OrderedDict>& modules) { + update(modules); + } + + /// Return the items in the `ModuleDict`. + std::vector>> items() const { + return modules_.pairs(); + } + + /// Return the keys in the `ModuleDict`. + std::vector keys() const { + return modules_.keys(); + } + + /// Return the values in the `ModuleDict`. + std::vector> values() const { + return modules_.values(); + } + + /// Return an iterator to the start of `ModuleDict`. + Iterator begin() { + return modules_.begin(); + } + + /// Return a const iterator to the start of `ModuleDict`. + ConstIterator begin() const { + return modules_.begin(); + } + + /// Return an iterator to the end of `ModuleDict`. + Iterator end() { + return modules_.end(); + } + + /// Return a const iterator to the end of `ModuleDict`. + ConstIterator end() const { + return modules_.end(); + } + + /// Return the number of items currently stored in the `ModuleDict`. + size_t size() const noexcept { + return modules_.size(); + } + + /// Return true if the `ModuleDict` is empty, otherwise return false. + bool empty() const noexcept { + return modules_.is_empty(); + } + + /// Check if the certain parameter with the key in the `ModuleDict`. + bool contains(const std::string& key) const noexcept { + return modules_.contains(key); + } + + /// Remove all items from the `ModuleDict`. + void clear() { + // Not remove the registration of modules to make it consistent with python + // version. + modules_.clear(); + } + + /// Special cloning function for `ModuleDict` because it does not use + /// `reset()`. + std::shared_ptr clone( + const std::optional& device = std::nullopt) const override { + auto clone = std::make_shared(); + for (const auto& module : modules_) { + clone->insert(module.key(), module.value()->clone(device)); + } + return clone; + } + + /// `reset()` is empty for `ModuleDict`, since it does not have parameters of + /// its own. + void reset() override {} + + /// Pretty prints the `ModuleDict` into the given `stream`. + void pretty_print(std::ostream& stream) const override { + stream << "torch::nn::ModuleDict"; + } + + /// Attempts to returns the `Module` associated with the given `key`. Throws + /// an exception if no such `key` is stored in the `ModuleDict`. Check + /// contains(key) before for a non-throwing way of access. + std::shared_ptr operator[](const std::string& key) const { + return modules_[key]; + } + + /// Attempts to return the module at the given key as the requested type. + /// Throws an exception if no such `key` is stored in the `ModuleDict`. + /// Check contains(key) before for a non-throwing way of access. + template + T& at(const std::string& key) { + static_assert( + torch::detail::is_module::value, + "Can only call ModuleList::at with an nn::Module type"); + auto module = modules_[key]->as(); + TORCH_CHECK( + module, + "Unable to cast module[", + key, + "] to ", + c10::demangle(typeid(T).name())); + return *module; + } + + /// Attempts to return the module at the given key as the requested type. + /// Throws an exception if no such `key` is stored in the `ModuleDict`. + /// Check contains(key) before for a non-throwing way of access. + template + const T& at(const std::string& key) const { + static_assert( + torch::detail::is_module::value, + "Can only call ModuleList::at with an nn::Module type"); + const auto module = modules_[key]->as(); + TORCH_CHECK( + module, + "Unable to cast module[", + key, + "] to ", + c10::demangle(typeid(T).name())); + return *module; + } + + /// Removes and returns the `Module` associated with the given `key`. + /// Throws an exception if no such `key` is stored in the `ModuleDict`. + /// Check contains(key) before for a non-throwing way of access. + std::shared_ptr pop(const std::string& key) { + auto module = modules_[key]; + modules_.erase(key); + // Not remove the registration of the module to make it consistent with + // python version. + return module; + } + + /// Updated the `ModuleDict` with a vector of key-module pairs. + void update( + const std::vector>>& + modules) { + for (auto& item : modules) { + insert(item.first, item.second); + } + } + + /// Updated the `ModuleDict` with key-value pairs from `OrderedDict` or + /// `ModuleDict`. + template + void update(const Container& container) { + for (auto& item : container) { + insert(item.key(), item.value()); + } + } + + private: + /// Private `OrderedDict` holding the key-Module pairs. + torch::OrderedDict> modules_; + + /// Insert a key-module pair by overwriting existing keys, + /// and register or replace the `Module`. + void insert(const std::string& key, std::shared_ptr module) { + if (contains(key)) { + modules_[key] = std::move(module); + replace_module(key, modules_[key]); + } else { + modules_.insert(key, std::move(module)); + register_module(key, modules_.back().value()); + } + } +}; + +/// A `ModuleHolder` subclass for `ModuleDictImpl`. +/// See the documentation for `ModuleDictImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(ModuleDict); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/modulelist.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/modulelist.h new file mode 100644 index 0000000000000000000000000000000000000000..dec16cb80fedf2a30081aa5e16ec0a3b324614fc --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/modulelist.h @@ -0,0 +1,272 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace torch::nn { + +/// A list of `Module`s that registers its elements. +/// +/// \rst +/// .. code-block:: cpp +/// +/// torch::nn::ModuleList mlist( +/// torch::nn::Linear(3, 4), +/// torch::nn::BatchNorm1d(4), +/// torch::nn::Dropout(0.5) +/// ); +/// +/// for (const auto &module : *mlist) { +/// module->pretty_print(std::cout); +/// } +/// +/// \endrst +/// +/// Why should you use `ModuleList` instead of a simple `std::vector`? The value +/// a `ModuleList` provides over manually calling a sequence of modules is that +/// it allows treating the whole container *as a single module*, such that +/// performing a transformation on the `ModuleList` applies to each of the +/// modules it stores (which are each a registered submodule of the +/// `ModuleList`). For example, calling +/// `.to(torch::kCUDA)` on a `ModuleList` will move each module in the list to +/// CUDA memory. For example: +/// +/// \rst +/// .. code-block:: cpp +/// +/// torch::nn::ModuleList mlist( +/// torch::nn::Linear(3, 4), +/// torch::nn::BatchNorm1d(4), +/// torch::nn::Dropout(0.5) +/// ); +/// +/// // Convert all modules to CUDA. +/// mlist->to(torch::kCUDA); +/// +/// \endrst +/// +/// Finally, `ModuleList` provides a lightweight container API, such as allowing +/// iteration over submodules, positional access, adding a new module after +/// construction via `push_back`, as well as joining two `ModuleList`s via +/// `extend`. +class ModuleListImpl : public Cloneable { + public: + using Iterator = std::vector>::iterator; + using ConstIterator = std::vector>::const_iterator; + + ModuleListImpl() = default; + + /// Constructs the `ModuleList` from a variadic list of modules. + template + explicit ModuleListImpl(Modules&&... modules) { + modules_.reserve(sizeof...(Modules)); + push_back_var(std::forward(modules)...); + } + + /// Special cloning function for `ModuleList` because it does not use + /// `reset()`. + std::shared_ptr clone( + const std::optional& device = std::nullopt) const override { + auto clone = std::make_shared(); + for (const auto& module : modules_) { + clone->push_back(module->clone(device)); + } + return clone; + } + + /// `reset()` is empty for `ModuleList`, since it does not have parameters of + /// its own. + void reset() override {} + + /// Pretty prints the `ModuleList` module into the given `stream`. + void pretty_print(std::ostream& stream) const override { + stream << "torch::nn::ModuleList"; + } + + void push_back(std::shared_ptr module) { + modules_.push_back(std::move(module)); + const auto index = modules_.size() - 1; + register_module(std::to_string(index), modules_[index]); + } + + /// Adds a new `Module` to the `ModuleList` container, moving or copying + /// it into a `shared_ptr` internally. This method allows passing value types, + /// and letting the container deal with the boxing. + template > + void push_back(M&& module) { + using Type = std::remove_reference_t; + push_back(std::make_shared(std::forward(module))); + } + + /// Unwraps the contained module of a `ModuleHolder` and adds it to the + /// `ModuleList`. + template + void push_back(const ModuleHolder& module_holder) { + push_back(module_holder.ptr()); + } + + /// Iterates over the container and calls `push_back()` on each value. + template + void extend(const Container& container) { + for (const auto& module : container) { + push_back(module); + } + } + + /// Returns an iterator to the start of the `ModuleList`. + Iterator begin() { + return modules_.begin(); + } + + /// Returns a const iterator to the start of the `ModuleList`. + ConstIterator begin() const { + return modules_.begin(); + } + + /// Returns an iterator to the end of the `ModuleList`. + Iterator end() { + return modules_.end(); + } + + /// Returns a const iterator to the end of the `ModuleList`. + ConstIterator end() const { + return modules_.end(); + } + + /// Attempts to return the module at the given index as the requested type. + /// Throws an exception if the index is out of bounds or the types do not + /// match. + template + T& at(size_t index) { + static_assert( + torch::detail::is_module::value, + "Can only call ModuleList::at with an nn::Module type"); + TORCH_CHECK(index < size(), "Index out of range"); + auto module = modules_[index]->as(); + TORCH_CHECK( + module, + "Unable to cast module[", + index, + "] to ", + c10::demangle(typeid(T).name())); + return *module; + } + + /// Attempts to return the module at the given index as the requested type. + /// Throws an exception if the index is out of bounds or the types do not + /// match. + template + const T& at(size_t index) const { + static_assert( + torch::detail::is_module::value, + "Can only call ModuleList::at with an nn::Module type"); + TORCH_CHECK(index < size(), "Index out of range"); + const auto module = modules_[index]->as(); + TORCH_CHECK( + module, + "Unable to cast module[", + index, + "] to ", + c10::demangle(typeid(T).name())); + return *module; + } + + /// Attempts to return a `std::shared_ptr` whose dynamic type is that of the + /// underlying module at the given index. Throws an exception if the index is + /// out of bounds. + std::shared_ptr ptr(size_t index) const { + TORCH_CHECK(index < size(), "Index out of range"); + return modules_[index]; + } + + /// Attempts to return a `std::shared_ptr` whose type is the one provided. + /// Throws an exception if the index is out of bounds or the types do not + /// match. + template + std::shared_ptr ptr(size_t index) const { + static_assert( + torch::detail::is_module::value, + "Can only call ModuleList::ptr with an nn::Module type"); + TORCH_CHECK(index < size(), "Index out of range"); + return std::dynamic_pointer_cast(modules_[index]); + } + + /// Like `ptr(index)`. + std::shared_ptr operator[](size_t index) const { + // This is the only method we can call without a type. + return ptr(index); + } + + /// The current size of the `ModuleList` container. + size_t size() const noexcept { + return modules_.size(); + } + + /// True if there are no modules in the `ModuleList`. + bool is_empty() const noexcept { + return size() == 0; + } + + void insert(size_t index, std::shared_ptr module) { + TORCH_CHECK(index <= size(), "Index out of range"); + + if (index == size()) + push_back(std::move(module)); + else { + modules_.insert( + modules_.begin() + Iterator::difference_type(index), + std::move(module)); + + for (const auto i : c10::irange(index, size() - 1)) { + (void)i; // Suppress unused variable warning + replace_module(std::to_string(index), modules_[index]); + } + register_module(std::to_string(size() - 1), modules_.back()); + } + } + + /// Unwraps the contained module of a `ModuleHolder` and inserts it in the + /// `ModuleList`. + template + void insert(size_t index, const ModuleHolder& module_holder) { + insert(index, module_holder.ptr()); + } + + /// inserts a new `Module` to the `ModuleList` container, moving or copying + /// it into a `shared_ptr` internally. This method allows passing value types, + /// and letting the container deal with the boxing. + template > + void insert(size_t index, M&& module) { + using Type = std::remove_reference_t; + insert(index, std::make_shared(std::forward(module))); + } + + private: + template + void push_back_var(Head&& head, Tail&&... tail) { + push_back(std::forward(head)); + // Recursively calls this method, until the parameter pack only thas this + // entry left. Then calls `push_back()` a final time (above). + push_back_var(std::forward(tail)...); + } + + /// The base case, when the list of modules is empty. + void push_back_var() {} + + // Box the AnyModules to give ModuleList reference semantics, like the rest of + // the API. Note that this is not required otherwise, this could just be a + // `vector`. + std::vector> modules_; +}; + +/// A `ModuleHolder` subclass for `ModuleListImpl`. +/// See the documentation for `ModuleListImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(ModuleList); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/named_any.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/named_any.h new file mode 100644 index 0000000000000000000000000000000000000000..68c5f5104c4bb4459518df26990a8c8b5cd2982a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/named_any.h @@ -0,0 +1,81 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace torch::nn { + +/// Stores a type erased `Module` with name. +/// +/// The `NamedAnyModule` class enables the following API for constructing +/// `nn::Sequential` with named submodules: +/// \rst +/// .. code-block:: cpp +/// +/// struct M : torch::nn::Module { +/// explicit M(int value_) : value(value_) {} +/// int value; +/// int forward() { +/// return value; +/// } +/// }; +/// +/// Sequential sequential({ +/// {"m1", std::make_shared(1)}, // shared pointer to `Module` is +/// supported {std::string("m2"), M(2)}, // `Module` is supported +/// {"linear1", Linear(10, 3)} // `ModuleHolder` is supported +/// }); +/// \endrst +class NamedAnyModule { + public: + /// Creates a `NamedAnyModule` from a (boxed) `Module`. + template + NamedAnyModule(std::string name, std::shared_ptr module_ptr) + : NamedAnyModule(std::move(name), AnyModule(std::move(module_ptr))) {} + + /// Creates a `NamedAnyModule` from a `Module`, moving or copying it + /// into a `shared_ptr` internally. + // NOTE: We need to use `std::remove_reference_t` to get rid of + // any reference components for make_unique. + template > + NamedAnyModule(std::string name, M&& module) + : NamedAnyModule( + std::move(name), + std::make_shared>( + std::forward(module))) {} + + /// Creates a `NamedAnyModule` from a `Module` that is unwrapped from + /// a `ModuleHolder`. + template + NamedAnyModule(std::string name, const ModuleHolder& module_holder) + : NamedAnyModule(std::move(name), module_holder.ptr()) {} + + /// Creates a `NamedAnyModule` from a type-erased `AnyModule`. + NamedAnyModule(std::string name, AnyModule any_module) + : name_(std::move(name)), module_(std::move(any_module)) {} + + /// Returns a reference to the name. + const std::string& name() const noexcept { + return name_; + } + + /// Returns a reference to the module. + AnyModule& module() noexcept { + return module_; + } + + /// Returns a const reference to the module. + const AnyModule& module() const noexcept { + return module_; + } + + private: + std::string name_; + AnyModule module_; +}; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/parameterdict.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/parameterdict.h new file mode 100644 index 0000000000000000000000000000000000000000..30a39b8403ff598355668b0f5afe06e06215e805 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/parameterdict.h @@ -0,0 +1,146 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace torch::nn { + +class ParameterDictImpl : public Cloneable { + public: + using Iterator = OrderedDict::Iterator; + using ConstIterator = OrderedDict::ConstIterator; + + ParameterDictImpl() = default; + + explicit ParameterDictImpl( + const torch::OrderedDict& params) { + parameters_ = params; + } + + /// `reset()` is empty for `ParameterDict`, since it does not have + /// parameters of its own. + void reset() override {} + + /// Pretty prints the `ParameterDict` module into the given `stream`. + void pretty_print(std::ostream& stream) const override { + stream << "torch::nn::ParameterDict(" << '\n'; + for (const auto& pair : parameters_) { + stream << '(' << pair.key() << ')' << ": Parameter containing: [" + << pair.value().scalar_type() << " of size " + << pair.value().sizes() << ']'; + ; + stream << '\n'; + } + stream << ')'; + } + + /// Insert the parameter along with the key into ParameterDict + /// The parameter is set to be require grad by default + Tensor& insert(const std::string& key, const Tensor& param) { + bool requires_grad = param.requires_grad(); + return register_parameter(key, param, requires_grad); + } + + /// Remove key from the ParameterDict and return its value, throw exception + /// if the key is not contained. Please check contains(key) before for a + /// non-throwing access. + Tensor pop(const std::string& key) { + torch::Tensor v = parameters_[key]; + parameters_.erase(key); + return v; + } + + /// Return the keys in the dict + ::std::vector keys() const { + return parameters_.keys(); + } + + /// Return the Values in the dict + ::std::vector values() const { + return parameters_.values(); + } + + /// Return an iterator to the start of ParameterDict + Iterator begin() { + return parameters_.begin(); + } + + /// Return a const iterator to the start of ParameterDict + ConstIterator begin() const { + return parameters_.begin(); + } + + /// Return an iterator to the end of ParameterDict + Iterator end() { + return parameters_.end(); + } + + /// Return a const iterator to the end of ParameterDict + ConstIterator end() const { + return parameters_.end(); + } + + /// Return the number of items currently stored in the ParameterDict + size_t size() const noexcept { + return parameters_.size(); + } + + /// Return true if the ParameterDict is empty, otherwise return false + bool empty() const noexcept { + return parameters_.is_empty(); + } + + /// Update the ParameterDict with the key-value pairs from + /// another ParameterDict, overwriting existing key + template + void update(const Container& container) { + for (auto& item : container) { + parameters_[item.key()] = item.value(); + } + } + + /// Remove all parameters in the ParameterDict + void clear() { + parameters_.clear(); + } + + /// Check if the certain parameter with the key in the ParameterDict + bool contains(const std::string& key) const noexcept { + return parameters_.contains(key); + } + + /// Returns the value associated with the given `key`. Throws an exception if + /// no such key is stored in the `ParameterDict`. Check contains(key) before + /// for a non-throwing way of access + const Tensor& get(const std::string& key) const { + return parameters_[key]; + } + + /// Returns the value associated with the given `key`. Throws an exception if + /// no such key is stored in the `ParameterDict`. Check contains(key) before + /// for a non-throwing way of access + Tensor& get(const std::string& key) { + return parameters_[key]; + } + + /// Returns the value associated with the given `key`. Throws an exception if + /// no such key is stored in the `ParameterDict`. Check contains(key) before + /// for a non-throwing way of access + Tensor& operator[](const std::string& key) { + return parameters_[key]; + } + + /// Returns the value associated with the given `key`. Throws an exception if + /// no such key is stored in the `ParameterDict`. Check contains(key) before + /// for a non-throwing way of access + const Tensor& operator[](const std::string& key) const { + return parameters_[key]; + } +}; + +TORCH_MODULE(ParameterDict); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/parameterlist.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/parameterlist.h new file mode 100644 index 0000000000000000000000000000000000000000..32c5b3dae6359dea7c2c5c55b91937acf88df8f1 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/parameterlist.h @@ -0,0 +1,167 @@ +#pragma once + +#include +#include + +#include + +namespace torch::nn { +class ParameterListImpl : public Cloneable { + public: + using Iterator = + std::vector::Item>::iterator; + using ConstIterator = std::vector< + OrderedDict::Item>::const_iterator; + + ParameterListImpl() = default; + + /// Constructs the `ParameterList` from a variadic list of ParameterList. + template + explicit ParameterListImpl(Tensors&&... params) { + parameters_.reserve(sizeof...(Tensors)); + push_back_var(std::forward(params)...); + } + + template + explicit ParameterListImpl(const Tensors&... params) { + parameters_.reserve(sizeof...(Tensors)); + push_back_var(std::forward(params)...); + } + + /// `reset()` is empty for `ParameterList`, since it does not have parameters + /// of its own. + void reset() override {} + + /// Pretty prints the `ParameterList` module into the given `stream`. + void pretty_print(std::ostream& stream) const override { + stream << "torch::nn::ParameterList(" << '\n'; + for (const auto& pair : parameters_) { + stream << '(' << pair.key() << ')' << ": Parameter containing: [" + << pair.value().scalar_type() << " of size " + << pair.value().sizes() << ']'; + ; + stream << '\n'; + } + stream << ')'; + } + + /// push a given parameter at the end of the list + void append(torch::Tensor&& param) { + bool requires_grad = param.requires_grad(); + register_parameter( + std::to_string(parameters_.size()), std::move(param), requires_grad); + } + + /// push a given parameter at the end of the list + void append(const torch::Tensor& param) { + bool requires_grad = param.requires_grad(); + register_parameter( + std::to_string(parameters_.size()), param, requires_grad); + } + + /// push a given parameter at the end of the list + /// And the key of the pair will be discarded, only the value + /// will be added into the `ParameterList` + void append(const OrderedDict::Item& pair) { + register_parameter( + std::to_string(parameters_.size()), + pair.value(), + pair.value().requires_grad()); + } + + /// extend parameters from a container to the end of the list + template + void extend(const Container& container) { + for (const auto& param : container) { + append(param); + } + } + + /// Returns an iterator to the start of the ParameterList + /// the iterator returned will be type of `OrderedDict::Item` + Iterator begin() { + return parameters_.begin(); + } + + /// Returns a const iterator to the start of the ParameterList + /// the iterator returned will be type of `OrderedDict::Item` + ConstIterator begin() const { + return parameters_.begin(); + } + + /// Returns an iterator to the end of the ParameterList + /// the iterator returned will be type of `OrderedDict::Item` + Iterator end() { + return parameters_.end(); + } + + /// Returns a const iterator to the end of the ParameterList + /// the iterator returned will be type of `OrderedDict::Item` + ConstIterator end() const { + return parameters_.end(); + } + + /// Returns the value associated with the given `key`. Throws an exception if + /// no such key is stored in the `ParameterList`. Check contains(key) before + /// for a non-throwing way of access + at::Tensor& at(size_t idx) { + TORCH_CHECK(idx < size(), "Index out of range"); + return parameters_[std::to_string(idx)]; + } + + /// Returns the value associated with the given `key`. Throws an exception if + /// no such key is stored in the `ParameterList`. Check contains(key) before + /// for a non-throwing way of access + const at::Tensor& at(size_t idx) const { + TORCH_CHECK(idx < size(), "Index out of range"); + return parameters_[std::to_string(idx)]; + } + + /// Returns the value associated with the given `key`. Throws an exception if + /// no such key is stored in the `ParameterList`. Check contains(key) before + /// for a non-throwing way of access + at::Tensor& operator[](size_t idx) { + return at(idx); + } + + /// Returns the value associated with the given `key`. Throws an exception if + /// no such key is stored in the `ParameterList`. Check contains(key) before + /// for a non-throwing way of access + const at::Tensor& operator[](size_t idx) const { + return at(idx); + } + + /// Return the size of the ParameterList + size_t size() const noexcept { + return parameters_.size(); + } + /// True if the ParameterList is empty + bool is_empty() const noexcept { + return parameters_.is_empty(); + } + + /// Overload the +=, so that two ParameterList could be incrementally added + template + Container& operator+=(const Container& other) { + extend(other); + return *this; + } + + private: + template + void push_back_var(Head&& head, Tail&&... tail) { + append(std::forward(head)); + // Recursively calls this method, until the parameter pack only thas this + // entry left. Then calls `push_back()` a final time (above). + push_back_var(std::forward(tail)...); + } + + /// The base case, when the list of modules is empty. + void push_back_var() {} +}; +TORCH_MODULE(ParameterList); +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/sequential.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/sequential.h new file mode 100644 index 0000000000000000000000000000000000000000..c5941b0bdb7798f26b3ebec29bcb497e108a1fa0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/container/sequential.h @@ -0,0 +1,387 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace torch::nn { + +/// A list of `Module`s that acts as a `Module` itself. +/// +/// A `Sequential` is fundamentally a list of `Module`s, each with a `forward()` +/// method. `Sequential` provides a `forward()` method of its own, which accepts +/// any input and forwards it to the first module it stores. It then "chains" +/// outputs to inputs sequentially for each subsequent module, finally returning +/// the output of the last module. For example: +/// +/// \rst +/// .. code-block:: cpp +/// +/// torch::nn::Sequential seq( +/// torch::nn::Linear(3, 4), +/// torch::nn::BatchNorm1d(4), +/// torch::nn::Dropout(0.5) +/// ); +/// +/// auto output = seq->forward(torch::ones(3)); +/// +/// \endrst +/// +/// This can conceptually be thought of as the following loop (using Python as +/// pseudocode): +/// +/// \rst +/// .. code-block:: python +/// +/// def forward(sequential, input): +/// for module in sequential: +/// input = module(input) +/// return input +/// +/// \endrst +/// +/// Why should you use `Sequential` instead of a simple `std::vector`? The value +/// a `Sequential` provides over manually calling a sequence of modules is that +/// it allows treating the whole container *as a single module*, such that +/// performing a transformation on the `Sequential` applies to each of the +/// modules it stores (which are each a registered submodule of the +/// `Sequential`). For example, calling +/// `.to(torch::kCUDA)` on a `Sequential` will move each module in the list to +/// CUDA memory. For example: +/// +/// \rst +/// .. code-block:: cpp +/// +/// torch::nn::Sequential seq( +/// torch::nn::Linear(3, 4), +/// torch::nn::BatchNorm1d(4), +/// torch::nn::Dropout(0.5) +/// ); +/// +/// // Convert all modules to CUDA. +/// seq->to(torch::kCUDA); +/// +/// \endrst +/// +/// Finally, `Sequential` provides a lightweight container API, such as allowing +/// iteration over submodules, positional access, adding a new module after +/// construction via `push_back`, as well as joining two `Sequential`s via +/// `extend`. +/// +/// \rst +/// .. attention:: +/// One current limitation of `Sequential` is that all except the first module +/// must accept a single argument. If your modules need to take multiple +/// arguments, you should define them to take and return tuples. +/// \endrst +class SequentialImpl : public Cloneable { + public: + using Iterator = std::vector::iterator; + using ConstIterator = std::vector::const_iterator; + + SequentialImpl() = default; + + /// Constructs the `Sequential` from a variadic list of modules. + template + explicit SequentialImpl(Modules&&... modules) { + modules_.reserve(sizeof...(Modules)); + push_back(std::forward(modules)...); + } + + /// Constructs the `Sequential` from an `OrderedDict` of named `AnyModule`s. + explicit SequentialImpl( + torch::OrderedDict&& ordered_dict) { + modules_.reserve(ordered_dict.size()); + for (auto& item : ordered_dict) { + push_back(item.key(), std::move(item.value())); + } + } + + /// Constructs the `Sequential` from a braced-init-list of named `AnyModule`s. + /// It enables the following use case: + /// `Sequential sequential({{"m1", M(1)}, {"m2", M(2)}})` + explicit SequentialImpl(std::initializer_list named_modules) { + modules_.reserve(named_modules.size()); + for (const auto& named_module : named_modules) { + push_back(named_module.name(), named_module.module()); + } + } + + /// Special cloning function for `Sequential` because it does not use + /// `reset()`. + std::shared_ptr clone( + const std::optional& device = std::nullopt) const override { + auto clone = std::make_shared(); + for (const auto& module : modules_) { + clone->push_back(module.clone(device)); + } + return clone; + } + + /// `reset()` is empty for `Sequential`, since it does not have parameters of + /// its own. + void reset() override {} + + /// Pretty prints the `Sequential` module into the given `stream`. + void pretty_print(std::ostream& stream) const override { + stream << "torch::nn::Sequential"; + } + + /// Feeds `inputs` to the first module and then chains outputs to inputs, + /// returning the last output. + /// + /// Conceptually the following loop in Python: + /// + /// \rst + /// .. code-block:: python + /// + /// def forward(sequential, input): + /// for module in sequential: + /// input = module(input) + /// return input + /// + /// \endrst + /// + /// The return type is taken as the first template parameter. It defaults to + /// `Tensor`. If the last module in the `Sequential` returns another type `T`, + /// you should call `forward(inputs)` instead of just `forward(inputs)`: + /// + /// \rst + /// .. code-block:: cpp + /// + /// torch::Tensor tensor = sequential1->forward(inputs); + /// int integer = sequential2->forward(inputs); + /// float value = sequential3->forward(inputs); + /// + /// \endrst + template + ReturnType forward(InputTypes&&... inputs) { + TORCH_CHECK(!is_empty(), "Cannot call forward() on an empty Sequential"); + + auto iterator = modules_.begin(); + auto input = iterator->any_forward(std::forward(inputs)...); + + for (++iterator; iterator != modules_.end(); ++iterator) { + input = iterator->any_forward(std::move(input)); + } + + // Check the return value and give a nice error message if the requested + // return type was incorrect. + if (auto* return_value = input.template try_get()) { + return std::move(*return_value); + } + TORCH_CHECK( + false, + "The type of the return value is ", + c10::demangle(input.type_info().name()), + ", but you asked for type ", + c10::demangle(typeid(ReturnType).name())); + } + + /// Adds a new (boxed) `Module` to the `Sequential` container. + template + void push_back(std::shared_ptr module_ptr) { + push_back(std::to_string(modules_.size()), std::move(module_ptr)); + } + + /// Adds a new named (boxed) `Module` to the `Sequential` container. + template + void push_back(std::string name, std::shared_ptr module_ptr) { + push_back(std::move(name), AnyModule(std::move(module_ptr))); + } + + /// Adds a new `Module` to the `Sequential` container, moving or copying it + /// into a `shared_ptr` internally. This method allows passing value types, + /// and letting the container deal with the boxing. This means you can write + /// `Sequential(Module(3, 4))` instead of + /// `Sequential(std::make_shared(3, 4))`. + template > + void push_back(M&& module) { + push_back(std::to_string(modules_.size()), std::forward(module)); + } + + /// Adds a new named `Module` to the `Sequential` container, moving or copying + /// it into a `shared_ptr` internally. This method allows passing value types, + /// and letting the container deal with the boxing. + template > + void push_back(std::string name, M&& module) { + using Type = typename std::remove_reference_t; + push_back(std::move(name), std::make_shared(std::forward(module))); + } + + /// Unwraps the contained module of a `ModuleHolder` and adds it to the + /// `Sequential`. + template + void push_back(const ModuleHolder& module_holder) { + push_back(std::to_string(modules_.size()), module_holder); + } + + /// Unwraps the contained named module of a `ModuleHolder` and adds it to the + /// `Sequential`. + template + void push_back(std::string name, const ModuleHolder& module_holder) { + push_back(std::move(name), module_holder.ptr()); + } + + /// Iterates over the container and calls `push_back()` on each value. + template + void extend(const Container& container) { + for (const auto& module : container) { + push_back(module); + } + } + + /// Adds a type-erased `AnyModule` to the `Sequential`. + void push_back(AnyModule any_module) { + push_back(std::to_string(modules_.size()), std::move(any_module)); + } + + void push_back(std::string name, AnyModule any_module) { + modules_.push_back(std::move(any_module)); + const auto index = modules_.size() - 1; + register_module(std::move(name), modules_[index].ptr()); + } + + /// Returns an iterator to the start of the `Sequential`. + Iterator begin() { + return modules_.begin(); + } + + /// Returns a const iterator to the start of the `Sequential`. + ConstIterator begin() const { + return modules_.begin(); + } + + /// Returns an iterator to the end of the `Sequential`. + Iterator end() { + return modules_.end(); + } + + /// Returns a const iterator to the end of the `Sequential`. + ConstIterator end() const { + return modules_.end(); + } + + /// Attempts to return the module at the given index as the requested type. + /// Throws an exception if the index is out of bounds or the types do not + /// match. + template + T& at(size_t index) { + static_assert( + torch::detail::is_module::value, + "Can only call Sequential::at with an nn::Module type"); + TORCH_CHECK(index < size(), "Index out of range"); + return modules_[index].get(); + } + + /// Attempts to return the module at the given index as the requested type. + /// Throws an exception if the index is out of bounds or the types do not + /// match. + template + const T& at(size_t index) const { + static_assert( + torch::detail::is_module::value, + "Can only call Sequential::at with an nn::Module type"); + TORCH_CHECK(index < size(), "Index out of range"); + return modules_[index].get(); + } + + /// Attempts to return a `std::shared_ptr` whose dynamic type is that of the + /// underlying module at the given index. Throws an exception if the index is + /// out of bounds. + std::shared_ptr ptr(size_t index) const { + TORCH_CHECK(index < size(), "Index out of range"); + return modules_[index].ptr(); + } + + /// Attempts to return a `std::shared_ptr` whose type is the one provided. + /// Throws an exception if the index is out of bounds or the types do not + /// match. + template + std::shared_ptr ptr(size_t index) const { + static_assert( + torch::detail::is_module::value, + "Can only call Sequential::ptr with an nn::Module type"); + TORCH_CHECK(index < size(), "Index out of range"); + return modules_[index].ptr(); + } + + /// Like `ptr(index)`. + std::shared_ptr operator[](size_t index) const { + // This is the only method we can call without a type. + return ptr(index); + } + + /// The current size of the `Sequential` container. + size_t size() const noexcept { + return modules_.size(); + } + + /// True if there are no modules in the `Sequential`. + bool is_empty() const noexcept { + return size() == 0; + } + + private: + /// Takes a First *and* Second parameter, to avoid ambiguity when a parameter + /// pack has only one type, in which case the template would be preferred, + /// even if the other `push_back` functions are better fits (e.g. `unique_ptr` + /// -> `shared_ptr` overload). + /// NOTE: We explicitly avoid matching this template with + /// `push_back(std::string("name"), module)` or `push_back("name", module)`, + /// since they should be handled by their respective `push_back` functions. + template < + typename First, + typename Second, + typename... Rest, + typename = std::enable_if_t< + !std::is_same_v && + // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + !std::is_same_v, std::decay_t>>> + void push_back(First&& first, Second&& second, Rest&&... rest) { + push_back(std::forward(first)); + // Recursively calls this method, until the parameter pack only thas this + // entry left. Then calls `push_back()` a final time (above). + push_back(std::forward(second), std::forward(rest)...); + } + + /// The base case, when the list of modules is empty. + void push_back() {} + + // Box the AnyModules to give Sequential reference semantics, like the rest of + // the API. Note that this is not required otherwise, this could just be a + // `vector`. + std::vector modules_; +}; + +/// A `ModuleHolder` subclass for `SequentialImpl`. +/// See the documentation for `SequentialImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +class Sequential : public torch::nn::ModuleHolder { + public: + using torch::nn::ModuleHolder::ModuleHolder; + + Sequential() = default; + + /// Constructs the `Sequential` from a braced-init-list of named `AnyModule`s. + /// It enables the following use case: + /// `Sequential sequential({{"m1", M(1)}, {"m2", M(2)}})` + Sequential(std::initializer_list named_modules) + : ModuleHolder(std::make_shared(named_modules)) {} +}; +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/conv.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/conv.h new file mode 100644 index 0000000000000000000000000000000000000000..9932da919ed97a41ee2ab78b092bdbc2f620dd6c --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/conv.h @@ -0,0 +1,448 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace torch::nn { + +/// Base class for all (dimension-specialized) convolution modules. +template +class ConvNdImpl : public torch::nn::Cloneable { + public: + explicit ConvNdImpl(detail::ConvNdOptions options_) + : options(std::move(options_)) { + ConvNdImpl::reset(); + } + + void reset() override { + TORCH_CHECK( + options.in_channels() > 0 && options.groups() > 0 && + options.out_channels() > 0, + "in_channels, groups and out_channels must be a positive integer."); + TORCH_CHECK( + options.in_channels() % options.groups() == 0, + "in_channels must be divisible by groups"); + TORCH_CHECK( + options.out_channels() % options.groups() == 0, + "out_channels must be divisible by groups"); + + std::visit( + c10::overloaded( + [&](enumtype::kValid) { + _reversed_padding_repeated_twice.resize(2 * D); + std::fill_n(_reversed_padding_repeated_twice.begin(), 2 * D, 0); + }, + [&](enumtype::kSame) { + for (const auto i : c10::irange(D)) { + const auto stride = (*options.stride())[i]; + TORCH_CHECK( + stride == 1, + "padding='same' is not supported for strided convolutions"); + } + + _reversed_padding_repeated_twice.resize(2 * D); + for (const auto i : c10::irange(D)) { + const auto dilation = (*options.dilation())[i]; + const auto kernel_size = (*options.kernel_size())[i]; + const auto total_padding = dilation * (kernel_size - 1); + auto left_pad = total_padding / 2; + auto right_pad = total_padding - left_pad; + _reversed_padding_repeated_twice[2 * i] = left_pad; + _reversed_padding_repeated_twice[2 * i + 1] = right_pad; + } + }, + [&](const ExpandingArray& pad) { + _reversed_padding_repeated_twice = + torch::nn::modules::utils::_reverse_repeat_vector(pad, 2); + }), + options.padding()); + + if (options.transposed()) { + std::vector weight_sizes = { + options.in_channels(), options.out_channels() / options.groups()}; + weight_sizes.insert( + weight_sizes.end(), + (*options.kernel_size()).begin(), + (*options.kernel_size()).end()); + weight = this->register_parameter("weight", torch::empty(weight_sizes)); + } else { + std::vector weight_sizes = { + options.out_channels(), options.in_channels() / options.groups()}; + weight_sizes.insert( + weight_sizes.end(), + (*options.kernel_size()).begin(), + (*options.kernel_size()).end()); + weight = this->register_parameter("weight", torch::empty(weight_sizes)); + } + + if (options.bias()) { + bias = this->register_parameter( + "bias", torch::empty({options.out_channels()})); + } else { + this->register_parameter("bias", Tensor(), /*requires_grad=*/false); + } + + reset_parameters(); + } + + void reset_parameters() { + init::kaiming_uniform_( + weight, + /*a=*/std::sqrt(5)); // NOLINT(cppcoreguidelines-avoid-magic-numbers) + + if (bias.defined()) { + auto [fan_in, fan_out] = init::_calculate_fan_in_and_fan_out(weight); + auto bound = 1 / std::sqrt(fan_in); + init::uniform_(bias, -bound, bound); + } + } + + /// Pretty prints the `Conv{1,2,3}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override { + stream << "torch::nn::Conv" << D << 'd' << '(' << options.in_channels() + << ", " << options.out_channels() + << ", kernel_size=" << options.kernel_size() + << ", stride=" << options.stride(); + std::visit( + c10::overloaded( + [&](enumtype::kValid) { stream << ", padding='valid'"; }, + [&](enumtype::kSame) { stream << ", padding='same'"; }, + [&](const ExpandingArray& pad) { + if (*pad != *ExpandingArray(0)) { + stream << ", padding=" << pad; + } + }), + options.padding()); + if (*options.dilation() != *ExpandingArray(1)) { + stream << ", dilation=" << options.dilation(); + } + if (*options.output_padding() != *ExpandingArray(0)) { + stream << ", output_padding=" << options.output_padding(); + } + if (options.groups() != 1) { + stream << ", groups=" << options.groups(); + } + if (!options.bias()) { + stream << ", bias=" << std::boolalpha << false; + } + if (!std::get_if(&options.padding_mode())) { + stream << ", padding_mode=" + << enumtype::get_enum_name(options.padding_mode()); + } + stream << ')'; + } + + /// The options with which this `Module` was constructed. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + detail::ConvNdOptions options; + + /// The learned kernel (or "weight"). + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + Tensor weight; + + /// The learned bias. Only defined if the `bias` option was true. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + Tensor bias; + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::vector _reversed_padding_repeated_twice; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Conv1d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies convolution over a 1-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Conv1d to learn about +/// the exact behavior of this module. +/// +/// See the documentation for `torch::nn::Conv1dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Conv1d model(Conv1dOptions(3, 2, 3).stride(1).bias(false)); +/// ``` +class TORCH_API Conv1dImpl : public ConvNdImpl<1, Conv1dImpl> { + public: + Conv1dImpl( + int64_t input_channels, + int64_t output_channels, + ExpandingArray<1> kernel_size) + : Conv1dImpl( + Conv1dOptions(input_channels, output_channels, kernel_size)) {} + explicit Conv1dImpl(Conv1dOptions options_); + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `Conv1dImpl`. +/// See the documentation for `Conv1dImpl` class to learn what methods it +/// provides, and examples of how to use `Conv1d` with +/// `torch::nn::Conv1dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Conv1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Conv2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies convolution over a 2-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Conv2d to learn about +/// the exact behavior of this module. +/// +/// See the documentation for `torch::nn::Conv2dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Conv2d model(Conv2dOptions(3, 2, 3).stride(1).bias(false)); +/// ``` +class TORCH_API Conv2dImpl : public ConvNdImpl<2, Conv2dImpl> { + public: + Conv2dImpl( + int64_t input_channels, + int64_t output_channels, + ExpandingArray<2> kernel_size) + : Conv2dImpl( + Conv2dOptions(input_channels, output_channels, kernel_size)) {} + explicit Conv2dImpl(Conv2dOptions options_); + Tensor forward(const Tensor& input); + + protected: + Tensor _conv_forward(const Tensor& input, const Tensor& weight); +}; + +/// A `ModuleHolder` subclass for `Conv2dImpl`. +/// See the documentation for `Conv2dImpl` class to learn what methods it +/// provides, and examples of how to use `Conv2d` with +/// `torch::nn::Conv2dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Conv2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Conv3d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies convolution over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Conv3d to learn about +/// the exact behavior of this module. +/// +/// See the documentation for `torch::nn::Conv3dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Conv3d model(Conv3dOptions(3, 2, 3).stride(1).bias(false)); +/// ``` +class TORCH_API Conv3dImpl : public ConvNdImpl<3, Conv3dImpl> { + public: + Conv3dImpl( + int64_t input_channels, + int64_t output_channels, + ExpandingArray<3> kernel_size) + : Conv3dImpl( + Conv3dOptions(input_channels, output_channels, kernel_size)) {} + explicit Conv3dImpl(Conv3dOptions options_); + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `Conv3dImpl`. +/// See the documentation for `Conv3dImpl` class to learn what methods it +/// provides, and examples of how to use `Conv3d` with +/// `torch::nn::Conv3dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Conv3d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~ ConvTranspose ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Base class for all (dimension-specialized) convolution transpose modules. +template +class ConvTransposeNdImpl : public ConvNdImpl { + public: + using torch::nn::ConvNdImpl::ConvNdImpl; + explicit ConvTransposeNdImpl(detail::ConvNdOptions options_) + : ConvNdImpl(options_) { + TORCH_INTERNAL_ASSERT( + std::holds_alternative>(this->options.padding()), + "ConvTranspose padding cannot be a string"); + } + + /// Pretty prints the `ConvTranspose{1,2,3}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override { + stream << "torch::nn::ConvTranspose" << D << 'd' << '(' + << this->options.in_channels() << ", " + << this->options.out_channels() + << ", kernel_size=" << this->options.kernel_size() + << ", stride=" << this->options.stride(); + const auto& pad = padding(); + if (*pad != *ExpandingArray(0)) { + stream << ", padding=" << pad; + } + if (*this->options.dilation() != *ExpandingArray(1)) { + stream << ", dilation=" << this->options.dilation(); + } + if (*this->options.output_padding() != *ExpandingArray(0)) { + stream << ", output_padding=" << this->options.output_padding(); + } + if (this->options.groups() != 1) { + stream << ", groups=" << this->options.groups(); + } + if (!this->options.bias()) { + stream << ", bias=" << std::boolalpha << false; + } + if (!std::get_if(&this->options.padding_mode())) { + stream << ", padding_mode=" + << enumtype::get_enum_name(this->options.padding_mode()); + } + stream << ')'; + } + + protected: + const ExpandingArray& padding() const { + return std::get>(this->options.padding()); + } + + std::vector _output_padding( + const Tensor& input, + const std::optional& output_size, + const ExpandingArray& stride, + const ExpandingArray& padding, + const ExpandingArray& kernel_size); +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ConvTranspose1d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the ConvTranspose1d function. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ConvTranspose1d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ConvTranspose1dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ConvTranspose1d model(ConvTranspose1dOptions(3, 2, +/// 3).stride(1).bias(false)); +/// ``` +class TORCH_API ConvTranspose1dImpl + : public ConvTransposeNdImpl<1, ConvTranspose1dImpl> { + public: + ConvTranspose1dImpl( + int64_t input_channels, + int64_t output_channels, + ExpandingArray<1> kernel_size) + : ConvTranspose1dImpl(ConvTranspose1dOptions( + input_channels, + output_channels, + kernel_size)) {} + explicit ConvTranspose1dImpl(ConvTranspose1dOptions options_); + Tensor forward( + const Tensor& input, + const std::optional& output_size = std::nullopt); + + protected: + FORWARD_HAS_DEFAULT_ARGS({1, AnyValue(std::optional())}) +}; + +/// A `ModuleHolder` subclass for `ConvTranspose1dImpl`. +/// See the documentation for `ConvTranspose1dImpl` class to learn what methods +/// it provides, and examples of how to use `ConvTranspose1d` with +/// `torch::nn::ConvTranspose1dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(ConvTranspose1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ConvTranspose2d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the ConvTranspose2d function. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ConvTranspose2d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ConvTranspose2dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ConvTranspose2d model(ConvTranspose2dOptions(3, 2, +/// 3).stride(1).bias(false)); +/// ``` +class TORCH_API ConvTranspose2dImpl + : public ConvTransposeNdImpl<2, ConvTranspose2dImpl> { + public: + ConvTranspose2dImpl( + int64_t input_channels, + int64_t output_channels, + ExpandingArray<2> kernel_size) + : ConvTranspose2dImpl(ConvTranspose2dOptions( + input_channels, + output_channels, + kernel_size)) {} + explicit ConvTranspose2dImpl(ConvTranspose2dOptions options_); + Tensor forward( + const Tensor& input, + const std::optional& output_size = std::nullopt); + + protected: + FORWARD_HAS_DEFAULT_ARGS({1, AnyValue(std::optional())}) +}; + +/// A `ModuleHolder` subclass for `ConvTranspose2dImpl`. +/// See the documentation for `ConvTranspose2dImpl` class to learn what methods +/// it provides, and examples of how to use `ConvTranspose2d` with +/// `torch::nn::ConvTranspose2dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(ConvTranspose2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ConvTranspose3d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the ConvTranspose3d function. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ConvTranspose3d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ConvTranspose3dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ConvTranspose3d model(ConvTranspose3dOptions(2, 2, +/// 2).stride(1).bias(false)); +/// ``` +class TORCH_API ConvTranspose3dImpl + : public ConvTransposeNdImpl<3, ConvTranspose3dImpl> { + public: + ConvTranspose3dImpl( + int64_t input_channels, + int64_t output_channels, + ExpandingArray<3> kernel_size) + : ConvTranspose3dImpl(ConvTranspose3dOptions( + input_channels, + output_channels, + kernel_size)) {} + explicit ConvTranspose3dImpl(ConvTranspose3dOptions options_); + Tensor forward( + const Tensor& input, + const std::optional& output_size = std::nullopt); + + protected: + FORWARD_HAS_DEFAULT_ARGS({1, AnyValue(std::optional())}) +}; + +/// A `ModuleHolder` subclass for `ConvTranspose3dImpl`. +/// See the documentation for `ConvTranspose3dImpl` class to learn what methods +/// it provides, and examples of how to use `ConvTranspose3d` with +/// `torch::nn::ConvTranspose3dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(ConvTranspose3d); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/distance.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/distance.h new file mode 100644 index 0000000000000000000000000000000000000000..ccc6a4d21bb7c1339c501e123b03d7e317d6cc46 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/distance.h @@ -0,0 +1,84 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace torch::nn { + +/// Returns the cosine similarity between :math:`x_1` and :math:`x_2`, computed +/// along `dim`. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.CosineSimilarity to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::CosineSimilarityOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// CosineSimilarity model(CosineSimilarityOptions().dim(0).eps(0.5)); +/// ``` +class TORCH_API CosineSimilarityImpl : public Cloneable { + public: + explicit CosineSimilarityImpl(const CosineSimilarityOptions& options_ = {}); + + void reset() override; + + /// Pretty prints the `CosineSimilarity` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input1, const Tensor& input2); + + /// The options with which this `Module` was constructed. + CosineSimilarityOptions options; +}; + +/// A `ModuleHolder` subclass for `CosineSimilarityImpl`. +/// See the documentation for `CosineSimilarityImpl` class to learn what methods +/// it provides, and examples of how to use `CosineSimilarity` with +/// `torch::nn::CosineSimilarityOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(CosineSimilarity); + +// ============================================================================ + +/// Returns the batchwise pairwise distance between vectors :math:`v_1`, +/// :math:`v_2` using the p-norm. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.PairwiseDistance to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::PairwiseDistanceOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// PairwiseDistance +/// model(PairwiseDistanceOptions().p(3).eps(0.5).keepdim(true)); +/// ``` +class TORCH_API PairwiseDistanceImpl : public Cloneable { + public: + explicit PairwiseDistanceImpl(const PairwiseDistanceOptions& options_ = {}); + + void reset() override; + + /// Pretty prints the `PairwiseDistance` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input1, const Tensor& input2); + + /// The options with which this `Module` was constructed. + PairwiseDistanceOptions options; +}; + +/// A `ModuleHolder` subclass for `PairwiseDistanceImpl`. +/// See the documentation for `PairwiseDistanceImpl` class to learn what methods +/// it provides, and examples of how to use `PairwiseDistance` with +/// `torch::nn::PairwiseDistanceOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(PairwiseDistance); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/dropout.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/dropout.h new file mode 100644 index 0000000000000000000000000000000000000000..803d6d0a0bd61b7733e6c9ab8dd2c8722e7c5481 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/dropout.h @@ -0,0 +1,184 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace torch::nn { + +namespace detail { + +template +class _DropoutNd : public torch::nn::Cloneable { + public: + _DropoutNd(double p) : _DropoutNd(DropoutOptions().p(p)) {} + + explicit _DropoutNd(const DropoutOptions& options_ = {}) : options(options_) { + _DropoutNd::reset(); + } + + void reset() override { + TORCH_CHECK( + options.p() >= 0. && options.p() <= 1., + "dropout probability has to be between 0 and 1, but got ", + options.p()); + } + + /// The options with which this `Module` was constructed. + DropoutOptions options; +}; + +} // namespace detail + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Dropout ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies dropout over a 1-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Dropout to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::DropoutOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Dropout model(DropoutOptions().p(0.42).inplace(true)); +/// ``` +class TORCH_API DropoutImpl : public detail::_DropoutNd { + public: + using detail::_DropoutNd::_DropoutNd; + + Tensor forward(Tensor input); + + /// Pretty prints the `Dropout` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `DropoutImpl`. +/// See the documentation for `DropoutImpl` class to learn what methods it +/// provides, and examples of how to use `Dropout` with +/// `torch::nn::DropoutOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Dropout); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Dropout2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies dropout over a 2-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Dropout2d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::Dropout2dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Dropout2d model(Dropout2dOptions().p(0.42).inplace(true)); +/// ``` +class TORCH_API Dropout2dImpl : public detail::_DropoutNd { + public: + using detail::_DropoutNd::_DropoutNd; + + Tensor forward(Tensor input); + + /// Pretty prints the `Dropout2d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `Dropout2dImpl`. +/// See the documentation for `Dropout2dImpl` class to learn what methods it +/// provides, and examples of how to use `Dropout2d` with +/// `torch::nn::Dropout2dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Dropout2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Dropout3d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies dropout over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Dropout3d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::Dropout3dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Dropout3d model(Dropout3dOptions().p(0.42).inplace(true)); +/// ``` +class TORCH_API Dropout3dImpl : public detail::_DropoutNd { + public: + using detail::_DropoutNd::_DropoutNd; + + Tensor forward(Tensor input); + + /// Pretty prints the `Dropout3d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `Dropout3dImpl`. +/// See the documentation for `Dropout3dImpl` class to learn what methods it +/// provides, and examples of how to use `Dropout3d` with +/// `torch::nn::Dropout3dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Dropout3d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ AlphaDropout ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies Alpha Dropout over the input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.AlphaDropout to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::AlphaDropoutOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// AlphaDropout model(AlphaDropoutOptions(0.2).inplace(true)); +/// ``` +class TORCH_API AlphaDropoutImpl : public detail::_DropoutNd { + public: + using detail::_DropoutNd::_DropoutNd; + + Tensor forward(const Tensor& input); + + /// Pretty prints the `AlphaDropout` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `AlphaDropoutImpl`. +/// See the documentation for `AlphaDropoutImpl` class to learn what methods it +/// provides, and examples of how to use `AlphaDropout` with +/// `torch::nn::AlphaDropoutOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(AlphaDropout); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FeatureAlphaDropout +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// See the documentation for `torch::nn::FeatureAlphaDropoutOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// FeatureAlphaDropout model(FeatureAlphaDropoutOptions(0.2).inplace(true)); +/// ``` +class TORCH_API FeatureAlphaDropoutImpl + : public detail::_DropoutNd { + public: + using detail::_DropoutNd::_DropoutNd; + + Tensor forward(const Tensor& input); + + /// Pretty prints the `FeatureAlphaDropout` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; +}; + +/// A `ModuleHolder` subclass for `FeatureAlphaDropoutImpl`. +/// See the documentation for `FeatureAlphaDropoutImpl` class to learn what +/// methods it provides, and examples of how to use `FeatureAlphaDropout` with +/// `torch::nn::FeatureAlphaDropoutOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(FeatureAlphaDropout); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/embedding.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/embedding.h new file mode 100644 index 0000000000000000000000000000000000000000..943a1240e0b7e96e54f4213906d904aace6a7d63 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/embedding.h @@ -0,0 +1,165 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace torch::nn { + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Embedding +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Performs a lookup in a fixed size embedding table. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Embedding to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::EmbeddingOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Embedding model(EmbeddingOptions(10, +/// 2).padding_idx(3).max_norm(2).norm_type(2.5).scale_grad_by_freq(true).sparse(true)); +/// ``` +class TORCH_API EmbeddingImpl : public torch::nn::Cloneable { + public: + EmbeddingImpl(int64_t num_embeddings, int64_t embedding_dim) + : EmbeddingImpl(EmbeddingOptions(num_embeddings, embedding_dim)) {} + explicit EmbeddingImpl(EmbeddingOptions options_); + + void reset() override; + + void reset_parameters(); + + /// Pretty prints the `Embedding` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// Performs a lookup on the embedding table stored in `weight` using the + /// `indices` supplied and returns the result. + Tensor forward(const Tensor& indices); + + /// The `Options` used to configure this `Embedding` module. + /// Changes to `EmbeddingOptions` *after construction* have no effect. + EmbeddingOptions options; + + /// The embedding table. + Tensor weight; +}; + +/// A `ModuleHolder` subclass for `EmbeddingImpl`. +/// See the documentation for `EmbeddingImpl` class to learn what methods it +/// provides, and examples of how to use `Embedding` with +/// `torch::nn::EmbeddingOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +class Embedding : public torch::nn::ModuleHolder { + public: + using torch::nn::ModuleHolder::ModuleHolder; + + /// See the documentation for `torch::nn::EmbeddingFromPretrainedOptions` + /// class to learn what optional arguments are supported for this function. + static Embedding from_pretrained( + const torch::Tensor& embeddings, + const EmbeddingFromPretrainedOptions& options = {}) { + TORCH_CHECK( + embeddings.dim() == 2, + "Embeddings parameter is expected to be 2-dimensional"); + + auto rows = embeddings.size(0); + auto cols = embeddings.size(1); + + Embedding embedding(EmbeddingOptions(rows, cols) + ._weight(embeddings) + .padding_idx(options.padding_idx()) + .max_norm(options.max_norm()) + .norm_type(options.norm_type()) + .scale_grad_by_freq(options.scale_grad_by_freq()) + .sparse(options.sparse())); + embedding->weight.set_requires_grad(!options.freeze()); + return embedding; + } +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ EmbeddingBag +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Computes sums or means of 'bags' of embeddings, without instantiating the +/// intermediate embeddings. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.EmbeddingBag to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::EmbeddingBagOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// EmbeddingBag model(EmbeddingBagOptions(10, +/// 2).max_norm(2).norm_type(2.5).scale_grad_by_freq(true).sparse(true).mode(torch::kSum).padding_idx(1)); +/// ``` +class TORCH_API EmbeddingBagImpl + : public torch::nn::Cloneable { + public: + EmbeddingBagImpl(int64_t num_embeddings, int64_t embedding_dim) + : EmbeddingBagImpl(EmbeddingBagOptions(num_embeddings, embedding_dim)) {} + explicit EmbeddingBagImpl(EmbeddingBagOptions options_); + + void reset() override; + + void reset_parameters(); + + /// Pretty prints the `EmbeddingBag` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The `Options` used to configure this `EmbeddingBag` module. + EmbeddingBagOptions options; + /// The embedding table. + Tensor weight; + + Tensor forward( + const Tensor& input, + const Tensor& offsets = {}, + const Tensor& per_sample_weights = {}); + + protected: + FORWARD_HAS_DEFAULT_ARGS({1, AnyValue(Tensor())}, {2, AnyValue(Tensor())}) +}; + +/// A `ModuleHolder` subclass for `EmbeddingBagImpl`. +/// See the documentation for `EmbeddingBagImpl` class to learn what methods it +/// provides, and examples of how to use `EmbeddingBag` with +/// `torch::nn::EmbeddingBagOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +class EmbeddingBag : public torch::nn::ModuleHolder { + public: + using torch::nn::ModuleHolder::ModuleHolder; + + /// See the documentation for `torch::nn::EmbeddingBagFromPretrainedOptions` + /// class to learn what optional arguments are supported for this function. + static EmbeddingBag from_pretrained( + const torch::Tensor& embeddings, + const EmbeddingBagFromPretrainedOptions& options = {}) { + TORCH_CHECK( + embeddings.dim() == 2, + "Embeddings parameter is expected to be 2-dimensional"); + + auto rows = embeddings.size(0); + auto cols = embeddings.size(1); + + EmbeddingBag embeddingbag( + EmbeddingBagOptions(rows, cols) + ._weight(embeddings) + .max_norm(options.max_norm()) + .norm_type(options.norm_type()) + .scale_grad_by_freq(options.scale_grad_by_freq()) + .mode(options.mode()) + .sparse(options.sparse()) + .padding_idx(options.padding_idx())); + embeddingbag->weight.set_requires_grad(!options.freeze()); + return embeddingbag; + } +}; +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/fold.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/fold.h new file mode 100644 index 0000000000000000000000000000000000000000..f5618bc1c2217d79356aeebfb49a5793d0ca4ae0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/fold.h @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace torch::nn { + +/// Applies fold over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Fold to learn about +/// the exact behavior of this module. +/// +/// See the documentation for `torch::nn::FoldOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Fold model(FoldOptions({8, 8}, {3, 3}).dilation(2).padding({2, +/// 1}).stride(2)); +/// ``` +class TORCH_API FoldImpl : public torch::nn::Cloneable { + public: + FoldImpl(ExpandingArray<2> output_size, ExpandingArray<2> kernel_size) + : FoldImpl(FoldOptions(output_size, kernel_size)) {} + explicit FoldImpl(const FoldOptions& options_); + + void reset() override; + + /// Pretty prints the `Fold` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input); + + /// The options with which this `Module` was constructed. + FoldOptions options; +}; + +/// A `ModuleHolder` subclass for `FoldImpl`. +/// See the documentation for `FoldImpl` class to learn what methods it +/// provides, and examples of how to use `Fold` with `torch::nn::FoldOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(Fold); + +// ============================================================================ + +/// Applies unfold over a 4-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Unfold to learn about +/// the exact behavior of this module. +/// +/// See the documentation for `torch::nn::UnfoldOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Unfold model(UnfoldOptions({2, 4}).dilation(2).padding({2, 1}).stride(2)); +/// ``` +class TORCH_API UnfoldImpl : public Cloneable { + public: + UnfoldImpl(ExpandingArray<2> kernel_size) + : UnfoldImpl(UnfoldOptions(kernel_size)) {} + explicit UnfoldImpl(const UnfoldOptions& options_); + + void reset() override; + + /// Pretty prints the `Unfold` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input); + + /// The options with which this `Module` was constructed. + UnfoldOptions options; +}; + +/// A `ModuleHolder` subclass for `UnfoldImpl`. +/// See the documentation for `UnfoldImpl` class to learn what methods it +/// provides, and examples of how to use `Unfold` with +/// `torch::nn::UnfoldOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Unfold); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/instancenorm.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/instancenorm.h new file mode 100644 index 0000000000000000000000000000000000000000..9aea46d525047f8cc408b89cdea1ad427a745656 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/instancenorm.h @@ -0,0 +1,153 @@ +#pragma once + +#include +#include +#include + +namespace torch::nn { + +/// Base class for all (dimension-specialized) instance norm modules +template +// NOLINTNEXTLINE(bugprone-crtp-constructor-accessibility) +class InstanceNormImpl + : public torch::nn::NormImplBase { + private: + inline Tensor apply_instance_norm(const Tensor& input) { + return torch::nn::functional::detail::instance_norm( + input, + this->running_mean, + this->running_var, + this->weight, + this->bias, + this->is_training() || !this->options.track_running_stats(), + this->options.momentum(), + this->options.eps()); + } + + inline Tensor handle_no_batch_input(const Tensor& input) { + return this->apply_instance_norm(input.unsqueeze(0)).squeeze(0); + } + + public: + using torch::nn::NormImplBase::NormImplBase; + + Tensor forward(const Tensor& input) { + this->_check_input_dim(input); + + // For InstanceNorm1D, 2D is unbatched and 3D is batched + // For InstanceNorm2D, 3D is unbatched and 4D is batched + // For InstanceNorm3D, 4D is unbatched and 5D is batched + // check if input does not have a batch-dim + if (input.dim() == D + 1) { + return this->handle_no_batch_input(input); + } + + return this->apply_instance_norm(input); + } + + /// Pretty prints the `InstanceNorm{1,2,3}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override { + stream << std::boolalpha << "torch::nn::InstanceNorm" << D << "d(" + << this->options.num_features() << ", " + << "eps=" << this->options.eps() << ", " + << "momentum=" << this->options.momentum() << ", " + << "affine=" << this->options.affine() << ", " + << "track_running_stats=" << this->options.track_running_stats() + << ')'; + } +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ InstanceNorm1d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the InstanceNorm1d function. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.InstanceNorm1d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::InstanceNorm1dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// InstanceNorm1d +/// model(InstanceNorm1dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +class TORCH_API InstanceNorm1dImpl + : public InstanceNormImpl<1, InstanceNorm1dImpl> { + protected: + void _check_input_dim(const Tensor& input) override; + + public: + using InstanceNormImpl<1, InstanceNorm1dImpl>::InstanceNormImpl; +}; + +/// A `ModuleHolder` subclass for `InstanceNorm1dImpl`. +/// See the documentation for `InstanceNorm1dImpl` class to learn what methods +/// it provides, and examples of how to use `InstanceNorm1d` with +/// `torch::nn::InstanceNorm1dOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(InstanceNorm1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ InstanceNorm2d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the InstanceNorm2d function. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.InstanceNorm2d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::InstanceNorm2dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// InstanceNorm2d +/// model(InstanceNorm2dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +class TORCH_API InstanceNorm2dImpl + : public InstanceNormImpl<2, InstanceNorm2dImpl> { + protected: + void _check_input_dim(const Tensor& input) override; + + public: + using InstanceNormImpl<2, InstanceNorm2dImpl>::InstanceNormImpl; +}; + +/// A `ModuleHolder` subclass for `InstanceNorm2dImpl`. +/// See the documentation for `InstanceNorm2dImpl` class to learn what methods +/// it provides, and examples of how to use `InstanceNorm2d` with +/// `torch::nn::InstanceNorm2dOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(InstanceNorm2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ InstanceNorm3d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the InstanceNorm3d function. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.InstanceNorm3d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::InstanceNorm3dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// InstanceNorm3d +/// model(InstanceNorm3dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +class TORCH_API InstanceNorm3dImpl + : public InstanceNormImpl<3, InstanceNorm3dImpl> { + protected: + void _check_input_dim(const Tensor& input) override; + + public: + using InstanceNormImpl<3, InstanceNorm3dImpl>::InstanceNormImpl; +}; + +/// A `ModuleHolder` subclass for `InstanceNorm3dImpl`. +/// See the documentation for `InstanceNorm3dImpl` class to learn what methods +/// it provides, and examples of how to use `InstanceNorm3d` with +/// `torch::nn::InstanceNorm3dOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(InstanceNorm3d); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/linear.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/linear.h new file mode 100644 index 0000000000000000000000000000000000000000..bcea231e4abfb703c6700fabca1eb05ef5836c2f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/linear.h @@ -0,0 +1,210 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace torch::nn { + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Identity ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// A placeholder identity operator that is argument-insensitive. +/// See https://pytorch.org/docs/main/generated/torch.nn.Identity.html to +/// learn about the exact behavior of this module. +class TORCH_API IdentityImpl : public Cloneable { + public: + void reset() override; + + /// Pretty prints the `Identity` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `IdentityImpl`. +/// See the documentation for `IdentityImpl` class to learn what methods it +/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(Identity); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Linear ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies a linear transformation with optional bias. +/// See https://pytorch.org/docs/main/generated/torch.nn.Linear.html to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::LinearOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Linear model(LinearOptions(5, 2).bias(false)); +/// ``` +class TORCH_API LinearImpl : public Cloneable { + public: + LinearImpl(int64_t in_features, int64_t out_features) + : LinearImpl(LinearOptions(in_features, out_features)) {} + explicit LinearImpl(const LinearOptions& options_); + + void reset() override; + + void reset_parameters(); + + /// Pretty prints the `Linear` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// Transforms the `input` tensor by multiplying with the `weight` and + /// optionally adding the `bias`, if `with_bias` is true in the options. + Tensor forward(const Tensor& input); + + /// The options used to configure this module. + LinearOptions options; + + /// The learned weight. + Tensor weight; + + /// The learned bias. If `bias` is false in the `options`, this tensor is + /// undefined. + Tensor bias; +}; + +/// A `ModuleHolder` subclass for `LinearImpl`. +/// See the documentation for `LinearImpl` class to learn what methods it +/// provides, and examples of how to use `Linear` with +/// `torch::nn::LinearOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Linear); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Flatten ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// A placeholder for Flatten operator +/// See https://pytorch.org/docs/main/generated/torch.nn.Flatten.html to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::FlattenOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Flatten model(FlattenOptions().start_dim(2).end_dim(4)); +/// ``` +class TORCH_API FlattenImpl : public Cloneable { + public: + explicit FlattenImpl(const FlattenOptions& options_ = {}); + + void reset() override; + + /// Pretty prints the `Flatten` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// Applies a flatten transform on the `input`. + Tensor forward(const Tensor& input); + + /// The options used to configure this module. + FlattenOptions options; +}; + +/// A `ModuleHolder` subclass for `FlattenImpl`. +/// See the documentation for `FlattenImpl` class to learn what methods it +/// provides, and examples of how to use `Flatten` with +/// `torch::nn::FlattenOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Flatten); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Unflatten +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// A placeholder for unflatten operator +/// See https://pytorch.org/docs/main/generated/torch.nn.Unflatten.html to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::UnflattenOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Unflatten model(UnflattenOptions(0, {2, 2})); +/// ``` +class TORCH_API UnflattenImpl : public Cloneable { + public: + UnflattenImpl(int64_t dim, std::vector sizes) + : UnflattenImpl(UnflattenOptions(dim, std::move(sizes))) {} + explicit UnflattenImpl(UnflattenOptions options_); + + void reset() override; + + /// Pretty prints the `Unflatten` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// Applies an unflatten transform on the `input`. + Tensor forward(const Tensor& input); + + /// The options used to configure this module. + UnflattenOptions options; +}; + +/// A `ModuleHolder` subclass for `UnflattenImpl`. +/// See the documentation for `UnflattenImpl` class to learn what methods it +/// provides, and examples of how to use `Unflatten` with +/// `torch::nn::UnflattenOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Unflatten); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Bilinear ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies a bilinear transformation with optional bias. +/// See https://pytorch.org/docs/main/generated/torch.nn.Bilinear.html to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::BilinearOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Bilinear model(BilinearOptions(3, 2, 4).bias(false)); +/// ``` +class TORCH_API BilinearImpl : public Cloneable { + public: + BilinearImpl(int64_t in1_features, int64_t in2_features, int64_t out_features) + : BilinearImpl( + BilinearOptions(in1_features, in2_features, out_features)) {} + explicit BilinearImpl(const BilinearOptions& options_); + + void reset() override; + + void reset_parameters(); + + /// Pretty prints the `Bilinear` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// Applies a bilinear transform on the `input1` and `input2` tensor by + /// multiplying with the `weight` and optionally adding the `bias`, if + /// `with_bias` is true in the options. + Tensor forward(const Tensor& input1, const Tensor& input2); + + /// The options used to configure this module. + BilinearOptions options; + + /// The learned weight. + Tensor weight; + + /// The learned bias. If `with_bias` is false in the `options`, this tensor is + /// undefined. + Tensor bias; +}; + +/// A `ModuleHolder` subclass for `BilinearImpl`. +/// See the documentation for `BilinearImpl` class to learn what methods it +/// provides, and examples of how to use `Bilinear` with +/// `torch::nn::BilinearOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Bilinear); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/loss.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/loss.h new file mode 100644 index 0000000000000000000000000000000000000000..08d7cb535bd92457ab6ab307f465aaac6e159624 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/loss.h @@ -0,0 +1,803 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace torch::nn { + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ L1Loss ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that measures the mean absolute error (MAE) between each +/// element in the input : math :`x` and target : `y`. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.L1Loss to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::L1LossOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// L1Loss model(L1LossOptions(torch::kNone)); +/// ``` +struct TORCH_API L1LossImpl : Cloneable { + explicit L1LossImpl(L1LossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `L1Loss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + L1LossOptions options; +}; + +/// A `ModuleHolder` subclass for `L1LossImpl`. +/// See the documentation for `L1LossImpl` class to learn what methods it +/// provides, and examples of how to use `L1Loss` with +/// `torch::nn::L1LossOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(L1Loss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ KLDivLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// The Kullback-Leibler divergence loss measure +/// See https://pytorch.org/docs/main/nn.html#torch.nn.KLDivLoss to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::KLDivLossOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// KLDivLoss model(KLDivLossOptions().reduction(torch::kNone)); +/// ``` +struct TORCH_API KLDivLossImpl : Cloneable { + explicit KLDivLossImpl(KLDivLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `KLDivLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + KLDivLossOptions options; +}; + +/// A `ModuleHolder` subclass for `KLDivLossImpl`. +/// See the documentation for `KLDivLossImpl` class to learn what methods it +/// provides, and examples of how to use `KLDivLoss` with +/// `torch::nn::KLDivLossOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(KLDivLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MSELoss ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that measures the mean squared error (squared L2 norm) +/// between each element in the input :math:`x` and target :math:`y`. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.MSELoss to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MSELossOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MSELoss model(MSELossOptions(torch::kNone)); +/// ``` +struct TORCH_API MSELossImpl : Cloneable { + explicit MSELossImpl(MSELossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `MSELoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + MSELossOptions options; +}; + +/// A `ModuleHolder` subclass for `MSELossImpl`. +/// See the documentation for `MSELossImpl` class to learn what methods it +/// provides, and examples of how to use `MSELoss` with +/// `torch::nn::MSELossOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(MSELoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BCELoss ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that measures the Binary Cross Entropy +/// between the target and the output. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.BCELoss to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::BCELossOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// BCELoss model(BCELossOptions().reduction(torch::kNone).weight(weight)); +/// ``` +struct TORCH_API BCELossImpl : Cloneable { + explicit BCELossImpl(BCELossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `BCELoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + BCELossOptions options; +}; + +/// A `ModuleHolder` subclass for `BCELossImpl`. +/// See the documentation for `BCELossImpl` class to learn what methods it +/// provides, and examples of how to use `BCELoss` with +/// `torch::nn::BCELossOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(BCELoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ HingeEmbeddingLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that measures the loss given an input tensor :math:`x` +/// and a labels tensor :math:`y` (containing 1 or -1). +/// See https://pytorch.org/docs/main/nn.html#torch.nn.HingeEmbeddingLoss to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::HingeEmbeddingLossOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// HingeEmbeddingLoss +/// model(HingeEmbeddingLossOptions().margin(4).reduction(torch::kNone)); +/// ``` +struct TORCH_API HingeEmbeddingLossImpl : Cloneable { + explicit HingeEmbeddingLossImpl(HingeEmbeddingLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `HingeEmbeddingLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + HingeEmbeddingLossOptions options; +}; + +/// A `ModuleHolder` subclass for `HingeEmbeddingLossImpl`. +/// See the documentation for `HingeEmbeddingLossImpl` class to learn what +/// methods it provides, and examples of how to use `HingeEmbeddingLoss` with +/// `torch::nn::HingeEmbeddingLossOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(HingeEmbeddingLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MultiMarginLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that optimizes a multi-class classification hinge +/// loss (margin-based loss) between input :math:`x` (a 2D mini-batch `Tensor`) +/// and output :math:`y` (which is a 1D tensor of target class indices, :math:`0 +/// \leq y \leq \text{x.size}(1)-1`). See +/// https://pytorch.org/docs/main/nn.html#torch.nn.MultiMarginLoss to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MultiMarginLossOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MultiMarginLoss model(MultiMarginLossOptions().margin(2).weight(weight)); +/// ``` +struct TORCH_API MultiMarginLossImpl : public Cloneable { + explicit MultiMarginLossImpl(MultiMarginLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `MultiMarginLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + MultiMarginLossOptions options; +}; + +/// A `ModuleHolder` subclass for `MultiMarginLossImpl`. +/// See the documentation for `MultiMarginLossImpl` class to learn what methods +/// it provides, and examples of how to use `MultiMarginLoss` with +/// `torch::nn::MultiMarginLossOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(MultiMarginLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CosineEmbeddingLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that measures the loss given input tensors +/// `input1`, `input2`, and a `Tensor` label `target` with values 1 or +/// -1. This is used for measuring whether two inputs are similar or +/// dissimilar, using the cosine distance, and is typically used for learning +/// nonlinear embeddings or semi-supervised learning. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.CosineEmbeddingLoss to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::CosineEmbeddingLossOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// CosineEmbeddingLoss model(CosineEmbeddingLossOptions().margin(0.5)); +/// ``` +struct TORCH_API CosineEmbeddingLossImpl + : public Cloneable { + explicit CosineEmbeddingLossImpl(CosineEmbeddingLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `CosineEmbeddingLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward( + const Tensor& input1, + const Tensor& input2, + const Tensor& target); + + /// The options with which this `Module` was constructed. + CosineEmbeddingLossOptions options; +}; + +/// A `ModuleHolder` subclass for `CosineEmbeddingLossImpl`. +/// See the documentation for `CosineEmbeddingLossImpl` class to learn what +/// methods it provides, and examples of how to use `CosineEmbeddingLoss` with +/// `torch::nn::CosineEmbeddingLossOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(CosineEmbeddingLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SmoothL1Loss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that uses a squared term if the absolute +/// element-wise error falls below beta and an L1 term otherwise. +/// It is less sensitive to outliers than the `MSELoss` and in some cases +/// prevents exploding gradients (e.g. see the paper `Fast R-CNN` by Ross +/// Girshick). See https://pytorch.org/docs/main/nn.html#torch.nn.SmoothL1Loss +/// to learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::SmoothL1LossOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// SmoothL1Loss model(SmoothL1LossOptions().reduction(torch::kNone).beta(0.5)); +/// ``` +struct TORCH_API SmoothL1LossImpl : public Cloneable { + explicit SmoothL1LossImpl(SmoothL1LossOptions options = {}); + + void reset() override; + + /// Pretty prints the `L1Loss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + SmoothL1LossOptions options; +}; + +/// A `ModuleHolder` subclass for `SmoothL1LossImpl`. +/// See the documentation for `SmoothL1LossImpl` class to learn what methods it +/// provides, and examples of how to use `SmoothL1Loss` with +/// `torch::nn::SmoothL1LossOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(SmoothL1Loss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ HuberLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that uses a squared term if the absolute +/// element-wise error falls below delta and a delta-scaled L1 term otherwise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.HuberLoss to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::HuberLossOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// HuberLoss model(HuberLossOptions().reduction(torch::kNone).delta(0.5)); +/// ``` +struct TORCH_API HuberLossImpl : public Cloneable { + explicit HuberLossImpl(HuberLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `HuberLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + HuberLossOptions options; +}; + +/// A `ModuleHolder` subclass for `HuberLossImpl`. +/// See the documentation for `HuberLossImpl` class to learn what methods it +/// provides, and examples of how to use `HuberLoss` with +/// `torch::nn::HuberLossOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(HuberLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MultiLabelMarginLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that optimizes a multi-class multi-classification +/// hinge loss (margin-based loss) between input :math:`x` (a 2D mini-batch +/// `Tensor`) and output :math:`y` (which is a 2D `Tensor` of target class +/// indices). See +/// https://pytorch.org/docs/main/nn.html#torch.nn.MultiLabelMarginLoss to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MultiLabelMarginLossOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MultiLabelMarginLoss model(MultiLabelMarginLossOptions(torch::kNone)); +/// ``` +struct TORCH_API MultiLabelMarginLossImpl + : public Cloneable { + explicit MultiLabelMarginLossImpl(MultiLabelMarginLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `L1Loss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + MultiLabelMarginLossOptions options; +}; + +/// A `ModuleHolder` subclass for `MultiLabelMarginLossImpl`. +/// See the documentation for `MultiLabelMarginLossImpl` class to learn what +/// methods it provides, and examples of how to use `MultiLabelMarginLoss` with +/// `torch::nn::MultiLabelMarginLossOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(MultiLabelMarginLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SoftMarginLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that optimizes a two-class classification +/// logistic loss between input tensor :math:`x` and target tensor :math:`y` +/// (containing 1 or -1). +/// See https://pytorch.org/docs/main/nn.html#torch.nn.SoftMarginLoss to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::SoftMarginLossOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// SoftMarginLoss model(SoftMarginLossOptions(torch::kNone)); +/// ``` +struct TORCH_API SoftMarginLossImpl : public Cloneable { + explicit SoftMarginLossImpl(SoftMarginLossOptions options_ = {}); + + /// Pretty prints the `SoftMarginLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + void reset() override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + SoftMarginLossOptions options; +}; + +/// A `ModuleHolder` subclass for `SoftMarginLossImpl`. +/// See the documentation for `SoftMarginLossImpl` class to learn what methods +/// it provides, and examples of how to use `SoftMarginLoss` with +/// `torch::nn::SoftMarginLossOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(SoftMarginLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MultiLabelSoftMarginLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that optimizes a multi-label one-versus-all +/// loss based on max-entropy, between input :math:`x` and target :math:`y` of +/// size :math:`(N, C)`. See +/// https://pytorch.org/docs/main/nn.html#torch.nn.MultiLabelSoftMarginLoss to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MultiLabelSoftMarginLossOptions` class +/// to learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MultiLabelSoftMarginLoss +/// model(MultiLabelSoftMarginLossOptions().reduction(torch::kNone).weight(weight)); +/// ``` +struct TORCH_API MultiLabelSoftMarginLossImpl + : public Cloneable { + explicit MultiLabelSoftMarginLossImpl( + MultiLabelSoftMarginLossOptions options_ = {}); + + /// Pretty prints the `MultiLabelSoftMarginLoss` module into the given + /// `stream`. + void pretty_print(std::ostream& stream) const override; + + void reset() override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + MultiLabelSoftMarginLossOptions options; +}; + +/// A `ModuleHolder` subclass for `MultiLabelSoftMarginLossImpl`. +/// See the documentation for `MultiLabelSoftMarginLossImpl` class to learn what +/// methods it provides, and examples of how to use `MultiLabelSoftMarginLoss` +/// with `torch::nn::MultiLabelSoftMarginLossOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(MultiLabelSoftMarginLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ TripletMarginLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that measures the triplet loss given an input +/// tensors :math:`x1`, :math:`x2`, :math:`x3` and a margin with a value greater +/// than :math:`0`. This is used for measuring a relative similarity between +/// samples. A triplet is composed by `a`, `p` and `n` (i.e., `anchor`, +/// `positive examples` and `negative examples` respectively). The +/// shapes of all input tensors should be :math:`(N, D)`. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.TripletMarginLoss to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::TripletMarginLossOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// TripletMarginLoss +/// model(TripletMarginLossOptions().margin(3).p(2).eps(1e-06).swap(false)); +/// ``` +struct TORCH_API TripletMarginLossImpl + : public Cloneable { + explicit TripletMarginLossImpl(TripletMarginLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `TripletMarginLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward( + const Tensor& anchor, + const Tensor& positive, + const Tensor& negative); + + /// The options with which this `Module` was constructed. + TripletMarginLossOptions options; +}; + +/// A `ModuleHolder` subclass for `TripletMarginLossImpl`. +/// See the documentation for `TripletMarginLossImpl` class to learn what +/// methods it provides, and examples of how to use `TripletMarginLoss` with +/// `torch::nn::TripletMarginLossOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(TripletMarginLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ TripletMarginWithDistanceLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that measures the triplet loss given input +/// tensors :math:`a`, :math:`p`, and :math:`n` (representing anchor, +/// positive, and negative examples, respectively); and a nonnegative, +/// real-valued function +/// ("distance function") used to compute the relationships between the anchor +/// and positive example ("positive distance") and the anchor and negative +/// example ("negative distance"). +/// See +/// https://pytorch.org/docs/main/nn.html#torch.nn.TripletMarginWithDistanceLoss +/// to learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::TripletMarginWithDistanceLossOptions` +/// class to learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// TripletMarginWithDistanceLoss +/// model(TripletMarginWithDistanceLossOptions().margin(3).swap(false)); +/// ``` +struct TORCH_API TripletMarginWithDistanceLossImpl + : public Cloneable { + explicit TripletMarginWithDistanceLossImpl( + TripletMarginWithDistanceLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `TripletMarginWithDistanceLoss` module into the given + /// `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward( + const Tensor& anchor, + const Tensor& positive, + const Tensor& negative); + + /// The options with which this `Module` was constructed. + TripletMarginWithDistanceLossOptions options; +}; + +/// A `ModuleHolder` subclass for `TripletMarginWithDistanceLossImpl`. +/// See the documentation for `TripletMarginWithDistanceLossImpl` class to learn +/// what methods it provides, and examples of how to use +/// `TripletMarginWithDistanceLoss` with +/// `torch::nn::TripletMarginWithDistanceLossOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(TripletMarginWithDistanceLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CTCLoss ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// The Connectionist Temporal Classification loss. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.CTCLoss to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::CTCLossOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// CTCLoss +/// model(CTCLossOptions().blank(42).zero_infinity(false).reduction(torch::kSum)); +/// ``` +struct TORCH_API CTCLossImpl : public Cloneable { + explicit CTCLossImpl(CTCLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `CTCLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward( + const Tensor& log_probs, + const Tensor& targets, + const Tensor& input_lengths, + const Tensor& target_lengths); + + /// The options with which this `Module` was constructed. + CTCLossOptions options; +}; + +/// A `ModuleHolder` subclass for `CTCLossImpl`. +/// See the documentation for `CTCLossImpl` class to learn what methods it +/// provides, and examples of how to use `CTCLoss` with +/// `torch::nn::CTCLossOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(CTCLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PoissonNLLLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Negative log likelihood loss with Poisson distribution of target. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.PoissonNLLLoss to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::PoissonNLLLossOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// PoissonNLLLoss +/// model(PoissonNLLLossOptions().log_input(false).full(true).eps(0.42).reduction(torch::kSum)); +/// ``` +struct TORCH_API PoissonNLLLossImpl : public Cloneable { + explicit PoissonNLLLossImpl(PoissonNLLLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `PoissonNLLLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& log_input, const Tensor& targets); + + /// The options with which this `Module` was constructed. + PoissonNLLLossOptions options; +}; + +/// A `ModuleHolder` subclass for `PoissonNLLLossImpl`. +/// See the documentation for `PoissonNLLLossImpl` class to learn what methods +/// it provides, and examples of how to use `PoissonNLLLoss` with +/// `torch::nn::PoissonNLLLossOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(PoissonNLLLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MarginRankingLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that measures the loss given +/// inputs :math:`x1`, :math:`x2`, two 1D mini-batch `Tensors`, +/// and a label 1D mini-batch tensor :math:`y` (containing 1 or -1). +/// See https://pytorch.org/docs/main/nn.html#torch.nn.MarginRankingLoss to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MarginRankingLossOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MarginRankingLoss +/// model(MarginRankingLossOptions().margin(0.5).reduction(torch::kSum)); +/// ``` +struct TORCH_API MarginRankingLossImpl + : public Cloneable { + explicit MarginRankingLossImpl(MarginRankingLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `MarginRankingLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward( + const Tensor& input1, + const Tensor& input2, + const Tensor& targets); + + /// The options with which this `Module` was constructed. + MarginRankingLossOptions options; +}; + +/// A `ModuleHolder` subclass for `MarginRankingLossImpl`. +/// See the documentation for `MarginRankingLossImpl` class to learn what +/// methods it provides, and examples of how to use `MarginRankingLoss` with +/// `torch::nn::MarginRankingLossOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(MarginRankingLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ NLLLoss ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// The negative log likelihood loss. It is useful to train a classification +/// problem with `C` classes. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.NLLLoss to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::NLLLossOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// NLLLoss model(NLLLossOptions().ignore_index(-100).reduction(torch::kMean)); +/// ``` +struct TORCH_API NLLLossImpl : public Cloneable { + explicit NLLLossImpl(NLLLossOptions options_ = {}); + + /// Pretty prints the `NLLLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + void reset() override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + NLLLossOptions options; + + /// A manual rescaling weight given to each class. + Tensor weight; +}; + +/// A `ModuleHolder` subclass for `NLLLossImpl`. +/// See the documentation for `NLLLossImpl` class to learn what methods it +/// provides, and examples of how to use `NLLLoss` with +/// `torch::nn::NLLLossOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(NLLLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CrossEntropyLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Creates a criterion that computes cross entropy loss between input and +/// target. See +/// https://pytorch.org/docs/main/nn.html#torch.nn.CrossEntropyLoss to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::CrossEntropyLossOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// CrossEntropyLoss +/// model(CrossEntropyLossOptions().ignore_index(-100).reduction(torch::kMean)); +/// ``` +struct TORCH_API CrossEntropyLossImpl : public Cloneable { + explicit CrossEntropyLossImpl(CrossEntropyLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `CrossEntropyLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + CrossEntropyLossOptions options; + + /// A manual rescaling weight given to each class. + Tensor weight; +}; + +/// A `ModuleHolder` subclass for `CrossEntropyLossImpl`. +/// See the documentation for `CrossEntropyLossImpl` class to learn what methods +/// it provides, and examples of how to use `CrossEntropyLoss` with +/// `torch::nn::CrossEntropyLossOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(CrossEntropyLoss); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BCEWithLogitsLoss +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// This loss combines a `Sigmoid` layer and the `BCELoss` in one single +/// class. This version is more numerically stable than using a plain `Sigmoid` +/// followed by a `BCELoss` as, by combining the operations into one layer, +/// we take advantage of the log-sum-exp trick for numerical stability. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.BCEWithLogitsLoss to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::BCEWithLogitsLossOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// BCEWithLogitsLoss +/// model(BCEWithLogitsLossOptions().reduction(torch::kNone).weight(weight)); +/// ``` +struct TORCH_API BCEWithLogitsLossImpl + : public Cloneable { + explicit BCEWithLogitsLossImpl(BCEWithLogitsLossOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `BCEWithLogitsLoss` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input, const Tensor& target); + + /// The options with which this `Module` was constructed. + BCEWithLogitsLossOptions options; + + /// A manual rescaling weight given to the loss of each batch element. + Tensor weight; + + /// A weight of positive examples. + Tensor pos_weight; +}; + +/// A `ModuleHolder` subclass for `BCEWithLogitsLossImpl`. +/// See the documentation for `BCEWithLogitsLossImpl` class to learn what +/// methods it provides, and examples of how to use `BCEWithLogitsLoss` with +/// `torch::nn::BCEWithLogitsLossOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(BCEWithLogitsLoss); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/normalization.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/normalization.h new file mode 100644 index 0000000000000000000000000000000000000000..2546e57ff1a86e1704ec31f1cf936075ab60e697 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/normalization.h @@ -0,0 +1,197 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace torch::nn { + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LayerNorm ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies Layer Normalization over a mini-batch of inputs as described in +/// the paper `Layer Normalization`_ . +/// See https://pytorch.org/docs/main/nn.html#torch.nn.LayerNorm to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::LayerNormOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// LayerNorm model(LayerNormOptions({2, +/// 2}).elementwise_affine(false).eps(2e-5)); +/// ``` +class TORCH_API LayerNormImpl : public torch::nn::Cloneable { + public: + LayerNormImpl(std::vector normalized_shape) + : LayerNormImpl(LayerNormOptions(std::move(normalized_shape))) {} + explicit LayerNormImpl(LayerNormOptions options_); + + void reset() override; + + void reset_parameters(); + + /// Pretty prints the `LayerNorm` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// Applies layer normalization over a mini-batch of inputs as described in + /// the paper `Layer Normalization`_ . + /// + /// The mean and standard-deviation are calculated separately over the last + /// certain number dimensions which have to be of the shape specified by + /// input `normalized_shape`. + /// + /// `Layer Normalization`: https://arxiv.org/abs/1607.06450 + Tensor forward(const Tensor& input); + + /// The options with which this module was constructed. + LayerNormOptions options; + + /// The learned weight. + /// Initialized to ones if the `elementwise_affine` option is set to `true` + /// upon construction. + Tensor weight; + + /// The learned bias. + /// Initialized to zeros `elementwise_affine` option is set to `true` upon + /// construction. + Tensor bias; +}; + +/// A `ModuleHolder` subclass for `LayerNormImpl`. +/// See the documentation for `LayerNormImpl` class to learn what methods it +/// provides, and examples of how to use `LayerNorm` with +/// `torch::nn::LayerNormOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(LayerNorm); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LocalResponseNorm +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies local response normalization over an input signal composed +/// of several input planes, where channels occupy the second dimension. +/// Applies normalization across channels. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.LocalResponseNorm to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::LocalResponseNormOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// LocalResponseNorm +/// model(LocalResponseNormOptions(2).alpha(0.0002).beta(0.85).k(2.)); +/// ``` +class TORCH_API LocalResponseNormImpl + : public Cloneable { + public: + LocalResponseNormImpl(int64_t size) + : LocalResponseNormImpl(LocalResponseNormOptions(size)) {} + explicit LocalResponseNormImpl(const LocalResponseNormOptions& options_); + + Tensor forward(const Tensor& input); + + void reset() override; + + /// Pretty prints the `LocalResponseNormImpl` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + LocalResponseNormOptions options; +}; + +/// A `ModuleHolder` subclass for `LocalResponseNormImpl`. +/// See the documentation for `LocalResponseNormImpl` class to learn what +/// methods it provides, and examples of how to use `LocalResponseNorm` with +/// `torch::nn::LocalResponseNormOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(LocalResponseNorm); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CrossMapLRN2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// See the documentation for `torch::nn::CrossMapLRN2dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// CrossMapLRN2d model(CrossMapLRN2dOptions(3).alpha(1e-5).beta(0.1).k(10)); +/// ``` +class TORCH_API CrossMapLRN2dImpl + : public torch::nn::Cloneable { + public: + CrossMapLRN2dImpl(int64_t size) + : CrossMapLRN2dImpl(CrossMapLRN2dOptions(size)) {} + explicit CrossMapLRN2dImpl(const CrossMapLRN2dOptions& options_) + : options(options_) {} + + void reset() override; + + /// Pretty prints the `CrossMapLRN2d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + torch::Tensor forward(const torch::Tensor& input); + + CrossMapLRN2dOptions options; +}; + +/// A `ModuleHolder` subclass for `CrossMapLRN2dImpl`. +/// See the documentation for `CrossMapLRN2dImpl` class to learn what methods it +/// provides, and examples of how to use `CrossMapLRN2d` with +/// `torch::nn::CrossMapLRN2dOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(CrossMapLRN2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GroupNorm ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies Group Normalization over a mini-batch of inputs as described in +/// the paper `Group Normalization`_ . +/// See https://pytorch.org/docs/main/nn.html#torch.nn.GroupNorm to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::GroupNormOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// GroupNorm model(GroupNormOptions(2, 2).eps(2e-5).affine(false)); +/// ``` +class TORCH_API GroupNormImpl : public torch::nn::Cloneable { + public: + GroupNormImpl(int64_t num_groups, int64_t num_channels) + : GroupNormImpl(GroupNormOptions(num_groups, num_channels)) {} + explicit GroupNormImpl(const GroupNormOptions& options_); + + void reset() override; + + void reset_parameters(); + + /// Pretty prints the `GroupNorm` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input); + + /// The options with which this module was constructed. + GroupNormOptions options; + + /// The learned weight. + Tensor weight; + + /// The learned bias. + Tensor bias; +}; + +/// A `ModuleHolder` subclass for `GroupNormImpl`. +/// See the documentation for `GroupNormImpl` class to learn what methods it +/// provides, and examples of how to use `GroupNorm` with +/// `torch::nn::GroupNormOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(GroupNorm); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/padding.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/padding.h new file mode 100644 index 0000000000000000000000000000000000000000..f35806e794a48f34ac05f13aebb5c8432f602803 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/padding.h @@ -0,0 +1,376 @@ +#pragma once + +#include +#include +#include + +#include + +namespace torch::nn { + +/// Base class for all (dimension-specialized) ReflectionPad modules. +template +class TORCH_API ReflectionPadImpl : public torch::nn::Cloneable { + public: + ReflectionPadImpl(ExpandingArray padding) + : ReflectionPadImpl(ReflectionPadOptions(padding)) {} + explicit ReflectionPadImpl(const ReflectionPadOptions& options_); + + void reset() override; + + Tensor forward(const Tensor& input); + + /// Pretty prints the `ReflectionPad{1,2}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + ReflectionPadOptions options; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ReflectionPad1d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies ReflectionPad over a 1-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ReflectionPad1d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ReflectionPad1dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ReflectionPad1d model(ReflectionPad1dOptions({3, 1})); +/// ``` +class TORCH_API ReflectionPad1dImpl + : public ReflectionPadImpl<1, ReflectionPad1dImpl> { + public: + using ReflectionPadImpl<1, ReflectionPad1dImpl>::ReflectionPadImpl; +}; + +/// A `ModuleHolder` subclass for `ReflectionPad1dImpl`. +/// See the documentation for `ReflectionPad1dImpl` class to learn what methods +/// it provides, and examples of how to use `ReflectionPad1d` with +/// `torch::nn::ReflectionPad1dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(ReflectionPad1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ReflectionPad2d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies ReflectionPad over a 2-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ReflectionPad2d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ReflectionPad2dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ReflectionPad2d model(ReflectionPad2dOptions({1, 1, 2, 0})); +/// ``` +class TORCH_API ReflectionPad2dImpl + : public ReflectionPadImpl<2, ReflectionPad2dImpl> { + public: + using ReflectionPadImpl<2, ReflectionPad2dImpl>::ReflectionPadImpl; +}; + +/// A `ModuleHolder` subclass for `ReflectionPad2dImpl`. +/// See the documentation for `ReflectionPad2dImpl` class to learn what methods +/// it provides, and examples of how to use `ReflectionPad2d` with +/// `torch::nn::ReflectionPad2dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(ReflectionPad2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ReflectionPad3d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies ReflectionPad over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ReflectionPad3d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ReflectionPad3dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ReflectionPad3d model(ReflectionPad3dOptions(1)); +/// ReflectionPad3d model(ReflectionPad3dOptions({1, 1, 2, 0, 1, 2})); +/// ``` +class TORCH_API ReflectionPad3dImpl + : public ReflectionPadImpl<3, ReflectionPad3dImpl> { + public: + using ReflectionPadImpl<3, ReflectionPad3dImpl>::ReflectionPadImpl; +}; + +/// A `ModuleHolder` subclass for `ReflectionPad3dImpl`. +/// See the documentation for `ReflectionPad3dImpl` class to learn what methods +/// it provides, and examples of how to use `ReflectionPad3d` with +/// `torch::nn::ReflectionPad3dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(ReflectionPad3d); + +// ============================================================================ + +/// Base class for all (dimension-specialized) ReplicationPad modules. +template +class TORCH_API ReplicationPadImpl : public torch::nn::Cloneable { + public: + ReplicationPadImpl(ExpandingArray padding) + : ReplicationPadImpl(ReplicationPadOptions(padding)) {} + explicit ReplicationPadImpl(const ReplicationPadOptions& options_); + + void reset() override; + + Tensor forward(const Tensor& input); + + /// Pretty prints the `ReplicationPad{1,2}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + ReplicationPadOptions options; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ReplicationPad1d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies ReplicationPad over a 1-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ReplicationPad1d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ReplicationPad1dOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ReplicationPad1d model(ReplicationPad1dOptions({3, 1})); +/// ``` +class TORCH_API ReplicationPad1dImpl + : public ReplicationPadImpl<1, ReplicationPad1dImpl> { + public: + using ReplicationPadImpl<1, ReplicationPad1dImpl>::ReplicationPadImpl; +}; + +/// A `ModuleHolder` subclass for `ReplicationPad1dImpl`. +/// See the documentation for `ReplicationPad1dImpl` class to learn what methods +/// it provides, and examples of how to use `ReplicationPad1d` with +/// `torch::nn::ReplicationPad1dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(ReplicationPad1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ReplicationPad2d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies ReplicationPad over a 2-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ReplicationPad2d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ReplicationPad2dOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ReplicationPad2d model(ReplicationPad2dOptions({1, 1, 2, 0})); +/// ``` +class TORCH_API ReplicationPad2dImpl + : public ReplicationPadImpl<2, ReplicationPad2dImpl> { + public: + using ReplicationPadImpl<2, ReplicationPad2dImpl>::ReplicationPadImpl; +}; + +/// A `ModuleHolder` subclass for `ReplicationPad2dImpl`. +/// See the documentation for `ReplicationPad2dImpl` class to learn what methods +/// it provides, and examples of how to use `ReplicationPad2d` with +/// `torch::nn::ReplicationPad2dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(ReplicationPad2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ReplicationPad3d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies ReplicationPad over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ReplicationPad3d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ReplicationPad3dOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ReplicationPad3d model(ReplicationPad3dOptions({1, 2, 1, 2, 1, 2})); +/// ``` +class TORCH_API ReplicationPad3dImpl + : public ReplicationPadImpl<3, ReplicationPad3dImpl> { + public: + using ReplicationPadImpl<3, ReplicationPad3dImpl>::ReplicationPadImpl; +}; + +/// A `ModuleHolder` subclass for `ReplicationPad3dImpl`. +/// See the documentation for `ReplicationPad3dImpl` class to learn what methods +/// it provides, and examples of how to use `ReplicationPad3d` with +/// `torch::nn::ReplicationPad3dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(ReplicationPad3d); + +// ============================================================================ + +/// Base class for all (dimension-specialized) ZeroPad modules. +template +class TORCH_API ZeroPadImpl : public torch::nn::Cloneable { + public: + ZeroPadImpl(ExpandingArray padding) + : ZeroPadImpl(ZeroPadOptions(padding)) {} + explicit ZeroPadImpl(const ZeroPadOptions& options_); + + void reset() override; + + Tensor forward(const Tensor& input); + + /// Pretty prints the `ZeroPad{1,2}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + ZeroPadOptions options; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ZeroPad1d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// Applies ZeroPad over a 1-D input. +class TORCH_API ZeroPad1dImpl : public ZeroPadImpl<1, ZeroPad1dImpl> { + public: + using ZeroPadImpl<1, ZeroPad1dImpl>::ZeroPadImpl; +}; + +/// A `ModuleHolder` subclass for `ZeroPad1dImpl`. +/// See the documentation for `ZeroPad1dImpl` class to learn what methods it +/// provides, and examples of how to use `ZeroPad1d` with +/// `torch::nn::ZeroPad1dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(ZeroPad1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ZeroPad2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// Applies ZeroPad over a 2-D input. +class TORCH_API ZeroPad2dImpl : public ZeroPadImpl<2, ZeroPad2dImpl> { + public: + using ZeroPadImpl<2, ZeroPad2dImpl>::ZeroPadImpl; +}; + +/// A `ModuleHolder` subclass for `ZeroPad2dImpl`. +/// See the documentation for `ZeroPad2dImpl` class to learn what methods it +/// provides, and examples of how to use `ZeroPad2d` with +/// `torch::nn::ZeroPad2dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(ZeroPad2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ZeroPad3d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// Applies ZeroPad over a 3-D input. +class TORCH_API ZeroPad3dImpl : public ZeroPadImpl<3, ZeroPad3dImpl> { + public: + using ZeroPadImpl<3, ZeroPad3dImpl>::ZeroPadImpl; +}; + +/// A `ModuleHolder` subclass for `ZeroPad3dImpl`. +/// See the documentation for `ZeroPad3dImpl` class to learn what methods it +/// provides, and examples of how to use `ZeroPad3d` with +/// `torch::nn::ZeroPad3dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(ZeroPad3d); + +// ============================================================================ + +/// Base class for all (dimension-specialized) ConstantPad modules. +template +class TORCH_API ConstantPadImpl : public torch::nn::Cloneable { + public: + ConstantPadImpl(ExpandingArray padding, double value) + : ConstantPadImpl(ConstantPadOptions(padding, value)) {} + explicit ConstantPadImpl(const ConstantPadOptions& options_); + + void reset() override; + + Tensor forward(const Tensor& input); + + /// Pretty prints the `ConstantPad{1,2}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + ConstantPadOptions options; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ConstantPad1d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies ConstantPad over a 1-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ConstantPad1d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ConstantPad1dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ConstantPad1d model(ConstantPad1dOptions({3, 1}, 3.5)); +/// ``` +class TORCH_API ConstantPad1dImpl + : public ConstantPadImpl<1, ConstantPad1dImpl> { + public: + using ConstantPadImpl<1, ConstantPad1dImpl>::ConstantPadImpl; +}; + +/// A `ModuleHolder` subclass for `ConstantPad1dImpl`. +/// See the documentation for `ConstantPad1dImpl` class to learn what methods it +/// provides, and examples of how to use `ConstantPad1d` with +/// `torch::nn::ConstantPad1dOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(ConstantPad1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ConstantPad2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies ConstantPad over a 2-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ConstantPad2d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ConstantPad2dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ConstantPad2d model(ConstantPad2dOptions({3, 0, 2, 1}, 3.5)); +/// ``` +class TORCH_API ConstantPad2dImpl + : public ConstantPadImpl<2, ConstantPad2dImpl> { + public: + using ConstantPadImpl<2, ConstantPad2dImpl>::ConstantPadImpl; +}; + +/// A `ModuleHolder` subclass for `ConstantPad2dImpl`. +/// See the documentation for `ConstantPad2dImpl` class to learn what methods it +/// provides, and examples of how to use `ConstantPad2d` with +/// `torch::nn::ConstantPad2dOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(ConstantPad2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ConstantPad3d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies ConstantPad over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.ConstantPad3d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::ConstantPad3dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// ConstantPad3d model(ConstantPad3dOptions({1, 2, 1, 2, 1, 2}, 3.5)); +/// ``` +class TORCH_API ConstantPad3dImpl + : public ConstantPadImpl<3, ConstantPad3dImpl> { + public: + using ConstantPadImpl<3, ConstantPad3dImpl>::ConstantPadImpl; +}; + +/// A `ModuleHolder` subclass for `ConstantPad3dImpl`. +/// See the documentation for `ConstantPad3dImpl` class to learn what methods it +/// provides, and examples of how to use `ConstantPad3d` with +/// `torch::nn::ConstantPad3dOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(ConstantPad3d); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/pixelshuffle.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/pixelshuffle.h new file mode 100644 index 0000000000000000000000000000000000000000..a33d295a4eccbb4ba531a3d65fc051fd2261bcfb --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/pixelshuffle.h @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include + +#include + +namespace torch::nn { + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PixelShuffle +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Rearranges elements in a tensor of shape :math:`(*, C \times r^2, H, W)` +/// to a tensor of shape :math:`(*, C, H \times r, W \times r)`, where r is an +/// upscale factor. See +/// https://pytorch.org/docs/main/nn.html#torch.nn.PixelShuffle to learn about +/// the exact behavior of this module. +/// +/// See the documentation for `torch::nn::PixelShuffleOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// PixelShuffle model(PixelShuffleOptions(5)); +/// ``` +struct TORCH_API PixelShuffleImpl + : public torch::nn::Cloneable { + explicit PixelShuffleImpl(const PixelShuffleOptions& options_); + + /// Pretty prints the `PixelShuffle` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input); + + void reset() override; + + /// The options with which this `Module` was constructed. + PixelShuffleOptions options; +}; + +/// A `ModuleHolder` subclass for `PixelShuffleImpl`. +/// See the documentation for `PixelShuffleImpl` class to learn what methods it +/// provides, and examples of how to use `PixelShuffle` with +/// `torch::nn::PixelShuffleOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(PixelShuffle); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PixelUnshuffle ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Reverses the PixelShuffle operation by rearranging elements in a tensor of +/// shape :math:`(*, C, H \times r, W \times r)` to a tensor of shape :math:`(*, +/// C \times r^2, H, W)`, where r is a downscale factor. See +/// https://pytorch.org/docs/main/nn.html#torch.nn.PixelUnshuffle to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::PixelUnshuffleOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// PixelUnshuffle model(PixelUnshuffleOptions(5)); +/// ``` +struct TORCH_API PixelUnshuffleImpl + : public torch::nn::Cloneable { + explicit PixelUnshuffleImpl(const PixelUnshuffleOptions& options_); + + /// Pretty prints the `PixelUnshuffle` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input); + + void reset() override; + + /// The options with which this `Module` was constructed. + PixelUnshuffleOptions options; +}; + +/// A `ModuleHolder` subclass for `PixelUnshuffleImpl`. +/// See the documentation for `PixelUnshuffleImpl` class to learn what methods +/// it provides, and examples of how to use `PixelUnshuffle` with +/// `torch::nn::PixelUnshuffleOptions`. See the documentation for `ModuleHolder` +/// to learn about PyTorch's module storage semantics. +TORCH_MODULE(PixelUnshuffle); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/pooling.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/pooling.h new file mode 100644 index 0000000000000000000000000000000000000000..2b90ecc186e883ceff84ee232f1469acba2b2266 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/pooling.h @@ -0,0 +1,777 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace torch::nn { + +/// Base class for all (dimension-specialized) avgpool modules. +template +class TORCH_API AvgPoolImpl : public torch::nn::Cloneable { + public: + AvgPoolImpl(ExpandingArray kernel_size) + : AvgPoolImpl(AvgPoolOptions(kernel_size)) {} + explicit AvgPoolImpl(const AvgPoolOptions& options_); + + void reset() override; + + /// Pretty prints the `AvgPool{1,2,3}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + AvgPoolOptions options; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ AvgPool1d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies avgpool over a 1-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.AvgPool1d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::AvgPool1dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// AvgPool1d model(AvgPool1dOptions(3).stride(2)); +/// ``` +class TORCH_API AvgPool1dImpl : public AvgPoolImpl<1, AvgPool1dImpl> { + public: + using AvgPoolImpl<1, AvgPool1dImpl>::AvgPoolImpl; + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `AvgPool1dImpl`. +/// See the documentation for `AvgPool1dImpl` class to learn what methods it +/// provides, and examples of how to use `AvgPool1d` with +/// `torch::nn::AvgPool1dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(AvgPool1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ AvgPool2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies avgpool over a 2-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.AvgPool2d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::AvgPool2dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// AvgPool2d model(AvgPool2dOptions({3, 2}).stride({2, 2})); +/// ``` +class TORCH_API AvgPool2dImpl : public AvgPoolImpl<2, AvgPool2dImpl> { + public: + using AvgPoolImpl<2, AvgPool2dImpl>::AvgPoolImpl; + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `AvgPool2dImpl`. +/// See the documentation for `AvgPool2dImpl` class to learn what methods it +/// provides, and examples of how to use `AvgPool2d` with +/// `torch::nn::AvgPool2dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(AvgPool2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ AvgPool3d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies avgpool over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.AvgPool3d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::AvgPool3dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// AvgPool3d model(AvgPool3dOptions(5).stride(2)); +/// ``` +class TORCH_API AvgPool3dImpl : public AvgPoolImpl<3, AvgPool3dImpl> { + public: + using AvgPoolImpl<3, AvgPool3dImpl>::AvgPoolImpl; + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `AvgPool3dImpl`. +/// See the documentation for `AvgPool3dImpl` class to learn what methods it +/// provides, and examples of how to use `AvgPool3d` with +/// `torch::nn::AvgPool3dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(AvgPool3d); + +// ============================================================================ + +/// Base class for all (dimension-specialized) maxpool modules. +template +class TORCH_API MaxPoolImpl : public torch::nn::Cloneable { + public: + MaxPoolImpl(ExpandingArray kernel_size) + : MaxPoolImpl(MaxPoolOptions(kernel_size)) {} + explicit MaxPoolImpl(const MaxPoolOptions& options_); + + void reset() override; + + /// Pretty prints the `MaxPool{1,2,3}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + MaxPoolOptions options; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MaxPool1d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies maxpool over a 1-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.MaxPool1d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MaxPool1dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MaxPool1d model(MaxPool1dOptions(3).stride(2)); +/// ``` +class TORCH_API MaxPool1dImpl : public MaxPoolImpl<1, MaxPool1dImpl> { + public: + using MaxPoolImpl<1, MaxPool1dImpl>::MaxPoolImpl; + Tensor forward(const Tensor& input); + + /// Returns the outputs and the indices of the max values. + /// Useful for `torch::nn::MaxUnpool1d` later. + std::tuple forward_with_indices(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `MaxPool1dImpl`. +/// See the documentation for `MaxPool1dImpl` class to learn what methods it +/// provides, and examples of how to use `MaxPool1d` with +/// `torch::nn::MaxPool1dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(MaxPool1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MaxPool2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies maxpool over a 2-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.MaxPool2d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MaxPool2dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MaxPool2d model(MaxPool2dOptions({3, 2}).stride({2, 2})); +/// ``` +class TORCH_API MaxPool2dImpl : public MaxPoolImpl<2, MaxPool2dImpl> { + public: + using MaxPoolImpl<2, MaxPool2dImpl>::MaxPoolImpl; + Tensor forward(const Tensor& input); + + /// Returns the outputs and the indices of the max values. + /// Useful for `torch::nn::MaxUnpool2d` later. + std::tuple forward_with_indices(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `MaxPool2dImpl`. +/// See the documentation for `MaxPool2dImpl` class to learn what methods it +/// provides, and examples of how to use `MaxPool2d` with +/// `torch::nn::MaxPool2dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(MaxPool2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MaxPool3d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies maxpool over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.MaxPool3d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MaxPool3dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MaxPool3d model(MaxPool3dOptions(3).stride(2)); +/// ``` +class TORCH_API MaxPool3dImpl : public MaxPoolImpl<3, MaxPool3dImpl> { + public: + using MaxPoolImpl<3, MaxPool3dImpl>::MaxPoolImpl; + Tensor forward(const Tensor& input); + + /// Returns the outputs and the indices of the max values. + /// Useful for `torch::nn::MaxUnpool3d` later. + std::tuple forward_with_indices(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `MaxPool3dImpl`. +/// See the documentation for `MaxPool3dImpl` class to learn what methods it +/// provides, and examples of how to use `MaxPool3d` with +/// `torch::nn::MaxPool3dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(MaxPool3d); + +// ============================================================================ + +/// Base class for all (dimension-specialized) adaptive maxpool modules. +template +class TORCH_API AdaptiveMaxPoolImpl : public torch::nn::Cloneable { + public: + AdaptiveMaxPoolImpl(output_size_t output_size) + : AdaptiveMaxPoolImpl( + AdaptiveMaxPoolOptions(output_size)) {} + explicit AdaptiveMaxPoolImpl( + const AdaptiveMaxPoolOptions& options_) + : options(options_) {} + + void reset() override {} + + /// Pretty prints the `AdaptiveMaxPool{1,2,3}d` module into the given + /// `stream`. + void pretty_print(std::ostream& stream) const override { + stream << "torch::nn::AdaptiveMaxPool" << D << 'd' + << "(output_size=" << options.output_size() << ')'; + } + + /// The options with which this `Module` was constructed. + AdaptiveMaxPoolOptions options; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~ AdaptiveMaxPool1d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies adaptive maxpool over a 1-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.AdaptiveMaxPool1d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::AdaptiveMaxPool1dOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// AdaptiveMaxPool1d model(AdaptiveMaxPool1dOptions(3)); +/// ``` +class TORCH_API AdaptiveMaxPool1dImpl + : public AdaptiveMaxPoolImpl<1, ExpandingArray<1>, AdaptiveMaxPool1dImpl> { + public: + using AdaptiveMaxPoolImpl<1, ExpandingArray<1>, AdaptiveMaxPool1dImpl>:: + AdaptiveMaxPoolImpl; + + Tensor forward(const Tensor& input); + + /// Returns the indices along with the outputs. + /// Useful to pass to nn.MaxUnpool1d. + std::tuple forward_with_indices(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `AdaptiveMaxPool1dImpl`. +/// See the documentation for `AdaptiveMaxPool1dImpl` class to learn what +/// methods it provides, and examples of how to use `AdaptiveMaxPool1d` with +/// `torch::nn::AdaptiveMaxPool1dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(AdaptiveMaxPool1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ AdaptiveMaxPool2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies adaptive maxpool over a 2-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.AdaptiveMaxPool2d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::AdaptiveMaxPool2dOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// AdaptiveMaxPool2d model(AdaptiveMaxPool2dOptions({3, 2})); +/// ``` +class TORCH_API AdaptiveMaxPool2dImpl : public AdaptiveMaxPoolImpl< + 2, + ExpandingArrayWithOptionalElem<2>, + AdaptiveMaxPool2dImpl> { + public: + using AdaptiveMaxPoolImpl< + 2, + ExpandingArrayWithOptionalElem<2>, + AdaptiveMaxPool2dImpl>::AdaptiveMaxPoolImpl; + + Tensor forward(const Tensor& input); + + /// Returns the indices along with the outputs. + /// Useful to pass to nn.MaxUnpool2d. + std::tuple forward_with_indices(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `AdaptiveMaxPool2dImpl`. +/// See the documentation for `AdaptiveMaxPool2dImpl` class to learn what +/// methods it provides, and examples of how to use `AdaptiveMaxPool2d` with +/// `torch::nn::AdaptiveMaxPool2dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(AdaptiveMaxPool2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ AdaptiveMaxPool3d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies adaptive maxpool over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.AdaptiveMaxPool3d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::AdaptiveMaxPool3dOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// AdaptiveMaxPool3d model(AdaptiveMaxPool3dOptions(3)); +/// ``` +class TORCH_API AdaptiveMaxPool3dImpl : public AdaptiveMaxPoolImpl< + 3, + ExpandingArrayWithOptionalElem<3>, + AdaptiveMaxPool3dImpl> { + public: + using AdaptiveMaxPoolImpl< + 3, + ExpandingArrayWithOptionalElem<3>, + AdaptiveMaxPool3dImpl>::AdaptiveMaxPoolImpl; + + Tensor forward(const Tensor& input); + + /// Returns the indices along with the outputs. + /// Useful to pass to nn.MaxUnpool3d. + std::tuple forward_with_indices(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `AdaptiveMaxPool3dImpl`. +/// See the documentation for `AdaptiveMaxPool3dImpl` class to learn what +/// methods it provides, and examples of how to use `AdaptiveMaxPool3d` with +/// `torch::nn::AdaptiveMaxPool3dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(AdaptiveMaxPool3d); + +// ============================================================================ + +/// Base class for all (dimension-specialized) adaptive avgpool modules. +template +class TORCH_API AdaptiveAvgPoolImpl : public torch::nn::Cloneable { + public: + AdaptiveAvgPoolImpl(output_size_t output_size) + : AdaptiveAvgPoolImpl( + AdaptiveAvgPoolOptions(output_size)) {} + explicit AdaptiveAvgPoolImpl( + const AdaptiveAvgPoolOptions& options_) + : options(options_) {} + + void reset() override {} + + /// Pretty prints the `AdaptiveAvgPool{1,2,3}d` module into the given + /// `stream`. + void pretty_print(std::ostream& stream) const override { + stream << "torch::nn::AdaptiveAvgPool" << D << 'd' + << "(output_size=" << options.output_size() << ')'; + } + + /// The options with which this `Module` was constructed. + AdaptiveAvgPoolOptions options; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~ AdaptiveAvgPool1d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies adaptive avgpool over a 1-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.AdaptiveAvgPool1d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::AdaptiveAvgPool1dOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// AdaptiveAvgPool1d model(AdaptiveAvgPool1dOptions(5)); +/// ``` +class TORCH_API AdaptiveAvgPool1dImpl + : public AdaptiveAvgPoolImpl<1, ExpandingArray<1>, AdaptiveAvgPool1dImpl> { + public: + using AdaptiveAvgPoolImpl<1, ExpandingArray<1>, AdaptiveAvgPool1dImpl>:: + AdaptiveAvgPoolImpl; + + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `AdaptiveAvgPool1dImpl`. +/// See the documentation for `AdaptiveAvgPool1dImpl` class to learn what +/// methods it provides, and examples of how to use `AdaptiveAvgPool1d` with +/// `torch::nn::AdaptiveAvgPool1dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(AdaptiveAvgPool1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~ AdaptiveAvgPool2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies adaptive avgpool over a 2-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.AdaptiveAvgPool2d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::AdaptiveAvgPool2dOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// AdaptiveAvgPool2d model(AdaptiveAvgPool2dOptions({3, 2})); +/// ``` +class TORCH_API AdaptiveAvgPool2dImpl : public AdaptiveAvgPoolImpl< + 2, + ExpandingArrayWithOptionalElem<2>, + AdaptiveAvgPool2dImpl> { + public: + using AdaptiveAvgPoolImpl< + 2, + ExpandingArrayWithOptionalElem<2>, + AdaptiveAvgPool2dImpl>::AdaptiveAvgPoolImpl; + + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `AdaptiveAvgPool2dImpl`. +/// See the documentation for `AdaptiveAvgPool2dImpl` class to learn what +/// methods it provides, and examples of how to use `AdaptiveAvgPool2d` with +/// `torch::nn::AdaptiveAvgPool2dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(AdaptiveAvgPool2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~ AdaptiveAvgPool3d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies adaptive avgpool over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.AdaptiveAvgPool3d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::AdaptiveAvgPool3dOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// AdaptiveAvgPool3d model(AdaptiveAvgPool3dOptions(3)); +/// ``` +class TORCH_API AdaptiveAvgPool3dImpl : public AdaptiveAvgPoolImpl< + 3, + ExpandingArrayWithOptionalElem<3>, + AdaptiveAvgPool3dImpl> { + public: + using AdaptiveAvgPoolImpl< + 3, + ExpandingArrayWithOptionalElem<3>, + AdaptiveAvgPool3dImpl>::AdaptiveAvgPoolImpl; + + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `AdaptiveAvgPool3dImpl`. +/// See the documentation for `AdaptiveAvgPool3dImpl` class to learn what +/// methods it provides, and examples of how to use `AdaptiveAvgPool3d` with +/// `torch::nn::AdaptiveAvgPool3dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(AdaptiveAvgPool3d); + +// ============================================================================ + +/// Base class for all (dimension-specialized) maxunpool modules. +template +class TORCH_API MaxUnpoolImpl : public torch::nn::Cloneable { + public: + MaxUnpoolImpl(ExpandingArray kernel_size) + : MaxUnpoolImpl(MaxUnpoolOptions(kernel_size)) {} + explicit MaxUnpoolImpl(const MaxUnpoolOptions& options_); + + void reset() override; + + /// Pretty prints the `MaxUnpool{1,2,3}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// The options with which this `Module` was constructed. + MaxUnpoolOptions options; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MaxUnpool1d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies maxunpool over a 1-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.MaxUnpool1d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MaxUnpool1dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MaxUnpool1d model(MaxUnpool1dOptions(3).stride(2).padding(1)); +/// ``` +class TORCH_API MaxUnpool1dImpl : public MaxUnpoolImpl<1, MaxUnpool1dImpl> { + public: + using MaxUnpoolImpl<1, MaxUnpool1dImpl>::MaxUnpoolImpl; + Tensor forward( + const Tensor& input, + const Tensor& indices, + const std::optional>& output_size = std::nullopt); + + protected: + FORWARD_HAS_DEFAULT_ARGS({2, AnyValue(std::optional>())}) +}; + +/// A `ModuleHolder` subclass for `MaxUnpool1dImpl`. +/// See the documentation for `MaxUnpool1dImpl` class to learn what methods it +/// provides, and examples of how to use `MaxUnpool1d` with +/// `torch::nn::MaxUnpool1dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(MaxUnpool1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MaxUnpool2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies maxunpool over a 2-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.MaxUnpool2d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MaxUnpool2dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MaxUnpool2d model(MaxUnpool2dOptions(3).stride(2).padding(1)); +/// ``` +class TORCH_API MaxUnpool2dImpl : public MaxUnpoolImpl<2, MaxUnpool2dImpl> { + public: + using MaxUnpoolImpl<2, MaxUnpool2dImpl>::MaxUnpoolImpl; + Tensor forward( + const Tensor& input, + const Tensor& indices, + const std::optional>& output_size = std::nullopt); + + protected: + FORWARD_HAS_DEFAULT_ARGS({2, AnyValue(std::optional>())}) +}; + +/// A `ModuleHolder` subclass for `MaxUnpool2dImpl`. +/// See the documentation for `MaxUnpool2dImpl` class to learn what methods it +/// provides, and examples of how to use `MaxUnpool2d` with +/// `torch::nn::MaxUnpool2dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(MaxUnpool2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MaxUnpool3d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies maxunpool over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.MaxUnpool3d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::MaxUnpool3dOptions` class to learn +/// what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// MaxUnpool3d model(MaxUnpool3dOptions(3).stride(2).padding(1)); +/// ``` +class TORCH_API MaxUnpool3dImpl : public MaxUnpoolImpl<3, MaxUnpool3dImpl> { + public: + using MaxUnpoolImpl<3, MaxUnpool3dImpl>::MaxUnpoolImpl; + Tensor forward( + const Tensor& input, + const Tensor& indices, + const std::optional>& output_size = std::nullopt); + + protected: + FORWARD_HAS_DEFAULT_ARGS({2, AnyValue(std::optional>())}) +}; + +/// A `ModuleHolder` subclass for `MaxUnpool3dImpl`. +/// See the documentation for `MaxUnpool3dImpl` class to learn what methods it +/// provides, and examples of how to use `MaxUnpool3d` with +/// `torch::nn::MaxUnpool3dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(MaxUnpool3d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FractionalMaxPool2d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies fractional maxpool over a 2-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.FractionalMaxPool2d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::FractionalMaxPool2dOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// FractionalMaxPool2d model(FractionalMaxPool2dOptions(5).output_size(1)); +/// ``` +class TORCH_API FractionalMaxPool2dImpl + : public torch::nn::Cloneable { + public: + FractionalMaxPool2dImpl(ExpandingArray<2> kernel_size) + : FractionalMaxPool2dImpl(FractionalMaxPool2dOptions(kernel_size)) {} + explicit FractionalMaxPool2dImpl(FractionalMaxPool2dOptions options_); + + void reset() override; + + /// Pretty prints the `FractionalMaxPool2d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input); + + /// Returns the outputs and the indices of the max values. + /// Useful for `torch::nn::MaxUnpool2d` later. + std::tuple forward_with_indices(const Tensor& input); + + /// The options with which this `Module` was constructed. + FractionalMaxPool2dOptions options; + + Tensor _random_samples; +}; + +/// A `ModuleHolder` subclass for `FractionalMaxPool2dImpl`. +/// See the documentation for `FractionalMaxPool2dImpl` class to learn what +/// methods it provides, and examples of how to use `FractionalMaxPool2d` with +/// `torch::nn::FractionalMaxPool2dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(FractionalMaxPool2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FractionalMaxPool3d +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies fractional maxpool over a 3-D input. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.FractionalMaxPool3d to +/// learn about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::FractionalMaxPool3dOptions` class to +/// learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// FractionalMaxPool3d model(FractionalMaxPool3dOptions(5).output_size(1)); +/// ``` +class TORCH_API FractionalMaxPool3dImpl + : public torch::nn::Cloneable { + public: + FractionalMaxPool3dImpl(ExpandingArray<3> kernel_size) + : FractionalMaxPool3dImpl(FractionalMaxPool3dOptions(kernel_size)) {} + explicit FractionalMaxPool3dImpl(FractionalMaxPool3dOptions options_); + + void reset() override; + + /// Pretty prints the `FractionalMaxPool3d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input); + + /// Returns the outputs and the indices of the max values. + /// Useful for `torch::nn::MaxUnpool3d` later. + std::tuple forward_with_indices(const Tensor& input); + + /// The options with which this `Module` was constructed. + FractionalMaxPool3dOptions options; + + Tensor _random_samples; +}; + +/// A `ModuleHolder` subclass for `FractionalMaxPool3dImpl`. +/// See the documentation for `FractionalMaxPool3dImpl` class to learn what +/// methods it provides, and examples of how to use `FractionalMaxPool3d` with +/// `torch::nn::FractionalMaxPool3dOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(FractionalMaxPool3d); + +// ============================================================================ + +/// Base class for all (dimension-specialized) lppool modules. +template +class TORCH_API LPPoolImpl : public torch::nn::Cloneable { + public: + LPPoolImpl(double norm_type, ExpandingArray kernel_size) + : LPPoolImpl(LPPoolOptions(norm_type, kernel_size)) {} + explicit LPPoolImpl(const LPPoolOptions& options_); + + void reset() override; + + /// Pretty prints the `LPPool{1,2}d` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + LPPoolOptions options; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LPPool1d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the LPPool1d function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.LPPool1d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::LPPool1dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// LPPool1d model(LPPool1dOptions(1, 2).stride(5).ceil_mode(true)); +/// ``` +class TORCH_API LPPool1dImpl : public LPPoolImpl<1, LPPool1dImpl> { + public: + using LPPoolImpl<1, LPPool1dImpl>::LPPoolImpl; + + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `LPPool1dImpl`. +/// See the documentation for `LPPool1dImpl` class to learn what methods it +/// provides, and examples of how to use `LPPool1d` with +/// `torch::nn::LPPool1dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(LPPool1d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LPPool2d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the LPPool2d function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.LPPool2d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::LPPool2dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// LPPool2d model(LPPool2dOptions(1, std::vector({3, 4})).stride({5, +/// 6}).ceil_mode(true)); +/// ``` +class TORCH_API LPPool2dImpl : public LPPoolImpl<2, LPPool2dImpl> { + public: + using LPPoolImpl<2, LPPool2dImpl>::LPPoolImpl; + + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `LPPool2dImpl`. +/// See the documentation for `LPPool2dImpl` class to learn what methods it +/// provides, and examples of how to use `LPPool2d` with +/// `torch::nn::LPPool2dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(LPPool2d); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LPPool3d ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Applies the LPPool3d function element-wise. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.LPPool3d to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::LPPool3dOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// LPPool3d model(LPPool3dOptions(1, std::vector({3, 4, 5})).stride( +/// {5, 6, 7}).ceil_mode(true)); +/// ``` +class TORCH_API LPPool3dImpl : public LPPoolImpl<3, LPPool3dImpl> { + public: + using LPPoolImpl<3, LPPool3dImpl>::LPPoolImpl; + + Tensor forward(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `LPPool3dImpl`. +/// See the documentation for `LPPool3dImpl` class to learn what methods it +/// provides, and examples of how to use `LPPool3d` with +/// `torch::nn::LPPool3dOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(LPPool3d); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/rnn.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/rnn.h new file mode 100644 index 0000000000000000000000000000000000000000..31779b329082fed38dcbcfde10eecd2cf8792084 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/rnn.h @@ -0,0 +1,399 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace torch::nn { + +namespace detail { +/// Base class for all RNN implementations (intended for code sharing). +template +class TORCH_API RNNImplBase : public torch::nn::Cloneable { + public: + explicit RNNImplBase(const RNNOptionsBase& options_); + + /// Initializes the parameters of the RNN module. + void reset() override; + + void reset_parameters(); + + /// Overrides `nn::Module::to()` to call `flatten_parameters()` after the + /// original operation. + void to(torch::Device device, torch::Dtype dtype, bool non_blocking = false) + override; + void to(torch::Dtype dtype, bool non_blocking = false) override; + void to(torch::Device device, bool non_blocking = false) override; + + /// Pretty prints the RNN module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + /// Modifies the internal storage of weights for optimization purposes. + /// + /// On CPU, this method should be called if any of the weight or bias vectors + /// are changed (i.e. weights are added or removed). On GPU, it should be + /// called __any time the storage of any parameter is modified__, e.g. any + /// time a parameter is assigned a new value. This allows using the fast path + /// in cuDNN implementations of respective RNN `forward()` methods. It is + /// called once upon construction, inside `reset()`. + void flatten_parameters(); + + std::vector all_weights() const; + + /// The RNN's options. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + RNNOptionsBase options_base; + + protected: + // Resets flat_weights_ + // Note: be v. careful before removing this, as 3rd party device types + // likely rely on this behavior to properly .to() modules like LSTM. + void reset_flat_weights(); + + void check_input(const Tensor& input, const Tensor& batch_sizes) const; + + std::tuple get_expected_hidden_size( + const Tensor& input, + const Tensor& batch_sizes) const; + + void check_hidden_size( + const Tensor& hx, + std::tuple expected_hidden_size, + std::string msg = "Expected hidden size {1}, got {2}") const; + + void check_forward_args(Tensor input, Tensor hidden, Tensor batch_sizes) + const; + + Tensor permute_hidden(Tensor hx, const Tensor& permutation) const; + + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::vector flat_weights_names_; + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::vector> all_weights_; + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::vector flat_weights_; +}; +} // namespace detail + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RNN ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// A multi-layer Elman RNN module with Tanh or ReLU activation. +/// See https://pytorch.org/docs/main/generated/torch.nn.RNN.html to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::RNNOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// RNN model(RNNOptions(128, +/// 64).num_layers(3).dropout(0.2).nonlinearity(torch::kTanh)); +/// ``` +class TORCH_API RNNImpl : public detail::RNNImplBase { + public: + RNNImpl(int64_t input_size, int64_t hidden_size) + : RNNImpl(RNNOptions(input_size, hidden_size)) {} + explicit RNNImpl(const RNNOptions& options_); + + std::tuple forward(const Tensor& input, Tensor hx = {}); + + protected: + FORWARD_HAS_DEFAULT_ARGS({1, AnyValue(Tensor())}) + + public: + std::tuple + forward_with_packed_input( + const torch::nn::utils::rnn::PackedSequence& packed_input, + Tensor hx = {}); + + RNNOptions options; + + protected: + std::tuple forward_helper( + const Tensor& input, + const Tensor& batch_sizes, + const Tensor& sorted_indices, + int64_t max_batch_size, + Tensor hx); +}; + +/// A `ModuleHolder` subclass for `RNNImpl`. +/// See the documentation for `RNNImpl` class to learn what methods it +/// provides, and examples of how to use `RNN` with `torch::nn::RNNOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(RNN); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LSTM ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// A multi-layer long-short-term-memory (LSTM) module. +/// See https://pytorch.org/docs/main/generated/torch.nn.LSTM.html to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::LSTMOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// LSTM model(LSTMOptions(2, +/// 4).num_layers(3).batch_first(false).bidirectional(true)); +/// ``` +class TORCH_API LSTMImpl : public detail::RNNImplBase { + public: + LSTMImpl(int64_t input_size, int64_t hidden_size) + : LSTMImpl(LSTMOptions(input_size, hidden_size)) {} + explicit LSTMImpl(const LSTMOptions& options_); + + std::tuple> forward( + const Tensor& input, + std::optional> hx_opt = {}); + + protected: + FORWARD_HAS_DEFAULT_ARGS( + {1, AnyValue(std::optional>())}) + + public: + std::tuple> + forward_with_packed_input( + const torch::nn::utils::rnn::PackedSequence& packed_input, + std::optional> hx_opt = {}); + + LSTMOptions options; + + protected: + void check_forward_args( + const Tensor& input, + std::tuple hidden, + const Tensor& batch_sizes) const; + + std::tuple get_expected_cell_size( + const Tensor& input, + const Tensor& batch_sizes) const; + + std::tuple permute_hidden( + std::tuple hx, + const Tensor& permutation) const; + + std::tuple> forward_helper( + const Tensor& input, + const Tensor& batch_sizes, + const Tensor& sorted_indices, + int64_t max_batch_size, + std::optional> hx_opt); +}; + +/// A `ModuleHolder` subclass for `LSTMImpl`. +/// See the documentation for `LSTMImpl` class to learn what methods it +/// provides, and examples of how to use `LSTM` with `torch::nn::LSTMOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(LSTM); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GRU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// A multi-layer gated recurrent unit (GRU) module. +/// See https://pytorch.org/docs/main/generated/torch.nn.GRU.html to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::GRUOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// GRU model(GRUOptions(2, +/// 4).num_layers(3).batch_first(false).bidirectional(true)); +/// ``` +class TORCH_API GRUImpl : public detail::RNNImplBase { + public: + GRUImpl(int64_t input_size, int64_t hidden_size) + : GRUImpl(GRUOptions(input_size, hidden_size)) {} + explicit GRUImpl(const GRUOptions& options_); + + std::tuple forward(const Tensor& input, Tensor hx = {}); + + protected: + FORWARD_HAS_DEFAULT_ARGS({1, AnyValue(torch::Tensor())}) + + public: + std::tuple + forward_with_packed_input( + const torch::nn::utils::rnn::PackedSequence& packed_input, + Tensor hx = {}); + + GRUOptions options; + + protected: + std::tuple forward_helper( + const Tensor& input, + const Tensor& batch_sizes, + const Tensor& sorted_indices, + int64_t max_batch_size, + Tensor hx); +}; + +/// A `ModuleHolder` subclass for `GRUImpl`. +/// See the documentation for `GRUImpl` class to learn what methods it +/// provides, and examples of how to use `GRU` with `torch::nn::GRUOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(GRU); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RNNCellImplBase +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +namespace detail { +/// Base class for all RNNCell implementations (intended for code sharing). +template +class TORCH_API RNNCellImplBase : public torch::nn::Cloneable { + public: + explicit RNNCellImplBase(const RNNCellOptionsBase& options_); + + /// Initializes the parameters of the RNNCell module. + void reset() override; + + void reset_parameters(); + + /// Pretty prints the RNN module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + RNNCellOptionsBase options_base; + + Tensor weight_ih; + Tensor weight_hh; + Tensor bias_ih; + Tensor bias_hh; + + protected: + void check_forward_input(const Tensor& input, const std::string& name) const; + virtual std::string get_nonlinearity_str() const; +}; +} // namespace detail + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RNNCell +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// An Elman RNN cell with tanh or ReLU non-linearity. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.RNNCell to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::RNNCellOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// RNNCell model(RNNCellOptions(20, +/// 10).bias(false).nonlinearity(torch::kReLU)); +/// ``` +class TORCH_API RNNCellImpl : public detail::RNNCellImplBase { + public: + RNNCellImpl(int64_t input_size, int64_t hidden_size) + : RNNCellImpl(RNNCellOptions(input_size, hidden_size)) {} + explicit RNNCellImpl(const RNNCellOptions& options_); + + Tensor forward(const Tensor& input, const Tensor& hx = {}); + + protected: + FORWARD_HAS_DEFAULT_ARGS({1, AnyValue(Tensor())}) + + public: + RNNCellOptions options; + + protected: + std::string get_nonlinearity_str() const override; +}; + +/// A `ModuleHolder` subclass for `RNNCellImpl`. +/// See the documentation for `RNNCellImpl` class to learn what methods it +/// provides, and examples of how to use `RNNCell` with +/// `torch::nn::RNNCellOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(RNNCell); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LSTMCell +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// A long short-term memory (LSTM) cell. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.LSTMCell to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::LSTMCellOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// LSTMCell model(LSTMCellOptions(20, 10).bias(false)); +/// ``` +class TORCH_API LSTMCellImpl : public detail::RNNCellImplBase { + public: + LSTMCellImpl(int64_t input_size, int64_t hidden_size) + : LSTMCellImpl(LSTMCellOptions(input_size, hidden_size)) {} + explicit LSTMCellImpl(const LSTMCellOptions& options_); + + std::tuple forward( + const Tensor& input, + std::optional> hx_opt = {}); + + protected: + FORWARD_HAS_DEFAULT_ARGS( + {1, AnyValue(std::optional>())}) + + public: + LSTMCellOptions options; +}; + +/// A `ModuleHolder` subclass for `LSTMCellImpl`. +/// See the documentation for `LSTMCellImpl` class to learn what methods it +/// provides, and examples of how to use `LSTMCell` with +/// `torch::nn::LSTMCellOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(LSTMCell); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GRUCell +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// A gated recurrent unit (GRU) cell. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.GRUCell to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::GRUCellOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// GRUCell model(GRUCellOptions(20, 10).bias(false)); +/// ``` +class TORCH_API GRUCellImpl : public detail::RNNCellImplBase { + public: + GRUCellImpl(int64_t input_size, int64_t hidden_size) + : GRUCellImpl(GRUCellOptions(input_size, hidden_size)) {} + explicit GRUCellImpl(const GRUCellOptions& options_); + + Tensor forward(const Tensor& input, const Tensor& hx = {}); + + protected: + FORWARD_HAS_DEFAULT_ARGS({1, AnyValue(Tensor())}) + + public: + GRUCellOptions options; +}; + +/// A `ModuleHolder` subclass for `GRUCellImpl`. +/// See the documentation for `GRUCellImpl` class to learn what methods it +/// provides, and examples of how to use `GRUCell` with +/// `torch::nn::GRUCellOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(GRUCell); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/transformer.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/transformer.h new file mode 100644 index 0000000000000000000000000000000000000000..d8f031051aff11caff1a072bfc386ab630f7271f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/transformer.h @@ -0,0 +1,141 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include + +namespace torch::nn { + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Transformer ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// A transformer model. User is able to modify the attributes as needed. The +/// architecture is based on the paper "Attention Is All You Need". Ashish +/// Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N +/// Gomez, Lukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. +/// In Advances in Neural Information Processing Systems, pages 6000-6010. +/// +/// See https://pytorch.org/docs/stable/generated/torch.nn.Transformer.html to +/// learn about the exact behavior of this transformer model +/// +/// See the documentation for `torch::nn::Transformer` class to learn what +/// constructor arguments are supported for this encoder layer model +/// +/// Example: +/// ``` +/// Transformer trans(TransformerOptions(512, 8)); +/// ``` +class TORCH_API TransformerImpl : public Cloneable { + public: + explicit TransformerImpl(TransformerOptions options_); + + /// forward function for Transformer Module + /// Args: + /// src: the sequence to the encoder (required). + /// tgt: the sequence to the decoder (required). + /// src_mask: the additive mask for the src sequence (optional). + /// tgt_mask: the additive mask for the tgt sequence (optional). + /// memory_mask: the additive mask for the encoder output (optional). + /// src_key_padding_mask: the ByteTensor mask for src keys per batch + /// (optional). tgt_key_padding_mask: the ByteTensor mask for tgt keys per + /// batch (optional). memory_key_padding_mask: the ByteTensor mask for + /// memory keys per batch (optional). + /// + /// Shape: + /// src: `(S, N, E)` + /// tgt: `(T, N, E)` + /// src_mask: `(S, S)` + /// tgt_mask: `(T, T)` + /// memory_mask: `(T, S)` + /// src_key_padding_mask: `(N, S)` + /// tgt_key_padding_mask: `(N, T)` + /// memory_key_padding_mask: `(N, S)` + /// + /// Note: + /// [src/tgt/memory]_mask ensures that position i is allowed to attend the + /// unmasked positions. If a ByteTensor is provided, the non-zero + /// positions are not allowed to attend while the zero positions will be + /// unchanged. If a BoolTensor is provided, positions with `True` are not + /// allowed to attend while `False` values will be unchanged. If a + /// FloatTensor is provided, it will be added to the attention weight. + /// + /// [src/tgt/memory]_key_padding_mask provides specified elements in the + /// key to be ignored by the attention. If a ByteTensor is provided, the + /// non-zero positions will be ignored while the zero positions will be + /// unchanged. If a BoolTensor is provided, the positions with the value + /// of `True` will be ignored while the position with the value of `False` + /// will be unchanged. + /// + /// output: `(T, N, E)` + /// + /// Note: + /// Due to the multi-head attention architecture in the transformer model, + /// the output sequence length of a transformer is same as the input + /// sequence (i.e. target) length of the decode. + /// + /// where + /// S is the source sequence length, + /// T is the target sequence length, + /// N is the batch size, + /// E is the feature number. + Tensor forward( + const Tensor& src, + const Tensor& tgt, + const Tensor& src_mask = {}, + const Tensor& tgt_mask = {}, + const Tensor& memory_mask = {}, + const Tensor& src_key_padding_mask = {}, + const Tensor& tgt_key_padding_mask = {}, + const Tensor& memory_key_padding_mask = {}); + + void reset() override; + + void reset_parameters(); + + /// Generate a square mask for the sequence. + /// The masked positions are filled with `-inf` in float type. + /// Unmasked positions are filled with `0.0` in float type. + /// Note: + /// 1. This function will always return a CPU tensor. + /// 2. This function requires the platform support IEEE754, since `-inf` is + /// guaranteed to + /// be valid only when IEEE754 is supported. If the platform doesn't + /// support IEEE754, this function will fill the mask with the smallest + /// float number instead of `-inf`, a one time warning will pop up as + /// well. + static Tensor generate_square_subsequent_mask(int64_t sz); + + protected: + FORWARD_HAS_DEFAULT_ARGS( + {2, AnyValue(Tensor())}, + {3, AnyValue(Tensor())}, + {4, AnyValue(Tensor())}, + {5, AnyValue(Tensor())}, + {6, AnyValue(Tensor())}, + {7, AnyValue(Tensor())}) + + public: + /// options with which this `Transformer` was constructed + TransformerOptions options; + + /// encoder module + AnyModule encoder; + + /// decoder module + AnyModule decoder; +}; + +/// A `ModuleHolder` subclass for `TransformerImpl`. +/// See the documentation for `TransformerImpl` class to learn what +/// methods it provides, and examples of how to use `Transformer` with +/// `torch::nn::TransformerOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(Transformer); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/transformercoder.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/transformercoder.h new file mode 100644 index 0000000000000000000000000000000000000000..44a7e8e75d1f3a87eac3bd76dc7b0b0a7e0f1892 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/transformercoder.h @@ -0,0 +1,152 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace torch::nn { + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ TransformerEncoder +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// TransformerEncoder module. +/// See +/// https://pytorch.org/docs/main/generated/torch.nn.TransformerEncoder.html +/// to learn about the exact behavior of this encoder layer module. +/// +/// See the documentation for `torch::nn::TransformerEncoder` class to learn +/// what constructor arguments are supported for this encoder module. +/// +/// Example: +/// ``` +/// TransformerEncoderLayer encoderLayer(TransformerEncoderLayerOptions(512, +/// 8).dropout(0.1)); TransformerEncoder +/// encoder(TransformerEncoderOptions(encoderLayer, +/// 6).norm(LayerNorm(LayerNormOptions({2})))); +/// ``` +class TORCH_API TransformerEncoderImpl + : public Cloneable { + public: + TransformerEncoderImpl( + TransformerEncoderLayer encoder_layer, + int64_t num_layers) + : TransformerEncoderImpl( + TransformerEncoderOptions(std::move(encoder_layer), num_layers)) {} + explicit TransformerEncoderImpl(TransformerEncoderOptions options_); + + Tensor forward( + const Tensor& src, + const Tensor& src_mask = {}, + const Tensor& src_key_padding_mask = {}); + + void reset() override; + + void reset_parameters(); + + protected: + FORWARD_HAS_DEFAULT_ARGS({1, AnyValue(Tensor())}, {2, AnyValue(Tensor())}) + + public: + /// options with which this `TransformerEncoder` was constructed + TransformerEncoderOptions options; + + /// module list that contains all the encoder layers + ModuleList layers = nullptr; + + /// optional normalization module + AnyModule norm; +}; + +/// A `ModuleHolder` subclass for `TransformerEncoderImpl`. +/// See the documentation for `TransformerEncoderImpl` class to learn what +/// methods it provides, and examples of how to use `TransformerEncoder` with +/// `torch::nn::TransformerEncoderOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(TransformerEncoder); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ TransformerDecoder +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// TransformerDecoder is a stack of N decoder layers. +/// See +/// https://pytorch.org/docs/main/generated/torch.nn.TransformerDecoder.html +/// to learn about the exact behavior of this decoder module +/// +/// See the documentation for `torch::nn::TransformerDecoderOptions` class to +/// learn what constructor arguments are supported for this decoder module +/// +/// Example: +/// ``` +/// TransformerDecoderLayer decoder_layer(TransformerDecoderLayerOptions(512, +/// 8).dropout(0.1)); TransformerDecoder +/// transformer_decoder(TransformerDecoderOptions(decoder_layer, +/// 6).norm(LayerNorm(LayerNormOptions({2})))); const auto memory = +/// torch::rand({10, 32, 512}); const auto tgt = torch::rand({20, 32, 512}); +/// auto out = transformer_decoder(tgt, memory); +/// ``` +class TORCH_API TransformerDecoderImpl + : public Cloneable { + public: + TransformerDecoderImpl( + TransformerDecoderLayer decoder_layer, + int64_t num_layers) + : TransformerDecoderImpl( + TransformerDecoderOptions(std::move(decoder_layer), num_layers)) {} + explicit TransformerDecoderImpl(TransformerDecoderOptions options_); + + void reset() override; + + void reset_parameters(); + + /// Pass the inputs (and mask) through the decoder layer in turn. + /// Args: + /// tgt: the sequence to the decoder layer (required). + /// memory: the sequence from the last layer of the encoder (required). + /// tgt_mask: the mask for the tgt sequence (optional). + /// memory_mask: the mask for the memory sequence (optional). + /// tgt_key_padding_mask: the mask for the tgt keys per batch + /// (optional). memory_key_padding_mask: the mask for the memory keys + /// per batch (optional). + Tensor forward( + const Tensor& tgt, + const Tensor& memory, + const Tensor& tgt_mask = {}, + const Tensor& memory_mask = {}, + const Tensor& tgt_key_padding_mask = {}, + const Tensor& memory_key_padding_mask = {}); + + /// The options used to configure this module. + TransformerDecoderOptions options; + + /// Cloned layers of decoder layers + ModuleList layers{nullptr}; + + /// optional layer normalization module + AnyModule norm; + + protected: + FORWARD_HAS_DEFAULT_ARGS( + {2, AnyValue(Tensor())}, + {3, AnyValue(Tensor())}, + {4, AnyValue(Tensor())}, + {5, AnyValue(Tensor())}) +}; + +/// A `ModuleHolder` subclass for `TransformerDecoderImpl`. +/// See the documentation for `TransformerDecoderImpl` class to learn what +/// methods it provides, and examples of how to use `TransformerDecoder` with +/// `torch::nn::TransformerDecoderOptions`. +/// See the documentation for `ModuleHolder` to learn about PyTorch's +/// module storage semantics. +TORCH_MODULE(TransformerDecoder); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/transformerlayer.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/transformerlayer.h new file mode 100644 index 0000000000000000000000000000000000000000..22abdccf6e0d939e26c902df29df0043f377be25 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/transformerlayer.h @@ -0,0 +1,193 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace torch::nn { + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ TransformerEncoderLayer +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// TransformerEncoderLayer module. +/// See +/// https://pytorch.org/docs/main/generated/torch.nn.TransformerEncoderLayer.html +/// to learn about the exact behavior of this encoder layer model +/// +/// See the documentation for `torch::nn::TransformerEncoderLayer` class to +/// learn what constructor arguments are supported for this encoder layer model +/// +/// Example: +/// ``` +/// TransformerEncoderLayer encoderLayer(TransformerEncoderLayerOptions(512, +/// 8).dropout(0.1)); +/// ``` +class TORCH_API TransformerEncoderLayerImpl + : public Cloneable { + public: + TransformerEncoderLayerImpl(int64_t d_model, int64_t nhead) + : TransformerEncoderLayerImpl( + TransformerEncoderLayerOptions(d_model, nhead)) {} + explicit TransformerEncoderLayerImpl(TransformerEncoderLayerOptions options_); + + Tensor forward( + const Tensor& src, + const Tensor& src_mask = {}, + const Tensor& src_key_padding_mask = {}); + + void reset() override; + + void reset_parameters(); + + protected: + FORWARD_HAS_DEFAULT_ARGS({1, AnyValue(Tensor())}, {2, AnyValue(Tensor())}) + + public: + /// options with which this `TransformerEncoderLayer` was constructed + TransformerEncoderLayerOptions options; + + /// self attention + MultiheadAttention self_attn = nullptr; + + /// feedforward first linear layer + Linear linear1 = nullptr; + + /// feedforward dropout layer + Dropout dropout = nullptr; + + /// feedforward second linear layer + Linear linear2 = nullptr; + + /// pre feedforward, normalization layer + LayerNorm norm1 = nullptr; + /// post feedforward, normalization layer + LayerNorm norm2 = nullptr; + + /// pre feedforward, dropout layer + Dropout dropout1 = nullptr; + /// post feedforward, dropout layer + Dropout dropout2 = nullptr; +}; + +/// A `ModuleHolder` subclass for `TransformerEncoderLayerImpl``. +/// See the documentation for `TransformerEncoderLayerImpl` class to learn what +/// methods it provides, and examples of how to use `TransformerEncoderLayer` +/// with `torch::nn::TransformerEncoderLayerOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(TransformerEncoderLayer); + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ TransformerDecoderLayer +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// TransformerDecoderLayer is made up of self-attn, multi-head-attn and +/// feedforward network. This standard decoder layer is based on the paper +/// "Attention Is All You Need". Ashish Vaswani, Noam Shazeer, Niki Parmar, +/// Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Lukasz Kaiser, and Illia +/// Polosukhin. 2017. Attention is all you need. In Advances in Neural +/// Information Processing Systems, pages 6000-6010. Users may modify or +/// implement in a different way during application. See +/// https://pytorch.org/docs/main/nn.html#transformer-layers to learn about +/// the exact behavior of this module. +/// +/// See the documentation for `torch::nn::TransformerDecoderLayerOptions` class +/// to learn what constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// TransformerDecoderLayer model(TransformerDecoderLayerOptions(512, +/// 8).dropout(0.2)); +/// ``` +class TORCH_API TransformerDecoderLayerImpl + : public Cloneable { + public: + TransformerDecoderLayerImpl(int64_t d_model, int64_t nhead) + : TransformerDecoderLayerImpl( + TransformerDecoderLayerOptions(d_model, nhead)) {} + explicit TransformerDecoderLayerImpl(TransformerDecoderLayerOptions options_); + + void reset() override; + + void reset_parameters(); + + /// Pass the inputs (and mask) through the decoder layer. + /// Args: + /// tgt: the sequence to the decoder layer (required). + /// memory: the sequence from the last layer of the encoder (required). + /// tgt_mask: the mask for the tgt sequence (optional). + /// memory_mask: the mask for the memory sequence (optional). + /// tgt_key_padding_mask: the mask for the tgt keys per batch + /// (optional). memory_key_padding_mask: the mask for the memory keys + /// per batch (optional). + Tensor forward( + Tensor tgt, + const Tensor& memory, + const Tensor& tgt_mask = {}, + const Tensor& memory_mask = {}, + const Tensor& tgt_key_padding_mask = {}, + const Tensor& memory_key_padding_mask = {}); + + /// The options used to configure this module. + TransformerDecoderLayerOptions options; + + /// self attention + MultiheadAttention self_attn{nullptr}; + + /// Dropout, post self attention + Dropout dropout1{nullptr}; + + /// Normalization, post self attention + LayerNorm norm1{nullptr}; + + /// Multi-headed attention + MultiheadAttention multihead_attn{nullptr}; + + /// Dropout, post multi-headed attention + Dropout dropout2{nullptr}; + + /// Normalization, post multi-headed attention + LayerNorm norm2{nullptr}; + + /// Feed forward first linear layer + Linear linear1{nullptr}; + + /// Feed forward dropout layer + Dropout dropout{nullptr}; + + /// Feed forward second linear layer + Linear linear2{nullptr}; + + /// Dropout, post feed forward + Dropout dropout3{nullptr}; + + /// Normalization, post feed forward + LayerNorm norm3{nullptr}; + + protected: + FORWARD_HAS_DEFAULT_ARGS( + {2, AnyValue(Tensor())}, + {3, AnyValue(Tensor())}, + {4, AnyValue(Tensor())}, + {5, AnyValue(Tensor())}) + + /// Apply activation based on configuration + Tensor activation(const Tensor& input); +}; + +/// A `ModuleHolder` subclass for `TransformerDecoderLayerImpl`. +/// See the documentation for `TransformerDecoderLayerImpl` class to learn what +/// methods it provides, and examples of how to use `TransformerDecoderLayer` +/// with `torch::nn::TransformerDecoderLayerOptions`. See the documentation for +/// `ModuleHolder` to learn about PyTorch's module storage semantics. +TORCH_MODULE(TransformerDecoderLayer); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/upsampling.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/upsampling.h new file mode 100644 index 0000000000000000000000000000000000000000..f6f2c847ecb33e32d489c437b104d664fb690c00 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/upsampling.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace torch::nn { + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Upsample ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +/// Upsamples a given multi-channel 1D (temporal), 2D (spatial) or 3D +/// (volumetric) data. +/// See https://pytorch.org/docs/main/nn.html#torch.nn.Upsample to learn +/// about the exact behavior of this module. +/// +/// See the documentation for `torch::nn::UpsampleOptions` class to learn what +/// constructor arguments are supported for this module. +/// +/// Example: +/// ``` +/// Upsample +/// model(UpsampleOptions().scale_factor({3}).mode(torch::kLinear).align_corners(false)); +/// ``` +class TORCH_API UpsampleImpl : public Cloneable { + public: + explicit UpsampleImpl(UpsampleOptions options_ = {}); + + void reset() override; + + /// Pretty prints the `Upsample` module into the given `stream`. + void pretty_print(std::ostream& stream) const override; + + Tensor forward(const Tensor& input); + + /// The options with which this `Module` was constructed. + UpsampleOptions options; +}; + +/// A `ModuleHolder` subclass for `UpsampleImpl`. +/// See the documentation for `UpsampleImpl` class to learn what methods it +/// provides, and examples of how to use `Upsample` with +/// `torch::nn::UpsampleOptions`. See the documentation for `ModuleHolder` to +/// learn about PyTorch's module storage semantics. +TORCH_MODULE(Upsample); + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/utils.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/utils.h new file mode 100644 index 0000000000000000000000000000000000000000..2b505f7ac8f40a9599e8e39d46e04e32fbe0b5dd --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/modules/utils.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include + +#include + +namespace torch::nn::modules::utils { + +// Reverse the order of `t` and repeat each element for `n` times. +// This can be used to translate padding arg used by Conv and Pooling modules +// to the ones used by `F::pad`. +// +// This mirrors `_reverse_repeat_tuple` in `torch/nn/modules/utils.py`. +inline std::vector _reverse_repeat_vector( + c10::ArrayRef t, + int64_t n) { + TORCH_INTERNAL_ASSERT(n >= 0); + std::vector ret; + ret.reserve(t.size() * n); + for (auto rit = t.rbegin(); rit != t.rend(); ++rit) { + for ([[maybe_unused]] const auto i : c10::irange(n)) { + ret.emplace_back(*rit); + } + } + return ret; +} + +inline std::vector _list_with_default( + c10::ArrayRef> out_size, + c10::IntArrayRef defaults) { + TORCH_CHECK( + defaults.size() > out_size.size(), + "Input dimension should be at least ", + out_size.size() + 1); + std::vector ret; + c10::IntArrayRef defaults_slice = + defaults.slice(defaults.size() - out_size.size(), out_size.size()); + for (const auto i : c10::irange(out_size.size())) { + auto v = out_size.at(i); + auto d = defaults_slice.at(i); + ret.emplace_back(v.has_value() ? v.value() : d); + } + return ret; +} + +} // namespace torch::nn::modules::utils diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options.h new file mode 100644 index 0000000000000000000000000000000000000000..fb6b619bfa0316e6a87432c49a34db8f24e482fe --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/activation.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/activation.h new file mode 100644 index 0000000000000000000000000000000000000000..5c7f70f422229d17c57734b451ef1116e87179a7 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/activation.h @@ -0,0 +1,712 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn { + +/// Options for the `ELU` module. +/// +/// Example: +/// ``` +/// ELU model(ELUOptions().alpha(42.42).inplace(true)); +/// ``` +struct TORCH_API ELUOptions { + /// The `alpha` value for the ELU formulation. Default: 1.0 + TORCH_ARG(double, alpha) = 1.0; + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace) = false; +}; + +namespace functional { +/// Options for `torch::nn::functional::elu`. +/// +/// See the documentation for `torch::nn::ELUOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::elu(x, F::ELUFuncOptions().alpha(0.42).inplace(true)); +/// ``` +using ELUFuncOptions = ELUOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `SELU` module. +/// +/// Example: +/// ``` +/// SELU model(SELUOptions().inplace(true)); +/// ``` +struct TORCH_API SELUOptions { + /* implicit */ SELUOptions(bool inplace = false); + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace); +}; + +namespace functional { +/// Options for `torch::nn::functional::selu`. +/// +/// See the documentation for `torch::nn::SELUOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::selu(input, F::SELUFuncOptions(false)); +/// ``` +using SELUFuncOptions = SELUOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `GLU` module. +/// +/// Example: +/// ``` +/// GLU model(GLUOptions(1)); +/// ``` +struct TORCH_API GLUOptions { + /* implicit */ GLUOptions(int64_t dim = -1); + + /// the dimension on which to split the input. Default: -1 + TORCH_ARG(int64_t, dim); +}; + +namespace functional { +/// Options for `torch::nn::functional::glu`. +/// +/// See the documentation for `torch::nn::GLUOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::glu(input, GLUFuncOptions(1)); +/// ``` +using GLUFuncOptions = GLUOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `GELU` module. +/// +/// Example: +/// ``` +/// GELU model(GELUOptions().approximate("none")); +/// ``` +struct TORCH_API GELUOptions { + /// Specifies the approximation to apply to the output. + TORCH_ARG(std::string, approximate) = "none"; +}; + +namespace functional { +/// Options for `torch::nn::functional::gelu`. +/// +/// See the documentation for `torch::nn::GELUOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::gelu(input, F::GELUFuncOptions().approximate("none")); +/// ``` +using GELUFuncOptions = GELUOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `Hardshrink` module. +/// +/// Example: +/// ``` +/// Hardshrink model(HardshrinkOptions().lambda(42.42)); +/// ``` +struct TORCH_API HardshrinkOptions { + /* implicit */ HardshrinkOptions(double lambda = 0.5); + + /// the `lambda` value for the Hardshrink formulation. Default: 0.5 + TORCH_ARG(double, lambda); +}; + +namespace functional { +/// Options for `torch::nn::functional::hardshrink`. +/// +/// See the documentation for `torch::nn::HardshrinkOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::hardshrink(x, F::HardshrinkFuncOptions().lambda(0.42)); +/// ``` +using HardshrinkFuncOptions = HardshrinkOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `Hardtanh` module. +/// +/// Example: +/// ``` +/// Hardtanh +/// model(HardtanhOptions().min_val(-42.42).max_val(0.42).inplace(true)); +/// ``` +struct TORCH_API HardtanhOptions { + /// minimum value of the linear region range. Default: -1 + TORCH_ARG(double, min_val) = -1.0; + + /// maximum value of the linear region range. Default: 1 + TORCH_ARG(double, max_val) = 1.0; + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace) = false; +}; + +namespace functional { +/// Options for `torch::nn::functional::hardtanh`. +/// +/// See the documentation for `torch::nn::HardtanhOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::hardtanh(x, +/// F::HardtanhFuncOptions().min_val(-1.0).max_val(1.0).inplace(true)); +/// ``` +using HardtanhFuncOptions = HardtanhOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `LeakyReLU` module. +/// +/// Example: +/// ``` +/// LeakyReLU model(LeakyReLUOptions().negative_slope(0.42).inplace(true)); +/// ``` +struct TORCH_API LeakyReLUOptions { + /// Controls the angle of the negative slope. Default: 1e-2 + TORCH_ARG(double, negative_slope) = 1e-2; + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace) = false; +}; + +namespace functional { +/// Options for `torch::nn::functional::leaky_relu`. +/// +/// See the documentation for `torch::nn::LeakyReLUOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::leaky_relu(x, +/// F::LeakyReLUFuncOptions().negative_slope(0.42).inplace(true)); +/// ``` +using LeakyReLUFuncOptions = LeakyReLUOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `Softmax` module. +/// +/// Example: +/// ``` +/// Softmax model(SoftmaxOptions(1)); +/// ``` +struct TORCH_API SoftmaxOptions { + SoftmaxOptions(int64_t dim); + + /// Dimension along which Softmax will be computed. + TORCH_ARG(int64_t, dim); +}; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::softmax`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::softmax(input, F::SoftmaxFuncOptions(1)); +/// ``` +struct TORCH_API SoftmaxFuncOptions { + SoftmaxFuncOptions(int64_t dim); + + /// Dimension along which Softmax will be computed. + TORCH_ARG(int64_t, dim); + + /// the desired data type of returned tensor. + /// If specified, the input tensor is casted to `dtype` before the operation + /// is performed. This is useful for preventing data type overflows. Default: + /// None. + TORCH_ARG(std::optional, dtype) = std::nullopt; +}; + +} // namespace functional + +// ============================================================================ + +/// Options for the `Softmin` module. +/// +/// Example: +/// ``` +/// Softmin model(SoftminOptions(1)); +/// ``` +struct TORCH_API SoftminOptions { + SoftminOptions(int64_t dim); + + /// Dimension along which Softmin will be computed. + TORCH_ARG(int64_t, dim); +}; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::softmin`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::softmin(input, F::SoftminFuncOptions(1)); +/// ``` +struct TORCH_API SoftminFuncOptions { + SoftminFuncOptions(int64_t dim); + + /// Dimension along which Softmin will be computed. + TORCH_ARG(int64_t, dim); + + /// the desired data type of returned tensor. + /// If specified, the input tensor is casted to `dtype` before the operation + /// is performed. This is useful for preventing data type overflows. Default: + /// None. + TORCH_ARG(std::optional, dtype) = std::nullopt; +}; + +} // namespace functional + +// ============================================================================ + +/// Options for the `LogSoftmax` module. +/// +/// Example: +/// ``` +/// LogSoftmax model(LogSoftmaxOptions(1)); +/// ``` +struct TORCH_API LogSoftmaxOptions { + LogSoftmaxOptions(int64_t dim); + + /// Dimension along which LogSoftmax will be computed. + TORCH_ARG(int64_t, dim); +}; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::log_softmax`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::log_softmax(input, LogSoftmaxFuncOptions(1)); +/// ``` +struct TORCH_API LogSoftmaxFuncOptions { + LogSoftmaxFuncOptions(int64_t dim); + + /// Dimension along which LogSoftmax will be computed. + TORCH_ARG(int64_t, dim); + + /// the desired data type of returned tensor. + /// If specified, the input tensor is casted to `dtype` before the operation + /// is performed. This is useful for preventing data type overflows. Default: + /// None. + TORCH_ARG(std::optional, dtype) = std::nullopt; +}; + +} // namespace functional + +// ============================================================================ + +/// Options for the `PReLU` module. +/// +/// Example: +/// ``` +/// PReLU model(PReLUOptions().num_parameters(42)); +/// ``` +struct TORCH_API PReLUOptions { + /// number of `a` to learn. Although it takes an int as input, there is only + /// two values are legitimate: 1, or the number of channels at input. Default: + /// 1 + TORCH_ARG(int64_t, num_parameters) = 1; + + /// the initial value of `a`. Default: 0.25 + TORCH_ARG(double, init) = 0.25; +}; + +// ============================================================================ + +/// Options for the `ReLU` module. +/// +/// Example: +/// ``` +/// ReLU model(ReLUOptions().inplace(true)); +/// ``` +struct TORCH_API ReLUOptions { + /* implicit */ ReLUOptions(bool inplace = false); + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace); +}; + +namespace functional { +/// Options for `torch::nn::functional::relu`. +/// +/// See the documentation for `torch::nn::ReLUOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::relu(x, F::ReLUFuncOptions().inplace(true)); +/// ``` +using ReLUFuncOptions = ReLUOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `ReLU6` module. +/// +/// Example: +/// ``` +/// ReLU6 model(ReLU6Options().inplace(true)); +/// ``` +struct TORCH_API ReLU6Options { + /* implicit */ ReLU6Options(bool inplace = false); + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace); +}; + +namespace functional { +/// Options for `torch::nn::functional::relu6`. +/// +/// See the documentation for `torch::nn::ReLU6Options` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::relu6(x, F::ReLU6FuncOptions().inplace(true)); +/// ``` +using ReLU6FuncOptions = ReLU6Options; +} // namespace functional + +// ============================================================================ + +/// Options for the `RReLU` module. +/// +/// Example: +/// ``` +/// RReLU model(RReLUOptions().lower(0.24).upper(0.42).inplace(true)); +/// ``` +struct TORCH_API RReLUOptions { + /// lower bound of the uniform distribution. Default: 1/8 + TORCH_ARG(double, lower) = 1.0 / 8.0; + + /// upper bound of the uniform distribution. Default: 1/3 + TORCH_ARG(double, upper) = 1.0 / 3.0; + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace) = false; +}; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::rrelu`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::rrelu(x, F::RReLUFuncOptions().lower(0.1).upper(0.4).inplace(true)); +/// ``` +struct TORCH_API RReLUFuncOptions { + /// lower bound of the uniform distribution. Default: 1/8 + TORCH_ARG(double, lower) = 1.0 / 8.0; + + /// upper bound of the uniform distribution. Default: 1/3 + TORCH_ARG(double, upper) = 1.0 / 3.0; + + TORCH_ARG(bool, training) = false; + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace) = false; +}; + +} // namespace functional + +// ============================================================================ + +/// Options for the `CELU` module. +/// +/// Example: +/// ``` +/// CELU model(CELUOptions().alpha(42.42).inplace(true)); +/// ``` +struct TORCH_API CELUOptions { + /// The `alpha` value for the CELU formulation. Default: 1.0 + TORCH_ARG(double, alpha) = 1.0; + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace) = false; +}; + +namespace functional { +/// Options for `torch::nn::functional::celu`. +/// +/// See the documentation for `torch::nn::CELUOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::celu(x, F::CELUFuncOptions().alpha(0.42).inplace(true)); +/// ``` +using CELUFuncOptions = CELUOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `Softplus` module. +/// +/// Example: +/// ``` +/// Softplus model(SoftplusOptions().beta(0.24).threshold(42.42)); +/// ``` +struct TORCH_API SoftplusOptions { + /// the `beta` value for the Softplus formulation. Default: 1 + TORCH_ARG(double, beta) = 1.0; + + /// values above this revert to a linear function. Default: 20 + TORCH_ARG(double, threshold) = 20.0; +}; + +namespace functional { +/// Options for `torch::nn::functional::softplus`. +/// +/// See the documentation for `torch::nn::SoftplusOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::softplus(x, F::SoftplusFuncOptions().beta(0.5).threshold(3.0)); +/// ``` +using SoftplusFuncOptions = SoftplusOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `Softshrink` module. +/// +/// Example: +/// ``` +/// Softshrink model(SoftshrinkOptions(42.42)); +/// ``` +struct TORCH_API SoftshrinkOptions { + /* implicit */ SoftshrinkOptions(double lambda = 0.5); + + /// the `lambda` value for the Softshrink formulation. Default: 0.5 + TORCH_ARG(double, lambda); +}; + +namespace functional { +/// Options for `torch::nn::functional::softshrink`. +/// +/// See the documentation for `torch::nn::SoftshrinkOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::softshrink(x, F::SoftshrinkFuncOptions(0.42)); +/// ``` +using SoftshrinkFuncOptions = SoftshrinkOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `Threshold` module. +/// +/// Example: +/// ``` +/// Threshold model(ThresholdOptions(42.42, 24.24).inplace(true)); +/// ``` +struct TORCH_API ThresholdOptions { + ThresholdOptions(double threshold, double value) + : threshold_(threshold), value_(value) {} + + /// The value to threshold at + TORCH_ARG(double, threshold); + + /// The value to replace with + TORCH_ARG(double, value); + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace) = false; +}; + +namespace functional { +/// Options for `torch::nn::functional::threshold`. +/// +/// See the documentation for `torch::nn::ThresholdOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::threshold(x, F::ThresholdFuncOptions(0.5, 0.5).inplace(true)); +/// ``` +using ThresholdFuncOptions = ThresholdOptions; +} // namespace functional + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::gumbel_softmax`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::gumbel_softmax(logits, F::GumbelSoftmaxFuncOptions().hard(true).dim(-1)); +/// ``` +struct TORCH_API GumbelSoftmaxFuncOptions { + /// non-negative scalar temperature + TORCH_ARG(double, tau) = 1.0; + + /// returned samples will be discretized as one-hot vectors, + /// but will be differentiated as if it is the soft sample in autograd. + /// Default: False + TORCH_ARG(bool, hard) = false; + + /// dimension along which softmax will be computed. Default: -1 + TORCH_ARG(int, dim) = -1; +}; + +} // namespace functional + +// ============================================================================ + +/// Options for the `MultiheadAttention` module. +/// +/// Example: +/// ``` +/// MultiheadAttention model(MultiheadAttentionOptions(20, 10).bias(false)); +/// ``` +struct TORCH_API MultiheadAttentionOptions { + MultiheadAttentionOptions(int64_t embed_dim, int64_t num_heads); + + /// total dimension of the model. + TORCH_ARG(int64_t, embed_dim); + + /// parallel attention heads. + TORCH_ARG(int64_t, num_heads); + + /// a Dropout layer on attn_output_weights. Default: 0.0. + TORCH_ARG(double, dropout) = 0.0; + + /// add bias as module parameter. Default: true. + TORCH_ARG(bool, bias) = true; + + /// add bias to the key and value sequences at dim=0. + TORCH_ARG(bool, add_bias_kv) = false; + + /// add a new batch of zeros to the key and value sequences at dim=1. + TORCH_ARG(bool, add_zero_attn) = false; + + /// total number of features in key. Default: std::nullopt. + TORCH_ARG(int64_t, kdim); + + /// total number of features in key. Default: std::nullopt. + TORCH_ARG(int64_t, vdim); +}; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::multi_head_attention_forward` +struct TORCH_API MultiheadAttentionForwardFuncOptions { + MultiheadAttentionForwardFuncOptions( + int64_t embed_dim_to_check, + int64_t num_heads, + Tensor in_proj_weight, + Tensor in_proj_bias, + Tensor bias_k, + Tensor bias_v, + bool add_zero_attn, + double dropout_p, + Tensor out_proj_weight, + Tensor out_proj_bias); + + TORCH_ARG(int64_t, embed_dim_to_check); + + TORCH_ARG(int64_t, num_heads); + + TORCH_ARG(Tensor, in_proj_weight); + + TORCH_ARG(Tensor, in_proj_bias); + + TORCH_ARG(Tensor, bias_k); + + TORCH_ARG(Tensor, bias_v); + + TORCH_ARG(bool, add_zero_attn); + + TORCH_ARG(double, dropout_p); + + TORCH_ARG(Tensor, out_proj_weight); + + TORCH_ARG(Tensor, out_proj_bias); + + TORCH_ARG(bool, training) = true; + + TORCH_ARG(Tensor, key_padding_mask); + + TORCH_ARG(bool, need_weights) = true; + + TORCH_ARG(Tensor, attn_mask); + + TORCH_ARG(bool, use_separate_proj_weight) = false; + + TORCH_ARG(Tensor, q_proj_weight); + + TORCH_ARG(Tensor, k_proj_weight); + + TORCH_ARG(Tensor, v_proj_weight); + + TORCH_ARG(Tensor, static_k); + + TORCH_ARG(Tensor, static_v); + + TORCH_ARG(bool, average_attn_weights) = true; +}; + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/adaptive.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/adaptive.h new file mode 100644 index 0000000000000000000000000000000000000000..e7d98ca6ce2c7b8f80a38f7b24a89612efcb223a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/adaptive.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +namespace torch::nn { + +/// Options for the `AdaptiveLogSoftmaxWithLoss` module. +/// +/// Example: +/// ``` +/// AdaptiveLogSoftmaxWithLoss model(AdaptiveLogSoftmaxWithLossOptions(8, 10, +/// {4, 8}).div_value(2.).head_bias(true)); +/// ``` +struct TORCH_API AdaptiveLogSoftmaxWithLossOptions { + /* implicit */ AdaptiveLogSoftmaxWithLossOptions( + int64_t in_features, + int64_t n_classes, + std::vector cutoffs); + + /// Number of features in the input tensor + TORCH_ARG(int64_t, in_features); + + /// Number of classes in the dataset + TORCH_ARG(int64_t, n_classes); + + /// Cutoffs used to assign targets to their buckets + TORCH_ARG(std::vector, cutoffs); + + /// value used as an exponent to compute sizes of the clusters. Default: 4.0 + TORCH_ARG(double, div_value) = 4.; + + /// If ``true``, adds a bias term to the 'head' of + /// the adaptive softmax. Default: false + TORCH_ARG(bool, head_bias) = false; +}; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/batchnorm.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/batchnorm.h new file mode 100644 index 0000000000000000000000000000000000000000..e935072ab5b728603cefd5539bc3e02a965952c0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/batchnorm.h @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include + +namespace torch::nn { + +/// Options for the `BatchNorm` module. +struct TORCH_API BatchNormOptions { + /* implicit */ BatchNormOptions(int64_t num_features); + + /// The number of features of the input tensor. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(int64_t, num_features); + + /// The epsilon value added for numerical stability. + /// Changing this parameter after construction __is effective__. + TORCH_ARG(double, eps) = 1e-5; + + /// A momentum multiplier for the mean and variance. + /// Changing this parameter after construction __is effective__. + TORCH_ARG(std::optional, momentum) = 0.1; + + /// Whether to learn a scale and bias that are applied in an affine + /// transformation on the input. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(bool, affine) = true; + + /// Whether to store and update batch statistics (mean and variance) in the + /// module. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(bool, track_running_stats) = true; +}; + +/// Options for the `BatchNorm1d` module. +/// +/// Example: +/// ``` +/// BatchNorm1d +/// model(BatchNorm1dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +using BatchNorm1dOptions = BatchNormOptions; + +/// Options for the `BatchNorm2d` module. +/// +/// Example: +/// ``` +/// BatchNorm2d +/// model(BatchNorm2dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +using BatchNorm2dOptions = BatchNormOptions; + +/// Options for the `BatchNorm3d` module. +/// +/// Example: +/// ``` +/// BatchNorm3d +/// model(BatchNorm3dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +using BatchNorm3dOptions = BatchNormOptions; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::batch_norm`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::batch_norm(input, mean, variance, +/// F::BatchNormFuncOptions().weight(weight).bias(bias).momentum(0.1).eps(1e-05).training(false)); +/// ``` +struct TORCH_API BatchNormFuncOptions { + TORCH_ARG(Tensor, weight); + + TORCH_ARG(Tensor, bias); + + TORCH_ARG(bool, training) = false; + + /// A momentum multiplier for the mean and variance. + /// Changing this parameter after construction __is effective__. + TORCH_ARG(double, momentum) = 0.1; + + /// The epsilon value added for numerical stability. + /// Changing this parameter after construction __is effective__. + TORCH_ARG(double, eps) = 1e-5; +}; + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/conv.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/conv.h new file mode 100644 index 0000000000000000000000000000000000000000..b7667791c578ca5b07e2283a6668a3d955bba435 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/conv.h @@ -0,0 +1,413 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace torch::nn { + +namespace detail { + +typedef std::variant< + enumtype::kZeros, + enumtype::kReflect, + enumtype::kReplicate, + enumtype::kCircular> + conv_padding_mode_t; + +template +using conv_padding_t = + std::variant, enumtype::kValid, enumtype::kSame>; + +/// Options for a `D`-dimensional convolution or convolution transpose module. +template +struct ConvNdOptions { + using padding_t = conv_padding_t; + ConvNdOptions( + int64_t in_channels, + int64_t out_channels, + ExpandingArray kernel_size) + : in_channels_(in_channels), + out_channels_(out_channels), + kernel_size_(std::move(kernel_size)) {} + + /// The number of channels the input volumes will have. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(int64_t, in_channels); + + /// The number of output channels the convolution should produce. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(int64_t, out_channels); + + /// The kernel size to use. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, kernel_size); + + /// The stride of the convolution. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, stride) = 1; + + /// The padding to add to the input volumes. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(padding_t, padding) = 0; + + public: + auto padding(std::initializer_list il) { + return padding(IntArrayRef{il}); + } + + /// The kernel dilation. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, dilation) = 1; + + /// If true, convolutions will be transpose convolutions (a.k.a. + /// deconvolutions). + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(bool, transposed) = false; + + /// For transpose convolutions, the padding to add to output volumes. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, output_padding) = 0; + + /// The number of convolution groups. + /// This parameter __can__ be changed after construction. + TORCH_ARG(int64_t, groups) = 1; + + /// Whether to add a bias after individual applications of the kernel. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(bool, bias) = true; + + /// Accepted values `torch::kZeros`, `torch::kReflect`, `torch::kReplicate` or + /// `torch::kCircular`. Default: `torch::kZeros` + TORCH_ARG(conv_padding_mode_t, padding_mode) = torch::kZeros; +}; + +} // namespace detail + +// ============================================================================ + +/// Options for a `D`-dimensional convolution module. +template +struct ConvOptions { + using padding_mode_t = detail::conv_padding_mode_t; + using padding_t = detail::conv_padding_t; + + ConvOptions( + int64_t in_channels, + int64_t out_channels, + ExpandingArray kernel_size) + : in_channels_(in_channels), + out_channels_(out_channels), + kernel_size_(std::move(kernel_size)) {} + + /// The number of channels the input volumes will have. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(int64_t, in_channels); + + /// The number of output channels the convolution should produce. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(int64_t, out_channels); + + /// The kernel size to use. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, kernel_size); + + /// The stride of the convolution. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, stride) = 1; + + /// The padding to add to the input volumes. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(padding_t, padding) = 0; + + public: + auto padding(std::initializer_list il) { + return padding(IntArrayRef{il}); + } + + /// The kernel dilation. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, dilation) = 1; + + /// The number of convolution groups. + /// This parameter __can__ be changed after construction. + TORCH_ARG(int64_t, groups) = 1; + + /// Whether to add a bias after individual applications of the kernel. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(bool, bias) = true; + + /// Accepted values `torch::kZeros`, `torch::kReflect`, `torch::kReplicate` or + /// `torch::kCircular`. Default: `torch::kZeros` + TORCH_ARG(padding_mode_t, padding_mode) = torch::kZeros; +}; + +/// `ConvOptions` specialized for the `Conv1d` module. +/// +/// Example: +/// ``` +/// Conv1d model(Conv1dOptions(3, 2, 3).stride(1).bias(false)); +/// ``` +using Conv1dOptions = ConvOptions<1>; + +/// `ConvOptions` specialized for the `Conv2d` module. +/// +/// Example: +/// ``` +/// Conv2d model(Conv2dOptions(3, 2, 3).stride(1).bias(false)); +/// ``` +using Conv2dOptions = ConvOptions<2>; + +/// `ConvOptions` specialized for the `Conv3d` module. +/// +/// Example: +/// ``` +/// Conv3d model(Conv3dOptions(3, 2, 3).stride(1).bias(false)); +/// ``` +using Conv3dOptions = ConvOptions<3>; + +// ============================================================================ + +namespace functional { + +/// Options for a `D`-dimensional convolution functional. +template +struct ConvFuncOptions { + using padding_t = torch::nn::detail::conv_padding_t; + + /// optional bias of shape `(out_channels)`. Default: ``None`` + TORCH_ARG(torch::Tensor, bias); + + /// The stride of the convolving kernel. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + TORCH_ARG(ExpandingArray, stride) = 1; + + /// Implicit paddings on both sides of the input. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + TORCH_ARG(padding_t, padding) = 0; + + public: + auto padding(std::initializer_list il) { + return padding(IntArrayRef{il}); + } + + /// The spacing between kernel elements. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + TORCH_ARG(ExpandingArray, dilation) = 1; + + /// Split input into groups, `in_channels` should be divisible by + /// the number of groups. + TORCH_ARG(int64_t, groups) = 1; +}; + +/// `ConvFuncOptions` specialized for `torch::nn::functional::conv1d`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv1d(x, weight, F::Conv1dFuncOptions().stride(1)); +/// ``` +using Conv1dFuncOptions = ConvFuncOptions<1>; + +/// `ConvFuncOptions` specialized for `torch::nn::functional::conv2d`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv2d(x, weight, F::Conv2dFuncOptions().stride(1)); +/// ``` +using Conv2dFuncOptions = ConvFuncOptions<2>; + +/// `ConvFuncOptions` specialized for `torch::nn::functional::conv3d`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv3d(x, weight, F::Conv3dFuncOptions().stride(1)); +/// ``` +using Conv3dFuncOptions = ConvFuncOptions<3>; + +} // namespace functional + +// ============================================================================ + +template +struct ConvTransposeOptions { + using padding_mode_t = detail::conv_padding_mode_t; + + ConvTransposeOptions( + int64_t in_channels, + int64_t out_channels, + ExpandingArray kernel_size) + : in_channels_(in_channels), + out_channels_(out_channels), + kernel_size_(std::move(kernel_size)) {} + + /// The number of channels the input volumes will have. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(int64_t, in_channels); + + /// The number of output channels the convolution should produce. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(int64_t, out_channels); + + /// The kernel size to use. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, kernel_size); + + /// The stride of the convolution. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, stride) = 1; + + /// The padding to add to the input volumes. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, padding) = 0; + + /// For transpose convolutions, the padding to add to output volumes. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, output_padding) = 0; + + /// The number of convolution groups. + /// This parameter __can__ be changed after construction. + TORCH_ARG(int64_t, groups) = 1; + + /// Whether to add a bias after individual applications of the kernel. + /// Changing this parameter after construction __has no effect__. + TORCH_ARG(bool, bias) = true; + + /// The kernel dilation. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + /// This parameter __can__ be changed after construction. + TORCH_ARG(ExpandingArray, dilation) = 1; + + /// Accepted values `torch::kZeros`, `torch::kReflect`, `torch::kReplicate` or + /// `torch::kCircular`. Default: `torch::kZeros` + TORCH_ARG(padding_mode_t, padding_mode) = torch::kZeros; +}; + +/// `ConvTransposeOptions` specialized for the `ConvTranspose1d` module. +/// +/// Example: +/// ``` +/// ConvTranspose1d model(ConvTranspose1dOptions(3, 2, +/// 3).stride(1).bias(false)); +/// ``` +using ConvTranspose1dOptions = ConvTransposeOptions<1>; + +/// `ConvTransposeOptions` specialized for the `ConvTranspose2d` module. +/// +/// Example: +/// ``` +/// ConvTranspose2d model(ConvTranspose2dOptions(3, 2, +/// 3).stride(1).bias(false)); +/// ``` +using ConvTranspose2dOptions = ConvTransposeOptions<2>; + +/// `ConvTransposeOptions` specialized for the `ConvTranspose3d` module. +/// +/// Example: +/// ``` +/// ConvTranspose3d model(ConvTranspose3dOptions(2, 2, +/// 2).stride(1).bias(false)); +/// ``` +using ConvTranspose3dOptions = ConvTransposeOptions<3>; + +// ============================================================================ + +namespace functional { + +/// Options for a `D`-dimensional convolution functional. +template +struct ConvTransposeFuncOptions { + /// optional bias of shape `(out_channels)`. Default: ``None`` + TORCH_ARG(torch::Tensor, bias); + + /// The stride of the convolving kernel. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + TORCH_ARG(ExpandingArray, stride) = 1; + + /// Implicit paddings on both sides of the input. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + TORCH_ARG(ExpandingArray, padding) = 0; + + /// Additional size added to one side of each dimension in the output shape. + /// Default: 0 + TORCH_ARG(ExpandingArray, output_padding) = 0; + + /// Split input into groups, `in_channels` should be divisible by + /// the number of groups. + TORCH_ARG(int64_t, groups) = 1; + + /// The spacing between kernel elements. + /// For a `D`-dim convolution, must be a single number or a list of `D` + /// numbers. + TORCH_ARG(ExpandingArray, dilation) = 1; +}; + +/// `ConvTransposeFuncOptions` specialized for +/// `torch::nn::functional::conv_transpose1d`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv_transpose1d(x, weight, F::ConvTranspose1dFuncOptions().stride(1)); +/// ``` +using ConvTranspose1dFuncOptions = ConvTransposeFuncOptions<1>; + +/// `ConvTransposeFuncOptions` specialized for +/// `torch::nn::functional::conv_transpose2d`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv_transpose2d(x, weight, F::ConvTranspose2dFuncOptions().stride(1)); +/// ``` +using ConvTranspose2dFuncOptions = ConvTransposeFuncOptions<2>; + +/// `ConvTransposeFuncOptions` specialized for +/// `torch::nn::functional::conv_transpose3d`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::conv_transpose3d(x, weight, F::ConvTranspose3dFuncOptions().stride(1)); +/// ``` +using ConvTranspose3dFuncOptions = ConvTransposeFuncOptions<3>; + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/distance.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/distance.h new file mode 100644 index 0000000000000000000000000000000000000000..7039068d7616b33932905ae2b3a78553176f2a4b --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/distance.h @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include + +namespace torch::nn { + +/// Options for the `CosineSimilarity` module. +/// +/// Example: +/// ``` +/// CosineSimilarity model(CosineSimilarityOptions().dim(0).eps(0.5)); +/// ``` +struct TORCH_API CosineSimilarityOptions { + /// Dimension where cosine similarity is computed. Default: 1 + TORCH_ARG(int64_t, dim) = 1; + /// Small value to avoid division by zero. Default: 1e-8 + TORCH_ARG(double, eps) = 1e-8; +}; + +namespace functional { +/// Options for `torch::nn::functional::cosine_similarity`. +/// +/// See the documentation for `torch::nn::CosineSimilarityOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::cosine_similarity(input1, input2, +/// F::CosineSimilarityFuncOptions().dim(1)); +/// ``` +using CosineSimilarityFuncOptions = CosineSimilarityOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `PairwiseDistance` module. +/// +/// Example: +/// ``` +/// PairwiseDistance +/// model(PairwiseDistanceOptions().p(3).eps(0.5).keepdim(true)); +/// ``` +struct TORCH_API PairwiseDistanceOptions { + /// The norm degree. Default: 2 + TORCH_ARG(double, p) = 2.0; + /// Small value to avoid division by zero. Default: 1e-6 + TORCH_ARG(double, eps) = 1e-6; + /// Determines whether or not to keep the vector dimension. Default: false + TORCH_ARG(bool, keepdim) = false; +}; + +namespace functional { +/// Options for `torch::nn::functional::pairwise_distance`. +/// +/// See the documentation for `torch::nn::PairwiseDistanceOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::pairwise_distance(input1, input2, F::PairwiseDistanceFuncOptions().p(1)); +/// ``` +using PairwiseDistanceFuncOptions = PairwiseDistanceOptions; +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/dropout.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/dropout.h new file mode 100644 index 0000000000000000000000000000000000000000..1710ebfd422e65d90e2fc64bd7f14bd14d2b90a0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/dropout.h @@ -0,0 +1,128 @@ +#pragma once + +#include +#include +#include + +namespace torch::nn { + +/// Options for the `Dropout` module. +/// +/// Example: +/// ``` +/// Dropout model(DropoutOptions().p(0.42).inplace(true)); +/// ``` +struct TORCH_API DropoutOptions { + /* implicit */ DropoutOptions(double p = 0.5); + + /// The probability of an element to be zeroed. Default: 0.5 + TORCH_ARG(double, p) = 0.5; + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace) = false; +}; + +/// Options for the `Dropout2d` module. +/// +/// Example: +/// ``` +/// Dropout2d model(Dropout2dOptions().p(0.42).inplace(true)); +/// ``` +using Dropout2dOptions = DropoutOptions; + +/// Options for the `Dropout3d` module. +/// +/// Example: +/// ``` +/// Dropout3d model(Dropout3dOptions().p(0.42).inplace(true)); +/// ``` +using Dropout3dOptions = DropoutOptions; + +/// Options for the `AlphaDropout` module. +/// +/// Example: +/// ``` +/// AlphaDropout model(AlphaDropoutOptions(0.2).inplace(true)); +/// ``` +using AlphaDropoutOptions = DropoutOptions; + +/// Options for the `FeatureAlphaDropout` module. +/// +/// Example: +/// ``` +/// FeatureAlphaDropout model(FeatureAlphaDropoutOptions(0.2).inplace(true)); +/// ``` +using FeatureAlphaDropoutOptions = DropoutOptions; + +namespace functional { + +/// Options for `torch::nn::functional::dropout`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::dropout(input, F::DropoutFuncOptions().p(0.5)); +/// ``` +struct TORCH_API DropoutFuncOptions { + /// The probability of an element to be zeroed. Default: 0.5 + TORCH_ARG(double, p) = 0.5; + + TORCH_ARG(bool, training) = true; + + /// can optionally do the operation in-place. Default: False + TORCH_ARG(bool, inplace) = false; +}; + +/// Options for `torch::nn::functional::dropout2d`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::dropout2d(input, F::Dropout2dFuncOptions().p(0.5)); +/// ``` +using Dropout2dFuncOptions = DropoutFuncOptions; + +/// Options for `torch::nn::functional::dropout3d`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::dropout3d(input, F::Dropout3dFuncOptions().p(0.5)); +/// ``` +using Dropout3dFuncOptions = DropoutFuncOptions; + +/// Options for `torch::nn::functional::alpha_dropout`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::alpha_dropout(input, +/// F::AlphaDropoutFuncOptions().p(0.5).training(false)); +/// ``` +struct TORCH_API AlphaDropoutFuncOptions { + TORCH_ARG(double, p) = 0.5; + + TORCH_ARG(bool, training) = false; + + TORCH_ARG(bool, inplace) = false; +}; + +/// Options for `torch::nn::functional::feature_alpha_dropout`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::feature_alpha_dropout(input, +/// F::FeatureAlphaDropoutFuncOptions().p(0.5).training(false)); +/// ``` +struct TORCH_API FeatureAlphaDropoutFuncOptions { + TORCH_ARG(double, p) = 0.5; + + TORCH_ARG(bool, training) = false; + + TORCH_ARG(bool, inplace) = false; +}; + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/embedding.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/embedding.h new file mode 100644 index 0000000000000000000000000000000000000000..ecf9c41d99680843027892ae7f153c4619be68f8 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/embedding.h @@ -0,0 +1,240 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn { + +/// Options for the `Embedding` module. +/// +/// Example: +/// ``` +/// Embedding model(EmbeddingOptions(10, +/// 2).padding_idx(3).max_norm(2).norm_type(2.5).scale_grad_by_freq(true).sparse(true)); +/// ``` +struct TORCH_API EmbeddingOptions { + EmbeddingOptions(int64_t num_embeddings, int64_t embedding_dim); + + /// The size of the dictionary of embeddings. + TORCH_ARG(int64_t, num_embeddings); + /// The size of each embedding vector. + TORCH_ARG(int64_t, embedding_dim); + /// If specified, the entries at `padding_idx` do not contribute to the + /// gradient; therefore, the embedding vector at `padding_idx` is not updated + /// during training, i.e. it remains as a fixed "pad". For a newly constructed + /// Embedding, the embedding vector at `padding_idx` will default to all + /// zeros, but can be updated to another value to be used as the padding + /// vector. + TORCH_ARG(std::optional, padding_idx) = std::nullopt; + /// If given, each embedding vector with norm larger than `max_norm` is + /// renormalized to have norm `max_norm`. + TORCH_ARG(std::optional, max_norm) = std::nullopt; + /// The p of the p-norm to compute for the `max_norm` option. Default ``2``. + TORCH_ARG(double, norm_type) = 2.; + /// If given, this will scale gradients by the inverse of frequency of the + /// words in the mini-batch. Default ``false``. + TORCH_ARG(bool, scale_grad_by_freq) = false; + /// If ``true``, gradient w.r.t. `weight` matrix will be a sparse tensor. + TORCH_ARG(bool, sparse) = false; + /// The learnable weights of the module of shape (num_embeddings, + /// embedding_dim) + TORCH_ARG(torch::Tensor, _weight); +}; + +// ============================================================================ + +/// Options for the `Embedding::from_pretrained` function. +struct TORCH_API EmbeddingFromPretrainedOptions { + /// If ``true``, the tensor does not get updated in the learning process. + /// Equivalent to ``embedding.weight.requires_grad_(false)``. Default: + /// ``true`` + TORCH_ARG(bool, freeze) = true; + /// If specified, the entries at `padding_idx` do not contribute to the + /// gradient; therefore, the embedding vector at `padding_idx` is not updated + /// during training, i.e. it remains as a fixed "pad". + TORCH_ARG(std::optional, padding_idx) = std::nullopt; + /// If given, each embedding vector with norm larger than `max_norm` is + /// renormalized to have norm `max_norm`. + TORCH_ARG(std::optional, max_norm) = std::nullopt; + /// The p of the p-norm to compute for the `max_norm` option. Default ``2``. + TORCH_ARG(double, norm_type) = 2.; + /// If given, this will scale gradients by the inverse of frequency of the + /// words in the mini-batch. Default ``false``. + TORCH_ARG(bool, scale_grad_by_freq) = false; + /// If ``true``, gradient w.r.t. `weight` matrix will be a sparse tensor. + TORCH_ARG(bool, sparse) = false; +}; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::embedding`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::embedding(input, weight, +/// F::EmbeddingFuncOptions().norm_type(2.5).scale_grad_by_freq(true).sparse(true)); +/// ``` +struct TORCH_API EmbeddingFuncOptions { + /// If specified, the entries at `padding_idx` do not contribute to the + /// gradient; therefore, the embedding vector at `padding_idx` is not updated + /// during training, i.e. it remains as a fixed "pad". + TORCH_ARG(std::optional, padding_idx) = std::nullopt; + /// If given, each embedding vector with norm larger than `max_norm` is + /// renormalized to have norm `max_norm`. + TORCH_ARG(std::optional, max_norm) = std::nullopt; + /// The p of the p-norm to compute for the `max_norm` option. Default ``2``. + TORCH_ARG(double, norm_type) = 2.; + /// If given, this will scale gradients by the inverse of frequency of the + /// words in the mini-batch. Default ``false``. + TORCH_ARG(bool, scale_grad_by_freq) = false; + /// If ``true``, gradient w.r.t. `weight` matrix will be a sparse tensor. + TORCH_ARG(bool, sparse) = false; +}; + +} // namespace functional + +// ============================================================================ + +typedef std::variant + EmbeddingBagMode; + +/// Options for the `EmbeddingBag` module. +/// +/// Example: +/// ``` +/// EmbeddingBag model(EmbeddingBagOptions(10, +/// 2).max_norm(2).norm_type(2.5).scale_grad_by_freq(true).sparse(true).mode(torch::kSum)); +/// ``` +struct TORCH_API EmbeddingBagOptions { + EmbeddingBagOptions(int64_t num_embeddings, int64_t embedding_dim); + + /// The size of the dictionary of embeddings. + TORCH_ARG(int64_t, num_embeddings); + /// The size of each embedding vector. + TORCH_ARG(int64_t, embedding_dim); + /// If given, each embedding vector with norm larger than `max_norm` is + /// renormalized to have norm `max_norm`. + TORCH_ARG(std::optional, max_norm) = std::nullopt; + /// The p of the p-norm to compute for the `max_norm` option. Default ``2``. + TORCH_ARG(double, norm_type) = 2.; + /// If given, this will scale gradients by the inverse of frequency of the + /// words in the mini-batch. Default ``false``. Note: this option is not + /// supported when ``mode="kMax"``. + TORCH_ARG(bool, scale_grad_by_freq) = false; + /// ``"kSum"``, ``"kMean"`` or ``"kMax"``. Specifies the way to reduce the + /// bag. ``"kSum"`` computes the weighted sum, taking `per_sample_weights` + /// into consideration. ``"kMean"`` computes the average of the values in the + /// bag, ``"kMax"`` computes the max value over each bag. + TORCH_ARG(EmbeddingBagMode, mode) = torch::kMean; + /// If ``true``, gradient w.r.t. `weight` matrix will be a sparse tensor. + /// Note: this option is not supported when ``mode="kMax"``. + TORCH_ARG(bool, sparse) = false; + /// The learnable weights of the module of shape (num_embeddings, + /// embedding_dim) + TORCH_ARG(torch::Tensor, _weight); + /// If ``true``, `offsets` has one additional element, where the last element + /// is equivalent to the size of `indices`. This matches the CSR format. + TORCH_ARG(bool, include_last_offset) = false; + /// If specified, the entries at `padding_idx` do not contribute to the + /// gradient; therefore, the embedding vector at padding_idx is not updated + /// during training, i.e. it remains as a fixed "pad". For a newly constructed + /// EmbeddingBag, the embedding vector at `padding_idx` will default to all + /// zeros, but can be updated to another value to be used as the padding + /// vector. Note that the embedding vector at `padding_idx` is excluded from + /// the reduction. + TORCH_ARG(std::optional, padding_idx) = std::nullopt; +}; + +// ============================================================================ + +/// Options for the `EmbeddingBag::from_pretrained` function. +struct TORCH_API EmbeddingBagFromPretrainedOptions { + /// If ``true``, the tensor does not get updated in the learning process. + /// Equivalent to ``embeddingbag.weight.requires_grad_(false)``. Default: + /// ``true`` + TORCH_ARG(bool, freeze) = true; + /// If given, each embedding vector with norm larger than `max_norm` is + /// renormalized to have norm `max_norm`. + TORCH_ARG(std::optional, max_norm) = std::nullopt; + /// The p of the p-norm to compute for the `max_norm` option. Default ``2``. + TORCH_ARG(double, norm_type) = 2.; + /// If given, this will scale gradients by the inverse of frequency of the + /// words in the mini-batch. Default ``false``. Note: this option is not + /// supported when ``mode="kMax"``. + TORCH_ARG(bool, scale_grad_by_freq) = false; + /// ``"kSum"``, ``"kMean"`` or ``"kMax"``. Specifies the way to reduce the + /// bag. ``"kSum"`` computes the weighted sum, taking `per_sample_weights` + /// into consideration. ``"kMean"`` computes the average of the values in the + /// bag, ``"kMax"`` computes the max value over each bag. + TORCH_ARG(EmbeddingBagMode, mode) = torch::kMean; + /// If ``true``, gradient w.r.t. `weight` matrix will be a sparse tensor. + /// Note: this option is not supported when ``mode="kMax"``. + TORCH_ARG(bool, sparse) = false; + /// If ``true``, `offsets` has one additional element, where the last element + /// is equivalent to the size of `indices`. This matches the CSR format. Note: + /// this option is currently only supported when ``mode="sum"``. + TORCH_ARG(bool, include_last_offset) = false; + /// If specified, the entries at `padding_idx` do not contribute to the + /// gradient; therefore, the embedding vector at padding_idx is not updated + /// during training, i.e. it remains as a fixed "pad". Note that the embedding + /// vector at `padding_idx` is excluded from the reduction. + TORCH_ARG(std::optional, padding_idx) = std::nullopt; +}; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::embedding_bag`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::embedding_bag(input, weight, +/// F::EmbeddingBagFuncOptions().mode(torch::kSum).offsets(offsets)); +/// ``` +struct TORCH_API EmbeddingBagFuncOptions { + /// Only used when `input` is 1D. `offsets` determines + /// the starting index position of each bag (sequence) in `input`. + TORCH_ARG(torch::Tensor, offsets); + /// If given, each embedding vector with norm larger than `max_norm` is + /// renormalized to have norm `max_norm`. + TORCH_ARG(std::optional, max_norm) = std::nullopt; + /// The p of the p-norm to compute for the `max_norm` option. Default ``2``. + TORCH_ARG(double, norm_type) = 2.; + /// If given, this will scale gradients by the inverse of frequency of the + /// words in the mini-batch. Default ``false``. Note: this option is not + /// supported when ``mode="kMax"``. + TORCH_ARG(bool, scale_grad_by_freq) = false; + /// ``"kSum"``, ``"kMean"`` or ``"kMax"``. Specifies the way to reduce the + /// bag. ``"kSum"`` computes the weighted sum, taking `per_sample_weights` + /// into consideration. ``"kMean"`` computes the average of the values in the + /// bag, ``"kMax"`` computes the max value over each bag. + TORCH_ARG(EmbeddingBagMode, mode) = torch::kMean; + /// If ``true``, gradient w.r.t. `weight` matrix will be a sparse tensor. + /// Note: this option is not supported when ``mode="kMax"``. + TORCH_ARG(bool, sparse) = false; + /// a tensor of float / double weights, or None to indicate all weights should + /// be taken to be 1. If specified, `per_sample_weights` must have exactly the + /// same shape as input and is treated as having the same `offsets`, if those + /// are not None. + TORCH_ARG(torch::Tensor, per_sample_weights); + /// If ``true``, `offsets` has one additional element, where the last element + /// is equivalent to the size of `indices`. This matches the CSR format. Note: + /// this option is currently only supported when ``mode="sum"``. + TORCH_ARG(bool, include_last_offset) = false; + /// If specified, the entries at `padding_idx` do not contribute to the + /// gradient; therefore, the embedding vector at padding_idx is not updated + /// during training, i.e. it remains as a fixed "pad". Note that the embedding + /// vector at `padding_idx` is excluded from the reduction. + TORCH_ARG(std::optional, padding_idx) = std::nullopt; +}; + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/fold.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/fold.h new file mode 100644 index 0000000000000000000000000000000000000000..eb11b39ee90fbe797767dc163216bebe701eacc2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/fold.h @@ -0,0 +1,95 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn { + +/// Options for the `Fold` module. +/// +/// Example: +/// ``` +/// Fold model(FoldOptions({8, 8}, {3, 3}).dilation(2).padding({2, +/// 1}).stride(2)); +/// ``` +struct TORCH_API FoldOptions { + FoldOptions(ExpandingArray<2> output_size, ExpandingArray<2> kernel_size) + : output_size_(output_size), kernel_size_(kernel_size) {} + + /// describes the spatial shape of the large containing tensor of the sliding + /// local blocks. It is useful to resolve the ambiguity when multiple input + /// shapes map to same number of sliding blocks, e.g., with stride > 0. + TORCH_ARG(ExpandingArray<2>, output_size); + + /// the size of the sliding blocks + TORCH_ARG(ExpandingArray<2>, kernel_size); + + /// controls the spacing between the kernel points; also known as the à trous + /// algorithm. + TORCH_ARG(ExpandingArray<2>, dilation) = 1; + + /// controls the amount of implicit zero-paddings on both sides for padding + /// number of points for each dimension before reshaping. + TORCH_ARG(ExpandingArray<2>, padding) = 0; + + /// controls the stride for the sliding blocks. + TORCH_ARG(ExpandingArray<2>, stride) = 1; +}; + +namespace functional { +/// Options for `torch::nn::functional::fold`. +/// +/// See the documentation for `torch::nn::FoldOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::fold(input, F::FoldFuncOptions({3, 2}, {2, 2})); +/// ``` +using FoldFuncOptions = FoldOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `Unfold` module. +/// +/// Example: +/// ``` +/// Unfold model(UnfoldOptions({2, 4}).dilation(2).padding({2, 1}).stride(2)); +/// ``` +struct TORCH_API UnfoldOptions { + UnfoldOptions(ExpandingArray<2> kernel_size) : kernel_size_(kernel_size) {} + + /// the size of the sliding blocks + TORCH_ARG(ExpandingArray<2>, kernel_size); + + /// controls the spacing between the kernel points; also known as the à trous + /// algorithm. + TORCH_ARG(ExpandingArray<2>, dilation) = 1; + + /// controls the amount of implicit zero-paddings on both sides for padding + /// number of points for each dimension before reshaping. + TORCH_ARG(ExpandingArray<2>, padding) = 0; + + /// controls the stride for the sliding blocks. + TORCH_ARG(ExpandingArray<2>, stride) = 1; +}; + +namespace functional { +/// Options for `torch::nn::functional::unfold`. +/// +/// See the documentation for `torch::nn::UnfoldOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::unfold(input, F::UnfoldFuncOptions({2, 2}).padding(1).stride(2)); +/// ``` +using UnfoldFuncOptions = UnfoldOptions; +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/instancenorm.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/instancenorm.h new file mode 100644 index 0000000000000000000000000000000000000000..b53b36867a6299106d5883435be0cdd74815e53c --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/instancenorm.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn { + +/// Options for the `InstanceNorm` module. +struct TORCH_API InstanceNormOptions { + /* implicit */ InstanceNormOptions(int64_t num_features); + + /// The number of features of the input tensor. + TORCH_ARG(int64_t, num_features); + + /// The epsilon value added for numerical stability. + TORCH_ARG(double, eps) = 1e-5; + + /// A momentum multiplier for the mean and variance. + TORCH_ARG(double, momentum) = 0.1; + + /// Whether to learn a scale and bias that are applied in an affine + /// transformation on the input. + TORCH_ARG(bool, affine) = false; + + /// Whether to store and update batch statistics (mean and variance) in the + /// module. + TORCH_ARG(bool, track_running_stats) = false; +}; + +/// Options for the `InstanceNorm1d` module. +/// +/// Example: +/// ``` +/// InstanceNorm1d +/// model(InstanceNorm1dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +using InstanceNorm1dOptions = InstanceNormOptions; + +/// Options for the `InstanceNorm2d` module. +/// +/// Example: +/// ``` +/// InstanceNorm2d +/// model(InstanceNorm2dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +using InstanceNorm2dOptions = InstanceNormOptions; + +/// Options for the `InstanceNorm3d` module. +/// +/// Example: +/// ``` +/// InstanceNorm3d +/// model(InstanceNorm3dOptions(4).eps(0.5).momentum(0.1).affine(false).track_running_stats(true)); +/// ``` +using InstanceNorm3dOptions = InstanceNormOptions; + +namespace functional { + +/// Options for `torch::nn::functional::instance_norm`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::instance_norm(input, +/// F::InstanceNormFuncOptions().running_mean(mean).running_var(variance).weight(weight).bias(bias).momentum(0.1).eps(1e-5)); +/// ``` +struct TORCH_API InstanceNormFuncOptions { + TORCH_ARG(Tensor, running_mean); + + TORCH_ARG(Tensor, running_var); + + TORCH_ARG(Tensor, weight); + + TORCH_ARG(Tensor, bias); + + TORCH_ARG(bool, use_input_stats) = true; + + TORCH_ARG(double, momentum) = 0.1; + + TORCH_ARG(double, eps) = 1e-5; +}; + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/linear.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/linear.h new file mode 100644 index 0000000000000000000000000000000000000000..918df0c09f27a16aa143d9b04b1c9f96fc265ed3 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/linear.h @@ -0,0 +1,82 @@ +#pragma once + +#include +#include +#include + +namespace torch::nn { + +/// Options for the `Linear` module. +/// +/// Example: +/// ``` +/// Linear model(LinearOptions(5, 2).bias(false)); +/// ``` +struct TORCH_API LinearOptions { + LinearOptions(int64_t in_features, int64_t out_features); + /// size of each input sample + TORCH_ARG(int64_t, in_features); + + /// size of each output sample + TORCH_ARG(int64_t, out_features); + + /// If set to false, the layer will not learn an additive bias. Default: true + TORCH_ARG(bool, bias) = true; +}; + +// ============================================================================ + +/// Options for the `Flatten` module. +/// +/// Example: +/// ``` +/// Flatten model(FlattenOptions().start_dim(2).end_dim(4)); +/// ``` +struct TORCH_API FlattenOptions { + /// first dim to flatten + TORCH_ARG(int64_t, start_dim) = 1; + /// last dim to flatten + TORCH_ARG(int64_t, end_dim) = -1; +}; + +// ============================================================================ + +/// Options for the `Unflatten` module. +/// +/// Example: +/// ``` +/// Unflatten model(UnflattenOptions(0, {2, 2})); +/// ``` +struct TORCH_API UnflattenOptions { + UnflattenOptions(int64_t dim, std::vector sizes); + + /// dim to unflatten + TORCH_ARG(int64_t, dim); + /// new shape of unflattened dim + TORCH_ARG(std::vector, sizes); +}; + +// ============================================================================ + +/// Options for the `Bilinear` module. +/// +/// Example: +/// ``` +/// Bilinear model(BilinearOptions(3, 2, 4).bias(false)); +/// ``` +struct TORCH_API BilinearOptions { + BilinearOptions( + int64_t in1_features, + int64_t in2_features, + int64_t out_features); + /// The number of features in input 1 (columns of the input1 matrix). + TORCH_ARG(int64_t, in1_features); + /// The number of features in input 2 (columns of the input2 matrix). + TORCH_ARG(int64_t, in2_features); + /// The number of output features to produce (columns of the output matrix). + TORCH_ARG(int64_t, out_features); + /// Whether to learn and add a bias after the bilinear transformation. + TORCH_ARG(bool, bias) = true; +}; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/loss.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/loss.h new file mode 100644 index 0000000000000000000000000000000000000000..b13fc9f970e72da6a219b43c5d9c72f0e3f4e46b --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/loss.h @@ -0,0 +1,800 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn { + +/// Options for the `L1Loss` module. +/// +/// Example: +/// ``` +/// L1Loss model(L1LossOptions(torch::kNone)); +/// ``` +struct TORCH_API L1LossOptions { + typedef std::variant + reduction_t; + + TORCH_OPTIONS_CTOR_VARIANT_ARG3(L1LossOptions, reduction, kNone, kMean, kSum) + + /// Specifies the reduction to apply to the output. + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::l1_loss`. +/// +/// See the documentation for `torch::nn::L1LossOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::l1_loss(input, target, F::L1LossFuncOptions(torch::kNone)); +/// ``` +using L1LossFuncOptions = L1LossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `KLDivLoss` module. +/// +/// Example: +/// ``` +/// KLDivLoss +/// model(KLDivLossOptions().reduction(torch::kNone).log_target(false)); +/// ``` +struct TORCH_API KLDivLossOptions { + typedef std::variant< + enumtype::kNone, + enumtype::kBatchMean, + enumtype::kSum, + enumtype::kMean> + reduction_t; + + TORCH_OPTIONS_CTOR_VARIANT_ARG4( + KLDivLossOptions, + reduction, + kNone, + kBatchMean, + kSum, + kMean) + + /// Specifies the reduction to apply to the output. + /// ``'none'`` | ``'batchmean'`` | ``'sum'`` | ``'mean'``. Default: ``'mean'`` + TORCH_ARG(reduction_t, reduction) = torch::kMean; + + /// Specifies whether `target` is accepted in the log space. Default: False + TORCH_ARG(bool, log_target) = false; +}; + +namespace functional { +/// Options for `torch::nn::functional::kl_div`. +/// +/// See the documentation for `torch::nn::KLDivLossOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::kl_div(input, target, +/// F::KLDivFuncOptions().reduction(torch::kNone).log_target(false)); +/// ``` +using KLDivFuncOptions = KLDivLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `MSELoss` module. +/// +/// Example: +/// ``` +/// MSELoss model(MSELossOptions(torch::kNone)); +/// ``` +struct TORCH_API MSELossOptions { + typedef std::variant + reduction_t; + + TORCH_OPTIONS_CTOR_VARIANT_ARG3(MSELossOptions, reduction, kNone, kMean, kSum) + + /// Specifies the reduction to apply to the output. + /// ``'none'`` | ``'mean'`` | ``'sum'``. Default: ``'mean'`` + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::mse_loss`. +/// +/// See the documentation for `torch::nn::MSELossOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::mse_loss(input, target, F::MSELossFuncOptions(torch::kNone)); +/// ``` +using MSELossFuncOptions = MSELossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `BCELoss` module. +/// +/// Example: +/// ``` +/// BCELoss model(BCELossOptions().reduction(torch::kNone).weight(weight)); +/// ``` +struct TORCH_API BCELossOptions { + typedef std::variant + reduction_t; + + /// A manual rescaling weight given to the loss of each batch element. + TORCH_ARG(Tensor, weight); + /// Specifies the reduction to apply to the output. + /// ``'none'`` | ``'mean'`` | ``'sum'``. Default: ``'mean'`` + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::binary_cross_entropy`. +/// +/// See the documentation for `torch::nn::BCELossOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::binary_cross_entropy(input, target, +/// F::BinaryCrossEntropyFuncOptions().weight(weight)); +/// ``` +using BinaryCrossEntropyFuncOptions = BCELossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `HingeEmbeddingLoss` module. +/// +/// Example: +/// ``` +/// HingeEmbeddingLoss +/// model(HingeEmbeddingLossOptions().margin(4).reduction(torch::kNone)); +/// ``` +struct TORCH_API HingeEmbeddingLossOptions { + typedef std::variant + reduction_t; + + /// Specifies the threshold for which the distance of a negative sample must + /// reach in order to incur zero loss. Default: 1 + TORCH_ARG(double, margin) = 1.0; + /// Specifies the reduction to apply to the output. Default: Mean + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::hinge_embedding_loss`. +/// +/// See the documentation for `torch::nn::HingeEmbeddingLossOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::hinge_embedding_loss(input, target, +/// F::HingeEmbeddingLossFuncOptions().margin(2)); +/// ``` +using HingeEmbeddingLossFuncOptions = HingeEmbeddingLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `MultiMarginLoss` module. +/// +/// Example: +/// ``` +/// MultiMarginLoss model(MultiMarginLossOptions().margin(2).weight(weight)); +/// ``` +struct TORCH_API MultiMarginLossOptions { + typedef std::variant + reduction_t; + + /// Has a default value of :math:`1`. :math:`1` and :math:`2` + /// are the only supported values. + TORCH_ARG(int64_t, p) = 1; + /// Has a default value of :math:`1`. + TORCH_ARG(double, margin) = 1.0; + /// A manual rescaling weight given to each + /// class. If given, it has to be a Tensor of size `C`. Otherwise, it is + /// treated as if having all ones. + TORCH_ARG(Tensor, weight); + /// Specifies the reduction to apply to the output: + /// ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be + /// applied, + /// ``'mean'``: the sum of the output will be divided by the number of + /// elements in the output, ``'sum'``: the output will be summed. Default: + /// ``'mean'`` + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::multi_margin_loss`. +/// +/// See the documentation for `torch::nn::MultiMarginLossOptions` class to learn +/// what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::multi_margin_loss(input, target, +/// F::MultiMarginLossFuncOptions().margin(2).weight(weight)); +/// ``` +using MultiMarginLossFuncOptions = MultiMarginLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `CosineEmbeddingLoss` module. +/// +/// Example: +/// ``` +/// CosineEmbeddingLoss model(CosineEmbeddingLossOptions().margin(0.5)); +/// ``` +struct TORCH_API CosineEmbeddingLossOptions { + typedef std::variant + reduction_t; + + /// Specifies the threshold for which the distance of a negative sample must + /// reach in order to incur zero loss. Should be a number from -1 to 1, 0 + /// to 0.5 is suggested. Default: 0.0 + TORCH_ARG(double, margin) = 0.0; + /// Specifies the reduction to apply to the output. Default: Mean + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::cosine_embedding_loss`. +/// +/// See the documentation for `torch::nn::CosineEmbeddingLossOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::cosine_embedding_loss(input1, input2, target, +/// F::CosineEmbeddingLossFuncOptions().margin(0.5)); +/// ``` +using CosineEmbeddingLossFuncOptions = CosineEmbeddingLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `MultiLabelMarginLoss` module. +/// +/// Example: +/// ``` +/// MultiLabelMarginLoss model(MultiLabelMarginLossOptions(torch::kNone)); +/// ``` +struct TORCH_API MultiLabelMarginLossOptions { + typedef std::variant + reduction_t; + + TORCH_OPTIONS_CTOR_VARIANT_ARG3( + MultiLabelMarginLossOptions, + reduction, + kNone, + kMean, + kSum) + + /// Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. + /// 'none': no reduction will be applied, 'mean': the sum of the output will + /// be divided by the number of elements in the output, 'sum': the output will + /// be summed. Default: 'mean' + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::multilabel_margin_loss`. +/// +/// See the documentation for `torch::nn::MultiLabelMarginLossOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::multilabel_margin_loss(input, target, +/// F::MultilabelMarginLossFuncOptions(torch::kNone)); +/// ``` +using MultilabelMarginLossFuncOptions = MultiLabelMarginLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `SoftMarginLoss` module. +/// +/// Example: +/// ``` +/// SoftMarginLoss model(SoftMarginLossOptions(torch::kNone)); +/// ``` +struct TORCH_API SoftMarginLossOptions { + typedef std::variant + reduction_t; + + TORCH_OPTIONS_CTOR_VARIANT_ARG3( + SoftMarginLossOptions, + reduction, + kNone, + kMean, + kSum) + + /// Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. + /// 'none': no reduction will be applied, 'mean': the sum of the output will + /// be divided by the number of elements in the output, 'sum': the output will + /// be summed. Default: 'mean' + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::soft_margin_loss`. +/// +/// See the documentation for `torch::nn::SoftMarginLossOptions` class to learn +/// what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::soft_margin_loss(input, target, +/// F::SoftMarginLossFuncOptions(torch::kNone)); +/// ``` +using SoftMarginLossFuncOptions = SoftMarginLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `MultiLabelSoftMarginLoss` module. +/// +/// Example: +/// ``` +/// MultiLabelSoftMarginLoss +/// model(MultiLabelSoftMarginLossOptions().reduction(torch::kNone).weight(weight)); +/// ``` +struct TORCH_API MultiLabelSoftMarginLossOptions { + typedef std::variant + reduction_t; + + /// A manual rescaling weight given to each + /// class. If given, it has to be a Tensor of size `C`. Otherwise, it is + /// treated as if having all ones. + TORCH_ARG(Tensor, weight); + + /// Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. + /// 'none': no reduction will be applied, 'mean': the sum of the output will + /// be divided by the number of elements in the output, 'sum': the output will + /// be summed. Default: 'mean' + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::multilabel_soft_margin_loss`. +/// +/// See the documentation for `torch::nn::MultiLabelSoftMarginLossOptions` class +/// to learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::multilabel_soft_margin_loss(input, target, +/// F::MultilabelSoftMarginLossFuncOptions().reduction(torch::kNone).weight(weight)); +/// ``` +using MultilabelSoftMarginLossFuncOptions = MultiLabelSoftMarginLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `TripletMarginLoss` module. +/// +/// Example: +/// ``` +/// TripletMarginLoss +/// model(TripletMarginLossOptions().margin(3).p(2).eps(1e-06).swap(false)); +/// ``` +struct TORCH_API TripletMarginLossOptions { + typedef std::variant + reduction_t; + + /// Specifies the threshold for which the distance of a negative sample must + /// reach in order to incur zero loss. Default: 1 + TORCH_ARG(double, margin) = 1.0; + /// Specifies the norm degree for pairwise distance. Default: 2 + TORCH_ARG(double, p) = 2.0; + TORCH_ARG(double, eps) = 1e-6; + /// The distance swap is described in detail in the paper Learning shallow + /// convolutional feature descriptors with triplet losses by V. Balntas, + /// E. Riba et al. Default: False + TORCH_ARG(bool, swap) = false; + /// Specifies the reduction to apply to the output. Default: Mean + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::triplet_margin_loss`. +/// +/// See the documentation for `torch::nn::TripletMarginLossOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::triplet_margin_loss(anchor, positive, negative, +/// F::TripletMarginLossFuncOptions().margin(1.0)); +/// ``` +using TripletMarginLossFuncOptions = TripletMarginLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `TripletMarginWithDistanceLoss` module. +/// +/// Example: +/// ``` +/// TripletMarginWithDistanceLoss +/// model(TripletMarginWithDistanceLossOptions().margin(3).swap(false)); +/// ``` +struct TORCH_API TripletMarginWithDistanceLossOptions { + typedef std::variant + reduction_t; + typedef std::function + distance_function_t; + + /// Specifies a nonnegative, real-valued function that quantifies the + /// closeness of two tensors. If not specified, `F::pairwise_distance` will + /// be used. Default: nullopt + TORCH_ARG(std::optional, distance_function) = + std::nullopt; + /// Specifies a nonnegative margin representing the minimum difference + /// between the positive and negative distances required for the loss to be 0. + /// Larger margins penalize cases where the negative examples are not distance + /// enough from the anchors, relative to the positives. Default: 1 + TORCH_ARG(double, margin) = 1.0; + /// Whether to use the distance swap described in the paper Learning shallow + /// convolutional feature descriptors with triplet losses by V. Balntas, + /// E. Riba et al. If True, and if the positive example is closer to the + /// negative example than the anchor is, swaps the positive example and the + /// anchor in the loss computation. Default: False + TORCH_ARG(bool, swap) = false; + /// Specifies the reduction to apply to the output. Default: Mean + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::triplet_margin_with_distance_loss`. +/// +/// See the documentation for `torch::nn::TripletMarginWithDistanceLossOptions` +/// class to learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::triplet_margin_with_distance_loss(anchor, positive, negative, +/// F::TripletMarginWithDistanceLossFuncOptions().margin(1.0)); +/// ``` +using TripletMarginWithDistanceLossFuncOptions = + TripletMarginWithDistanceLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `CTCLoss` module. +/// +/// Example: +/// ``` +/// CTCLoss +/// model(CTCLossOptions().blank(42).zero_infinity(false).reduction(torch::kSum)); +/// ``` +struct TORCH_API CTCLossOptions { + typedef std::variant + reduction_t; + + /// blank label. Default `0`. + TORCH_ARG(int64_t, blank) = 0; + /// Specifies the reduction to apply to the output. Default: Mean + TORCH_ARG(reduction_t, reduction) = torch::kMean; + /// Whether to zero infinite losses and the associated gradients. + /// Default: `false`. Infinite losses mainly occur when the inputs are + /// too short to be aligned to the targets. + TORCH_ARG(bool, zero_infinity) = false; +}; + +namespace functional { +/// Options for `torch::nn::functional::ctc_loss`. +/// +/// See the documentation for `torch::nn::CTCLossOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::ctc_loss(log_probs, targets, input_lengths, target_lengths, +/// F::CTCLossFuncOptions().reduction(torch::kNone)); +/// ``` +using CTCLossFuncOptions = CTCLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `SmoothL1Loss` module. +/// +/// Example: +/// ``` +/// SmoothL1Loss model(SmoothL1LossOptions().reduction(torch::kNone).beta(0.5)); +/// ``` +struct TORCH_API SmoothL1LossOptions { + typedef std::variant + reduction_t; + + TORCH_OPTIONS_CTOR_VARIANT_ARG3( + SmoothL1LossOptions, + reduction, + kNone, + kMean, + kSum) + + /// Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. + /// 'none': no reduction will be applied, 'mean': the sum of the output will + /// be divided by the number of elements in the output, 'sum': the output will + /// be summed. Default: 'mean' + TORCH_ARG(reduction_t, reduction) = torch::kMean; + /// Specifies the threshold at which to change between L1 and L2 loss. + /// If beta is not specified, a value of 1.0 will be used. + /// Default: nullopt + TORCH_ARG(std::optional, beta) = std::nullopt; +}; + +namespace functional { +/// Options for `torch::nn::functional::smooth_l1_loss`. +/// +/// See the documentation for `torch::nn::SmoothL1LossOptions` class to learn +/// what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::smooth_l1_loss(input, target, F::SmoothL1LossFuncOptions(torch::kNone)); +/// ``` +using SmoothL1LossFuncOptions = SmoothL1LossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `HuberLoss` module. +/// +/// Example: +/// ``` +/// HuberLoss model(HuberLossOptions().reduction(torch::kNone).delta(0.5)); +/// ``` +struct TORCH_API HuberLossOptions { + typedef std::variant + reduction_t; + + TORCH_OPTIONS_CTOR_VARIANT_ARG3( + HuberLossOptions, + reduction, + kNone, + kMean, + kSum) + + /// Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. + /// 'none': no reduction will be applied, 'mean': the sum of the output will + /// be divided by the number of elements in the output, 'sum': the output will + /// be summed. Default: 'mean' + TORCH_ARG(reduction_t, reduction) = torch::kMean; + /// Specifies the threshold at which to change between L1 and L2 loss. + /// Default: 1.0 + TORCH_ARG(double, delta) = 1.0; +}; + +namespace functional { +/// Options for `torch::nn::functional::huber_loss`. +/// +/// See the documentation for `torch::nn::HuberLossOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::huber_loss(input, target, F::HuberLossFuncOptions(torch::kNone)); +/// ``` +using HuberLossFuncOptions = HuberLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `PoissonNLLLoss` module. +/// +/// Example: +/// ``` +/// PoissonNLLLoss +/// model(PoissonNLLLossOptions().log_input(false).full(true).eps(0.42).reduction(torch::kSum)); +/// ``` +struct TORCH_API PoissonNLLLossOptions { + typedef std::variant + reduction_t; + + /// if true the loss is computed as `exp(input) - target * input`, + /// if false the loss is `input - target * log(input + eps)`. + TORCH_ARG(bool, log_input) = true; + /// whether to compute full loss, i.e. to add the Stirling approximation term + /// target * log(target) - target + 0.5 * log(2 * pi * target). + TORCH_ARG(bool, full) = false; + /// Small value to avoid evaluation of `log(0)` when `log_input = false`. + /// Default: 1e-8 + TORCH_ARG(double, eps) = 1e-8; + /// Specifies the reduction to apply to the output. Default: Mean + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::poisson_nll_loss`. +/// +/// See the documentation for `torch::nn::PoissonNLLLossOptions` class to learn +/// what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::poisson_nll_loss(input, target, +/// F::PoissonNLLLossFuncOptions().reduction(torch::kNone)); +/// ``` +using PoissonNLLLossFuncOptions = PoissonNLLLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `MarginRankingLoss` module. +/// +/// Example: +/// ``` +/// MarginRankingLoss +/// model(MarginRankingLossOptions().margin(0.5).reduction(torch::kSum)); +/// ``` +struct TORCH_API MarginRankingLossOptions { + typedef std::variant + reduction_t; + + /// Has a default value of `0`. + TORCH_ARG(double, margin) = 0; + /// Specifies the reduction to apply to the output. Default: Mean + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::margin_ranking_loss`. +/// +/// See the documentation for `torch::nn::MarginRankingLossOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::margin_ranking_loss(input1, input2, target, +/// F::MarginRankingLossFuncOptions().margin(0.5).reduction(torch::kSum)); +/// ``` +using MarginRankingLossFuncOptions = MarginRankingLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `NLLLoss` module. +/// +/// Example: +/// ``` +/// NLLLoss model(NLLLossOptions().ignore_index(-100).reduction(torch::kMean)); +/// ``` +struct TORCH_API NLLLossOptions { + typedef std::variant + reduction_t; + + /// A manual rescaling weight given to each + /// class. If given, it has to be a Tensor of size `C`. Otherwise, it is + /// treated as if having all ones. + TORCH_ARG(Tensor, weight); + /// Specifies a target value that is ignored + /// and does not contribute to the input gradient. + TORCH_ARG(int64_t, ignore_index) = -100; + /// Specifies the reduction to apply to the output. Default: Mean + TORCH_ARG(reduction_t, reduction) = torch::kMean; +}; + +namespace functional { +/// Options for `torch::nn::functional::nll_loss`. +/// +/// See the documentation for `torch::nn::NLLLossOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::nll_loss(input, target, +/// F::NLLLossFuncOptions().ignore_index(-100).reduction(torch::kMean)); +/// ``` +using NLLLossFuncOptions = NLLLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `CrossEntropyLoss` module. +/// +/// Example: +/// ``` +/// CrossEntropyLoss +/// model(CrossEntropyLossOptions().ignore_index(-100).reduction(torch::kMean)); +/// ``` +struct TORCH_API CrossEntropyLossOptions { + typedef std::variant + reduction_t; + + /// A manual rescaling weight given to each class. If given, has to be a + /// Tensor of size C + TORCH_ARG(Tensor, weight); + /// Specifies a target value that is ignored + /// and does not contribute to the input gradient. + TORCH_ARG(int64_t, ignore_index) = -100; + /// Specifies the reduction to apply to the output. Default: Mean + TORCH_ARG(reduction_t, reduction) = torch::kMean; + /// Specifies the amount of smoothing when computing the loss. Default: 0.0 + TORCH_ARG(double, label_smoothing) = 0.0; +}; + +namespace functional { +/// Options for `torch::nn::functional::cross_entropy`. +/// +/// See the documentation for `torch::nn::CrossEntropyLossOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::cross_entropy(input, target, +/// F::CrossEntropyFuncOptions().ignore_index(-100).reduction(torch::kMean)); +/// ``` +using CrossEntropyFuncOptions = CrossEntropyLossOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `BCEWithLogitsLoss` module. +/// +/// Example: +/// ``` +/// BCEWithLogitsLoss +/// model(BCEWithLogitsLossOptions().reduction(torch::kNone).weight(weight)); +/// ``` +struct TORCH_API BCEWithLogitsLossOptions { + typedef std::variant + reduction_t; + /// A manual rescaling weight given to the loss of each batch element. + /// If given, has to be a Tensor of size `nbatch`. + TORCH_ARG(Tensor, weight); + /// Specifies the reduction to apply to the output. Default: Mean + TORCH_ARG(reduction_t, reduction) = torch::kMean; + /// A weight of positive examples. + /// Must be a vector with length equal to the number of classes. + TORCH_ARG(Tensor, pos_weight); +}; + +namespace functional { +/// Options for `torch::nn::functional::binary_cross_entropy_with_logits`. +/// +/// See the documentation for `torch::nn::BCEWithLogitsLossOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::binary_cross_entropy_with_logits(input, target, +/// F::BinaryCrossEntropyWithLogitsFuncOptions().pos_weight(pos_weight).reduction(torch::kSum)); +/// ``` +using BinaryCrossEntropyWithLogitsFuncOptions = BCEWithLogitsLossOptions; +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/normalization.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/normalization.h new file mode 100644 index 0000000000000000000000000000000000000000..ded94404a67845d853482a26519c9e3b89ab4ecc --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/normalization.h @@ -0,0 +1,190 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn { + +/// Options for the `LayerNorm` module. +/// +/// Example: +/// ``` +/// LayerNorm model(LayerNormOptions({2, +/// 2}).elementwise_affine(false).eps(2e-5)); +/// ``` +struct TORCH_API LayerNormOptions { + /* implicit */ LayerNormOptions(std::vector normalized_shape); + /// input shape from an expected input. + TORCH_ARG(std::vector, normalized_shape); + /// a value added to the denominator for numerical stability. ``Default: + /// 1e-5``. + TORCH_ARG(double, eps) = 1e-5; + /// a boolean value that when set to ``true``, this module + /// has learnable per-element affine parameters initialized to ones (for + /// weights) and zeros (for biases). ``Default: true``. + TORCH_ARG(bool, elementwise_affine) = true; +}; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::layer_norm`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::layer_norm(input, F::LayerNormFuncOptions({2, 2}).eps(2e-5)); +/// ``` +struct TORCH_API LayerNormFuncOptions { + /* implicit */ LayerNormFuncOptions(std::vector normalized_shape); + /// input shape from an expected input. + TORCH_ARG(std::vector, normalized_shape); + + TORCH_ARG(Tensor, weight); + + TORCH_ARG(Tensor, bias); + + /// a value added to the denominator for numerical stability. ``Default: + /// 1e-5``. + TORCH_ARG(double, eps) = 1e-5; +}; + +} // namespace functional + +// ============================================================================ + +/// Options for the `LocalResponseNorm` module. +/// +/// Example: +/// ``` +/// LocalResponseNorm +/// model(LocalResponseNormOptions(2).alpha(0.0002).beta(0.85).k(2.)); +/// ``` +struct TORCH_API LocalResponseNormOptions { + /* implicit */ LocalResponseNormOptions(int64_t size) : size_(size) {} + /// amount of neighbouring channels used for normalization + TORCH_ARG(int64_t, size); + + /// multiplicative factor. Default: 1e-4 + TORCH_ARG(double, alpha) = 1e-4; + + /// exponent. Default: 0.75 + TORCH_ARG(double, beta) = 0.75; + + /// additive factor. Default: 1 + TORCH_ARG(double, k) = 1.; +}; + +namespace functional { +/// Options for `torch::nn::functional::local_response_norm`. +/// +/// See the documentation for `torch::nn::LocalResponseNormOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::local_response_norm(x, F::LocalResponseNormFuncOptions(2)); +/// ``` +using LocalResponseNormFuncOptions = LocalResponseNormOptions; +} // namespace functional + +// ============================================================================ + +/// Options for the `CrossMapLRN2d` module. +/// +/// Example: +/// ``` +/// CrossMapLRN2d model(CrossMapLRN2dOptions(3).alpha(1e-5).beta(0.1).k(10)); +/// ``` +struct TORCH_API CrossMapLRN2dOptions { + CrossMapLRN2dOptions(int64_t size); + + TORCH_ARG(int64_t, size); + + TORCH_ARG(double, alpha) = 1e-4; + + TORCH_ARG(double, beta) = 0.75; + + TORCH_ARG(int64_t, k) = 1; +}; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::normalize`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::normalize(input, F::NormalizeFuncOptions().p(1).dim(-1)); +/// ``` +struct TORCH_API NormalizeFuncOptions { + /// The exponent value in the norm formulation. Default: 2.0 + TORCH_ARG(double, p) = 2.0; + /// The dimension to reduce. Default: 1 + TORCH_ARG(int64_t, dim) = 1; + /// Small value to avoid division by zero. Default: 1e-12 + TORCH_ARG(double, eps) = 1e-12; + /// the output tensor. If `out` is used, this + /// operation won't be differentiable. + TORCH_ARG(std::optional, out) = std::nullopt; +}; + +} // namespace functional + +// ============================================================================ + +/// Options for the `GroupNorm` module. +/// +/// Example: +/// ``` +/// GroupNorm model(GroupNormOptions(2, 2).eps(2e-5).affine(false)); +/// ``` +struct TORCH_API GroupNormOptions { + /* implicit */ GroupNormOptions(int64_t num_groups, int64_t num_channels); + + /// number of groups to separate the channels into + TORCH_ARG(int64_t, num_groups); + /// number of channels expected in input + TORCH_ARG(int64_t, num_channels); + /// a value added to the denominator for numerical stability. Default: 1e-5 + TORCH_ARG(double, eps) = 1e-5; + /// a boolean value that when set to ``true``, this module + /// has learnable per-channel affine parameters initialized to ones (for + /// weights) and zeros (for biases). Default: ``true``. + TORCH_ARG(bool, affine) = true; +}; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::group_norm`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::group_norm(input, F::GroupNormFuncOptions(2).eps(2e-5)); +/// ``` +struct TORCH_API GroupNormFuncOptions { + /* implicit */ GroupNormFuncOptions(int64_t num_groups); + + /// number of groups to separate the channels into + TORCH_ARG(int64_t, num_groups); + + TORCH_ARG(Tensor, weight); + + TORCH_ARG(Tensor, bias); + + /// a value added to the denominator for numerical stability. Default: 1e-5 + TORCH_ARG(double, eps) = 1e-5; +}; + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/padding.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/padding.h new file mode 100644 index 0000000000000000000000000000000000000000..82da24b20085957129b5cb1022051fc7ad19a1f3 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/padding.h @@ -0,0 +1,217 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace torch::nn { + +/// Options for a `D`-dimensional ReflectionPad module. +template +struct TORCH_API ReflectionPadOptions { + ReflectionPadOptions(ExpandingArray padding) : padding_(padding) {} + + /// The size of the padding. + /// If it is `int`, uses the same padding in all boundaries. + /// If it is a 2-`tuple` (for ReflectionPad1d), uses (padding_left, + /// padding_right). If it is a 4-`tuple` (for ReflectionPad2d), uses + /// (padding_left, padding_right, padding_top, padding_bottom). If it is a + /// 6-`tuple` (for ReflectionPad3d), uses (padding_left, padding_right, + /// padding_top, padding_bottom, padding_front, padding_back). + + TORCH_ARG(ExpandingArray, padding); +}; + +/// `ReflectionPadOptions` specialized for the `ReflectionPad1d` module. +/// +/// Example: +/// ``` +/// ReflectionPad1d model(ReflectionPad1dOptions({3, 1})); +/// ``` +using ReflectionPad1dOptions = ReflectionPadOptions<1>; + +/// `ReflectionPadOptions` specialized for the `ReflectionPad2d` module. +/// +/// Example: +/// ``` +/// ReflectionPad2d model(ReflectionPad2dOptions({1, 1, 2, 0})); +/// ``` +using ReflectionPad2dOptions = ReflectionPadOptions<2>; + +/// `ReflectionPadOptions` specialized for the `ReflectionPad3d` module. +/// +/// Example: +/// ``` +/// ReflectionPad3d model(ReflectionPad3dOptions({1, 1, 2, 0, 1, 1})); +/// ``` +using ReflectionPad3dOptions = ReflectionPadOptions<3>; + +// ============================================================================ + +/// Options for a `D`-dimensional ReplicationPad module. +template +struct TORCH_API ReplicationPadOptions { + ReplicationPadOptions(ExpandingArray padding) : padding_(padding) {} + + /// The size of the padding. + /// - If it is `int`, uses the same padding in all boundaries. + /// - If it is a 2-`tuple` (for ReplicationPad1d), uses (padding_left, + /// padding_right). + /// - If it is a 4-`tuple` (for ReplicationPad2d), uses (padding_left, + /// padding_right, padding_top, padding_bottom). + /// - If it is a 6-`tuple` (for ReplicationPad3d), uses + /// (padding_left, padding_right, padding_top, padding_bottom, + /// padding_front, padding_back). + TORCH_ARG(ExpandingArray, padding); +}; + +/// `ReplicationPadOptions` specialized for the `ReplicationPad1d` module. +/// +/// Example: +/// ``` +/// ReplicationPad1d model(ReplicationPad1dOptions({3, 1})); +/// ``` +using ReplicationPad1dOptions = ReplicationPadOptions<1>; + +/// `ReplicationPadOptions` specialized for the `ReplicationPad2d` module. +/// +/// Example: +/// ``` +/// ReplicationPad2d model(ReplicationPad2dOptions({1, 1, 2, 0})); +/// ``` +using ReplicationPad2dOptions = ReplicationPadOptions<2>; + +/// `ReplicationPadOptions` specialized for the `ReplicationPad3d` module. +/// +/// Example: +/// ``` +/// ReplicationPad3d model(ReplicationPad3dOptions({1, 2, 1, 2, 1, 2})); +/// ``` +using ReplicationPad3dOptions = ReplicationPadOptions<3>; + +// ============================================================================ + +template +struct TORCH_API ZeroPadOptions { + ZeroPadOptions(ExpandingArray padding) : padding_(padding) {} + + /// The size of the padding. + /// - If it is `int`, uses the same padding in all boundaries. + /// - If it is a 2-`tuple` (for ZeroPad1d), uses (padding_left, + /// padding_right). + /// - If it is a 4-`tuple` (for ZeroPad2d), uses (padding_left, padding_right, + /// padding_top, padding_bottom). + /// - If it is a 6-`tuple` (for ZeroPad3d), uses + /// (padding_left, padding_right, padding_top, padding_bottom, + /// padding_front, padding_back). + TORCH_ARG(ExpandingArray, padding); +}; + +/// `ZeroPadOptions` specialized for the `ZeroPad1d` module. +/// +/// Example: +/// ``` +/// ConstantPad1d model(ConstantPad1dOptions({3, 1}); +/// ``` +using ZeroPad1dOptions = ZeroPadOptions<1>; + +/// `ZeroPadOptions` specialized for the `ZeroPad2d` module. +/// +/// Example: +/// ``` +/// ConstantPad2d model(ConstantPad2dOptions({1, 1, 2, 0}); +/// ``` +using ZeroPad2dOptions = ZeroPadOptions<2>; + +/// `ZeroPadOptions` specialized for the `ZeroPad3d` module. +/// +/// Example: +/// ``` +/// ConstantPad3d model(ConstantPad3dOptions({1, 2, 1, 2, 1, 2}); +/// ``` +using ZeroPad3dOptions = ZeroPadOptions<3>; + +// ============================================================================ + +/// Options for a `D`-dimensional ConstantPad module. +template +struct TORCH_API ConstantPadOptions { + ConstantPadOptions(ExpandingArray padding, double value) + : padding_(padding), value_(value) {} + + /// The size of the padding. + /// - If it is `int`, uses the same padding in all boundaries. + /// - If it is a 2-`tuple` (for ConstantPad1d), uses (padding_left, + /// padding_right). + /// - If it is a 4-`tuple` (for ConstantPad2d), uses (padding_left, + /// padding_right, padding_top, padding_bottom). + /// - If it is a 6-`tuple` (for ConstantPad3d), uses + /// (padding_left, padding_right, padding_top, padding_bottom, + /// padding_front, padding_back). + TORCH_ARG(ExpandingArray, padding); + + /// Fill value for constant padding. + TORCH_ARG(double, value); +}; + +/// `ConstantPadOptions` specialized for the `ConstantPad1d` module. +/// +/// Example: +/// ``` +/// ConstantPad1d model(ConstantPad1dOptions({3, 1}, 3.5)); +/// ``` +using ConstantPad1dOptions = ConstantPadOptions<1>; + +/// `ConstantPadOptions` specialized for the `ConstantPad2d` module. +/// +/// Example: +/// ``` +/// ConstantPad2d model(ConstantPad2dOptions({3, 0, 2, 1}, 3.5)); +/// ``` +using ConstantPad2dOptions = ConstantPadOptions<2>; + +/// `ConstantPadOptions` specialized for the `ConstantPad3d` module. +/// +/// Example: +/// ``` +/// ConstantPad3d model(ConstantPad3dOptions({1, 2, 1, 2, 1, 2}, 3.5)); +/// ``` +using ConstantPad3dOptions = ConstantPadOptions<3>; + +// ============================================================================ + +namespace functional { + +/// Options for `torch::nn::functional::pad`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::pad(input, F::PadFuncOptions({1, 2, 2, 1, 1, +/// 2}).mode(torch::kReplicate)); +/// ``` +struct TORCH_API PadFuncOptions { + typedef std::variant< + enumtype::kConstant, + enumtype::kReflect, + enumtype::kReplicate, + enumtype::kCircular> + mode_t; + + PadFuncOptions(std::vector pad); + + /// m-elements tuple, where m/2 <= input dimensions and m is even. + TORCH_ARG(std::vector, pad); + + /// "constant", "reflect", "replicate" or "circular". Default: "constant" + TORCH_ARG(mode_t, mode) = torch::kConstant; + + /// fill value for "constant" padding. Default: 0 + TORCH_ARG(double, value) = 0; +}; + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/pixelshuffle.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/pixelshuffle.h new file mode 100644 index 0000000000000000000000000000000000000000..11413aa2b36e2e9ac1eecbf002a2dc7998f29ddb --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/pixelshuffle.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include + +namespace torch::nn { + +/// Options for the `PixelShuffle` module. +/// +/// Example: +/// ``` +/// PixelShuffle model(PixelShuffleOptions(5)); +/// ``` +struct TORCH_API PixelShuffleOptions { + PixelShuffleOptions(int64_t upscale_factor) + : upscale_factor_(upscale_factor) {} + + /// Factor to increase spatial resolution by + TORCH_ARG(int64_t, upscale_factor); +}; + +/// Options for the `PixelUnshuffle` module. +/// +/// Example: +/// ``` +/// PixelUnshuffle model(PixelUnshuffleOptions(5)); +/// ``` +struct TORCH_API PixelUnshuffleOptions { + /* implicit */ PixelUnshuffleOptions(int64_t downscale_factor) + : downscale_factor_(downscale_factor) {} + + /// Factor to decrease spatial resolution by + TORCH_ARG(int64_t, downscale_factor); +}; + +namespace functional { +/// Options for `torch::nn::functional::pixel_shuffle`. +/// +/// See the documentation for `torch::nn::PixelShuffleOptions` class to learn +/// what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::pixel_shuffle(x, F::PixelShuffleFuncOptions(2)); +/// ``` +using PixelShuffleFuncOptions = PixelShuffleOptions; + +/// Options for `torch::nn::functional::pixel_unshuffle`. +/// +/// See the documentation for `torch::nn::PixelUnshuffleOptions` class to learn +/// what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::pixel_unshuffle(x, F::PixelUnshuffleFuncOptions(2)); +/// ``` +using PixelUnshuffleFuncOptions = PixelUnshuffleOptions; +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/pooling.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/pooling.h new file mode 100644 index 0000000000000000000000000000000000000000..0963692dae8277526acb654f6e116b58a4a76eaa --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/pooling.h @@ -0,0 +1,594 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn { + +/// Options for a `D`-dimensional avgpool module. +template +struct AvgPoolOptions { + AvgPoolOptions(ExpandingArray kernel_size) + : kernel_size_(kernel_size), stride_(kernel_size) {} + + /// the size of the window to take an average over + TORCH_ARG(ExpandingArray, kernel_size); + + /// the stride of the window. Default value is `kernel_size` + TORCH_ARG(ExpandingArray, stride); + + /// implicit zero padding to be added on both sides + TORCH_ARG(ExpandingArray, padding) = 0; + + /// when True, will use `ceil` instead of `floor` to compute the output shape + TORCH_ARG(bool, ceil_mode) = false; + + /// when True, will include the zero-padding in the averaging calculation + TORCH_ARG(bool, count_include_pad) = true; + + /// if specified, it will be used as divisor, otherwise size of the pooling + /// region will be used. + + TORCH_ARG(std::optional, divisor_override) = std::nullopt; +}; + +/// `AvgPoolOptions` specialized for the `AvgPool1d` module. +/// +/// Example: +/// ``` +/// AvgPool1d model(AvgPool1dOptions(3).stride(2)); +/// ``` +using AvgPool1dOptions = AvgPoolOptions<1>; + +/// `AvgPoolOptions` specialized for the `AvgPool2d` module. +/// +/// Example: +/// ``` +/// AvgPool2d model(AvgPool2dOptions({3, 2}).stride({2, 2})); +/// ``` +using AvgPool2dOptions = AvgPoolOptions<2>; + +/// `AvgPoolOptions` specialized for the `AvgPool3d` module. +/// +/// Example: +/// ``` +/// AvgPool3d model(AvgPool3dOptions(5).stride(2)); +/// ``` +using AvgPool3dOptions = AvgPoolOptions<3>; + +namespace functional { +/// Options for `torch::nn::functional::avg_pool1d`. +/// +/// See the documentation for `torch::nn::AvgPool1dOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::avg_pool1d(x, F::AvgPool1dFuncOptions(3).stride(2)); +/// ``` +using AvgPool1dFuncOptions = AvgPool1dOptions; +} // namespace functional + +namespace functional { +/// Options for `torch::nn::functional::avg_pool2d`. +/// +/// See the documentation for `torch::nn::AvgPool2dOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::avg_pool2d(x, F::AvgPool2dFuncOptions(3).stride(2)); +/// ``` +using AvgPool2dFuncOptions = AvgPool2dOptions; +} // namespace functional + +namespace functional { +/// Options for `torch::nn::functional::avg_pool3d`. +/// +/// See the documentation for `torch::nn::AvgPool3dOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::avg_pool3d(x, F::AvgPool3dFuncOptions(3).stride(2)); +/// ``` +using AvgPool3dFuncOptions = AvgPool3dOptions; +} // namespace functional + +// ============================================================================ + +/// Options for a `D`-dimensional maxpool module. +template +struct MaxPoolOptions { + MaxPoolOptions(ExpandingArray kernel_size) + : kernel_size_(kernel_size), stride_(kernel_size) {} + + /// the size of the window to take a max over + TORCH_ARG(ExpandingArray, kernel_size); + + /// the stride of the window. Default value is `kernel_size + TORCH_ARG(ExpandingArray, stride); + + /// implicit zero padding to be added on both sides + TORCH_ARG(ExpandingArray, padding) = 0; + + /// a parameter that controls the stride of elements in the window + TORCH_ARG(ExpandingArray, dilation) = 1; + + /// when True, will use `ceil` instead of `floor` to compute the output shape + TORCH_ARG(bool, ceil_mode) = false; +}; + +/// `MaxPoolOptions` specialized for the `MaxPool1d` module. +/// +/// Example: +/// ``` +/// MaxPool1d model(MaxPool1dOptions(3).stride(2)); +/// ``` +using MaxPool1dOptions = MaxPoolOptions<1>; + +/// `MaxPoolOptions` specialized for the `MaxPool2d` module. +/// +/// Example: +/// ``` +/// MaxPool2d model(MaxPool2dOptions({3, 2}).stride({2, 2})); +/// ``` +using MaxPool2dOptions = MaxPoolOptions<2>; + +/// `MaxPoolOptions` specialized for the `MaxPool3d` module. +/// +/// Example: +/// ``` +/// MaxPool3d model(MaxPool3dOptions(3).stride(2)); +/// ``` +using MaxPool3dOptions = MaxPoolOptions<3>; + +namespace functional { +/// Options for `torch::nn::functional::max_pool1d` and +/// `torch::nn::functional::max_pool1d_with_indices`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_pool1d(x, F::MaxPool1dFuncOptions(3).stride(2)); +/// ``` +using MaxPool1dFuncOptions = MaxPool1dOptions; +} // namespace functional + +namespace functional { +/// Options for `torch::nn::functional::max_pool2d` and +/// `torch::nn::functional::max_pool2d_with_indices`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_pool2d(x, F::MaxPool2dFuncOptions(3).stride(2)); +/// ``` +using MaxPool2dFuncOptions = MaxPool2dOptions; +} // namespace functional + +namespace functional { +/// Options for `torch::nn::functional::max_pool3d` and +/// `torch::nn::functional::max_pool3d_with_indices`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_pool3d(x, F::MaxPool3dFuncOptions(3).stride(2)); +/// ``` +using MaxPool3dFuncOptions = MaxPool3dOptions; +} // namespace functional + +// ============================================================================ + +/// Options for a `D`-dimensional adaptive maxpool module. +template +struct AdaptiveMaxPoolOptions { + AdaptiveMaxPoolOptions(output_size_t output_size) + : output_size_(output_size) {} + + /// the target output size + TORCH_ARG(output_size_t, output_size); +}; + +/// `AdaptiveMaxPoolOptions` specialized for the `AdaptiveMaxPool1d` module. +/// +/// Example: +/// ``` +/// AdaptiveMaxPool1d model(AdaptiveMaxPool1dOptions(3)); +/// ``` +using AdaptiveMaxPool1dOptions = AdaptiveMaxPoolOptions>; + +/// `AdaptiveMaxPoolOptions` specialized for the `AdaptiveMaxPool2d` module. +/// +/// Example: +/// ``` +/// AdaptiveMaxPool2d model(AdaptiveMaxPool2dOptions({3, 2})); +/// ``` +using AdaptiveMaxPool2dOptions = + AdaptiveMaxPoolOptions>; + +/// `AdaptiveMaxPoolOptions` specialized for the `AdaptiveMaxPool3d` module. +/// +/// Example: +/// ``` +/// AdaptiveMaxPool3d model(AdaptiveMaxPool3dOptions(3)); +/// ``` +using AdaptiveMaxPool3dOptions = + AdaptiveMaxPoolOptions>; + +namespace functional { +/// Options for `torch::nn::functional::adaptive_max_pool1d` and +/// `torch::nn::functional::adaptive_max_pool1d_with_indices` +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_max_pool1d(x, F::AdaptiveMaxPool1dFuncOptions(3)); +/// ``` +using AdaptiveMaxPool1dFuncOptions = AdaptiveMaxPool1dOptions; +} // namespace functional + +namespace functional { +/// Options for `torch::nn::functional::adaptive_max_pool2d` and +/// `torch::nn::functional::adaptive_max_pool2d_with_indices` +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_max_pool2d(x, F::AdaptiveMaxPool2dFuncOptions(3)); +/// ``` +using AdaptiveMaxPool2dFuncOptions = AdaptiveMaxPool2dOptions; +} // namespace functional + +namespace functional { +/// Options for `torch::nn::functional::adaptive_max_pool3d` and +/// `torch::nn::functional::adaptive_max_pool3d_with_indices` +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_max_pool3d(x, F::AdaptiveMaxPool3dFuncOptions(3)); +/// ``` +using AdaptiveMaxPool3dFuncOptions = AdaptiveMaxPool3dOptions; +} // namespace functional + +// ============================================================================ + +/// Options for a `D`-dimensional adaptive avgpool module. +template +struct AdaptiveAvgPoolOptions { + AdaptiveAvgPoolOptions(output_size_t output_size) + : output_size_(output_size) {} + + /// the target output size + TORCH_ARG(output_size_t, output_size); +}; + +/// `AdaptiveAvgPoolOptions` specialized for the `AdaptiveAvgPool1d` module. +/// +/// Example: +/// ``` +/// AdaptiveAvgPool1d model(AdaptiveAvgPool1dOptions(5)); +/// ``` +using AdaptiveAvgPool1dOptions = AdaptiveAvgPoolOptions>; + +/// `AdaptiveAvgPoolOptions` specialized for the `AdaptiveAvgPool2d` module. +/// +/// Example: +/// ``` +/// AdaptiveAvgPool2d model(AdaptiveAvgPool2dOptions({3, 2})); +/// ``` +using AdaptiveAvgPool2dOptions = + AdaptiveAvgPoolOptions>; + +/// `AdaptiveAvgPoolOptions` specialized for the `AdaptiveAvgPool3d` module. +/// +/// Example: +/// ``` +/// AdaptiveAvgPool3d model(AdaptiveAvgPool3dOptions(3)); +/// ``` +using AdaptiveAvgPool3dOptions = + AdaptiveAvgPoolOptions>; + +namespace functional { +/// Options for `torch::nn::functional::adaptive_avg_pool1d`. +/// +/// See the documentation for `torch::nn::AdaptiveAvgPool1dOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_avg_pool1d(x, F::AdaptiveAvgPool1dFuncOptions(3)); +/// ``` +using AdaptiveAvgPool1dFuncOptions = AdaptiveAvgPool1dOptions; +} // namespace functional + +namespace functional { +/// Options for `torch::nn::functional::adaptive_avg_pool2d`. +/// +/// See the documentation for `torch::nn::AdaptiveAvgPool2dOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_avg_pool2d(x, F::AdaptiveAvgPool2dFuncOptions(3)); +/// ``` +using AdaptiveAvgPool2dFuncOptions = AdaptiveAvgPool2dOptions; +} // namespace functional + +namespace functional { +/// Options for `torch::nn::functional::adaptive_avg_pool3d`. +/// +/// See the documentation for `torch::nn::AdaptiveAvgPool3dOptions` class to +/// learn what arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::adaptive_avg_pool3d(x, F::AdaptiveAvgPool3dFuncOptions(3)); +/// ``` +using AdaptiveAvgPool3dFuncOptions = AdaptiveAvgPool3dOptions; +} // namespace functional + +// ============================================================================ + +/// Options for a `D`-dimensional maxunpool module. +template +struct MaxUnpoolOptions { + MaxUnpoolOptions(ExpandingArray kernel_size) + : kernel_size_(kernel_size), stride_(kernel_size) {} + + /// the size of the window to take a max over + TORCH_ARG(ExpandingArray, kernel_size); + + /// the stride of the window. Default value is `kernel_size + TORCH_ARG(ExpandingArray, stride); + + /// implicit zero padding to be added on both sides + TORCH_ARG(ExpandingArray, padding) = 0; +}; + +/// `MaxUnpoolOptions` specialized for the `MaxUnpool1d` module. +/// +/// Example: +/// ``` +/// MaxUnpool1d model(MaxUnpool1dOptions(3).stride(2).padding(1)); +/// ``` +using MaxUnpool1dOptions = MaxUnpoolOptions<1>; + +/// `MaxUnpoolOptions` specialized for the `MaxUnpool2d` module. +/// +/// Example: +/// ``` +/// MaxUnpool2d model(MaxUnpool2dOptions(3).stride(2).padding(1)); +/// ``` +using MaxUnpool2dOptions = MaxUnpoolOptions<2>; + +/// `MaxUnpoolOptions` specialized for the `MaxUnpool3d` module. +/// +/// Example: +/// ``` +/// MaxUnpool3d model(MaxUnpool3dOptions(3).stride(2).padding(1)); +/// ``` +using MaxUnpool3dOptions = MaxUnpoolOptions<3>; + +// ============================================================================ + +namespace functional { + +/// Options for a `D`-dimensional maxunpool functional. +template +struct MaxUnpoolFuncOptions { + MaxUnpoolFuncOptions(ExpandingArray kernel_size) + : kernel_size_(kernel_size), stride_(kernel_size) {} + + /// the size of the window to take a max over + TORCH_ARG(ExpandingArray, kernel_size); + + /// the stride of the window. Default value is `kernel_size + TORCH_ARG(ExpandingArray, stride); + + /// implicit zero padding to be added on both sides + TORCH_ARG(ExpandingArray, padding) = 0; + + /// the targeted output size + TORCH_ARG(std::optional>, output_size) = std::nullopt; +}; + +/// `MaxUnpoolFuncOptions` specialized for +/// `torch::nn::functional::max_unpool1d`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_unpool1d(x, indices, +/// F::MaxUnpool1dFuncOptions(3).stride(2).padding(1)); +/// ``` +using MaxUnpool1dFuncOptions = MaxUnpoolFuncOptions<1>; + +/// `MaxUnpoolFuncOptions` specialized for +/// `torch::nn::functional::max_unpool2d`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_unpool2d(x, indices, +/// F::MaxUnpool2dFuncOptions(3).stride(2).padding(1)); +/// ``` +using MaxUnpool2dFuncOptions = MaxUnpoolFuncOptions<2>; + +/// `MaxUnpoolFuncOptions` specialized for +/// `torch::nn::functional::max_unpool3d`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::max_unpool3d(x, indices, F::MaxUnpool3dFuncOptions(3)); +/// ``` +using MaxUnpool3dFuncOptions = MaxUnpoolFuncOptions<3>; + +} // namespace functional + +// ============================================================================ + +/// Options for a `D`-dimensional fractional maxpool module. +template +struct FractionalMaxPoolOptions { + FractionalMaxPoolOptions(ExpandingArray kernel_size) + : kernel_size_(kernel_size) {} + + /// the size of the window to take a max over + TORCH_ARG(ExpandingArray, kernel_size); + + /// the target output size of the image + TORCH_ARG(std::optional>, output_size) = std::nullopt; + + /// If one wants to have an output size as a ratio of the input size, this + /// option can be given. This has to be a number or tuple in the range (0, 1) + using ExpandingArrayDouble = torch::ExpandingArray; + TORCH_ARG(std::optional, output_ratio) = std::nullopt; + + TORCH_ARG(torch::Tensor, _random_samples); +}; + +/// `FractionalMaxPoolOptions` specialized for the `FractionalMaxPool2d` module. +/// +/// Example: +/// ``` +/// FractionalMaxPool2d model(FractionalMaxPool2dOptions(5).output_size(1)); +/// ``` +using FractionalMaxPool2dOptions = FractionalMaxPoolOptions<2>; + +/// `FractionalMaxPoolOptions` specialized for the `FractionalMaxPool3d` module. +/// +/// Example: +/// ``` +/// FractionalMaxPool3d model(FractionalMaxPool3dOptions(5).output_size(1)); +/// ``` +using FractionalMaxPool3dOptions = FractionalMaxPoolOptions<3>; + +namespace functional { +/// Options for `torch::nn::functional::fractional_max_pool2d` and +/// `torch::nn::functional::fractional_max_pool2d_with_indices` +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::fractional_max_pool2d(x, +/// F::FractionalMaxPool2dFuncOptions(3).output_size(2)); +/// ``` +using FractionalMaxPool2dFuncOptions = FractionalMaxPool2dOptions; +} // namespace functional + +namespace functional { +/// Options for `torch::nn::functional::fractional_max_pool3d` and +/// `torch::nn::functional::fractional_max_pool3d_with_indices` +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::fractional_max_pool3d(x, +/// F::FractionalMaxPool3dFuncOptions(3).output_size(2)); +/// ``` +using FractionalMaxPool3dFuncOptions = FractionalMaxPool3dOptions; +} // namespace functional + +// ============================================================================ + +/// Options for a `D`-dimensional lppool module. +template +struct LPPoolOptions { + LPPoolOptions(double norm_type, ExpandingArray kernel_size) + : norm_type_(norm_type), + kernel_size_(kernel_size), + stride_(kernel_size) {} + + TORCH_ARG(double, norm_type); + + // the size of the window to take an average over + TORCH_ARG(ExpandingArray, kernel_size); + + // the stride of the window. Default value is `kernel_size` + TORCH_ARG(ExpandingArray, stride); + + // when True, will use `ceil` instead of `floor` to compute the output shape + TORCH_ARG(bool, ceil_mode) = false; +}; + +/// `LPPoolOptions` specialized for the `LPPool1d` module. +/// +/// Example: +/// ``` +/// LPPool1d model(LPPool1dOptions(1, 2).stride(5).ceil_mode(true)); +/// ``` +using LPPool1dOptions = LPPoolOptions<1>; + +/// `LPPoolOptions` specialized for the `LPPool2d` module. +/// +/// Example: +/// ``` +/// LPPool2d model(LPPool2dOptions(1, std::vector({3, 4})).stride({5, +/// 6}).ceil_mode(true)); +/// ``` +using LPPool2dOptions = LPPoolOptions<2>; + +/// `LPPoolOptions` specialized for the `LPPool3d` module. +/// +/// Example: +/// ``` +/// LPPool3d model(LPPool3dOptions(1, std::vector({3, 4, 5})).stride( +/// {5, 6, 7}).ceil_mode(true)); +/// ``` +using LPPool3dOptions = LPPoolOptions<3>; + +namespace functional { +/// Options for `torch::nn::functional::lp_pool1d`. +/// +/// See the documentation for `torch::nn::LPPool1dOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::lp_pool1d(x, F::LPPool1dFuncOptions(2, 3).stride(2)); +/// ``` +using LPPool1dFuncOptions = LPPool1dOptions; +} // namespace functional + +namespace functional { +/// Options for `torch::nn::functional::lp_pool2d`. +/// +/// See the documentation for `torch::nn::LPPool2dOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::lp_pool2d(x, F::LPPool2dFuncOptions(2, {2, 3}).stride(2)); +/// ``` +using LPPool2dFuncOptions = LPPool2dOptions; +} // namespace functional + +namespace functional { +/// Options for `torch::nn::functional::lp_pool3d`. +/// +/// See the documentation for `torch::nn::LPPool3dOptions` class to learn what +/// arguments are supported. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::lp_pool3d(x, F::LPPool3dFuncOptions(2, {2, 3, 4}).stride(2)); +/// ``` +using LPPool3dFuncOptions = LPPool3dOptions; +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/rnn.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/rnn.h new file mode 100644 index 0000000000000000000000000000000000000000..8607139a27ecb2a265ba86a4ac1e3e23de39673a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/rnn.h @@ -0,0 +1,234 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn { + +namespace detail { + +/// Common options for RNN, LSTM and GRU modules. +struct TORCH_API RNNOptionsBase { + typedef std::variant< + enumtype::kLSTM, + enumtype::kGRU, + enumtype::kRNN_TANH, + enumtype::kRNN_RELU> + rnn_options_base_mode_t; + + RNNOptionsBase( + rnn_options_base_mode_t mode, + int64_t input_size, + int64_t hidden_size); + + TORCH_ARG(rnn_options_base_mode_t, mode); + /// The number of features of a single sample in the input sequence `x`. + TORCH_ARG(int64_t, input_size); + /// The number of features in the hidden state `h`. + TORCH_ARG(int64_t, hidden_size); + /// The number of recurrent layers (cells) to use. + TORCH_ARG(int64_t, num_layers) = 1; + /// Whether a bias term should be added to all linear operations. + TORCH_ARG(bool, bias) = true; + /// If true, the input sequence should be provided as `(batch, sequence, + /// features)`. If false (default), the expected layout is `(sequence, batch, + /// features)`. + TORCH_ARG(bool, batch_first) = false; + /// If non-zero, adds dropout with the given probability to the output of each + /// RNN layer, except the final layer. + TORCH_ARG(double, dropout) = 0.0; + /// Whether to make the RNN bidirectional. + TORCH_ARG(bool, bidirectional) = false; + /// Cell projection dimension. If 0, projections are not added. Can only be + /// used for LSTMs. + TORCH_ARG(int64_t, proj_size) = 0; +}; + +} // namespace detail + +/// Options for the `RNN` module. +/// +/// Example: +/// ``` +/// RNN model(RNNOptions(128, +/// 64).num_layers(3).dropout(0.2).nonlinearity(torch::kTanh)); +/// ``` +struct TORCH_API RNNOptions { + typedef std::variant nonlinearity_t; + + RNNOptions(int64_t input_size, int64_t hidden_size); + + /// The number of expected features in the input `x` + TORCH_ARG(int64_t, input_size); + /// The number of features in the hidden state `h` + TORCH_ARG(int64_t, hidden_size); + /// Number of recurrent layers. E.g., setting ``num_layers=2`` + /// would mean stacking two RNNs together to form a `stacked RNN`, + /// with the second RNN taking in outputs of the first RNN and + /// computing the final results. Default: 1 + TORCH_ARG(int64_t, num_layers) = 1; + /// The non-linearity to use. Can be either ``torch::kTanh`` or + /// ``torch::kReLU``. Default: ``torch::kTanh`` + TORCH_ARG(nonlinearity_t, nonlinearity) = torch::kTanh; + /// If ``false``, then the layer does not use bias weights `b_ih` and `b_hh`. + /// Default: ``true`` + TORCH_ARG(bool, bias) = true; + /// If ``true``, then the input and output tensors are provided + /// as `(batch, seq, feature)`. Default: ``false`` + TORCH_ARG(bool, batch_first) = false; + /// If non-zero, introduces a `Dropout` layer on the outputs of each + /// RNN layer except the last layer, with dropout probability equal to + /// `dropout`. Default: 0 + TORCH_ARG(double, dropout) = 0.0; + /// If ``true``, becomes a bidirectional RNN. Default: ``false`` + TORCH_ARG(bool, bidirectional) = false; +}; + +/// Options for the `LSTM` module. +/// +/// Example: +/// ``` +/// LSTM model(LSTMOptions(2, +/// 4).num_layers(3).batch_first(false).bidirectional(true)); +/// ``` +struct TORCH_API LSTMOptions { + LSTMOptions(int64_t input_size, int64_t hidden_size); + + /// The number of expected features in the input `x` + TORCH_ARG(int64_t, input_size); + /// The number of features in the hidden state `h` + TORCH_ARG(int64_t, hidden_size); + /// Number of recurrent layers. E.g., setting ``num_layers=2`` + /// would mean stacking two LSTMs together to form a `stacked LSTM`, + /// with the second LSTM taking in outputs of the first LSTM and + /// computing the final results. Default: 1 + TORCH_ARG(int64_t, num_layers) = 1; + /// If ``false``, then the layer does not use bias weights `b_ih` and `b_hh`. + /// Default: ``true`` + TORCH_ARG(bool, bias) = true; + /// If ``true``, then the input and output tensors are provided + /// as (batch, seq, feature). Default: ``false`` + TORCH_ARG(bool, batch_first) = false; + /// If non-zero, introduces a `Dropout` layer on the outputs of each + /// LSTM layer except the last layer, with dropout probability equal to + /// `dropout`. Default: 0 + TORCH_ARG(double, dropout) = 0.0; + /// If ``true``, becomes a bidirectional LSTM. Default: ``false`` + TORCH_ARG(bool, bidirectional) = false; + /// Cell projection dimension. If 0, projections are not added + TORCH_ARG(int64_t, proj_size) = 0; +}; + +/// Options for the `GRU` module. +/// +/// Example: +/// ``` +/// GRU model(GRUOptions(2, +/// 4).num_layers(3).batch_first(false).bidirectional(true)); +/// ``` +struct TORCH_API GRUOptions { + GRUOptions(int64_t input_size, int64_t hidden_size); + + /// The number of expected features in the input `x` + TORCH_ARG(int64_t, input_size); + /// The number of features in the hidden state `h` + TORCH_ARG(int64_t, hidden_size); + /// Number of recurrent layers. E.g., setting ``num_layers=2`` + /// would mean stacking two GRUs together to form a `stacked GRU`, + /// with the second GRU taking in outputs of the first GRU and + /// computing the final results. Default: 1 + TORCH_ARG(int64_t, num_layers) = 1; + /// If ``false``, then the layer does not use bias weights `b_ih` and `b_hh`. + /// Default: ``true`` + TORCH_ARG(bool, bias) = true; + /// If ``true``, then the input and output tensors are provided + /// as (batch, seq, feature). Default: ``false`` + TORCH_ARG(bool, batch_first) = false; + /// If non-zero, introduces a `Dropout` layer on the outputs of each + /// GRU layer except the last layer, with dropout probability equal to + /// `dropout`. Default: 0 + TORCH_ARG(double, dropout) = 0.0; + /// If ``true``, becomes a bidirectional GRU. Default: ``false`` + TORCH_ARG(bool, bidirectional) = false; +}; + +namespace detail { + +/// Common options for RNNCell, LSTMCell and GRUCell modules +struct TORCH_API RNNCellOptionsBase { + RNNCellOptionsBase( + int64_t input_size, + int64_t hidden_size, + bool bias, + int64_t num_chunks); + TORCH_ARG(int64_t, input_size); + TORCH_ARG(int64_t, hidden_size); + TORCH_ARG(bool, bias); + TORCH_ARG(int64_t, num_chunks); +}; + +} // namespace detail + +/// Options for the `RNNCell` module. +/// +/// Example: +/// ``` +/// RNNCell model(RNNCellOptions(20, +/// 10).bias(false).nonlinearity(torch::kReLU)); +/// ``` +struct TORCH_API RNNCellOptions { + typedef std::variant nonlinearity_t; + + RNNCellOptions(int64_t input_size, int64_t hidden_size); + + /// The number of expected features in the input `x` + TORCH_ARG(int64_t, input_size); + /// The number of features in the hidden state `h` + TORCH_ARG(int64_t, hidden_size); + /// If ``false``, then the layer does not use bias weights `b_ih` and `b_hh`. + /// Default: ``true`` + TORCH_ARG(bool, bias) = true; + /// The non-linearity to use. Can be either ``torch::kTanh`` or + /// ``torch::kReLU``. Default: ``torch::kTanh`` + TORCH_ARG(nonlinearity_t, nonlinearity) = torch::kTanh; +}; + +/// Options for the `LSTMCell` module. +/// +/// Example: +/// ``` +/// LSTMCell model(LSTMCellOptions(20, 10).bias(false)); +/// ``` +struct TORCH_API LSTMCellOptions { + LSTMCellOptions(int64_t input_size, int64_t hidden_size); + + /// The number of expected features in the input `x` + TORCH_ARG(int64_t, input_size); + /// The number of features in the hidden state `h` + TORCH_ARG(int64_t, hidden_size); + /// If ``false``, then the layer does not use bias weights `b_ih` and `b_hh`. + /// Default: ``true`` + TORCH_ARG(bool, bias) = true; +}; + +/// Options for the `GRUCell` module. +/// +/// Example: +/// ``` +/// GRUCell model(GRUCellOptions(20, 10).bias(false)); +/// ``` +struct TORCH_API GRUCellOptions { + GRUCellOptions(int64_t input_size, int64_t hidden_size); + + /// The number of expected features in the input `x` + TORCH_ARG(int64_t, input_size); + /// The number of features in the hidden state `h` + TORCH_ARG(int64_t, hidden_size); + /// If ``false``, then the layer does not use bias weights `b_ih` and `b_hh`. + /// Default: ``true`` + TORCH_ARG(bool, bias) = true; +}; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/transformer.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/transformer.h new file mode 100644 index 0000000000000000000000000000000000000000..44878220e5982e3d827b179fd452e1aef29bc8fd --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/transformer.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace torch::nn { + +/// Options for the `Transformer` module +/// +/// Example: +/// ``` +/// TransformerOptions options; +/// TransformerOptions options(16, 4); +/// auto options = TransformerOptions().d_model(4).nhead(2).dropout(0.0); +/// ``` +struct TORCH_API TransformerOptions { + // The following constructors are commonly used + // Please don't add more unless it is proved as a common usage + TransformerOptions() = default; + TransformerOptions(int64_t d_model, int64_t nhead); + TransformerOptions( + int64_t d_model, + int64_t nhead, + int64_t num_encoder_layers, + int64_t num_decoder_layers); + + /// the number of expected features in the encoder/decoder inputs + /// (default=512) + TORCH_ARG(int64_t, d_model) = 512; + + /// the number of heads in the multiheadattention models (default=8) + TORCH_ARG(int64_t, nhead) = 8; + + /// the number of sub-encoder-layers in the encoder (default=6) + TORCH_ARG(int64_t, num_encoder_layers) = 6; + + /// the number of sub-decoder-layers in the decoder (default=6) + TORCH_ARG(int64_t, num_decoder_layers) = 6; + + /// the dimension of the feedforward network model (default=2048) + TORCH_ARG(int64_t, dim_feedforward) = 2048; + + /// the dropout value (default=0.1) + TORCH_ARG(double, dropout) = 0.1; + + /// the activation function of encoder/decoder intermediate layer + /// (default=``torch::kReLU``) + TORCH_ARG(activation_t, activation) = torch::kReLU; + + /// custom encoder (default=None) + TORCH_ARG(AnyModule, custom_encoder); + + /// custom decoder (default=None) + TORCH_ARG(AnyModule, custom_decoder); +}; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/transformercoder.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/transformercoder.h new file mode 100644 index 0000000000000000000000000000000000000000..93cf2f449fa1d581fc99ca14eeee848d80be7a2d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/transformercoder.h @@ -0,0 +1,74 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace torch::nn { + +/// Options for the `TransformerEncoder` +/// +/// Example: +/// ``` +/// TransformerEncoderLayer encoderLayer(TransformerEncoderLayerOptions(512, +/// 8).dropout(0.1)); auto options = TransformerEncoderOptions(encoderLayer, +/// 6).norm(LayerNorm(LayerNormOptions({2}))); +/// ``` +struct TORCH_API TransformerEncoderOptions { + // This constructor will keep a shallow copy of encoder_layer, so it keeps all + // the data in encoder_layer. + TransformerEncoderOptions( + TransformerEncoderLayer encoder_layer, + int64_t num_layers); + // This constructor will create a new TransformerEncoderLayer obj based on + // passed in encoder_layer_options. + TransformerEncoderOptions( + const TransformerEncoderLayerOptions& encoder_layer_options, + int64_t num_layers); + + /// transformer Encoder Layer + TORCH_ARG(TransformerEncoderLayer, encoder_layer) = nullptr; + + /// number of encoder layers + TORCH_ARG(int64_t, num_layers); + + /// normalization module + TORCH_ARG(AnyModule, norm); +}; + +/// Options for the `TransformerDecoder` module. +/// +/// Example: +/// ``` +/// TransformerDecoderLayer decoder_layer(TransformerDecoderLayerOptions(512, +/// 8).dropout(0.1)); auto options = TransformerDecoderOptions(decoder_layer, +/// 6)norm(LayerNorm(LayerNormOptions({2}))); TransformerDecoder +/// transformer_decoder(options); +/// ``` +struct TORCH_API TransformerDecoderOptions { + // This constructor will keep a ref of passed in decoder_layer, + // so it keeps all the data in decoder_layer. + TransformerDecoderOptions( + TransformerDecoderLayer decoder_layer, + int64_t num_layers); + // This constructor will create a new TransformerDecoderLayer obj, + // based on passed in decoder_layer_options. + TransformerDecoderOptions( + const TransformerDecoderLayerOptions& decoder_layer_options, + int64_t num_layers); + + /// decoder layer to be cloned + TORCH_ARG(TransformerDecoderLayer, decoder_layer) = nullptr; + + /// number of decoder layers + TORCH_ARG(int64_t, num_layers); + + /// normalization module + TORCH_ARG(AnyModule, norm); +}; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/transformerlayer.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/transformerlayer.h new file mode 100644 index 0000000000000000000000000000000000000000..b2ec50f016cb29c6805bee4570b6f829f9006ef4 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/transformerlayer.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn { + +using activation_t = std::variant< + enumtype::kReLU, + enumtype::kGELU, + std::function>; + +/// Options for the `TransformerEncoderLayer` +/// +/// Example: +/// ``` +/// auto options = TransformerEncoderLayer(512, 8).dropout(0.2); +/// ``` +struct TORCH_API TransformerEncoderLayerOptions { + /* implicit */ TransformerEncoderLayerOptions(int64_t d_model, int64_t nhead); + + /// the number of expected features in the input + TORCH_ARG(int64_t, d_model); + + /// the number of heads in the multiheadattention models + TORCH_ARG(int64_t, nhead); + + /// the dimension of the feedforward network model, default is 2048 + TORCH_ARG(int64_t, dim_feedforward) = 2048; + + /// the dropout value, default is 0.1 + TORCH_ARG(double, dropout) = 0.1; + + /// the activation function of intermediate layer, can be ``torch::kReLU``, + /// ``torch::GELU``, or a unary callable. Default: ``torch::kReLU`` + TORCH_ARG(activation_t, activation) = torch::kReLU; +}; + +// ============================================================================ + +/// Options for the `TransformerDecoderLayer` module. +/// +/// Example: +/// ``` +/// TransformerDecoderLayer model(TransformerDecoderLayerOptions(512, +/// 8).dropout(0.2)); +/// ``` +struct TORCH_API TransformerDecoderLayerOptions { + TransformerDecoderLayerOptions(int64_t d_model, int64_t nhead); + + /// number of expected features in the input + TORCH_ARG(int64_t, d_model); + + /// number of heads in the multiheadattention models + TORCH_ARG(int64_t, nhead); + + /// dimension of the feedforward network model. Default: 2048 + TORCH_ARG(int64_t, dim_feedforward) = 2048; + + /// dropout value. Default: 1 + TORCH_ARG(double, dropout) = 0.1; + + /// activation function of intermediate layer, can be ``torch::kGELU``, + /// ``torch::kReLU``, or a unary callable. Default: ``torch::kReLU`` + TORCH_ARG(activation_t, activation) = torch::kReLU; +}; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/upsampling.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/upsampling.h new file mode 100644 index 0000000000000000000000000000000000000000..10a58843ba7342b6be69eda8cefec58a4092ae67 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/upsampling.h @@ -0,0 +1,108 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace torch::nn { + +/// Options for the `Upsample` module. +/// +/// Example: +/// ``` +/// Upsample +/// model(UpsampleOptions().scale_factor(std::vector({3})).mode(torch::kLinear).align_corners(false)); +/// ``` +struct TORCH_API UpsampleOptions { + /// output spatial sizes. + TORCH_ARG(std::optional>, size) = std::nullopt; + + /// multiplier for spatial size. + TORCH_ARG(std::optional>, scale_factor) = std::nullopt; + + /// the upsampling algorithm: one of "nearest", "linear", "bilinear", + /// "bicubic" and "trilinear". Default: "nearest" + typedef std::variant< + enumtype::kNearest, + enumtype::kLinear, + enumtype::kBilinear, + enumtype::kBicubic, + enumtype::kTrilinear> + mode_t; + TORCH_ARG(mode_t, mode) = torch::kNearest; + + /// if "True", the corner pixels of the input and output tensors are + /// aligned, and thus preserving the values at those pixels. This only has + /// effect when :attr:`mode` is "linear", "bilinear", "bicubic", or + /// "trilinear". Default: "False" + TORCH_ARG(std::optional, align_corners) = std::nullopt; +}; + +namespace functional { + +/// Options for `torch::nn::functional::interpolate`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::interpolate(input, +/// F::InterpolateFuncOptions().size(std::vector({4})).mode(torch::kNearest)); +/// ``` +struct TORCH_API InterpolateFuncOptions { + typedef std::variant< + enumtype::kNearest, + enumtype::kLinear, + enumtype::kBilinear, + enumtype::kBicubic, + enumtype::kTrilinear, + enumtype::kArea, + enumtype::kNearestExact> + mode_t; + + /// output spatial sizes. + TORCH_ARG(std::optional>, size) = std::nullopt; + + /// multiplier for spatial size. + TORCH_ARG(std::optional>, scale_factor) = std::nullopt; + + /// the upsampling algorithm: one of "nearest", "linear", "bilinear", + /// "bicubic", "trilinear", "area", "nearest-exact". Default: "nearest" + TORCH_ARG(mode_t, mode) = torch::kNearest; + + /// Geometrically, we consider the pixels of the input and output as squares + /// rather than points. If set to "True", the input and output tensors are + /// aligned by the center points of their corner pixels, preserving the values + /// at the corner pixels. If set to "False", the input and output tensors + /// are aligned by the corner points of their corner pixels, and the + /// interpolation uses edge value padding for out-of-boundary values, making + /// this operation *independent* of input size when `scale_factor` is + /// kept the same. It is *required* when interpolating mode is "linear", + /// "bilinear", "bicubic" or "trilinear". Default: "False" + TORCH_ARG(std::optional, align_corners) = std::nullopt; + + /// recompute the scale_factor for use in the + /// interpolation calculation. When `scale_factor` is passed as a parameter, + /// it is used to compute the `output_size`. If `recompute_scale_factor` is + /// `true` or not specified, a new `scale_factor` will be computed based on + /// the output and input sizes for use in the interpolation computation (i.e. + /// the computation will be identical to if the computed `output_size` were + /// passed-in explicitly). Otherwise, the passed-in `scale_factor` will be + /// used in the interpolation computation. Note that when `scale_factor` is + /// floating-point, the recomputed scale_factor may differ from the one passed + /// in due to rounding and precision issues. + TORCH_ARG(std::optional, recompute_scale_factor) = std::nullopt; + + /// flag to apply anti-aliasing. Using anti-alias + /// option together with :attr:`align_corners` equals "False", interpolation + /// result would match Pillow result for downsampling operation. Supported + /// modes: "bilinear". Default: "False". + TORCH_ARG(bool, antialias) = false; +}; + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/vision.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/vision.h new file mode 100644 index 0000000000000000000000000000000000000000..617cbe9a215a719870cc675b7ece3e8f1be18bff --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/options/vision.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::nn::functional { + +/// Options for `torch::nn::functional::grid_sample`. +/// +/// Example: +/// ``` +/// namespace F = torch::nn::functional; +/// F::grid_sample(input, grid, +/// F::GridSampleFuncOptions().mode(torch::kBilinear).padding_mode(torch::kZeros).align_corners(true)); +/// ``` +struct TORCH_API GridSampleFuncOptions { + typedef std:: + variant + mode_t; + typedef std:: + variant + padding_mode_t; + + /// interpolation mode to calculate output values. Default: Bilinear + TORCH_ARG(mode_t, mode) = torch::kBilinear; + /// padding mode for outside grid values. Default: Zeros + TORCH_ARG(padding_mode_t, padding_mode) = torch::kZeros; + /// Specifies perspective to pixel as point. Default: false + TORCH_ARG(std::optional, align_corners) = std::nullopt; +}; + +} // namespace torch::nn::functional diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/parallel/data_parallel.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/parallel/data_parallel.h new file mode 100644 index 0000000000000000000000000000000000000000..924a04c5f6a1df4c946d6f3db40e090ddbdfcc40 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/parallel/data_parallel.h @@ -0,0 +1,295 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace torch::nn { + +namespace { + +// Note [Replicating Modules] +// ~~~~~~~~~~~~~~~~~~~~~~~~~~ +// +// Module replication is implemented in the following two steps: +// 1) create a module replica on each destination device using Module.clone(). +// 2) manually add a gradient edge pointing from every parameter X in every +// module replica to the same parameter X in the original module, using +// ReduceAdd as the grad_fn. +// +// ReduceAdd can ONLY be used during the backward pass of data parallel. Forward +// pass cannot use this function as it does not setup gradient function and +// history at all. Do NOT try to use ReduceAdd for any other purposes. +// +// NB: An alternative is to add Broadcast and ReduceAddCoalesce to +// torch/csrc/autograd/functions/comm.cpp as normal autograd functions, +// implement a Replicatable (like cloneable) class and add it as a friend class +// in Module.h. In the forward pass, the Replicatable could use the Broadcast +// function to replicate every module parameter and set gradient functions using +// ReduceAddCoalesce (like how it is implemented in Python). However, unlike in +// Python, where changes to Linear._parameters["weight"] would also apply to +// Linear.weight (using Linear as an example), Linear.weight and +// Linear.parameters_["weight"] are two tensor objects pointing to the same +// TensorImpl. Assigning a new tensor to Linear.parameters_["weight"] will not +// change Linear.weight. To make this work, we will have to: +// 1) force every module to also inherit from Replicatable +// 2) force every module to implement an additional function, e.g., +// Replicatable::load_params(), to pick up changes from parameters_ to their +// own member fields. +// This will be an overkill as Replicatable will only be used in data_parallel, +// not even ddp. + +// Autograd function for the replicate step in data parallel. This is only used +// in data parallel, and should not be exposed as a user API. +struct ReduceAdd : public autograd::Node { + explicit ReduceAdd(const at::Device& destination_device) + : destination_device_(destination_device) {}; + ~ReduceAdd() override = default; + + // NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved) + autograd::variable_list apply(autograd::variable_list&& inputs) override { + TORCH_CHECK( + !torch::autograd::compute_requires_grad(inputs), + "ReduceAdd can only be used during the backward pass of data parallel."); + + Tensor output = torch::zeros_like(inputs[0], {destination_device_}); + + for (auto& input : inputs) { + TORCH_CHECK( + input.sizes() == inputs[0].sizes(), + "All inputs of ReduceAdd must have the same size, but got ", + input.sizes(), + " and ", + inputs[0].sizes()); + + TORCH_CHECK( + input.dtype() == inputs[0].dtype(), + "All inputs of ReduceAdd must have the same dtype, but got ", + input.dtype(), + " and ", + inputs[0].dtype()); + + // TODO: use nccl reduce + output.add_(input.to(destination_device_)); + } + + return {std::move(output)}; + } + + private: + at::Device destination_device_; +}; + +} // namespace + +// A friend function to Module, it recursively sets gradient edges pointing from +// every parameter X in every module replica to the same parameter X in the +// original module. See [Replicating Modules] +template +void replicate_grad_edges( + const std::shared_ptr& module, + const std::vector>& replicas, + const std::vector& devices) { + for (auto& parameter : module->named_parameters(/*recurse=*/false)) { + auto grad_fn = c10::make_intrusive((*parameter).device()); + grad_fn->set_next_edges(autograd::collect_next_edges(*parameter)); + + for (const auto i : c10::irange(devices.size())) { + autograd::set_history(replicas[i]->parameters_[parameter.key()], grad_fn); + } + } + + for (auto& buffer : module->named_buffers(/*recurse=*/false)) { + if (buffer.value().requires_grad()) { + auto grad_fn = c10::make_intrusive((*buffer).device()); + grad_fn->set_next_edges(autograd::collect_next_edges(*buffer)); + + for (const auto i : c10::irange(devices.size())) { + autograd::set_history(replicas[i]->buffers_[buffer.key()], grad_fn); + } + } + } + + for (auto& child : module->children_) { + std::vector> child_replicas; + child_replicas.reserve(devices.size()); + for (auto& replica : replicas) { + child_replicas.push_back(replica->children_[child.key()]); + } + + // recursively set gradient edges for all children + replicate_grad_edges(*child, child_replicas, devices); + } +} + +namespace parallel { + +/// Replicates a module on the given list of devices. +/// A replica is created by calling `clone()` on the module. For this, the +/// module must inherit from `nn::Cloneable`, or define its own `clone()` +/// method, which is expected to perform a deep copy of the module. +template +std::vector> replicate( + const std::shared_ptr& module, + const std::vector& devices) { + std::vector> replicas; + replicas.reserve(devices.size()); + for (const auto& device : devices) { + replicas.push_back( + std::dynamic_pointer_cast(module->clone(device))); + } + // Configure gradient edges to point from replica parameters to original + // module parameters. See [Replicating Modules] + replicate_grad_edges(module, replicas, devices); + return replicas; +} + +/// Replicates a module holder on the given list of devices. +/// This method allows calling `replicate()` with a module holder, such as +/// `Linear`. +template +std::vector> replicate( + const ModuleHolder& module, + const std::vector& devices) { + auto ptrs = replicate(module.ptr(), devices); + return std::vector>(ptrs.begin(), ptrs.end()); +} + +/// Applies the given inputs to the given modules in a parallel fashion. +/// Conceptually, a thread is spawned for each `(module, input)` pair, in which +/// `forward()` is called on the module with its corresponding input. The +/// outputs of the individual calls are stored in a vector and returned. +/// +/// The first exception caught by any thread is stashed and rethrown after all +/// threads have completed their operation. +/// +/// Further remarks: +/// 1. The length of the module container must match the length of the inputs. +/// 2. If a list of devices is supplied, it must match the list of modules in +/// length. Each device will be set to the current default device during the +/// invocation of the respective module. This means any tensors allocated on the +/// default device inside the module will be constructed on this device. +template +std::vector parallel_apply( + std::vector& modules, + const std::vector& inputs, + const std::optional>& devices = std::nullopt) { + TORCH_CHECK( + modules.size() == inputs.size(), "Must have as many inputs as modules"); + if (devices) { + TORCH_CHECK( + modules.size() == devices->size(), + "Must have as many devices as modules"); + } + + std::vector outputs(modules.size()); + std::mutex mutex; + + // std::exception_ptr can be passed between threads: + // > An instance of std::exception_ptr may be passed to another function, + // > possibly on another thread, where the exception may be rethrown [...]. + // https://en.cppreference.com/w/cpp/error/exception_ptr + std::exception_ptr exception; + + at::parallel_for( + /*begin=*/0, + /*end=*/modules.size(), + /*grain_size=*/1, + [&modules, &inputs, &devices, &outputs, &mutex, &exception]( + int64_t index, int64_t stop) { + for (; index < stop; ++index) { + try { + auto output = modules[index]->forward(inputs[index]); + output = + output.to(devices ? (*devices)[index] : inputs[index].device()); + std::lock_guard lock(mutex); + outputs[index] = output; + } catch (...) { + std::lock_guard lock(mutex); + if (!exception) { + exception = std::current_exception(); + } + } + } + }); + + if (exception) { + std::rethrow_exception(exception); + } + + return outputs; +} + +/// Evaluates `module(input)` in parallel across the given `devices`. If +/// `devices` is not supplied, the invocation is parallelized across all +/// available CUDA devices. If `output_device` is supplied, the final, combined +/// tensor will be placed on this device. If not, it defaults to the first +/// device in `devices`. +/// +/// In detail, this method performs the following four distinct steps: +/// 1. *Scatter* the input to the given devices, +/// 2. *Replicate* (deep clone) the model on each device, +/// 3. *Evaluate* each module with its input on its device, +/// 4. *Gather* the outputs of each replica into a single output tensor, located +/// on the `output_device`. +template +Tensor data_parallel( + ModuleType module, + Tensor input, + std::optional> devices = std::nullopt, + std::optional output_device = std::nullopt, + int64_t dim = 0) { + if (!devices) { + const auto device_count = torch::cuda::device_count(); + TORCH_CHECK( + device_count > 0, "Expected at least one CUDA device to be available"); + devices = std::vector(); + devices->reserve(device_count); + for (const auto index : c10::irange(device_count)) { + devices->emplace_back(kCUDA, static_cast(index)); + } + } + if (!output_device) { + output_device = devices->front(); + } + + if (devices->size() == 1) { + module->to(devices->front()); + input = input.to(devices->front()); + return module->forward(std::move(input)).to(*output_device); + } + + autograd::Scatter scatter(*devices, /*chunk_sizes=*/std::nullopt, dim); + auto scattered_inputs = fmap(scatter.apply({std::move(input)})); + // Input tensor might not be big enough to scale across all available devices + if (scattered_inputs.size() < devices->size()) { + devices->resize( + scattered_inputs.size(), + Device(DeviceType::COMPILE_TIME_MAX_DEVICE_TYPES)); + } + + auto replicas = replicate(module, *devices); + auto outputs = parallel_apply(replicas, scattered_inputs, *devices); + return autograd::Gather(*output_device, dim) + .apply(fmap(std::move(outputs))) + .front(); +} + +} // namespace parallel +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/pimpl-inl.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/pimpl-inl.h new file mode 100644 index 0000000000000000000000000000000000000000..b78f14ef6f0e9e45540590dbf8b61acbfb187e02 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/pimpl-inl.h @@ -0,0 +1,76 @@ +// This class exists only to do SFINAE on abstract types `T` that are really +// `ModuleHolder`, because there's no good way to say that `T` is a +// `ModuleHolder` over some unknown type `ModuleType`. With this, you can do +// `enable_if_t>`. +struct ModuleHolderIndicator {}; + +// A type trait that is true for types that are `ModuleHolder`s. +template +using is_module_holder = + std::is_base_of>; + +template +using disable_if_module_holder_t = + std::enable_if_t::value>; + +// A collection of templates that answer the question whether a type `T` is a +// `ModuleHolder`, and if so whether its contained type is of type `C`. This is +// tricky because it is hard to short circuit in template metaprogramming. A +// naive and incorrect solution to this problem would be something like +// `disable_if::value && typename T::ContainedType == C>`. +// This would disable all types that are not `ModuleHolder`s, because even +// though the `is_module_holder::value` may be `false` for such types the +// `T::ContainedType` access would be ill-formed and thus fail the whole +// expression by the rules of SFINAE. Instead we have to use template +// specialization to statically branch on the first condition +// (`is_module_holder`) and are only then allowed to query +// `T::ContainedType` in the branch for which the condition was true. + +// Base template. +template +struct is_module_holder_of_impl; + +// False branch. `T` is not a `ModuleHolder` and thus not a `ModuleHolder` with +// contained type `C`. +template +struct is_module_holder_of_impl : std::false_type {}; + +// True branch. `T` is a `ModuleHolder` and thus we can legit access its +// `ContainedType` and compare it against `C`. +template +struct is_module_holder_of_impl + : std::is_same {}; + +// Helper template. +template +struct is_module_holder_of : is_module_holder_of_impl< + is_module_holder::value, + std::decay_t, + std::decay_t> {}; + +// A collection of templates that allow deducing the return type of the +// `forward()` method, but only if a module actually has a `forward()` method, +// and otherwise deduces to the type `void`. + +template +struct return_type_of_forward_impl; + +template +struct return_type_of_forward_impl { + using type = decltype(::std::declval().forward(::std::declval()...)); +}; + +template +struct return_type_of_forward_impl { + using type = void; +}; + +template +using return_type_of_forward = return_type_of_forward_impl< + torch::detail::has_forward::value, + C, + Args...>; + +template +using return_type_of_forward_t = + typename return_type_of_forward::type; diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/pimpl.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/pimpl.h new file mode 100644 index 0000000000000000000000000000000000000000..f28906d188d0bffad1ba09cbd093cb3c021ad664 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/pimpl.h @@ -0,0 +1,200 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace torch { +namespace detail { +// Dump all the template metaprogramming in this file. +#include +} // namespace detail + +namespace nn { + +/// A `ModuleHolder` is essentially a wrapper around `std::shared_ptr` where +/// `M` is an `nn::Module` subclass, with convenient constructors defined for +/// the kind of constructions we want to allow for our modules. +template +class ModuleHolder : torch::detail::ModuleHolderIndicator { + protected: + /// The module pointer this class wraps. + /// NOTE: Must be placed at the top of the class so that we can use it with + /// trailing return types below. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::shared_ptr impl_; + + public: + using ContainedType = Contained; + + /// Default constructs the contained module if it has a default constructor, + /// else produces a static error. + /// + /// NOTE: This uses the behavior of template + /// classes in C++ that constructors (or any methods) are only compiled when + /// actually used. + ModuleHolder() : impl_(default_construct()) { + static_assert( + std::is_default_constructible_v, + "You are trying to default construct a module which has " + "no default constructor. Use = nullptr to give it the empty state " + "(e.g. `Linear linear = nullptr;` instead of `Linear linear;`)."); + } + + /// Constructs the `ModuleHolder` with an empty contained value. Access to + /// the underlying module is not permitted and will throw an exception, until + /// a value is assigned. + /* implicit */ ModuleHolder(std::nullptr_t) : impl_(nullptr) {} + + /// Constructs the `ModuleHolder` with a contained module, forwarding all + /// arguments to its constructor. + template < + typename Head, + typename... Tail, + typename = std::enable_if_t< + !(torch::detail::is_module_holder_of::value && + (sizeof...(Tail) == 0))>> + explicit ModuleHolder(Head&& head, Tail&&... tail) + : impl_(new Contained( + std::forward(head), + std::forward(tail)...)) {} + + /// Constructs the `ModuleHolder` from a pointer to the contained type. + /// Example: `Linear(std::make_shared(...))`. + /* implicit */ ModuleHolder(std::shared_ptr module) + : impl_(std::move(module)) {} + + /// Returns true if the `ModuleHolder` contains a module, or false if it is + /// `nullptr`. + explicit operator bool() const noexcept { + return !is_empty(); + } + + /// Forwards to the contained module. + Contained* operator->() { + return get(); + } + + /// Forwards to the contained module. + const Contained* operator->() const { + return get(); + } + + /// Returns a reference to the contained module. + Contained& operator*() { + return *get(); + } + + /// Returns a const reference to the contained module. + const Contained& operator*() const { + return *get(); + } + + /// Returns a shared pointer to the underlying module. + const std::shared_ptr& ptr() const { + TORCH_CHECK(!is_empty(), "Accessing empty ModuleHolder"); + return impl_; + } + + /// Returns a pointer to the underlying module. + Contained* get() { + TORCH_CHECK(!is_empty(), "Accessing empty ModuleHolder"); + return impl_.get(); + } + + /// Returns a const pointer to the underlying module. + const Contained* get() const { + TORCH_CHECK(!is_empty(), "Accessing empty ModuleHolder"); + return impl_.get(); + } + + /// Calls the `forward()` method of the contained module. + template + auto operator()(Args&&... args) + -> torch::detail::return_type_of_forward_t { + // This will not compile if the module does not have a `forward()` method + // (as expected). + // NOTE: `std::forward` is qualified to prevent VS2017 emitting + // error C2872: 'std': ambiguous symbol + return impl_->forward(::std::forward(args)...); + } + + /// Forwards to the subscript operator of the contained module. + /// NOTE: std::forward is qualified to prevent VS2017 emitting + /// error C2872: 'std': ambiguous symbol + template + auto operator[](Arg&& arg) { + return (*impl_)[::std::forward(arg)]; + } + + /// Returns true if the `ModuleHolder` does not contain a module. + bool is_empty() const noexcept { + return impl_ == nullptr; + } + + private: + template + std::shared_ptr default_construct() { + if constexpr (std::is_default_constructible_v) { + return std::make_shared(); + } else { + return nullptr; + } + } +}; + +/// Pretty prints the given `Module` into the `ostream`. +template +std::ostream& operator<<( + std::ostream& stream, + const nn::ModuleHolder& module) { + return stream << *module; +} + +/// Serializes a `ModuleHolder` into an `OutputArchive`. +template +serialize::OutputArchive& operator<<( + serialize::OutputArchive& archive, + const nn::ModuleHolder& module) { + return archive << module.ptr(); +} + +/// Deserializes a `ModuleHolder` from an `InputArchive`. +template +serialize::InputArchive& operator>>( + serialize::InputArchive& archive, + nn::ModuleHolder& module) { + return archive >> module.ptr(); +} + +} // namespace nn +} // namespace torch + +// Workaround for CUDA 10.2 and below not allowing attribute unused on +// using declarations. +#ifdef __CUDACC__ +#define TORCH_UNUSED_EXCEPT_CUDA +#else +#define TORCH_UNUSED_EXCEPT_CUDA [[maybe_unused]] +#endif + +/// Defines a class `Name` which inherits from `nn::ModuleHolder` to provide a +/// wrapper over a `std::shared_ptr`. +/// `Impl` is a type alias for `ImplType` which provides a way to call static +/// method of `ImplType`. +#define TORCH_MODULE_IMPL(Name, ImplType) \ + class Name : public torch::nn::ModuleHolder { /* NOLINT */ \ + public: \ + using torch::nn::ModuleHolder::ModuleHolder; \ + using Impl TORCH_UNUSED_EXCEPT_CUDA = ImplType; \ + } + +/// Like `TORCH_MODULE_IMPL`, but defaults the `ImplType` name to `Impl`. +#define TORCH_MODULE(Name) TORCH_MODULE_IMPL(Name, Name##Impl) diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils.h new file mode 100644 index 0000000000000000000000000000000000000000..47a5e2cd799f0ec56ab60e2a59b52dd9a98a653a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils.h @@ -0,0 +1,5 @@ +#pragma once + +#include +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils/clip_grad.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils/clip_grad.h new file mode 100644 index 0000000000000000000000000000000000000000..0478e6ae831850884f87d4f857a41cc1e26d9e8c --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils/clip_grad.h @@ -0,0 +1,144 @@ +#pragma once + +#include + +#include +#include +#include + +namespace torch::nn::utils { + +// Clips gradient norm of a vector of Tensors. +// See +// https://pytorch.org/docs/stable/nn.html?highlight=clip_grad_norm#torch.nn.utils.clip_grad_norm_ +// for more details about this module. +// +// Difference with the python version: unlike the python version, even when +// skipping the finiteness checks (error_if_nonfinite = false), this function +// will introduce a device <=> CPU synchronization (for devices where that makes +// sense!) in order to return a CPU-side `double`. This C++ version therefore +// cannot be run fully asynchronously w.r.t. the device of the gradients. +inline double clip_grad_norm_( + const std::vector& parameters, + double max_norm, + double norm_type = 2.0, + bool error_if_nonfinite = false) { + std::vector params_with_grad; + + for (const auto& param : parameters) { + auto& grad = param.grad(); + if (grad.defined()) { + params_with_grad.push_back(param); + } + } + + if (params_with_grad.empty()) { + return 0.0; + } + + Tensor total_norm_tensor; + if (norm_type == std::numeric_limits::infinity()) { + std::vector norms; + norms.reserve(params_with_grad.size()); + + for (const auto& param : params_with_grad) { + norms.emplace_back(param.grad().data().abs().max()); + } + total_norm_tensor = + (norms.size() == 1) ? norms[0] : torch::max(torch::stack(norms)); + } else if (norm_type == 0) { + total_norm_tensor = + torch::full({}, static_cast(params_with_grad.size())); + } else { + std::vector norms; + norms.reserve(params_with_grad.size()); + + for (const auto& param : params_with_grad) { + norms.emplace_back(param.grad().data().norm(norm_type)); + } + total_norm_tensor = + (norms.size() == 1) ? norms[0] : torch::stack(norms).norm(norm_type); + } + + // When possible (ie when skipping the finiteness check), we avoid + // synchronizing the CPU and the gradients' device until the very end to + // preserve async execution on the device. When checking for finite-ness, this + // optional ensures we only sync once. + std::optional total_norm = std::nullopt; + if (error_if_nonfinite) { + total_norm = total_norm_tensor.item().toDouble(); + TORCH_CHECK( + std::isfinite(*total_norm), + "The total norm of order ", + norm_type, + " for gradients from `parameters` ", + "is non-finite, so it cannot be clipped. To disable this error and scale ", + "the gradients with the non-finite norm anyway, set ", + "`error_if_nonfinite=false`"); + } + + auto clip_coef = max_norm / (total_norm_tensor + 1e-6); + auto clip_coef_clamped = + torch::clamp(clip_coef, std::nullopt /* min */, 1.0 /* max */); + for (auto& param : params_with_grad) { + param.grad().data().mul_(clip_coef_clamped); + } + + if (!total_norm.has_value()) { + total_norm = total_norm_tensor.item().toDouble(); + } + return *total_norm; +} + +// A wrapper around clip_grad_norm_ that allows us to call the function with a +// braced-init-list of Tensors. +inline double clip_grad_norm_( + std::initializer_list parameters, + double max_norm, + double norm_type = 2.0, + bool error_if_nonfinite = false) { + return clip_grad_norm_( + std::vector(parameters), max_norm, norm_type, error_if_nonfinite); +} + +// A wrapper around clip_grad_norm_ that allows us to call the function with a +// single Tensor. +inline double clip_grad_norm_( + Tensor parameter, + double max_norm, + double norm_type = 2.0, + bool error_if_nonfinite = false) { + std::vector params = {std::move(parameter)}; + return clip_grad_norm_(params, max_norm, norm_type, error_if_nonfinite); +} + +// Clips gradient of an iterable of parameters at specified value. +// Gradients are modified in-place. +// See https://pytorch.org/docs/stable/nn.html#clip-grad-value +// for more details about this module. +inline void clip_grad_value_( + const std::vector& parameters, + double clip_value) { + for (const auto& param : parameters) { + if (param.grad().defined()) { + param.grad().data().clamp_(-clip_value, clip_value); + } + } +} + +// A wrapper around clip_grad_value_ that allows us to call the function with a +// braced-init-list of Tensors. +inline void clip_grad_value_( + std::initializer_list parameters, + double clip_value) { + clip_grad_value_(std::vector(parameters), clip_value); +} + +// A wrapper around clip_grad_value_ that allows us to call the function with a +// single Tensor. +inline void clip_grad_value_(Tensor parameter, double clip_value) { + std::vector params = {std::move(parameter)}; + clip_grad_value_(params, clip_value); +} + +} // namespace torch::nn::utils diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils/convert_parameters.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils/convert_parameters.h new file mode 100644 index 0000000000000000000000000000000000000000..044c1123721ead7597c637669c070528d55d2b0a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils/convert_parameters.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include + +namespace torch::nn::utils { + +// This helper function is to check if the parameters are located +// in the same device. Currently, the conversion between model parameters +// and single vector form is not supported for multiple allocations, +// e.g. parameters in different GPUs, or mixture of CPU/GPU. +inline std::optional _check_param_device( + const torch::Tensor& param, + std::optional old_param_device) { + // Meet the first parameter + if (old_param_device == std::nullopt) { + old_param_device = param.is_cuda() ? param.get_device() : -1; + } else { + bool warn = false; + if (param.is_cuda()) { // Check if in same GPU + warn = (param.get_device() != old_param_device); + } else { // Check if in CPU + warn = (old_param_device != -1); + } + if (warn) { + TORCH_CHECK( + false, + "Found two parameters on different devices, ", + "this is currently not supported."); + } + } + + return old_param_device; +} + +// Convert parameters to one vector +inline torch::Tensor parameters_to_vector( + const std::vector& parameters) { + std::optional param_device; + + std::vector vec; + vec.reserve(parameters.size()); + + for (const torch::Tensor& param : parameters) { + // Ensure the parameters are located in the same device + param_device = _check_param_device(param, param_device); + + vec.push_back(param.view(-1)); + } + + return torch::cat(vec); +} + +// Convert one vector to the parameters +inline void vector_to_parameters( + const torch::Tensor& vec, + const std::vector& parameters) { + // Flag for the device where the parameter is located + std::optional param_device; + + // Pointer for slicing the vector for each parameter + int64_t pointer = 0; + for (const torch::Tensor& param : parameters) { + // Ensure the parameters are located in the same device + param_device = _check_param_device(param, param_device); + + // The length of the parameter + auto num_param = param.numel(); + // Slice the vector, reshape it, and replace the old data of the parameter + param.set_data( + vec.slice(0, pointer, pointer + num_param).view_as(param).data()); + + // Increment the pointer + pointer += num_param; + } +} + +} // namespace torch::nn::utils diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils/rnn.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils/rnn.h new file mode 100644 index 0000000000000000000000000000000000000000..8b8fbae583fea731b403b30eb8408e43ef579662 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/nn/utils/rnn.h @@ -0,0 +1,348 @@ +#pragma once + +#include +#include + +#include + +namespace torch::nn::utils::rnn { + +inline Tensor invert_permutation(const Tensor& permutation) { + if (!permutation.defined()) { + return torch::Tensor(); + } + Tensor output = + torch::empty_like(permutation, torch::MemoryFormat::Contiguous); + output.scatter_( + 0, + permutation, + torch::arange(0, permutation.numel(), permutation.device())); + return output; +} + +/// Holds the data and list of `batch_sizes` of a packed sequence. +/// +/// All RNN modules accept packed sequences as inputs. +/// +/// Note: +/// Instances of this class should never be created manually. They are meant +/// to be instantiated by functions like `pack_padded_sequence`. +/// +/// Batch sizes represent the number elements at each sequence step in +/// the batch, not the varying sequence lengths passed to +/// `pack_padded_sequence`. For instance, given data ``abc`` and ``x`` +/// the :class:`PackedSequence` would contain data ``axbc`` with +/// ``batch_sizes=[2,1,1]``. +/// +/// Attributes: +/// data (Tensor): Tensor containing packed sequence +/// batch_sizes (Tensor): Tensor of integers holding +/// information about the batch size at each sequence step +/// sorted_indices (Tensor, optional): Tensor of integers holding how this +/// :class:`PackedSequence` is constructed from sequences. +/// unsorted_indices (Tensor, optional): Tensor of integers holding how this +/// to recover the original sequences with correct order. +/// +/// .. note:: +/// `data` can be on arbitrary device and of arbitrary dtype. +/// `sorted_indices` and `unsorted_indices` must be ``torch::kInt64`` +/// tensors on the same device as `data`. +/// +/// However, `batch_sizes` should always be a CPU ``torch::kInt64`` tensor. +/// +/// This invariant is maintained throughout `PackedSequence` class, +/// and all functions that construct a `PackedSequence` in libtorch +/// (i.e., they only pass in tensors conforming to this constraint). +class PackedSequence { + public: + explicit PackedSequence( + Tensor data, + Tensor batch_sizes, + Tensor sorted_indices = {}, + Tensor unsorted_indices = {}) { + // NB: if unsorted_indices is provided, it should be the inverse permutation + // to sorted_indices. Don't assert it here because the PackedSequence ctor + // should only be used internally. + if (!unsorted_indices.defined()) { + unsorted_indices = invert_permutation(sorted_indices); + } + TORCH_CHECK( + batch_sizes.device().type() == kCPU, + "batch_sizes should always be on CPU. " + "Instances of PackedSequence should never be created manually. " + "They should be instantiated by functions like pack_sequence " + "and pack_padded_sequences in nn::utils::rnn. " + "https://pytorch.org/docs/stable/nn.html#torch.nn.utils.rnn.pack_sequence"); + data_ = std::move(data); + batch_sizes_ = std::move(batch_sizes); + sorted_indices_ = std::move(sorted_indices); + unsorted_indices_ = std::move(unsorted_indices); + } + + const Tensor& data() const { + return data_; + } + + const Tensor& batch_sizes() const { + return batch_sizes_; + } + + const Tensor& sorted_indices() const { + return sorted_indices_; + } + + const Tensor& unsorted_indices() const { + return unsorted_indices_; + } + + PackedSequence pin_memory() const { + // Why not convert `batch_sizes`? + // See NOTE [ device and dtype of a PackedSequence ] + return PackedSequence( + data_.pin_memory(), + batch_sizes_, + sorted_indices_.defined() ? sorted_indices_.pin_memory() : Tensor(), + unsorted_indices_.defined() ? unsorted_indices_.pin_memory() + : Tensor()); + } + + PackedSequence to(TensorOptions options) const { + // Performs dtype and/or device conversion on `data_`. + // + // If the ``data_`` Tensor already has the correct `torch::Dtype` + // and `torch::Device`, then ``self`` is returned. + // Otherwise, returns a copy with the desired configuration. + + // Why not convert `batch_sizes`? + // See NOTE [ device and dtype of a PackedSequence ] + Tensor data = data_.to(options); + if (data.is_same(data_)) { + return *this; + } else { + // Does not forward device or dtype args, device is set from data.device() + Tensor sorted_indices = sorted_indices_.defined() + ? sorted_indices_.to( + options.device(data.device()).dtype(sorted_indices_.dtype())) + : Tensor(); + Tensor unsorted_indices = unsorted_indices_.defined() + ? unsorted_indices_.to( + options.device(data.device()).dtype(unsorted_indices_.dtype())) + : Tensor(); + return PackedSequence( + std::move(data), + batch_sizes_, + std::move(sorted_indices), + std::move(unsorted_indices)); + } + } + + PackedSequence cuda() const { + return to(kCUDA); + } + + PackedSequence cpu() const { + return to(kCPU); + } + + /// Returns true if `data_` stored on a gpu + bool is_cuda() const { + return data_.is_cuda(); + } + + /// Returns true if `data_` stored on in pinned memory + bool is_pinned() const { + return data_.is_pinned(); + } + + private: + Tensor data_; + Tensor batch_sizes_; + Tensor sorted_indices_; + Tensor unsorted_indices_; +}; + +/// Packs a Tensor containing padded sequences of variable length. +/// +/// `input` can be of size ``T x B x *`` where `T` is the length of the +/// longest sequence (equal to ``lengths[0]``), ``B`` is the batch size, and +/// ``*`` is any number of dimensions (including 0). If ``batch_first`` is +/// ``true``, ``B x T x *`` `input` is expected. +/// +/// For unsorted sequences, use `enforce_sorted = false`. If `enforce_sorted` is +/// ``true``, the sequences should be sorted by length in a decreasing order, +/// i.e. +/// ``input[:,0]`` should be the longest sequence, and ``input[:,B-1]`` the +/// shortest one. +/// +/// Note: +/// This function accepts any input that has at least two dimensions. You +/// can apply it to pack the labels, and use the output of the RNN with +/// them to compute the loss directly. A Tensor can be retrieved from +/// a `PackedSequence` object by calling its ``.data()`` function. +/// +/// Arguments: +/// input (Tensor): padded batch of variable length sequences. +/// lengths (Tensor): list of sequences lengths of each batch element. +/// batch_first (bool, optional): if ``true``, the input is expected in ``B +/// x T x *`` +/// format. Default: ``false``. +/// enforce_sorted (bool, optional): if ``true``, the input is expected to +/// contain sequences sorted by length in a decreasing order. If +/// ``false``, this condition is not checked. Default: ``true``. +/// +/// Returns: +/// a `PackedSequence` object +inline PackedSequence pack_padded_sequence( + Tensor input, + Tensor lengths, + bool batch_first = false, + bool enforce_sorted = true) { + lengths = lengths.to(kInt64); + Tensor sorted_indices; + if (enforce_sorted) { + sorted_indices = Tensor(); + } else { + std::tie(lengths, sorted_indices) = + torch::sort(lengths, /*dim=*/-1, /*descending=*/true); + sorted_indices = sorted_indices.to(input.device()); + int64_t batch_dim = batch_first ? 0 : 1; + input = input.index_select(batch_dim, sorted_indices); + } + + auto [data, batch_sizes] = + torch::_pack_padded_sequence(input, lengths, batch_first); + return PackedSequence( + std::move(data), std::move(batch_sizes), std::move(sorted_indices), {}); +} + +/// Pads a packed batch of variable length sequences. +/// +/// It is an inverse operation to `pack_padded_sequence`. +/// +/// The returned Tensor's data will be of size ``T x B x *``, where `T` is the +/// length of the longest sequence and `B` is the batch size. If ``batch_first`` +/// is true, the data will be transposed into ``B x T x *`` format. +/// +/// Batch elements will be ordered decreasingly by their length. +/// +/// Arguments: +/// sequence (PackedSequence): batch to pad +/// batch_first (bool, optional): if ``true``, the output will be in ``B x T +/// x *`` +/// format. +/// padding_value (double, optional): values for padded elements. +/// total_length (int64_t, optional): if specified, the output will be +/// padded to +/// have length `total_length`. This method will throw error +/// if `total_length` is less than the max sequence length in +/// `sequence`. +/// +/// Returns: +/// Tuple of Tensor containing the padded sequence, and a Tensor +/// containing the list of lengths of each sequence in the batch. +inline std::tuple pad_packed_sequence( + const PackedSequence& sequence, + bool batch_first = false, + double padding_value = 0.0, + std::optional total_length = std::nullopt) { + int64_t max_seq_length = sequence.batch_sizes().size(0); + if (total_length.has_value()) { + int64_t total_length_val = total_length.value(); + TORCH_CHECK( + total_length_val >= max_seq_length, + "Expected total_length to be at least the length " + "of the longest sequence in input, but got " + "total_length=", + total_length_val, + " and max sequence length being ", + max_seq_length); + max_seq_length = total_length_val; + } + auto [padded_output, lengths] = torch::_pad_packed_sequence( + sequence.data(), + sequence.batch_sizes(), + batch_first, + padding_value, + max_seq_length); + const Tensor& unsorted_indices = sequence.unsorted_indices(); + if (unsorted_indices.defined()) { + int64_t batch_dim = batch_first ? 0 : 1; + return std::make_tuple( + padded_output.index_select(batch_dim, unsorted_indices), + lengths.index({unsorted_indices.cpu()})); + } + return std::make_tuple(padded_output, lengths); +} + +/// Pad a list of variable length Tensors with ``padding_value`` +/// +/// ``pad_sequence`` stacks a list of Tensors along a new dimension, +/// and pads them to equal length. For example, if the input is list of +/// sequences with size ``L x *`` and if batch_first is false, and ``T x B x *`` +/// otherwise. +/// +/// `B` is batch size. It is equal to the number of elements in ``sequences``. +/// `T` is length of the longest sequence. +/// `L` is length of the sequence. +/// `*` is any number of trailing dimensions, including none. +/// +/// Note: +/// This function returns a Tensor of size ``T x B x *`` or ``B x T x *`` +/// where `T` is the length of the longest sequence. This function assumes +/// trailing dimensions and type of all the Tensors in sequences are same. +/// +/// Arguments: +/// sequences (torch::ArrayRef): list of variable length sequences. +/// batch_first (bool, optional): output will be in ``B x T x *`` if true, +/// or in +/// ``T x B x *`` otherwise +/// padding_value (double, optional): value for padded elements. Default: 0. +/// padding_side (str, optional): the side to pad the sequences on. Default: +/// "right". +/// +/// Returns: +/// Tensor of size ``T x B x *`` if `batch_first` is ``false``. +/// Tensor of size ``B x T x *`` otherwise +inline Tensor pad_sequence( + ArrayRef sequences, + bool batch_first = false, + double padding_value = 0, + std::string_view padding_side = "right") { + return at::pad_sequence(sequences, batch_first, padding_value, padding_side); +} + +/// Packs a list of variable length Tensors +/// +/// ``sequences`` should be a list of Tensors of size ``L x *``, where `L` is +/// the length of a sequence and `*` is any number of trailing dimensions, +/// including zero. +/// +/// For unsorted sequences, use `enforce_sorted = false`. If ``enforce_sorted`` +/// is ``true``, the sequences should be sorted in the order of decreasing +/// length. +/// +/// +/// Arguments: +/// sequences (torch::ArrayRef): A list of sequences of decreasing +/// length. enforce_sorted (bool, optional): if ``true``, checks that the +/// input +/// contains sequences sorted by length in a decreasing order. If +/// ``false``, this condition is not checked. Default: ``true``. +/// +/// Returns: +/// a `PackedSequence` object +inline PackedSequence pack_sequence( + ArrayRef sequences, + bool enforce_sorted = true) { + Tensor lengths = torch::empty({(int64_t)sequences.size()}, kInt64); + for (const auto i : c10::irange(sequences.size())) { + lengths[static_cast(i)] = sequences[i].size(0); + } + return pack_padded_sequence( + at::pad_sequence(sequences), + std::move(lengths), + /*batch_first=*/false, + /*enforce_sorted=*/enforce_sorted); +} + +} // namespace torch::nn::utils::rnn diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim.h new file mode 100644 index 0000000000000000000000000000000000000000..efcec794021b7b73f54c18d7cf9d646d27321412 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim.h @@ -0,0 +1,13 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/adagrad.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/adagrad.h new file mode 100644 index 0000000000000000000000000000000000000000..159b68a8b9434c4da97789ee5358369e77f0260e --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/adagrad.h @@ -0,0 +1,99 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +namespace torch::serialize { +class OutputArchive; +class InputArchive; +} // namespace torch::serialize + +namespace torch::optim { + +struct TORCH_API AdagradOptions + : public OptimizerCloneableOptions { + AdagradOptions(double lr = 1e-2); + TORCH_ARG(double, lr) = 1e-2; + TORCH_ARG(double, lr_decay) = 0; + TORCH_ARG(double, weight_decay) = 0; + TORCH_ARG(double, initial_accumulator_value) = 0; + TORCH_ARG(double, eps) = 1e-10; + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const AdagradOptions& lhs, + const AdagradOptions& rhs); + double get_lr() const override; + void set_lr(const double lr) override; +}; + +struct TORCH_API AdagradParamState + : public OptimizerCloneableParamState { + TORCH_ARG(torch::Tensor, sum); + TORCH_ARG(int64_t, step) = 0; + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const AdagradParamState& lhs, + const AdagradParamState& rhs); +}; + +class TORCH_API Adagrad : public Optimizer { + public: + explicit Adagrad( + const std::vector& param_groups, + AdagradOptions defaults = {}) + : Optimizer(param_groups, std::make_unique(defaults)) { + TORCH_CHECK(defaults.lr() >= 0, "Invalid learning rate: ", defaults.lr()); + TORCH_CHECK( + defaults.lr_decay() >= 0, + "Invalid lr_decay value: ", + defaults.lr_decay()); + TORCH_CHECK( + defaults.weight_decay() >= 0, + "Invalid weight_decay value: ", + defaults.weight_decay()); + TORCH_CHECK( + defaults.initial_accumulator_value() >= 0, + "Invalid initial_accumulator_value value: ", + defaults.initial_accumulator_value()); + TORCH_CHECK(defaults.eps() >= 0, "Invalid epsilon value: ", defaults.eps()); + + for (const auto& group : param_groups_) { + for (const auto& p : group.params()) { + auto state = std::make_unique(); + state->step(0); + state->sum(torch::full_like( + p.data(), + defaults.initial_accumulator_value(), + at::MemoryFormat::Preserve)); + state_[p.unsafeGetTensorImpl()] = std::move(state); + } + } + } + + explicit Adagrad(std::vector params, AdagradOptions defaults = {}) + : Adagrad({OptimizerParamGroup(std::move(params))}, std::move(defaults)) { + } + + torch::Tensor step(LossClosure closure = nullptr) override; + void save(serialize::OutputArchive& archive) const override; + void load(serialize::InputArchive& archive) override; + + private: + template + static void serialize(Self& self, Archive& archive) { + _TORCH_OPTIM_SERIALIZE_WITH_TEMPLATE_ARG(Adagrad); + } +}; +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/adam.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/adam.h new file mode 100644 index 0000000000000000000000000000000000000000..155261d2a97a7c9e437fcdf5b550a1fc77eff44a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/adam.h @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace torch::serialize { +class OutputArchive; +class InputArchive; +} // namespace torch::serialize + +namespace torch::optim { + +struct TORCH_API AdamOptions : public OptimizerCloneableOptions { + AdamOptions(double lr = 1e-3); + TORCH_ARG(double, lr) = 1e-3; + typedef std::tuple betas_t; + TORCH_ARG(betas_t, betas) = std::make_tuple(0.9, 0.999); + TORCH_ARG(double, eps) = 1e-8; + TORCH_ARG(double, weight_decay) = 0; + TORCH_ARG(bool, amsgrad) = false; + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const AdamOptions& lhs, + const AdamOptions& rhs); + double get_lr() const override; + void set_lr(const double lr) override; +}; + +struct TORCH_API AdamParamState + : public OptimizerCloneableParamState { + TORCH_ARG(int64_t, step) = 0; + TORCH_ARG(torch::Tensor, exp_avg); + TORCH_ARG(torch::Tensor, exp_avg_sq); + TORCH_ARG(torch::Tensor, max_exp_avg_sq); + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const AdamParamState& lhs, + const AdamParamState& rhs); +}; + +class TORCH_API Adam : public Optimizer { + public: + explicit Adam( + const std::vector& param_groups, + AdamOptions defaults = {}) + : Optimizer(param_groups, std::make_unique(defaults)) { + TORCH_CHECK(defaults.lr() >= 0, "Invalid learning rate: ", defaults.lr()); + TORCH_CHECK(defaults.eps() >= 0, "Invalid epsilon value: ", defaults.eps()); + auto betas = defaults.betas(); + TORCH_CHECK( + 0 <= std::get<0>(betas) && std::get<0>(betas) < 1.0, + "Invalid beta parameter at index 0: ", + std::get<0>(betas)); + TORCH_CHECK( + 0 <= std::get<1>(betas) && std::get<1>(betas) < 1.0, + "Invalid beta parameter at index 1: ", + std::get<1>(betas)); + TORCH_CHECK( + defaults.weight_decay() >= 0, + "Invalid weight_decay value: ", + defaults.weight_decay()); + } + explicit Adam(std::vector params, AdamOptions defaults = {}) + : Adam({OptimizerParamGroup(std::move(params))}, std::move(defaults)) {} + + torch::Tensor step(LossClosure closure = nullptr) override; + void save(serialize::OutputArchive& archive) const override; + void load(serialize::InputArchive& archive) override; + + private: + template + static void serialize(Self& self, Archive& archive) { + _TORCH_OPTIM_SERIALIZE_WITH_TEMPLATE_ARG(Adam); + } +}; +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/adamw.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/adamw.h new file mode 100644 index 0000000000000000000000000000000000000000..31cfb309d47c863ac13fd9232c7ef9f5837d1ffe --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/adamw.h @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace torch::serialize { +class OutputArchive; +class InputArchive; +} // namespace torch::serialize + +namespace torch::optim { + +struct TORCH_API AdamWOptions : public OptimizerCloneableOptions { + AdamWOptions(double lr = 1e-3); + TORCH_ARG(double, lr) = 1e-3; + typedef std::tuple betas_t; + TORCH_ARG(betas_t, betas) = std::make_tuple(0.9, 0.999); + TORCH_ARG(double, eps) = 1e-8; + TORCH_ARG(double, weight_decay) = 1e-2; + TORCH_ARG(bool, amsgrad) = false; + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const AdamWOptions& lhs, + const AdamWOptions& rhs); + double get_lr() const override; + void set_lr(const double lr) override; +}; + +struct TORCH_API AdamWParamState + : public OptimizerCloneableParamState { + TORCH_ARG(int64_t, step) = 0; + TORCH_ARG(torch::Tensor, exp_avg); + TORCH_ARG(torch::Tensor, exp_avg_sq); + TORCH_ARG(torch::Tensor, max_exp_avg_sq); + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const AdamWParamState& lhs, + const AdamWParamState& rhs); +}; + +class TORCH_API AdamW : public Optimizer { + public: + explicit AdamW( + const std::vector& param_groups, + AdamWOptions defaults = {}) + : Optimizer(param_groups, std::make_unique(defaults)) { + TORCH_CHECK(defaults.lr() >= 0, "Invalid learning rate: ", defaults.lr()); + TORCH_CHECK(defaults.eps() >= 0, "Invalid epsilon value: ", defaults.eps()); + auto betas = defaults.betas(); + TORCH_CHECK( + 0 <= std::get<0>(betas) && std::get<0>(betas) < 1.0, + "Invalid beta parameter at index 0: ", + std::get<0>(betas)); + TORCH_CHECK( + 0 <= std::get<1>(betas) && std::get<1>(betas) < 1.0, + "Invalid beta parameter at index 1: ", + std::get<1>(betas)); + TORCH_CHECK( + defaults.weight_decay() >= 0, + "Invalid weight_decay value: ", + defaults.weight_decay()); + } + explicit AdamW(std::vector params, AdamWOptions defaults = {}) + : AdamW({OptimizerParamGroup(std::move(params))}, std::move(defaults)) {} + + torch::Tensor step(LossClosure closure = nullptr) override; + void save(serialize::OutputArchive& archive) const override; + void load(serialize::InputArchive& archive) override; + + private: + template + static void serialize(Self& self, Archive& archive) { + _TORCH_OPTIM_SERIALIZE_WITH_TEMPLATE_ARG(AdamW); + } +}; +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/lbfgs.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/lbfgs.h new file mode 100644 index 0000000000000000000000000000000000000000..f2fcff6dae731b52dc2ae0ae9c2472eee2515876 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/lbfgs.h @@ -0,0 +1,100 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace torch::optim { + +struct TORCH_API LBFGSOptions : public OptimizerCloneableOptions { + LBFGSOptions(double lr = 1); + TORCH_ARG(double, lr) = 1; + TORCH_ARG(int64_t, max_iter) = 20; + TORCH_ARG(std::optional, max_eval) = std::nullopt; + TORCH_ARG(double, tolerance_grad) = 1e-7; + TORCH_ARG(double, tolerance_change) = 1e-9; + TORCH_ARG(int64_t, history_size) = 100; + TORCH_ARG(std::optional, line_search_fn) = std::nullopt; + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const LBFGSOptions& lhs, + const LBFGSOptions& rhs); + double get_lr() const override; + void set_lr(const double lr) override; +}; + +struct TORCH_API LBFGSParamState + : public OptimizerCloneableParamState { + TORCH_ARG(int64_t, func_evals) = 0; + TORCH_ARG(int64_t, n_iter) = 0; + TORCH_ARG(double, t) = 0; + TORCH_ARG(double, prev_loss) = 0; + TORCH_ARG(Tensor, d); + TORCH_ARG(Tensor, H_diag); + TORCH_ARG(Tensor, prev_flat_grad); + TORCH_ARG(std::deque, old_dirs); + TORCH_ARG(std::deque, old_stps); + TORCH_ARG(std::deque, ro); + TORCH_ARG(std::optional>, al) = std::nullopt; + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const LBFGSParamState& lhs, + const LBFGSParamState& rhs); +}; + +class TORCH_API LBFGS : public Optimizer { + public: + explicit LBFGS( + const std::vector& param_groups, + LBFGSOptions defaults = {}) + : Optimizer(param_groups, std::make_unique(defaults)) { + TORCH_CHECK( + param_groups_.size() == 1, + "LBFGS doesn't support per-parameter options (parameter groups)"); + if (defaults.max_eval() == std::nullopt) { + auto max_eval_val = (defaults.max_iter() * 5) / 4; + static_cast(param_groups_[0].options()) + .max_eval(max_eval_val); + static_cast(*defaults_).max_eval(max_eval_val); + } + _numel_cache = std::nullopt; + } + explicit LBFGS(std::vector params, LBFGSOptions defaults = {}) + : LBFGS({OptimizerParamGroup(std::move(params))}, std::move(defaults)) {} + + Tensor step(LossClosure closure) override; + void save(serialize::OutputArchive& archive) const override; + void load(serialize::InputArchive& archive) override; + + private: + std::optional _numel_cache; + int64_t _numel(); + Tensor _gather_flat_grad(); + void _add_grad(const double step_size, const Tensor& update); + std::tuple _directional_evaluate( + const LossClosure& closure, + const std::vector& x, + double t, + const Tensor& d); + void _set_param(const std::vector& params_data); + std::vector _clone_param(); + + template + static void serialize(Self& self, Archive& archive) { + _TORCH_OPTIM_SERIALIZE_WITH_TEMPLATE_ARG(LBFGS); + } +}; +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/optimizer.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/optimizer.h new file mode 100644 index 0000000000000000000000000000000000000000..60de3c3b2cac1de55c253da34f4c4f143eeaeec1 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/optimizer.h @@ -0,0 +1,458 @@ +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +// Forward declarations confuse Doxygen +#ifndef DOXYGEN_SHOULD_SKIP_THIS +namespace at { +class Tensor; +} // namespace at + +namespace torch { +using at::Tensor; +namespace serialize { +class OutputArchive; +class InputArchive; +} // namespace serialize +} // namespace torch +#endif // DOXYGEN_SHOULD_SKIP_THIS + +namespace torch::optim { + +class TORCH_API OptimizerParamState { + public: + OptimizerParamState() = default; + OptimizerParamState(const OptimizerParamState&) = default; + OptimizerParamState& operator=(const OptimizerParamState&) = default; + OptimizerParamState(OptimizerParamState&&) noexcept = default; + OptimizerParamState& operator=(OptimizerParamState&&) noexcept = default; + virtual std::unique_ptr clone() const; + virtual void serialize(torch::serialize::InputArchive& archive); + virtual void serialize(torch::serialize::OutputArchive& archive) const; + virtual ~OptimizerParamState() = default; +}; + +template +class OptimizerCloneableParamState : public OptimizerParamState { + std::unique_ptr clone() const override { + return std::make_unique(static_cast(*this)); + } +}; + +class TORCH_API OptimizerOptions { + public: + OptimizerOptions() = default; + OptimizerOptions(const OptimizerOptions&) = default; + OptimizerOptions& operator=(const OptimizerOptions&) = default; + OptimizerOptions(OptimizerOptions&&) noexcept = default; + OptimizerOptions& operator=(OptimizerOptions&&) noexcept = default; + virtual std::unique_ptr clone() const; + virtual void serialize(torch::serialize::InputArchive& archive); + virtual void serialize(torch::serialize::OutputArchive& archive) const; + virtual ~OptimizerOptions() = default; + virtual double get_lr() const; + virtual void set_lr(const double lr); +}; + +// Forward declarations for optimizer option types +struct SGDOptions; +struct AdamOptions; +struct AdamWOptions; +struct AdagradOptions; +struct RMSpropOptions; +struct LBFGSOptions; + +/** + * OptimizerCloneableOptions provides parameter group inheritance functionality + * for PyTorch C++ optimizer options. When creating parameter groups with + * partial options (e.g., AdamOptions().weight_decay(0.1)), fields not + * explicitly set by the user inherit from the optimizer's default values, + * while explicitly set fields are preserved. + * + * This enables Python-like behavior in C++: + * ```cpp + * // Python equivalent: + * // optimizer = Adam([{'params': params1, 'weight_decay': 0.1}], lr=0.01) + * // Result: weight_decay=0.1 preserved, lr=0.01 inherited + * + * AdamOptions defaults; + * defaults.lr(0.01).weight_decay(0.05); + * + * std::vector groups; + * groups.emplace_back(params1, std::make_unique( + * AdamOptions().weight_decay(0.1))); // Only weight_decay specified + * + * Adam optimizer(groups, defaults); + * // Result: group inherits lr=0.01, preserves weight_decay=0.1 + * ``` + * + * **Implementation**: Uses SFINAE-based field detection and constructor-default + * comparison to distinguish explicitly set fields from default values. + * Fields that match constructor defaults are inherited; others are preserved. + */ +template +class OptimizerCloneableOptions : public OptimizerOptions { + private: + std::unique_ptr clone() const override { + return std::make_unique(static_cast(*this)); + } + + // SFINAE field detection - detects optimizer fields using public accessor + // methods + template + struct _has_lr : std::false_type {}; + template + struct _has_lr().lr())>> + : std::true_type {}; + + template + struct _has_momentum : std::false_type {}; + template + struct _has_momentum< + T, + std::void_t().momentum())>> + : std::true_type {}; + + template + struct _has_weight_decay : std::false_type {}; + template + struct _has_weight_decay< + T, + std::void_t().weight_decay())>> + : std::true_type {}; + + template + struct _has_dampening : std::false_type {}; + template + struct _has_dampening< + T, + std::void_t().dampening())>> + : std::true_type {}; + + template + struct _has_nesterov : std::false_type {}; + template + struct _has_nesterov< + T, + std::void_t().nesterov())>> + : std::true_type {}; + + template + struct _has_betas : std::false_type {}; + template + struct _has_betas().betas())>> + : std::true_type {}; + + template + struct _has_eps : std::false_type {}; + template + struct _has_eps().eps())>> + : std::true_type {}; + + template + struct _has_amsgrad : std::false_type {}; + template + struct _has_amsgrad< + T, + std::void_t().amsgrad())>> + : std::true_type {}; + + // Optimizer-specific field detection + template + struct _has_lr_decay : std::false_type {}; + template + struct _has_lr_decay< + T, + std::void_t().lr_decay())>> + : std::true_type {}; + + template + struct _has_alpha : std::false_type {}; + template + struct _has_alpha().alpha())>> + : std::true_type {}; + + template + struct _has_centered : std::false_type {}; + template + struct _has_centered< + T, + std::void_t().centered())>> + : std::true_type {}; + + template + struct _has_initial_accumulator_value : std::false_type {}; + template + struct _has_initial_accumulator_value< + T, + std::void_t< + decltype(std::declval().initial_accumulator_value())>> + : std::true_type {}; + + // LBFGS-specific fields with appropriate types + template + struct _has_max_iter : std::false_type {}; + template + struct _has_max_iter< + T, + std::void_t().max_iter())>> + : std::true_type {}; + + template + struct _has_max_eval : std::false_type {}; + template + struct _has_max_eval< + T, + std::void_t().max_eval())>> + : std::true_type {}; + + template + struct _has_tolerance_grad : std::false_type {}; + template + struct _has_tolerance_grad< + T, + std::void_t().tolerance_grad())>> + : std::true_type {}; + + template + struct _has_tolerance_change : std::false_type {}; + template + struct _has_tolerance_change< + T, + std::void_t().tolerance_change())>> + : std::true_type {}; + + template + struct _has_history_size : std::false_type {}; + template + struct _has_history_size< + T, + std::void_t().history_size())>> + : std::true_type {}; + + template + struct _has_line_search_fn : std::false_type {}; + template + struct _has_line_search_fn< + T, + std::void_t().line_search_fn())>> + : std::true_type {}; + + /** + * Merges user-specified options with optimizer defaults using + * constructor-default comparison to detect explicitly set fields. + * + * Algorithm: + * 1. Start with optimizer defaults as base + * 2. Create fresh constructor instance for comparison + * 3. If user_value != constructor_default → user explicitly set it → preserve + * 4. If user_value == constructor_default → user didn't set it → inherit from + * defaults + * + * Implementation is in optimizer.cpp to anchor vtable/typeinfo. + */ + void _merge_by_comparison( + const Derived& defaults, + const Derived& user_options); + + // Friend class for controlled access to private _merge_by_comparison method + friend class Optimizer; +}; + +/// Stores parameters in the param_group and stores a pointer to the +/// OptimizerOptions +class TORCH_API OptimizerParamGroup { + public: + // NOTE: In order to store `OptimizerParamGroup` in a `std::vector`, it has to + // be copy-constructible. + OptimizerParamGroup(const OptimizerParamGroup& param_group) + : params_(param_group.params()), + options_( + param_group.has_options() ? param_group.options().clone() + : nullptr) {} + OptimizerParamGroup(OptimizerParamGroup&& param_group) = default; + OptimizerParamGroup(std::vector params) + : params_(std::move(params)) {} + OptimizerParamGroup( + std::vector params, + std::unique_ptr options) + : params_(std::move(params)), options_(std::move(options)) {} + + OptimizerParamGroup& operator=(const OptimizerParamGroup& param_group) = + delete; + OptimizerParamGroup& operator=(OptimizerParamGroup&& param_group) noexcept = + default; + ~OptimizerParamGroup() = default; + bool has_options() const; + OptimizerOptions& options(); + const OptimizerOptions& options() const; + void set_options(std::unique_ptr options); + std::vector& params(); + const std::vector& params() const; + + protected: + std::vector params_; + std::unique_ptr options_; +}; + +class TORCH_API Optimizer { + public: + // The copy constructor is deleted, because the user should use the + // `state_dict` / `load_state_dict` API to copy an optimizer instead. + Optimizer(const Optimizer& optimizer) = delete; + Optimizer(Optimizer&& optimizer) = default; + Optimizer& operator=(const Optimizer& optimizer) = delete; + Optimizer& operator=(Optimizer&& optimizer) = default; + + explicit Optimizer( + const std::vector& param_groups, + std::unique_ptr defaults) + : defaults_(std::move(defaults)) { + for (const auto& param_group : param_groups) { + add_param_group(param_group); + } + } + + /// Constructs the `Optimizer` from a vector of parameters. + explicit Optimizer( + std::vector parameters, + std::unique_ptr defaults) + : Optimizer( + {OptimizerParamGroup(std::move(parameters))}, + std::move(defaults)) {} + + /// Adds the given param_group to the optimizer's param_group list. + void add_param_group(const OptimizerParamGroup& param_group); + + virtual ~Optimizer() = default; + + using LossClosure = std::function; + /// A loss function closure, which is expected to return the loss value. + virtual Tensor step(LossClosure closure = nullptr) = 0; + + /// Adds the given vector of parameters to the optimizer's parameter list. + void add_parameters(const std::vector& parameters); + + /// Zeros out the gradients of all parameters. + void zero_grad(bool set_to_none = true); + + /// Provides a const reference to the parameters in the first param_group this + /// optimizer holds. + const std::vector& parameters() const noexcept; + + /// Provides a reference to the parameters in the first param_group this + /// optimizer holds. + std::vector& parameters() noexcept; + + /// Returns the number of parameters referenced by the optimizer. + size_t size() const noexcept; + + OptimizerOptions& defaults() noexcept; + + const OptimizerOptions& defaults() const noexcept; + + /// Provides a reference to the param_groups this optimizer holds. + std::vector& param_groups() noexcept; + + /// Provides a const reference to the param_groups this optimizer holds. + const std::vector& param_groups() const noexcept; + + /// Provides a reference to the state this optimizer holds + ska::flat_hash_map>& + state() noexcept; + + /// Provides a const reference to the state this optimizer holds + const ska::flat_hash_map>& state() + const noexcept; + + /// Serializes the optimizer state into the given `archive`. + virtual void save(serialize::OutputArchive& archive) const; + + /// Deserializes the optimizer state from the given `archive`. + virtual void load(serialize::InputArchive& archive); + + private: + /// Helper function to try merging for a specific optimizer type + template + static bool _try_merge_optimizer_type( + std::unique_ptr& final_options, + const OptimizerOptions& user_options, + const OptimizerOptions& defaults) { + auto* typed_final = dynamic_cast(final_options.get()); + auto* typed_user = dynamic_cast(&user_options); + auto* typed_defaults = dynamic_cast(&defaults); + + if (typed_final && typed_user && typed_defaults) { + typed_final->_merge_by_comparison(*typed_defaults, *typed_user); + return true; + } + return false; + } + + /// Simple variadic dispatch helper - try all optimizer types in one call + template + static void _try_merge_all_optimizer_types( + std::unique_ptr& final_options, + const OptimizerOptions& user_options, + const OptimizerOptions& defaults) { + // Try each optimizer type until one succeeds - much cleaner than manual + // chain + (void)(_try_merge_optimizer_type( + final_options, user_options, defaults) || + ...); + } + + /// Convenience function with all known PyTorch optimizers + static void _try_merge_all_optimizers( + std::unique_ptr& final_options, + const OptimizerOptions& user_options, + const OptimizerOptions& defaults); + + protected: + std::vector param_groups_; + ska::flat_hash_map> state_; + std::unique_ptr defaults_; +}; + +/* How do we decide whether to serialize undefined tensors or + std::nullopt values into the output archive? +Answer: we strictly follow the behavior of Python API. To be more specific: + +For optimizer options: +a) For undefined tensor: currently no tensor is used as an options argument in +Python API, so we don't need to worry about it now. b) For std::nullopt value: +we serialize std::nullopt values into the output archive, to follow the exact +same behavior as Python API. + +For optimizer param state: +a) For undefined tensor: in param state, undefined tensor in C++ impl is +equivalent to missing key in Python impl. Since we don't serialize missing keys +in Python API, we skip undefined tensors when serializing the param state. b) +For std::nullopt value: in param state, std::nullopt value in C++ impl is +equivalent to missing key in Python impl. Since we don't serialize missing keys +in Python API, we skip std::nullopt values when serializing the param state. */ + +/// Serializes an `Optimizer` into an `OutputArchive`. +TORCH_API serialize::OutputArchive& operator<<( + serialize::OutputArchive& archive, + const Optimizer& optimizer); + +/// Deserializes a `Tensor` from an `InputArchive`. +TORCH_API serialize::InputArchive& operator>>( + serialize::InputArchive& archive, + Optimizer& optimizer); + +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/rmsprop.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/rmsprop.h new file mode 100644 index 0000000000000000000000000000000000000000..7cf842b82a40eadf53363db77f233eaf50158d95 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/rmsprop.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace torch::serialize { +class OutputArchive; +class InputArchive; +} // namespace torch::serialize + +namespace torch::optim { + +struct TORCH_API RMSpropOptions + : public OptimizerCloneableOptions { + RMSpropOptions(double lr = 1e-2); + TORCH_ARG(double, lr) = 1e-2; + TORCH_ARG(double, alpha) = 0.99; + TORCH_ARG(double, eps) = 1e-8; + TORCH_ARG(double, weight_decay) = 0; + TORCH_ARG(double, momentum) = 0; + TORCH_ARG(bool, centered) = false; + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const RMSpropOptions& lhs, + const RMSpropOptions& rhs); + double get_lr() const override; + void set_lr(const double lr) override; +}; + +struct TORCH_API RMSpropParamState + : public OptimizerCloneableParamState { + TORCH_ARG(int64_t, step) = 0; + TORCH_ARG(torch::Tensor, square_avg); + TORCH_ARG(torch::Tensor, momentum_buffer); + TORCH_ARG(torch::Tensor, grad_avg); + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const RMSpropParamState& lhs, + const RMSpropParamState& rhs); +}; + +class TORCH_API RMSprop : public Optimizer { + public: + explicit RMSprop( + const std::vector& param_groups, + RMSpropOptions defaults = {}) + : Optimizer(param_groups, std::make_unique(defaults)) { + TORCH_CHECK(defaults.lr() >= 0, "Invalid learning rate: ", defaults.lr()); + TORCH_CHECK(defaults.eps() >= 0, "Invalid epsilon value: ", defaults.eps()); + TORCH_CHECK( + defaults.momentum() >= 0, + "Invalid momentum value: ", + defaults.momentum()); + TORCH_CHECK( + defaults.weight_decay() >= 0, + "Invalid weight_decay value: ", + defaults.weight_decay()); + TORCH_CHECK( + defaults.alpha() >= 0, "Invalid alpha value: ", defaults.alpha()); + } + + explicit RMSprop(std::vector params, RMSpropOptions defaults = {}) + : RMSprop({OptimizerParamGroup(std::move(params))}, std::move(defaults)) { + } + + torch::Tensor step(LossClosure closure = nullptr) override; + void save(serialize::OutputArchive& archive) const override; + void load(serialize::InputArchive& archive) override; + + private: + template + static void serialize(Self& self, Archive& archive) { + _TORCH_OPTIM_SERIALIZE_WITH_TEMPLATE_ARG(RMSprop); + } +}; +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/schedulers/lr_scheduler.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/schedulers/lr_scheduler.h new file mode 100644 index 0000000000000000000000000000000000000000..8212f150c54e77e4e7eab1e24844f37561433526 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/schedulers/lr_scheduler.h @@ -0,0 +1,38 @@ +#pragma once + +#include + +#include + +namespace torch::optim { + +class TORCH_API LRScheduler { + public: + // This class needs to take a reference of an optimizer from outside such that + // it can modify its learning rates; due to this the lifetime of said + // optimizer must be maintained + LRScheduler(torch::optim::Optimizer& optimizer); + + virtual ~LRScheduler() = default; + + void step(); + + protected: + // A vector of learning rates is calculated and returned from the specific + // subclass. A vector is returned with each element being a separate learning + // rate for each param group - although the normal use case would be to return + // a vector of identical elements. + virtual std::vector get_lrs() = 0; + + // Get current learning rates from the optimizer + std::vector get_current_lrs() const; + + unsigned step_count_{}; + + private: + void set_optimizer_lrs(const std::vector& learning_rates); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + torch::optim::Optimizer& optimizer_; +}; +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/schedulers/reduce_on_plateau_scheduler.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/schedulers/reduce_on_plateau_scheduler.h new file mode 100644 index 0000000000000000000000000000000000000000..94fdad6a0cc070ae64167439bea6c656ebf28c96 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/schedulers/reduce_on_plateau_scheduler.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include + +#include + +#include + +namespace torch::optim { + +class TORCH_API ReduceLROnPlateauScheduler { + public: + enum SchedulerMode { min, max }; + enum ThresholdMode { rel, abs }; + ReduceLROnPlateauScheduler( + Optimizer& optimizer, + SchedulerMode mode = min, + float factor = 0.1, + int patience = 10, + double threshold = 1e-4, + ThresholdMode threshold_mode = rel, + int cooldown = 0, + const std::vector& min_lr = std::vector(), + double eps = 1e-8, + bool verbose = false); + + virtual ~ReduceLROnPlateauScheduler() = default; + + void step(float metric); + + private: + void reset(); + void reduce_lr(int epoch); + bool in_cooldown() const; + bool is_better(float a); + void init_is_better( + SchedulerMode mode, + double threshold, + ThresholdMode threshold_mode); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Optimizer& optimizer; + SchedulerMode mode{}; + float mode_worse{}; + float factor; + int patience; + double threshold{}; + ThresholdMode threshold_mode{}; + int cooldown{}; + int cooldown_counter{}; + std::vector min_lrs; + double eps; + float best{}; + bool verbose; + int last_epoch{}; + int num_bad_epochs{}; +}; +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/schedulers/step_lr.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/schedulers/step_lr.h new file mode 100644 index 0000000000000000000000000000000000000000..afa3eacee15006ee4ea94712523ec0484585edf2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/schedulers/step_lr.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace torch::optim { + +class TORCH_API StepLR : public LRScheduler { + public: + StepLR( + torch::optim::Optimizer& optimizer, + const unsigned step_size, + const double gamma = 0.1); + + private: + std::vector get_lrs() override; + + const unsigned step_size_; + const double gamma_; +}; +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/serialize.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/serialize.h new file mode 100644 index 0000000000000000000000000000000000000000..e10c2637747b5d9564247776f4544b3a964e3f28 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/serialize.h @@ -0,0 +1,315 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace torch::optim { +namespace detail { +// Utility function to save state +template +void serialize( + serialize::OutputArchive& archive, + const ska::flat_hash_map>& + state) { + for (const auto& item : state) { + serialize::OutputArchive param_state_archive(archive.compilation_unit()); + std::string tensorimpl_key = + std::to_string(reinterpret_cast(item.first)); + const DerivedOptimizerParamState& curr_state = + static_cast(*(item.second)); + curr_state.serialize(param_state_archive); + archive.write(tensorimpl_key, param_state_archive); + } +} + +// Utility function to load state +template +void serialize( + serialize::InputArchive& archive, + ska::flat_hash_map>& state) { + std::vector tensorimpl_keys = archive.keys(); + for (const std::string& tensorimpl_key : tensorimpl_keys) { + serialize::InputArchive param_state_archive; + archive.read(tensorimpl_key, param_state_archive); + DerivedOptimizerParamState param_state; + param_state.serialize(param_state_archive); + // NOLINTNEXTLINE(performance-no-int-to-ptr) + state[reinterpret_cast(std::stoull(tensorimpl_key))] = + std::make_unique(param_state); + } +} + +// Utility function to save param_groups +template +void serialize( + serialize::OutputArchive& archive, + const std::vector& param_groups) { + archive.write( + "param_groups/size", + torch::tensor(static_cast(param_groups.size()))); + for (const auto i : c10::irange(param_groups.size())) { + serialize::OutputArchive param_group_archive(archive.compilation_unit()); + std::vector params = param_groups[i].params(); + param_group_archive.write( + "params/size", torch::tensor(static_cast(params.size()))); + for (const auto index : c10::irange(params.size())) { + param_group_archive.write( + "params/" + std::to_string(index), + IValue(std::to_string( + reinterpret_cast(params[index].unsafeGetTensorImpl())))); + } + const DerivedOptimizerParamOptions& param_group_options = + static_cast( + param_groups[i].options()); + serialize::OutputArchive param_group_options_archive( + param_group_archive.compilation_unit()); + param_group_options.serialize(param_group_options_archive); + param_group_archive.write("options", param_group_options_archive); + archive.write("param_groups/" + std::to_string(i), param_group_archive); + } +} + +// Utility function to load param_groups +// We take as input vector of pair of string and unique_ptr to optimizer options +// so that we can retain the state for each param by using the old tensor impl +// keys (saved during serialization) and map the new tensor impl keys to the +// correct state for each param +template +void serialize( + serialize::InputArchive& archive, + std::vector< + std::pair, std::unique_ptr>>& + param_groups) { + torch::Tensor param_groups_size_tensor; + archive.read("param_groups/size", param_groups_size_tensor); + const int64_t param_groups_size = param_groups_size_tensor.item(); + for (const auto i : c10::irange(param_groups_size)) { + serialize::InputArchive param_group_archive; + archive.read("param_groups/" + std::to_string(i), param_group_archive); + torch::Tensor size_tensor; + param_group_archive.read("params/size", size_tensor); + const int64_t size = size_tensor.item(); + std::vector params; + for (const auto index : c10::irange(size)) { + IValue ivalue; + param_group_archive.read("params/" + std::to_string(index), ivalue); + std::string element = ivalue.toStringRef(); + params.emplace_back(element); + } + serialize::InputArchive param_group_options_archive; + param_group_archive.read("options", param_group_options_archive); + DerivedOptimizerParamOptions param_group_options(0); + param_group_options.serialize(param_group_options_archive); + param_groups.emplace_back(std::make_pair( + params, + std::make_unique(param_group_options))); + } +} +} // namespace detail + +// Note: These functions are all called `serialize()` so they can be called +// inside a template where the archive type is a template type and can thus be +// passed such that the appropriate overload is selected. + +/// Utility function to save a value of `int64_t` type. +void serialize( + serialize::OutputArchive& archive, + const std::string& key, + const int64_t& value); + +/// Utility function to load a value of `int64_t` type. +void serialize( + serialize::InputArchive& archive, + const std::string& key, + int64_t& value); + +/// Utility function to save a vector of step buffers. +void serialize( + serialize::OutputArchive& archive, + const std::string& key, + const std::vector& steps); + +/// Utility function to load a vector of step buffers. +void serialize( + serialize::InputArchive& archive, + const std::string& key, + std::vector& steps); + +// Utility function to save state and param_groups +template < + typename DerivedOptimizerParamState, + typename DerivedOptimizerParamOptions> +void serialize(serialize::OutputArchive& archive, const Optimizer& optimizer) { + archive.write("pytorch_version", IValue("1.5.0")); + serialize::OutputArchive state_archive(archive.compilation_unit()); + detail::serialize( + state_archive, optimizer.state()); + archive.write("state", state_archive); + + serialize::OutputArchive param_groups_archive(archive.compilation_unit()); + detail::serialize( + param_groups_archive, optimizer.param_groups()); + archive.write("param_groups", param_groups_archive); +} + +// Utility function to load state and param_groups and update state +template < + typename DerivedOptimizerParamState, + typename DerivedOptimizerParamOptions> +void serialize(serialize::InputArchive& archive, Optimizer& optimizer) { + IValue pytorch_version; + archive.read("pytorch_version", pytorch_version); + TORCH_INTERNAL_ASSERT(pytorch_version.toStringRef() == "1.5.0"); + serialize::InputArchive state_archive; + archive.read("state", state_archive); + ska::flat_hash_map> saved_state; + detail::serialize(state_archive, saved_state); + + serialize::InputArchive param_groups_archive; + archive.read("param_groups", param_groups_archive); + std::vector< + std::pair, std::unique_ptr>> + saved_param_groups; + detail::serialize( + param_groups_archive, saved_param_groups); + + // update state and optimizer options + TORCH_CHECK( + saved_param_groups.size() == optimizer.param_groups().size(), + "loaded state dict has a different number of parameter groups"); + for (const auto i : c10::irange(saved_param_groups.size())) { + std::vector param_group_old_keys = saved_param_groups[i].first; + std::vector params = optimizer.param_groups()[i].params(); + TORCH_CHECK( + param_group_old_keys.size() == params.size(), + "loaded state dict contains a parameter group that has a different size than the optimizer's parameter group"); + + for (const auto idx : c10::irange(params.size())) { + auto param_group_old_key = + // NOLINTNEXTLINE(performance-no-int-to-ptr) + reinterpret_cast(std::stoull(param_group_old_keys[idx])); + if (saved_state.find(param_group_old_key) != saved_state.end()) { + optimizer.state()[params[idx].unsafeGetTensorImpl()] = + std::move(saved_state[param_group_old_key]); + } + } + + auto& saved_options = reinterpret_cast( + *saved_param_groups[i].second); + auto& current_options = reinterpret_cast( + optimizer.param_groups()[i].options()); + current_options = saved_options; + } +} + +/// Utility function to save a vector of buffers. +template +void serialize( + serialize::OutputArchive& archive, + const std::string& key, + const BufferContainer& buffers) { + archive.write( + key + "/size", torch::tensor(static_cast(buffers.size()))); + for (const auto index : c10::irange(buffers.size())) { + archive.write( + key + "/" + std::to_string(index), buffers[index], /*is_buffer=*/true); + } +} + +/// Utility function to load a vector of buffers. +template +void serialize( + serialize::InputArchive& archive, + const std::string& key, + BufferContainer& buffers) { + buffers.clear(); + torch::Tensor size_tensor; + archive.read(key + "/size", size_tensor); + const size_t size = size_tensor.item(); + for (const auto index : c10::irange(size)) { + buffers.emplace_back(); + archive.read( + key + "/" + std::to_string(index), buffers.back(), /*is_buffer=*/true); + } +} + +template +c10::List deque_to_list(const std::deque& dq) { + c10::List list; + list.reserve(dq.size()); + for (const auto& e : dq) { + list.emplace_back(e); + } + return list; +} + +template +std::deque list_to_deque(const c10::List& list) { + std::deque dq; + for (const auto& e : list) { + dq.emplace_back(e); + } + return dq; +} + +#define _TORCH_OPTIM_SERIALIZE(name) \ + torch::optim::serialize(archive, #name, self.name) + +#define _TORCH_OPTIM_SERIALIZE_WITH_TEMPLATE_ARG(OptimizerName) \ + torch::optim::serialize( \ + archive, self) + +#define _TORCH_OPTIM_SERIALIZE_TORCH_ARG(name) \ + { \ + auto ivalue = torch::IValue(name()); \ + /* do not serialize if name is an undefined tensor*/ \ + if (!(ivalue.isTensor() && \ + ivalue.unsafeToTensorImpl() == \ + at::UndefinedTensorImpl::singleton())) { \ + archive.write(#name, ivalue); \ + } \ + } + +#define _TORCH_OPTIM_SERIALIZE_TORCH_ARG_DEQUE(name) \ + { \ + c10::IValue ivalue = torch::IValue(deque_to_list(name())); \ + archive.write(#name, ivalue); \ + } + +#define _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(T, name) \ + { \ + c10::IValue ivalue; \ + bool exists = archive.try_read(#name, ivalue); \ + if (exists) { \ + name(ivalue.to()); \ + } else { \ + constexpr bool is_tensor_type = std::is_base_of_v; \ + TORCH_INTERNAL_ASSERT(is_tensor_type); \ + } \ + } + +#define _TORCH_OPTIM_DESERIALIZE_TORCH_ARG_OPTIONAL(T, name) \ + { \ + c10::IValue ivalue; \ + bool exists = archive.try_read(#name, ivalue); \ + if (exists) { \ + name(ivalue.toOptional()); \ + } \ + } + +#define _TORCH_OPTIM_DESERIALIZE_TORCH_ARG_DEQUE(T, name) \ + { \ + c10::IValue ivalue; \ + archive.read(#name, ivalue); \ + auto list = ivalue.to>(); \ + name(list_to_deque(list)); \ + } + +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/sgd.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/sgd.h new file mode 100644 index 0000000000000000000000000000000000000000..0984ba25868d428d9217eeefbcbfdd5ddb439d7d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/optim/sgd.h @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace torch::serialize { +class OutputArchive; +class InputArchive; +} // namespace torch::serialize + +namespace torch::optim { + +struct TORCH_API SGDOptions : public OptimizerCloneableOptions { + SGDOptions(double lr); + TORCH_ARG(double, lr); + TORCH_ARG(double, momentum) = 0; + TORCH_ARG(double, dampening) = 0; + TORCH_ARG(double, weight_decay) = 0; + TORCH_ARG(bool, nesterov) = false; + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const SGDOptions& lhs, + const SGDOptions& rhs); + double get_lr() const override; + void set_lr(const double lr) override; +}; + +struct TORCH_API SGDParamState + : public OptimizerCloneableParamState { + TORCH_ARG(torch::Tensor, momentum_buffer); + + public: + void serialize(torch::serialize::InputArchive& archive) override; + void serialize(torch::serialize::OutputArchive& archive) const override; + TORCH_API friend bool operator==( + const SGDParamState& lhs, + const SGDParamState& rhs); +}; + +class TORCH_API SGD : public Optimizer { + public: + explicit SGD( + const std::vector& param_groups, + SGDOptions defaults) + : Optimizer(param_groups, std::make_unique(defaults)) { + TORCH_CHECK(defaults.lr() >= 0, "Invalid learning rate: ", defaults.lr()); + TORCH_CHECK( + defaults.momentum() >= 0, + "Invalid momentum value: ", + defaults.momentum()); + TORCH_CHECK( + defaults.weight_decay() >= 0, + "Invalid weight_decay value: ", + defaults.weight_decay()); + TORCH_CHECK( + !defaults.nesterov() || + (defaults.momentum() > 0 && defaults.dampening() == 0), + "Nesterov momentum requires a momentum and zero dampening"); + } + + explicit SGD(std::vector params, SGDOptions defaults) + : SGD({OptimizerParamGroup(std::move(params))}, std::move(defaults)) {} + + torch::Tensor step(LossClosure closure = nullptr) override; + + void save(serialize::OutputArchive& archive) const override; + void load(serialize::InputArchive& archive) override; + + private: + template + static void serialize(Self& self, Archive& archive) { + _TORCH_OPTIM_SERIALIZE_WITH_TEMPLATE_ARG(SGD); + } +}; +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/ordered_dict.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/ordered_dict.h new file mode 100644 index 0000000000000000000000000000000000000000..4eed65300bd989888973db3d080c8538adc306f2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/ordered_dict.h @@ -0,0 +1,516 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace torch { +/// An ordered dictionary implementation, akin to Python's `OrderedDict`. +template +class OrderedDict { + public: + /// A (key, value) pair. + class Item; + + // The lifetime of an iterator is bound to the lifetime of the `OrderedDict`. + // Further, any `insert()` operation may invalidate all iterators + // pointing into the vector. + using Iterator = typename std::vector::iterator; + using ConstIterator = typename std::vector::const_iterator; + + /// Constructs the `OrderedDict` with a short description of the kinds of keys + /// stored in the `OrderedDict`. This description is used in error messages + /// thrown by the `OrderedDict`. + explicit OrderedDict(std::string key_description = "Key"); + + /// Copy constructs this `OrderedDict` from `other`. + OrderedDict(const OrderedDict& other); + + /// Assigns items from `other` to this `OrderedDict`. + OrderedDict& operator=(const OrderedDict& other); + + // NB: Move works by default, because you can move-construct vectors of const + // values. I tried to make this noexcept (conditional on the move constructors + // of index_ and items_ being noexcept) but the obvious spelling didn't + // compile on Windows. + OrderedDict(OrderedDict&& other) noexcept = default; + OrderedDict& operator=(OrderedDict&& other) noexcept = default; + + ~OrderedDict() = default; + + /// Constructs a new `OrderedDict` and pre-populates it with the given + /// `Item`s. + /*implicit */ OrderedDict(std::initializer_list initializer_list); + + /// Returns the key description string the `OrderedDict` was constructed with. + const std::string& key_description() const noexcept; + + // Element Access + + /// Returns the very first item in the `OrderedDict` and throws an exception + /// if it is empty. + Item& front(); + + /// Returns the very first item in the `OrderedDict` and throws an exception + /// if it is empty. + const Item& front() const; + + /// Returns the very last item in the `OrderedDict` and throws an exception + /// if it is empty. + Item& back(); + + /// Returns the very last item in the `OrderedDict` and throws an exception + /// if it is empty. + const Item& back() const; + + /// Returns the item at the `index`-th position in the `OrderedDict`. Throws + /// an exception if the index is out of bounds. + Item& operator[](size_t index); + + /// Returns the item at the `index`-th position in the `OrderedDict`. Throws + /// an exception if the index is out of bounds. + const Item& operator[](size_t index) const; + + /// Returns the value associated with the given `key`. Throws an exception if + /// no such key is stored in the `OrderedDict`. Use `find()` for a + /// non-throwing way of accessing a value if it is present. + Value& operator[](const Key& key); + + /// Returns the value associated with the given `key`. Throws an exception if + /// no such key is stored in the `OrderedDict`. Use `find()` for a + /// non-throwing way of accessing a value if it is present. + const Value& operator[](const Key& key) const; + + // Lookup + + /// Returns a pointer to the value associated with the given key, or a + /// `nullptr` if no such key is stored in the `OrderedDict`. + Value* find(const Key& key) noexcept; + + /// Returns a pointer to the value associated with the given key, or a + /// `nullptr` if no such key is stored in the `OrderedDict`. + const Value* find(const Key& key) const noexcept; + + /// Returns true if the key is present in the `OrderedDict`. + bool contains(const Key& key) const noexcept; + + // Iterators + + /// Returns an iterator to the first item in the `OrderedDict`. Iteration is + /// ordered. + Iterator begin(); + + /// Returns an iterator to the first item in the `OrderedDict`. Iteration is + /// ordered. + ConstIterator begin() const; + + /// Returns an iterator one past the last item in the `OrderedDict`. + Iterator end(); + + /// Returns an iterator one past the last item in the `OrderedDict`. + ConstIterator end() const; + + // Capacity + + /// Returns the number of items currently stored in the `OrderedDict`. + size_t size() const noexcept; + + /// Returns true if the `OrderedDict` contains no elements. + bool is_empty() const noexcept; + + /// Resizes internal storage to fit at least `requested_capacity` items + /// without requiring reallocation. + void reserve(size_t requested_capacity); + + // Modifiers + + /// Inserts a new `(key, value)` pair into the `OrderedDict`. Throws an + /// exception if the key is already present. If insertion is successful, + /// immediately returns a reference to the inserted value. + template + Value& insert(K&& key, V&& value); + + /// Inserts a new `(key, value)` pair into the `OrderedDict`. Throws an + /// exception if the key is already present. If insertion is successful, + /// immediately returns a reference to the inserted value. + Value& insert(Key key, Value&& value); + + /// Inserts all items from `other` into this `OrderedDict`. If any key from + /// `other` is already present in this `OrderedDict`, an exception is thrown. + void update(OrderedDict&& other); + + /// Inserts all items from `other` into this `OrderedDict`. If any key from + /// `other` is already present in this `OrderedDict`, an exception is thrown. + void update(const OrderedDict& other); + + /// Removes the item that has `key` from this `OrderedDict` if exists and if + /// it doesn't an exception is thrown. + void erase(const Key& key); + + /// Removes all items from this `OrderedDict`. + void clear(); + + // Observers + + /// Returns the items stored in the `OrderedDict`. + const std::vector& items() const noexcept; + + /// Returns a newly allocated vector and copies all keys from this + /// `OrderedDict` into the vector. + ::std::vector keys() const; + + /// Returns a newly allocated vector and copies all values from this + /// `OrderedDict` into the vector. + ::std::vector values() const; + + /// Returns a newly allocated vector and copies all keys and values from this + /// `OrderedDict` into a vector of `std::pair`. + ::std::vector> pairs() const; + + /// Returns true if both dicts contain the same keys and values, in the same + /// order. + template + friend bool operator==( + const OrderedDict& a, + const OrderedDict& b); + + private: + /// A mapping from a key to an index into the `items_` vector. + ::std::unordered_map index_; + + /// The items stored in the `OrderedDict`. + ::std::vector items_; + + /// A description of the keys stored in the `OrderedDict`. + ::std::string key_description_{"Key"}; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ OrderedDict::Item ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +template +class OrderedDict::Item { + public: + /// Constructs a new item. + Item(Key key, Value value) : pair_(std::move(key), std::move(value)) {} + + /// Returns a reference to the value. + Value& operator*() { + return value(); + } + + /// Returns a reference to the value. + const Value& operator*() const { + return value(); + } + + /// Allows access to the value using the arrow operator. + Value* operator->() { + return &value(); + } + + /// Allows access to the value using the arrow operator. + const Value* operator->() const { + return &value(); + } + + /// Returns a reference to the key. + const Key& key() const noexcept { + return pair_.first; + } + + /// Returns a reference to the value. + Value& value() noexcept { + return pair_.second; + } + + /// Returns a reference to the value. + const Value& value() const noexcept { + return pair_.second; + } + + /// Returns a `(key, value)` pair. + const std::pair& pair() const noexcept { + return pair_; + } + + private: + /// This is stored as an std::pair because it will make Python binding a lot, + /// lot easier. + ::std::pair pair_; +}; + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ OrderedDict ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +template +OrderedDict::OrderedDict(std::string key_description) + : key_description_(std::move(key_description)) {} + +template +OrderedDict::OrderedDict(const OrderedDict& other) + : index_(other.index_), key_description_(other.key_description_) { + // Copy we have to do ourselves, because items' keys are const, so we have to + // re-insert the items. + for (const auto& item : other.items_) { + items_.push_back(item); + } +} + +template +OrderedDict& OrderedDict::operator=( + const OrderedDict& other) { + index_ = other.index_; + items_.clear(); + for (auto& item : other.items_) { + items_.push_back(item); + } + key_description_ = other.key_description_; + return *this; +} + +template +OrderedDict::OrderedDict( + std::initializer_list initializer_list) + : OrderedDict("Key") { + items_.reserve(initializer_list.size()); + for (auto& item : initializer_list) { + // Copy the key here and move it into the index. + items_.emplace_back(item.key(), std::move(item.value())); + index_.emplace(std::move(item.key()), size() - 1); + } +} + +template +typename OrderedDict::Iterator OrderedDict::begin() { + return items_.begin(); +} + +template +typename OrderedDict::ConstIterator OrderedDict::begin() + const { + return items_.begin(); +} + +template +typename OrderedDict::Iterator OrderedDict::end() { + return items_.end(); +} + +template +typename OrderedDict::ConstIterator OrderedDict::end() + const { + return items_.end(); +} + +template +typename OrderedDict::Item& OrderedDict::front() { + TORCH_CHECK(!items_.empty(), "Called front() on an empty OrderedDict"); + return items_.front(); +} + +template +const typename OrderedDict::Item& OrderedDict::front() + const { + TORCH_CHECK(!items_.empty(), "Called front() on an empty OrderedDict"); + return items_.front(); +} + +template +typename OrderedDict::Item& OrderedDict::back() { + TORCH_CHECK(!items_.empty(), "Called back() on an empty OrderedDict"); + return items_.back(); +} + +template +const typename OrderedDict::Item& OrderedDict::back() + const { + TORCH_CHECK(!items_.empty(), "Called back() on an empty OrderedDict"); + return items_.back(); +} + +template +typename OrderedDict::Item& OrderedDict::operator[]( + size_t index) { + TORCH_CHECK(index < items_.size(), "Index ", index, " is out of bounds"); + return items_[index]; +} + +template +const typename OrderedDict::Item& OrderedDict:: +operator[](size_t index) const { + TORCH_CHECK(index < items_.size(), "Index ", index, " is out of bounds"); + return items_[index]; +} + +template +Value& OrderedDict::operator[](const Key& key) { + if (auto* value = find(key)) { + return *value; + } + TORCH_CHECK(false, key_description_, " '", key, "' is not defined"); +} + +template +const Value& OrderedDict::operator[](const Key& key) const { + if (auto* value = find(key)) { + return *value; + } + TORCH_CHECK(false, key_description_, " '", key, "' is not defined"); +} + +template +template +Value& OrderedDict::insert(K&& key, V&& value) { + TORCH_CHECK( + index_.count(key) == 0, key_description_, " '", key, "' already defined"); + // Copy `key` here and move it into the index. + items_.emplace_back(key, std::forward(value)); + index_.emplace(std::forward(key), size() - 1); + return items_.back().value(); +} + +template +Value& OrderedDict::insert(Key key, Value&& value) { + return insert(std::move(key), std::move(value)); +} + +template +void OrderedDict::update(OrderedDict&& other) { + reserve(size() + other.size()); + for (auto&& item : std::move(other)) { + // We want to call `insert()` to prevent duplicate keys. + insert(std::move(item.key()), std::move(item.value())); + } +} + +template +void OrderedDict::update(const OrderedDict& other) { + reserve(size() + other.size()); + for (auto& item : other) { + // We want to call `insert()` to prevent duplicate keys. + insert(item.key(), item.value()); + } +} + +template +Value* OrderedDict::find(const Key& key) noexcept { + auto iterator = index_.find(key); + if (iterator == index_.end()) { + return nullptr; + } + return &items_[iterator->second].value(); +} + +template +const Value* OrderedDict::find(const Key& key) const noexcept { + auto iterator = index_.find(key); + if (iterator == index_.end()) { + return nullptr; + } + return &items_[iterator->second].value(); +} + +template +void OrderedDict::erase(const Key& key) { + auto it = index_.find(key); + TORCH_CHECK(it != index_.end(), "Key '", key, "' doesn't exist"); + + auto index = it->second; + index_.erase(it); + items_.erase(items_.begin() + index); + + for (auto& pair : index_) + if (pair.second > index) + --pair.second; +} + +template +bool OrderedDict::contains(const Key& key) const noexcept { + return find(key) != nullptr; +} + +template +void OrderedDict::clear() { + index_.clear(); + items_.clear(); +} + +template +size_t OrderedDict::size() const noexcept { + return items_.size(); +} + +template +bool OrderedDict::is_empty() const noexcept { + return items_.empty(); +} + +template +const std::string& OrderedDict::key_description() const noexcept { + return key_description_; +} + +template +const std::vector::Item>& OrderedDict< + Key, + Value>::items() const noexcept { + return items_; +} + +template +::std::vector OrderedDict::keys() const { + std::vector keys; + keys.reserve(size()); + for (const auto& item : items_) { + keys.push_back(item.key()); + } + return keys; +} + +template +::std::vector OrderedDict::values() const { + std::vector values; + values.reserve(size()); + for (const auto& item : items_) { + values.push_back(item.value()); + } + return values; +} + +template +::std::vector> OrderedDict::pairs() const { + std::vector> values; + values.reserve(size()); + for (const auto& item : items_) { + values.push_back(item.pair()); + } + return values; +} + +template +void OrderedDict::reserve(size_t requested_capacity) { + index_.reserve(requested_capacity); + items_.reserve(requested_capacity); +} + +template +bool operator==( + const torch::OrderedDict& a, + const torch::OrderedDict& b) { + using Item = typename torch::OrderedDict::Item; + if (a.index_ != b.index_) + return false; + if (a.items_.size() != b.items_.size()) + return false; + // NOTE: There's no point in comparing keys for items_, as we already know + // that index is equal. + return std::equal( + a.items_.begin(), + a.items_.end(), + b.items_.begin(), + [](const Item& a, const Item& b) { return a.value() == b.value(); }); +} + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/print.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/print.h new file mode 100644 index 0000000000000000000000000000000000000000..7fb38d2795bed414a96842e9496b2f523831814a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/print.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace torch { + +/// Set whether to use scientific notation when printing tensors +/// +/// When enabled (default), tensors with large or small values will use +/// scientific notation (e.g., 1.0000e+05). When disabled, they will use +/// fixed-point notation (e.g., 100000.0). +/// +/// \param enabled Whether to enable scientific notation +TORCH_API void set_printoption_sci_mode(bool enabled); + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/python.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/python.h new file mode 100644 index 0000000000000000000000000000000000000000..2174d87f6ea8b7b9e2f2ae149359299156f203e5 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/python.h @@ -0,0 +1,259 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace torch::python { +namespace detail { +inline Device py_object_to_device(py::object object) { + PyObject* obj = object.ptr(); + if (THPDevice_Check(obj)) { + return reinterpret_cast(obj)->device; + } + TORCH_CHECK_TYPE(false, "Expected device"); +} + +inline Dtype py_object_to_dtype(py::object object) { + PyObject* obj = object.ptr(); + if (THPDtype_Check(obj)) { + return reinterpret_cast(obj)->scalar_type; + } + TORCH_CHECK_TYPE(false, "Expected dtype"); +} + +template +using PyModuleClass = + py::class_>; + +/// Dynamically creates a subclass of `torch.nn.cpp.ModuleWrapper` that is also +/// a subclass of `torch.nn.Module`, and passes it the user-provided C++ module +/// to which it delegates all calls. +template +void bind_cpp_module_wrapper( + const py::module& module, + PyModuleClass cpp_class, + const char* name) { + // Grab the `torch.nn.cpp.ModuleWrapper` class, which we'll subclass + // with a dynamically created class below. + py::object cpp_module = + py::module::import("torch.nn.cpp").attr("ModuleWrapper"); + + // Grab the `type` class which we'll use as a metaclass to create a new class + // dynamically. + py::object type_metaclass = + py::reinterpret_borrow((PyObject*)&PyType_Type); + + // The `ModuleWrapper` constructor copies all functions to its own `__dict__` + // in its constructor, but we do need to give our dynamic class a constructor. + // Inside, we construct an instance of the original C++ module we're binding + // (the `torch::nn::Module` subclass), and then forward it to the + // `ModuleWrapper` constructor. + py::dict attributes; + + // `type()` always needs a `str`, but pybind11's `str()` method always creates + // a `unicode` object. + py::object name_str = py::str(name); + + // Dynamically create the subclass of `ModuleWrapper`, which is a subclass of + // `torch.nn.Module`, and will delegate all calls to the C++ module we're + // binding. + py::object wrapper_class = + type_metaclass(name_str, py::make_tuple(cpp_module), attributes); + + // The constructor of the dynamic class calls `ModuleWrapper.__init__()`, + // which replaces its methods with those of the C++ module. + wrapper_class.attr("__init__") = py::cpp_function( + [cpp_module, cpp_class]( + const py::object& self, + const py::args& args, + const py::kwargs& kwargs) { + cpp_module.attr("__init__")(self, cpp_class(*args, **kwargs)); + }, + py::is_method(wrapper_class)); + + // Calling `my_module.my_class` now means that `my_class` is a subclass of + // `ModuleWrapper`, and whose methods call into the C++ module we're binding. + module.attr(name) = wrapper_class; +} +} // namespace detail + +/// Adds method bindings for a pybind11 `class_` that binds an `nn::Module` +/// subclass. +/// +/// Say you have a pybind11 class object created with `py::class_(m, +/// "Net")`. This function will add all the necessary `.def()` calls to bind the +/// `nn::Module` base class' methods, such as `train()`, `eval()` etc. into +/// Python. +/// +/// Users should prefer to use `bind_module` if possible. +template +py::class_ add_module_bindings( + py::class_ module) { + // clang-format off + return module + .def("train", + [](ModuleType& module, bool mode) { module.train(mode); }, + py::arg("mode") = true) + .def("eval", [](ModuleType& module) { module.eval(); }) + .def("clone", [](ModuleType& module) { return module.clone(); }) + .def_property_readonly( + "training", [](ModuleType& module) { return module.is_training(); }) + .def("zero_grad", [](ModuleType& module) { module.zero_grad(); }) + .def_property_readonly( "_parameters", [](ModuleType& module) { + return module.named_parameters(/*recurse=*/false); + }) + .def("parameters", [](ModuleType& module, bool recurse) { + return module.parameters(recurse); + }, + py::arg("recurse") = true) + .def("named_parameters", [](ModuleType& module, bool recurse) { + return module.named_parameters(recurse); + }, + py::arg("recurse") = true) + .def_property_readonly("_buffers", [](ModuleType& module) { + return module.named_buffers(/*recurse=*/false); + }) + .def("buffers", [](ModuleType& module, bool recurse) { + return module.buffers(recurse); }, + py::arg("recurse") = true) + .def("named_buffers", [](ModuleType& module, bool recurse) { + return module.named_buffers(recurse); + }, + py::arg("recurse") = true) + .def_property_readonly( + "_modules", [](ModuleType& module) { return module.named_children(); }) + .def("modules", [](ModuleType& module) { return module.modules(); }) + .def("named_modules", + [](ModuleType& module, const py::object& /* unused */, std::string prefix, bool remove_duplicate /* unused */) { + return module.named_modules(std::move(prefix)); + }, + py::arg("memo") = py::none(), + py::arg("prefix") = std::string(), + py::arg("remove_duplicate") = true) + .def("children", [](ModuleType& module) { return module.children(); }) + .def("named_children", + [](ModuleType& module) { return module.named_children(); }) + .def("to", [](ModuleType& module, py::object object, bool non_blocking) { + if (THPDevice_Check(object.ptr())) { + module.to( + reinterpret_cast(object.ptr())->device, + non_blocking); + } else { + module.to(detail::py_object_to_dtype(object), non_blocking); + } + }, + py::arg("dtype_or_device"), + py::arg("non_blocking") = false) + .def("to", + [](ModuleType& module, + const py::object& device, + const py::object& dtype, + bool non_blocking) { + if (device.is_none()) { + module.to(detail::py_object_to_dtype(dtype), non_blocking); + } else if (dtype.is_none()) { + module.to(detail::py_object_to_device(device), non_blocking); + } else { + module.to( + detail::py_object_to_device(device), + detail::py_object_to_dtype(dtype), + non_blocking); + } + }, + py::arg("device"), + py::arg("dtype"), + py::arg("non_blocking") = false) + .def("cuda", [](ModuleType& module) { module.to(kCUDA); }) + .def("cpu", [](ModuleType& module) { module.to(kCPU); }) + .def("float", [](ModuleType& module) { module.to(kFloat32); }) + .def("double", [](ModuleType& module) { module.to(kFloat64); }) + .def("half", [](ModuleType& module) { module.to(kFloat16); }) + .def("__str__", [](ModuleType& module) { return module.name(); }) + .def("__repr__", [](ModuleType& module) { return module.name(); }); + // clang-format on +} + +/// Creates a pybind11 class object for an `nn::Module` subclass type and adds +/// default bindings. +/// +/// After adding the default bindings, the class object is returned, such that +/// you can add more bindings. +/// +/// Example usage: +/// \rst +/// .. code-block:: cpp +/// +/// struct Net : torch::nn::Module { +/// Net(int in, int out) { } +/// torch::Tensor forward(torch::Tensor x) { return x; } +/// }; +/// +/// PYBIND11_MODULE(my_module, m) { +/// torch::python::bind_module(m, "Net") +/// .def(py::init()) +/// .def("forward", &Net::forward); +/// } +/// \endrst +template +std::enable_if_t< + !torch::detail::has_forward::value || force_enable, + detail::PyModuleClass> +bind_module(py::module module, const char* name) { + py::module cpp = module.def_submodule("cpp"); + auto cpp_class = + add_module_bindings(detail::PyModuleClass(cpp, name)); + detail::bind_cpp_module_wrapper(module, cpp_class, name); + return cpp_class; +} + +/// Creates a pybind11 class object for an `nn::Module` subclass type and adds +/// default bindings. +/// +/// After adding the default bindings, the class object is returned, such that +/// you can add more bindings. +/// +/// If the class has a `forward()` method, it is automatically exposed as +/// `forward()` and `__call__` in Python. +/// +/// Example usage: +/// \rst +/// .. code-block:: cpp +/// +/// struct Net : torch::nn::Module { +/// Net(int in, int out) { } +/// torch::Tensor forward(torch::Tensor x) { return x; } +/// }; +/// +/// PYBIND11_MODULE(my_module, m) { +/// torch::python::bind_module(m, "Net") +/// .def(py::init()) +/// .def("forward", &Net::forward); +/// } +/// \endrst +template < + typename ModuleType, + typename = std::enable_if_t::value>> +detail::PyModuleClass bind_module( + py::module module, + const char* name) { + return bind_module(module, name) + .def("forward", &ModuleType::forward) + .def("__call__", &ModuleType::forward); +} +} // namespace torch::python diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/python/init.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/python/init.h new file mode 100644 index 0000000000000000000000000000000000000000..716b8b2283919c4b72e53b1e93653f6d391da1ab --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/python/init.h @@ -0,0 +1,8 @@ +#pragma once + +#include + +namespace torch::python { +/// Initializes Python bindings for the C++ frontend. +void init_bindings(PyObject* module); +} // namespace torch::python diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize.h new file mode 100644 index 0000000000000000000000000000000000000000..8aa3e2d7c4954683e5a73c20d28061d1bd0464d7 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize.h @@ -0,0 +1,144 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace torch { + +/// Serializes the given `value`. +/// There must be an overload of `operator<<` between `serialize::OutputArchive` +/// and `Value` for this method to be well-formed. Currently, such an overload +/// is provided for (subclasses of): +/// +/// - `torch::nn::Module`, +/// - `torch::optim::Optimizer` +/// - `torch::Tensor` +/// +/// To perform the serialization, a `serialize::OutputArchive` is constructed, +/// and all arguments after the `value` are forwarded to its `save_to` method. +/// For example, you can pass a filename, or an `ostream`. +/// +/// \rst +/// .. code-block:: cpp +/// +/// torch::nn::Linear model(3, 4); +/// torch::save(model, "model.pt"); +/// +/// torch::optim::SGD sgd(model->parameters(), 0.9); // 0.9 is learning rate +/// std::ostringstream stream; +/// // Note that the same stream cannot be used in multiple torch::save(...) +/// // invocations, otherwise the header will be corrupted. +/// torch::save(sgd, stream); +/// +/// auto tensor = torch::ones({3, 4}); +/// torch::save(tensor, "my_tensor.pt"); +/// \endrst +template +void save(const Value& value, SaveToArgs&&... args) { + serialize::OutputArchive archive(std::make_shared()); + archive << value; + archive.save_to(std::forward(args)...); +} + +/// Serializes the given `tensor_vec` of type `std::vector`. +/// +/// To perform the serialization, a `serialize::OutputArchive` is constructed, +/// and all arguments after the `tensor_vec` are forwarded to its `save_to` +/// method. For example, you can pass a filename, or an `ostream`. +/// +/// \rst +/// .. code-block:: cpp +/// +/// std::vector tensor_vec = { torch::randn({1, 2}), +/// torch::randn({3, 4}) }; torch::save(tensor_vec, "my_tensor_vec.pt"); +/// +/// std::vector tensor_vec = { torch::randn({5, 6}), +/// torch::randn({7, 8}) }; std::ostringstream stream; +/// // Note that the same stream cannot be used in multiple torch::save(...) +/// // invocations, otherwise the header will be corrupted. +/// torch::save(tensor_vec, stream); +/// \endrst +template +void save(const std::vector& tensor_vec, SaveToArgs&&... args) { + serialize::OutputArchive archive(std::make_shared()); + for (const auto i : c10::irange(tensor_vec.size())) { + auto& value = tensor_vec[i]; + archive.write(std::to_string(i), value); + } + archive.save_to(std::forward(args)...); +} + +TORCH_API std::vector pickle_save(const torch::IValue& ivalue); +TORCH_API torch::IValue pickle_load(const std::vector& data); + +/// Deserializes the given `value`. +/// There must be an overload of `operator>>` between `serialize::InputArchive` +/// and `Value` for this method to be well-formed. Currently, such an overload +/// is provided for (subclasses of): +/// +/// - `torch::nn::Module`, +/// - `torch::optim::Optimizer` +/// - `torch::Tensor` +/// +/// To perform the serialization, a `serialize::InputArchive` is constructed, +/// and all arguments after the `value` are forwarded to its `load_from` method. +/// For example, you can pass a filename, or an `istream`. +/// +/// \rst +/// .. code-block:: cpp +/// +/// torch::nn::Linear model(3, 4); +/// torch::load(model, "model.pt"); +/// +/// torch::optim::SGD sgd(model->parameters(), 0.9); // 0.9 is learning rate +/// std::istringstream stream("..."); +/// torch::load(sgd, stream); +/// +/// auto tensor = torch::ones({3, 4}); +/// torch::load(tensor, "my_tensor.pt"); +/// \endrst +template +void load(Value& value, LoadFromArgs&&... args) { + serialize::InputArchive archive; + archive.load_from(std::forward(args)...); + archive >> value; +} + +/// Deserializes the given `tensor_vec` of type `std::vector`. +/// +/// To perform the serialization, a `serialize::InputArchive` is constructed, +/// and all arguments after the `value` are forwarded to its `load_from` method. +/// For example, you can pass a filename, or an `istream`. +/// +/// \rst +/// .. code-block:: cpp +/// +/// std::vector tensor_vec; +/// torch::load(tensor_vec, "my_tensor_vec.pt"); +/// +/// std::vector tensor_vec; +/// std::istringstream stream("..."); +/// torch::load(tensor_vec, stream); +/// \endrst +template +void load(std::vector& tensor_vec, LoadFromArgs&&... args) { + serialize::InputArchive archive; + archive.load_from(std::forward(args)...); + + // NOTE: The number of elements in the serialized `std::vector` + // is not known ahead of time, so we need a while-loop to increment the index, + // and use `archive.try_read(...)` to check whether we have reached the end of + // the serialized `std::vector`. + size_t index = 0; + torch::Tensor value; + while (archive.try_read(std::to_string(index), value)) { + tensor_vec.push_back(std::move(value)); + value = torch::Tensor(); + index++; + } +} +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/archive.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/archive.h new file mode 100644 index 0000000000000000000000000000000000000000..3a80de488cc2d22ab3b671b2c422a8b75bef27fd --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/archive.h @@ -0,0 +1,4 @@ +#pragma once + +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/input-archive.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/input-archive.h new file mode 100644 index 0000000000000000000000000000000000000000..e15f25125325a443a2880aec1bced67c6a1f72ea --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/input-archive.h @@ -0,0 +1,115 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace at { +class Tensor; +} // namespace at + +namespace torch { +using at::Tensor; +namespace jit { +struct Module; +} // namespace jit +} // namespace torch + +namespace torch::serialize { + +/// A recursive representation of tensors that can be deserialized from a file +/// or stream. In most cases, users should not have to interact with this class, +/// and should instead use `torch::load`. +class TORCH_API InputArchive final { + public: + /// Default-constructs the `InputArchive`. + InputArchive(); + + // Move is allowed. + InputArchive(InputArchive&&) = default; + InputArchive& operator=(InputArchive&&) = default; + + // Copy is disallowed. + InputArchive(InputArchive&) = delete; + InputArchive& operator=(InputArchive&) = delete; + + ~InputArchive() = default; + + /// Reads an `IValue` associated with a given `key`. + void read(const std::string& key, c10::IValue& ivalue); + + /// Reads an `IValue` associated with a given `key`. If there is no `IValue` + /// associated with the `key`, this returns false, otherwise it returns true. + bool try_read(const std::string& key, c10::IValue& ivalue); + + /// Reads a `tensor` associated with a given `key`. If there is no `tensor` + /// associated with the `key`, this returns false, otherwise it returns true. + /// If the tensor is expected to be a buffer (not differentiable), `is_buffer` + /// must be `true`. + bool try_read(const std::string& key, Tensor& tensor, bool is_buffer = false); + + /// Reads a `tensor` associated with a given `key`. + /// If the tensor is expected to be a buffer (not differentiable), `is_buffer` + /// must be `true`. + void read(const std::string& key, Tensor& tensor, bool is_buffer = false); + + /// Reads a `InputArchive` associated with a given `key`. If there is no + /// `InputArchive` associated with the `key`, this returns false, otherwise + /// it returns true. + bool try_read(const std::string& key, InputArchive& archive); + + /// Reads an `InputArchive` associated with a given `key`. + /// The archive can thereafter be used for further deserialization of the + /// nested data. + void read(const std::string& key, InputArchive& archive); + + /// Loads the `InputArchive` from a serialized representation stored in the + /// file at `filename`. Storage are remapped using device option. If device + /// is not specified, the module is loaded to the original device. + void load_from( + const std::string& filename, + std::optional device = std::nullopt); + + /// Loads the `InputArchive` from a serialized representation stored in the + /// given `stream`. Storage are remapped using device option. If device + /// is not specified, the module is loaded to the original device. + void load_from( + std::istream& stream, + std::optional device = std::nullopt); + + // Loads given the specified flat array. + void load_from( + const char* data, + size_t size, + std::optional device = std::nullopt); + + // Loads given the specified read and size functions. + void load_from( + const std::function& + read_func, + const std::function& size_func, + std::optional device = std::nullopt); + + // Returns the vector of keys in the input archive. + std::vector keys(); + + /// Forwards all arguments to `read()`. + /// Useful for generic code that can be reused for both `InputArchive` and + /// `OutputArchive` (where `operator()` forwards to `write()`). + template + void operator()(Ts&&... ts) { + read(std::forward(ts)...); + } + + private: + jit::Module module_; + std::string hierarchy_prefix_; +}; +} // namespace torch::serialize diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/output-archive.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/output-archive.h new file mode 100644 index 0000000000000000000000000000000000000000..1d6fb7bf211d3e3c11f865a65d10b331979ec123 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/output-archive.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace at { +class Tensor; +} // namespace at + +namespace torch { +using at::Tensor; +namespace jit { +struct Module; +} // namespace jit +} // namespace torch + +namespace torch::serialize { +class TORCH_API OutputArchive final { + public: + explicit OutputArchive(std::shared_ptr cu); + explicit OutputArchive() + : cu_(std::make_shared()), + module_("__torch__.Module", cu_) {} + + // Move is allowed. + OutputArchive(OutputArchive&&) = default; + OutputArchive& operator=(OutputArchive&&) = default; + + // Copy is disallowed. + OutputArchive(OutputArchive&) = delete; + OutputArchive& operator=(OutputArchive&) = delete; + + std::shared_ptr compilation_unit() const { + return cu_; + } + + /// Writes an `IValue` to the `OutputArchive`. + void write(const std::string& key, const c10::IValue& ivalue); + + /// Writes a `(key, tensor)` pair to the `OutputArchive`, and marks it as + /// being or not being a buffer (non-differentiable tensor). + void write( + const std::string& key, + const Tensor& tensor, + bool is_buffer = false); + + /// Writes a nested `OutputArchive` under the given `key` to this + /// `OutputArchive`. + void write(const std::string& key, OutputArchive& nested_archive); + + /// Saves the `OutputArchive` into a serialized representation in a file at + /// `filename`. + void save_to(const std::string& filename); + + /// Saves the `OutputArchive` into a serialized representation into the given + /// `stream`. + void save_to(std::ostream& stream); + + /// Saves the `OutputArchive` into a serialized representation using the + /// given writer function. + void save_to(const std::function& func); + + /// Forwards all arguments to `write()`. + /// Useful for generic code that can be reused for both `OutputArchive` and + /// `InputArchive` (where `operator()` forwards to `read()`). + template + void operator()(Ts&&... ts) { + write(std::forward(ts)...); + } + + private: + std::shared_ptr cu_; + jit::Module module_; +}; +} // namespace torch::serialize diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/tensor.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/tensor.h new file mode 100644 index 0000000000000000000000000000000000000000..c1c43d1def971ff8fcf738cd208f897d580d92de --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/serialize/tensor.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace torch { +inline serialize::OutputArchive& operator<<( + serialize::OutputArchive& archive, + const Tensor& tensor) { + archive.write("0", tensor); + return archive; +} + +inline serialize::InputArchive& operator>>( + serialize::InputArchive& archive, + Tensor& tensor) { + archive.read("0", tensor); + return archive; +} +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/sparse.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/sparse.h new file mode 100644 index 0000000000000000000000000000000000000000..91d0733424ec7305a656a61e93ad0058b300cd8a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/sparse.h @@ -0,0 +1,3 @@ +#pragma once + +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/special.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/special.h new file mode 100644 index 0000000000000000000000000000000000000000..20b88f2dbe84caa435b9829a4373cb3177887f92 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/special.h @@ -0,0 +1,1403 @@ +#pragma once + +#include +#include + +namespace torch::special { + +/// Computes the natural logarithm of the absolute value of the gamma function +/// See https://pytorch.org/docs/main/special.html#torch.special.gammaln. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::gammaln(t); +/// ``` +inline Tensor gammaln(const Tensor& self) { + return torch::special_gammaln(self); +} + +inline Tensor& gammaln_out(Tensor& result, const Tensor& self) { + return torch::special_gammaln_out(result, self); +} + +/// Computes the regularized lower incomplete gamma function +/// See https://pytorch.org/docs/main/special.html#torch.special.gammainc. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// auto s = torch::randn(128, dtype=kDouble); +/// torch::special::gammainc(s, t); +/// ``` +inline Tensor gammainc(const Tensor& self, const Tensor& other) { + return torch::special_gammainc(self, other); +} + +inline Tensor& gammainc_out( + Tensor& result, + const Tensor& self, + const Tensor& other) { + return torch::special_gammainc_out(result, self, other); +} + +/// Computes the regularized upper incomplete gamma function +/// See https://pytorch.org/docs/main/special.html#torch.special.gammainc. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// auto s = torch::randn(128, dtype=kDouble); +/// torch::special::gammaincc(s, t); +/// ``` +inline Tensor gammaincc(const Tensor& self, const Tensor& other) { + return torch::special_gammaincc(self, other); +} + +inline Tensor& gammaincc_out( + Tensor& result, + const Tensor& self, + const Tensor& other) { + return torch::special_gammaincc_out(result, self, other); +} + +/// Computes the multivariate log-gamma function with dimension `p`, elementwise +/// See https://pytorch.org/docs/main/special.html#torch.special.multigammaln. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::multigammaln(t, 1); +/// ``` +inline Tensor multigammaln(const Tensor& self, int64_t p) { + return torch::special_multigammaln(self, p); +} + +inline Tensor& multigammaln_out(Tensor& result, const Tensor& self, int64_t p) { + return torch::special_multigammaln_out(result, self, p); +} + +/// Computes the nth derivative of the digamma function on the input. +/// See https:://pytorch.org/docs/main/special.html#torch.special.polygamma. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::polygamma(2, t); +/// ``` +inline Tensor polygamma(int64_t n, const Tensor& self) { + return torch::special_polygamma(n, self); +} + +inline Tensor& polygamma_out(Tensor& result, int64_t n, const Tensor& self) { + return torch::special_polygamma_out(result, n, self); +} + +/// Computes the logarithmic derivative of the gamma function on input +/// See https://pytorch.org/docs/main/special.html#torch.special.psi +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::psi(t); +/// ``` +inline Tensor psi(const Tensor& self) { + return torch::special_psi(self); +} + +inline Tensor& psi_out(Tensor& result, const Tensor& self) { + return torch::special_psi_out(result, self); +} + +/// Computes the logarithmic derivative of the gamma function on input +/// See https://pytorch.org/docs/main/special.html#torch.special.digamma +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::digamma(t); +/// ``` +inline Tensor digamma(const Tensor& self) { + return torch::special_digamma(self); +} + +inline Tensor& digamma_out(Tensor& result, const Tensor& self) { + return torch::special_digamma_out(result, self); +} + +/// Computes entropy of input, elementwise +/// See https://pytorch.org/docs/main/special.html#torch.special.entr. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::entr(t); +/// ``` +inline Tensor entr(const Tensor& self) { + return torch::special_entr(self); +} + +inline Tensor& entr_out(Tensor& result, const Tensor& self) { + return torch::special_entr_out(result, self); +} + +/// Computes the error function +/// See https://pytorch.org/docs/main/special.html#torch.special.erf. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::erf(t); +/// ``` +inline Tensor erf(const Tensor& self) { + return torch::special_erf(self); +} + +inline Tensor& erf_out(Tensor& result, const Tensor& self) { + return torch::special_erf_out(result, self); +} + +/// Computes the complementary error function +/// See https://pytorch.org/docs/main/special.html#torch.special.erfc. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::erfc(t); +/// ``` +inline Tensor erfc(const Tensor& self) { + return torch::special_erfc(self); +} + +inline Tensor& erfc_out(Tensor& result, const Tensor& self) { + return torch::special_erfc_out(result, self); +} + +/// Computes the scaled complementary error function +/// See https://pytorch.org/docs/main/special.html#torch.special.erfcx. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::erfcx(t); +/// ``` +inline Tensor erfcx(const Tensor& self) { + return torch::special_erfcx(self); +} + +inline Tensor& erfcx_out(Tensor& result, const Tensor& self) { + return torch::special_erfcx_out(result, self); +} + +/// Computes the inverse error function +/// See https://pytorch.org/docs/main/special.html#torch.special.erfinv. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::erfinv(t); +/// ``` +inline Tensor erfinv(const Tensor& self) { + return torch::special_erfinv(self); +} + +inline Tensor& erfinv_out(Tensor& result, const Tensor& self) { + return torch::special_erfinv_out(result, self); +} + +/// Computes the log of summed exponentials of each row of input in the given +/// dimension dim See +/// https://pytorch.org/docs/main/special.html#torch.special.logsumexp. +/// +/// Example: +/// ``` +/// auto t = torch::randn(3, 3); +/// torch::special::logsumexp(t, 1); +/// ``` +inline Tensor logsumexp(const Tensor& self, IntArrayRef dims, bool keepdim) { + return torch::special_logsumexp(self, dims, keepdim); +} + +inline Tensor& logsumexp_out( + Tensor& result, + const Tensor& self, + IntArrayRef dims, + bool keepdim) { + return torch::special_logsumexp_out(result, self, dims, keepdim); +} + +/// Computes the argument, x, for which the area under the Gaussian probability +/// density function (integrated from minus infinity to x) is equal to input, +/// elementwise. See +/// https://pytorch.org/docs/main/special.html#torch.special.ndtri +/// +/// Example: +/// ``` +/// auto t = torch::rand(128, dtype=kDouble); +/// torch::special::ndtri(t); +/// ``` +inline Tensor ndtri(const Tensor& self) { + return torch::special_ndtri(self); +} + +inline Tensor& ndtri_out(Tensor& result, const Tensor& self) { + return torch::special_ndtri_out(result, self); +} + +/// Computes the log of area under the standard Gaussian probability density +/// function, integrated from minus infinity to :attr:`input`, elementwise See +/// https://pytorch.org/docs/main/special.html#torch.special.log_ndtr +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::log_ndtr(t); +/// ``` +inline Tensor log_ndtr(const Tensor& self) { + return torch::special_log_ndtr(self); +} + +inline Tensor& log_ndtr_out(Tensor& result, const Tensor& self) { + return torch::special_log_ndtr_out(result, self); +} + +/// Computes the logit of input, elementwise. +/// See https://pytorch.org/docs/main/special.html#torch.special.logit. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::logit(t); +/// ``` +inline Tensor logit(const Tensor& self) { + return torch::special_logit(self); +} + +inline Tensor& logit_out(Tensor& result, const Tensor& self) { + return torch::special_logit_out(result, self); +} + +/// Computes the expit (also known as the logistic sigmoid function) of input, +/// elementwise See +/// https://pytorch.org/docs/main/special.html#torch.special.expit. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::expit(t); +/// ``` +inline Tensor expit(const Tensor& self) { + return torch::special_expit(self); +} + +inline Tensor& expit_out(Tensor& result, const Tensor& self) { + return torch::special_expit_out(result, self); +} + +/// Computes the base two exponential function of :attr:`input`, elementwise +/// See https://pytorch.org/docs/main/special.html#torch.special.exp2. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::exp2(t); +/// ``` +inline Tensor exp2(const Tensor& self) { + return torch::special_exp2(self); +} + +inline Tensor& exp2_out(Tensor& result, const Tensor& self) { + return torch::special_exp2_out(result, self); +} + +/// Computes the exponential of the elements minus 1, elementwise +/// See https://pytorch.org/docs/main/special.html#torch.special.expm1. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::expm1(t); +/// ``` +inline Tensor expm1(const Tensor& self) { + return torch::special_expm1(self); +} + +inline Tensor& expm1_out(Tensor& result, const Tensor& self) { + return torch::special_expm1_out(result, self); +} + +/// Computes x * log(y) for inputs, elementwise +/// See https://pytorch.org/docs/main/special.html#torch.special.xlogy. +/// +/// Example: +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto y = torch::randn(128, dtype=kDouble); +/// torch::special::xlogy(x, y); +/// ``` +inline Tensor xlogy(const Tensor& self, const Tensor& other) { + return torch::special_xlogy(self, other); +} + +inline Tensor xlogy(const Scalar& self, const Tensor& other) { + return torch::special_xlogy(self, other); +} + +inline Tensor xlogy(const Tensor& self, const Scalar& other) { + return torch::special_xlogy(self, other); +} + +inline Tensor& xlogy_out( + Tensor& result, + const Tensor& self, + const Tensor& other) { + return torch::special_xlogy_out(result, self, other); +} + +inline Tensor& xlogy_out( + Tensor& result, + const Scalar& self, + const Tensor& other) { + return torch::special_xlogy_out(result, self, other); +} + +inline Tensor& xlogy_out( + Tensor& result, + const Tensor& self, + const Scalar& other) { + return torch::special_xlogy_out(result, self, other); +} + +/// Computes x * log1p(y) for inputs, elementwise +/// See https://pytorch.org/docs/main/special.html#torch.special.xlog1py. +/// +/// Example: +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto y = torch::randn(128, dtype=kDouble); +/// torch::special::xlog1py(x, y); +/// ``` +inline Tensor xlog1py(const Tensor& self, const Tensor& other) { + return torch::special_xlog1py(self, other); +} + +inline Tensor xlog1py(const Scalar& self, const Tensor& other) { + return torch::special_xlog1py(self, other); +} + +inline Tensor xlog1py(const Tensor& self, const Scalar& other) { + return torch::special_xlog1py(self, other); +} + +inline Tensor& xlog1py_out( + Tensor& result, + const Tensor& self, + const Tensor& other) { + return torch::special_xlog1py_out(result, self, other); +} + +inline Tensor& xlog1py_out( + Tensor& result, + const Scalar& self, + const Tensor& other) { + return torch::special_xlog1py_out(result, self, other); +} + +inline Tensor& xlog1py_out( + Tensor& result, + const Tensor& self, + const Scalar& other) { + return torch::special_xlog1py_out(result, self, other); +} + +/// Computes Hurwitz Zeta function for inputs, elementwise +/// See https://pytorch.org/docs/main/special.html#torch.special.zeta. +/// +/// Example: +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto y = torch::randn(128, dtype=kDouble); +/// torch::special::zeta(x, y); +/// ``` +inline Tensor zeta(const Tensor& self, const Tensor& other) { + return torch::special_zeta(self, other); +} + +inline Tensor zeta(const Scalar& self, const Tensor& other) { + return torch::special_zeta(self, other); +} + +inline Tensor zeta(const Tensor& self, const Scalar& other) { + return torch::special_zeta(self, other); +} + +inline Tensor& zeta_out( + Tensor& result, + const Tensor& self, + const Tensor& other) { + return torch::special_zeta_out(result, self, other); +} + +inline Tensor& zeta_out( + Tensor& result, + const Scalar& self, + const Tensor& other) { + return torch::special_zeta_out(result, self, other); +} + +inline Tensor& zeta_out( + Tensor& result, + const Tensor& self, + const Scalar& other) { + return torch::special_zeta_out(result, self, other); +} + +/// Computes the zeroth order modified Bessel function of the first kind of +/// input, elementwise See +/// https://pytorch.org/docs/main/special.html#torch.special.i0 +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::i0(t); +/// ``` +inline Tensor i0(const Tensor& self) { + return torch::special_i0(self); +} + +inline Tensor& i0_out(Tensor& result, const Tensor& self) { + return torch::special_i0_out(result, self); +} + +/// Computes the area under the standard Gaussian probability density function, +/// integrated from minus infinity to :attr:`input`, elementwise +/// See https://pytorch.org/docs/main/special.html#torch.special.ndtr +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::ndtr(t); +/// ``` +inline Tensor ndtr(const Tensor& self) { + return torch::special_ndtr(self); +} + +inline Tensor& ndtr_out(Tensor& result, const Tensor& self) { + return torch::special_ndtr_out(result, self); +} + +/// Computes the exponentially scaled zeroth order modified Bessel function of +/// the first kind See +/// https://pytorch.org/docs/main/special.html#torch.special.i0e. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::i0e(t); +/// ``` +inline Tensor i0e(const Tensor& self) { + return torch::special_i0e(self); +} + +inline Tensor& i0e_out(Tensor& result, const Tensor& self) { + return torch::special_i0e_out(result, self); +} + +/// Computes the first order modified Bessel function of the first kind +/// See https://pytorch.org/docs/main/special.html#torch.special.i1. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::i1(t); +/// ``` +inline Tensor i1(const Tensor& self) { + return torch::special_i1(self); +} + +inline Tensor& i1_out(Tensor& result, const Tensor& self) { + return torch::special_i1_out(result, self); +} + +/// Computes the exponentially scaled first order modified Bessel function of +/// the first kind See +/// https://pytorch.org/docs/main/special.html#torch.special.i1e. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::i1e(t); +/// ``` +inline Tensor i1e(const Tensor& self) { + return torch::special_i1e(self); +} + +inline Tensor& i1e_out(Tensor& result, const Tensor& self) { + return torch::special_i1e_out(result, self); +} + +/// Computes the sinc of input, elementwise +/// See https://pytorch.org/docs/main/special.html#torch.special.sinc. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::sinc(t); +/// ``` +inline Tensor sinc(const Tensor& self) { + return torch::special_sinc(self); +} + +inline Tensor& sinc_out(Tensor& result, const Tensor& self) { + return torch::special_sinc_out(result, self); +} + +/// Rounds the elements of the input +/// See https://pytorch.org/docs/main/special.html#torch.special.round. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::round(t); +/// ``` +inline Tensor round(const Tensor& self) { + return torch::special_round(self); +} + +inline Tensor& round_out(Tensor& result, const Tensor& self) { + return torch::special_round_out(result, self); +} + +/// Computes log(1 + x) of the input, elementwise +/// See https://pytorch.org/docs/main/special.html#torch.special.log1p. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, dtype=kDouble); +/// torch::special::log1p(t); +/// ``` +inline Tensor log1p(const Tensor& self) { + return torch::special_log1p(self); +} + +inline Tensor& log1p_out(Tensor& result, const Tensor& self) { + return torch::special_log1p_out(result, self); +} + +/// Computes log followed by softmax(x) of the input +/// See https://pytorch.org/docs/main/special.html#torch.special.log_softmax. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, 128, dtype=kDouble); +/// torch::special::log_softmax(t, 0); +/// ``` +inline Tensor log_softmax( + const Tensor& self, + int64_t dim, + std::optional dtype) { + return torch::special_log_softmax(self, dim, dtype); +} + +/// Computes softmax of the input along a given dimension +/// See https://pytorch.org/docs/main/special.html#torch.special.softmax. +/// +/// Example: +/// ``` +/// auto t = torch::randn(128, 128, dtype=kDouble); +/// torch::special::softmax(t, 0); +/// ``` +inline Tensor softmax( + const Tensor& self, + int64_t dim, + std::optional dtype) { + return torch::special_softmax(self, dim, dtype); +} + +/// Airy function Ai. +/// +/// See https://pytorch.org/docs/main/special.html#torch.special.airy_ai. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::airy_ai(x); +/// ``` +inline Tensor airy_ai(const Tensor& x) { + return torch::special_airy_ai(x); +} + +inline Tensor& airy_ai_out(Tensor& y, const Tensor& x) { + return torch::special_airy_ai_out(y, x); +} + +/// Bessel function of the first kind of order 0. +/// +/// See https://pytorch.org/docs/main/special.html#torch.special.bessel_j0. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::bessel_j0(x); +/// ``` +inline Tensor bessel_j0(const Tensor& self) { + return torch::special_bessel_j0(self); +} + +inline Tensor& bessel_j0_out(Tensor& result, const Tensor& self) { + return torch::special_bessel_j0_out(result, self); +} + +/// Bessel function of the first kind of order 1. +/// +/// See https://pytorch.org/docs/main/special.html#torch.special.bessel_j1. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::bessel_j1(x); +/// ``` +inline Tensor bessel_j1(const Tensor& self) { + return torch::special_bessel_j1(self); +} + +inline Tensor& bessel_j1_out(Tensor& result, const Tensor& self) { + return torch::special_bessel_j1_out(result, self); +} + +/// Bessel function of the second kind of order 0. +/// +/// See https://pytorch.org/docs/main/special.html#torch.special.bessel_y0. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::bessel_y0(x); +/// ``` +inline Tensor bessel_y0(const Tensor& self) { + return torch::special_bessel_y0(self); +} + +inline Tensor& bessel_y0_out(Tensor& result, const Tensor& self) { + return torch::special_bessel_y0_out(result, self); +} + +/// Bessel function of the second kind of order 1. +/// +/// See https://pytorch.org/docs/main/special.html#torch.special.bessel_y1. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::bessel_y1(x); +/// ``` +inline Tensor bessel_y1(const Tensor& self) { + return torch::special_bessel_y1(self); +} + +inline Tensor& bessel_y1_out(Tensor& result, const Tensor& self) { + return torch::special_bessel_y1_out(result, self); +} + +/// Chebyshev polynomial of the first kind. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.chebyshev_polynomial_t. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::chebyshev_polynomial_t(x, n); +/// ``` +inline Tensor chebyshev_polynomial_t(const Tensor& x, const Tensor& n) { + return torch::special_chebyshev_polynomial_t(x, n); +} + +inline Tensor chebyshev_polynomial_t(const Scalar& x, const Tensor& n) { + return torch::special_chebyshev_polynomial_t(x, n); +} + +inline Tensor chebyshev_polynomial_t(const Tensor& x, const Scalar& n) { + return torch::special_chebyshev_polynomial_t(x, n); +} + +inline Tensor& chebyshev_polynomial_t_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_chebyshev_polynomial_t_out(output, x, n); +} + +inline Tensor& chebyshev_polynomial_t_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_chebyshev_polynomial_t_out(output, x, n); +} + +inline Tensor& chebyshev_polynomial_t_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_chebyshev_polynomial_t_out(output, x, n); +} + +/// Chebyshev polynomial of the second kind. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.chebyshev_polynomial_u. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::chebyshev_polynomial_u(x, n); +/// ``` +inline Tensor chebyshev_polynomial_u(const Tensor& x, const Tensor& n) { + return torch::special_chebyshev_polynomial_u(x, n); +} + +inline Tensor chebyshev_polynomial_u(const Scalar& x, const Tensor& n) { + return torch::special_chebyshev_polynomial_u(x, n); +} + +inline Tensor chebyshev_polynomial_u(const Tensor& x, const Scalar& n) { + return torch::special_chebyshev_polynomial_u(x, n); +} + +inline Tensor& chebyshev_polynomial_u_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_chebyshev_polynomial_u_out(output, x, n); +} + +inline Tensor& chebyshev_polynomial_u_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_chebyshev_polynomial_u_out(output, x, n); +} + +inline Tensor& chebyshev_polynomial_u_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_chebyshev_polynomial_u_out(output, x, n); +} + +/// Chebyshev polynomial of the third kind. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.chebyshev_polynomial_v. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::chebyshev_polynomial_v(x, n); +/// ``` +inline Tensor chebyshev_polynomial_v(const Tensor& x, const Tensor& n) { + return torch::special_chebyshev_polynomial_v(x, n); +} + +inline Tensor chebyshev_polynomial_v(const Scalar& x, const Tensor& n) { + return torch::special_chebyshev_polynomial_v(x, n); +} + +inline Tensor chebyshev_polynomial_v(const Tensor& x, const Scalar& n) { + return torch::special_chebyshev_polynomial_v(x, n); +} + +inline Tensor& chebyshev_polynomial_v_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_chebyshev_polynomial_v_out(output, x, n); +} + +inline Tensor& chebyshev_polynomial_v_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_chebyshev_polynomial_v_out(output, x, n); +} + +inline Tensor& chebyshev_polynomial_v_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_chebyshev_polynomial_v_out(output, x, n); +} + +/// Chebyshev polynomial of the fourth kind. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.chebyshev_polynomial_w. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::chebyshev_polynomial_w(x, n); +/// ``` +inline Tensor chebyshev_polynomial_w(const Tensor& x, const Tensor& n) { + return torch::special_chebyshev_polynomial_w(x, n); +} + +inline Tensor chebyshev_polynomial_w(const Scalar& x, const Tensor& n) { + return torch::special_chebyshev_polynomial_w(x, n); +} + +inline Tensor chebyshev_polynomial_w(const Tensor& x, const Scalar& n) { + return torch::special_chebyshev_polynomial_w(x, n); +} + +inline Tensor& chebyshev_polynomial_w_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_chebyshev_polynomial_w_out(output, x, n); +} + +inline Tensor& chebyshev_polynomial_w_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_chebyshev_polynomial_w_out(output, x, n); +} + +inline Tensor& chebyshev_polynomial_w_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_chebyshev_polynomial_w_out(output, x, n); +} + +/// Physicist’s Hermite polynomial. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.hermite_polynomial_h. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::hermite_polynomial_h(x, n); +/// ``` +inline Tensor hermite_polynomial_h(const Tensor& x, const Tensor& n) { + return torch::special_hermite_polynomial_h(x, n); +} + +inline Tensor hermite_polynomial_h(const Scalar& x, const Tensor& n) { + return torch::special_hermite_polynomial_h(x, n); +} + +inline Tensor hermite_polynomial_h(const Tensor& x, const Scalar& n) { + return torch::special_hermite_polynomial_h(x, n); +} + +inline Tensor& hermite_polynomial_h_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_hermite_polynomial_h_out(output, x, n); +} + +inline Tensor& hermite_polynomial_h_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_hermite_polynomial_h_out(output, x, n); +} + +inline Tensor& hermite_polynomial_h_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_hermite_polynomial_h_out(output, x, n); +} + +/// Probabilist’s Hermite polynomial. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.hermite_polynomial_he. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::hermite_polynomial_he(x, n); +/// ``` +inline Tensor hermite_polynomial_he(const Tensor& x, const Tensor& n) { + return torch::special_hermite_polynomial_he(x, n); +} + +inline Tensor hermite_polynomial_he(const Scalar& x, const Tensor& n) { + return torch::special_hermite_polynomial_he(x, n); +} + +inline Tensor hermite_polynomial_he(const Tensor& x, const Scalar& n) { + return torch::special_hermite_polynomial_he(x, n); +} + +inline Tensor& hermite_polynomial_he_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_hermite_polynomial_he_out(output, x, n); +} + +inline Tensor& hermite_polynomial_he_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_hermite_polynomial_he_out(output, x, n); +} + +inline Tensor& hermite_polynomial_he_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_hermite_polynomial_he_out(output, x, n); +} + +/// Laguerre polynomial. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.laguerre_polynomial_l. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::laguerre_polynomial_l(x, n); +/// ``` +inline Tensor laguerre_polynomial_l(const Tensor& x, const Tensor& n) { + return torch::special_laguerre_polynomial_l(x, n); +} + +inline Tensor laguerre_polynomial_l(const Scalar& x, const Tensor& n) { + return torch::special_laguerre_polynomial_l(x, n); +} + +inline Tensor laguerre_polynomial_l(const Tensor& x, const Scalar& n) { + return torch::special_laguerre_polynomial_l(x, n); +} + +inline Tensor& laguerre_polynomial_l_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_laguerre_polynomial_l_out(output, x, n); +} + +inline Tensor& laguerre_polynomial_l_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_laguerre_polynomial_l_out(output, x, n); +} + +inline Tensor& laguerre_polynomial_l_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_laguerre_polynomial_l_out(output, x, n); +} + +/// Legendre polynomial. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.legendre_polynomial_p. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::legendre_polynomial_p(x, n); +/// ``` +inline Tensor legendre_polynomial_p(const Tensor& x, const Tensor& n) { + return torch::special_legendre_polynomial_p(x, n); +} + +inline Tensor legendre_polynomial_p(const Scalar& x, const Tensor& n) { + return torch::special_legendre_polynomial_p(x, n); +} + +inline Tensor legendre_polynomial_p(const Tensor& x, const Scalar& n) { + return torch::special_legendre_polynomial_p(x, n); +} + +inline Tensor& legendre_polynomial_p_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_legendre_polynomial_p_out(output, x, n); +} + +inline Tensor& legendre_polynomial_p_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_legendre_polynomial_p_out(output, x, n); +} + +inline Tensor& legendre_polynomial_p_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_legendre_polynomial_p_out(output, x, n); +} + +/// Modified Bessel function of the first kind of order 0. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.modified_bessel_i0. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::modified_bessel_i0(x); +/// ``` +inline Tensor modified_bessel_i0(const Tensor& self) { + return torch::special_modified_bessel_i0(self); +} + +inline Tensor& modified_bessel_i0_out(Tensor& result, const Tensor& self) { + return torch::special_modified_bessel_i0_out(result, self); +} + +/// Modified Bessel function of the first kind of order 1. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.modified_bessel_i1. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::modified_bessel_i1(x); +/// ``` +inline Tensor modified_bessel_i1(const Tensor& self) { + return torch::special_modified_bessel_i1(self); +} + +inline Tensor& modified_bessel_i1_out(Tensor& result, const Tensor& self) { + return torch::special_modified_bessel_i1_out(result, self); +} + +/// Modified Bessel function of the second kind of order 0. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.modified_bessel_k0. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::modified_bessel_k0(x); +/// ``` +inline Tensor modified_bessel_k0(const Tensor& self) { + return torch::special_modified_bessel_k0(self); +} + +inline Tensor& modified_bessel_k0_out(Tensor& result, const Tensor& self) { + return torch::special_modified_bessel_k0_out(result, self); +} + +/// Modified Bessel function of the second kind of order 1. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.modified_bessel_k1. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::modified_bessel_k1(x); +/// ``` +inline Tensor modified_bessel_k1(const Tensor& self) { + return torch::special_modified_bessel_k1(self); +} + +inline Tensor& modified_bessel_k1_out(Tensor& result, const Tensor& self) { + return torch::special_modified_bessel_k1_out(result, self); +} + +/// Scaled modified Bessel function of the second kind of order 0. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.scaled_modified_bessel_k0. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::scaled_modified_bessel_k0(x); +/// ``` +inline Tensor scaled_modified_bessel_k0(const Tensor& x) { + return torch::special_scaled_modified_bessel_k0(x); +} + +inline Tensor& scaled_modified_bessel_k0_out(Tensor& y, const Tensor& x) { + return torch::special_scaled_modified_bessel_k0_out(y, x); +} + +/// Scaled modified Bessel function of the second kind of order 1. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.scaled_modified_bessel_k1. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::scaled_modified_bessel_k1(x); +/// ``` +inline Tensor scaled_modified_bessel_k1(const Tensor& x) { + return torch::special_scaled_modified_bessel_k1(x); +} + +inline Tensor& scaled_modified_bessel_k1_out(Tensor& y, const Tensor& x) { + return torch::special_scaled_modified_bessel_k1_out(y, x); +} + +/// Shifted Chebyshev polynomial of the first kind. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.shifted_chebyshev_polynomial_t. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::shifted_chebyshev_polynomial_t(x, n); +/// ``` +inline Tensor shifted_chebyshev_polynomial_t(const Tensor& x, const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_t(x, n); +} + +inline Tensor shifted_chebyshev_polynomial_t(const Scalar& x, const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_t(x, n); +} + +inline Tensor shifted_chebyshev_polynomial_t(const Tensor& x, const Scalar& n) { + return torch::special_shifted_chebyshev_polynomial_t(x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_t_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_t_out(output, x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_t_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_t_out(output, x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_t_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_shifted_chebyshev_polynomial_t_out(output, x, n); +} + +/// Shifted Chebyshev polynomial of the second kind. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.shifted_chebyshev_polynomial_u. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::shifted_chebyshev_polynomial_u(x, n); +/// ``` +inline Tensor shifted_chebyshev_polynomial_u(const Tensor& x, const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_u(x, n); +} + +inline Tensor shifted_chebyshev_polynomial_u(const Scalar& x, const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_u(x, n); +} + +inline Tensor shifted_chebyshev_polynomial_u(const Tensor& x, const Scalar& n) { + return torch::special_shifted_chebyshev_polynomial_u(x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_u_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_u_out(output, x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_u_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_u_out(output, x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_u_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_shifted_chebyshev_polynomial_u_out(output, x, n); +} + +/// Shifted Chebyshev polynomial of the third kind. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.shifted_chebyshev_polynomial_v. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::shifted_chebyshev_polynomial_v(x, n); +/// ``` +inline Tensor shifted_chebyshev_polynomial_v(const Tensor& x, const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_v(x, n); +} + +inline Tensor shifted_chebyshev_polynomial_v(const Scalar& x, const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_v(x, n); +} + +inline Tensor shifted_chebyshev_polynomial_v(const Tensor& x, const Scalar& n) { + return torch::special_shifted_chebyshev_polynomial_v(x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_v_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_v_out(output, x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_v_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_v_out(output, x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_v_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_shifted_chebyshev_polynomial_v_out(output, x, n); +} + +/// Shifted Chebyshev polynomial of the fourth kind. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.shifted_chebyshev_polynomial_w. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// auto n = torch::randn(128, dtype=kDouble); +/// +/// torch::special::shifted_chebyshev_polynomial_w(x, n); +/// ``` +inline Tensor shifted_chebyshev_polynomial_w(const Tensor& x, const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_w(x, n); +} + +inline Tensor shifted_chebyshev_polynomial_w(const Scalar& x, const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_w(x, n); +} + +inline Tensor shifted_chebyshev_polynomial_w(const Tensor& x, const Scalar& n) { + return torch::special_shifted_chebyshev_polynomial_w(x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_w_out( + Tensor& output, + const Tensor& x, + const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_w_out(output, x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_w_out( + Tensor& output, + const Scalar& x, + const Tensor& n) { + return torch::special_shifted_chebyshev_polynomial_w_out(output, x, n); +} + +inline Tensor& shifted_chebyshev_polynomial_w_out( + Tensor& output, + const Tensor& x, + const Scalar& n) { + return torch::special_shifted_chebyshev_polynomial_w_out(output, x, n); +} + +/// Spherical Bessel function of the first kind of order 0. +/// +/// See +/// https://pytorch.org/docs/main/special.html#torch.special.spherical_bessel_j0. +/// +/// Example: +/// +/// ``` +/// auto x = torch::randn(128, dtype=kDouble); +/// +/// torch::special::spherical_bessel_j0(x); +/// ``` +inline Tensor spherical_bessel_j0(const Tensor& x) { + return torch::special_spherical_bessel_j0(x); +} + +inline Tensor& spherical_bessel_j0_out(Tensor& y, const Tensor& x) { + return torch::special_spherical_bessel_j0_out(y, x); +} +} // namespace torch::special diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/torch.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/torch.h new file mode 100644 index 0000000000000000000000000000000000000000..aed9f82b0d0177b14e32bc1b381228a4b87ce896 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/torch.h @@ -0,0 +1,8 @@ +#pragma once + +#include + +#ifdef TORCH_API_INCLUDE_EXTENSION_H +#include + +#endif // defined(TORCH_API_INCLUDE_EXTENSION_H) diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/types.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/types.h new file mode 100644 index 0000000000000000000000000000000000000000..d220832ea042c643e5f969e5dcbfee255b5bd973 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/types.h @@ -0,0 +1,70 @@ +#pragma once + +#include + +#include + +#include +#include + +#include + +namespace torch { + +// NOTE [ Exposing declarations in `at::` to `torch::` ] +// +// The following line `using namespace at;` is responsible for exposing all +// declarations in `at::` namespace to `torch::` namespace. +// +// According to the rules laid out in +// https://en.cppreference.com/w/cpp/language/qualified_lookup, section +// "Namespace members": +// ``` +// Qualified lookup within the scope of a namespace N first considers all +// declarations that are located in N and all declarations that are located in +// the inline namespace members of N (and, transitively, in their inline +// namespace members). If there are no declarations in that set then it +// considers declarations in all namespaces named by using-directives found in N +// and in all transitive inline namespace members of N. +// ``` +// +// This means that if both `at::` and `torch::` namespaces have a function with +// the same signature (e.g. both `at::func()` and `torch::func()` exist), after +// `namespace torch { using namespace at; }`, when we call `torch::func()`, the +// `func()` function defined in `torch::` namespace will always be called, and +// the `func()` function defined in `at::` namespace is always hidden. +using namespace at; // NOLINT + +#if !defined(FBCODE_CAFFE2) && !defined(C10_NODEPRECATED) +using std::nullopt; // NOLINT +using std::optional; // NOLINT +#endif + +using Dtype = at::ScalarType; + +/// Fixed width dtypes. +constexpr auto kUInt8 = at::kByte; +constexpr auto kInt8 = at::kChar; +constexpr auto kInt16 = at::kShort; +constexpr auto kInt32 = at::kInt; +constexpr auto kInt64 = at::kLong; +constexpr auto kUInt16 = at::kUInt16; +constexpr auto kUInt32 = at::kUInt32; +constexpr auto kUInt64 = at::kUInt64; +constexpr auto kFloat16 = at::kHalf; +constexpr auto kFloat32 = at::kFloat; +constexpr auto kFloat64 = at::kDouble; + +/// Rust-style short dtypes. +constexpr auto kU8 = kUInt8; +constexpr auto kU16 = kUInt16; +constexpr auto kU32 = kUInt32; +constexpr auto kU64 = kUInt64; +constexpr auto kI8 = kInt8; +constexpr auto kI16 = kInt16; +constexpr auto kI32 = kInt32; +constexpr auto kI64 = kInt64; +constexpr auto kF16 = kFloat16; +constexpr auto kF32 = kFloat32; +constexpr auto kF64 = kFloat64; +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/utils.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/utils.h new file mode 100644 index 0000000000000000000000000000000000000000..d5363b9bb99d16059b6a2fab190d65717d4a32b6 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/utils.h @@ -0,0 +1,117 @@ +#pragma once + +#include +#include +#include +#include +#include + +// NOLINTBEGIN(misc-unused-using-decls) +namespace torch { + +/// A RAII, thread-local guard that disabled gradient calculation. +/// +/// Disabling gradient calculation is useful for inference, when you are sure +/// that you will not call `at::Tensor::backward`. It will reduce memory +/// consumption for computations that would otherwise have `requires_grad() == +/// true`. +/// +/// In this mode, the result of every computation will have +/// `requires_grad() == false`, even when the inputs have `requires_grad() == +/// true`. +/// +/// This context manager is thread-local; it will not affect computation +/// in other threads. +/// +/// Example: +/// @code +/// auto x = torch::tensor({1.}, torch::requires_grad()); +/// { +/// torch::NoGradGuard no_grad; +/// auto y = x * 2; +/// std::cout << y.requires_grad() << std::endl; // prints `false` +/// } +/// { +/// auto doubler = [](torch::Tensor x) { +/// torch::NoGradGuard no_grad; +/// return x * 2; +/// }; +/// auto z = doubler(x); +/// std::cout << z.requires_grad() << std::endl; // prints `false` +/// } +/// @endcode +using NoGradGuard = at::NoGradGuard; + +/// A RAII, thread-local guard that sets gradient calculation to on or off. +/// +/// ``AutoGradMode`` will enable or disable grads based on its argument +/// `enabled`. +/// +/// This context manager is thread-local; it will not affect computation +/// in other threads. +/// +/// \param enabled: Flag whether to enable grad (``true``), or disable +/// (``false``). This can be used to conditionally enable +/// gradients. +/// +/// Example: +/// @code +/// auto x = torch::tensor({1.}, torch::requires_grad()); +/// { +/// torch::AutoGradMode enable_grad(true); +/// auto y = x * 2; +/// std::cout << y.requires_grad() << std::endl; // prints `true` +/// } +/// { +/// torch::AutoGradMode enable_grad(false); +/// auto y = x * 2; +/// std::cout << y.requires_grad() << std::endl; // prints `false` +/// } +/// @endcode +using AutoGradMode = at::AutoGradMode; + +/// Sets the global random seed for all newly created CPU and CUDA tensors. +using at::manual_seed; + +// Called during new thread initialization +using at::init_num_threads; + +// Returns the number of threads used in parallel region. +using at::get_num_threads; + +// Sets the number of threads to be used in parallel region. +using at::set_num_threads; + +// Returns the number of threads used for inter-op parallelism. +using at::get_num_interop_threads; + +// Sets the number of threads to be used for inter-op parallelism. +using at::set_num_interop_threads; + +// Returns true if both t1, t2 are undefined or both are defined and equal +inline bool equal_if_defined(const Tensor& t1, const Tensor& t2) { + return ( + (!t1.defined() && !t2.defined()) || + (t1.defined() && t2.defined() && torch::equal(t1, t2))); +} + +// RecordFunction API +using at::addGlobalCallback; +using at::addThreadLocalCallback; +using at::CallbackHandle; +using at::clearCallbacks; +using at::clearGlobalCallbacks; +using at::clearThreadLocalCallbacks; +using at::DisableRecordFunctionGuard; +using at::enableRecordFunction; +using at::hasCallbacks; +using at::hasGlobalCallbacks; +using at::hasThreadLocalCallbacks; +using at::isRecordFunctionEnabled; +using at::RecordFunction; +using at::RecordFunctionCallback; +using at::RecordFunctionGuard; +using at::removeCallback; + +} // namespace torch +// NOLINTEND(misc-unused-using-decls) diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/version.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/version.h new file mode 100644 index 0000000000000000000000000000000000000000..3158ddc81bba1437fced81e12a7c388052fc564d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/version.h @@ -0,0 +1 @@ +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/include/torch/xpu.h b/western_ai/usa/pytorch/torch/csrc/api/include/torch/xpu.h new file mode 100644 index 0000000000000000000000000000000000000000..edb607ee00a69b4c09cedb40ce01d24388c53626 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/include/torch/xpu.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include +#include + +namespace torch::xpu { + +/// Returns the number of XPU devices available. +size_t TORCH_API device_count(); + +/// Returns true if at least one XPU device is available. +bool TORCH_API is_available(); + +/// Sets the seed for the current GPU. +void TORCH_API manual_seed(uint64_t seed); + +/// Sets the seed for all available GPUs. +void TORCH_API manual_seed_all(uint64_t seed); + +/// Waits for all kernels in all streams on a XPU device to complete. +void TORCH_API synchronize(int64_t device_index); + +} // namespace torch::xpu diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/cuda.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..234520821eba7aee7d030878ac1b5ca996d092ba --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/cuda.cpp @@ -0,0 +1,62 @@ +#include + +#include +#include +#include + +namespace torch::cuda { + +c10::DeviceIndex device_count() { + return at::detail::getCUDAHooks().deviceCount(); +} + +bool is_available() { + // NB: the semantics of this are different from at::globalContext().hasCUDA(); + // ATen's function tells you if you have a working driver and CUDA build, + // whereas this function also tells you if you actually have any GPUs. + // This function matches the semantics of at::cuda::is_available() + return cuda::device_count() > 0; +} + +bool cudnn_is_available() { + return is_available() && at::detail::getCUDAHooks().hasCuDNN(); +} + +/// Sets the seed for the current GPU. +void manual_seed(uint64_t seed) { + if (is_available()) { + auto index = at::detail::getCUDAHooks().getCurrentDevice(); + auto gen = at::detail::getCUDAHooks().getDefaultGenerator(index); + { + // See Note [Acquire lock when using random generators] + std::lock_guard lock(gen.mutex()); + gen.set_current_seed(seed); + } + } +} + +/// Sets the seed for all available GPUs. +void manual_seed_all(uint64_t seed) { + auto num_gpu = device_count(); + for (const auto i : c10::irange(num_gpu)) { + auto gen = at::detail::getCUDAHooks().getDefaultGenerator(i); + { + // See Note [Acquire lock when using random generators] + std::lock_guard lock(gen.mutex()); + gen.set_current_seed(seed); + } + } +} + +void synchronize(int64_t device_index) { + TORCH_CHECK(is_available(), "No CUDA GPUs are available"); + auto num_gpus = cuda::device_count(); + TORCH_CHECK( + device_index < 0 || device_index < num_gpus, + "Device index out of range: ", + device_index); + at::detail::getCUDAHooks().deviceSynchronize( + static_cast(device_index)); +} + +} // namespace torch::cuda diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/data/datasets/mnist.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/data/datasets/mnist.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1df81e35a78a98f33b51a9b0c3a70beae125ae27 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/data/datasets/mnist.cpp @@ -0,0 +1,125 @@ +#include + +#include +#include + +#include + +#include +#include +#include + +namespace torch::data::datasets { +namespace { +constexpr uint32_t kTrainSize = 60000; +constexpr uint32_t kTestSize = 10000; +constexpr uint32_t kImageMagicNumber = 2051; +constexpr uint32_t kTargetMagicNumber = 2049; +constexpr uint32_t kImageRows = 28; +constexpr uint32_t kImageColumns = 28; +constexpr const char* kTrainImagesFilename = "train-images-idx3-ubyte"; +constexpr const char* kTrainTargetsFilename = "train-labels-idx1-ubyte"; +constexpr const char* kTestImagesFilename = "t10k-images-idx3-ubyte"; +constexpr const char* kTestTargetsFilename = "t10k-labels-idx1-ubyte"; + +bool check_is_little_endian() { + const uint32_t word = 1; + return reinterpret_cast(&word)[0] == 1; +} + +constexpr uint32_t flip_endianness(uint32_t value) { + return ((value & 0xffu) << 24u) | ((value & 0xff00u) << 8u) | + ((value & 0xff0000u) >> 8u) | ((value & 0xff000000u) >> 24u); +} + +uint32_t read_int32(std::ifstream& stream) { + static const bool is_little_endian = check_is_little_endian(); + uint32_t value = 0; + AT_ASSERT(stream.read(reinterpret_cast(&value), sizeof value)); + return is_little_endian ? flip_endianness(value) : value; +} + +uint32_t expect_int32(std::ifstream& stream, uint32_t expected) { + const auto value = read_int32(stream); + TORCH_CHECK( + value == expected, + "Expected to read number ", + expected, + " but found ", + value, + " instead"); + return value; +} + +std::string join_paths(std::string head, const std::string& tail) { + if (head.back() != '/') { + head.push_back('/'); + } + head += tail; + return head; +} + +Tensor read_images(const std::string& root, bool train) { + const auto path = + join_paths(root, train ? kTrainImagesFilename : kTestImagesFilename); + std::ifstream images(path, std::ios::binary); + TORCH_CHECK(images, "Error opening images file at ", path); + + const auto count = train ? kTrainSize : kTestSize; + + // From http://yann.lecun.com/exdb/mnist/ + expect_int32(images, kImageMagicNumber); + expect_int32(images, count); + expect_int32(images, kImageRows); + expect_int32(images, kImageColumns); + + auto tensor = + torch::empty({count, 1, kImageRows, kImageColumns}, torch::kByte); + images.read(reinterpret_cast(tensor.data_ptr()), tensor.numel()); + return tensor.to(torch::kFloat32).div_(255); +} + +Tensor read_targets(const std::string& root, bool train) { + const auto path = + join_paths(root, train ? kTrainTargetsFilename : kTestTargetsFilename); + std::ifstream targets(path, std::ios::binary); + TORCH_CHECK(targets, "Error opening targets file at ", path); + + const auto count = train ? kTrainSize : kTestSize; + + expect_int32(targets, kTargetMagicNumber); + expect_int32(targets, count); + + auto tensor = torch::empty(count, torch::kByte); + targets.read(reinterpret_cast(tensor.data_ptr()), count); + return tensor.to(torch::kInt64); +} +} // namespace + +MNIST::MNIST(const std::string& root, Mode mode) + : images_(read_images(root, mode == Mode::kTrain)), + targets_(read_targets(root, mode == Mode::kTrain)) {} + +Example<> MNIST::get(size_t index) { + return { + images_[static_cast(index)], + targets_[static_cast(index)]}; +} + +std::optional MNIST::size() const { + return images_.size(0); +} + +bool MNIST::is_train() const noexcept { + return images_.size(0) == kTrainSize; +} + +const Tensor& MNIST::images() const { + return images_; +} + +const Tensor& MNIST::targets() const { + return targets_; +} + +} // namespace torch::data::datasets diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/distributed.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/distributed.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f5a95986d8b7deb7003bc59b2349f67f03d14c6a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/distributed.cpp @@ -0,0 +1,159 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace torch::data::samplers { + +DistributedRandomSampler::DistributedRandomSampler( + size_t size, + size_t num_replicas, + size_t rank, + bool allow_duplicates) + : DistributedSampler(size, num_replicas, rank, allow_duplicates) { + // shuffle first time. + DistributedRandomSampler::reset(size_); +} + +std::optional> DistributedRandomSampler::next( + size_t batch_size) { + if (sample_index_ == end_index_) { + return std::nullopt; + } + + size_t end = sample_index_ + batch_size; + if (end > end_index_) { + end = end_index_; + } + + auto iter = all_indices_.begin(); + std::vector res( + iter + static_cast(sample_index_), + iter + static_cast(end)); + sample_index_ = end; + return res; +} + +void DistributedRandomSampler::reset(std::optional new_size) { + size_ = new_size.value_or(size_); + populate_indices(); + + std::mt19937 rand(epoch_); + std::shuffle(all_indices_.begin(), all_indices_.end(), rand); + sample_index_ = begin_index_; +} + +void DistributedRandomSampler::populate_indices() { + size_t num_local_samples = local_sample_count(); + size_t sample_count = + num_replicas_ == 1 ? size_ : num_local_samples * num_replicas_; + all_indices_.resize(sample_count); + std::iota(std::begin(all_indices_), std::end(all_indices_), 0); + for (const auto i : c10::irange(size_, sample_count)) { + // we may have added duplicate samples to make all + // replicas to have the same number of samples. + all_indices_[i] = i - size_; + } + begin_index_ = rank_ * num_local_samples; + end_index_ = begin_index_ + num_local_samples; + sample_index_ = begin_index_; +} + +void DistributedRandomSampler::save(serialize::OutputArchive& archive) const { + archive.write( + "sample_index_", + torch::tensor(static_cast(sample_index_)), + /*is_buffer=*/true); + archive.write( + "epoch_", + torch::tensor(static_cast(epoch_)), + /*is_buffer=*/true); +} + +void DistributedRandomSampler::load(serialize::InputArchive& archive) { + auto tensor = torch::empty(1, torch::kInt64); + archive.read("epoch_", tensor, /*is_buffer=*/true); + epoch_ = tensor.item(); + // call reset() after loading epoch_ to populate indices. + reset(size_); + + tensor = torch::empty(1, torch::kInt64); + archive.read("sample_index_", tensor, /*is_buffer=*/true); + sample_index_ = tensor.item(); +} + +size_t DistributedRandomSampler::index() const noexcept { + return sample_index_; +} + +DistributedSequentialSampler::DistributedSequentialSampler( + size_t size, + size_t num_replicas, + size_t rank, + bool allow_duplicates) + : DistributedSampler(size, num_replicas, rank, allow_duplicates) { + populate_indices(); +} + +std::optional> DistributedSequentialSampler::next( + size_t batch_size) { + if (sample_index_ == end_index_) { + return std::nullopt; + } + + size_t end = sample_index_ + batch_size; + if (end > end_index_) { + end = end_index_; + } + + std::vector res(end - sample_index_); + std::iota(std::begin(res), std::end(res), sample_index_); + if (end >= size_) { + for (size_t& index : res) { + index = index % size_; + } + } + sample_index_ = end; + return res; +} + +void DistributedSequentialSampler::reset(std::optional new_size) { + size_t size = new_size.value_or(size_); + if (size != size_) { + size_ = size; + populate_indices(); + } else { + sample_index_ = begin_index_; + } +} + +void DistributedSequentialSampler::populate_indices() { + begin_index_ = rank_ * local_sample_count(); + end_index_ = begin_index_ + local_sample_count(); + sample_index_ = begin_index_; +} + +void DistributedSequentialSampler::save( + serialize::OutputArchive& archive) const { + archive.write( + "sample_index_", + torch::tensor(static_cast(sample_index_)), + /*is_buffer=*/true); +} + +void DistributedSequentialSampler::load(serialize::InputArchive& archive) { + auto tensor = torch::empty(1, torch::kInt64); + archive.read("sample_index_", tensor, /*is_buffer=*/true); + sample_index_ = tensor.item(); +} + +size_t DistributedSequentialSampler::index() const noexcept { + return sample_index_; +} + +} // namespace torch::data::samplers diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/random.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/random.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7781cf1f5f5a950fd0e252e9c9798e9ff236f2f2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/random.cpp @@ -0,0 +1,71 @@ +#include +#include +#include + +#include +#include +#include + +namespace torch::data::samplers { +RandomSampler::RandomSampler(int64_t size, Dtype index_dtype) + : indices_(torch::randperm(size, index_dtype)) {} + +RandomSampler::~RandomSampler() = default; + +void RandomSampler::reset(std::optional new_size) { + // This allocates a new chunk of memory every time (just FYI). It should be + // amortized over the entire epoch hopefully. + const auto size = new_size.value_or(static_cast(indices_.numel())); + indices_ = torch::randperm(static_cast(size), indices_.options()); + index_ = 0; +} + +std::optional> RandomSampler::next(size_t batch_size) { + AT_ASSERT(index_ <= indices_.numel()); + const size_t remaining_indices = indices_.numel() - index_; + if (remaining_indices == 0) { + return std::nullopt; + } + std::vector index_batch(std::min(batch_size, remaining_indices)); + auto slice = indices_.slice(/*dim=*/0, index_, index_ + index_batch.size()); + // You may want to store your indices with 32-bit or less, but here we need + // to upcast to 64-bit. A batch itself won't hold too many indices, so that + // should be ok. Note that if this indeed results in a type promotion, there + // will be two allocations: one for the upcast slice, and one for the + // returned `index_batch` vector. + slice = slice.to(torch::kInt64); + const auto* data = slice.const_data_ptr(); + std::copy(data, data + index_batch.size(), index_batch.begin()); + index_ += static_cast(index_batch.size()); + return index_batch; +} + +void RandomSampler::save(serialize::OutputArchive& archive) const { + archive.write( + "index", + torch::tensor(index_, torch::kInt64), + /*is_buffer=*/true); + archive.write( + "indices", + indices_, + /*is_buffer=*/true); +} + +void RandomSampler::load(serialize::InputArchive& archive) { + auto tensor = torch::empty(1, torch::kInt64); + archive.read( + "index", + tensor, + /*is_buffer=*/true); + index_ = tensor.item(); + archive.read( + "indices", + indices_, + /*is_buffer=*/true); +} + +size_t RandomSampler::index() const noexcept { + return index_; +} + +} // namespace torch::data::samplers diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/sequential.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/sequential.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8b590c538bb09f9bf63f45e07eede8aeb1505d08 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/sequential.cpp @@ -0,0 +1,51 @@ +#include +#include +#include + +#include +#include +#include + +namespace torch::data::samplers { +SequentialSampler::SequentialSampler(size_t size) : size_(size) {} + +void SequentialSampler::reset(std::optional new_size) { + if (new_size.has_value()) { + size_ = *new_size; + } + index_ = 0; +} + +std::optional> SequentialSampler::next(size_t batch_size) { + const auto remaining_indices = size_ - index_; + if (remaining_indices == 0) { + return std::nullopt; + } + std::vector index_batch(std::min(batch_size, remaining_indices)); + for (auto& i : index_batch) { + i = index_++; + } + return index_batch; +} + +void SequentialSampler::save(serialize::OutputArchive& archive) const { + archive.write( + "index", + torch::tensor(static_cast(index_), torch::kInt64), + /*is_buffer=*/true); +} + +void SequentialSampler::load(serialize::InputArchive& archive) { + auto tensor = torch::empty(1, torch::kInt64); + archive.read( + "index", + tensor, + /*is_buffer=*/true); + index_ = tensor.item(); +} + +size_t SequentialSampler::index() const noexcept { + return index_; +} + +} // namespace torch::data::samplers diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/stream.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/stream.cpp new file mode 100644 index 0000000000000000000000000000000000000000..440db3b9ff32f8427ade67b37badb9126556ce7c --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/data/samplers/stream.cpp @@ -0,0 +1,57 @@ +#include +#include +#include + +#include + +#include + +namespace torch::data::samplers { + +BatchSize::BatchSize(size_t size) : size_(size) {} +size_t BatchSize::size() const noexcept { + return size_; +} +BatchSize::operator size_t() const noexcept { + return size_; +} + +StreamSampler::StreamSampler(size_t epoch_size) : epoch_size_(epoch_size) {} + +void StreamSampler::reset(std::optional new_size) { + if (new_size.has_value()) { + epoch_size_ = *new_size; + } + examples_retrieved_so_far_ = 0; +} + +std::optional StreamSampler::next(size_t batch_size) { + AT_ASSERT(examples_retrieved_so_far_ <= epoch_size_); + if (examples_retrieved_so_far_ == epoch_size_) { + return std::nullopt; + } + if (examples_retrieved_so_far_ + batch_size > epoch_size_) { + batch_size = epoch_size_ - examples_retrieved_so_far_; + } + examples_retrieved_so_far_ += batch_size; + return BatchSize(batch_size); +} + +void StreamSampler::save(serialize::OutputArchive& archive) const { + archive.write( + "examples_retrieved_so_far", + torch::tensor( + static_cast(examples_retrieved_so_far_), torch::kInt64), + /*is_buffer=*/true); +} + +void StreamSampler::load(serialize::InputArchive& archive) { + auto tensor = torch::empty(1, torch::kInt64); + archive.read( + "examples_retrieved_so_far", + tensor, + /*is_buffer=*/true); + examples_retrieved_so_far_ = tensor.item(); +} + +} // namespace torch::data::samplers diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/enum.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/enum.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9a8fd995a98986f097af61cae20293aab29c6e90 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/enum.cpp @@ -0,0 +1,40 @@ +#include + +TORCH_ENUM_DEFINE(Linear) +TORCH_ENUM_DEFINE(Conv1D) +TORCH_ENUM_DEFINE(Conv2D) +TORCH_ENUM_DEFINE(Conv3D) +TORCH_ENUM_DEFINE(ConvTranspose1D) +TORCH_ENUM_DEFINE(ConvTranspose2D) +TORCH_ENUM_DEFINE(ConvTranspose3D) +TORCH_ENUM_DEFINE(Sigmoid) +TORCH_ENUM_DEFINE(Tanh) +TORCH_ENUM_DEFINE(ReLU) +TORCH_ENUM_DEFINE(GELU) +TORCH_ENUM_DEFINE(LeakyReLU) +TORCH_ENUM_DEFINE(FanIn) +TORCH_ENUM_DEFINE(FanOut) +TORCH_ENUM_DEFINE(Constant) +TORCH_ENUM_DEFINE(Reflect) +TORCH_ENUM_DEFINE(Replicate) +TORCH_ENUM_DEFINE(Circular) +TORCH_ENUM_DEFINE(Nearest) +TORCH_ENUM_DEFINE(Bilinear) +TORCH_ENUM_DEFINE(Bicubic) +TORCH_ENUM_DEFINE(Trilinear) +TORCH_ENUM_DEFINE(Area) +TORCH_ENUM_DEFINE(NearestExact) +TORCH_ENUM_DEFINE(Sum) +TORCH_ENUM_DEFINE(Mean) +TORCH_ENUM_DEFINE(Max) +TORCH_ENUM_DEFINE(None) +TORCH_ENUM_DEFINE(BatchMean) +TORCH_ENUM_DEFINE(Zeros) +TORCH_ENUM_DEFINE(Border) +TORCH_ENUM_DEFINE(Reflection) +TORCH_ENUM_DEFINE(RNN_TANH) +TORCH_ENUM_DEFINE(RNN_RELU) +TORCH_ENUM_DEFINE(LSTM) +TORCH_ENUM_DEFINE(GRU) +TORCH_ENUM_DEFINE(Valid) +TORCH_ENUM_DEFINE(Same) diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/imethod.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/imethod.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a04bcf6c9e629ef0497398fc6b46cdd5c3cd255e --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/imethod.cpp @@ -0,0 +1,15 @@ +#include + +namespace torch { + +const std::vector& IMethod::getArgumentNames() const { + if (isArgumentNamesInitialized_) { + return argumentNames_; + } + + isArgumentNamesInitialized_ = true; + setArgumentNames(argumentNames_); + return argumentNames_; +} + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/jit.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/jit.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5e9aa2cfa6acb0353bb31ea2356bf29c50b7b98a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/jit.cpp @@ -0,0 +1,17 @@ +#include + +#include +#include + +#include +#include + +namespace torch::jit { + +std::shared_ptr compile(const std::string& source) { + auto module = std::make_shared(); + module->define(std::nullopt, source, nativeResolver(), nullptr); + return module; +} + +} // namespace torch::jit diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/mps.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/mps.cpp new file mode 100644 index 0000000000000000000000000000000000000000..32e6ffc89a27a065da4992975d6a5d52a2f519b3 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/mps.cpp @@ -0,0 +1,38 @@ +#include +#include + +namespace torch::mps { + +bool is_available() { + return at::detail::getMPSHooks().hasMPS(); +} + +/// Sets the seed for the MPS's default generator. +void manual_seed(uint64_t seed) { + if (is_available()) { + auto gen = at::detail::getMPSHooks().getDefaultGenerator(); + { + // See Note [Acquire lock when using random generators] + std::lock_guard lock(gen.mutex()); + gen.set_current_seed(seed); + } + } +} + +void synchronize() { + at::detail::getMPSHooks().deviceSynchronize(); +} + +void commit() { + at::detail::getMPSHooks().commitStream(); +} + +MTLCommandBuffer_t get_command_buffer() { + return at::detail::getMPSHooks().getCommandBuffer(); +} + +DispatchQueue_t get_dispatch_queue() { + return at::detail::getMPSHooks().getDispatchQueue(); +} + +} // namespace torch::mps diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/init.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/init.cpp new file mode 100644 index 0000000000000000000000000000000000000000..39dea23d292f430a46379974eb81179ecb846a02 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/init.cpp @@ -0,0 +1,256 @@ +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include + +namespace torch::nn::init { +namespace { +struct Fan { + explicit Fan(Tensor& tensor) { + const auto dimensions = tensor.ndimension(); + TORCH_CHECK( + dimensions >= 2, + "Fan in and fan out cannot be computed for tensor with fewer than 2 dimensions"); + + if (dimensions == 2) { + in = tensor.size(1); + out = tensor.size(0); + } else { + in = tensor.size(1) * tensor[0][0].numel(); + out = tensor.size(0) * tensor[0][0].numel(); + } + } + + int64_t in; + int64_t out; +}; + +double calculate_kaiming_std( + Tensor tensor, + double a, + FanModeType mode, + NonlinearityType nonlinearity) { + NoGradGuard guard; + Fan fan(tensor); + const auto gain = calculate_gain(nonlinearity, a); + double std = 0.0; + + if (std::holds_alternative(mode)) { + std = gain / std::sqrt(fan.in); + } else { + std = gain / std::sqrt(fan.out); + } + return std; +} +} // namespace + +double calculate_gain(NonlinearityType nonlinearity, double param) { + if (std::holds_alternative(nonlinearity)) { + return 5.0 / 3.0; + } else if (std::holds_alternative(nonlinearity)) { + return std::sqrt(2.0); + } else if (std::holds_alternative(nonlinearity)) { + return std::sqrt(2.0 / (1 + pow(param, 2))); + } + + return 1.0; +} + +// NOLINTNEXTLINE(performance-unnecessary-value-param) +Tensor constant_(Tensor tensor, Scalar value) { + NoGradGuard guard; + return tensor.fill_(value); +} + +Tensor dirac_(Tensor tensor) { + NoGradGuard guard; + + TORCH_CHECK( + tensor.ndimension() >= 3 && tensor.ndimension() <= 5, + "Only tensors with 3, 4, or 5 dimensions are supported"); + + const auto sizes = tensor.sizes(); + const auto min_dim = std::min(sizes[0], sizes[1]); + + tensor.zero_(); + for (const auto d : c10::irange(min_dim)) { + // NOLINTNEXTLINE(bugprone-switch-missing-default-case) + switch (tensor.ndimension()) { + case 3: // Temporal convolution + tensor[d][d][sizes[2] / 2] = 1; + break; + case 4: // Spatial convolution + tensor[d][d][sizes[2] / 2][sizes[3] / 2] = 1; + break; + case 5: // Volumetric convolution + tensor[d][d][sizes[2] / 2][sizes[3] / 2][sizes[4] / 2] = 1; + break; + } + } + + return tensor; +} + +Tensor eye_(Tensor matrix) { + NoGradGuard guard; + TORCH_CHECK( + matrix.ndimension() == 2, "Only tensors with 2 dimensions are supported"); + return torch::eye_out(matrix, matrix.size(0), matrix.size(1)); +} + +// NOLINTNEXTLINE(performance-unnecessary-value-param) +Tensor normal_(Tensor tensor, double mean, double std) { + NoGradGuard guard; + return tensor.normal_(mean, std); +} + +// NOLINTNEXTLINE(performance-unnecessary-value-param) +Tensor ones_(Tensor tensor) { + NoGradGuard guard; + return tensor.fill_(1); +} + +Tensor orthogonal_(Tensor tensor, double gain) { + NoGradGuard guard; + + TORCH_CHECK( + tensor.ndimension() >= 2, + "Only tensors with 2 or more dimensions are supported"); + + const auto rows = tensor.size(0); + const auto columns = tensor.numel() / rows; + auto flattened = torch::randn({rows, columns}); + + if (rows < columns) { + flattened.t_(); + } + + // Compute the qr factorization + auto [q, r] = torch::linalg_qr(flattened); + // Make Q uniform according to https://arxiv.org/pdf/math-ph/0609050.pdf + auto d = torch::diag(r, 0); + auto ph = d.sign(); + q *= ph; + + if (rows < columns) { + q.t_(); + } + + tensor.view_as(q).copy_(q); + tensor.mul_(gain); + + return tensor; +} + +Tensor sparse_(Tensor tensor, double sparsity, double std) { + NoGradGuard guard; + + TORCH_CHECK( + tensor.ndimension() == 2, "Only tensors with 2 dimensions are supported"); + + const auto rows = tensor.size(0); + const auto columns = tensor.size(1); + const int64_t num_zeros = std::ceil(sparsity * static_cast(rows)); + tensor.normal_(0, std); + for (const auto column : c10::irange(columns)) { + auto row_indices = torch::randperm(rows, tensor.options().dtype(kLong)); + auto zero_indices = + row_indices.slice(/*dim=*/0, /*start=*/0, /*end=*/num_zeros); + tensor.index_put_( + {zero_indices, torch::tensor(column, tensor.options().dtype(kLong))}, + torch::zeros(num_zeros, tensor.options())); + } + + return tensor; +} + +// NOLINTNEXTLINE(performance-unnecessary-value-param) +Tensor uniform_(Tensor tensor, double low, double high) { + NoGradGuard guard; + return tensor.uniform_(low, high); +} + +Tensor kaiming_uniform_( + // NOLINTNEXTLINE(performance-unnecessary-value-param) + Tensor tensor, + double a, + FanModeType mode, + NonlinearityType nonlinearity) { + NoGradGuard guard; + auto std = calculate_kaiming_std(tensor, a, mode, nonlinearity); + // Calculate uniform bounds from standard deviation + const auto bound = std::sqrt(3.0) * std; + return tensor.uniform_(-bound, bound); +} + +Tensor kaiming_normal_( + // NOLINTNEXTLINE(performance-unnecessary-value-param) + Tensor tensor, + double a, + FanModeType mode, + NonlinearityType nonlinearity) { + NoGradGuard guard; + + auto std = calculate_kaiming_std(tensor, a, mode, nonlinearity); + return tensor.normal_(0, std); +} + +Tensor xavier_normal_(Tensor tensor, double gain) { + NoGradGuard guard; + + Fan fan(tensor); + const auto std = + gain * std::sqrt(2.0 / static_cast(fan.in + fan.out)); + return tensor.normal_(0, std); +} + +Tensor xavier_uniform_(Tensor tensor, double gain) { + NoGradGuard guard; + Fan fan(tensor); + const auto std = + gain * std::sqrt(2.0 / static_cast(fan.in + fan.out)); + // Calculate uniform bounds from standard deviation with + const auto a = std::sqrt(3.0) * std; + return tensor.uniform_(-a, a); +} + +// NOLINTNEXTLINE(performance-unnecessary-value-param) +Tensor zeros_(Tensor tensor) { + NoGradGuard guard; + return tensor.zero_(); +} + +std::tuple _calculate_fan_in_and_fan_out( + const Tensor& tensor) { + const auto dimensions = tensor.dim(); + TORCH_CHECK( + dimensions >= 2, + "Fan in and fan out cannot be computed " + "for tensor with fewer than 2 dimensions") + + int64_t fan_in = 0, fan_out = 0; + if (dimensions == 2) { // Linear + fan_in = tensor.size(1); + fan_out = tensor.size(0); + } else { + const auto num_input_fmaps = tensor.size(1); + const auto num_output_fmaps = tensor.size(0); + int64_t receptive_field_size = 1; + if (tensor.dim() > 2) { + receptive_field_size = tensor[0][0].numel(); + } + fan_in = num_input_fmaps * receptive_field_size; + fan_out = num_output_fmaps * receptive_field_size; + } + return std::tie(fan_in, fan_out); +} + +} // namespace torch::nn::init diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/module.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/module.cpp new file mode 100644 index 0000000000000000000000000000000000000000..afb96bd1c409cc3d955206f1ce1243106fb9fa34 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/module.cpp @@ -0,0 +1,415 @@ +#include + +#include + +#include + +#include +#include +#include + +namespace torch::nn { +namespace { +/// Joins names hierarchically: "name_prefix.name" if `name_prefix` is +/// non-empty, else just "name". +std::string join_name(const std::string& name_prefix, const std::string& name) { + size_t total_size = name.size(); + if (!name_prefix.empty()) { + total_size += name_prefix.size() + 1; + } + std::string full_name; + full_name.reserve(total_size); + if (!name_prefix.empty()) { + full_name += name_prefix; + full_name.push_back('.'); + } + full_name += name; + return full_name; +} +} // namespace + +Module::Module() + : parameters_("Parameter"), buffers_("Buffer"), children_("Submodule") {} + +Module::Module(std::string name) : Module() { + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + name_ = std::move(name); +} + +const std::string& Module::name() const noexcept { + // If the name optional is empty at this point, we grab the name of the + // dynamic type via RTTI. Note that we cannot do this in the constructor, + // because in the constructor of a base class `this` always refers to the base + // type. Inheritance effectively does not work in constructors. Also this note + // from http://en.cppreference.com/w/cpp/language/typeid: + // If typeid is used on an object under construction or destruction (in a + // destructor or in a constructor, including constructor's initializer list + // or default member initializers), then the std::type_info object referred + // to by this typeid represents the class that is being constructed or + // destroyed even if it is not the most-derived class. + if (!name_.has_value()) { + name_ = c10::demangle(typeid(*this).name()); +#if defined(_WIN32) + // Windows adds "struct" or "class" as a prefix. + if (name_->find("struct ") == 0) { + name_->erase(name_->begin(), name_->begin() + 7); + } else if (name_->find("class ") == 0) { + name_->erase(name_->begin(), name_->begin() + 6); + } +#endif // defined(_WIN32) + } + return *name_; +} + +std::shared_ptr Module::clone( + const std::optional& device) const { + TORCH_CHECK( + false, + "clone() has not been implemented for ", + name(), + ". Subclass torch::nn::Cloneable<", + name(), + "> instead of torch::nn::Module to inherit the ability to clone."); +} + +void Module::apply(const ModuleApplyFunction& function) { + function(*this); + apply_to_submodules( + [&function](const std::string&, const std::shared_ptr& module) { + function(*module); + }); +} + +void Module::apply(const ConstModuleApplyFunction& function) const { + function(*this); + apply_to_submodules( + [&function](const std::string&, const std::shared_ptr& module) { + function(*module); + }); +} + +void Module::apply( + const NamedModuleApplyFunction& function, + const std::string& name_prefix) { + function(/*name=*/name_prefix, *this); + apply_to_submodules( + [&function]( + const std::string& name, const std::shared_ptr& module) { + function(name, *module); + }, + name_prefix); +} + +void Module::apply( + const ConstNamedModuleApplyFunction& function, + const std::string& name_prefix) const { + function(/*name=*/name_prefix, *this); + apply_to_submodules( + [&function]( + const std::string& name, const std::shared_ptr& module) { + function(name, *module); + }, + name_prefix); +} + +void Module::apply(const ModulePointerApplyFunction& function) const { + function(shared_from_this_checked()); + apply_to_submodules( + [&function](const std::string&, const std::shared_ptr& module) { + function(module); + }); +} + +void Module::apply( + const NamedModulePointerApplyFunction& function, + const std::string& name_prefix) const { + function( + /*name=*/name_prefix, shared_from_this_checked()); + apply_to_submodules(function, name_prefix); +} + +std::vector Module::parameters(bool recurse) const { + return named_parameters(recurse).values(); +} + +OrderedDict Module::named_parameters(bool recurse) const { + OrderedDict result; + if (!recurse) { + for (const auto& parameter : parameters_) { + if (parameter.value().defined()) { + result.insert(parameter.key(), parameter.value()); + } + } + } else { + apply([&result](const std::string& name, const Module& module) { + for (const auto& parameter : module.named_parameters(/*recurse=*/false)) { + TORCH_INTERNAL_ASSERT(parameter.value().defined()); + result.insert(join_name(name, parameter.key()), parameter.value()); + } + }); + } + return result; +} + +std::vector Module::buffers(bool recurse) const { + return named_buffers(recurse).values(); +} + +OrderedDict Module::named_buffers(bool recurse) const { + OrderedDict result; + if (!recurse) { + for (const auto& buffer : buffers_) { + if (buffer.value().defined()) { + result.insert(buffer.key(), buffer.value()); + } + } + } else { + apply([&result](const std::string& name, const Module& module) { + for (const auto& buffer : module.named_buffers(/*recurse=*/false)) { + TORCH_INTERNAL_ASSERT(buffer.value().defined()); + result.insert(join_name(name, buffer.key()), buffer.value()); + } + }); + } + return result; +} + +std::vector> Module::modules(bool include_self) const { + std::vector> result; + if (include_self) { + apply([&result](const std::shared_ptr& module) { + result.push_back(module); + }); + } else { + apply_to_submodules( + [&result](const std::string&, const std::shared_ptr& module) { + result.push_back(module); + }); + } + return result; +} + +OrderedDict> Module::named_modules( + const std::string& name_prefix, + bool include_self) const { + OrderedDict> result; + if (include_self) { + apply( + [&result]( + const std::string& key, const std::shared_ptr& module) { + result.insert(key, module); + }, + name_prefix); + } else { + apply_to_submodules( + [&result]( + const std::string& key, const std::shared_ptr& module) { + result.insert(key, module); + }, + name_prefix); + } + return result; +} + +std::vector> Module::children() const { + return children_.values(); +} + +OrderedDict> Module::named_children() + const { + return children_; +} + +void Module::train(bool on) { + for (auto& child : children_) { + child.value()->train(on); + } + is_training_ = on; +} + +void Module::eval() { + train(/*on=*/false); +} + +void Module::to(torch::Device device, torch::Dtype dtype, bool non_blocking) { + to_impl(device, dtype, non_blocking); +} + +void Module::to(torch::Dtype dtype, bool non_blocking) { + to_impl(dtype, non_blocking); +} + +void Module::to(torch::Device device, bool non_blocking) { + to_impl(device, non_blocking); +} + +bool Module::is_training() const noexcept { + return is_training_; +} + +void Module::zero_grad(bool set_to_none) { + for (auto& child : children_) { + child.value()->zero_grad(set_to_none); + } + for (auto& parameter : named_parameters(/*recurse=*/false)) { + auto& grad = parameter->mutable_grad(); + if (grad.defined()) { + grad = grad.detach(); + + if (set_to_none) + grad.reset(); + else + grad.zero_(); + } + } +} + +void Module::save(serialize::OutputArchive& archive) const { + for (const auto& parameter : named_parameters(/*recurse=*/false)) { + archive.write(parameter.key(), parameter.value()); + } + for (const auto& buffer : named_buffers(/*recurse=*/false)) { + archive.write(buffer.key(), buffer.value(), /*is_buffer=*/true); + } + for (const auto& child : children_) { + if (child.value()->is_serializable()) { + serialize::OutputArchive child_archive(archive.compilation_unit()); + child.value()->save(child_archive); + archive.write(child.key(), child_archive); + } + } +} + +void Module::load(serialize::InputArchive& archive) { + for (auto& parameter : named_parameters(/*recurse=*/false)) { + archive.read(parameter.key(), parameter.value()); + } + for (auto& buffer : named_buffers(/*recurse=*/false)) { + archive.read(buffer.key(), buffer.value(), /*is_buffer=*/true); + } + for (const auto& child : children_) { + if (child.value()->is_serializable()) { + serialize::InputArchive child_archive; + archive.read(child.key(), child_archive); + child.value()->load(child_archive); + } + } +} + +bool Module::is_serializable() const { + return true; +} + +Tensor& Module::register_parameter( + std::string name, + Tensor tensor, + bool requires_grad) { + TORCH_CHECK(!name.empty(), "Parameter name must not be empty"); + TORCH_CHECK( + name.find('.') == std::string::npos, + "Parameter name must not contain a dot (got '", + name, + "')"); + if (!tensor.defined()) { + if (requires_grad) { + TORCH_WARN( + "An undefined tensor cannot require grad. ", + "Ignoring the `requires_grad=true` function parameter."); + } + } else { + tensor.set_requires_grad(requires_grad); + } + return parameters_.insert(std::move(name), std::move(tensor)); +} + +Tensor& Module::register_buffer(std::string name, Tensor tensor) { + TORCH_CHECK(!name.empty(), "Buffer name must not be empty"); + TORCH_CHECK( + name.find('.') == std::string::npos, + "Buffer name must not contain a dot (got '", + name, + "')"); + return buffers_.insert(std::move(name), std::move(tensor)); +} + +void Module::unregister_module(const std::string& name) { + TORCH_CHECK( + children_.contains(name), + "No Module with name `", + name, + "` is registered"); + children_.erase(name); +} + +void Module::pretty_print(std::ostream& stream) const { + stream << name(); +} + +void Module::pretty_print_recursive( + std::ostream& stream, + const std::string& indentation) const { + pretty_print(stream); + if (!children_.is_empty()) { + stream << "(\n"; + const std::string next_indentation = indentation + " "; + for (const auto& child : children_) { + stream << next_indentation << '(' << child.key() << "): "; + child.value()->pretty_print_recursive(stream, next_indentation); + stream << '\n'; + } + stream << indentation << ')'; + } +} + +void Module::clone_(Module& other, const std::optional& device) {} + +void Module::apply_to_submodules( + const NamedModulePointerApplyFunction& function, + const std::string& name_prefix) const { + for (const auto& child : children_) { + auto qualified_name = join_name(name_prefix, child.key()); + function(qualified_name, child.value()); + child.value()->apply_to_submodules(function, qualified_name); + } +} + +std::shared_ptr Module::shared_from_this_checked() const { + std::shared_ptr ptr; + try { + ptr = shared_from_this(); + } catch (const std::bad_weak_ptr&) { + TORCH_CHECK( + false, + "It looks like you attempted to retrieve your top-level module " + "as a shared_ptr, but it is not stored in a shared_ptr. " + "Use std::make_shared<", + name(), + "> instead of creating your module on " + "the stack, or alternatively do not try to access your top-level " + "module at all by passing /*include_self=*/false " + "to modules() or named_modules()"); + } + return std::const_pointer_cast(ptr); +} + +std::ostream& operator<<(std::ostream& stream, const nn::Module& module) { + module.pretty_print_recursive(stream, ""); + return stream; +} + +serialize::OutputArchive& operator<<( + serialize::OutputArchive& archive, + const std::shared_ptr& module) { + TORCH_CHECK(module != nullptr, "Cannot serialize empty module"); + module->save(archive); + return archive; +} + +serialize::InputArchive& operator>>( + serialize::InputArchive& archive, + const std::shared_ptr& module) { + TORCH_CHECK(module != nullptr, "Cannot deserialize empty module"); + module->load(archive); + return archive; +} +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/_functions.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/_functions.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4c109e0dc19bdbbc92c60f79456f21082af8619d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/_functions.cpp @@ -0,0 +1,139 @@ +#include +#include + +using namespace torch::autograd; + +namespace torch::nn::functions { + +Variable CrossMapLRN2d::forward( + AutogradContext* ctx, + const Variable& input, + const CrossMapLRN2dOptions& options) { + ctx->saved_data["size"] = options.size(); + ctx->saved_data["alpha"] = options.alpha(); + ctx->saved_data["beta"] = options.beta(); + ctx->saved_data["k"] = options.k(); + ctx->saved_data["scale"] = torch::Tensor(); + + TORCH_CHECK(input.dim() == 4); + + ctx->saved_data["scale"] = ctx->saved_data["scale"].toTensor().defined() + ? ctx->saved_data["scale"] + : torch::empty({0}, input.options()); + + torch::Tensor output = torch::empty({0}, input.options()); + + int64_t channels = input.size(1); + + output.resize_as_(input); + ctx->saved_data["scale"].toTensor().resize_as_(input); + + /// use output storage as temporary buffer + auto input_square = output; + torch::pow_out(input_square, input, 2); + + int64_t pre_pad = + static_cast((ctx->saved_data["size"].toInt() - 1) / 2 + 1); + int64_t pre_pad_crop = pre_pad > channels ? channels : pre_pad; + + auto scale_first = ctx->saved_data["scale"].toTensor().select(1, 0); + scale_first.zero_(); + + /// compute first feature map normalization + for (const auto c : c10::irange(pre_pad_crop)) { + scale_first.add_(input_square.select(1, c)); + } + + /// reuse computations for next feature maps normalization + /// by adding the next feature map and removing the previous + torch::Tensor scale_previous, scale_current, square_next, square_previous; + + for (const auto c : c10::irange(1, channels)) { + scale_previous = ctx->saved_data["scale"].toTensor().select(1, c - 1); + scale_current = ctx->saved_data["scale"].toTensor().select(1, c); + scale_current.copy_(scale_previous); + + if (c < channels - pre_pad + 1) { + square_next = input_square.select(1, c + pre_pad - 1); + scale_current.add_(square_next, 1); + } + + if (c > pre_pad) { + square_previous = input_square.select(1, c - pre_pad); + scale_current.add_(square_previous, -1); + } + } + + ctx->saved_data["scale"] + .toTensor() + .mul_( + ctx->saved_data["alpha"].toDouble() / + static_cast(ctx->saved_data["size"].toInt())) + .add_(ctx->saved_data["k"].toInt()); + + torch::pow_out( + output, + ctx->saved_data["scale"].toTensor(), + -ctx->saved_data["beta"].toDouble()); + output.mul_(input); + + ctx->save_for_backward({input, output}); + return output; +} + +variable_list CrossMapLRN2d::backward( + AutogradContext* ctx, + variable_list grad_outputs) { + auto const& grad_output = grad_outputs[0]; + auto input = ctx->get_saved_variables()[0]; + auto output = ctx->get_saved_variables()[1]; + auto grad_input = torch::empty({0}, grad_output.options()); + + int64_t batch_size = input.size(0); + int64_t channels = input.size(1); + int64_t input_height = input.size(2); + int64_t input_width = input.size(3); + + auto padded_ratio = torch::empty( + {channels + ctx->saved_data["size"].toInt() - 1, + input_height, + input_width}, + input.options()); + auto accum_ratio = torch::empty({input_height, input_width}, input.options()); + double cache_ratio_value = 2 * ctx->saved_data["alpha"].toDouble() * + ctx->saved_data["beta"].toDouble() / + static_cast(ctx->saved_data["size"].toInt()); + int64_t inversePrePad = + (ctx->saved_data["size"].toInt() - + (ctx->saved_data["size"].toInt() - 1) / 2); + + grad_input.resize_as_(input); + torch::pow_out( + grad_input, + ctx->saved_data["scale"].toTensor(), + -ctx->saved_data["beta"].toDouble()) + .mul_(grad_output); + + padded_ratio.zero_(); + auto padded_ratio_center = padded_ratio.narrow(0, inversePrePad, channels); + + for (const auto n : c10::irange(batch_size)) { + torch::mul_out(padded_ratio_center, grad_output[n], output[n]); + padded_ratio_center.div_(ctx->saved_data["scale"].toTensor()[n]); + torch::sum_out( + accum_ratio, + padded_ratio.narrow(0, 0, ctx->saved_data["size"].toInt() - 1), + 0, + /*keepdim=*/false); + for (const auto c : c10::irange(channels)) { + accum_ratio.add_(padded_ratio[c + ctx->saved_data["size"].toInt() - 1]); + grad_input[n][c].addcmul_(input[n][c], accum_ratio, -cache_ratio_value); + accum_ratio.add_(padded_ratio[c], -1); + } + } + + return variable_list{ + grad_input, Variable(), Variable(), Variable(), Variable()}; +} + +} // namespace torch::nn::functions diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/activation.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/activation.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8378ab2775812b8172aad195facff16c496b04e0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/activation.cpp @@ -0,0 +1,569 @@ +#include +#include +#include + +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +ELUImpl::ELUImpl(const ELUOptions& options_) : options(options_) {} + +Tensor ELUImpl::forward(Tensor input) { + return F::detail::elu(std::move(input), options.alpha(), options.inplace()); +} + +void ELUImpl::reset() {} + +void ELUImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::ELU(alpha=" << options.alpha(); + if (options.inplace()) { + stream << std::boolalpha << ", inplace=" << options.inplace(); + } + stream << ')'; +} + +// ============================================================================ + +SELUImpl::SELUImpl(const SELUOptions& options_) : options(options_) {} + +Tensor SELUImpl::forward(Tensor input) { + return F::detail::selu(std::move(input), options.inplace()); +} + +void SELUImpl::reset() {} + +void SELUImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::SELU("; + if (options.inplace()) { + stream << std::boolalpha << "inplace=" << options.inplace(); + } + stream << ')'; +} + +// ============================================================================ + +HardshrinkImpl::HardshrinkImpl(const HardshrinkOptions& options_) + : options(options_) {} + +Tensor HardshrinkImpl::forward(const Tensor& input) { + return F::detail::hardshrink(input, options.lambda()); +} + +void HardshrinkImpl::reset() {} + +void HardshrinkImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::Hardshrink(" << options.lambda() + << ')'; +} + +// ============================================================================ + +HardtanhImpl::HardtanhImpl(const HardtanhOptions& options_) + : options(options_) { + HardtanhImpl::reset(); +} + +Tensor HardtanhImpl::forward(Tensor input) { + return F::detail::hardtanh( + std::move(input), + options.min_val(), + options.max_val(), + options.inplace()); +} + +void HardtanhImpl::reset() { + TORCH_CHECK( + options.max_val() > options.min_val(), + "max_val must be greater than min_val"); +} + +void HardtanhImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha + << "torch::nn::Hardtanh(min_val=" << options.min_val() + << ", max_val=" << options.max_val(); + if (options.inplace()) { + stream << std::boolalpha << ", inplace=" << options.inplace(); + } + stream << ')'; +} + +// ============================================================================ + +LeakyReLUImpl::LeakyReLUImpl(const LeakyReLUOptions& options_) + : options(options_) {} + +Tensor LeakyReLUImpl::forward(Tensor input) { + return F::detail::leaky_relu( + std::move(input), options.negative_slope(), options.inplace()); +} + +void LeakyReLUImpl::reset() {} + +void LeakyReLUImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha + << "torch::nn::LeakyReLU(negative_slope=" << options.negative_slope(); + if (options.inplace()) { + stream << std::boolalpha << ", inplace=" << options.inplace(); + } + stream << ')'; +} + +// ============================================================================ + +Tensor LogSigmoidImpl::forward(const Tensor& input) { + return F::logsigmoid(input); +} + +void LogSigmoidImpl::reset() {} + +void LogSigmoidImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::LogSigmoid()"; +} + +// ============================================================================ + +SoftmaxImpl::SoftmaxImpl(const SoftmaxOptions& options_) : options(options_) {} + +void SoftmaxImpl::reset() {} + +void SoftmaxImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Softmax(dim=" << options.dim() << ')'; +} + +Tensor SoftmaxImpl::forward(const Tensor& input) { + return F::detail::softmax(input, options.dim(), std::nullopt); +} + +// ============================================================================ + +SoftminImpl::SoftminImpl(const SoftminOptions& options_) : options(options_) {} + +void SoftminImpl::reset() {} + +void SoftminImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Softmin(dim=" << options.dim() << ')'; +} + +Tensor SoftminImpl::forward(const Tensor& input) { + return F::detail::softmin(input, options.dim(), std::nullopt); +} + +// ============================================================================ + +LogSoftmaxImpl::LogSoftmaxImpl(const LogSoftmaxOptions& options_) + : options(options_) {} + +void LogSoftmaxImpl::reset() {} + +void LogSoftmaxImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::LogSoftmax(dim=" << options.dim() << ')'; +} + +Tensor LogSoftmaxImpl::forward(const Tensor& input) { + return F::detail::log_softmax(input, options.dim(), std::nullopt); +} + +// ============================================================================ + +void Softmax2dImpl::reset() {} + +void Softmax2dImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Softmax2d()"; +} + +Tensor Softmax2dImpl::forward(const Tensor& input) { + TORCH_CHECK( + input.dim() == 4 || input.dim() == 3, + "Softmax2d requires a 3D or 4D tensor as input"); + return F::detail::softmax(input, /*dim=*/-3, std::nullopt); +} + +// ============================================================================ + +PReLUImpl::PReLUImpl(const PReLUOptions& options_) : options(options_) { + PReLUImpl::reset(); +} + +Tensor PReLUImpl::forward(const Tensor& input) { + return F::prelu(input, weight); +} + +void PReLUImpl::reset() { + weight = register_parameter( + "weight", torch::full(options.num_parameters(), options.init())); +} + +void PReLUImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::PReLU(num_parameters=" << options.num_parameters() + << ')'; +} + +// ============================================================================ + +ReLUImpl::ReLUImpl(const ReLUOptions& options_) : options(options_) {} + +Tensor ReLUImpl::forward(Tensor input) { + return F::detail::relu(std::move(input), options.inplace()); +} + +void ReLUImpl::reset() {} + +void ReLUImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::ReLU("; + if (options.inplace()) { + stream << std::boolalpha << "inplace=" << options.inplace(); + } + stream << ')'; +} + +// ============================================================================ + +ReLU6Impl::ReLU6Impl(const ReLU6Options& options_) : options(options_) {} + +Tensor ReLU6Impl::forward(Tensor input) { + return F::detail::relu6(std::move(input), options.inplace()); +} + +void ReLU6Impl::reset() {} + +void ReLU6Impl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::ReLU6("; + if (options.inplace()) { + stream << std::boolalpha << "inplace=" << options.inplace(); + } + stream << ')'; +} + +// ============================================================================ + +RReLUImpl::RReLUImpl(const RReLUOptions& options_) : options(options_) {} + +Tensor RReLUImpl::forward(Tensor input) { + return F::detail::rrelu( + std::move(input), + options.lower(), + options.upper(), + is_training(), + options.inplace()); +} + +void RReLUImpl::reset() {} + +void RReLUImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::RReLU(lower=" << options.lower() + << ", upper=" << options.upper(); + if (options.inplace()) { + stream << std::boolalpha << ", inplace=" << options.inplace(); + } + stream << ')'; +} + +// ============================================================================ + +CELUImpl::CELUImpl(const CELUOptions& options_) : options(options_) {} + +Tensor CELUImpl::forward(Tensor input) { + return F::detail::celu(std::move(input), options.alpha(), options.inplace()); +} + +void CELUImpl::reset() {} + +void CELUImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::CELU(alpha=" << options.alpha(); + if (options.inplace()) { + stream << std::boolalpha << ", inplace=" << options.inplace(); + } + stream << ')'; +} + +// ============================================================================ + +GLUImpl::GLUImpl(const GLUOptions& options_) : options(options_) {} + +Tensor GLUImpl::forward(const Tensor& input) { + return F::detail::glu(input, options.dim()); +} + +void GLUImpl::reset() {} + +void GLUImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::GLU(dim=" << options.dim() << ')'; +} + +// ============================================================================ + +GELUImpl::GELUImpl(GELUOptions options_) : options(std::move(options_)) {} + +Tensor GELUImpl::forward(const Tensor& input) { + return F::detail::gelu(input, options.approximate()); +} + +void GELUImpl::reset() {} + +void GELUImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::GELU()"; +} + +// ============================================================================ + +Tensor SiLUImpl::forward(const Tensor& input) { + return F::silu(input); +} + +void SiLUImpl::reset() {} + +void SiLUImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::SiLU()"; +} + +// ============================================================================ + +Tensor MishImpl::forward(const Tensor& input) { + return F::mish(input); +} + +void MishImpl::reset() {} + +void MishImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Mish()"; +} + +// ============================================================================ + +Tensor SigmoidImpl::forward(const Tensor& input) { + return torch::sigmoid(input); +} + +void SigmoidImpl::reset() {} + +void SigmoidImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Sigmoid()"; +} + +// ============================================================================ + +SoftplusImpl::SoftplusImpl(const SoftplusOptions& options_) + : options(options_) {} + +Tensor SoftplusImpl::forward(const Tensor& input) { + return F::detail::softplus(input, options.beta(), options.threshold()); +} + +void SoftplusImpl::reset() {} + +void SoftplusImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Softplus(beta=" << options.beta() + << ", threshold=" << options.threshold() << ')'; +} + +// ============================================================================ + +SoftshrinkImpl::SoftshrinkImpl(const SoftshrinkOptions& options_) + : options(options_) {} + +Tensor SoftshrinkImpl::forward(const Tensor& input) { + return F::detail::softshrink(input, options.lambda()); +} + +void SoftshrinkImpl::reset() {} + +void SoftshrinkImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Softshrink(" << options.lambda() << ')'; +} + +// ============================================================================ + +Tensor SoftsignImpl::forward(const Tensor& input) { + return F::softsign(input); +} + +void SoftsignImpl::reset() {} + +void SoftsignImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Softsign()"; +} + +// ============================================================================ + +Tensor TanhImpl::forward(const Tensor& input) { + return torch::tanh(input); +} + +void TanhImpl::reset() {} + +void TanhImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Tanh()"; +} + +// ============================================================================ + +Tensor TanhshrinkImpl::forward(const Tensor& input) { + return F::tanhshrink(input); +} + +void TanhshrinkImpl::reset() {} + +void TanhshrinkImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Tanhshrink()"; +} + +// ============================================================================ + +ThresholdImpl::ThresholdImpl(const ThresholdOptions& options_) + : options(options_) {} + +Tensor ThresholdImpl::forward(Tensor input) { + return F::detail::threshold( + std::move(input), + options.threshold(), + options.value(), + options.inplace()); +} + +void ThresholdImpl::reset() {} + +void ThresholdImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Threshold(threshold=" << options.threshold() + << ", value=" << options.value(); + if (options.inplace()) { + stream << std::boolalpha << ", inplace=" << options.inplace(); + } + stream << ')'; +} + +// ============================================================================ + +MultiheadAttentionImpl::MultiheadAttentionImpl( + const MultiheadAttentionOptions& options_) + : Cloneable("torch::nn::MultiheadAttention"), options(options_) { + MultiheadAttentionImpl::reset(); +} + +std::tuple MultiheadAttentionImpl::forward( + const Tensor& query, + const Tensor& key, + const Tensor& value, + const Tensor& key_padding_mask, + bool need_weights, + const Tensor& attn_mask, + bool average_attn_weights) { + if (!_qkv_same_embed_dim) { + return F::multi_head_attention_forward( + query, + key, + value, + F::MultiheadAttentionForwardFuncOptions( + /*embed_dim_to_check=*/options.embed_dim(), + /*num_heads=*/options.num_heads(), + /*in_proj_weight=*/in_proj_weight, + /*in_proj_bias=*/in_proj_bias, + /*bias_k=*/bias_k, + /*bias_v=*/bias_v, + /*add_zero_attn=*/options.add_zero_attn(), + /*dropout_p=*/options.dropout(), + /*out_proj_weight=*/out_proj->weight, + /*out_proj_bias=*/out_proj->bias) + .training(is_training()) + .key_padding_mask(key_padding_mask) + .need_weights(need_weights) + .attn_mask(attn_mask) + .use_separate_proj_weight(true) + .q_proj_weight(q_proj_weight) + .k_proj_weight(k_proj_weight) + .v_proj_weight(v_proj_weight) + .average_attn_weights(average_attn_weights)); + } else { + return F::multi_head_attention_forward( + query, + key, + value, + F::MultiheadAttentionForwardFuncOptions( + /*embed_dim_to_check=*/options.embed_dim(), + /*num_heads=*/options.num_heads(), + /*in_proj_weight=*/in_proj_weight, + /*in_proj_bias=*/in_proj_bias, + /*bias_k=*/bias_k, + /*bias_v=*/bias_v, + /*add_zero_attn=*/options.add_zero_attn(), + /*dropout_p=*/options.dropout(), + /*out_proj_weight=*/out_proj->weight, + /*out_proj_bias=*/out_proj->bias) + .training(is_training()) + .key_padding_mask(key_padding_mask) + .need_weights(need_weights) + .attn_mask(attn_mask) + .average_attn_weights(average_attn_weights)); + } +} + +void MultiheadAttentionImpl::reset() { + _qkv_same_embed_dim = options.kdim() == options.embed_dim() && + options.vdim() == options.embed_dim(); + TORCH_CHECK(options.num_heads() > 0, "num_heads must be greater than 0"); + head_dim = options.embed_dim() / options.num_heads(); + TORCH_CHECK( + head_dim * options.num_heads() == options.embed_dim(), + "embed_dim must be divisible by num_heads"); + if (!_qkv_same_embed_dim) { + q_proj_weight = register_parameter( + "q_proj_weight", + torch::empty({options.embed_dim(), options.embed_dim()})); + k_proj_weight = register_parameter( + "k_proj_weight", torch::empty({options.embed_dim(), options.kdim()})); + v_proj_weight = register_parameter( + "v_proj_weight", torch::empty({options.embed_dim(), options.vdim()})); + register_parameter("in_proj_weight", {}, /*requires_grad=*/false); + } else { + in_proj_weight = register_parameter( + "in_proj_weight", + torch::empty({3 * options.embed_dim(), options.embed_dim()})); + register_parameter("q_proj_weight", {}, /*requires_grad=*/false); + register_parameter("k_proj_weight", {}, /*requires_grad=*/false); + register_parameter("v_proj_weight", {}, /*requires_grad=*/false); + } + if (options.bias()) { + in_proj_bias = register_parameter( + "in_proj_bias", torch::empty(3 * options.embed_dim())); + } else { + register_parameter("in_proj_bias", {}, /*requires_grad=*/false); + } + out_proj = register_module( + "out_proj", + Linear(LinearOptions(options.embed_dim(), options.embed_dim()) + .bias(options.bias()))); + if (options.add_bias_kv()) { + bias_k = + register_parameter("bias_k", torch::empty({1, 1, options.embed_dim()})); + bias_v = + register_parameter("bias_v", torch::empty({1, 1, options.embed_dim()})); + } else { + bias_k.reset(); + bias_v.reset(); + } + _reset_parameters(); +} + +void MultiheadAttentionImpl::_reset_parameters() { + using namespace torch::nn::init; + if (_qkv_same_embed_dim) { + xavier_uniform_(in_proj_weight); + } else { + xavier_uniform_(q_proj_weight); + xavier_uniform_(k_proj_weight); + xavier_uniform_(v_proj_weight); + } + if (in_proj_bias.defined()) { + constant_(in_proj_bias, 0.); + constant_(out_proj->bias, 0.); + } + if (bias_k.defined()) { + xavier_normal_(bias_k); + } + if (bias_v.defined()) { + xavier_normal_(bias_v); + } +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/adaptive.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/adaptive.cpp new file mode 100644 index 0000000000000000000000000000000000000000..83fa661107d38a5b3ea1e911121180237e24cd6f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/adaptive.cpp @@ -0,0 +1,222 @@ +#include +#include +#include +#include + +namespace F = torch::nn::functional; + +using namespace torch::indexing; + +namespace torch::nn { + +ASMoutput::ASMoutput(Tensor output_, double loss_) + : output(std::move(output_)), loss(loss_) {} + +AdaptiveLogSoftmaxWithLossImpl::AdaptiveLogSoftmaxWithLossImpl( + AdaptiveLogSoftmaxWithLossOptions options_) + : options(std::move(options_)), + shortlist_size(0), + n_clusters(0), + head_size(0) { + AdaptiveLogSoftmaxWithLossImpl::reset(); +} + +void AdaptiveLogSoftmaxWithLossImpl::reset() { + TORCH_CHECK( + !options.cutoffs().empty(), + "cutoffs should be a sequence of length larger than 0"); + TORCH_CHECK( + std::is_sorted(options.cutoffs().begin(), options.cutoffs().end()) && + *std::min_element( + options.cutoffs().begin(), options.cutoffs().end()) > 0 && + *std::max_element( + options.cutoffs().begin(), options.cutoffs().end()) <= + (options.n_classes() - 1) && + std::set(options.cutoffs().begin(), options.cutoffs().end()) + .size() == options.cutoffs().size(), + "cutoffs should be a sequence of unique, positive integers sorted in an increasing order, ", + "where each value is between 1 and n_classes-1"); + TORCH_CHECK(options.div_value() != 0, "div_value should not be equal to 0"); + + cutoffs = options.cutoffs(); + cutoffs.push_back(options.n_classes()); + + shortlist_size = cutoffs[0]; + n_clusters = static_cast(cutoffs.size() - 1); + head_size = shortlist_size + n_clusters; + + head = this->register_module( + "head", + Linear(LinearOptions(options.in_features(), head_size) + .bias(options.head_bias()))); + tail = this->register_module("tail", ModuleList()); + + for (const auto i : c10::irange(n_clusters)) { + int64_t hsz = static_cast(std::floor( + static_cast(options.in_features()) / + std::pow(options.div_value(), (i + 1)))); + int64_t osz = cutoffs[i + 1] - cutoffs[i]; + + Sequential projection( + Linear(LinearOptions(options.in_features(), hsz).bias(false)), + Linear(LinearOptions(hsz, osz).bias(false))); + tail->push_back(projection); + } +} + +void AdaptiveLogSoftmaxWithLossImpl::reset_parameters() { + head->reset_parameters(); + for (const auto i : c10::irange(tail->size())) { + auto i2h = tail[i]->children()[0]->as(); + auto h2o = tail[i]->children()[1]->as(); + i2h->reset_parameters(); + h2o->reset_parameters(); + } +} + +ASMoutput AdaptiveLogSoftmaxWithLossImpl::forward( + const Tensor& input_, + const Tensor& target_) { + auto targ_dim = target_.dim(); + + TORCH_CHECK( + targ_dim == 1 || targ_dim == 0, + "0D or 1D target tensor expected, multi-target not supported"); + + if (targ_dim == 1) { + TORCH_CHECK( + input_.dim() == 2, + "1D target tensor expects 2D input tensors, but found inputs with sizes ", + input_.sizes(), + "."); + } else { + TORCH_CHECK( + input_.dim() == 1, + "0D target tensor expects 1D input tensors, but found inputs with sizes ", + input_.sizes(), + "."); + } + + bool is_batched = (targ_dim > 0); + Tensor input = is_batched ? input_ : input_.unsqueeze(0); + Tensor target = is_batched ? target_ : target_.unsqueeze(0); + + int64_t used_rows = 0; + const int64_t batch_size = target.size(0); + + Tensor output = input.new_zeros(batch_size); + Tensor gather_inds = target.new_empty(batch_size); + + auto cutoff_values = cutoffs; + cutoff_values.insert(cutoff_values.begin(), 0); + + for (const auto i : c10::irange(cutoff_values.size() - 1)) { + int64_t low_idx = cutoff_values[i]; + int64_t high_idx = cutoff_values[i + 1]; + + const Tensor target_mask = (target >= low_idx) * (target < high_idx); + const Tensor row_indices = target_mask.nonzero().squeeze(); + + if (row_indices.numel() == 0) { + continue; + } + + if (i == 0) { + gather_inds.index_copy_(0, row_indices, target.index({target_mask})); + } else { + Tensor relative_target = target.index({target_mask}) - low_idx; + Tensor input_subset = input.index_select(0, row_indices); + + const Tensor cluster_output = + tail[i - 1]->as()->forward(input_subset); + int64_t cluster_index = shortlist_size + static_cast(i) - 1; + + gather_inds.index_fill_(0, row_indices, cluster_index); + + const Tensor cluster_logprob = F::log_softmax(cluster_output, 1); + const Tensor local_logprob = + cluster_logprob.gather(1, relative_target.unsqueeze(1)); + output.index_copy_(0, row_indices, local_logprob.squeeze(1)); + } + + used_rows += row_indices.numel(); + } + + TORCH_CHECK( + used_rows == batch_size, + "Target values should be in [0, ", + options.n_classes() - 1, + "], " + "but values in range [", + target.min().item().toDouble(), + ", ", + target.max().item().toDouble(), + "] " + "were found. "); + + const Tensor head_output = head(input); + const Tensor head_logprob = F::log_softmax(head_output, 1); + output += head_logprob.gather(1, gather_inds.unsqueeze(1)).squeeze(); + const double loss = (-output).mean().item().toDouble(); + + if (!is_batched) { + output = output.squeeze(0); + } + + return ASMoutput(output, loss); +} + +Tensor AdaptiveLogSoftmaxWithLossImpl::_get_full_log_prob( + const Tensor& input, + const Tensor& head_output) { + Tensor out = input.new_empty({head_output.size(0), options.n_classes()}); + const Tensor head_logprob = F::log_softmax(head_output, 1); + + out.index_put_( + {Slice(), Slice(None, shortlist_size)}, + head_logprob.index({Slice(), Slice(None, shortlist_size)})); + + for (const auto i : c10::irange(cutoffs.size() - 1)) { + int64_t start_idx = cutoffs[i]; + int64_t stop_idx = cutoffs[i + 1]; + const Tensor cluster_output = tail[i]->as()->forward(input); + const Tensor cluster_logprob = F::log_softmax(cluster_output, 1); + auto output_logprob = cluster_logprob + + head_logprob.index({Slice(), static_cast(shortlist_size + i)}) + .unsqueeze(1); + + out.index_put_({Slice(), Slice(start_idx, stop_idx)}, output_logprob); + } + return out; +} + +Tensor AdaptiveLogSoftmaxWithLossImpl::AdaptiveLogSoftmaxWithLossImpl::log_prob( + const Tensor& input) { + const Tensor head_output = head(input); + return _get_full_log_prob(input, head_output); +} + +Tensor AdaptiveLogSoftmaxWithLossImpl::predict(const Tensor& input) { + const Tensor head_output = head(input); + Tensor output = torch::argmax(head_output, 1); + auto not_in_shortlist = (output >= shortlist_size); + auto all_in_shortlist = bitwise_not(not_in_shortlist.any()); + + if (all_in_shortlist.item().toBool()) { + return output; + } else if (not_in_shortlist.all().item().toBool()) { + const Tensor log_prob = _get_full_log_prob(input, head_output); + return torch::argmax(log_prob, 1); + } else { + const Tensor log_prob = _get_full_log_prob( + input.index({not_in_shortlist}), head_output.index({not_in_shortlist})); + output.index_put_({not_in_shortlist}, torch::argmax(log_prob, 1)); + return output; + } +} + +void AdaptiveLogSoftmaxWithLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::AdaptiveLogSoftmaxWithLoss"; +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/batchnorm.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/batchnorm.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7bbf80178b6ff2f1c831a8fccb0c1089dd5be97f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/batchnorm.cpp @@ -0,0 +1,30 @@ +#include +#include + +#include + +namespace torch::nn { + +void BatchNorm1dImpl::_check_input_dim(const Tensor& input) { + TORCH_CHECK( + input.dim() == 2 || input.dim() == 3, + "expected 2D or 3D input (got ", + input.dim(), + "D input)"); +} + +void BatchNorm2dImpl::_check_input_dim(const Tensor& input) { + TORCH_CHECK( + input.dim() == 4, "expected 4D input (got ", input.dim(), "D input)"); +} + +void BatchNorm3dImpl::_check_input_dim(const Tensor& input) { + TORCH_CHECK( + input.dim() == 5, "expected 5D input (got ", input.dim(), "D input)"); +} + +template class BatchNormImplBase<1, BatchNorm1dImpl>; +template class BatchNormImplBase<2, BatchNorm2dImpl>; +template class BatchNormImplBase<3, BatchNorm3dImpl>; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/container/functional.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/container/functional.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8b55498276bc29595e4a007c5e9b87b4b5307303 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/container/functional.cpp @@ -0,0 +1,27 @@ +#include + +#include +#include + +namespace torch::nn { +FunctionalImpl::FunctionalImpl(Function function) + : function_(std::move(function)) {} + +void FunctionalImpl::reset() {} + +void FunctionalImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Functional()"; +} + +Tensor FunctionalImpl::forward(Tensor input) { + return function_(std::move(input)); +} + +Tensor FunctionalImpl::operator()(Tensor input) { + return forward(std::move(input)); +} + +bool FunctionalImpl::is_serializable() const { + return false; +} +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/conv.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/conv.cpp new file mode 100644 index 0000000000000000000000000000000000000000..174435a3113795d624e8ab0a0b88f7ab566105b2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/conv.cpp @@ -0,0 +1,343 @@ +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace F = torch::nn::functional; + +static F::PadFuncOptions::mode_t _get_pad_mode_from_conv_padding_mode( + torch::nn::detail::conv_padding_mode_t conv_padding_mode) { + F::PadFuncOptions::mode_t pad_mode; + if (std::holds_alternative(conv_padding_mode)) { + pad_mode = torch::kReflect; + } else if (std::holds_alternative( + conv_padding_mode)) { + pad_mode = torch::kReplicate; + } else if (std::holds_alternative( + conv_padding_mode)) { + pad_mode = torch::kCircular; + } else { + TORCH_CHECK( + false, + "Unsupported conv padding mode: ", + torch::enumtype::get_enum_name(conv_padding_mode)); + } + return pad_mode; +} + +namespace torch::nn { +Conv1dImpl::Conv1dImpl(Conv1dOptions options_) + : ConvNdImpl(detail::ConvNdOptions<1>( + /*in_channels=*/options_.in_channels(), + /*out_channels=*/options_.out_channels(), + /*kernel_size=*/options_.kernel_size()) + .stride(options_.stride()) + .padding(options_.padding()) + .dilation(options_.dilation()) + .transposed(false) + .output_padding(0) + .groups(options_.groups()) + .bias(options_.bias()) + .padding_mode(options_.padding_mode())) {} + +Tensor Conv1dImpl::forward(const Tensor& input) { + if (!std::get_if(&options.padding_mode())) { + return F::detail::conv1d( + F::pad( + input, + F::PadFuncOptions(_reversed_padding_repeated_twice) + .mode(_get_pad_mode_from_conv_padding_mode( + options.padding_mode()))), + weight, + bias, + options.stride(), + /*padding=*/0, + options.dilation(), + options.groups()); + } + return F::detail::conv1d( + input, + weight, + bias, + options.stride(), + options.padding(), + options.dilation(), + options.groups()); +} + +Conv2dImpl::Conv2dImpl(Conv2dOptions options_) + : ConvNdImpl(detail::ConvNdOptions<2>( + /*in_channels=*/options_.in_channels(), + /*out_channels=*/options_.out_channels(), + /*kernel_size=*/options_.kernel_size()) + .stride(options_.stride()) + .padding(options_.padding()) + .dilation(options_.dilation()) + .transposed(false) + .output_padding(0) + .groups(options_.groups()) + .bias(options_.bias()) + .padding_mode(options_.padding_mode())) {} + +Tensor Conv2dImpl::_conv_forward(const Tensor& input, const Tensor& weight) { + if (!std::get_if(&options.padding_mode())) { + return F::detail::conv2d( + F::pad( + input, + F::PadFuncOptions(_reversed_padding_repeated_twice) + .mode(_get_pad_mode_from_conv_padding_mode( + options.padding_mode()))), + weight, + bias, + options.stride(), + /*padding=*/0, + options.dilation(), + options.groups()); + } + return F::detail::conv2d( + input, + weight, + bias, + options.stride(), + options.padding(), + options.dilation(), + options.groups()); +} + +Tensor Conv2dImpl::forward(const Tensor& input) { + return _conv_forward(input, weight); +} + +Conv3dImpl::Conv3dImpl(Conv3dOptions options_) + : ConvNdImpl(detail::ConvNdOptions<3>( + /*in_channels=*/options_.in_channels(), + /*out_channels=*/options_.out_channels(), + /*kernel_size=*/options_.kernel_size()) + .stride(options_.stride()) + .padding(options_.padding()) + .dilation(options_.dilation()) + .transposed(false) + .output_padding(0) + .groups(options_.groups()) + .bias(options_.bias()) + .padding_mode(options_.padding_mode())) {} + +Tensor Conv3dImpl::forward(const Tensor& input) { + if (!std::get_if(&options.padding_mode())) { + return F::detail::conv3d( + F::pad( + input, + F::PadFuncOptions(_reversed_padding_repeated_twice) + .mode(_get_pad_mode_from_conv_padding_mode( + options.padding_mode()))), + weight, + bias, + options.stride(), + /*padding=*/0, + options.dilation(), + options.groups()); + } + return F::detail::conv3d( + input, + weight, + bias, + options.stride(), + options.padding(), + options.dilation(), + options.groups()); +} + +template class ConvNdImpl<1, Conv1dImpl>; +template class ConvNdImpl<2, Conv2dImpl>; +template class ConvNdImpl<3, Conv3dImpl>; + +// ============================================================================ + +template +std::vector ConvTransposeNdImpl::_output_padding( + const Tensor& input, + const std::optional& output_size, + const ExpandingArray& stride, + const ExpandingArray& padding, + const ExpandingArray& kernel_size) { + std::vector ret; + std::optional output_size_ = output_size; + + if (output_size_ == std::nullopt) { + ret = at::IntArrayRef(this->options.output_padding()).vec(); + } else { + auto k = input.dim() - 2; + if (output_size_.value().size() == static_cast(k + 2)) { + output_size_ = output_size_.value().slice(2); + } + if (output_size_.value().size() != static_cast(k)) { + TORCH_CHECK( + false, + "output_size must have ", + k, + " or ", + k + 2, + " elements (got ", + output_size_.value().size(), + ")"); + } + + std::vector min_sizes; + std::vector max_sizes; + for (const auto d : c10::irange(k)) { + int64_t dim_size = + ((input.sizes()[d + 2] - 1) * (*stride)[d] - 2 * (*padding)[d] + + (*kernel_size)[d]); + min_sizes.push_back(dim_size); + max_sizes.push_back(min_sizes[d] + (*stride)[d] - 1); + } + + for (const auto i : c10::irange(output_size_.value().size())) { + int64_t size = output_size_.value()[i]; + int64_t min_size = min_sizes[i]; + int64_t max_size = max_sizes[i]; + if (size < min_size || size > max_size) { + TORCH_CHECK( + false, + "requested an output size of ", + output_size_.value(), + ", but valid sizes range " + "from ", + min_sizes, + " to ", + max_sizes, + " (for an input of ", + input.sizes().slice(2), + ")"); + } + } + + for (const auto d : c10::irange(k)) { + ret.push_back(output_size_.value()[d] - min_sizes[d]); + } + } + return ret; +} + +ConvTranspose1dImpl::ConvTranspose1dImpl(ConvTranspose1dOptions options_) + : ConvTransposeNdImpl(detail::ConvNdOptions<1>( + /*in_channels=*/options_.in_channels(), + /*out_channels=*/options_.out_channels(), + /*kernel_size=*/options_.kernel_size()) + .stride(options_.stride()) + .padding(options_.padding()) + .dilation(options_.dilation()) + .transposed(true) + .output_padding(options_.output_padding()) + .groups(options_.groups()) + .bias(options_.bias()) + .padding_mode(options_.padding_mode())) {} + +Tensor ConvTranspose1dImpl::forward( + const Tensor& input, + const std::optional& output_size) { + if (!std::get_if(&options.padding_mode())) { + TORCH_CHECK( + false, "Only `zeros` padding mode is supported for ConvTranspose1d"); + } + + const auto& pad = padding(); + std::vector output_padding = _output_padding( + input, output_size, options.stride(), pad, options.kernel_size()); + + return F::detail::conv_transpose1d( + input, + weight, + bias, + options.stride(), + pad, + output_padding, + options.groups(), + options.dilation()); +} + +ConvTranspose2dImpl::ConvTranspose2dImpl(ConvTranspose2dOptions options_) + : ConvTransposeNdImpl(detail::ConvNdOptions<2>( + /*in_channels=*/options_.in_channels(), + /*out_channels=*/options_.out_channels(), + /*kernel_size=*/options_.kernel_size()) + .stride(options_.stride()) + .padding(options_.padding()) + .dilation(options_.dilation()) + .transposed(true) + .output_padding(options_.output_padding()) + .groups(options_.groups()) + .bias(options_.bias()) + .padding_mode(options_.padding_mode())) {} + +Tensor ConvTranspose2dImpl::forward( + const Tensor& input, + const std::optional& output_size) { + if (!std::get_if(&options.padding_mode())) { + TORCH_CHECK( + false, "Only `zeros` padding mode is supported for ConvTranspose2d"); + } + + const auto& pad = padding(); + std::vector output_padding = _output_padding( + input, output_size, options.stride(), pad, options.kernel_size()); + + return F::detail::conv_transpose2d( + input, + weight, + bias, + options.stride(), + pad, + output_padding, + options.groups(), + options.dilation()); +} + +ConvTranspose3dImpl::ConvTranspose3dImpl(ConvTranspose3dOptions options_) + : ConvTransposeNdImpl(detail::ConvNdOptions<3>( + /*in_channels=*/options_.in_channels(), + /*out_channels=*/options_.out_channels(), + /*kernel_size=*/options_.kernel_size()) + .stride(options_.stride()) + .padding(options_.padding()) + .dilation(options_.dilation()) + .transposed(true) + .output_padding(options_.output_padding()) + .groups(options_.groups()) + .bias(options_.bias()) + .padding_mode(options_.padding_mode())) {} + +Tensor ConvTranspose3dImpl::forward( + const Tensor& input, + const std::optional& output_size) { + if (!std::get_if(&options.padding_mode())) { + TORCH_CHECK( + false, "Only `zeros` padding mode is supported for ConvTranspose3d"); + } + + const auto& pad = padding(); + std::vector output_padding = _output_padding( + input, output_size, options.stride(), pad, options.kernel_size()); + + return F::detail::conv_transpose3d( + input, + weight, + bias, + options.stride(), + pad, + output_padding, + options.groups(), + options.dilation()); +} + +template class ConvTransposeNdImpl<1, ConvTranspose1dImpl>; +template class ConvTransposeNdImpl<2, ConvTranspose2dImpl>; +template class ConvTransposeNdImpl<3, ConvTranspose3dImpl>; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/distance.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/distance.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3376dcbd6ccceca52e963c5eb29d968bc81f6381 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/distance.cpp @@ -0,0 +1,41 @@ +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +CosineSimilarityImpl::CosineSimilarityImpl( + const CosineSimilarityOptions& options_) + : options(options_) {} + +void CosineSimilarityImpl::reset() {} + +void CosineSimilarityImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::CosineSimilarity" + << "(dim=" << options.dim() << ", eps=" << options.eps() << ')'; +} + +Tensor CosineSimilarityImpl::forward(const Tensor& x1, const Tensor& x2) { + return F::detail::cosine_similarity(x1, x2, options.dim(), options.eps()); +} + +// ============================================================================ + +PairwiseDistanceImpl::PairwiseDistanceImpl( + const PairwiseDistanceOptions& options_) + : options(options_) {} + +void PairwiseDistanceImpl::reset() {} + +void PairwiseDistanceImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::PairwiseDistance" + << "(p=" << options.p() << ", eps=" << options.eps() + << ", keepdim=" << options.keepdim() << ')'; +} + +Tensor PairwiseDistanceImpl::forward(const Tensor& x1, const Tensor& x2) { + return F::detail::pairwise_distance( + x1, x2, options.p(), options.eps(), options.keepdim()); +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/dropout.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/dropout.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3b9f92350bb549314cd2244d30a1d2e0203b2340 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/dropout.cpp @@ -0,0 +1,69 @@ +#include +#include + +#include +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +Tensor DropoutImpl::forward(Tensor input) { + return F::detail::dropout( + std::move(input), options.p(), is_training(), options.inplace()); +} + +void DropoutImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::Dropout(p=" << options.p() + << ", inplace=" << options.inplace() << ')'; +} + +// ============================================================================ + +Tensor Dropout2dImpl::forward(Tensor input) { + return F::detail::dropout2d( + std::move(input), options.p(), is_training(), options.inplace()); +} + +void Dropout2dImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::Dropout2d(p=" << options.p() + << ", inplace=" << options.inplace() << ')'; +} + +// ============================================================================ + +Tensor Dropout3dImpl::forward(Tensor input) { + return F::detail::dropout3d( + std::move(input), options.p(), is_training(), options.inplace()); +} + +void Dropout3dImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::Dropout3d(p=" << options.p() + << ", inplace=" << options.inplace() << ')'; +} + +// ============================================================================ + +Tensor AlphaDropoutImpl::forward(const Tensor& input) { + return F::detail::alpha_dropout( + input, options.p(), is_training(), /*inplace=*/false); +} + +void AlphaDropoutImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::AlphaDropout(p=" << options.p() + << ", inplace=" << options.inplace() << ')'; +} + +// ============================================================================ + +Tensor FeatureAlphaDropoutImpl::forward(const Tensor& input) { + return F::detail::feature_alpha_dropout( + input, options.p(), is_training(), /*inplace=*/false); +} + +void FeatureAlphaDropoutImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::FeatureAlphaDropout(p=" << options.p() + << ", inplace=" << options.inplace() << ')'; +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/embedding.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/embedding.cpp new file mode 100644 index 0000000000000000000000000000000000000000..17705a45670b74fc5c86d64151376bad86a57b35 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/embedding.cpp @@ -0,0 +1,185 @@ +#include + +#include +#include + +#include +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { +EmbeddingImpl::EmbeddingImpl(EmbeddingOptions options_) + : options(std::move(options_)) { + EmbeddingImpl::reset(); +} + +void EmbeddingImpl::reset() { + if (options.padding_idx().has_value()) { + if (options.padding_idx() > 0) { + TORCH_CHECK( + options.padding_idx() < options.num_embeddings(), + "Padding_idx must be within num_embeddings"); + } else if (options.padding_idx() < 0) { + TORCH_CHECK( + options.padding_idx() >= -options.num_embeddings(), + "Padding_idx must be within num_embedding"); + options.padding_idx( + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + options.num_embeddings() + *options.padding_idx()); + } + } + + if (!options._weight().defined()) { + weight = register_parameter( + "weight", + torch::empty({options.num_embeddings(), options.embedding_dim()})); + reset_parameters(); + } else { + TORCH_CHECK( + options._weight().sizes() == + torch::IntArrayRef( + {options.num_embeddings(), options.embedding_dim()}), + "Shape of _weight does not match num_embeddings and embedding_dim"); + weight = register_parameter("weight", options._weight()); + } +} + +void EmbeddingImpl::reset_parameters() { + torch::nn::init::normal_(weight); + if (options.padding_idx().has_value()) { + torch::NoGradGuard no_grad; + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + weight[*options.padding_idx()].fill_(0); + } +} + +void EmbeddingImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Embedding(num_embeddings=" << options.num_embeddings() + << ", embedding_dim=" << options.embedding_dim(); + auto const& padding_idx_opt = options.padding_idx(); + if (padding_idx_opt.has_value()) { + stream << ", padding_idx=" << padding_idx_opt.value(); + } + auto const& max_norm_opt = options.max_norm(); + if (max_norm_opt.has_value()) { + stream << ", max_norm=" << max_norm_opt.value(); + } + if (options.norm_type() != 2) { + stream << ", norm_type=" << options.norm_type(); + } + if (options.scale_grad_by_freq()) { + stream << ", scale_grad_by_freq=" << std::boolalpha + << options.scale_grad_by_freq(); + } + if (options.sparse()) { + stream << ", sparse=" << std::boolalpha << options.sparse(); + } + stream << ')'; +} + +torch::Tensor EmbeddingImpl::forward(const Tensor& input) { + return F::detail::embedding( + input, + weight, + options.padding_idx(), + options.max_norm(), + options.norm_type(), + options.scale_grad_by_freq(), + options.sparse()); +} + +EmbeddingBagImpl::EmbeddingBagImpl(EmbeddingBagOptions options_) + : options(std::move(options_)) { + EmbeddingBagImpl::reset(); +} + +void EmbeddingBagImpl::reset() { + auto const& padding_idx_opt = options.padding_idx(); + if (padding_idx_opt.has_value()) { + auto padding_idx = padding_idx_opt.value(); + if (padding_idx > 0) { + TORCH_CHECK( + padding_idx < options.num_embeddings(), + "Padding_idx must be within num_embeddings"); + } else if (padding_idx < 0) { + TORCH_CHECK( + padding_idx >= -options.num_embeddings(), + "Padding_idx must be within num_embedding"); + options.padding_idx(options.num_embeddings() + padding_idx); + } + } + if (!options._weight().defined()) { + weight = register_parameter( + "weight", + torch::empty({options.num_embeddings(), options.embedding_dim()})); + reset_parameters(); + } else { + TORCH_CHECK( + options._weight().sizes() == + torch::IntArrayRef( + {options.num_embeddings(), options.embedding_dim()}), + "Shape of weight does not match num_embeddings and embedding_dim"); + weight = register_parameter("weight", options._weight()); + } +} + +void EmbeddingBagImpl::reset_parameters() { + auto const& padding_idx_opt = options.padding_idx(); + if (padding_idx_opt.has_value()) { + torch::NoGradGuard no_grad; + weight[*padding_idx_opt].fill_(0); + } + torch::nn::init::normal_(weight); +} + +torch::Tensor EmbeddingBagImpl::forward( + const Tensor& input, + const Tensor& offsets, + const Tensor& per_sample_weights) { + return F::detail::embedding_bag( + input, + weight, + offsets, + options.max_norm(), + options.norm_type(), + options.scale_grad_by_freq(), + options.mode(), + options.sparse(), + per_sample_weights, + options.include_last_offset(), + options.padding_idx()); +} + +void EmbeddingBagImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::EmbeddingBag(num_embeddings=" + << options.num_embeddings() + << ", embedding_dim=" << options.embedding_dim(); + auto const& max_norm_opt = options.max_norm(); + if (max_norm_opt.has_value()) { + stream << ", max_norm=" << *max_norm_opt; + } + if (options.norm_type() != 2) { + stream << ", norm_type=" << options.norm_type(); + } + if (options.scale_grad_by_freq()) { + stream << ", scale_grad_by_freq=" << std::boolalpha + << options.scale_grad_by_freq(); + } + if (options.sparse()) { + stream << ", sparse=" << std::boolalpha << options.sparse(); + } + if (!std::get_if(&options.mode())) { + stream << ", mode=" << torch::enumtype::get_enum_name(options.mode()); + } + if (options.include_last_offset()) { + stream << ", include_last_offset=" << std::boolalpha + << options.include_last_offset(); + } + auto const& padding_idx_opt = options.padding_idx(); + if (padding_idx_opt.has_value()) { + stream << ", padding_idx=" << padding_idx_opt.value(); + } + stream << ')'; +} +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/fold.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/fold.cpp new file mode 100644 index 0000000000000000000000000000000000000000..26f1f39a9c4f3d90a59d9de107f826a9808a862e --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/fold.cpp @@ -0,0 +1,53 @@ +#include + +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +FoldImpl::FoldImpl(const FoldOptions& options_) : options(options_) {} + +void FoldImpl::reset() {} + +void FoldImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Fold(output_size=" << options.output_size() + << ", kernel_size=" << options.kernel_size() + << ", dilation=" << options.dilation() + << ", padding=" << options.padding() << ", stride=" << options.stride() + << ')'; +} + +Tensor FoldImpl::forward(const Tensor& input) { + return F::detail::fold( + input, + options.output_size(), + options.kernel_size(), + options.dilation(), + options.padding(), + options.stride()); +} + +// ============================================================================ + +UnfoldImpl::UnfoldImpl(const UnfoldOptions& options_) : options(options_) {} + +void UnfoldImpl::reset() {} + +void UnfoldImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Unfold(kernel_size=" << options.kernel_size() + << ", dilation=" << options.dilation() + << ", padding=" << options.padding() << ", stride=" << options.stride() + << ')'; +} + +Tensor UnfoldImpl::forward(const Tensor& input) { + return F::detail::unfold( + input, + options.kernel_size(), + options.dilation(), + options.padding(), + options.stride()); +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/instancenorm.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/instancenorm.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f83904b7e26501f2483142e6e660ed8232e6b83e --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/instancenorm.cpp @@ -0,0 +1,32 @@ +#include +#include + +namespace torch::nn { + +void InstanceNorm1dImpl::_check_input_dim(const Tensor& input) { + if (input.dim() != 3 && input.dim() != 2) { + TORCH_CHECK( + false, "expected 2D or 3D input (got ", input.dim(), "D input)"); + } +} + +void InstanceNorm2dImpl::_check_input_dim(const Tensor& input) { + if (input.dim() != 4 && input.dim() != 3) { + TORCH_CHECK( + false, "expected 3D or 4D input (got ", input.dim(), "D input)"); + } +} + +void InstanceNorm3dImpl::_check_input_dim(const Tensor& input) { + if (input.dim() != 5 && + input.dim() != 4) { // NOLINT(cppcoreguidelines-avoid-magic-numbers) + TORCH_CHECK( + false, "expected 4D or 5D input (got ", input.dim(), "D input)"); + } +} + +template class InstanceNormImpl<1, InstanceNorm1dImpl>; +template class InstanceNormImpl<2, InstanceNorm2dImpl>; +template class InstanceNormImpl<3, InstanceNorm3dImpl>; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/linear.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/linear.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a47e98ac7ff5683b0c74ac2ff82d5b25e88fd2f0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/linear.cpp @@ -0,0 +1,143 @@ +#include +#include +#include + +#include +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +void IdentityImpl::reset() {} + +void IdentityImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Identity()"; +} + +Tensor IdentityImpl::forward(const Tensor& input) { + return input; +} + +// ============================================================================ + +LinearImpl::LinearImpl(const LinearOptions& options_) : options(options_) { + LinearImpl::reset(); +} + +void LinearImpl::reset() { + weight = register_parameter( + "weight", torch::empty({options.out_features(), options.in_features()})); + if (options.bias()) { + bias = register_parameter("bias", torch::empty(options.out_features())); + } else { + bias = register_parameter("bias", {}, /*requires_grad=*/false); + } + + reset_parameters(); +} + +void LinearImpl::reset_parameters() { + torch::nn::init::kaiming_uniform_( + weight, std::sqrt(5)); // NOLINT(cppcoreguidelines-avoid-magic-numbers) + if (bias.defined()) { + auto [fan_in, fan_out] = + torch::nn::init::_calculate_fan_in_and_fan_out(weight); + const auto bound = 1 / std::sqrt(fan_in); + torch::nn::init::uniform_(bias, -bound, bound); + } +} + +void LinearImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha + << "torch::nn::Linear(in_features=" << options.in_features() + << ", out_features=" << options.out_features() + << ", bias=" << options.bias() << ')'; +} + +Tensor LinearImpl::forward(const Tensor& input) { + return F::linear(input, weight, bias); +} + +// ============================================================================ + +FlattenImpl::FlattenImpl(const FlattenOptions& options_) : options(options_) {} + +void FlattenImpl::reset() {} + +void FlattenImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Flatten(start_dim=" << options.start_dim() + << ", end_dim=" << options.end_dim() << ')'; +} + +Tensor FlattenImpl::forward(const Tensor& input) { + return input.flatten(options.start_dim(), options.end_dim()); +} + +// ============================================================================ + +UnflattenImpl::UnflattenImpl(UnflattenOptions options_) + : options(std::move(options_)) {} + +void UnflattenImpl::reset() {} + +void UnflattenImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Unflatten(dim=" << options.dim() + << ", unflattened_size={"; + auto sizes = options.sizes(); + TORCH_CHECK(!sizes.empty(), "Unflatten requires non-empty sizes"); + size_t i = 0; + for (; i < sizes.size() - 1; ++i) { + stream << sizes[i] << ", "; + } + stream << sizes[i] << "})"; +} + +Tensor UnflattenImpl::forward(const Tensor& input) { + return input.unflatten(options.dim(), options.sizes()); +} + +// ============================================================================ + +BilinearImpl::BilinearImpl(const BilinearOptions& options_) + : options(options_) { + BilinearImpl::reset(); +} + +void BilinearImpl::reset() { + weight = register_parameter( + "weight", + torch::empty( + {options.out_features(), + options.in1_features(), + options.in2_features()})); + if (options.bias()) { + bias = register_parameter("bias", torch::empty(options.out_features())); + } else { + bias = register_parameter("bias", torch::Tensor(), /*requires_grad=*/false); + } + + reset_parameters(); +} + +void BilinearImpl::reset_parameters() { + const auto bound = 1.0 / std::sqrt(weight.size(1)); + init::uniform_(weight, -bound, bound); + if (bias.defined()) { + init::uniform_(bias, -bound, bound); + } +} + +void BilinearImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha + << "torch::nn::Bilinear(in1_features=" << options.in1_features() + << ", in2_features=" << options.in2_features() + << ", out_features=" << options.out_features() + << ", bias=" << options.bias() << ')'; +} + +Tensor BilinearImpl::forward(const Tensor& input1, const Tensor& input2) { + return F::bilinear(input1, input2, weight, bias); +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/loss.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/loss.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6052900631f843a92bc979189343fe5dc3374989 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/loss.cpp @@ -0,0 +1,426 @@ +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +L1LossImpl::L1LossImpl(L1LossOptions options_) : options(options_) {} + +void L1LossImpl::reset() {} + +void L1LossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::L1Loss()"; +} + +Tensor L1LossImpl::forward(const Tensor& input, const Tensor& target) { + return F::detail::l1_loss(input, target, options.reduction()); +} + +// ============================================================================ + +KLDivLossImpl::KLDivLossImpl(KLDivLossOptions options_) : options(options_) {} + +void KLDivLossImpl::reset() {} + +void KLDivLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::KLDivLoss()"; +} + +Tensor KLDivLossImpl::forward(const Tensor& input, const Tensor& target) { + return F::detail::kl_div( + input, target, options.reduction(), options.log_target()); +} + +// ============================================================================ + +MSELossImpl::MSELossImpl(MSELossOptions options_) : options(options_) {} + +void MSELossImpl::reset() {} + +void MSELossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::MSELoss()"; +} + +Tensor MSELossImpl::forward(const Tensor& input, const Tensor& target) { + return F::detail::mse_loss(input, target, options.reduction()); +} + +// ============================================================================ + +BCELossImpl::BCELossImpl(BCELossOptions options_) + : options(std::move(options_)) { + BCELossImpl::reset(); +} + +void BCELossImpl::reset() { + register_buffer("weight", options.weight()); +} + +void BCELossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::BCELoss()"; +} + +Tensor BCELossImpl::forward(const Tensor& input, const Tensor& target) { + return F::detail::binary_cross_entropy( + input, target, options.weight(), options.reduction()); +} + +// ============================================================================ + +HingeEmbeddingLossImpl::HingeEmbeddingLossImpl( + HingeEmbeddingLossOptions options_) + : options(options_) {} + +void HingeEmbeddingLossImpl::reset() {} + +void HingeEmbeddingLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::HingeEmbeddingLoss(margin=" << options.margin() << ')'; +} + +Tensor HingeEmbeddingLossImpl::forward( + const Tensor& input, + const Tensor& target) { + return F::detail::hinge_embedding_loss( + input, target, options.margin(), options.reduction()); +} + +// ============================================================================ + +MultiMarginLossImpl::MultiMarginLossImpl(MultiMarginLossOptions options_) + : options(std::move(options_)) { + MultiMarginLossImpl::reset(); +} + +void MultiMarginLossImpl::reset() { + TORCH_CHECK( + (options.p() == 1) || (options.p() == 2), + "only p == 1 and p == 2 supported"); + TORCH_CHECK(!options.weight().defined() || options.weight().dim() == 1); + + register_buffer("weight", options.weight()); +} + +void MultiMarginLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::MultiMarginLoss(p=" << options.p() + << ", margin=" << options.margin() << ", weight=" << options.weight() + << ", reduction=" << enumtype::get_enum_name(options.reduction()) + << ')'; +} + +Tensor MultiMarginLossImpl::forward(const Tensor& input, const Tensor& target) { + return F::detail::multi_margin_loss( + input, + target, + options.p(), + options.margin(), + options.weight(), + options.reduction()); +} + +// ============================================================================ + +CosineEmbeddingLossImpl::CosineEmbeddingLossImpl( + CosineEmbeddingLossOptions options_) + : options(options_) {} + +void CosineEmbeddingLossImpl::reset() {} + +void CosineEmbeddingLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::CosineEmbeddingLoss(margin=" << options.margin() << ')'; +} + +Tensor CosineEmbeddingLossImpl::forward( + const Tensor& input1, + const Tensor& input2, + const Tensor& target) { + return F::detail::cosine_embedding_loss( + input1, input2, target, options.margin(), options.reduction()); +} +// ============================================================================ + +MultiLabelSoftMarginLossImpl::MultiLabelSoftMarginLossImpl( + torch::nn::MultiLabelSoftMarginLossOptions options_) + : options(std::move(options_)) { + MultiLabelSoftMarginLossImpl::reset(); +} + +void MultiLabelSoftMarginLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::MultiLabelSoftMarginLoss()"; +} + +void MultiLabelSoftMarginLossImpl::reset() { + register_buffer("weight", options.weight()); +} + +Tensor MultiLabelSoftMarginLossImpl::forward( + const Tensor& input, + const Tensor& target) { + return F::detail::multilabel_soft_margin_loss( + input, target, options.weight(), options.reduction()); +} + +// ============================================================================ + +TripletMarginLossImpl::TripletMarginLossImpl(TripletMarginLossOptions options_) + : options(options_) {} + +void TripletMarginLossImpl::reset() {} + +void TripletMarginLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::TripletMarginLoss(margin=" << options.margin() + << ", p=" << options.p() << ", eps=" << options.eps() << std::boolalpha + << ", swap=" << options.swap() << ')'; +} + +Tensor TripletMarginLossImpl::forward( + const Tensor& anchor, + const Tensor& positive, + const Tensor& negative) { + return F::detail::triplet_margin_loss( + anchor, + positive, + negative, + options.margin(), + options.p(), + options.eps(), + options.swap(), + options.reduction()); +} + +// ============================================================================ + +TripletMarginWithDistanceLossImpl::TripletMarginWithDistanceLossImpl( + TripletMarginWithDistanceLossOptions options_) + : options(std::move(options_)) {} + +void TripletMarginWithDistanceLossImpl::reset() {} + +void TripletMarginWithDistanceLossImpl::pretty_print( + std::ostream& stream) const { + stream << "torch::nn::TripletMarginWithDistanceLoss(margin=" + << options.margin() << std::boolalpha << ", swap=" << options.swap() + << ')'; +} + +Tensor TripletMarginWithDistanceLossImpl::forward( + const Tensor& anchor, + const Tensor& positive, + const Tensor& negative) { + return F::detail::triplet_margin_with_distance_loss( + anchor, + positive, + negative, + options.distance_function(), + options.margin(), + options.swap(), + options.reduction()); +} + +// ============================================================================ + +MultiLabelMarginLossImpl::MultiLabelMarginLossImpl( + torch::nn::MultiLabelMarginLossOptions options_) + : options(options_) {} + +void MultiLabelMarginLossImpl::reset() {} + +void MultiLabelMarginLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::MultiLabelMarginLoss()"; +} + +Tensor MultiLabelMarginLossImpl::forward( + const Tensor& input, + const Tensor& target) { + return F::detail::multilabel_margin_loss(input, target, options.reduction()); +} + +// ============================================================================ + +SoftMarginLossImpl::SoftMarginLossImpl( + torch::nn::SoftMarginLossOptions options_) + : options(options_) {} + +void SoftMarginLossImpl::reset() {} + +void SoftMarginLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::SoftMarginLoss()"; +} + +Tensor SoftMarginLossImpl::forward(const Tensor& input, const Tensor& target) { + return F::detail::soft_margin_loss(input, target, options.reduction()); +} + +// ============================================================================ + +SmoothL1LossImpl::SmoothL1LossImpl(torch::nn::SmoothL1LossOptions options_) + : options(options_) {} + +void SmoothL1LossImpl::reset() {} + +void SmoothL1LossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::SmoothL1Loss"; +} + +Tensor SmoothL1LossImpl::forward(const Tensor& input, const Tensor& target) { + return F::detail::smooth_l1_loss( + input, target, options.reduction(), options.beta()); +} + +// ============================================================================ + +HuberLossImpl::HuberLossImpl(torch::nn::HuberLossOptions options_) + : options(options_) {} + +void HuberLossImpl::reset() {} + +void HuberLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::HuberLoss"; +} + +Tensor HuberLossImpl::forward(const Tensor& input, const Tensor& target) { + return F::detail::huber_loss( + input, target, options.reduction(), options.delta()); +} + +// ============================================================================ + +CTCLossImpl::CTCLossImpl(CTCLossOptions options_) : options(options_) {} + +void CTCLossImpl::reset() {} + +void CTCLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::CTCLoss()"; +} + +Tensor CTCLossImpl::forward( + const Tensor& log_probs, + const Tensor& targets, + const Tensor& input_lengths, + const Tensor& target_lengths) { + return F::detail::ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + options.blank(), + options.reduction(), + options.zero_infinity()); +} + +// ============================================================================ + +PoissonNLLLossImpl::PoissonNLLLossImpl(PoissonNLLLossOptions options_) + : options(options_) {} + +void PoissonNLLLossImpl::reset() {} + +void PoissonNLLLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::PoissonNLLLoss()"; +} + +Tensor PoissonNLLLossImpl::forward( + const Tensor& log_input, + const Tensor& target) { + return F::detail::poisson_nll_loss( + log_input, + target, + options.log_input(), + options.full(), + options.eps(), + options.reduction()); +} + +// ============================================================================ + +MarginRankingLossImpl::MarginRankingLossImpl(MarginRankingLossOptions options_) + : options(options_) {} + +void MarginRankingLossImpl::reset() {} + +void MarginRankingLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::MarginRankingLoss()"; +} + +Tensor MarginRankingLossImpl::forward( + const Tensor& input1, + const Tensor& input2, + const Tensor& target) { + return F::detail::margin_ranking_loss( + input1, input2, target, options.margin(), options.reduction()); +} + +// ============================================================================ + +NLLLossImpl::NLLLossImpl(NLLLossOptions options_) + : options(std::move(options_)) { + NLLLossImpl::reset(); +} + +void NLLLossImpl::reset() { + weight = register_buffer("weight", options.weight()); +} + +void NLLLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::NLLLoss()"; +} + +Tensor NLLLossImpl::forward(const Tensor& input, const Tensor& target) { + return F::detail::nll_loss( + input, target, weight, options.ignore_index(), options.reduction()); +} + +// ============================================================================ + +CrossEntropyLossImpl::CrossEntropyLossImpl(CrossEntropyLossOptions options_) + : options(std::move(options_)) { + CrossEntropyLossImpl::reset(); +} + +void CrossEntropyLossImpl::reset() { + weight = register_buffer("weight", options.weight()); +} + +void CrossEntropyLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::CrossEntropyLoss()"; +} + +Tensor CrossEntropyLossImpl::forward( + const Tensor& input, + const Tensor& target) { + return F::detail::cross_entropy( + input, + target, + weight, + options.ignore_index(), + options.reduction(), + options.label_smoothing()); +} + +// ============================================================================ + +BCEWithLogitsLossImpl::BCEWithLogitsLossImpl(BCEWithLogitsLossOptions options_) + : options(std::move(options_)) { + BCEWithLogitsLossImpl::reset(); +} + +void BCEWithLogitsLossImpl::reset() { + weight = register_buffer("weight", options.weight()); + pos_weight = register_buffer("pos_weight", options.pos_weight()); +} + +void BCEWithLogitsLossImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::BCEWithLogitsLoss()"; +} + +Tensor BCEWithLogitsLossImpl::forward( + const Tensor& input, + const Tensor& target) { + return F::detail::binary_cross_entropy_with_logits( + input, + target, + options.weight(), + options.reduction(), + options.pos_weight()); +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/normalization.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/normalization.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8587ee23ee81333c9464574ced3bd882d0821928 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/normalization.cpp @@ -0,0 +1,119 @@ +#include + +#include + +#include +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +LayerNormImpl::LayerNormImpl(LayerNormOptions options_) + : options(std::move(options_)) { + LayerNormImpl::reset(); +} + +void LayerNormImpl::reset() { + if (options.elementwise_affine()) { + weight = + register_parameter("weight", torch::empty(options.normalized_shape())); + bias = register_parameter("bias", torch::empty(options.normalized_shape())); + } else { + weight = + register_parameter("weight", torch::Tensor(), /*requires_grad=*/false); + bias = register_parameter("bias", torch::Tensor(), /*requires_grad=*/false); + } + reset_parameters(); +} + +void LayerNormImpl::reset_parameters() { + if (options.elementwise_affine()) { + torch::nn::init::ones_(weight); + torch::nn::init::zeros_(bias); + } +} + +void LayerNormImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::LayerNorm(" + << torch::IntArrayRef(options.normalized_shape()) + << ", eps=" << options.eps() + << ", elementwise_affine=" << options.elementwise_affine() << ')'; +} + +torch::Tensor LayerNormImpl::forward(const Tensor& input) { + return F::detail::layer_norm( + input, options.normalized_shape(), weight, bias, options.eps()); +} + +// ============================================================================ + +LocalResponseNormImpl::LocalResponseNormImpl( + const LocalResponseNormOptions& options_) + : options(options_) {} + +Tensor LocalResponseNormImpl::forward(const Tensor& input) { + return F::detail::local_response_norm( + input, options.size(), options.alpha(), options.beta(), options.k()); +} + +void LocalResponseNormImpl::reset() {} + +void LocalResponseNormImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::LocalResponseNorm(" << options.size() + << ", alpha=" << options.alpha() << ", beta=" << options.beta() + << ", k=" << options.k() << ')'; +} + +// ============================================================================ + +void CrossMapLRN2dImpl::reset() {} + +void CrossMapLRN2dImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::CrossMapLRN2d(" << options.size() + << ", alpha=" << options.alpha() << ", beta=" << options.beta() + << ", k=" << options.k() << ')'; +} + +torch::Tensor CrossMapLRN2dImpl::forward(const torch::Tensor& input) { + return functions::CrossMapLRN2d::apply(input, options); +} + +// ============================================================================ + +GroupNormImpl::GroupNormImpl(const GroupNormOptions& options_) + : options(options_) { // NOLINT(modernize-pass-by-value) + GroupNormImpl::reset(); +} + +void GroupNormImpl::reset() { + if (options.affine()) { + weight = register_parameter("weight", torch::empty(options.num_channels())); + bias = register_parameter("bias", torch::empty(options.num_channels())); + } else { + weight = + register_parameter("weight", torch::Tensor(), /*requires_grad=*/false); + bias = register_parameter("bias", torch::Tensor(), /*requires_grad=*/false); + } + reset_parameters(); +} + +void GroupNormImpl::reset_parameters() { + if (options.affine()) { + torch::nn::init::ones_(weight); + torch::nn::init::zeros_(bias); + } +} + +torch::Tensor GroupNormImpl::forward(const Tensor& input) { + return F::detail::group_norm( + input, options.num_groups(), weight, bias, options.eps()); +} + +void GroupNormImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::GroupNorm(" << options.num_groups() + << ", " << options.num_channels() << ", eps=" << options.eps() + << ", affine=" << options.affine() << ')'; +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/padding.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/padding.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6cb35e614dfb86dabefcac1abab2b3ffc92f4fa8 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/padding.cpp @@ -0,0 +1,106 @@ +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +template +ReflectionPadImpl::ReflectionPadImpl( + const ReflectionPadOptions& options_) + : options(options_) {} + +template +void ReflectionPadImpl::reset() {} + +template +Tensor ReflectionPadImpl::forward(const Tensor& input) { + return F::detail::pad(input, options.padding(), torch::kReflect, 0); +} + +template +void ReflectionPadImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::ReflectionPad" << D << 'd' + << "(padding=" << options.padding() << ')'; +} + +template class ReflectionPadImpl<1, ReflectionPad1dImpl>; +template class ReflectionPadImpl<2, ReflectionPad2dImpl>; +template class ReflectionPadImpl<3, ReflectionPad3dImpl>; + +// ============================================================================ + +template +ReplicationPadImpl::ReplicationPadImpl( + const ReplicationPadOptions& options_) + : options(options_) {} + +template +void ReplicationPadImpl::reset() {} + +template +Tensor ReplicationPadImpl::forward(const Tensor& input) { + return F::detail::pad(input, options.padding(), torch::kReplicate, 0); +} + +template +void ReplicationPadImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::ReplicationPad" << D << 'd' + << "(padding=" << options.padding() << ')'; +} + +template class ReplicationPadImpl<1, ReplicationPad1dImpl>; +template class ReplicationPadImpl<2, ReplicationPad2dImpl>; +template class ReplicationPadImpl<3, ReplicationPad3dImpl>; + +// ============================================================================ + +template +ZeroPadImpl::ZeroPadImpl(const ZeroPadOptions& options_) + : options(options_) {} + +template +void ZeroPadImpl::reset() {} + +template +Tensor ZeroPadImpl::forward(const Tensor& input) { + return F::detail::pad(input, options.padding(), torch::kConstant, 0); +} + +template +void ZeroPadImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::ZeroPad" << D << 'd' << "(padding=" << options.padding() + << ')'; +} + +template class ZeroPadImpl<1, ZeroPad1dImpl>; +template class ZeroPadImpl<2, ZeroPad2dImpl>; +template class ZeroPadImpl<3, ZeroPad3dImpl>; + +// ============================================================================ + +template +ConstantPadImpl::ConstantPadImpl( + const ConstantPadOptions& options_) + : options(options_) {} + +template +void ConstantPadImpl::reset() {} + +template +Tensor ConstantPadImpl::forward(const Tensor& input) { + return F::detail::pad( + input, options.padding(), torch::kConstant, options.value()); +} + +template +void ConstantPadImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::ConstantPad" << D << 'd' + << "(padding=" << options.padding() << ", value=" << options.value() + << ')'; +} + +template class ConstantPadImpl<1, ConstantPad1dImpl>; +template class ConstantPadImpl<2, ConstantPad2dImpl>; +template class ConstantPadImpl<3, ConstantPad3dImpl>; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/pixelshuffle.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/pixelshuffle.cpp new file mode 100644 index 0000000000000000000000000000000000000000..35597f96dcfe7e4cd2b81d79f3a1b534c8c1b2b6 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/pixelshuffle.cpp @@ -0,0 +1,35 @@ +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +PixelShuffleImpl::PixelShuffleImpl(const PixelShuffleOptions& options_) + : options(options_) {} + +void PixelShuffleImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::PixelShuffle(upscale_factor=" + << options.upscale_factor() << ')'; +} + +void PixelShuffleImpl::reset() {} + +Tensor PixelShuffleImpl::forward(const Tensor& input) { + return F::detail::pixel_shuffle(input, options.upscale_factor()); +} + +PixelUnshuffleImpl::PixelUnshuffleImpl(const PixelUnshuffleOptions& options_) + : options(options_) {} + +void PixelUnshuffleImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::PixelUnshuffle(downscale_factor=" + << options.downscale_factor() << ')'; +} + +void PixelUnshuffleImpl::reset() {} + +Tensor PixelUnshuffleImpl::forward(const Tensor& input) { + return F::detail::pixel_unshuffle(input, options.downscale_factor()); +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/pooling.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/pooling.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e5ed51d05f01696421ccc541b0a2985ea07f8f2d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/pooling.cpp @@ -0,0 +1,440 @@ +#include + +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +template +AvgPoolImpl::AvgPoolImpl(const AvgPoolOptions& options_) + : options(options_) {} + +template +void AvgPoolImpl::reset() {} + +template +void AvgPoolImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::AvgPool" << D << 'd' + << "(kernel_size=" << options.kernel_size() + << ", stride=" << options.stride() << ", padding=" << options.padding() + << ')'; +} + +Tensor AvgPool1dImpl::forward(const Tensor& input) { + return F::detail::avg_pool1d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.ceil_mode(), + options.count_include_pad()); +} + +Tensor AvgPool2dImpl::forward(const Tensor& input) { + return F::detail::avg_pool2d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.ceil_mode(), + options.count_include_pad(), + options.divisor_override()); +} + +Tensor AvgPool3dImpl::forward(const Tensor& input) { + return F::detail::avg_pool3d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.ceil_mode(), + options.count_include_pad(), + options.divisor_override()); +} + +template class AvgPoolImpl<1, AvgPool1dImpl>; +template class AvgPoolImpl<2, AvgPool2dImpl>; +template class AvgPoolImpl<3, AvgPool3dImpl>; + +// ============================================================================ + +template +MaxPoolImpl::MaxPoolImpl(const MaxPoolOptions& options_) + : options(options_) {} + +template +void MaxPoolImpl::reset() {} + +template +void MaxPoolImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::MaxPool" << D << 'd' + << "(kernel_size=" << options.kernel_size() + << ", stride=" << options.stride() << ", padding=" << options.padding() + << ", dilation=" << options.dilation() + << ", ceil_mode=" << options.ceil_mode() << ')'; +} + +Tensor MaxPool1dImpl::forward(const Tensor& input) { + return F::detail::max_pool1d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +std::tuple MaxPool1dImpl::forward_with_indices( + const Tensor& input) { + return F::detail::max_pool1d_with_indices( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +Tensor MaxPool2dImpl::forward(const Tensor& input) { + return F::detail::max_pool2d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +std::tuple MaxPool2dImpl::forward_with_indices( + const Tensor& input) { + return F::detail::max_pool2d_with_indices( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +Tensor MaxPool3dImpl::forward(const Tensor& input) { + return F::detail::max_pool3d( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +std::tuple MaxPool3dImpl::forward_with_indices( + const Tensor& input) { + return F::detail::max_pool3d_with_indices( + input, + options.kernel_size(), + options.stride(), + options.padding(), + options.dilation(), + options.ceil_mode()); +} + +template class MaxPoolImpl<1, MaxPool1dImpl>; +template class MaxPoolImpl<2, MaxPool2dImpl>; +template class MaxPoolImpl<3, MaxPool3dImpl>; + +// ============================================================================ + +Tensor AdaptiveMaxPool1dImpl::forward(const Tensor& input) { + return F::detail::adaptive_max_pool1d(input, options.output_size()); +} + +std::tuple AdaptiveMaxPool1dImpl::forward_with_indices( + const Tensor& input) { + return F::detail::adaptive_max_pool1d_with_indices( + input, options.output_size()); +} + +Tensor AdaptiveMaxPool2dImpl::forward(const Tensor& input) { + return F::detail::adaptive_max_pool2d(input, options.output_size()); +} + +std::tuple AdaptiveMaxPool2dImpl::forward_with_indices( + const Tensor& input) { + return F::detail::adaptive_max_pool2d_with_indices( + input, options.output_size()); +} + +Tensor AdaptiveMaxPool3dImpl::forward(const Tensor& input) { + return F::detail::adaptive_max_pool3d(input, options.output_size()); +} + +std::tuple AdaptiveMaxPool3dImpl::forward_with_indices( + const Tensor& input) { + return F::detail::adaptive_max_pool3d_with_indices( + input, options.output_size()); +} + +template class AdaptiveMaxPoolImpl<1, ExpandingArray<1>, AdaptiveMaxPool1dImpl>; +template class AdaptiveMaxPoolImpl< + 2, + ExpandingArrayWithOptionalElem<2>, + AdaptiveMaxPool2dImpl>; +template class AdaptiveMaxPoolImpl< + 3, + ExpandingArrayWithOptionalElem<3>, + AdaptiveMaxPool3dImpl>; + +// ============================================================================ + +Tensor AdaptiveAvgPool1dImpl::forward(const Tensor& input) { + return F::detail::adaptive_avg_pool1d(input, options.output_size()); +} + +Tensor AdaptiveAvgPool2dImpl::forward(const Tensor& input) { + return F::detail::adaptive_avg_pool2d(input, options.output_size()); +} + +Tensor AdaptiveAvgPool3dImpl::forward(const Tensor& input) { + return F::detail::adaptive_avg_pool3d(input, options.output_size()); +} + +template class AdaptiveAvgPoolImpl<1, ExpandingArray<1>, AdaptiveAvgPool1dImpl>; +template class AdaptiveAvgPoolImpl< + 2, + ExpandingArrayWithOptionalElem<2>, + AdaptiveAvgPool2dImpl>; +template class AdaptiveAvgPoolImpl< + 3, + ExpandingArrayWithOptionalElem<3>, + AdaptiveAvgPool3dImpl>; + +// ============================================================================ + +template +MaxUnpoolImpl::MaxUnpoolImpl(const MaxUnpoolOptions& options_) + : options(options_) {} + +template +void MaxUnpoolImpl::reset() {} + +template +void MaxUnpoolImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::MaxUnpool" << D << 'd' + << "(kernel_size=" << options.kernel_size() + << ", stride=" << options.stride() << ", padding=" << options.padding() + << ')'; +} + +Tensor MaxUnpool1dImpl::forward( + const Tensor& input, + const Tensor& indices, + const std::optional>& output_size) { + return F::detail::max_unpool1d( + input, + indices, + options.kernel_size(), + options.stride(), + options.padding(), + output_size); +} + +Tensor MaxUnpool2dImpl::forward( + const Tensor& input, + const Tensor& indices, + const std::optional>& output_size) { + return F::detail::max_unpool2d( + input, + indices, + options.kernel_size(), + options.stride(), + options.padding(), + output_size); +} + +Tensor MaxUnpool3dImpl::forward( + const Tensor& input, + const Tensor& indices, + const std::optional>& output_size) { + return F::detail::max_unpool3d( + input, + indices, + options.kernel_size(), + options.stride(), + options.padding(), + output_size); +} + +template class MaxUnpoolImpl<1, MaxUnpool1dImpl>; +template class MaxUnpoolImpl<2, MaxUnpool2dImpl>; +template class MaxUnpoolImpl<3, MaxUnpool3dImpl>; + +// ============================================================================ + +FractionalMaxPool2dImpl::FractionalMaxPool2dImpl( + FractionalMaxPool2dOptions options_) + : options(std::move(options_)) { + FractionalMaxPool2dImpl::reset(); +} + +void FractionalMaxPool2dImpl::reset() { + _random_samples = + register_buffer("_random_samples", options._random_samples()); + if (options.output_size() == std::nullopt && + options.output_ratio() == std::nullopt) { + TORCH_CHECK( + false, + "FractionalMaxPool2d requires specifying either ", + "an output size, or a pooling ratio"); + } + if (options.output_size().has_value() && options.output_ratio().has_value()) { + TORCH_CHECK( + false, "only one of output_size and output_ratio may be specified"); + } + if (options.output_ratio().has_value()) { + at::ArrayRef output_ratio = + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + at::ArrayRef(options.output_ratio().value()); + if (!(0 < output_ratio[0] && output_ratio[0] < 1 && 0 < output_ratio[1] && + output_ratio[1] < 1)) { + TORCH_CHECK( + false, + "output_ratio must be between 0 and 1 (got ", + output_ratio, + ")"); + } + } +} + +Tensor FractionalMaxPool2dImpl::forward(const Tensor& input) { + return F::detail::fractional_max_pool2d( + input, + options.kernel_size(), + options.output_size(), + options.output_ratio(), + _random_samples); +} + +std::tuple FractionalMaxPool2dImpl::forward_with_indices( + const Tensor& input) { + return F::detail::fractional_max_pool2d_with_indices( + input, + options.kernel_size(), + options.output_size(), + options.output_ratio(), + _random_samples); +} + +void FractionalMaxPool2dImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::FractionalMaxPool2d()"; +} + +FractionalMaxPool3dImpl::FractionalMaxPool3dImpl( + FractionalMaxPool3dOptions options_) + : options(std::move(options_)) { + FractionalMaxPool3dImpl::reset(); +} + +void FractionalMaxPool3dImpl::reset() { + _random_samples = + register_buffer("_random_samples", options._random_samples()); + if (options.output_size() == std::nullopt && + options.output_ratio() == std::nullopt) { + TORCH_CHECK( + false, + "FractionalMaxPool3d requires specifying either ", + "an output size, or a pooling ratio"); + } + if (options.output_size().has_value() && options.output_ratio().has_value()) { + TORCH_CHECK( + false, "only one of output_size and output_ratio may be specified"); + } + if (options.output_ratio().has_value()) { + at::ArrayRef output_ratio = + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + at::ArrayRef(options.output_ratio().value()); + if (!(0 < output_ratio[0] && output_ratio[0] < 1 && 0 < output_ratio[1] && + output_ratio[1] < 1 && 0 < output_ratio[2] && output_ratio[2] < 1)) { + TORCH_CHECK( + false, + "output_ratio must be between 0 and 1 (got ", + output_ratio, + ")"); + } + } +} + +Tensor FractionalMaxPool3dImpl::forward(const Tensor& input) { + return F::detail::fractional_max_pool3d( + input, + options.kernel_size(), + options.output_size(), + options.output_ratio(), + _random_samples); +} + +std::tuple FractionalMaxPool3dImpl::forward_with_indices( + const Tensor& input) { + return F::detail::fractional_max_pool3d_with_indices( + input, + options.kernel_size(), + options.output_size(), + options.output_ratio(), + _random_samples); +} + +void FractionalMaxPool3dImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::FractionalMaxPool3d()"; +} + +// ============================================================================ + +template +LPPoolImpl::LPPoolImpl(const LPPoolOptions& options_) + : options(options_) {} + +template +void LPPoolImpl::reset() {} + +template +void LPPoolImpl::pretty_print(std::ostream& stream) const { + stream << std::boolalpha << "torch::nn::LPPool" << D << "d(" + << "norm_type=" << options.norm_type() << ", " + << "kernel_size=" << options.kernel_size() << ", " + << "stride=" << options.stride() << ", " + << "ceil_mode=" << options.ceil_mode() << ')'; +} + +Tensor LPPool1dImpl::forward(const Tensor& input) { + return F::detail::lp_pool1d( + input, + options.norm_type(), + options.kernel_size(), + options.stride(), + options.ceil_mode()); +} + +template class LPPoolImpl<1, LPPool1dImpl>; + +Tensor LPPool2dImpl::forward(const Tensor& input) { + return F::detail::lp_pool2d( + input, + options.norm_type(), + options.kernel_size(), + options.stride(), + options.ceil_mode()); +} + +template class LPPoolImpl<2, LPPool2dImpl>; + +Tensor LPPool3dImpl::forward(const Tensor& input) { + return F::detail::lp_pool3d( + input, + options.norm_type(), + options.kernel_size(), + options.stride(), + options.ceil_mode()); +} + +template class LPPoolImpl<3, LPPool3dImpl>; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/rnn.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/rnn.cpp new file mode 100644 index 0000000000000000000000000000000000000000..83d5636314e9f285ab41a597c9d4927bbbeeec4f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/rnn.cpp @@ -0,0 +1,1021 @@ +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace torch::nn::utils::rnn; + +namespace torch::nn { + +/// These must line up with the CUDNN mode codes: +/// https://docs.nvidia.com/deeplearning/cudnn/backend/latest/api/cudnn-adv-library.html#cudnnrnnmode-t +enum class CuDNNMode { RNN_RELU = 0, RNN_TANH = 1, LSTM = 2, GRU = 3 }; + +static CuDNNMode get_cudnn_mode_for_rnn( + detail::RNNOptionsBase::rnn_options_base_mode_t mode) { + if (std::holds_alternative(mode)) { + return CuDNNMode::RNN_RELU; + } else if (std::holds_alternative(mode)) { + return CuDNNMode::RNN_TANH; + } else if (std::holds_alternative(mode)) { + return CuDNNMode::LSTM; + } else if (std::holds_alternative(mode)) { + return CuDNNMode::GRU; + } else { + TORCH_CHECK(false, "Unknown mode: ", torch::enumtype::get_enum_name(mode)); + } +} + +static Tensor apply_permutation( + const Tensor& tensor, + const Tensor& permutation, + int64_t dim = 1) { + return tensor.index_select(dim, permutation); +} + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RNNImplBase ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +namespace detail { +template +RNNImplBase::RNNImplBase(const RNNOptionsBase& options_) + : options_base(options_) { + RNNImplBase::reset(); +} + +template +void RNNImplBase::reset() { + const int64_t num_directions = options_base.bidirectional() ? 2 : 1; + + TORCH_CHECK( + 0 <= options_base.dropout() && options_base.dropout() <= 1, + "dropout should be a number in range [0, 1] ", + "representing the probability of an element being ", + "zeroed"); + + if (options_base.dropout() > 0 && options_base.num_layers() == 1) { + TORCH_WARN( + "dropout option adds dropout after all but last ", + "recurrent layer, so non-zero dropout expects ", + "num_layers greater than 1, but got dropout=", + options_base.dropout(), + " and ", + "num_layers=", + options_base.num_layers()); + } + + TORCH_CHECK( + options_base.hidden_size() > 0, "hidden_size must be greater than zero"); + + TORCH_CHECK( + options_base.num_layers() > 0, "num_layers must be greater than zero"); + + TORCH_CHECK( + 0 <= options_base.proj_size() && + options_base.proj_size() < options_base.hidden_size(), + "proj_size has to be a positive integer, smaller than ", + "hidden_size or zero to disable projections"); + + if (options_base.proj_size() > 0) { + TORCH_CHECK( + std::get_if(&options_base.mode()), + "proj_size argument is only supported for LSTM, not RNN or GRU"); + } + + int64_t gate_size = 0; + if (std::holds_alternative(options_base.mode())) { + gate_size = 4 * options_base.hidden_size(); + } else if (std::holds_alternative(options_base.mode())) { + gate_size = 3 * options_base.hidden_size(); + // NOLINTNEXTLINE(bugprone-branch-clone) + } else if (std::holds_alternative(options_base.mode())) { + gate_size = options_base.hidden_size(); + } else if (std::holds_alternative(options_base.mode())) { + gate_size = options_base.hidden_size(); + } else { + TORCH_CHECK( + false, + "Unrecognized RNN mode: " + + torch::enumtype::get_enum_name(options_base.mode())); + } + + flat_weights_names_ = {}; + all_weights_ = {}; + + for (const auto layer : c10::irange(options_base.num_layers())) { + for (const auto direction : c10::irange(num_directions)) { + int64_t real_hidden_size = options_base.proj_size() > 0 + ? options_base.proj_size() + : options_base.hidden_size(); + int64_t layer_input_size = layer == 0 ? options_base.input_size() + : real_hidden_size * num_directions; + + auto w_ih = torch::empty({gate_size, layer_input_size}); + auto w_hh = torch::empty({gate_size, real_hidden_size}); + auto b_ih = torch::empty({gate_size}); + // Second bias vector included for CuDNN compatibility. Only one + // bias vector is needed in standard definition. + auto b_hh = torch::empty({gate_size}); + std::vector layer_params = {w_ih, w_hh}; + + std::string suffix = direction == 1 ? "_reverse" : ""; + std::vector param_names = { + "weight_ih_l{layer}{suffix}", "weight_hh_l{layer}{suffix}"}; + if (options_base.bias()) { + param_names.emplace_back("bias_ih_l{layer}{suffix}"); + param_names.emplace_back("bias_hh_l{layer}{suffix}"); + layer_params.emplace_back(b_ih); + layer_params.emplace_back(b_hh); + } + if (options_base.proj_size() > 0) { + auto w_hr = torch::empty( + {options_base.proj_size(), options_base.hidden_size()}); + layer_params.emplace_back(w_hr); + param_names.emplace_back("weight_hr_l{layer}{suffix}"); + } + for (auto& param_name : param_names) { + std::string x = std::regex_replace( + param_name, std::regex("\\{layer\\}"), c10::str(layer)); + param_name = + std::regex_replace(x, std::regex("\\{suffix\\}"), c10::str(suffix)); + } + + for (const auto i : c10::irange(param_names.size())) { + this->register_parameter(param_names[i], std::move(layer_params[i])); + } + flat_weights_names_.insert( + flat_weights_names_.end(), param_names.begin(), param_names.end()); + all_weights_.emplace_back(std::move(param_names)); + } + } + + flat_weights_ = {}; + for (const auto& wn : flat_weights_names_) { + auto named_parameters = this->named_parameters(/*recurse=*/false); + if (named_parameters.contains(wn)) { + flat_weights_.emplace_back(named_parameters[wn]); + } else { + flat_weights_.emplace_back(); + } + } + + this->flatten_parameters(); + this->reset_parameters(); +} + +template +void RNNImplBase::flatten_parameters() { + // Resets parameter data pointer so that they can use faster code paths. + // + // Right now, this works only if the module is on the GPU and cuDNN is + // enabled. Otherwise, it's a no-op. + + // Short-circuits if flat_weights_ is only partially instantiated + if (flat_weights_.size() != flat_weights_names_.size()) { + return; + } + + // Short-circuits if any tensor in self.flat_weights_ is not acceptable to + // cuDNN or the tensors in flat_weights_ are of different dtypes + + auto first_fw = flat_weights_[0]; + auto dtype = first_fw.dtype(); + for (const auto& fw : flat_weights_) { + if (!(fw.dtype() == dtype) || !fw.is_cuda() || + !torch::cudnn_is_acceptable(fw)) { + return; + } + } + + // If any parameters alias, we fall back to the slower, copying code path. + // This is a sufficient check, because overlapping parameter buffers that + // don't completely alias would break the assumptions of the uniqueness check + // in Module::named_parameters(). + std::unordered_set unique_data_ptrs; + for (const auto& p : flat_weights_) { + unique_data_ptrs.emplace(p.data_ptr()); + } + if (unique_data_ptrs.size() != flat_weights_.size()) { + return; + } + + { + torch::DeviceGuard device_guard(first_fw.device()); + + // Note: no_grad() is necessary since _cudnn_rnn_flatten_weight is + // an inplace operation on self.flat_weights_ + { + torch::NoGradGuard no_grad; + if (torch::_use_cudnn_rnn_flatten_weight()) { + int64_t num_weights = options_base.bias() ? 4 : 2; + if (options_base.proj_size() > 0) { + ++num_weights; + } + torch::_cudnn_rnn_flatten_weight( + flat_weights_, + num_weights, + options_base.input_size(), + static_cast(get_cudnn_mode_for_rnn(options_base.mode())), + options_base.hidden_size(), + options_base.proj_size(), + options_base.num_layers(), + options_base.batch_first(), + options_base.bidirectional()); + } + } + } +} + +template +void RNNImplBase::reset_flat_weights() { + flat_weights_ = {}; + for (const auto& wn : flat_weights_names_) { + auto named_parameters = this->named_parameters(/*recurse=*/false); + if (named_parameters.contains(wn)) { + flat_weights_.emplace_back(named_parameters[wn]); + } else { + flat_weights_.emplace_back(); + } + } +} + +template +void RNNImplBase::to( + torch::Device device, + torch::Dtype dtype, + bool non_blocking) { + nn::Module::to(device, dtype, non_blocking); + reset_flat_weights(); + flatten_parameters(); +} + +template +void RNNImplBase::to(torch::Dtype dtype, bool non_blocking) { + nn::Module::to(dtype, non_blocking); + reset_flat_weights(); + flatten_parameters(); +} + +template +void RNNImplBase::to(torch::Device device, bool non_blocking) { + nn::Module::to(device, non_blocking); + reset_flat_weights(); + flatten_parameters(); +} + +template +void RNNImplBase::reset_parameters() { + const double stdv = 1.0 / std::sqrt(options_base.hidden_size()); + for (auto& weight : this->parameters()) { + init::uniform_(weight, -stdv, stdv); + } +} + +template +void RNNImplBase::check_input( + const Tensor& input, + const Tensor& batch_sizes) const { + int64_t expected_input_dim = batch_sizes.defined() ? 2 : 3; + TORCH_CHECK( + input.dim() == expected_input_dim, + "input must have ", + expected_input_dim, + " dimensions, got ", + input.dim()); + TORCH_CHECK( + options_base.input_size() == input.size(-1), + "input.size(-1) must be equal to input_size. Expected ", + options_base.input_size(), + ", got ", + input.size(-1)); +} + +template +std::tuple RNNImplBase:: + get_expected_hidden_size(const Tensor& input, const Tensor& batch_sizes) + const { + int64_t mini_batch = 0; + if (batch_sizes.defined()) { + mini_batch = batch_sizes[0].item(); + } else { + mini_batch = options_base.batch_first() ? input.size(0) : input.size(1); + } + int64_t num_directions = options_base.bidirectional() ? 2 : 1; + int64_t real_hidden_size = options_base.proj_size() > 0 + ? options_base.proj_size() + : options_base.hidden_size(); + return std::make_tuple( + options_base.num_layers() * num_directions, mini_batch, real_hidden_size); +} + +template +void RNNImplBase::check_hidden_size( + const Tensor& hx, + std::tuple expected_hidden_size, + std::string msg) const { + auto expected_hidden_size_vec = std::vector({ + std::get<0>(expected_hidden_size), + std::get<1>(expected_hidden_size), + std::get<2>(expected_hidden_size), + }); + if (hx.sizes() != expected_hidden_size_vec) { + msg = std::regex_replace( + msg, std::regex("\\{1\\}"), c10::str(expected_hidden_size_vec)); + msg = std::regex_replace(msg, std::regex("\\{2\\}"), c10::str(hx.sizes())); + TORCH_CHECK(false, msg); + } +} + +template +void RNNImplBase::check_forward_args( + Tensor input, + Tensor hidden, + Tensor batch_sizes) const { + this->check_input(input, batch_sizes); + auto expected_hidden_size = + this->get_expected_hidden_size(input, batch_sizes); + + this->check_hidden_size(hidden, expected_hidden_size); +} + +template +Tensor RNNImplBase::permute_hidden( + Tensor hx, + const Tensor& permutation) const { + if (!permutation.defined()) { + return hx; + } + return apply_permutation(hx, permutation); +} + +template +void RNNImplBase::pretty_print(std::ostream& stream) const { + const std::string name = this->name(); + const std::string name_without_impl = name.substr(0, name.size() - 4); + stream << std::boolalpha << name_without_impl + << "(input_size=" << options_base.input_size() + << ", hidden_size=" << options_base.hidden_size() + << ", num_layers=" << options_base.num_layers() + << ", bias=" << options_base.bias() + << ", batch_first=" << options_base.batch_first() + << ", dropout=" << options_base.dropout() + << ", bidirectional=" << options_base.bidirectional(); + if (options_base.proj_size() > 0) { + stream << ", proj_size=" << options_base.proj_size(); + } + stream << ')'; +} + +template +std::vector RNNImplBase::all_weights() const { + std::vector result = {}; + auto named_parameters = this->named_parameters(/*recurse=*/false); + for (const auto& weights : all_weights_) { + for (const auto& weight : weights) { + result.emplace_back(named_parameters[weight]); + } + } + return result; +} + +template class RNNImplBase; +template class RNNImplBase; +template class RNNImplBase; +} // namespace detail + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RNN ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +static detail::RNNOptionsBase::rnn_options_base_mode_t +compute_rnn_options_base_mode(RNNOptions::nonlinearity_t nonlinearity) { + if (std::holds_alternative(nonlinearity)) { + return torch::kRNN_TANH; + } else if (std::holds_alternative(nonlinearity)) { + return torch::kRNN_RELU; + } else { + TORCH_CHECK( + false, + "Unknown nonlinearity ", + torch::enumtype::get_enum_name(nonlinearity)); + } +} + +RNNImpl::RNNImpl(const RNNOptions& options_) + : detail::RNNImplBase( + detail::RNNOptionsBase( + compute_rnn_options_base_mode(options_.nonlinearity()), + options_.input_size(), + options_.hidden_size()) + .num_layers(options_.num_layers()) + .bias(options_.bias()) + .batch_first(options_.batch_first()) + .dropout(options_.dropout()) + .bidirectional(options_.bidirectional())), + options(options_) {} + +std::tuple RNNImpl::forward_helper( + const Tensor& input, + const Tensor& batch_sizes, + const Tensor& sorted_indices, + int64_t max_batch_size, + Tensor hx) { + if (!hx.defined()) { + int64_t num_directions = options_base.bidirectional() ? 2 : 1; + hx = torch::zeros( + {options_base.num_layers() * num_directions, + max_batch_size, + options_base.hidden_size()}, + torch::dtype(input.dtype()).device(input.device())); + } else { + // Each batch of the hidden state should match the input sequence that + // the user believes he/she is passing in. + hx = this->permute_hidden(hx, sorted_indices); + } + + this->check_forward_args(input, hx, batch_sizes); + + std::tuple result; + if (!batch_sizes.defined()) { + if (std::holds_alternative(options_base.mode())) { + result = torch::rnn_tanh( + input, + hx, + flat_weights_, + options_base.bias(), + options_base.num_layers(), + options_base.dropout(), + this->is_training(), + options_base.bidirectional(), + options_base.batch_first()); + } else if (std::holds_alternative( + options_base.mode())) { + result = torch::rnn_relu( + input, + hx, + flat_weights_, + options_base.bias(), + options_base.num_layers(), + options_base.dropout(), + this->is_training(), + options_base.bidirectional(), + options_base.batch_first()); + } else { + TORCH_CHECK( + false, + "Unknown mode: ", + torch::enumtype::get_enum_name(options_base.mode())); + } + } else { + if (std::holds_alternative(options_base.mode())) { + result = torch::rnn_tanh( + input, + batch_sizes, + hx, + flat_weights_, + options_base.bias(), + options_base.num_layers(), + options_base.dropout(), + this->is_training(), + options_base.bidirectional()); + } else if (std::holds_alternative( + options_base.mode())) { + result = torch::rnn_relu( + input, + batch_sizes, + hx, + flat_weights_, + options_base.bias(), + options_base.num_layers(), + options_base.dropout(), + this->is_training(), + options_base.bidirectional()); + } else { + TORCH_CHECK( + false, + "Unknown mode: ", + torch::enumtype::get_enum_name(options_base.mode())); + } + } + return result; +} + +std::tuple RNNImpl::forward(const Tensor& input, Tensor hx) { + auto batch_sizes = torch::Tensor(); + auto max_batch_size = + options_base.batch_first() ? input.size(0) : input.size(1); + auto sorted_indices = torch::Tensor(); + auto unsorted_indices = torch::Tensor(); + + auto [output, hidden] = this->forward_helper( + input, batch_sizes, sorted_indices, max_batch_size, std::move(hx)); + + return std::make_tuple( + output, this->permute_hidden(hidden, unsorted_indices)); +} + +std::tuple RNNImpl::forward_with_packed_input( + const PackedSequence& packed_input, + Tensor hx) { + const auto& input = packed_input.data(); + const auto& batch_sizes = packed_input.batch_sizes(); + const auto& sorted_indices = packed_input.sorted_indices(); + const auto& unsorted_indices = packed_input.unsorted_indices(); + auto max_batch_size = batch_sizes[0].item(); + + auto [output, hidden] = this->forward_helper( + input, batch_sizes, sorted_indices, max_batch_size, std::move(hx)); + + auto output_packed = + PackedSequence(output, batch_sizes, sorted_indices, unsorted_indices); + return std::make_tuple( + std::move(output_packed), this->permute_hidden(hidden, unsorted_indices)); +} + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LSTM ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +LSTMImpl::LSTMImpl(const LSTMOptions& options_) + : detail::RNNImplBase(detail::RNNOptionsBase( + torch::kLSTM, + options_.input_size(), + options_.hidden_size()) + .num_layers(options_.num_layers()) + .bias(options_.bias()) + .batch_first(options_.batch_first()) + .dropout(options_.dropout()) + .bidirectional(options_.bidirectional()) + .proj_size(options_.proj_size())), + options(options_) {} + +std::tuple LSTMImpl::get_expected_cell_size( + const Tensor& input, + const Tensor& batch_sizes) const { + int64_t mini_batch = 0; + if (batch_sizes.defined()) { + mini_batch = batch_sizes[0].item(); + } else { + mini_batch = options_base.batch_first() ? input.size(0) : input.size(1); + } + int64_t num_directions = options_base.bidirectional() ? 2 : 1; + return std::make_tuple( + options_base.num_layers() * num_directions, + mini_batch, + options_base.hidden_size()); +} + +void LSTMImpl::check_forward_args( + const Tensor& input, + std::tuple hidden, + const Tensor& batch_sizes) const { + this->check_input(input, batch_sizes); + this->check_hidden_size( + std::get<0>(hidden), + this->get_expected_hidden_size(input, batch_sizes), + "Expected hidden[0] size {1}, got {2}"); + this->check_hidden_size( + std::get<1>(hidden), + this->get_expected_cell_size(input, batch_sizes), + "Expected hidden[1] size {1}, got {2}"); +} + +std::tuple LSTMImpl::permute_hidden( + std::tuple hx, + const Tensor& permutation) const { + if (!permutation.defined()) { + return hx; + } + return std::make_tuple( + apply_permutation(std::get<0>(hx), permutation), + apply_permutation(std::get<1>(hx), permutation)); +} + +std::tuple> LSTMImpl::forward_helper( + const Tensor& input, + const Tensor& batch_sizes, + const Tensor& sorted_indices, + int64_t max_batch_size, + std::optional> hx_opt) { + std::tuple hx; + if (!hx_opt.has_value()) { + int64_t num_directions = options.bidirectional() ? 2 : 1; + int64_t real_hidden_size = + options.proj_size() > 0 ? options.proj_size() : options.hidden_size(); + auto h_zeros = torch::zeros( + {options.num_layers() * num_directions, + max_batch_size, + real_hidden_size}, + torch::dtype(input.dtype()).device(input.device())); + auto c_zeros = torch::zeros( + {options.num_layers() * num_directions, + max_batch_size, + options.hidden_size()}, + torch::dtype(input.dtype()).device(input.device())); + hx = std::make_tuple(std::move(h_zeros), std::move(c_zeros)); + } else { + hx = hx_opt.value(); + // Each batch of the hidden state should match the input sequence that + // the user believes he/she is passing in. + hx = this->permute_hidden(hx, sorted_indices); + } + + this->check_forward_args(input, hx, batch_sizes); + std::tuple result; + if (!batch_sizes.defined()) { + result = torch::lstm( + input, + {std::move(std::get<0>(hx)), std::move(std::get<1>(hx))}, + flat_weights_, + options.bias(), + options.num_layers(), + options.dropout(), + this->is_training(), + options.bidirectional(), + options.batch_first()); + } else { + result = torch::lstm( + input, + batch_sizes, + {std::move(std::get<0>(hx)), std::move(std::get<1>(hx))}, + flat_weights_, + options.bias(), + options.num_layers(), + options.dropout(), + this->is_training(), + options.bidirectional()); + } + auto output = std::move(std::get<0>(result)); + auto hidden = std::make_tuple( + std::move(std::get<1>(result)), std::move(std::get<2>(result))); + + return std::make_tuple(std::move(output), std::move(hidden)); +} + +std::tuple> LSTMImpl::forward( + const Tensor& input, + std::optional> hx_opt) { + auto batch_sizes = torch::Tensor(); + auto max_batch_size = options.batch_first() ? input.size(0) : input.size(1); + auto sorted_indices = torch::Tensor(); + auto unsorted_indices = torch::Tensor(); + + auto [output, hidden] = this->forward_helper( + input, batch_sizes, sorted_indices, max_batch_size, std::move(hx_opt)); + + return std::make_tuple( + output, this->permute_hidden(hidden, unsorted_indices)); +} + +std::tuple> LSTMImpl:: + forward_with_packed_input( + const PackedSequence& packed_input, + std::optional> hx_opt) { + const auto& input = packed_input.data(); + const auto& batch_sizes = packed_input.batch_sizes(); + const auto& sorted_indices = packed_input.sorted_indices(); + const auto& unsorted_indices = packed_input.unsorted_indices(); + auto max_batch_size = batch_sizes[0].item(); + + auto [output, hidden] = this->forward_helper( + input, batch_sizes, sorted_indices, max_batch_size, std::move(hx_opt)); + + auto output_packed = + PackedSequence(output, batch_sizes, sorted_indices, unsorted_indices); + return std::make_tuple( + std::move(output_packed), this->permute_hidden(hidden, unsorted_indices)); +} + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GRU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +GRUImpl::GRUImpl(const GRUOptions& options_) + : detail::RNNImplBase( + detail::RNNOptionsBase( + torch::kGRU, + options_.input_size(), + options_.hidden_size()) + .num_layers(options_.num_layers()) + .bias(options_.bias()) + .batch_first(options_.batch_first()) + .dropout(options_.dropout()) + .bidirectional(options_.bidirectional())), + options(options_) {} + +std::tuple GRUImpl::forward_helper( + const Tensor& input, + const Tensor& batch_sizes, + const Tensor& sorted_indices, + int64_t max_batch_size, + Tensor hx) { + if (!hx.defined()) { + int64_t num_directions = options.bidirectional() ? 2 : 1; + hx = torch::zeros( + {options.num_layers() * num_directions, + max_batch_size, + options.hidden_size()}, + torch::dtype(input.dtype()).device(input.device())); + } else { + // Each batch of the hidden state should match the input sequence that + // the user believes he/she is passing in. + hx = this->permute_hidden(hx, sorted_indices); + } + + this->check_forward_args(input, hx, batch_sizes); + std::tuple result; + if (!batch_sizes.defined()) { + result = torch::gru( + input, + hx, + flat_weights_, + options.bias(), + options.num_layers(), + options.dropout(), + this->is_training(), + options.bidirectional(), + options.batch_first()); + } else { + result = torch::gru( + input, + batch_sizes, + hx, + flat_weights_, + options.bias(), + options.num_layers(), + options.dropout(), + this->is_training(), + options.bidirectional()); + } + return result; +} + +std::tuple GRUImpl::forward(const Tensor& input, Tensor hx) { + auto batch_sizes = torch::Tensor(); + auto max_batch_size = options.batch_first() ? input.size(0) : input.size(1); + auto sorted_indices = torch::Tensor(); + auto unsorted_indices = torch::Tensor(); + + auto [output, hidden] = this->forward_helper( + input, batch_sizes, sorted_indices, max_batch_size, std::move(hx)); + + return std::make_tuple( + output, this->permute_hidden(hidden, unsorted_indices)); +} + +std::tuple GRUImpl::forward_with_packed_input( + const PackedSequence& packed_input, + Tensor hx) { + const auto& input = packed_input.data(); + const auto& batch_sizes = packed_input.batch_sizes(); + const auto& sorted_indices = packed_input.sorted_indices(); + const auto& unsorted_indices = packed_input.unsorted_indices(); + auto max_batch_size = batch_sizes[0].item(); + + auto [output, hidden] = this->forward_helper( + input, batch_sizes, sorted_indices, max_batch_size, std::move(hx)); + + auto output_packed = + PackedSequence(output, batch_sizes, sorted_indices, unsorted_indices); + return std::make_tuple( + std::move(output_packed), this->permute_hidden(hidden, unsorted_indices)); +} + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RNNCellImplBase +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +namespace detail { +template +RNNCellImplBase::RNNCellImplBase(const RNNCellOptionsBase& options_) + : options_base(options_) { + RNNCellImplBase::reset(); +} + +template +void RNNCellImplBase::reset() { + weight_ih = this->register_parameter( + "weight_ih", + torch::empty( + {options_base.num_chunks() * options_base.hidden_size(), + options_base.input_size()})); + weight_hh = this->register_parameter( + "weight_hh", + torch::empty( + {options_base.num_chunks() * options_base.hidden_size(), + options_base.hidden_size()})); + + if (options_base.bias()) { + bias_ih = this->register_parameter( + "bias_ih", + torch::empty({options_base.num_chunks() * options_base.hidden_size()})); + bias_hh = this->register_parameter( + "bias_hh", + torch::empty({options_base.num_chunks() * options_base.hidden_size()})); + } else { + bias_ih = + this->register_parameter("bias_ih", Tensor(), /*requires_grad=*/false); + bias_hh = + this->register_parameter("bias_hh", Tensor(), /*requires_grad=*/false); + } + + reset_parameters(); +} + +template +void RNNCellImplBase::reset_parameters() { + const double stdv = 1.0 / std::sqrt(options_base.hidden_size()); + for (auto& weight : this->parameters()) { + init::uniform_(weight, -stdv, stdv); + } +} + +template +void RNNCellImplBase::pretty_print(std::ostream& stream) const { + const std::string name = this->name(); + const std::string name_without_impl = name.substr(0, name.size() - 4); + stream << name_without_impl << '(' << options_base.input_size() << ", " + << options_base.hidden_size(); + if (!options_base.bias()) { + stream << ", bias=" << std::boolalpha << false; + } + auto nonlinearity_str = this->get_nonlinearity_str(); + if (!nonlinearity_str.empty() && nonlinearity_str != "kTanh") { + stream << ", nonlinearity=" << nonlinearity_str; + } + stream << ')'; +} + +template +void RNNCellImplBase::check_forward_input( + const Tensor& input, + const std::string& name) const { + TORCH_CHECK( + input.dim() == 1 || input.dim() == 2, + "Expected ", + name.c_str(), + " to be 1D or 2D, got ", + input.dim(), + "D instead"); +} + +template +std::string RNNCellImplBase::get_nonlinearity_str() const { + return ""; +} + +template class RNNCellImplBase; +template class RNNCellImplBase; +template class RNNCellImplBase; +} // namespace detail + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RNNCell +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +RNNCellImpl::RNNCellImpl(const RNNCellOptions& options_) + : detail::RNNCellImplBase(detail::RNNCellOptionsBase( + options_.input_size(), + options_.hidden_size(), + options_.bias(), + /*num_chunks=*/1)), + options(options_) {} + +Tensor RNNCellImpl::forward(const Tensor& input, const Tensor& hx) { + this->check_forward_input(input, "input"); + this->check_forward_input(hx, "hidden"); + + Tensor r_hx, ret; + + bool is_batched = input.dim() == 2; + Tensor r_input = is_batched ? input : input.unsqueeze(0); + + if (!hx.defined()) { + r_hx = torch::zeros( + {input.size(0), options.hidden_size()}, + torch::dtype(input.dtype()).device(input.device())); + } else { + r_hx = is_batched ? hx : hx.unsqueeze(0); + } + + if (std::holds_alternative(options.nonlinearity())) { + ret = torch::rnn_tanh_cell( + r_input, r_hx, weight_ih, weight_hh, bias_ih, bias_hh); + } else if (std::holds_alternative(options.nonlinearity())) { + ret = torch::rnn_relu_cell( + r_input, r_hx, weight_ih, weight_hh, bias_ih, bias_hh); + } else { + TORCH_CHECK( + false, + "Unknown nonlinearity: ", + torch::enumtype::get_enum_name(options.nonlinearity())); + } + + if (!is_batched) { + ret = ret.squeeze(0); + } + + return ret; +} + +std::string RNNCellImpl::get_nonlinearity_str() const { + return get_enum_name(options.nonlinearity()); +} + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LSTMCell +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +LSTMCellImpl::LSTMCellImpl(const LSTMCellOptions& options_) + : detail::RNNCellImplBase(detail::RNNCellOptionsBase( + options_.input_size(), + options_.hidden_size(), + options_.bias(), + /*num_chunks=*/4)), + options(options_) {} + +std::tuple LSTMCellImpl::forward( + const Tensor& input, + std::optional> hx_opt) { + this->check_forward_input(input, "input"); + if (hx_opt.has_value()) { + this->check_forward_input(std::get<0>(hx_opt.value()), "hx[0]"); + this->check_forward_input(std::get<1>(hx_opt.value()), "hx[1]"); + } + + std::tuple r_hx, ret; + + bool is_batched = input.dim() == 2; + Tensor r_input = is_batched ? input : input.unsqueeze(0); + + if (!hx_opt.has_value()) { + auto zeros = torch::zeros( + {input.size(0), options.hidden_size()}, + torch::dtype(input.dtype()).device(input.device())); + r_hx = std::make_tuple(zeros, zeros); + } else { + if (!is_batched) { + r_hx = std::make_tuple( + std::get<0>(hx_opt.value()).unsqueeze(0), + std::get<1>(hx_opt.value()).unsqueeze(0)); + } else { + r_hx = hx_opt.value(); + } + } + + ret = torch::lstm_cell( + r_input, + {std::get<0>(r_hx), std::get<1>(r_hx)}, + weight_ih, + weight_hh, + bias_ih, + bias_hh); + + if (!is_batched) { + ret = std::make_tuple( + std::get<0>(ret).squeeze(0), std::get<1>(ret).squeeze(0)); + } + + return ret; +} + +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GRUCell +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +GRUCellImpl::GRUCellImpl(const GRUCellOptions& options_) + : detail::RNNCellImplBase(detail::RNNCellOptionsBase( + options_.input_size(), + options_.hidden_size(), + options_.bias(), + /*num_chunks=*/3)), + options(options_) {} + +Tensor GRUCellImpl::forward(const Tensor& input, const Tensor& hx) { + this->check_forward_input(input, "input"); + this->check_forward_input(hx, "hidden"); + + Tensor r_hx, ret; + + bool is_batched = input.dim() == 2; + Tensor r_input = is_batched ? input : input.unsqueeze(0); + + if (!hx.defined()) { + r_hx = torch::zeros( + {input.size(0), options.hidden_size()}, + torch::dtype(input.dtype()).device(input.device())); + } else { + r_hx = is_batched ? hx : hx.unsqueeze(0); + } + + ret = torch::gru_cell(r_input, r_hx, weight_ih, weight_hh, bias_ih, bias_hh); + + if (!is_batched) { + ret = ret.squeeze(0); + } + + return ret; +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/transformer.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/transformer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..077197686ef922373ab0ecb2803535b212899eff --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/transformer.cpp @@ -0,0 +1,479 @@ +#include +#include +#include +#include +#include + +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +// ========================TransformerEncoderLayerImpl========================= +TransformerEncoderLayerImpl::TransformerEncoderLayerImpl( + TransformerEncoderLayerOptions options_) + : options(std::move(options_)) { + TransformerEncoderLayerImpl::reset(); +} + +void TransformerEncoderLayerImpl::reset() { + // NOTE: reset() is for initializing the model only, calling reset() after the + // model is created will throw exceptions. Call reset_parameter() if the + // created model needs a reset + + self_attn = this->register_module( + "self_attn", + MultiheadAttention( + MultiheadAttentionOptions(options.d_model(), options.nhead()) + .dropout(options.dropout()))); + + linear1 = this->register_module( + "linear1", Linear(options.d_model(), options.dim_feedforward())); + dropout = this->register_module("dropout", Dropout(options.dropout())); + linear2 = this->register_module( + "linear2", Linear(options.dim_feedforward(), options.d_model())); + + norm1 = this->register_module( + "norm1", LayerNorm(LayerNormOptions({options.d_model()}))); + norm2 = this->register_module( + "norm2", LayerNorm(LayerNormOptions({options.d_model()}))); + + dropout1 = this->register_module("dropout1", Dropout(options.dropout())); + dropout2 = this->register_module("dropout2", Dropout(options.dropout())); +} + +void TransformerEncoderLayerImpl::reset_parameters() { + // TODO xinyu: standardrize reset_parameters virtual funcs + self_attn->_reset_parameters(); + + linear1->reset_parameters(); + // dropout->reset_parameters(); + linear2->reset_parameters(); + + norm1->reset_parameters(); + norm2->reset_parameters(); + + // dropout1->reset_parameters(); + // dropout2->reset_parameters(); +} + +Tensor TransformerEncoderLayerImpl::forward( + const Tensor& src, + const Tensor& src_mask, + const Tensor& src_key_padding_mask) { + // multihead attention + Tensor src2 = std::get<0>(self_attn( + src, src, src, src_key_padding_mask, /*need_weights=*/true, src_mask)); + // add & norm + Tensor ret = norm1(src + dropout1(src2)); + + // feedforward + if (std::holds_alternative(options.activation())) { + src2 = linear2(dropout(F::gelu(linear1(ret)))); + } else if (std::holds_alternative(options.activation())) { + src2 = linear2(dropout(F::relu(linear1(ret)))); + } else if (std::holds_alternative>( + options.activation())) { + auto callable_activation = + std::get>(options.activation()); + src2 = linear2(dropout(callable_activation(linear1(ret)))); + } else { + TORCH_CHECK(false, "activation should be kGELU, kReLU, or a callable"); + } + + // add & norm + return norm2(ret + dropout2(src2)); +} + +// ========================TransformerDecoderLayerImpl========================= +TransformerDecoderLayerImpl::TransformerDecoderLayerImpl( + TransformerDecoderLayerOptions options_) + : options(std::move(options_)) { + TransformerDecoderLayerImpl::reset(); +} + +void TransformerDecoderLayerImpl::reset() { + // NOTE: reset() is for initializing the model only, calling reset() after the + // model is created will cause throwing exceptions. Call reset_parameter() if + // the created model needs a reset. + + // initialize self attention + self_attn = this->register_module( + "self_attn", + MultiheadAttention( + MultiheadAttentionOptions(options.d_model(), options.nhead()) + .dropout(options.dropout()))); + + // initialize multihead attention + multihead_attn = this->register_module( + "multihead_attn", + MultiheadAttention( + MultiheadAttentionOptions(options.d_model(), options.nhead()) + .dropout(options.dropout()))); + + // Initialize Feed forward first linear layer + linear1 = this->register_module( + "linear1", Linear(options.d_model(), options.dim_feedforward())); + // initialize Feed forward dropout layer + dropout = this->register_module("dropout", Dropout(options.dropout())); + // initialize Feed forward second linear layer + linear2 = this->register_module( + "linear2", Linear(options.dim_feedforward(), options.d_model())); + + // initialize Normalization, post self attention + norm1 = this->register_module( + "norm1", LayerNorm(LayerNormOptions({options.d_model()}))); + // initialize post multi-headed attention Normalization + norm2 = this->register_module( + "norm2", LayerNorm(LayerNormOptions({options.d_model()}))); + // initialize normalization, post feed forward + norm3 = this->register_module( + "norm3", LayerNorm(LayerNormOptions({options.d_model()}))); + + // initialize Dropout, post self attention + dropout1 = this->register_module("dropout1", Dropout(options.dropout())); + // initialize post multi-headed attention dropout layer + dropout2 = this->register_module("dropout2", Dropout(options.dropout())); + // initialize dropout, post feed forward + dropout3 = this->register_module("dropout3", Dropout(options.dropout())); +} + +void TransformerDecoderLayerImpl::reset_parameters() { + // TODO xinyu: standardrize reset_parameters virtual funcs + self_attn->_reset_parameters(); + multihead_attn->_reset_parameters(); + + linear1->reset_parameters(); + // dropout->reset_parameters(); + linear2->reset_parameters(); + + norm1->reset_parameters(); + norm2->reset_parameters(); + norm3->reset_parameters(); + // dropout1->reset_parameters(); + // dropout2->reset_parameters(); + // dropout3->reset_parameters(); +} + +/// Pass the inputs (and mask) through the decoder layer. +Tensor TransformerDecoderLayerImpl::forward( + Tensor tgt, + const Tensor& memory, + const Tensor& tgt_mask, + const Tensor& memory_mask, + const Tensor& tgt_key_padding_mask, + const Tensor& memory_key_padding_mask) { + Tensor tgt2 = std::get<0>(self_attn( + tgt, // query + tgt, // key + tgt, // value + tgt_key_padding_mask, // key_padding_mask + false, // need_weights + tgt_mask) // attn_mask + ); + tgt = tgt + dropout1(tgt2); + tgt = norm1(tgt); + + tgt2 = std::get<0>(multihead_attn( + tgt, // query + memory, // key + memory, // value + memory_key_padding_mask, // key_padding_mask + false, // need_weights + memory_mask) // attn_mask + ); + tgt = tgt + dropout2(tgt2); + tgt = norm2(tgt); + + tgt2 = linear2(dropout(activation(linear1(tgt)))); + tgt = tgt + dropout3(tgt2); + tgt = norm3(tgt); + + return tgt; +} + +Tensor TransformerDecoderLayerImpl::activation(const Tensor& input) { + if (std::holds_alternative(options.activation())) { + return F::gelu(input); + } else if (std::holds_alternative(options.activation())) { + return F::relu(input); + } else if (std::holds_alternative>( + options.activation())) { + auto callable_activation = + std::get>(options.activation()); + return callable_activation(input); + } else { + TORCH_CHECK(false, "activation should be kGELU, kReLU, or a callable"); + } +} + +// ========================TransformerEncoderImpl========================= +TransformerEncoderImpl::TransformerEncoderImpl( + TransformerEncoderOptions options_) + : options(std::move(options_)) { + TransformerEncoderImpl::reset(); +} + +void TransformerEncoderImpl::reset() { + layers = this->register_module("layers", ModuleList()); + for ([[maybe_unused]] const auto i : c10::irange(options.num_layers())) { + layers->push_back(options.encoder_layer()->clone()); + } + + if (!options.norm().is_empty()) { + norm = options.norm().clone(); + this->register_module("norm", norm.ptr()); + } +} + +void TransformerEncoderImpl::reset_parameters() { + TORCH_CHECK( + layers->size() == static_cast(options.num_layers()), + "TransformerEncoder should have", + options.num_layers(), + " encoder layers, but got ", + layers->size()); + + size_t num_layers = layers->size(); + for (const auto i : c10::irange(num_layers)) { + layers->at(i).reset_parameters(); + } + // a. No way to know whether module in AnyModule has api to reset_parameters, + // so replace instead b. Allow user to add/delete normalization module when + // reset parameters + if (!norm.is_empty()) { + this->unregister_module("norm"); + norm = AnyModule(); + } + if (!options.norm().is_empty()) { + norm = options.norm().clone(); + this->register_module("norm", norm.ptr()); + } +} + +Tensor TransformerEncoderImpl::forward( + const Tensor& src, + const Tensor& src_mask, + const Tensor& src_key_padding_mask) { + size_t num_layers = layers->size(); + Tensor output; + if (num_layers > 0) { + output = layers->at(0).forward( + src, src_mask, src_key_padding_mask); + } + for (const auto i : c10::irange(1, num_layers)) { + output = layers->at(i).forward( + output, src_mask, src_key_padding_mask); + } + + if (!norm.is_empty()) { + output = norm.forward(num_layers == 0 ? src : output); + } + return output; +} + +// ========================TransformerDecoderImpl========================= +TransformerDecoderImpl::TransformerDecoderImpl( + TransformerDecoderOptions options_) + : options(std::move(options_)) { + TransformerDecoderImpl::reset(); +} + +void TransformerDecoderImpl::reset() { + layers = this->register_module("layers", ModuleList()); + for ([[maybe_unused]] const auto i : c10::irange(options.num_layers())) { + layers->push_back(options.decoder_layer()->clone()); + } + + if (!options.norm().is_empty()) { + norm = options.norm().clone(); + this->register_module("norm", norm.ptr()); + } +} + +void TransformerDecoderImpl::reset_parameters() { + TORCH_CHECK( + layers->size() == static_cast(options.num_layers()), + "TransformerDecoder should have", + options.num_layers(), + " decoder layers, but got ", + layers->size()); + + size_t num_layers = layers->size(); + for (const auto i : c10::irange(num_layers)) { + layers->at(i).reset_parameters(); + } + // a. No way to know whether module in AnyModule has api to reset_parameters, + // so replace instead b. Allow user to add/delete normalization module when + // reset parameters + if (!norm.is_empty()) { + this->unregister_module("norm"); + norm = AnyModule(); + } + if (!options.norm().is_empty()) { + norm = options.norm().clone(); + this->register_module("norm", norm.ptr()); + } +} + +Tensor TransformerDecoderImpl::forward( + const Tensor& tgt, + const Tensor& memory, + const Tensor& tgt_mask, + const Tensor& memory_mask, + const Tensor& tgt_key_padding_mask, + const Tensor& memory_key_padding_mask) { + size_t num_layers = layers->size(); + Tensor output; + if (num_layers > 0) { + output = layers->at(0).forward( + tgt, + memory, + tgt_mask, + memory_mask, + tgt_key_padding_mask, + memory_key_padding_mask); + } + for (const auto i : c10::irange(1, num_layers)) { + output = layers->at(i).forward( + output, + memory, + tgt_mask, + memory_mask, + tgt_key_padding_mask, + memory_key_padding_mask); + } + + if (!norm.is_empty()) { + output = norm.forward(num_layers == 0 ? tgt : output); + } + + return output; +} + +// =======================================TransformerImpl================================ +TransformerImpl::TransformerImpl(TransformerOptions options_) + : options(std::move(options_)) { + TransformerImpl::reset(); +} + +void TransformerImpl::reset() { + // set up encoder + if (options.custom_encoder().is_empty()) { + LayerNorm norm(LayerNormOptions({options.d_model()})); + TransformerEncoder trans_encoder( + TransformerEncoderOptions( + TransformerEncoderLayerOptions(options.d_model(), options.nhead()) + .dim_feedforward(options.dim_feedforward()) + .dropout(options.dropout()) + .activation(options.activation()), + options.num_encoder_layers()) + .norm(AnyModule(norm))); + + this->encoder = AnyModule(trans_encoder); + } else { + this->encoder = options.custom_encoder().clone(); + } + this->register_module("encoder", this->encoder.ptr()); + + // set up decoder + if (options.custom_decoder().is_empty()) { + LayerNorm norm(LayerNormOptions({options.d_model()})); + TransformerDecoder trans_decoder( + TransformerDecoderOptions( + TransformerDecoderLayerOptions(options.d_model(), options.nhead()) + .dim_feedforward(options.dim_feedforward()) + .dropout(options.dropout()) + .activation(options.activation()), + options.num_decoder_layers()) + .norm(AnyModule(norm))); + + this->decoder = AnyModule(trans_decoder); + } else { + this->decoder = options.custom_decoder().clone(); + } + this->register_module("decoder", this->decoder.ptr()); + + reset_parameters(); +} + +void TransformerImpl::reset_parameters() { + auto parameters = this->parameters(); + for (auto& param : parameters) { + if (param.dim() > 1) { + torch::nn::init::xavier_uniform_(param); + } + } +} + +Tensor TransformerImpl::forward( + const Tensor& src, + const Tensor& tgt, + const Tensor& src_mask, + const Tensor& tgt_mask, + const Tensor& memory_mask, + const Tensor& src_key_padding_mask, + const Tensor& tgt_key_padding_mask, + const Tensor& memory_key_padding_mask) { + TORCH_CHECK( + src.dim() == 3 && tgt.dim() == 3, + "src and tgt should have 3 dimensions, but got ", + src.dim(), + " and ", + tgt.dim()); + + TORCH_CHECK( + src.size(1) == tgt.size(1), + "src and tgt should have equal batch size (at dim 1), but got ", + src.size(1), + " and ", + tgt.size(1)); + + TORCH_CHECK( + src.size(2) == options.d_model() && tgt.size(2) == options.d_model(), + "src and tgt should have same feature size as d_model (at dim 2), but got ", + src.size(2), + " and ", + tgt.size(2), + " while d_model is ", + options.d_model()); + + Tensor memory = + this->encoder.forward(src, src_mask, src_key_padding_mask); + Tensor output = this->decoder.forward( + tgt, + memory, + tgt_mask, + memory_mask, + tgt_key_padding_mask, + memory_key_padding_mask); + + return output; +} + +Tensor TransformerImpl::generate_square_subsequent_mask(int64_t sz) { + // Treat 0 dim valid here + TORCH_CHECK( + sz >= 0, + "Input size must be non-negative to generate a valid square subsequent mask, but got ", + sz); + + // check IEEE754 support here since -inf is not guaranteed to be valid on non + // IEEE754 platform + if (std::numeric_limits::is_iec559) { + return torch::triu( + torch::full({sz, sz}, -std::numeric_limits::infinity()), 1); + } + // if IEEE754 is not supported, we use the smallest float number in current + // platform + else { + TORCH_WARN_ONCE( + "IEEE754 is not supported on this platform, generate_square_subsequent_mask will fill " + "the mask with smallest float number on this platform instead of -inf"); + return torch::triu( + torch::full({sz, sz}, std::numeric_limits::lowest()), 1); + } +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/upsampling.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/upsampling.cpp new file mode 100644 index 0000000000000000000000000000000000000000..511c6a2632ef006e484637ba4d534580a9652e26 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/modules/upsampling.cpp @@ -0,0 +1,48 @@ +#include + +namespace F = torch::nn::functional; + +namespace torch::nn { + +UpsampleImpl::UpsampleImpl(UpsampleOptions options_) + : options(std::move(options_)) {} + +void UpsampleImpl::reset() {} + +void UpsampleImpl::pretty_print(std::ostream& stream) const { + stream << "torch::nn::Upsample("; + if (options.scale_factor().has_value()) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + stream << "scale_factor=" << at::ArrayRef(*options.scale_factor()); + } else { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + stream << "size=" << at::ArrayRef(options.size().value()); + } + stream << ", mode=" << enumtype::get_enum_name(options.mode()) << ')'; +} + +Tensor UpsampleImpl::forward(const Tensor& input) { + F::InterpolateFuncOptions::mode_t mode; + if (std::holds_alternative(options.mode())) { + mode = torch::kNearest; + } else if (std::holds_alternative(options.mode())) { + mode = torch::kLinear; + } else if (std::holds_alternative(options.mode())) { + mode = torch::kBilinear; + } else if (std::holds_alternative(options.mode())) { + mode = torch::kBicubic; + } else if (std::holds_alternative(options.mode())) { + mode = torch::kTrilinear; + } + + return F::detail::interpolate( + input, + options.size(), + options.scale_factor(), + mode, + options.align_corners(), + std::nullopt, + false); +} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/activation.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/activation.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6652ea14413ad232146302a9188125f1fd66faff --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/activation.cpp @@ -0,0 +1,62 @@ +#include + +namespace torch::nn { + +SELUOptions::SELUOptions(bool inplace) : inplace_(inplace) {} + +GLUOptions::GLUOptions(int64_t dim) : dim_(dim) {} + +HardshrinkOptions::HardshrinkOptions(double lambda) : lambda_(lambda) {} + +SoftmaxOptions::SoftmaxOptions(int64_t dim) : dim_(dim) {} + +SoftminOptions::SoftminOptions(int64_t dim) : dim_(dim) {} + +LogSoftmaxOptions::LogSoftmaxOptions(int64_t dim) : dim_(dim) {} + +ReLUOptions::ReLUOptions(bool inplace) : inplace_(inplace) {} + +ReLU6Options::ReLU6Options(bool inplace) : inplace_(inplace) {} + +SoftshrinkOptions::SoftshrinkOptions(double lambda) : lambda_(lambda) {} + +MultiheadAttentionOptions::MultiheadAttentionOptions( + int64_t embed_dim, + int64_t num_heads) + : embed_dim_(embed_dim), + num_heads_(num_heads), + kdim_(embed_dim), + vdim_(embed_dim) {} + +namespace functional { + +SoftmaxFuncOptions::SoftmaxFuncOptions(int64_t dim) : dim_(dim) {} + +SoftminFuncOptions::SoftminFuncOptions(int64_t dim) : dim_(dim) {} + +LogSoftmaxFuncOptions::LogSoftmaxFuncOptions(int64_t dim) : dim_(dim) {} + +MultiheadAttentionForwardFuncOptions::MultiheadAttentionForwardFuncOptions( + int64_t embed_dim_to_check, + int64_t num_heads, + Tensor in_proj_weight, + Tensor in_proj_bias, + Tensor bias_k, + Tensor bias_v, + bool add_zero_attn, + double dropout_p, + Tensor out_proj_weight, + Tensor out_proj_bias) + : embed_dim_to_check_(embed_dim_to_check), + num_heads_(num_heads), + in_proj_weight_(std::move(in_proj_weight)), + in_proj_bias_(std::move(in_proj_bias)), + bias_k_(std::move(bias_k)), + bias_v_(std::move(bias_v)), + add_zero_attn_(add_zero_attn), + dropout_p_(dropout_p), + out_proj_weight_(std::move(out_proj_weight)), + out_proj_bias_(std::move(out_proj_bias)) {} + +} // namespace functional +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/adaptive.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/adaptive.cpp new file mode 100644 index 0000000000000000000000000000000000000000..21400ca6d2d16545b122f75551f94611400a70fd --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/adaptive.cpp @@ -0,0 +1,13 @@ +#include + +namespace torch::nn { + +AdaptiveLogSoftmaxWithLossOptions::AdaptiveLogSoftmaxWithLossOptions( + int64_t in_features, + int64_t n_classes, + std::vector cutoffs) + : in_features_(in_features), + n_classes_(n_classes), + cutoffs_(std::move(cutoffs)) {} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/batchnorm.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/batchnorm.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4f8751fa3fb160840d0233129dbdc22d875c4c06 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/batchnorm.cpp @@ -0,0 +1,8 @@ +#include + +namespace torch::nn { + +BatchNormOptions::BatchNormOptions(int64_t num_features) + : num_features_(num_features) {} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/conv.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/conv.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f385ac7d42fa331c8c40177d25b6823c00a28fac --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/conv.cpp @@ -0,0 +1,21 @@ +#include + +namespace torch::nn { + +template struct ConvOptions<1>; +template struct ConvOptions<2>; +template struct ConvOptions<3>; + +namespace functional { + +template struct ConvFuncOptions<1>; +template struct ConvFuncOptions<2>; +template struct ConvFuncOptions<3>; + +template struct ConvTransposeFuncOptions<1>; +template struct ConvTransposeFuncOptions<2>; +template struct ConvTransposeFuncOptions<3>; + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/dropout.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/dropout.cpp new file mode 100644 index 0000000000000000000000000000000000000000..21abcfe0bb8a451b8c3a130b9b2504cb5a374d8a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/dropout.cpp @@ -0,0 +1,7 @@ +#include + +namespace torch::nn { + +DropoutOptions::DropoutOptions(double p) : p_(p) {} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/embedding.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/embedding.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3e5dcac5ea8d107bc8330ba4b1a326158746b7b6 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/embedding.cpp @@ -0,0 +1,13 @@ +#include + +namespace torch::nn { +EmbeddingOptions::EmbeddingOptions( + int64_t num_embeddings, + int64_t embedding_dim) + : num_embeddings_(num_embeddings), embedding_dim_(embedding_dim) {} + +EmbeddingBagOptions::EmbeddingBagOptions( + int64_t num_embeddings, + int64_t embedding_dim) + : num_embeddings_(num_embeddings), embedding_dim_(embedding_dim) {} +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/instancenorm.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/instancenorm.cpp new file mode 100644 index 0000000000000000000000000000000000000000..75c2b3b1067a74a9ff8a874733b4f019bd0ffb5e --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/instancenorm.cpp @@ -0,0 +1,8 @@ +#include + +namespace torch::nn { + +InstanceNormOptions::InstanceNormOptions(int64_t num_features) + : num_features_(num_features) {} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/linear.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/linear.cpp new file mode 100644 index 0000000000000000000000000000000000000000..34a355950cb62529bfdbc4932101d2c532e7d433 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/linear.cpp @@ -0,0 +1,19 @@ +#include + +namespace torch::nn { + +LinearOptions::LinearOptions(int64_t in_features, int64_t out_features) + : in_features_(in_features), out_features_(out_features) {} + +BilinearOptions::BilinearOptions( + int64_t in1_features, + int64_t in2_features, + int64_t out_features) + : in1_features_(in1_features), + in2_features_(in2_features), + out_features_(out_features) {} + +UnflattenOptions::UnflattenOptions(int64_t dim, std::vector sizes) + : dim_(dim), sizes_(std::move(sizes)) {} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/normalization.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/normalization.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e9c439146d363d4c3302e871788405c643ac6291 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/normalization.cpp @@ -0,0 +1,24 @@ +#include + +namespace torch::nn { + +LayerNormOptions::LayerNormOptions(std::vector normalized_shape) + : normalized_shape_(std::move(normalized_shape)) {} + +CrossMapLRN2dOptions::CrossMapLRN2dOptions(int64_t size) : size_(size) {} + +GroupNormOptions::GroupNormOptions(int64_t num_groups, int64_t num_channels) + : num_groups_(num_groups), num_channels_(num_channels) {} + +namespace functional { + +LayerNormFuncOptions::LayerNormFuncOptions( + std::vector normalized_shape) + : normalized_shape_(std::move(normalized_shape)) {} + +GroupNormFuncOptions::GroupNormFuncOptions(int64_t num_groups) + : num_groups_(num_groups) {} + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/padding.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/padding.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ba7dd9ac2850fd69bfe91ddc5c4baab6484c7696 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/padding.cpp @@ -0,0 +1,23 @@ +#include + +namespace torch::nn { + +template struct ReflectionPadOptions<1>; +template struct ReflectionPadOptions<2>; + +template struct ReplicationPadOptions<1>; +template struct ReplicationPadOptions<2>; +template struct ReplicationPadOptions<3>; + +template struct ConstantPadOptions<1>; +template struct ConstantPadOptions<2>; +template struct ConstantPadOptions<3>; + +namespace functional { + +PadFuncOptions::PadFuncOptions(std::vector pad) + : pad_(std::move(pad)) {} + +} // namespace functional + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/pooling.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/pooling.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bf91a8fafaeb3367f01067b1c3772f1449fcd20d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/pooling.cpp @@ -0,0 +1,29 @@ +#include + +namespace torch::nn { + +template struct AvgPoolOptions<1>; +template struct AvgPoolOptions<2>; +template struct AvgPoolOptions<3>; + +template struct MaxPoolOptions<1>; +template struct MaxPoolOptions<2>; +template struct MaxPoolOptions<3>; + +template struct AdaptiveMaxPoolOptions>; +template struct AdaptiveMaxPoolOptions>; +template struct AdaptiveMaxPoolOptions>; + +template struct AdaptiveAvgPoolOptions>; +template struct AdaptiveAvgPoolOptions>; +template struct AdaptiveAvgPoolOptions>; + +template struct MaxUnpoolOptions<1>; +template struct MaxUnpoolOptions<2>; +template struct MaxUnpoolOptions<3>; + +template struct LPPoolOptions<1>; +template struct LPPoolOptions<2>; +template struct LPPoolOptions<3>; + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/rnn.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/rnn.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3f7a11af058a1f4face4f89ecf540b555193f40f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/rnn.cpp @@ -0,0 +1,47 @@ +#include + +namespace torch::nn { + +namespace detail { + +RNNOptionsBase::RNNOptionsBase( + rnn_options_base_mode_t mode, + int64_t input_size, + int64_t hidden_size) + : mode_(mode), input_size_(input_size), hidden_size_(hidden_size) {} + +} // namespace detail + +RNNOptions::RNNOptions(int64_t input_size, int64_t hidden_size) + : input_size_(input_size), hidden_size_(hidden_size) {} + +LSTMOptions::LSTMOptions(int64_t input_size, int64_t hidden_size) + : input_size_(input_size), hidden_size_(hidden_size) {} + +GRUOptions::GRUOptions(int64_t input_size, int64_t hidden_size) + : input_size_(input_size), hidden_size_(hidden_size) {} + +namespace detail { + +RNNCellOptionsBase::RNNCellOptionsBase( + int64_t input_size, + int64_t hidden_size, + bool bias, + int64_t num_chunks) + : input_size_(input_size), + hidden_size_(hidden_size), + bias_(bias), + num_chunks_(num_chunks) {} + +} // namespace detail + +RNNCellOptions::RNNCellOptions(int64_t input_size, int64_t hidden_size) + : input_size_(input_size), hidden_size_(hidden_size) {} + +LSTMCellOptions::LSTMCellOptions(int64_t input_size, int64_t hidden_size) + : input_size_(input_size), hidden_size_(hidden_size) {} + +GRUCellOptions::GRUCellOptions(int64_t input_size, int64_t hidden_size) + : input_size_(input_size), hidden_size_(hidden_size) {} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/transformer.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/transformer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2ca0c06eed9becc9fdb8bcb8296528951326fd29 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/transformer.cpp @@ -0,0 +1,50 @@ +#include +#include +#include + +namespace torch::nn { + +TransformerEncoderLayerOptions::TransformerEncoderLayerOptions( + int64_t d_model, + int64_t nhead) + : d_model_(d_model), nhead_(nhead) {} + +TransformerDecoderLayerOptions::TransformerDecoderLayerOptions( + int64_t d_model, + int64_t nhead) + : d_model_(d_model), nhead_(nhead) {} + +TransformerEncoderOptions::TransformerEncoderOptions( + TransformerEncoderLayer encoder_layer, + int64_t num_layers) + : encoder_layer_(std::move(encoder_layer)), num_layers_(num_layers) {} + +TransformerEncoderOptions::TransformerEncoderOptions( + const TransformerEncoderLayerOptions& encoder_layer_options, + int64_t num_layers) + : encoder_layer_(encoder_layer_options), num_layers_(num_layers) {} + +TransformerDecoderOptions::TransformerDecoderOptions( + TransformerDecoderLayer decoder_layer, + int64_t num_layers) + : decoder_layer_(std::move(decoder_layer)), num_layers_(num_layers) {} + +TransformerDecoderOptions::TransformerDecoderOptions( + const TransformerDecoderLayerOptions& decoder_layer_options, + int64_t num_layers) + : decoder_layer_(decoder_layer_options), num_layers_(num_layers) {} + +TransformerOptions::TransformerOptions(int64_t d_model, int64_t nhead) + : d_model_(d_model), nhead_(nhead) {} + +TransformerOptions::TransformerOptions( + int64_t d_model, + int64_t nhead, + int64_t num_encoder_layers, + int64_t num_decoder_layers) + : d_model_(d_model), + nhead_(nhead), + num_encoder_layers_(num_encoder_layers), + num_decoder_layers_(num_decoder_layers) {} + +} // namespace torch::nn diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/vision.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/vision.cpp new file mode 100644 index 0000000000000000000000000000000000000000..79af2bae4f646f783197fddadc7a8dcce1940204 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/nn/options/vision.cpp @@ -0,0 +1 @@ +#include diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/optim/adagrad.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/optim/adagrad.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fc9f13dea987bdb9ad14b1537d755bcaacc5ae80 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/optim/adagrad.cpp @@ -0,0 +1,150 @@ +#include + +#include +#include + +#include + +#include + +namespace torch::optim { + +AdagradOptions::AdagradOptions(double lr) : lr_(lr) {} + +bool operator==(const AdagradOptions& lhs, const AdagradOptions& rhs) { + return (lhs.lr() == rhs.lr()) && (lhs.lr_decay() == rhs.lr_decay()) && + (lhs.weight_decay() == rhs.weight_decay()) && + (lhs.initial_accumulator_value() == rhs.initial_accumulator_value()) && + (lhs.eps() == rhs.eps()); +} + +void AdagradOptions::serialize(torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(lr); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(lr_decay); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(weight_decay); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(initial_accumulator_value); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(eps); +} + +void AdagradOptions::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, lr); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, lr_decay); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, weight_decay); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, initial_accumulator_value); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, eps); +} + +double AdagradOptions::get_lr() const { + return lr(); +} + +void AdagradOptions::set_lr(const double lr) { + this->lr(lr); +} + +bool operator==(const AdagradParamState& lhs, const AdagradParamState& rhs) { + return (lhs.step() == rhs.step()) && torch::equal(lhs.sum(), rhs.sum()); +} + +void AdagradParamState::serialize( + torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(step); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(sum); +} + +void AdagradParamState::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(int64_t, step); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, sum); +} + +/// Adapted from +/// https://github.com/pytorch/pytorch/blob/master/torch/optim/adagrad.py +Tensor Adagrad::step(LossClosure closure) { + NoGradGuard no_grad; + Tensor loss = {}; + if (closure != nullptr) { + at::AutoGradMode enable_grad(true); + loss = closure(); + } + for (auto& group : param_groups_) { + for (auto& p : group.params()) { + if (!p.grad().defined()) { + continue; + } + auto grad = p.grad(); + TORCH_INTERNAL_ASSERT( + state_[p.unsafeGetTensorImpl()] != nullptr, + "state found NULL for the Tensor ", + p); + auto& state = + static_cast(*state_[p.unsafeGetTensorImpl()]); + auto& options = static_cast(group.options()); + + state.step(state.step() + 1); + + if (options.weight_decay() != 0) { + TORCH_CHECK( + !p.grad().is_sparse(), + "weight_decay option is not compatible with sparse gradients"); + grad = grad.add(p, options.weight_decay()); + } + const auto clr = options.lr() / + (1 + static_cast(state.step() - 1) * options.lr_decay()); + + if (grad.is_sparse()) { + grad = grad.coalesce(); + auto grad_indices = grad._indices(); + auto grad_values = grad._values(); + auto size = grad.sizes(); + + auto make_sparse = [&](const Tensor& values) -> Tensor { + if (grad_indices.dim() == 0 || values.dim() == 0) { + return torch::empty({0}, grad.options()).resize_as_(grad); + } + return torch::sparse_coo_tensor( + grad_indices, values, size, grad.options()); + }; + state.sum(state.sum().add_(make_sparse(grad_values.pow(2)))); + auto std = state.sum().sparse_mask(grad); + const auto std_values = std._values().sqrt_().add_(options.eps()); + + p.add_(make_sparse(grad_values / std_values), -clr); + } else { + state.sum(state.sum().addcmul_(grad, grad, 1.0)); + const auto std = state.sum().sqrt().add_(options.eps()); + p.addcdiv_(grad, std, -clr); + } + } + } + return loss; +} + +void Adagrad::save(serialize::OutputArchive& archive) const { + serialize(*this, archive); +} + +void Adagrad::load(serialize::InputArchive& archive) { + IValue pytorch_version; + if (archive.try_read("pytorch_version", pytorch_version)) { + serialize(*this, archive); + } else { // deserializing archives saved in old format (prior to + // version 1.5.0) + TORCH_WARN( + "Your serialized Adagrad optimizer is still using the old serialization format. " + "You should re-save your Adagrad optimizer to use the new serialization format."); + std::vector sum_buffers; + std::vector step_buffers; + torch::optim::serialize(archive, "sum_buffers", sum_buffers); + torch::optim::serialize(archive, "step_buffers", step_buffers); + // since there were no param_groups prior to version 1.5.0, assuming all + // tensors are now in one param_group + std::vector params = param_groups_.at(0).params(); + for (const auto idx : c10::irange(params.size())) { + auto state = std::make_unique(); + state->step(step_buffers[idx]); + state->sum(sum_buffers[idx]); + state_[params[idx].unsafeGetTensorImpl()] = std::move(state); + } + } +} +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/optim/adam.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/optim/adam.cpp new file mode 100644 index 0000000000000000000000000000000000000000..396b9c5c91705516e75fdabc15760884d260890f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/optim/adam.cpp @@ -0,0 +1,179 @@ +#include + +#include + +#include + +#include +#include + +namespace torch::optim { + +AdamOptions::AdamOptions(double lr) : lr_(lr) {} + +bool operator==(const AdamOptions& lhs, const AdamOptions& rhs) { + return (lhs.lr() == rhs.lr()) && + (std::get<0>(lhs.betas()) == std::get<0>(rhs.betas())) && + (std::get<1>(lhs.betas()) == std::get<1>(rhs.betas())) && + (lhs.eps() == rhs.eps()) && + (lhs.weight_decay() == rhs.weight_decay() && + (lhs.amsgrad() == rhs.amsgrad())); +} + +void AdamOptions::serialize(torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(lr); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(betas); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(eps); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(weight_decay); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(amsgrad); +} + +void AdamOptions::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, lr); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(betas_t, betas); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, eps); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, weight_decay); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(bool, amsgrad); +} + +double AdamOptions::get_lr() const { + return lr(); +} + +void AdamOptions::set_lr(const double lr) { + this->lr(lr); +} + +bool operator==(const AdamParamState& lhs, const AdamParamState& rhs) { + return (lhs.step() == rhs.step()) && + torch::equal(lhs.exp_avg(), rhs.exp_avg()) && + torch::equal(lhs.exp_avg_sq(), rhs.exp_avg_sq()) && + torch::equal_if_defined(lhs.max_exp_avg_sq(), rhs.max_exp_avg_sq()); +} + +void AdamParamState::serialize(torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(step); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(exp_avg); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(exp_avg_sq); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(max_exp_avg_sq); +} + +void AdamParamState::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(int64_t, step); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, exp_avg); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, exp_avg_sq); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, max_exp_avg_sq); +} + +Tensor Adam::step(LossClosure closure) { + NoGradGuard no_grad; + Tensor loss = {}; + if (closure != nullptr) { + at::AutoGradMode enable_grad(true); + loss = closure(); + } + for (auto& group : param_groups_) { + for (auto& p : group.params()) { + if (!p.grad().defined()) { + continue; + } + auto grad = p.grad(); + TORCH_CHECK(!grad.is_sparse(), "Adam does not support sparse gradients" /*, please consider SparseAdam instead*/); + auto param_state = state_.find(p.unsafeGetTensorImpl()); + auto& options = static_cast(group.options()); + + // State initialization + if (param_state == state_.end()) { + auto state = std::make_unique(); + state->step(0); + // Exponential moving average of gradient values + state->exp_avg(torch::zeros_like(p, MemoryFormat::Preserve)); + // Exponential moving average of squared gradient values + state->exp_avg_sq(torch::zeros_like(p, MemoryFormat::Preserve)); + if (options.amsgrad()) { + // Maintains max of all exp. moving avg. of sq. grad. values + state->max_exp_avg_sq(torch::zeros_like(p, MemoryFormat::Preserve)); + } + state_[p.unsafeGetTensorImpl()] = std::move(state); + } + + auto& state = + static_cast(*state_[p.unsafeGetTensorImpl()]); + auto& exp_avg = state.exp_avg(); + auto& exp_avg_sq = state.exp_avg_sq(); + auto& max_exp_avg_sq = state.max_exp_avg_sq(); + + state.step(state.step() + 1); + auto beta1 = std::get<0>(options.betas()); + auto beta2 = std::get<1>(options.betas()); + + auto bias_correction1 = 1 - std::pow(beta1, state.step()); + auto bias_correction2 = 1 - std::pow(beta2, state.step()); + + if (options.weight_decay() != 0) { + grad = grad.add(p, options.weight_decay()); + } + + // Decay the first and second moment running average coefficient + exp_avg.mul_(beta1).add_(grad, 1 - beta1); + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, 1 - beta2); + + Tensor denom; + if (options.amsgrad()) { + // Maintains the maximum of all 2nd moment running avg. till now + torch::max_out(max_exp_avg_sq, exp_avg_sq, max_exp_avg_sq); + // Use the max. for normalizing running avg. of gradient + denom = (max_exp_avg_sq.sqrt() / sqrt(bias_correction2)) + .add_(options.eps()); + } else { + denom = + (exp_avg_sq.sqrt() / sqrt(bias_correction2)).add_(options.eps()); + } + + auto step_size = options.lr() / bias_correction1; + p.addcdiv_(exp_avg, denom, -step_size); + } + } + return loss; +} + +void Adam::save(serialize::OutputArchive& archive) const { + serialize(*this, archive); +} + +void Adam::load(serialize::InputArchive& archive) { + IValue pytorch_version; + if (archive.try_read("pytorch_version", pytorch_version)) { + serialize(*this, archive); + } else { // deserializing archives saved in old format (prior to + // version 1.5.0) + TORCH_WARN( + "Your serialized Adam optimizer is still using the old serialization format. " + "You should re-save your Adam optimizer to use the new serialization format."); + std::vector step_buffers; + std::vector exp_average_buffers; + std::vector exp_average_sq_buffers; + std::vector max_exp_average_sq_buffers; + torch::optim::serialize(archive, "step_buffers", step_buffers); + torch::optim::serialize( + archive, "exp_average_buffers", exp_average_buffers); + torch::optim::serialize( + archive, "exp_average_sq_buffers", exp_average_sq_buffers); + torch::optim::serialize( + archive, "max_exp_average_sq_buffers", max_exp_average_sq_buffers); + // since there were no param_groups prior to version 1.5.0, assuming all + // tensors are now in one param_group + std::vector params = param_groups_.at(0).params(); + for (const auto idx : c10::irange(step_buffers.size())) { + auto state = std::make_unique(); + state->step(step_buffers.at(idx)); + state->exp_avg(exp_average_buffers.at(idx)); + state->exp_avg_sq(exp_average_sq_buffers.at(idx)); + if (idx < max_exp_average_sq_buffers.size()) { + state->max_exp_avg_sq(max_exp_average_sq_buffers.at(idx)); + } + state_[params.at(idx).unsafeGetTensorImpl()] = std::move(state); + } + } +} +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/optim/adamw.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/optim/adamw.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d1d6f4fe8cedc6abb1493a3adc6c4d4bfd03843a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/optim/adamw.cpp @@ -0,0 +1,180 @@ +#include + +#include + +#include + +#include +#include + +namespace torch::optim { + +AdamWOptions::AdamWOptions(double lr) : lr_(lr) {} + +bool operator==(const AdamWOptions& lhs, const AdamWOptions& rhs) { + return (lhs.lr() == rhs.lr()) && + (std::get<0>(lhs.betas()) == std::get<0>(rhs.betas())) && + (std::get<1>(lhs.betas()) == std::get<1>(rhs.betas())) && + (lhs.eps() == rhs.eps()) && (lhs.weight_decay() == rhs.weight_decay()) && + (lhs.amsgrad() == rhs.amsgrad()); +} + +void AdamWOptions::serialize(torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(lr); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(betas); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(eps); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(weight_decay); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(amsgrad); +} + +void AdamWOptions::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, lr); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(betas_t, betas); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, eps); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, weight_decay); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(bool, amsgrad); +} + +double AdamWOptions::get_lr() const { + return lr(); +} + +void AdamWOptions::set_lr(const double lr) { + this->lr(lr); +} + +bool operator==(const AdamWParamState& lhs, const AdamWParamState& rhs) { + return (lhs.step() == rhs.step()) && + torch::equal(lhs.exp_avg(), rhs.exp_avg()) && + torch::equal(lhs.exp_avg_sq(), rhs.exp_avg_sq()) && + torch::equal_if_defined(lhs.max_exp_avg_sq(), rhs.max_exp_avg_sq()); +} + +void AdamWParamState::serialize( + torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(step); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(exp_avg); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(exp_avg_sq); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(max_exp_avg_sq); +} + +void AdamWParamState::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(int64_t, step); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, exp_avg); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, exp_avg_sq); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, max_exp_avg_sq); +} + +Tensor AdamW::step(LossClosure closure) { + NoGradGuard no_grad; + Tensor loss = {}; + if (closure != nullptr) { + at::AutoGradMode enable_grad(true); + loss = closure(); + } + for (auto& group : param_groups_) { + for (auto& p : group.params()) { + if (!p.grad().defined()) { + continue; + } + const auto& grad = p.grad(); + TORCH_CHECK(!grad.is_sparse(), "AdamW does not support sparse gradients" /*, please consider SparseAdamW instead*/); + auto param_state = state_.find(p.unsafeGetTensorImpl()); + auto& options = static_cast(group.options()); + + // Perform step weight decay + if (options.weight_decay() != 0) { + p.mul_(1 - options.lr() * options.weight_decay()); + } + + // State initialization + if (param_state == state_.end()) { + auto state = std::make_unique(); + state->step(0); + // Exponential moving average of gradient values + state->exp_avg(torch::zeros_like(p, MemoryFormat::Preserve)); + // Exponential moving average of squared gradient values + state->exp_avg_sq(torch::zeros_like(p, MemoryFormat::Preserve)); + if (options.amsgrad()) { + // Maintains max of all exp. moving avg. of sq. grad. values + state->max_exp_avg_sq(torch::zeros_like(p, MemoryFormat::Preserve)); + } + state_[p.unsafeGetTensorImpl()] = std::move(state); + } + + auto& state = + static_cast(*state_[p.unsafeGetTensorImpl()]); + auto& exp_avg = state.exp_avg(); + auto& exp_avg_sq = state.exp_avg_sq(); + auto& max_exp_avg_sq = state.max_exp_avg_sq(); + + state.step(state.step() + 1); + auto beta1 = std::get<0>(options.betas()); + auto beta2 = std::get<1>(options.betas()); + + auto bias_correction1 = 1 - std::pow(beta1, state.step()); + auto bias_correction2 = 1 - std::pow(beta2, state.step()); + + // Decay the first and second moment running average coefficient + exp_avg.mul_(beta1).add_(grad, 1 - beta1); + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, 1 - beta2); + + Tensor denom; + if (options.amsgrad()) { + // Maintains the maximum of all 2nd moment running avg. till now + torch::max_out(max_exp_avg_sq, exp_avg_sq, max_exp_avg_sq); + // Use the max. for normalizing running avg. of gradient + denom = (max_exp_avg_sq.sqrt() / sqrt(bias_correction2)) + .add_(options.eps()); + } else { + denom = + (exp_avg_sq.sqrt() / sqrt(bias_correction2)).add_(options.eps()); + } + + auto step_size = options.lr() / bias_correction1; + p.addcdiv_(exp_avg, denom, -step_size); + } + } + return loss; +} + +void AdamW::save(serialize::OutputArchive& archive) const { + serialize(*this, archive); +} + +void AdamW::load(serialize::InputArchive& archive) { + IValue pytorch_version; + if (archive.try_read("pytorch_version", pytorch_version)) { + serialize(*this, archive); + } else { // deserializing archives saved in old format (prior to + // version 1.5.0) + TORCH_WARN( + "Your serialized AdamW optimizer is still using the old serialization format. " + "You should re-save your AdamW optimizer to use the new serialization format."); + std::vector step_buffers; + std::vector exp_average_buffers; + std::vector exp_average_sq_buffers; + std::vector max_exp_average_sq_buffers; + torch::optim::serialize(archive, "step_buffers", step_buffers); + torch::optim::serialize( + archive, "exp_average_buffers", exp_average_buffers); + torch::optim::serialize( + archive, "exp_average_sq_buffers", exp_average_sq_buffers); + torch::optim::serialize( + archive, "max_exp_average_sq_buffers", max_exp_average_sq_buffers); + // since there were no param_groups prior to version 1.5.0, assuming all + // tensors are now in one param_group + std::vector params = param_groups_.at(0).params(); + for (const auto idx : c10::irange(step_buffers.size())) { + auto state = std::make_unique(); + state->step(step_buffers.at(idx)); + state->exp_avg(exp_average_buffers.at(idx)); + state->exp_avg_sq(exp_average_sq_buffers.at(idx)); + if (idx < max_exp_average_sq_buffers.size()) { + state->max_exp_avg_sq(max_exp_average_sq_buffers.at(idx)); + } + state_[params.at(idx).unsafeGetTensorImpl()] = std::move(state); + } + } +} +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/optim/lbfgs.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/optim/lbfgs.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b7d3f5353d34962214945a5146279184f4d95f7a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/optim/lbfgs.cpp @@ -0,0 +1,642 @@ +#include + +#include +#include + +#include + +#include +#include +#include +#include + +namespace torch::optim { + +LBFGSOptions::LBFGSOptions(double lr) : lr_(lr) {} + +bool operator==(const LBFGSOptions& lhs, const LBFGSOptions& rhs) { + return (lhs.lr() == rhs.lr()) && (lhs.max_iter() == rhs.max_iter()) && + (lhs.max_eval() == rhs.max_eval()) && + (lhs.tolerance_grad() == rhs.tolerance_grad()) && + (lhs.tolerance_change() == rhs.tolerance_change() && + (lhs.history_size() == rhs.history_size())) && + (lhs.line_search_fn() == rhs.line_search_fn()); +} + +void LBFGSOptions::serialize(torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(lr); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(max_iter); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(max_eval); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(tolerance_grad); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(tolerance_change); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(history_size); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(line_search_fn); +} + +void LBFGSOptions::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, lr); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(int64_t, max_iter); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG_OPTIONAL(int64_t, max_eval); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, tolerance_grad); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, tolerance_change); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(int64_t, history_size); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG_OPTIONAL(std::string, line_search_fn); +} + +double LBFGSOptions::get_lr() const { + return lr(); +} + +void LBFGSOptions::set_lr(const double lr) { + this->lr(lr); +} + +template +static bool if_container_equal(T lhs, T rhs) { + if (!(lhs.size() == rhs.size())) + return false; + for (const auto i : c10::irange(lhs.size())) { + if (!torch::equal(lhs.at(i), rhs.at(i))) + return false; + } + return true; +} + +bool operator==(const LBFGSParamState& lhs, const LBFGSParamState& rhs) { + auto isNull = [](const std::optional>& val) { + return val == std::nullopt; + }; + return (lhs.func_evals() == rhs.func_evals()) && + (lhs.n_iter() == rhs.n_iter()) && (lhs.t() == rhs.t()) && + (lhs.prev_loss() == rhs.prev_loss()) && + torch::equal_if_defined(lhs.d(), rhs.d()) && + torch::equal_if_defined(lhs.H_diag(), rhs.H_diag()) && + torch::equal_if_defined(lhs.prev_flat_grad(), rhs.prev_flat_grad()) && + if_container_equal(lhs.old_dirs(), rhs.old_dirs()) && + if_container_equal(lhs.old_stps(), rhs.old_stps()) && + if_container_equal(lhs.ro(), rhs.ro()) && + ((isNull(lhs.al()) && isNull(rhs.al())) || + (!isNull(lhs.al()) && !isNull(rhs.al()) && + if_container_equal(*lhs.al(), *rhs.al()))); +} + +void LBFGSParamState::serialize( + torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(func_evals); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(n_iter); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(t); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(prev_loss); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(d); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(H_diag); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(prev_flat_grad); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG_DEQUE(old_dirs); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG_DEQUE(old_stps); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG_DEQUE(ro); + // Python version only serializes state vars if explicitly defined + if (al().has_value()) { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(al); + } +} + +void LBFGSParamState::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(int64_t, func_evals); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(int64_t, n_iter); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, t); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, prev_loss); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, d); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, H_diag); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, prev_flat_grad); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG_DEQUE(std::deque, old_dirs); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG_DEQUE(std::deque, old_stps); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG_DEQUE(std::deque, ro); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG_OPTIONAL(std::vector, al); +} + +Tensor LBFGS::_gather_flat_grad() { + std::vector views; + for (const auto& p : param_groups_.at(0).params()) { + if (!p.grad().defined()) { + views.emplace_back(p.new_empty({p.numel()}).zero_()); + } else if (p.grad().is_sparse()) { + views.emplace_back(p.grad().to_dense().view(-1)); + } else { + views.emplace_back(p.grad().view(-1)); + } + } + return torch::cat(views, 0); +} + +int64_t LBFGS::_numel() { + if (_numel_cache == std::nullopt) { + int64_t res = 0; + for (const auto& p : param_groups_.at(0).params()) { + res += p.numel(); + } + _numel_cache = res; + } + return *_numel_cache; +} + +void LBFGS::_add_grad(const double step_size, const Tensor& update) { + int64_t offset = 0; + for (auto& p : param_groups_.at(0).params()) { + auto numel = p.numel(); + // view as to avoid deprecated pointwise semantics + p.add_( + update.index({at::indexing::Slice(offset, offset + numel)}).view_as(p), + step_size); + offset += numel; + } + TORCH_INTERNAL_ASSERT(offset == _numel()); +} + +void LBFGS::_set_param(const std::vector& params_data) { + auto& _params = param_groups_.at(0).params(); + TORCH_INTERNAL_ASSERT(params_data.size() == _params.size()); + for (const auto i : c10::irange(_params.size())) { + _params.at(i).copy_(params_data.at(i)); + } +} + +std::vector LBFGS::_clone_param() { + std::vector result; + for (const auto& p : param_groups_.at(0).params()) { + result.emplace_back(p.clone(at::MemoryFormat::Contiguous)); + } + return result; +} + +std::tuple LBFGS::_directional_evaluate( + const LossClosure& closure, + const std::vector& x, + double t, + const Tensor& d) { + _add_grad(t, d); + double loss = 0; + { + torch::AutoGradMode enable_grad(true); + loss = closure().item(); + } + auto flat_grad = _gather_flat_grad(); + _set_param(x); + return std::make_tuple(loss, std::move(flat_grad)); +} + +static double _cubic_interpolate( + double x1, + double f1, + double g1, + double x2, + double f2, + double g2, + std::optional> bounds = std::nullopt) { + // ported from https://github.com/torch/optim/blob/master/polyinterp.lua + // Compute bounds of interpolation area + auto [xmin_bound, xmax_bound] = + (bounds.has_value()) ? (*bounds) : std::minmax({x1, x2}); + // Code for most common case: cubic interpolation of 2 points + // w/ function and derivative values for both + // Solution in this case (where x2 is the farthest point): + // d1 = g1 + g2 - 3*(f1-f2)/(x1-x2); + // d2 = sqrt(d1^2 - g1*g2); + // min_pos = x2 - (x2 - x1)*((g2 + d2 - d1)/(g2 - g1 + 2*d2)); + // t_new = min(max(min_pos,xmin_bound),xmax_bound); + + auto d1 = (g1 + g2) - (3 * (f1 - f2) / (x1 - x2)); + auto d2_square = std::pow(d1, 2) - g1 * g2; + if (d2_square >= 0) { + auto d2 = std::sqrt(d2_square); + double min_pos = 0; + if (x1 <= x2) { + min_pos = x2 - ((x2 - x1) * ((g2 + d2 - d1) / (g2 - g1 + 2 * d2))); + } else { + min_pos = x1 - ((x1 - x2) * ((g1 + d2 - d1) / (g1 - g2 + 2 * d2))); + } + return std::clamp(min_pos, xmin_bound, xmax_bound); + } else { + return (xmin_bound + xmax_bound) / 2; + } +} + +using Function = std::function( + const std::vector& x, + double t, + const Tensor& d)>; +static std::tuple _strong_wolfe( + const Function& obj_func, + const std::vector& x, + double t, + const Tensor& d, + double f, + Tensor g, + const Tensor& gtd, + double c1 = 1e-4, + double c2 = 0.9, // // NOLINT(cppcoreguidelines-avoid-magic-numbers) + double tolerance_change = 1e-9, + double max_ls = 25) { // NOLINT(cppcoreguidelines-avoid-magic-numbers) + + auto val = [](const Tensor& t) { return t.item(); }; + + auto d_norm = val(d.abs().max()); + g = g.clone(at::MemoryFormat::Contiguous); + // evaluate objective and gradient using initial step + auto [f_new, g_new] = obj_func(x, t, d); + int64_t ls_func_evals = 1; + auto gtd_new = g_new.dot(d); + + // bracket an interval containing a point satisfying the Wolfe criteria + double t_prev = 0; + auto f_prev = f; + auto g_prev = g; + auto gtd_prev = gtd; + bool done = false; + auto ls_iter = 0; + std::vector bracket, bracket_f; + std::vector bracket_g, bracket_gtd; + + while (ls_iter < max_ls) { + // check conditions + if ((f_new > (f + c1 * t * val(gtd))) || + (ls_iter > 1 && (f_new >= f_prev))) { + bracket = {t_prev, t}; + bracket_f = {f_prev, f_new}; + bracket_g = {g_prev, g_new.clone(at::MemoryFormat::Contiguous)}; + bracket_gtd = {gtd_prev, gtd_new}; + break; + } + if (std::abs(val(gtd_new)) <= (-c2 * val(gtd))) { + bracket = {t, t}; + bracket_f = {f_new, f_new}; + bracket_g = {g_new, g_new}; + done = true; + break; + } + if (val(gtd_new) >= 0) { + bracket = {t_prev, t}; + bracket_f = {f_prev, f_new}; + bracket_g = {g_prev, g_new.clone(at::MemoryFormat::Contiguous)}; + bracket_gtd = {gtd_prev, gtd_new}; + break; + } + // interpolate + auto min_step = t + + 0.01 * (t - t_prev); // NOLINT(cppcoreguidelines-avoid-magic-numbers) + auto max_step = t * 10; // NOLINT(cppcoreguidelines-avoid-magic-numbers) + auto tmp = t; + t = _cubic_interpolate( + t_prev, + f_prev, + val(gtd_prev), + t, + f_new, + val(gtd_new), + std::make_pair(min_step, max_step)); + // next step + t_prev = tmp; + f_prev = f_new; + g_prev = g_new.clone(at::MemoryFormat::Contiguous); + gtd_prev = gtd_new; + std::tie(f_new, g_new) = obj_func(x, t, d); + ls_func_evals += 1; + gtd_new = g_new.dot(d); + ls_iter += 1; + } + // reached max number of iterations? + if (ls_iter == max_ls) { + bracket = {0, t}; + bracket_f = {f, f_new}; + bracket_g = {g, g_new}; + } + + // zoom phase: we now have a point satisfying the criteria, or + // a bracket around it. We refine the bracket until we find the + // exact point satisfying the criteria + bool insuf_progress = false; + // find high and low points in bracket + auto [low_pos, high_pos] = bracket_f[0] <= bracket_f[1] + ? std::make_tuple(0, 1) + : std::make_tuple(1, 0); + while (!done && (ls_iter < max_ls)) { + // compute new trial value + t = _cubic_interpolate( + bracket[0], + bracket_f[0], + val(bracket_gtd[0]), + bracket[1], + bracket_f[1], + val(bracket_gtd[1])); + + // test that we are making sufficient progress: + // in case `t` is so close to boundary, we mark that we are making + // insufficient progress, and if + // + we have made insufficient progress in the last step, or + // + `t` is at one of the boundary, + // we will move `t` to a position which is `0.1 * len(bracket)` + // away from the nearest boundary point. + double bracket_max = std::max(bracket[0], bracket[1]); + auto bracket_min = std::min(bracket[0], bracket[1]); + auto eps = 0.1 * + (bracket_max - + bracket_min); // // NOLINT(cppcoreguidelines-avoid-magic-numbers) + if (std::min(bracket_max - t, t - bracket_min) < eps) { + // interpolation close to boundary + if (insuf_progress || (t >= bracket_max) || (t <= bracket_min)) { + // evaluate at 0.1 away from boundary + t = (std::abs(t - bracket_max) < std::abs(t - bracket_min)) + ? bracket_max - eps + : bracket_min + eps; + insuf_progress = false; + } else { + insuf_progress = true; + } + } else { + insuf_progress = false; + } + + // Evaluate new point + std::tie(f_new, g_new) = obj_func(x, t, d); + ls_func_evals += 1; + gtd_new = g_new.dot(d); + ls_iter += 1; + + if ((f_new > (f + c1 * t * val(gtd))) || (f_new >= bracket_f[low_pos])) { + // Armijo condition not satisfied or not lower than lowest point + // # Armijo condition not satisfied or not lower than lowest point + bracket[high_pos] = t; + bracket_f[high_pos] = f_new; + bracket_g[high_pos] = g_new.clone(at::MemoryFormat::Contiguous); + bracket_gtd[high_pos] = gtd_new; + std::tie(low_pos, high_pos) = bracket_f[0] <= bracket_f[1] + ? std::make_tuple(0, 1) + : std::make_tuple(1, 0); + } else { + if (val(at::abs(gtd_new)) <= (-c2 * val(gtd))) { + // Wolfe conditions satisfied + done = true; + } else if ((val(gtd_new) * (bracket[high_pos] - bracket[low_pos])) >= 0) { + // old high becomes new low + bracket[high_pos] = bracket[low_pos]; + bracket_f[high_pos] = bracket_f[low_pos]; + bracket_g[high_pos] = bracket_g[low_pos]; + bracket_gtd[high_pos] = bracket_gtd[low_pos]; + } + + // new point becomes new low + bracket[low_pos] = t; + bracket_f[low_pos] = f_new; + bracket_g[low_pos] = g_new.clone(at::MemoryFormat::Contiguous); + bracket_gtd[low_pos] = gtd_new; + } + + // line-search bracket is so small + if ((std::abs(bracket[1] - bracket[0]) * d_norm) < tolerance_change) + break; + } + + // return stuff + t = bracket[low_pos]; + f_new = bracket_f[low_pos]; + g_new = bracket_g[low_pos]; + return std::make_tuple(f_new, g_new, t, ls_func_evals); +} + +Tensor LBFGS::step(LossClosure closure) { + NoGradGuard no_grad; + TORCH_CHECK(closure != nullptr, "LBFGS requires a closure function"); + TORCH_INTERNAL_ASSERT(param_groups_.size() == 1); + auto val = [](const Tensor& t) { return t.item(); }; + + auto& group = param_groups_.at(0); + auto& _params = group.params(); + const auto& options = static_cast(group.options()); + auto lr = options.lr(); + auto max_iter = options.max_iter(); + auto max_eval = options.max_eval(); + auto tolerance_grad = options.tolerance_grad(); + auto tolerance_change = options.tolerance_change(); + auto line_search_fn = options.line_search_fn(); + auto history_size = options.history_size(); + + // NOTE: LBFGS has only global state, but we register it as state for + // the first param, because this helps with casting in load_state_dict + auto param_state = state_.find(_params.at(0).unsafeGetTensorImpl()); + if (param_state == state_.end()) { + state_[_params.at(0).unsafeGetTensorImpl()] = + std::make_unique(); + } + auto& state = static_cast( + *state_[_params.at(0).unsafeGetTensorImpl()]); + // evaluate initial f(x) and df/dx + Tensor orig_loss; + { + torch::AutoGradMode enable_grad(true); + orig_loss = closure(); + } + + auto loss = val(orig_loss); + auto current_evals = 1; + state.func_evals(state.func_evals() + 1); + auto flat_grad = _gather_flat_grad(); + auto opt_cond = (val(flat_grad.abs().max()) <= tolerance_grad); + + // optimal condition + if (opt_cond) { + return orig_loss; + } + + // tensors cached in state (for tracing) + auto& d = state.d(); + auto& t = state.t(); + auto& old_dirs = state.old_dirs(); + auto& old_stps = state.old_stps(); + auto& ro = state.ro(); + auto& H_diag = state.H_diag(); + auto& prev_flat_grad = state.prev_flat_grad(); + auto& prev_loss = state.prev_loss(); + + int n_iter = 0; + + // optimize for a max of max_iter iterations + while (n_iter < max_iter) { + // keep track of nb of iterations + n_iter += 1; + state.n_iter(state.n_iter() + 1); + + // compute gradient descent direction + if (state.n_iter() == 1) { + d = flat_grad.neg(); + H_diag = torch::tensor(1); + old_dirs = {}; + old_stps = {}; + ro = {}; + } else { + // do lbfgs update (update memory) + auto y = flat_grad.sub(prev_flat_grad); + auto s = d.mul(t); + auto ys = y.dot(s); // y*s + if (val(ys) > 1e-10) { // NOLINT(cppcoreguidelines-avoid-magic-numbers) + // updating memory + if (static_cast(old_dirs.size()) == history_size) { + // shift history by one (limited-memory) + old_dirs.pop_front(); + old_stps.pop_front(); + ro.pop_front(); + } + // store new direction/step + old_dirs.emplace_back(y); + old_stps.emplace_back(s); + ro.emplace_back(1. / ys); + + // update scale of initial Hessian approximation + H_diag = ys / y.dot(y); // (y*y) + } + + // compute the approximate (L-BFGS) inverse Hessian + // multiplied by the gradient + int64_t num_old = static_cast(old_dirs.size()); + + if (state.al() == std::nullopt) { + state.al(std::vector(history_size)); + } + auto& al = state.al(); + + // iteration in L-BFGS loop collapsed to use just one buffer + auto q = flat_grad.neg(); + for (int64_t i = num_old - 1; i > -1; i--) { + (*al).at(i) = old_stps.at(i).dot(q) * ro.at(i); + q.add_(old_dirs.at(i), -val((*al).at(i))); + } + + // multiply by initial Hessian + // r/d is the final direction + auto r = torch::mul(q, H_diag); + d = r; + for (const auto i : c10::irange(num_old)) { + auto be_i = old_dirs.at(i).dot(r) * ro.at(i); + r.add_(old_stps.at(i), val((*al).at(i) - be_i)); + } + } + + if (!prev_flat_grad.defined()) { + prev_flat_grad = flat_grad.clone(at::MemoryFormat::Contiguous); + } else { + prev_flat_grad.copy_(flat_grad); + } + prev_loss = loss; + + // ############################################################ + // # compute step length + // ############################################################ + // reset initial guess for step size + if (state.n_iter() == 1) { + t = std::min(1., 1. / val(flat_grad.abs().sum())) * lr; + } else { + t = lr; + } + + // directional derivative + auto gtd = flat_grad.dot(d); // g * d + + // directional derivative is below tolerance + if (val(gtd) > -tolerance_change) + break; + + // optional line search: user function + auto ls_func_evals = 0; + if (line_search_fn.has_value()) { + TORCH_CHECK( + *line_search_fn == "strong_wolfe", + "only 'strong_wolfe' is supported"); + auto x_init = _clone_param(); + auto obj_func = + [&](const std::vector& x, double t, const Tensor& d) { + return _directional_evaluate(closure, x, t, d); + }; + std::tie(loss, flat_grad, t, ls_func_evals) = + _strong_wolfe(obj_func, x_init, t, d, loss, flat_grad, gtd); + _add_grad(t, d); + opt_cond = (val(flat_grad.abs().max()) <= tolerance_grad); + } else { + // no line search, simply move with fixed-step + _add_grad(t, d); + if (n_iter != max_iter) { + // re-evaluate function only if not in last iteration + // the reason we do this: in a stochastic setting, + // no use to re-evaluate that function here + { + torch::AutoGradMode enable_grad(true); + loss = val(closure()); + } + flat_grad = _gather_flat_grad(); + opt_cond = val(torch::max(flat_grad.abs())) <= tolerance_grad; + ls_func_evals = 1; + } + } + // update func eval + current_evals += ls_func_evals; + state.func_evals(state.func_evals() + ls_func_evals); + + // ############################################################ + // # check conditions + // ############################################################ + if (n_iter == max_iter) + break; + + if (current_evals >= *max_eval) + break; + + // optimal condition + if (opt_cond) + break; + + // lack of progress + if (val(d.mul(t).abs().max()) <= tolerance_change) + break; + + if (std::abs(loss - prev_loss) < tolerance_change) + break; + } + + return orig_loss; +} + +void LBFGS::save(serialize::OutputArchive& archive) const { + serialize(*this, archive); +} + +void LBFGS::load(serialize::InputArchive& archive) { + IValue pytorch_version; + if (archive.try_read("pytorch_version", pytorch_version)) { + serialize(*this, archive); + } else { // deserializing archives saved in old format (prior to + // version 1.5.0) + TORCH_WARN( + "Your serialized LBFGS optimizer is still using the old serialization format. " + "The func_evals and n_iter value in state will be set to 0, ro will be set to an empty deque " + "and al will be set to std::nullopt because the old LBFGS optimizer didn't save these values." + "You should re-save your LBFGS optimizer to use the new serialization format."); + Tensor d, t, H_diag, prev_flat_grad, prev_loss; + std::deque old_dirs, old_stps; + archive("d", d, /*is_buffer=*/true); + archive("t", t, /*is_buffer=*/true); + archive("H_diag", H_diag, /*is_buffer=*/true); + archive("prev_flat_grad", prev_flat_grad, /*is_buffer=*/true); + archive("prev_loss", prev_loss, /*is_buffer=*/true); + torch::optim::serialize(archive, "old_dirs", old_dirs); + torch::optim::serialize(archive, "old_stps", old_stps); + + // NOTE: LBFGS has only global state, but we register it as state for + // the first param, because this helps with casting in load_state_dict + auto state = std::make_unique(); + state->d(d); + state->t(t.item()); + state->H_diag(H_diag); + state->prev_flat_grad(prev_flat_grad); + state->prev_loss(prev_loss.item()); + state->old_dirs(old_dirs); + state->old_stps(old_stps); + state_[param_groups_.at(0).params().at(0).unsafeGetTensorImpl()] = + std::move(state); + } +} +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/optim/optimizer.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/optim/optimizer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..51e392346418c1ef5d9de88414764d1330b2d8db --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/optim/optimizer.cpp @@ -0,0 +1,393 @@ +#include + +#include +#include + +// Include complete type definitions for all optimizers to enable dynamic_cast +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace torch::optim { + +// Implementation of OptimizerCloneableOptions::_merge_by_comparison +// Moved here to anchor vtable/typeinfo for template instantiations +template +void OptimizerCloneableOptions::_merge_by_comparison( + const Derived& defaults, + const Derived& user_options) { + auto* result = static_cast(this); + *result = defaults; // Start with optimizer defaults + + // Create constructor defaults instance for comparison + Derived constructor_defaults = []() { + if constexpr (std::is_default_constructible_v) { + return Derived{}; + } else { + // Handle optimizers requiring constructor parameters + if constexpr (std::is_same_v) { + return Derived(1e-3); + } else if constexpr (std::is_same_v) { + return Derived(1e-2); + } else if constexpr (std::is_same_v) { + return Derived(1e-2); + } else if constexpr (std::is_same_v) { + return Derived(1); + } else { + return Derived{}; + } + } + }(); + + // Merge fields: preserve user-set values, inherit defaults for unset values + + if constexpr (OptimizerCloneableOptions::_has_lr::value) { + if (user_options.lr() != constructor_defaults.lr()) { + result->lr(user_options.lr()); + } + } + if constexpr (OptimizerCloneableOptions::_has_momentum< + Derived>::value) { + if (user_options.momentum() != constructor_defaults.momentum()) { + result->momentum(user_options.momentum()); + } + } + if constexpr (OptimizerCloneableOptions::_has_weight_decay< + Derived>::value) { + if (user_options.weight_decay() != constructor_defaults.weight_decay()) { + result->weight_decay(user_options.weight_decay()); + } + } + if constexpr (OptimizerCloneableOptions::_has_dampening< + Derived>::value) { + if (user_options.dampening() != constructor_defaults.dampening()) { + result->dampening(user_options.dampening()); + } + } + if constexpr (OptimizerCloneableOptions::_has_nesterov< + Derived>::value) { + if (user_options.nesterov() != constructor_defaults.nesterov()) { + result->nesterov(user_options.nesterov()); + } + } + if constexpr (OptimizerCloneableOptions::_has_betas< + Derived>::value) { + if (user_options.betas() != constructor_defaults.betas()) { + result->betas(user_options.betas()); + } + } + if constexpr (OptimizerCloneableOptions::_has_eps::value) { + if (user_options.eps() != constructor_defaults.eps()) { + result->eps(user_options.eps()); + } + } + if constexpr (OptimizerCloneableOptions::_has_amsgrad< + Derived>::value) { + if (user_options.amsgrad() != constructor_defaults.amsgrad()) { + result->amsgrad(user_options.amsgrad()); + } + } + + // Optimizer-specific fields - automatically detected and handled + if constexpr (OptimizerCloneableOptions::_has_lr_decay< + Derived>::value) { + if (user_options.lr_decay() != constructor_defaults.lr_decay()) { + result->lr_decay(user_options.lr_decay()); + } + } + if constexpr (OptimizerCloneableOptions::_has_alpha< + Derived>::value) { + if (user_options.alpha() != constructor_defaults.alpha()) { + result->alpha(user_options.alpha()); + } + } + if constexpr (OptimizerCloneableOptions::_has_centered< + Derived>::value) { + if (user_options.centered() != constructor_defaults.centered()) { + result->centered(user_options.centered()); + } + } + if constexpr (OptimizerCloneableOptions< + Derived>::_has_initial_accumulator_value::value) { + if (user_options.initial_accumulator_value() != + constructor_defaults.initial_accumulator_value()) { + result->initial_accumulator_value( + user_options.initial_accumulator_value()); + } + } + + // LBFGS-specific fields with appropriate types + if constexpr (OptimizerCloneableOptions::_has_max_iter< + Derived>::value) { + if (user_options.max_iter() != constructor_defaults.max_iter()) { + result->max_iter(user_options.max_iter()); + } + } + if constexpr (OptimizerCloneableOptions::_has_max_eval< + Derived>::value) { + if (user_options.max_eval() != constructor_defaults.max_eval()) { + result->max_eval(user_options.max_eval()); + } + } + if constexpr (OptimizerCloneableOptions::_has_tolerance_grad< + Derived>::value) { + if (user_options.tolerance_grad() != + constructor_defaults.tolerance_grad()) { + result->tolerance_grad(user_options.tolerance_grad()); + } + } + if constexpr (OptimizerCloneableOptions::_has_tolerance_change< + Derived>::value) { + if (user_options.tolerance_change() != + constructor_defaults.tolerance_change()) { + result->tolerance_change(user_options.tolerance_change()); + } + } + if constexpr (OptimizerCloneableOptions::_has_history_size< + Derived>::value) { + if (user_options.history_size() != constructor_defaults.history_size()) { + result->history_size(user_options.history_size()); + } + } + if constexpr (OptimizerCloneableOptions::_has_line_search_fn< + Derived>::value) { + if (user_options.line_search_fn() != + constructor_defaults.line_search_fn()) { + result->line_search_fn(user_options.line_search_fn()); + } + } +} + +// Explicit template instantiations to anchor vtable/typeinfo +// These instantiations ensure the compiler generates the full class definition +// and vtable for each OptimizerCloneableOptions specialization +template class OptimizerCloneableOptions; +template class OptimizerCloneableOptions; +template class OptimizerCloneableOptions; +template class OptimizerCloneableOptions; +template class OptimizerCloneableOptions; +template class OptimizerCloneableOptions; + +// Simple implementation using variadic template helper +void Optimizer::_try_merge_all_optimizers( + std::unique_ptr& final_options, + const OptimizerOptions& user_options, + const OptimizerOptions& defaults) { + // Clean one-liner replaces the entire repetitive dispatch chain + _try_merge_all_optimizer_types< + SGDOptions, + AdamOptions, + AdamWOptions, + AdagradOptions, + RMSpropOptions, + LBFGSOptions>(final_options, user_options, defaults); +} + +bool OptimizerParamGroup::has_options() const { + return options_ != nullptr; +} + +OptimizerOptions& OptimizerParamGroup::options() { + TORCH_CHECK(has_options()); + return *options_; +} + +const OptimizerOptions& OptimizerParamGroup::options() const { + TORCH_CHECK(has_options()); + return *options_; +} + +void OptimizerParamGroup::set_options( + std::unique_ptr options) { + options_ = std::move(options); +} + +std::vector& OptimizerParamGroup::params() { + return params_; +} + +const std::vector& OptimizerParamGroup::params() const { + return params_; +} + +std::unique_ptr OptimizerParamState::clone() const { + TORCH_CHECK( + false, + "clone() has not been implemented for torch::optim::OptimizerParamState. ", + "Subclass torch::optim::OptimizerCloneableParamState ", + "instead of torch::optim::OptimizerParamState to inherit the ability to clone."); +} + +void OptimizerParamState::serialize(torch::serialize::InputArchive& archive) { + TORCH_CHECK( + false, + "void serialize(torch::serialize::InputArchive& archive) has not been implemented for torch::optim::OptimizerParamState. ", + "You must override it in your subclass of torch::optim::OptimizerCloneableParamState."); +} + +void OptimizerParamState::serialize( + torch::serialize::OutputArchive& archive) const { + TORCH_CHECK( + false, + "void serialize(torch::serialize::OutputArchive& archive) has not been implemented for torch::optim::OptimizerParamState. ", + "You must override it in your subclass of torch::optim::OptimizerCloneableParamState."); +} + +double OptimizerOptions::get_lr() const { + TORCH_CHECK( + false, + "double get_lr() has not been overridden and implemented in subclass of torch::optim::OptimizerOptions, you must override it in your subclass."); +} + +void OptimizerOptions::set_lr(const double lr) { + TORCH_CHECK( + false, + "double set_lr() has not been overridden and implemented in subclass of torch::optim::OptimizerOptions, you must override it in your subclass."); +} + +std::unique_ptr OptimizerOptions::clone() const { + TORCH_CHECK( + false, + "clone() has not been implemented for torch::optim::OptimizerOptions. ", + "Subclass torch::optim::OptimizerCloneableOptions ", + "instead of torch::optim::OptimizerOptions to inherit the ability to clone."); +} + +void OptimizerOptions::serialize(torch::serialize::InputArchive& archive) { + TORCH_CHECK( + false, + "void serialize(torch::serialize::InputArchive& archive) has not been implemented for torch::optim::OptimizerOptions. ", + "You must override it in your subclass of torch::optim::OptimizerCloneableOptions."); +} + +void OptimizerOptions::serialize( + torch::serialize::OutputArchive& archive) const { + TORCH_CHECK( + false, + "void serialize(torch::serialize::OutputArchive& archive) has not been implemented for torch::optim::OptimizerOptions. ", + "You must override it in your subclass of torch::optim::OptimizerCloneableOptions."); +} + +void Optimizer::add_param_group(const OptimizerParamGroup& param_group) { + for (const auto& param : param_group.params()) { + TORCH_CHECK(param.is_leaf(), "can't optimize a non-leaf Tensor"); + } + TORCH_INTERNAL_ASSERT(defaults_ != nullptr); + OptimizerParamGroup param_group_(param_group.params()); + if (!param_group.has_options()) { + // No options provided - use defaults directly + param_group_.set_options(defaults_->clone()); + } else { + // Options provided - merge user's explicit settings with defaults for + // parameter group inheritance This enables Python-C++ API parity by + // honoring user intent while inheriting missing parameters + auto final_options = defaults_->clone(); + + // Simple variadic dispatch - try all known optimizer types + _try_merge_all_optimizers(final_options, param_group.options(), *defaults_); + + // If no merging was done (custom optimizer), final_options already contains + // defaults + param_group_.set_options(std::move(final_options)); + } + for (const auto& p : param_group_.params()) { + TORCH_CHECK( + state_.count(p.unsafeGetTensorImpl()) == 0, + "some parameters appear in more than one parameter group"); + } + param_groups_.emplace_back(std::move(param_group_)); +} + +void Optimizer::add_parameters(const std::vector& parameters) { + TORCH_WARN("Optimizer::add_parameters() will be removed in PyTorch 1.6"); + auto& parameters_ = param_groups_[0].params(); + parameters_.insert(parameters_.end(), parameters.begin(), parameters.end()); +} + +void Optimizer::zero_grad(bool set_to_none) { + for (auto& group : param_groups_) { + for (auto& p : group.params()) { + if (p.mutable_grad().defined()) { + p.mutable_grad().detach_(); + if (set_to_none) + p.mutable_grad().reset(); + else + p.mutable_grad().zero_(); + } + } + } +} + +const std::vector& Optimizer::parameters() const noexcept { + TORCH_WARN("Optimizer::parameters() will be removed in PyTorch 1.6"); + return param_groups_.at(0).params(); +} + +std::vector& Optimizer::parameters() noexcept { + TORCH_WARN("Optimizer::parameters() will be removed in PyTorch 1.6"); + return param_groups_.at(0).params(); +} + +size_t Optimizer::size() const noexcept { + TORCH_WARN("Optimizer::size() will be removed in PyTorch 1.6"); + size_t count = 0; + for (const auto& group : param_groups_) { + count += group.params().size(); + } + return count; +} + +OptimizerOptions& Optimizer::defaults() noexcept { + return *defaults_; +} + +const OptimizerOptions& Optimizer::defaults() const noexcept { + return *defaults_; +} + +std::vector& Optimizer::param_groups() noexcept { + return param_groups_; +} + +const std::vector& Optimizer::param_groups() + const noexcept { + return param_groups_; +} + +ska::flat_hash_map>& Optimizer:: + state() noexcept { + return state_; +} + +const ska::flat_hash_map>& +Optimizer::state() const noexcept { + return state_; +} + +void Optimizer::save(serialize::OutputArchive& archive) const {} +void Optimizer::load(serialize::InputArchive& archive) {} + +/// Serializes an `Optimizer` into an `OutputArchive`. +serialize::OutputArchive& operator<<( + serialize::OutputArchive& archive, + const Optimizer& optimizer) { + optimizer.save(archive); + return archive; +} + +/// Deserializes a `Tensor` from an `InputArchive`. +serialize::InputArchive& operator>>( + serialize::InputArchive& archive, + Optimizer& optimizer) { + optimizer.load(archive); + return archive; +} + +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/optim/rmsprop.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/optim/rmsprop.cpp new file mode 100644 index 0000000000000000000000000000000000000000..23f9780f6a4f03ef6140fe99f3dc8958488496f5 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/optim/rmsprop.cpp @@ -0,0 +1,177 @@ +#include + +#include + +#include + +#include + +namespace torch::optim { + +RMSpropOptions::RMSpropOptions(double lr) : lr_(lr) {} + +bool operator==(const RMSpropOptions& lhs, const RMSpropOptions& rhs) { + return (lhs.lr() == rhs.lr()) && (lhs.alpha() == rhs.alpha()) && + (lhs.eps() == rhs.eps()) && (lhs.weight_decay() == rhs.weight_decay()) && + (lhs.momentum() == rhs.momentum()) && (lhs.centered() == rhs.centered()); +} + +void RMSpropOptions::serialize(torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(lr); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(alpha); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(eps); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(weight_decay); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(momentum); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(centered); +} + +void RMSpropOptions::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, lr); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, alpha); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, eps); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, weight_decay); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, momentum); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(bool, centered); +} + +double RMSpropOptions::get_lr() const { + return lr(); +} + +void RMSpropOptions::set_lr(const double lr) { + this->lr(lr); +} + +bool operator==(const RMSpropParamState& lhs, const RMSpropParamState& rhs) { + return (lhs.step() == rhs.step()) && + torch::equal(lhs.square_avg(), rhs.square_avg()) && + torch::equal_if_defined(lhs.momentum_buffer(), rhs.momentum_buffer()) && + torch::equal_if_defined(lhs.grad_avg(), rhs.grad_avg()); +} + +void RMSpropParamState::serialize( + torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(step); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(square_avg); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(momentum_buffer); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(grad_avg); +} + +void RMSpropParamState::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(int64_t, step); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, square_avg); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, momentum_buffer); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, grad_avg); +} + +/// Adapted from +/// https://github.com/pytorch/pytorch/blob/master/torch/optim/rmsprop.py +Tensor RMSprop::step(LossClosure closure) { + NoGradGuard no_grad; + Tensor loss = {}; + if (closure != nullptr) { + at::AutoGradMode enable_grad(true); + loss = closure(); + } + for (auto& group : param_groups_) { + for (auto& p : group.params()) { + if (!p.grad().defined()) { + continue; + } + auto grad = p.grad(); + TORCH_CHECK( + !grad.is_sparse(), "RMSprop does not support sparse gradients"); + auto param_state = state_.find(p.unsafeGetTensorImpl()); + auto& options = static_cast(group.options()); + + // State initialization + if (param_state == state_.end()) { + auto state = std::make_unique(); + state->step(0); + state->square_avg(torch::zeros_like(p, MemoryFormat::Preserve)); + if (options.momentum() > 0) { + state->momentum_buffer(torch::zeros_like(p, MemoryFormat::Preserve)); + } + if (options.centered()) { + state->grad_avg(torch::zeros_like(p, MemoryFormat::Preserve)); + } + state_[p.unsafeGetTensorImpl()] = std::move(state); + } + + auto& state = + static_cast(*state_[p.unsafeGetTensorImpl()]); + auto& square_avg = state.square_avg(); + auto alpha = options.alpha(); + + state.step(state.step() + 1); + + if (options.weight_decay() != 0) { + grad = grad.add(p, options.weight_decay()); + } + + square_avg.mul_(alpha).addcmul_(grad, grad, 1 - alpha); + + Tensor avg; + if (options.centered()) { + auto& grad_avg = state.grad_avg(); + grad_avg.mul_(alpha).add_(grad, 1 - alpha); + avg = square_avg.addcmul(grad_avg, grad_avg, -1) + .sqrt_() + .add_(options.eps()); + } else { + avg = square_avg.sqrt().add_(options.eps()); + } + + if (options.momentum() > 0) { + auto& buf = state.momentum_buffer(); + buf.mul_(options.momentum()).addcdiv_(grad, avg); + // Need to avoid version tracking for parameter. + p.add_(buf, -options.lr()); + } else { + // Need to avoid version tracking for parameter. + p.addcdiv_(grad, avg, -options.lr()); + } + } + } + return loss; +} + +void RMSprop::save(serialize::OutputArchive& archive) const { + serialize(*this, archive); +} + +void RMSprop::load(serialize::InputArchive& archive) { + IValue pytorch_version; + if (archive.try_read("pytorch_version", pytorch_version)) { + serialize(*this, archive); + } else { // deserializing archives saved in old format (prior to + // version 1.5.0) + TORCH_WARN( + "Your serialized RMSprop optimizer is still using the old serialization format. " + "The step value in state will be set to 0 because the old RMSprop optimizer didn't track the step value." + "You should re-save your RMSprop optimizer to use the new serialization format."); + std::vector square_average_buffers; + std::vector momentum_buffers; + std::vector grad_average_buffers; + torch::optim::serialize( + archive, "square_average_buffers", square_average_buffers); + torch::optim::serialize(archive, "momentum_buffers", momentum_buffers); + torch::optim::serialize( + archive, "grad_average_buffers", grad_average_buffers); + // since there were no param_groups prior to version 1.5.0, assuming all + // tensors are now in one param_group + std::vector params = param_groups_.at(0).params(); + for (const auto idx : c10::irange(square_average_buffers.size())) { + auto state = std::make_unique(); + state->square_avg(square_average_buffers[idx]); + if (idx < momentum_buffers.size()) { + state->momentum_buffer(momentum_buffers.at(idx)); + } + if (idx < grad_average_buffers.size()) { + state->grad_avg(grad_average_buffers.at(idx)); + } + state_[params[idx].unsafeGetTensorImpl()] = std::move(state); + } + } +} +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/optim/schedulers/lr_scheduler.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/optim/schedulers/lr_scheduler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cb61bd489f4178a221324c022a4ccddbc0dab672 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/optim/schedulers/lr_scheduler.cpp @@ -0,0 +1,41 @@ +#include +#include + +namespace torch::optim { + +LRScheduler::LRScheduler(torch::optim::Optimizer& optimizer) + : optimizer_(optimizer) {} + +void LRScheduler::step() { + std::vector learning_rates = get_lrs(); + set_optimizer_lrs(learning_rates); + step_count_++; +} + +void LRScheduler::set_optimizer_lrs(const std::vector& learning_rates) { + // Check the number of learning rates is equal to the number of parameters + // groups in the optimizer + TORCH_CHECK( + learning_rates.size() == optimizer_.param_groups().size(), + "Number of learning rates not equal to the number of param groups\n", + "Number of learning rates given: ", + learning_rates.size(), + "\nNumber of param groups: ", + optimizer_.param_groups().size()); + + for (const auto i : c10::irange(optimizer_.param_groups().size())) { + optimizer_.param_groups()[i].options().set_lr(learning_rates[i]); + } +} + +std::vector LRScheduler::get_current_lrs() const { + std::vector learnings_rates(optimizer_.param_groups().size()); + if (!learnings_rates.empty()) { + for (const auto i : c10::irange(optimizer_.param_groups().size())) { + learnings_rates[i] = optimizer_.param_groups()[i].options().get_lr(); + } + } + return learnings_rates; +} + +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/optim/schedulers/reduce_on_plateau_scheduler.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/optim/schedulers/reduce_on_plateau_scheduler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..edfdfbb82f71fd8bafa5b38ceba7777f9b8d6b84 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/optim/schedulers/reduce_on_plateau_scheduler.cpp @@ -0,0 +1,121 @@ +#include + +#include + +namespace torch::optim { + +ReduceLROnPlateauScheduler::ReduceLROnPlateauScheduler( + Optimizer& optimizer, + SchedulerMode mode, + float factor, + int patience, + double threshold, + ThresholdMode threshold_mode, + int cooldown, + const std::vector& min_lr, + double eps, + bool verbose) + : optimizer(optimizer) { + if (min_lr.empty()) { + this->min_lrs = std::vector(optimizer.param_groups().size()); + } else { + // Check if number of learning rates is equal to the number of parameters + // groups in the optimizer + TORCH_CHECK( + min_lr.size() == optimizer.param_groups().size(), + "Number of learning rates not equal to the number of param groups\n", + "Number of learning rates given: ", + min_lr.size(), + "\nNumber of param groups: ", + optimizer.param_groups().size()); + this->min_lrs = min_lr; + } + + TORCH_CHECK(factor < 1.0, "Factor should be < 1.0."); + this->factor = factor; + this->patience = patience; + this->cooldown = cooldown; + this->eps = eps; + this->verbose = verbose; + + init_is_better(mode, threshold, threshold_mode); + reset(); +} + +void ReduceLROnPlateauScheduler::step(float metrics) { + last_epoch++; + + if (is_better(metrics)) { + best = metrics; + num_bad_epochs = 0; + } else { + num_bad_epochs++; + } + + if (in_cooldown()) { + cooldown_counter--; + num_bad_epochs = 0; + } + + if (num_bad_epochs > patience) { + reduce_lr(last_epoch); + cooldown_counter = cooldown; + num_bad_epochs = 0; + } +} + +void ReduceLROnPlateauScheduler::reduce_lr(int epoch) { + for (std::size_t i = 0; i < optimizer.param_groups().size(); i++) { + auto old_lr = optimizer.param_groups()[i].options().get_lr(); + auto new_lr = std::fmax(old_lr * factor, min_lrs[i]); + if (old_lr - new_lr > eps) { + optimizer.param_groups()[i].options().set_lr(new_lr); + if (verbose) { + std::cout << std::setprecision(4) << "Epoch " << epoch + << ": reducing learning rate of group " << i << " to " + << new_lr << '\n'; + } + } + } +} + +void ReduceLROnPlateauScheduler::reset() { + this->cooldown_counter = 0; + this->num_bad_epochs = 0; + this->last_epoch = 0; + this->best = mode_worse; +} + +bool ReduceLROnPlateauScheduler::in_cooldown() const { + return cooldown_counter > 0; +} + +bool ReduceLROnPlateauScheduler::is_better(float a) { + if (mode == min && threshold_mode == rel) { + auto rel_epsilon = 1.0 - threshold; + return a < best * rel_epsilon; + } else if (mode == min && threshold_mode == abs) { + return a < best - threshold; + } else if (mode == max && threshold_mode == rel) { + auto rel_epsilon = 1.0 + threshold; + return a > best * rel_epsilon; + } else { + return a > best * threshold; + } +} + +void ReduceLROnPlateauScheduler::init_is_better( + SchedulerMode mode, + double threshold, + ThresholdMode threshold_mode) { + if (mode == min) { + mode_worse = std::numeric_limits::max(); + } else { + mode_worse = std::numeric_limits::min(); + } + + this->mode = mode; + this->threshold_mode = threshold_mode; + this->threshold = threshold; +} +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/optim/schedulers/step_lr.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/optim/schedulers/step_lr.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3edd155542d05e8ff64ef512dbfd8db63debf573 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/optim/schedulers/step_lr.cpp @@ -0,0 +1,24 @@ +#include + +namespace torch::optim { + +StepLR::StepLR( + torch::optim::Optimizer& optimizer, + const unsigned step_size, + const double gamma) + : LRScheduler(optimizer), step_size_(step_size), gamma_(gamma) {} + +std::vector StepLR::get_lrs() { + if (step_count_ == 0 || step_count_ % step_size_ != 0) + return get_current_lrs(); + else { + std::vector lrs = get_current_lrs(); + std::transform( + lrs.begin(), lrs.end(), lrs.begin(), [this](const double& v) { + return this->gamma_ * v; + }); + return lrs; + } +} + +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/optim/serialize.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/optim/serialize.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bca5269acd72ed6c8b2a950b70d115d179caf6ae --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/optim/serialize.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include +#include + +namespace torch::optim { +void serialize( + serialize::OutputArchive& archive, + const std::string& key, + const int64_t& value) { + archive.write(key, IValue(value)); +} + +void serialize( + serialize::InputArchive& archive, + const std::string& key, + int64_t& value) { + IValue ivalue; + archive.read(key, ivalue); + value = ivalue.toInt(); +} + +void serialize( + serialize::OutputArchive& archive, + const std::string& key, + const std::vector& steps) { + std::vector tensors; + tensors.reserve(steps.size()); + for (const auto& step : steps) { + tensors.push_back(torch::tensor(static_cast(step))); + } + serialize(archive, key, tensors); +} + +void serialize( + serialize::InputArchive& archive, + const std::string& key, + std::vector& steps) { + steps.clear(); + std::vector tensors; + serialize(archive, key, tensors); + for (const auto& step : tensors) { + steps.push_back(step.item()); + } +} +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/optim/sgd.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/optim/sgd.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f909277f17ffeff38b1e091303a242bac269a8df --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/optim/sgd.cpp @@ -0,0 +1,130 @@ +#include + +#include +#include +#include + +#include + +#include + +namespace torch::optim { + +SGDOptions::SGDOptions(double lr) : lr_(lr) {} + +bool operator==(const SGDOptions& lhs, const SGDOptions& rhs) { + return (lhs.lr() == rhs.lr()) && (lhs.momentum() == rhs.momentum()) && + (lhs.dampening() == rhs.dampening()) && + (lhs.weight_decay() == rhs.weight_decay()) && + (lhs.nesterov() == rhs.nesterov()); +} + +void SGDOptions::serialize(torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(lr); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(momentum); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(dampening); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(weight_decay); + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(nesterov); +} + +void SGDOptions::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, lr); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, momentum); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, dampening); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(double, weight_decay); + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(bool, nesterov); +} + +double SGDOptions::get_lr() const { + return lr(); +} + +void SGDOptions::set_lr(const double lr) { + this->lr(lr); +} + +bool operator==(const SGDParamState& lhs, const SGDParamState& rhs) { + return torch::equal(lhs.momentum_buffer(), rhs.momentum_buffer()); +} + +void SGDParamState::serialize(torch::serialize::OutputArchive& archive) const { + _TORCH_OPTIM_SERIALIZE_TORCH_ARG(momentum_buffer); +} + +void SGDParamState::serialize(torch::serialize::InputArchive& archive) { + _TORCH_OPTIM_DESERIALIZE_TORCH_ARG(Tensor, momentum_buffer); +} + +Tensor SGD::step(LossClosure closure) { + NoGradGuard no_grad; + Tensor loss = {}; + if (closure != nullptr) { + at::AutoGradMode enable_grad(true); + loss = closure(); + } + for (auto& group : param_groups_) { + auto& options = static_cast(group.options()); + auto weight_decay = options.weight_decay(); + auto momentum = options.momentum(); + auto dampening = options.dampening(); + auto nesterov = options.nesterov(); + + for (auto& p : group.params()) { + if (!p.grad().defined()) { + continue; + } + auto d_p = p.grad().data(); + if (weight_decay != 0) { + d_p = d_p.add(p.data(), weight_decay); + } + if (momentum != 0) { + Tensor buf; + auto param_state = state_.find(p.unsafeGetTensorImpl()); + if (param_state == state_.end()) { + buf = d_p.detach().clone(); + auto state = std::make_unique(); + state->momentum_buffer(buf); + state_[p.unsafeGetTensorImpl()] = std::move(state); + } else { + buf = static_cast(*param_state->second) + .momentum_buffer(); + buf.mul_(momentum).add_(d_p, 1 - dampening); + } + if (nesterov) { + d_p = d_p.add(buf, momentum); + } else { + d_p = buf; + } + } + p.data().add_(d_p, -1 * options.lr()); + } + } + return loss; +} + +void SGD::save(serialize::OutputArchive& archive) const { + serialize(*this, archive); +} + +void SGD::load(serialize::InputArchive& archive) { + IValue pytorch_version; + if (archive.try_read("pytorch_version", pytorch_version)) { + serialize(*this, archive); + } else { // deserializing archives saved in old format (prior to + // version 1.5.0) + TORCH_WARN( + "Your serialized SGD optimizer is still using the old serialization format. " + "You should re-save your SGD optimizer to use the new serialization format."); + std::vector momentum_buffers; + torch::optim::serialize(archive, "momentum_buffers", momentum_buffers); + // since there were no param_groups prior to version 1.5.0, assuming all + // tensors are now in one param_group + std::vector params = param_groups_.at(0).params(); + for (const auto idx : c10::irange(momentum_buffers.size())) { + auto state = std::make_unique(); + state->momentum_buffer(momentum_buffers[idx]); + state_[params[idx].unsafeGetTensorImpl()] = std::move(state); + } + } +} +} // namespace torch::optim diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/print.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/print.cpp new file mode 100644 index 0000000000000000000000000000000000000000..098d2fa1c95f24d1203e2a9f9e37b4676c41b3d9 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/print.cpp @@ -0,0 +1,10 @@ +#include +#include + +namespace torch { + +void set_printoption_sci_mode(bool enabled) { + at::set_printoption_sci_mode(enabled); +} + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/python/init.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/python/init.cpp new file mode 100644 index 0000000000000000000000000000000000000000..06d060afa8d89b757dc0002c403e0870bdb607af --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/python/init.cpp @@ -0,0 +1,75 @@ +#include +#include + +#include +#include + +#include + +#include + +namespace py = pybind11; + +namespace pybind11::detail { +#define ITEM_TYPE_CASTER(T, Name) \ + template <> \ + struct type_caster::Item> { \ + public: \ + using Item = typename torch::OrderedDict::Item; \ + using PairCaster = make_caster>; \ + PYBIND11_TYPE_CASTER(Item, _("Ordered" #Name "DictItem")); \ + bool load(handle src, bool convert) { \ + return PairCaster().load(src, convert); \ + } \ + static handle cast( \ + const Item& src, \ + return_value_policy policy, \ + handle parent) { \ + return PairCaster::cast( \ + src.pair(), std::move(policy), std::move(parent)); \ + } \ + } + +// NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) +ITEM_TYPE_CASTER(torch::Tensor, Tensor); +// NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) +ITEM_TYPE_CASTER(std::shared_ptr, Module); +} // namespace pybind11::detail + +namespace torch::python { +namespace { +template +void bind_ordered_dict(py::module module, const char* dict_name) { + using ODict = OrderedDict; + // clang-format off + py::class_(module, dict_name) + .def("items", &ODict::items) + .def("keys", &ODict::keys) + .def("values", &ODict::values) + .def("__iter__", [](const ODict& dict) { + return py::make_iterator(dict.begin(), dict.end()); + }, py::keep_alive<0, 1>()) + .def("__len__", &ODict::size) + .def("__contains__", &ODict::contains) + .def("__getitem__", [](const ODict& dict, const std::string& key) { + return dict[key]; + }) + .def("__getitem__", [](const ODict& dict, size_t index) { + return dict[index]; + }); + // clang-format on +} +} // namespace + +void init_bindings(PyObject* module) { + py::module m = py::handle(module).cast(); + py::module cpp = m.def_submodule("cpp"); + + bind_ordered_dict(cpp, "OrderedTensorDict"); + bind_ordered_dict>(cpp, "OrderedModuleDict"); + + py::module nn = cpp.def_submodule("nn"); + add_module_bindings( + py::class_>(nn, "Module")); +} +} // namespace torch::python diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/serialize.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/serialize.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f8c964d8393f4381c3657aa8162a303725ac438b --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/serialize.cpp @@ -0,0 +1,16 @@ +#include +#include + +#include + +namespace torch { + +std::vector pickle_save(const at::IValue& ivalue) { + return jit::pickle_save(ivalue); +} + +torch::IValue pickle_load(const std::vector& data) { + return jit::pickle_load(data); +} + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/serialize/input-archive.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/serialize/input-archive.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0235600cb4b87f4b75f7172bea777d055e1332c0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/serialize/input-archive.cpp @@ -0,0 +1,178 @@ +#include + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace torch::serialize { + +InputArchive::InputArchive() + : module_("Module", std::make_shared()) {} + +void InputArchive::read(const std::string& key, c10::IValue& ivalue) { + ivalue = module_.attr(key); +} + +bool InputArchive::try_read(const std::string& key, c10::IValue& ivalue) { + if (!module_.hasattr(key)) { + return false; + } + ivalue = module_.attr(key); + return true; +} + +bool InputArchive::try_read( + const std::string& key, + Tensor& tensor, + bool is_buffer) { + if (!module_.hasattr(key)) { + return false; + } + auto iv = module_.attr(key); + if (!iv.isTensor()) { + return false; + } + auto read_tensor = iv.toTensor(); + // clang-format on + if (tensor.defined()) { + torch::NoGradGuard guard; + if (tensor.device() != read_tensor.device()) { + tensor.set_data(read_tensor); + } else { + tensor.set_(read_tensor); + } + } else { + tensor = std::move(read_tensor); + } + return true; +} + +void InputArchive::read( + const std::string& key, + Tensor& tensor, + bool is_buffer) { + TORCH_CHECK( + try_read(key, tensor, is_buffer), + "No such serialized tensor '", + hierarchy_prefix_, + key, + "'"); +} + +bool InputArchive::try_read(const std::string& key, InputArchive& archive) { + if (!module_.hasattr(key)) { + return false; + } + auto iv = module_.attr(key); + if (!iv.isModule()) { + return false; + } + archive.module_ = iv.toModule(); + archive.hierarchy_prefix_ = hierarchy_prefix_ + key + "."; + return true; +} + +void InputArchive::read(const std::string& key, InputArchive& archive) { + TORCH_CHECK( + try_read(key, archive), + "No such serialized submodule: '", + hierarchy_prefix_, + key, + "'"); +} + +void InputArchive::load_from( + const std::string& filename, + std::optional device /*= std::nullopt*/) { + module_ = torch::jit::load(filename, device); +} + +void InputArchive::load_from( + std::istream& stream, + std::optional device /*= std::nullopt*/) { + module_ = torch::jit::load(stream, device); +} + +void InputArchive::load_from( + const char* data, + size_t size, + std::optional device /*= std::nullopt*/) { + using caffe2::serialize::ReadAdapterInterface; + class OurAdapter : public ReadAdapterInterface { + public: + OurAdapter(const char* data, size_t size) : data_(data), size_(size) {} + size_t size() const override { + return size_; + } + size_t read( + uint64_t pos, + void* buf, + size_t n, + [[maybe_unused]] const char* what = "") const override { + if (pos >= size_) { + return 0; + } + size_t nread = std::min(static_cast(pos) + n, size_) - pos; + memcpy(buf, data_ + pos, nread); + return nread; + } + + private: + const char* data_; + size_t size_; + }; + module_ = torch::jit::load(std::make_unique(data, size), device); +} + +void InputArchive::load_from( + const std::function& read_func, + const std::function& size_func, + std::optional device /*= std::nullopt*/) { + using caffe2::serialize::ReadAdapterInterface; + class OurAdapter : public ReadAdapterInterface { + public: + OurAdapter( + const std::function& read_func, + const std::function& size_func) + : read_func_(read_func), size_func_(size_func) {} + size_t size() const override { + return size_func_(); + } + size_t read(uint64_t pos, void* buf, size_t n, const char* what = "") + const override { + (void)what; + return read_func_(pos, buf, n); + } + + private: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const std::function& read_func_; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const std::function& size_func_; + }; + module_ = torch::jit::load( + std::make_unique(read_func, size_func), device); +} + +std::vector InputArchive::keys() { + std::vector all_keys; + all_keys.reserve(module_.named_attributes(/*recurse=*/false).size()); + + for (const torch::jit::NameValue& s : + module_.named_attributes(/*recurse=*/false)) { + all_keys.push_back(s.name); + } + + return all_keys; +} + +} // namespace torch::serialize diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/serialize/output-archive.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/serialize/output-archive.cpp new file mode 100644 index 0000000000000000000000000000000000000000..487ef9fa52e7be5b518b941a3767ddb465ee78a7 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/serialize/output-archive.cpp @@ -0,0 +1,49 @@ +#include + +#include +#include + +#include +#include + +#include + +#include +#include +#include + +namespace torch::serialize { +OutputArchive::OutputArchive(std::shared_ptr cu) + : cu_(std::move(cu)), + module_("__torch__.Module", cu_, /*shouldMangle=*/true) {} + +void OutputArchive::write(const std::string& key, const c10::IValue& ivalue) { + module_.register_attribute(key, ivalue.type(), ivalue); +} + +void OutputArchive::write( + const std::string& key, + const Tensor& tensor, + bool is_buffer) { + module_.register_parameter(key, tensor, is_buffer); +} + +void OutputArchive::write( + const std::string& key, + OutputArchive& nested_archive) { + module_.register_module(key, nested_archive.module_); +} + +void OutputArchive::save_to(const std::string& filename) { + jit::ExportModule(module_, filename); +} + +void OutputArchive::save_to(std::ostream& stream) { + jit::ExportModule(module_, stream); +} + +void OutputArchive::save_to( + const std::function& func) { + jit::ExportModule(module_, func); +} +} // namespace torch::serialize diff --git a/western_ai/usa/pytorch/torch/csrc/api/src/xpu.cpp b/western_ai/usa/pytorch/torch/csrc/api/src/xpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cba2f36fdf8a4a10219eccc91bb3f257684a49be --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/api/src/xpu.cpp @@ -0,0 +1,45 @@ +#include +#include + +namespace torch::xpu { + +size_t device_count() { + return at::detail::getXPUHooks().deviceCount(); +} + +bool is_available() { + return xpu::device_count() > 0; +} + +void manual_seed(uint64_t seed) { + if (is_available()) { + auto index = at::detail::getXPUHooks().getCurrentDevice(); + auto gen = at::detail::getXPUHooks().getDefaultGenerator(index); + { + // See Note [Acquire lock when using random generators] + std::lock_guard lock(gen.mutex()); + gen.set_current_seed(seed); + } + } +} + +/// Sets the seed for all available GPUs. +void manual_seed_all(uint64_t seed) { + auto num_gpu = device_count(); + for (const auto i : c10::irange(num_gpu)) { + auto gen = at::detail::getXPUHooks().getDefaultGenerator(i); + { + // See Note [Acquire lock when using random generators] + std::lock_guard lock(gen.mutex()); + gen.set_current_seed(seed); + } + } +} + +void synchronize(int64_t device_index) { + TORCH_CHECK(is_available(), "No XPU are available"); + at::detail::getXPUHooks().deviceSynchronize( + static_cast(device_index)); +} + +} // namespace torch::xpu diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/FunctionsManual.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/FunctionsManual.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e4181dfd7692c46476edc08dc3f09d54ddabc3af --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/FunctionsManual.cpp @@ -0,0 +1,8303 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +// Helper functions for autogenerated code +// These used to be inlined into the codegened Functions.cpp + +namespace torch::autograd::generated::details { + +using at::areAnyTensorSubclassLike; +using at::IntArrayRef; +using at::OptionalIntArrayRef; +using at::Scalar; +using at::Tensor; +using at::TensorList; +using at::native::detail::GridSamplerInterpolation; +using at::native::detail::GridSamplerPadding; + +const char* kCudnnDoubleBackwardMsg = + "Double backwards is not supported for CuDNN RNNs due to limitations in the CuDNN API. To run double backwards, please disable the CuDNN backend temporarily while running the forward pass of your RNN. For example: \nwith torch.backends.cudnn.flags(enabled=False):\n output = model(inputs)"; + +Tensor apply_loss_reduction(const Tensor& unreduced, int64_t reduction) { + if (reduction == at::Reduction::Mean) { + return unreduced.mean(); + } else if (reduction == at::Reduction::Sum) { + return unreduced.sum(); + } + return unreduced; +} + +static bool isDefined(const std::optional& t) { + return t.has_value() && t->defined(); +} + +Tensor toNonOptTensor(const std::optional& t) { + return t.has_value() ? *t : Tensor(); +} + +Tensor toNonOptFwGrad(const std::optional& t) { + return (t.has_value() && t->defined()) ? t->_fw_grad(/*level */ 0) : Tensor(); +} + +Tensor toNonOptPrimal(const std::optional& t) { + if (t.has_value() && t->defined()) { + if (t->unsafeGetTensorImpl()->is_wrapped_number()) { + return *t; + } + return t->_fw_primal(/* level */ 0); + } + return Tensor(); +} + +void update_wrapped_number(Tensor& input, Tensor& output) { + if (input.unsafeGetTensorImpl()->is_wrapped_number()) { + output.unsafeGetTensorImpl()->set_wrapped_number(true); + } +} + +void copy_range(variable_list& out, IndexRange range, const Tensor& t) { + TORCH_CHECK(range.second <= out.size()); + TORCH_CHECK( + range.second - range.first == 1, "inconsistent range for Tensor output"); + out[range.first] = t; +} + +void copy_range(variable_list& out, IndexRange range, at::ArrayRef t) { + TORCH_CHECK(range.second <= out.size()); + TORCH_CHECK( + range.second - range.first == t.size(), + "inconsistent range for TensorList output"); + std::copy( // NOLINT(modernize-use-ranges) + t.begin(), + t.end(), + out.begin() + static_cast(range.first)); +} + +Tensor copysign_tensor_self_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& result) { + auto ratio = result / self; + ratio.masked_fill_(self == 0, 0); + return grad * ratio; +} + +template +static T not_implemented_base(const char* name, const char* reason) { + std::string msg = + c10::str("the derivative for '", name, "' is not implemented."); + if (reason[0] != '\0') { + msg = c10::str(msg, " ", reason); + }; + TORCH_CHECK_NOT_IMPLEMENTED(false, msg); +} + +Tensor not_implemented(const char* name, const char* reason) { + return not_implemented_base(name, reason); +} + +std::vector not_implemented_list(const char* name, const char* reason) { + return not_implemented_base>(name, reason); +} + +Tensor maybe_multiply(const Tensor& t, const Scalar& s) { + bool is_one = false; + if (s.isFloatingPoint()) { + is_one = s.toSymFloat() == 1; + } else if (s.isIntegral(true)) { + is_one = s.toSymInt() == 1; + } + + if (is_one) { + return t; + } else { + return t * s; + } +} + +int64_t _safe_size(IntArrayRef sizes, IntArrayRef dim) { + int64_t size = 1; + if (sizes.empty()) { + return 1; + } + for (auto d : dim) { + d = at::maybe_wrap_dim(d, static_cast(sizes.size())); + size *= sizes[d]; + } + return size; +} + +static c10::SymInt _safe_size(c10::SymIntArrayRef sizes, c10::IntArrayRef dim) { + c10::SymInt size = 1; + if (sizes.empty()) { + return 1; + } + for (auto d : dim) { + d = at::maybe_wrap_dim(d, static_cast(sizes.size())); + size *= sizes[d]; + } + return size; +} + +Tensor handle_r_to_c(ScalarType self_st, Tensor gradient_result) { + if (!at::isComplexType(self_st) && gradient_result.is_complex()) { + // R -> C + return at::real(gradient_result); + } + return gradient_result; +} + +static Tensor handle_r_to_c(const Tensor& self, Tensor gradient_result) { + if (!self.is_complex() && gradient_result.is_complex()) { + // R -> C + return at::real(gradient_result); + } + return gradient_result; +} + +Tensor restore_reduced_dims( + const Tensor& output, + IntArrayRef dims, + bool keepdim) { + if (keepdim) { + return output; + } + auto total_dims = output.dim() + dims.size(); + std::vector target_shape(total_dims, 0); + for (int64_t i : dims) { + if (i < 0) { + i = static_cast(total_dims) + i; + } + target_shape[i] = 1; + } + int64_t j = 0; + for (const c10::SymInt& i : output.sym_sizes()) { + while (target_shape[j] > 0) + j++; + target_shape[j++] = i; + } + return output.reshape_symint(target_shape); +} + +Tensor scale_grad_by_count( + const Tensor& grad, + const Tensor& mask, + IntArrayRef dims) { + return (grad / mask.sum(dims, true)) * mask; +} + +Tensor amaxamin_jvp( + const Tensor& x, + const Tensor& dx, + const Tensor& result, + IntArrayRef dim, + bool keepdim) { + auto mask = x == restore_reduced_dims(result, dim, keepdim); + return at::where(mask, dx, 0.).sum(dim, keepdim) / mask.sum(dim, keepdim); +} + +// Builds the dims vector for aminmax: all dims when dim is nullopt, +// or the single wrapped dim otherwise. +static std::vector _aminmax_dims( + const Tensor& self, + std::optional dim_opt) { + if (dim_opt.has_value()) { + return {at::maybe_wrap_dim(*dim_opt, self.dim())}; + } + std::vector dims(self.dim()); + std::iota(dims.begin(), dims.end(), 0); + return dims; +} + +Tensor aminmax_backward( + const Tensor& self, + std::optional dim, + bool keepdim, + const Tensor& grad_min, + const Tensor& grad_max, + const Tensor& min, + const Tensor& max) { + auto dims_vec = _aminmax_dims(self, dim); + IntArrayRef dims(dims_vec); + Tensor result; + + if (grad_min.defined()) { + auto grad_min_expanded = restore_reduced_dims(grad_min, dims, keepdim); + auto min_mask = (self == restore_reduced_dims(min, dims, keepdim)); + result = scale_grad_by_count(grad_min_expanded, min_mask, dims); + } + + if (grad_max.defined()) { + auto grad_max_expanded = restore_reduced_dims(grad_max, dims, keepdim); + auto max_mask = (self == restore_reduced_dims(max, dims, keepdim)); + auto grad_max_result = + scale_grad_by_count(grad_max_expanded, max_mask, dims); + + if (result.defined()) { + if (!areAnyTensorSubclassLike({result, grad_max_result})) { + result.add_(grad_max_result); + } else { + result = result + grad_max_result; + } + } else { + result = grad_max_result; + } + } + + return result; +} + +Tensor aminmax_jvp( + const Tensor& self_p, + const Tensor& self_t, + const Tensor& result, + std::optional dim, + bool keepdim) { + auto dims_vec = _aminmax_dims(self_p, dim); + return amaxamin_jvp(self_p, self_t, result, IntArrayRef(dims_vec), keepdim); +} + +std::tuple _euclidean_dist_backward( + const Tensor& grad, + const Tensor& x1, + const Tensor& x2, + const Tensor& res) { + if (!grad.defined()) { + return std::tuple(Tensor(), Tensor()); + } + // handle case at 0 where we return a subgradient containing 0 + Tensor ratio = grad / res; + ratio.masked_fill_(res == 0, 0); + return std::tuple{ + x1 * ratio.sum(-1, true) - ratio.matmul(x2), + x2 * ratio.sum(-2, false).unsqueeze(-1) - ratio.mT().matmul(x1)}; +} + +Tensor norm_backward( + const Tensor& grad, + const Tensor& self, + const std::optional& p_, + const Tensor& norm) { + return norm_backward(grad, self, p_, norm, {}, true); +} + +Tensor norm_backward( + Tensor grad, + const Tensor& self, + const std::optional& p_, + Tensor norm, + IntArrayRef dim, + bool keepdim) { + // NB: We mask fill the NaNs in the output to be zero but still do float + // division + // by zero, which ASAN complains about. One way to appease ASAN is to fill + // the problematic values with something arbitrary before the division, + // but we decide not to due to the perf hit. Instead we just silence ASAN + // where necessary + size_t ndim = self.dim(); + double p = p_.value_or(2.0).toDouble(); + Tensor self_scaled; + Tensor scale_v; + + if (!keepdim && self.dim() != 0) { + grad = unsqueeze_multiple(grad, dim, ndim); + norm = unsqueeze_multiple(norm, dim, ndim); + } + + if (p == 0.0) { + return {}; + } else if (p == 1.0) { + return self.sgn() * grad; + } else if (p == 2.0) { + return grad * (self / norm).masked_fill_(norm == 0, 0); + } else if (std::isinf(p)) { + // Derivative of amax(abs(self), dim, keepdim) but respecting nans + // We create a mask of `argmax`: it's argmax if self.abs() == norm or it's + // NaN + auto self_abs = self.abs(); + auto mask = self_abs.eq(norm).logical_or(self_abs.isnan()); + return self.sgn() * ((grad / mask.sum(dim, true)) * mask); + } else if (p < 1.0) { + self_scaled = + self.sgn() * self.abs().pow_(p - 1).masked_fill_(self == 0, 0); + return self_scaled * grad * norm.pow(1 - p); + } else if (p < 2.0) { + self_scaled = self.sgn() * self.abs().pow_(p - 1); + scale_v = grad / norm.pow(p - 1); + scale_v.masked_fill_(norm == 0, 0); + return self_scaled * scale_v; + } else { + self_scaled = self * self.abs().pow_(p - 2); + scale_v = grad / norm.pow(p - 1); + scale_v.masked_fill_(norm == 0, 0); + return self_scaled * scale_v; + } +} + +// See norm_backward above for a note on ignoring the sanitizer +Tensor norm_jvp( + const Tensor& self_p, + const Tensor& self_t, + const std::optional& p_, + Tensor norm, + IntArrayRef dim, + bool keepdim) { + // NB: currently norm_jvp is also reused for dist's jvp (which haas two + // differentiable inputs) + // but self_t still cannot be a ZT because that would require both self_t + // and other_t to be ZT + TORCH_INTERNAL_ASSERT(!self_t._is_zerotensor()); + size_t ndim = self_p.dim(); // composite compliance? + double p = p_.value_or(2.0).toDouble(); + + if (p == 0.0) { + return at::zeros_like(norm); + } else if (p == 1.0) { + auto result = self_p.sgn(); + result = areAnyTensorSubclassLike({self_t}) ? result.mul(self_t.conj()) + : result.mul_(self_t.conj()); + result = at::real(result); + return result.sum(dim, keepdim); + } else if (p == 2.0) { + auto result = self_p.mul(self_t.conj()); + result = at::real(result); + result = result.sum(dim, keepdim); + return result.div_(norm).masked_fill_(norm == 0, 0); + } else if (std::isinf(p)) { + if (!keepdim && self_p.dim() != 0) { + norm = unsqueeze_multiple(norm, dim, ndim); + } + const auto self_isnan = self_p.isnan(); + const auto norm_isnan = norm.isnan(); + const auto& self_and_norm_isnan = areAnyTensorSubclassLike({norm}) + ? self_isnan.logical_and(norm_isnan) + : self_isnan.logical_and_(norm_isnan); + const auto is_eq_max = + (self_p.abs() == norm).logical_or_(self_and_norm_isnan).type_as(norm); + auto nb_max = is_eq_max.count_nonzero(dim); + if (self_p.dim() != 0) { + nb_max = unsqueeze_multiple(nb_max, dim, ndim); + } + return (at::real(self_p.sgn() * self_t.conj()) * is_eq_max / nb_max) + .sum(dim, keepdim); + } else if (p < 1.0) { + auto sumpow_t = (self_p.abs().pow_(p - 1).masked_fill_(self_p == 0, 0) * + at::real(self_p.sgn() * self_t.conj())) + .sum(dim, keepdim); + return sumpow_t * norm.pow(1 - p); + } else if (p < 2.0) { + auto sumpow_t = + (self_p.abs().pow_(p - 1) * at::real(self_p.sgn() * self_t.conj())) + .sum(dim, keepdim); + auto out = sumpow_t / norm.pow(p - 1); + return out.masked_fill_(norm == 0, 0); + } else { + auto sumpow_t = + (self_p.abs().pow_(p - 2) * at::real(self_p * self_t.conj())) + .sum(dim, keepdim); + auto out = sumpow_t / norm.pow(p - 1); + return out.masked_fill_(norm == 0, 0); + } +} + +Tensor norm_jvp( + const Tensor& self_p, + const Tensor& self_t, + const std::optional& p_, + Tensor norm) { + return norm_jvp(self_p, self_t, p_, std::move(norm), {}, true); +} + +Tensor _nested_from_padded_backward( + const Tensor& grad, + const Tensor& input, + bool do_transform_0213) { + if (do_transform_0213) { + auto new_sizes = { + input.size(0), input.size(2), (input.size(1) * input.size(3))}; + auto out = grad.to_padded_tensor(0, new_sizes); + auto expand_last_dim_size = { + input.size(0), input.size(2), input.size(1), input.size(3)}; + return out.view(expand_last_dim_size).permute({0, 2, 1, 3}); + } + return grad.to_padded_tensor(0, input.sizes()); +} + +std::tuple linear_double_backward( + const variable_list& grads, + const Tensor& self, + const Tensor& grad_output, + const Tensor& weight) { + if (!grad_output.defined()) { + return std::make_tuple(Tensor(), Tensor(), Tensor()); + } + + Tensor grad_self, grad_grad_output, grad_weight; + + if (grads[1].defined()) { + grad_self = + (grad_output.dim() == 1 ? grad_output.unsqueeze(0) : grad_output) + .matmul(grads[1]); + if (grad_output.dim() == 1) { + grad_self = grad_self.squeeze(0); + } + } + if (grads[0].defined()) { + grad_weight = + (grad_output.dim() == 1 ? grad_output.unsqueeze(1) : grad_output.mT()) + .matmul(grads[0].dim() == 1 ? grads[0].unsqueeze(0) : grads[0]); + } + + if (grads[0].defined() || grads[1].defined() || grads[2].defined()) { + grad_grad_output = at::zeros_like(grad_output); + if (grad_output.dim() == 1) { + grad_grad_output = grad_grad_output.unsqueeze(0); + } + } + + if (grads[0].defined()) { + grad_grad_output = grad_grad_output + + (grads[0].dim() == 1 ? grads[0].unsqueeze(0) : grads[0]) + .matmul(weight.mT()); + } + if (grads[1].defined()) { + grad_grad_output = grad_grad_output + + (self.dim() == 1 ? self.unsqueeze(0) : self).matmul(grads[1].mT()); + } + if (grads[2].defined()) { + grad_grad_output = grad_grad_output + grads[2]; + } + if (grad_grad_output.defined() && grad_output.dim() == 1) { + grad_grad_output = grad_grad_output.squeeze(0); + } + + return std::make_tuple( + std::move(grad_self), + std::move(grad_grad_output), + std::move(grad_weight)); +} + +Tensor linalg_vector_norm_jvp( + const Tensor& self_p, + const Tensor& self_t, + const Scalar& scalar_ord, + Tensor norm, + const at::OptionalIntArrayRef& opt_dim, + bool keepdim) { + // No need to handle the dtype arg as it's handled via broadcasting in the + // function + auto dim = opt_dim.value_or(IntArrayRef({})); + return norm_jvp(self_p, self_t, scalar_ord, std::move(norm), dim, keepdim); +} + +Tensor linalg_vector_norm_backward( + Tensor grad, + const Tensor& self, + const Scalar& scalar_ord, + Tensor norm, + const at::OptionalIntArrayRef& opt_dim, + bool keepdim) { + // No need to handle the dtype arg as it's handled via broadcasting in the + // function + auto dim = opt_dim.value_or(IntArrayRef({})); + return norm_backward( + std::move(grad), self, scalar_ord, std::move(norm), dim, keepdim); +} + +Tensor pow_backward(Tensor grad, const Tensor& self, const Scalar& exponent) { + if (exponent.equal(0.0)) { + return at::zeros_like(self, LEGACY_CONTIGUOUS_MEMORY_FORMAT); + } else { + auto grad_lambda = [&](auto exp) { + return grad * (exp * self.pow(exp - 1)).conj(); + }; + Tensor out = (exponent.isComplex()) + ? grad_lambda(exponent.toComplexDouble()) + : grad_lambda(exponent.toDouble()); + return handle_r_to_c(self, std::move(out)); + } +} + +Tensor pow_backward_self( + const Tensor& grad, + const Tensor& self, + const Tensor& exponent) { + auto out = at::where( + exponent == 0.0, + at::scalar_tensor(0.0, grad.options()), + grad * (exponent * self.pow(exponent - 1)).conj()); + return handle_r_to_c(self, std::move(out)); +} + +// Caveats: +// We define d(a^b)/db at a = 0 and b < 0 to be -inf. This is due to +// d(a^b)/db -> -inf for a fixed b as a -> +0 +// Currently, tensorflow defines d(a^b)/db = nan for a = 0 and b < 0. +// +// We define d(a^b)/db = 0 for a = 0 and b = 0 by continuity as +// d(a^b)/db = 0 for a > 0 and b -> +0. +// Currently, tensorflow agrees with us. +Tensor pow_backward_exponent( + const Tensor& grad, + const Tensor& self, + const Tensor& exponent, + const Tensor& result) { + Tensor cond; + if (exponent.is_complex()) { + auto is_real_exp = + at::logical_and(at::imag(exponent) == 0, at::real(exponent) >= 0); + cond = at::logical_and(self == 0, is_real_exp); + } else { + cond = at::logical_and(self == 0, exponent >= 0); + } + auto promoted_dtype = at::result_type(self, exponent); + // `.to()` is no-op if dtype is same. + auto self_ = self.to(promoted_dtype); + + auto out = grad * + at::where(cond, + at::scalar_tensor(0.0, grad.options()), + (result * self_.log()).conj()); + return handle_r_to_c(exponent, std::move(out)); +} + +Tensor pow_backward_exponent( + const Tensor& grad, + const Scalar& base, + const Tensor& exponent, + const Tensor& result) { + auto grad_lambda = [](const Tensor& a, const Scalar& b) { + return (a * b.log()).conj(); + }; + auto promoted_dtype = at::result_type(base, exponent); + auto base_ = at::isComplexType(promoted_dtype) + ? Scalar(base.toComplexDouble()) + : Scalar(base.toDouble()); + if (base.equal(0.0)) { + auto cond = [](const auto& exp) { + if (exp.is_complex()) { + return at::logical_and(at::imag(exp) == 0, at::real(exp) >= 0); + } else { + return exp >= 0; + } + }; + auto out = grad * + at::where(cond(exponent), + at::zeros({}, grad.options()), + grad_lambda(result, base_)); + return handle_r_to_c(exponent, std::move(out)); + } else { + auto out = grad * grad_lambda(result, base_); + return handle_r_to_c(exponent, std::move(out)); + } +} + +Tensor angle_backward(const Tensor& grad, const Tensor& self) { + if (self.is_complex()) { + return at::where( + self == 0.0, + at::zeros({}, self.options()), + grad * self / self.abs().pow(2) * + Scalar(c10::complex{0.0, 1.0})); + } else { + return at::zeros_like(self, at::MemoryFormat::Preserve); + } +} + +Tensor mvlgamma_backward(const Tensor& grad, const Tensor& self, int64_t p) { + Tensor args = at::arange( + -static_cast(p) / 2. + 0.5, + 0.5, + 0.5, + // use strided here regardless of self's layout; useful for e.g. NJT + self.options().layout(c10::kStrided)); + args = args.add(self.unsqueeze(-1)); + return grad * args.digamma_().sum(-1); +} + +Tensor sgn_backward(const Tensor& x, const Tensor& gx, const Tensor& sgn) { + if (x.is_complex()) { + auto abs = x.abs(); + return ((gx - (sgn * sgn) * gx.conj()) / (2. * abs)) + .masked_fill_(abs == 0., 0.); + } else { + return at::_efficientzerotensor(sgn.sizes(), sgn.options()); + } +} + +Tensor masked_fill_backward(const Tensor& grad, const Tensor& mask) { + // masked_select does not work well with functorch, as its shape is + // data-dependent + return areAnyTensorSubclassLike({grad, mask}) + ? at::where(mask, grad, 0).sum() + : grad.masked_select(mask).sum(); +} + +template +Tensor mul_tensor_backward( + const Tensor& grad, + const T& other, + ScalarType self_st) { + auto out = grad * other.conj(); + return handle_r_to_c(self_st, std::move(out)); +} +template Tensor mul_tensor_backward(const Tensor&, const Tensor&, ScalarType); +template Tensor mul_tensor_backward(const Tensor&, const Scalar&, ScalarType); + +template +Tensor div_tensor_self_backward( + const Tensor& grad, + const T& other, + ScalarType self_st, + const std::optional& rounding_mode) { + if (rounding_mode.has_value()) { + return at::zeros_like(grad, grad.options().dtype(self_st)); + } + + auto result = grad / other.conj(); + return handle_r_to_c(self_st, std::move(result)); +} +template Tensor div_tensor_self_backward( + const Tensor&, + const Tensor&, + ScalarType, + const std::optional&); +template Tensor div_tensor_self_backward( + const Tensor&, + const Scalar&, + ScalarType, + const std::optional&); + +Tensor div_tensor_other_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& other, + const std::optional& rounding_mode) { + if (rounding_mode.has_value()) { + return at::zeros_like(grad, grad.options().dtype(other.scalar_type())); + } + + auto result = -grad * ((self / other) / other).conj(); + return handle_r_to_c(other, std::move(result)); +} + +Tensor permute_backwards(const Tensor& grad, IntArrayRef fwd_dims) { + // invert the permutation + auto ndims = fwd_dims.size(); + std::vector dims(ndims); + for (const auto i : c10::irange(ndims)) { + dims[at::maybe_wrap_dim(fwd_dims[i], static_cast(ndims))] = + static_cast(i); + } + return grad.permute(dims); +} + +Tensor rad2deg_backward(const Tensor& grad) { + constexpr double M_180_PI = + 57.295779513082320876798154814105170332405472466564; + return at::mul(grad, Scalar(M_180_PI)); +} + +Tensor deg2rad_backward(const Tensor& grad) { + constexpr double M_PI_180 = + 0.017453292519943295769236907684886127134428718885417; + return at::mul(grad, Scalar(M_PI_180)); +} + +Tensor unsqueeze_multiple( + const Tensor& t, + OptionalIntArrayRef opt_dim, + size_t n_dims) { + if (opt_dim.has_value()) { + IntArrayRef dim = opt_dim.value(); + auto dim_size = dim.size(); + // Optimisation for two common cases + if (dim_size == 0) { + return t; + } else if (dim_size == 1) { + return t.unsqueeze(dim[0]); + } + } + auto dims_to_unsqueeze = at::dim_list_to_bitset(opt_dim, n_dims); + Tensor res = t; + for (const auto i : c10::irange(n_dims)) { + if (dims_to_unsqueeze[i]) { + res = res.unsqueeze(static_cast(i)); + } + } + return res; +} + +Tensor sum_backward( + const Tensor& grad, + c10::SymIntArrayRef sizes, + OptionalIntArrayRef opt_dims, + bool keepdim) { + if (!keepdim && !sizes.empty()) { + if (opt_dims.has_value() && !opt_dims.value().empty()) { + return unsqueeze_multiple(grad, opt_dims, sizes.size()) + .expand_symint(sizes); + } + } + return grad.expand_symint(sizes); +} + +Tensor sum_backward( + const Tensor& grad, + c10::SymIntArrayRef sizes, + c10::IntArrayRef dims, + bool keepdim) { + if (!keepdim && !sizes.empty() && !dims.empty()) { + // we are only using `keepdim=true` path for SymInts for now + TORCH_CHECK_NOT_IMPLEMENTED( + false, + "Only the keepdim=true path is implemented to support symints in autograd"); + } else { + return grad.expand_symint(sizes); + } +} + +Tensor nansum_backward( + const Tensor& grad, + const Tensor& self, + at::OptionalIntArrayRef dims, + bool keepdim) { + return sum_backward(grad, self.sym_sizes(), dims, keepdim) * + self.isnan().logical_not(); +} + +Tensor mean_backward( + const Tensor& grad, + c10::SymIntArrayRef shape, + OptionalIntArrayRef opt_dim, + c10::SymInt numel, + bool keepdim) { + bool is_all_reduce = !opt_dim.has_value() || opt_dim.value().empty(); + auto n = + is_all_reduce ? std::move(numel) : _safe_size(shape, opt_dim.value()); + return sum_backward(grad, shape, opt_dim, keepdim) / std::move(n); +} + +std::vector reverse_list_symint(const c10::SymIntArrayRef list) { + auto result = std::vector(); + result.reserve(list.size()); + // NOLINTNEXTLINE(modernize-loop-convert) + for (auto iter = list.rbegin(); iter != list.rend(); iter++) { + result.push_back(*iter); + } + return result; +} + +std::vector reverse_list(const IntArrayRef list) { + auto result = std::vector(); + result.reserve(list.size()); + // NOLINTNEXTLINE(modernize-loop-convert) + for (auto iter = list.rbegin(); iter != list.rend(); iter++) { + result.emplace_back(*iter); + } + return result; +} + +Tensor prod_safe_zeros_backward( + const Tensor& grad, + const Tensor& inp, + int64_t dim) { + if (inp.sym_numel() == 0) { + // When input has a zero sized dimension (empty tensor), + // we don't need to actually compute the grads. + // So we just reshape `grad` as `input`. + return grad.expand_as(inp); + } + + if (inp.sym_size(dim) == 1) { + return grad; + } + + auto ones_size = inp.sym_sizes().vec(); + ones_size[dim] = 1; + Tensor ones = at::ones_symint(ones_size, grad.options()); + Tensor exclusive_normal_nocp = + at::cat({ones, inp.narrow_symint(dim, 0, inp.sym_size(dim) - 1)}, dim); + Tensor exclusive_normal = exclusive_normal_nocp.cumprod(dim); + + Tensor narrow_reverse = + inp.narrow_symint(dim, 1, inp.sym_size(dim) - 1).flip(dim); + Tensor exclusive_reverse_nocp = + at::cat({std::move(ones), std::move(narrow_reverse)}, dim); + Tensor exclusive_reverse = exclusive_reverse_nocp.cumprod(dim).flip(dim); + + return grad * (exclusive_normal * exclusive_reverse).conj(); +} + +// note that the gradient for prod is equivalent to: +// cumprod(exclusive, normal) * cumprod(exclusive, reverse), e.g.: +// input: [ a, b, c] +// cumprod(exclusive, normal): [1 , a, a * b] +// cumprod(exclusive, reverse): [b * c, c, 1] +// product: [b * c, a * c, a * b] +// and this is safe under input with 0s. +Tensor prod_backward( + const Tensor& grad, + const Tensor& input, + const Tensor& result) { + if (input.dim() == 0) { + return grad; + } + if (input.is_meta() || isTensorSubclassLike(input)) { + // For Composite Compliance, always take the safer (and slower) path + return prod_safe_zeros_backward(grad, input.contiguous().view(-1), 0) + .view_as(input); + } + Tensor zero_idx = (input == 0).nonzero(); + if (zero_idx.sym_numel() == 0) { + return grad * (result / input).conj(); + } else if (!at::GradMode::is_enabled() && zero_idx.sym_size(0) > 1) { + return at::zeros_like(input, LEGACY_CONTIGUOUS_MEMORY_FORMAT); + } else { + return prod_safe_zeros_backward(grad, input.contiguous().view(-1), 0) + .view_as(input); + } +} + +Tensor prod_backward( + Tensor grad, + const Tensor& input, + Tensor result, + int64_t dim, + bool keepdim) { + if (input.dim() == 0) { + return grad; + } + dim = at::maybe_wrap_dim(dim, input.dim()); + if (!keepdim) { + // `prod` reduces the dimension at `dim`, + // so, unsqueeze `grad` and `result` at dim. + grad = grad.unsqueeze(dim); + result = result.unsqueeze(dim); + } + if (input.is_meta() || isTensorSubclassLike(input)) { + // For Composite Compliance, always take the safer (and slower) path + return prod_safe_zeros_backward(grad, input, dim); + } + + Tensor zero_mask = (input == 0); + Tensor slice_zero_count = zero_mask.sum(dim, true); + int64_t total_zeros = slice_zero_count.sum().item(); + if (total_zeros == 0) { + return grad * (result / input).conj(); + } else { + return prod_safe_zeros_backward(grad, input, dim); + } +} + +template +static Tensor generic_solve_jvp( + solve_f solve, + const Tensor& X, + const Tensor& A, + const Tensor& dA, + const Tensor& dB) { + auto is_vector_case = at::native::linalg_solve_is_vector_rhs(dA, dB); + auto dA_contrib = + is_vector_case ? dA.matmul(X.unsqueeze(-1)).squeeze(-1) : dA.matmul(X); + // In general, + // dX = solve(A, dB - dA_contrib), but this behavior is different for + // lu_solve. For refer to lu_solve_jvp for more details on this. + return solve(A, dB, dA_contrib); +} + +Tensor cumsum_backward(const Tensor& grad, int64_t dim) { + // Trivial case + if (grad.sym_numel() <= 1 || grad.sym_size(dim) == 1) { + return grad; + } + return grad.flip(dim).cumsum(dim).flip(dim); +} + +Tensor logsumexp_backward( + Tensor grad, + const Tensor& self, + Tensor result, + IntArrayRef dim, + bool keepdim) { + if (!keepdim && self.dim() != 0) { + grad = unsqueeze_multiple(grad, dim, self.dim()); + result = unsqueeze_multiple(result, dim, self.dim()); + } + return grad * (self - result).exp().conj(); +} + +Tensor logcumsumexp_backward( + Tensor grad, + const Tensor& self, + const Tensor& result, + int64_t dim) { + if (grad.dim() == 0 || grad.sym_numel() == 0) { + return grad; + } + + // Reference: + // https://github.com/tensorflow/tensorflow/blob/2a5910906a0e0f3dbc186ff9db6386d81a63448c/tensorflow/python/ops/math_grad.py#L1832-L1863 + + auto scalar_min = AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2( + at::ScalarType::Half, + at::ScalarType::BFloat16, + at::typeMetaToScalarType(grad.dtype()), + "logcumsumexp_backward", + []() { return c10::Scalar(std::numeric_limits::lowest()); }); + + auto reverse_logcumsumexp = [dim](const auto& x) { + return at::flip(at::logcumsumexp(at::flip(x, {dim}), dim), {dim}); + }; + + if (!at::is_complex(grad)) { + auto grad_min = at::scalar_tensor(scalar_min, grad.options()); + auto log_abs_grad = grad.abs().log(); + auto log_grad_positive = at::where(grad > 0, log_abs_grad, grad_min); + auto log_grad_negative = at::where(grad < 0, log_abs_grad, grad_min); + + auto output_pos = + (reverse_logcumsumexp(log_grad_positive - result) + self).exp(); + auto output_neg = + (reverse_logcumsumexp(log_grad_negative - result) + self).exp(); + + return output_pos - output_neg; + } else { + // no trick separating the positive and negative required + auto log_grad = grad.conj().log(); + auto output = (reverse_logcumsumexp(log_grad - result) + self).exp(); + return output.conj(); + } +} + +Tensor logcumsumexp_jvp( + const Tensor& self_p, + const Tensor& self_t, + int64_t dim) { + // Mostly taken from logsumexp_jvp + + // NB: for simplicity, we recompute some values that can be reused from + // forward + auto self_p_exp = [&self_p, dim]() { + if (!at::is_complex(self_p)) { + return (self_p - std::get<0>(at::max(self_p, dim, true))) + .exp(); // Use the exp-normalize trick + } else { + // at::max doesn't support complex128 + return self_p.exp(); + } + }(); + + auto cumsumexp_p = self_p_exp.cumsum(dim); + + TORCH_INTERNAL_ASSERT(!self_t._is_zerotensor()) + + constexpr double eps = 1e-13; + + if (areAnyTensorSubclassLike({self_p, self_t})) { + auto result = (self_p_exp * self_t).cumsum(dim); + result /= cumsumexp_p.add_(eps); + return result; + } else { + self_p_exp *= self_t; + auto cumsumexp_t = self_p_exp.cumsum(dim); + return cumsumexp_t /= cumsumexp_p.add_(eps); + } +} + +Tensor unbind_backward(const variable_list& grads, int64_t dim) { + c10::SymIntArrayRef sizes; + at::TensorOptions o; + for (const auto& v : grads) { + if (v.defined()) { + sizes = v.sym_sizes(); + o = static_cast(v).options(); + break; + } + } + auto grads_tensors = fmap(grads, [&](const Variable& v) { + return ( + v.defined() ? static_cast(v) + : at::zeros({}, o).expand_symint(sizes)); + }); + return at::stack(grads_tensors, dim); +} + +Tensor unbind_backward_nested( + const variable_list& grads, + const Tensor& nt_sizes, + int64_t dim, + const at::TensorOptions& options) { + std::vector grads_tensors; + grads_tensors.reserve(grads.size()); + for (int64_t i : c10::irange(static_cast(grads.size()))) { + if (grads[i].defined()) { + grads_tensors.push_back(static_cast(grads[i])); + } else { + const auto component_size = nt_sizes[i].contiguous(); + const c10::IntArrayRef grad_size( + component_size.const_data_ptr(), component_size.size(0)); + grads_tensors.push_back(at::zeros(grad_size, options)); + } + } + + return at::_nested_tensor_from_tensor_list(grads_tensors); +} + +Tensor unbind_backward_nested_jagged( + const variable_list& grads, + const Tensor& self, + int64_t dim) { + TORCH_INTERNAL_ASSERT( + dim == 0, "unbind_backward_nested_jagged() only supports dim=0") + auto grad_nt = at::zeros_like(self); + auto unbound_grads = grad_nt.unbind(); + for (int64_t i : c10::irange(static_cast(grads.size()))) { + if (grads[i].defined()) { + unbound_grads[i].copy_(static_cast(grads[i])); + } + } + + return grad_nt; +} + +Tensor unsqueeze_to(const Tensor& self, c10::SymIntArrayRef sym_sizes) { + auto result = self; + + auto nDims = sym_sizes.size(); + for (const auto dim : c10::irange(nDims)) { + if (sym_sizes[dim] == 1) { + result = result.unsqueeze(static_cast(dim)); + } + } + return result; +} + +Tensor unsqueeze_to( + const Tensor& self, + IntArrayRef dims, + c10::SymIntArrayRef sym_sizes) { + const auto ndim = sym_sizes.size(); + auto mask = at::dim_list_to_bitset(dims, ndim); + + Tensor result = self; + for (const auto d : c10::irange(ndim)) { + if (mask.test(d) && sym_sizes[d] == 1) { + result = result.unsqueeze(static_cast(d)); + } + } + return result; +} + +Tensor unsqueeze_to( + const Tensor& self, + int64_t dim, + c10::SymIntArrayRef sym_sizes) { + return unsqueeze_to(self, IntArrayRef{dim}, sym_sizes); +} + +std::vector cat_tensors_backward( + const Tensor& grad, + const std::vector>& sizes, + const std::vector& dtypes, + int64_t dim) { + std::vector grad_inputs(sizes.size()); + if (!grad.defined()) { + return grad_inputs; + } + dim = at::legacy_cat_wrap_dim_symint(dim, sizes); + c10::SymInt accumulate = 0; + + Tensor grad_; + bool grad_is_complex = grad.is_complex(); + if (grad_is_complex) { + grad_ = at::real(grad); + } + for (const auto i : c10::irange(sizes.size())) { + Tensor grad_val; + if (!at::isComplexType(dtypes[i]) && grad_is_complex) { + // R -> C + grad_val = grad_; + } else { + grad_val = grad; + } + auto& shape = sizes[i]; + // If input was empty tensor, gradInput should be empty tensor. + if (shape.size() == 1) { + if (TORCH_GUARD_OR_FALSE(shape[0].sym_eq(0))) { + grad_inputs[i] = at::zeros({0}, grad_val.options()); + continue; + } + } + const auto& size = shape[dim]; + accumulate += size; + grad_inputs[i] = grad_val.narrow_symint(dim, accumulate - size, size); + } + return grad_inputs; +} + +std::vector stack_tensors_backward( + const Tensor& grad, + int64_t dim, + const std::vector& dtypes) { + std::vector grad_inputs(dtypes.size()); + if (!grad.defined()) { + return grad_inputs; + } + bool grad_is_complex = grad.is_complex(); + for (const auto i : c10::irange(dtypes.size())) { + auto gr = grad.select(dim, static_cast(i)); + if (grad_is_complex && !at::isComplexType(dtypes[i])) { + gr = at::real(gr); + } + grad_inputs[i] = gr; + } + return grad_inputs; +} + +std::vector block_diag_backward( + const Tensor& grad, + const std::vector>& sizes, + const std::vector& dtypes) { + std::vector grad_inputs(sizes.size()); + if (!grad.defined()) { + return grad_inputs; + } + Tensor real_view_of_grad; + bool grad_is_complex = grad.is_complex(); + if (grad_is_complex) { + real_view_of_grad = at::real(grad); + } + + int64_t cur_dim0 = 0; + int64_t cur_dim1 = 0; + + for (const auto i : c10::irange(sizes.size())) { + // R -> C + Tensor grad_val = (!at::isComplexType(dtypes[i]) && grad_is_complex) + ? real_view_of_grad + : grad; + + auto& shape = sizes[i]; + // If input was empty tensor, gradInput should be empty tensor. + if (shape.size() == 1 && shape[0] == 0) { + grad_inputs[i] = at::zeros({0}, grad_val.options()); + continue; + } + // 0d case + int64_t dim0 = 1; + int64_t dim1 = 1; + // 2d case + if (shape.size() == 2) { + dim0 = shape[0]; + dim1 = shape[1]; + // 1d case + } else if (shape.size() == 1) { + dim1 = shape[0]; + } + auto slice = grad_val.slice(0, cur_dim0, cur_dim0 + dim0) + .slice(1, cur_dim1, cur_dim1 + dim1); + if (shape.size() == 1) { + slice = slice.squeeze(-1); + } else if (shape.empty()) { + slice = slice.squeeze(-1).squeeze(-1); + } + grad_inputs[i] = slice; + cur_dim0 += dim0; + cur_dim1 += dim1; + } + return grad_inputs; +} + +Tensor clamp_backward( + const Tensor& grad, + const Tensor& self, + const std::optional& min, + const std::optional& max) { + // clamp: gradients not defined on min and max, so we return the subgradient 1 + // for these cases. + if (max && min) { + auto zero = at::scalar_tensor(0., grad.options()); + return where((self >= *min).logical_and_(self <= *max), grad, zero); + } else if (min) { + auto zero = at::scalar_tensor(0., grad.options()); + return where(self >= *min, grad, zero); + } else if (max) { + auto zero = at::scalar_tensor(0., grad.options()); + return where(self <= *max, grad, zero); + } else { + return grad; + } +} + +Tensor clamp_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& min, + const Tensor& max) { + // clamp: gradients not defined on min and max, so we return the subgradient 1 + // for these cases. + if (max.defined() && min.defined()) { + auto zero = at::scalar_tensor(0., grad.options()); + const auto self_ge_min = self >= min; + const auto self_le_max = self <= max; + const auto& pred = areAnyTensorSubclassLike({self, min, max}) + ? self_ge_min.logical_and(self_le_max) + : self_ge_min.logical_and_(self_le_max); + return where(pred, grad, zero); + } else if (min.defined()) { + auto zero = at::scalar_tensor(0., grad.options()); + return where(self >= min, grad, zero); + } else if (max.defined()) { + auto zero = at::scalar_tensor(0., grad.options()); + return where(self <= max, grad, zero); + } else { + return grad; + } +} + +std::tuple clamp_backward_min_max( + const Tensor& grad, + const Tensor& self, + const Tensor& min, + const Tensor& max, + const std::array& grad_input_mask) { + // If min > max, min has no gradient + std::tuple ret; + if (!grad.defined()) { + return ret; + } + + auto zero = at::scalar_tensor(0., grad.options()); + if (max.defined() && min.defined()) { + if (grad_input_mask[0]) { + const auto self_lt_min = self < min; + const auto min_lt_max = min < max; + const auto& pred = areAnyTensorSubclassLike({self, min, max}) + ? self_lt_min.logical_and(min_lt_max) + : self_lt_min.logical_and_(min_lt_max); + std::get<0>(ret) = where(pred, grad, zero); + } + if (grad_input_mask[1]) { + const auto self_gt_max = self > max; + const auto max_lt_min = max < min; + const auto& pred = areAnyTensorSubclassLike({self, min, max}) + ? self_gt_max.logical_or(max_lt_min) + : self_gt_max.logical_or_(max_lt_min); + std::get<1>(ret) = where(pred, grad, zero); + } + } else if (min.defined() && grad_input_mask[0]) { + std::get<0>(ret) = where(self < min, grad, zero); + } else if (max.defined() && grad_input_mask[1]) { + std::get<1>(ret) = where(self > max, grad, zero); + } + return ret; +} + +at::Tensor clamp_jvp( + const Tensor& self_p, + const Tensor& self_t, + const Tensor& min_p, + const Tensor& min_t, + const Tensor& max_p, + const Tensor& max_t) { + if (min_p.defined() && max_p.defined()) { + return where( + min_p > max_p, + max_t, + where(self_p < min_p, min_t, where(self_p > max_p, max_t, self_t))); + } else if (min_p.defined()) { + return where(self_p > min_p, self_t, min_t); + } else if (max_p.defined()) { + return where(self_p < max_p, self_t, max_t); + } else { + return self_t; + } +} + +Tensor convolution_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& bias_p, + const Tensor& bias_t, + at::SymIntArrayRef stride, + at::SymIntArrayRef padding, + at::SymIntArrayRef dilation, + bool transposed, + at::SymIntArrayRef output_padding, + const c10::SymInt& groups) { + auto bias_t_opt = + bias_t.defined() ? std::optional(bias_t) : std::nullopt; + return ( + at::convolution_symint( + input_t, + weight_p, + std::nullopt, + stride, + padding, + dilation, + transposed, + output_padding, + groups) + + at::convolution_symint( + input_p, + weight_t, + bias_t_opt, + stride, + padding, + dilation, + transposed, + output_padding, + groups)); +} + +Tensor _convolution_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& bias_p, + const Tensor& bias_t, + at::SymIntArrayRef stride, + at::SymIntArrayRef padding, + at::SymIntArrayRef dilation, + bool transposed, + at::SymIntArrayRef output_padding, + const c10::SymInt& groups, + bool benchmark, + bool deterministic, + bool cudnn_enabled, + bool allow_tf32) { + auto bias_t_opt = + bias_t.defined() ? std::optional(bias_t) : std::nullopt; + return ( + at::_convolution_symint( + input_t, + weight_p, + std::nullopt, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + benchmark, + deterministic, + cudnn_enabled, + allow_tf32) + + at::_convolution_symint( + input_p, + weight_t, + bias_t_opt, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + benchmark, + deterministic, + cudnn_enabled, + allow_tf32)); +} + +Tensor convolution_backward_jvp_grad_bias( + const Tensor& grad_out_t, + const Tensor& grad_bias) { + if (!grad_bias.defined()) { + return Tensor(); + } + int64_t dim = grad_out_t.dim() - 2; + if (dim == 1) { + // Cannot pass initializer list due to overload ambiguity + auto dimlist = std::vector{0, 2}; + return grad_out_t.sum(dimlist); + } else if (dim == 2) { + return grad_out_t.sum({0, 2, 3}); + } else if (dim == 3) { + return grad_out_t.sum({0, 2, 3, 4}); + } else { + TORCH_INTERNAL_ASSERT( + false, + "convolution_backward_jvp_grad_bias expected dim of grad_out_t to be 3, 4, or 5, but got: ", + grad_out_t.dim()); + } +} + +// This function is used by load_derivatives.py to replace tensor.strides() +// calls that appear in derivative formulas. If the tensor has requires_grad +// set, this function returns its strides or an empty array if the tensor +// is sparse. If requires_grad is not set, an empty array is returned since +// there will be no backward pass. There has one special case, if input is +// MKLDNN tensor and has requires_grad set, just return an empty array, the +// reason is that MKLDNN tensor is a opaque tensor which has not stride info. +// +// This function only supports the case where `input` is the tensor whose +// single derivative is being calculated. +// +// This function does not support `self` derivatives for inplace functions. +// +// Args: +// input Tensor to call .strides() on +// input_name Name of `input` tensor, from derivative formula +at::SymIntArrayRef strides_or_error( + const Tensor& input, + std::string_view const& input_name) { + // TODO: Ideally, this function would never be called if requires_grad is + // not set. Once codegen is updated to avoid the call, we can remove this + // check. + if (input.requires_grad()) { + if (input.is_mkldnn()) + return {}; + if (input.is_sparse() || at::sparse_csr::is_sparse_compressed(input)) + return {}; + return input.sym_strides(); + } else { + return {}; + } +} + +Tensor mm_mat1_backward( + const Tensor& grad, + const Tensor& mat2, + at::SymIntArrayRef mat1_sizes, + at::SymIntArrayRef mat1_strides, + c10::Layout mat1_layout, + const Scalar& alpha) { + if (grad.layout() == c10::kStrided && mat2.layout() == c10::kStrided && + mat1_layout == c10::kStrided) { + // if input was column-major, return grad as column-order for efficiency + if (mat1_strides[0] == 1 && mat1_strides[1] == mat1_sizes[0]) { + return maybe_multiply(mat2.conj().mm(grad.t()).t(), alpha.conj()); + } + } + + // General fallback, should work for any layout + return maybe_multiply(grad.mm(mat2.t().conj()), alpha.conj()); +} + +Tensor mm_mat2_backward( + const Tensor& grad, + const Tensor& mat1, + at::SymIntArrayRef mat2_sizes, + at::SymIntArrayRef mat2_strides, + c10::Layout mat2_layout, + const Scalar& alpha) { + if (grad.layout() == c10::kStrided && mat1.layout() == c10::kStrided && + mat2_layout == c10::kStrided) { + // if input was column-major, return grad as column-order for efficiency + if (mat2_strides[0] == 1 && mat2_strides[1] == mat2_sizes[0]) { + return maybe_multiply(grad.t().mm(mat1.conj()).t(), alpha.conj()); + } + } + + // General fallback, should work for any layout + return maybe_multiply(mat1.t().conj().mm(grad), alpha.conj()); +} + +Tensor _grouped_mm_mat1_backward( + const Tensor& grad, + const Tensor& mat2, + at::SymIntArrayRef mat1_sizes, + at::SymIntArrayRef mat1_strides, + c10::Layout mat1_layout, + std::optional offs, + const Scalar& alpha) { + TORCH_CHECK( + grad.layout() == c10::kStrided && mat2.layout() == c10::kStrided && + mat1_layout == c10::kStrided, + "only strided layout supported for grouped mm"); + // if input was column-major, return grad as column-order for efficiency + if (offs.has_value() && !offs->defined()) { + offs = std::nullopt; + } + auto mat1_dim = mat1_sizes.size(); + if (mat1_strides[mat1_dim - 2] == 1 && + mat1_strides[mat1_dim - 1] == mat1_sizes[mat1_dim - 2]) { + auto grad_inp = + (at::_grouped_mm(mat2, grad.transpose(-2, -1), offs)).transpose(-2, -1); + return maybe_multiply(grad_inp, alpha.conj()); + } else { + auto grad_inp = (at::_grouped_mm(grad, mat2.transpose(-2, -1), offs)); + return maybe_multiply(grad_inp, alpha.conj()); + } +} + +Tensor _grouped_mm_mat2_backward( + const Tensor& grad, + const Tensor& mat1, + at::SymIntArrayRef mat2_sizes, + at::SymIntArrayRef mat2_strides, + c10::Layout mat2_layout, + std::optional offs, + const Scalar& alpha) { + TORCH_CHECK( + grad.layout() == c10::kStrided && mat1.layout() == c10::kStrided && + mat2_layout == c10::kStrided, + "only strided layout supported for grouped mm"); + // if input was column-major, return grad as column-order for efficiency + auto mat2_dim = mat2_sizes.size(); + if (offs.has_value() && !offs->defined()) { + offs = std::nullopt; + } + if (mat2_strides[mat2_dim - 2] == 1 && + mat2_strides[mat2_dim - 1] == mat2_sizes[mat2_dim - 2]) { + auto grad_inp = + at::_grouped_mm(grad.transpose(-2, -1), mat1, offs).transpose(-2, -1); + return maybe_multiply(grad_inp, alpha.conj()); + } else { + auto grad_inp = at::_grouped_mm(mat1.transpose(-2, -1), grad, offs); + return maybe_multiply(grad_inp, alpha.conj()); + } +} + +Tensor mm_mat1_sparse_backward( + const Tensor& grad, + const Tensor& mat1, + const Tensor& mat2, + const Scalar& alpha) { + if (grad.layout() == c10::kStrided && mat2.layout() == c10::kStrided && + mat1.is_sparse()) { + auto sparse = mat1.coalesce(); + Tensor grad_sparse = maybe_multiply(grad.mm(mat2.conj().t()), alpha); + return grad_sparse.sparse_mask(sparse); + } else if ( + grad.layout() == c10::kStrided && mat2.layout() == c10::kStrided && + mat1.is_sparse_csr()) { + // zero must to have mat1 sparsity pattern: + auto zero = mat1.clone(); + zero.values().zero_(); + return at::sparse_sampled_addmm(zero, grad, mat2.mH(), 1.0, alpha); + } else if ( + grad.layout() == c10::kStrided && mat2.layout() == c10::kStrided && + mat1.layout() == c10::kStrided) { + return maybe_multiply(grad.mm(mat2.mH()), alpha); + } + TORCH_CHECK( + false, + "sparse_addmm_sparse_backward: unsupported combination of layouts", + ", grad: ", + grad.layout(), + ", mat1: ", + mat1.layout(), + ", mat2: ", + mat2.layout()); +} + +static Tensor sparse_mask_like_grad( + const Tensor& x, + const Tensor& gx, + bool accumulate_matches) { + if (x.is_coalesced() && gx.is_coalesced()) { + if (x._nnz() >= gx._nnz()) { + // search into x is faster + return gx._sparse_mask_projection(x, accumulate_matches); + } else { + // search into gx is faster + return gx.sparse_mask(x); + } + } else if (x.is_coalesced()) { + return gx.sparse_mask(x); + } else if (gx.is_coalesced()) { + return gx._sparse_mask_projection(x, accumulate_matches); + } else { + if (x._nnz() >= gx._nnz()) { + // gx.coalesce() is likely faster + return gx.coalesce()._sparse_mask_projection(x, accumulate_matches); + } else { + // x.coalesce() is likely faster + return gx.sparse_mask(x.coalesce()); + } + } +} + +std::tuple sparse_sampled_addmm_backward( + const Tensor& grad, + const Tensor& self, + const std::optional& mat1, + const std::optional& mat2, + const Scalar& alpha, + const Scalar& beta, + const std::array& grad_input_mask) { + if (!grad.defined()) { + return std::make_tuple(Tensor{}, Tensor{}, Tensor{}); + } + + const auto grad_projected = grad.sparse_mask(self); + const auto self_requires_grad = grad_input_mask[0]; + const auto mat1_requires_grad = grad_input_mask[1]; + const auto mat2_requires_grad = grad_input_mask[2]; + return std::make_tuple( + self_requires_grad ? maybe_multiply(grad, beta.conj()) : Tensor{}, + mat1_requires_grad + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + ? maybe_multiply(grad_projected.mm(mat2->mH()), alpha.conj()) + : Tensor{}, + mat2_requires_grad + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + ? maybe_multiply(mat1->mH().mm(grad_projected), alpha.conj()) + : Tensor{}); +} + +Tensor sparse_mask_backward( + const Tensor& grad, + const Tensor& mask, + const c10::Layout self_layout) { + // NOTE: sparse_mask accumulates matches, so the backward step has to + // accumulate as well. + const auto self_grad = + sparse_mask_like_grad(mask, grad, /*accumulate_matches=*/true); + return self_layout == at::kStrided ? self_grad.to_dense() : self_grad; +} + +Tensor sparse_sparse_matmul_backward( + const Tensor& grad, + const Tensor& a, + const Tensor& b, + int64_t grad_order) { + /* + To implement the backward algorithm for sparse matrix-matrix matmul (SPMM) we + can start from the following definition for dense tensors: + + c = a @ b + then + a_grad = c_grad @ b^H + b_grad = a^H @ c_grad + + So for sparse matrices we can use the following definition: + + if grad_order == 0: + a_grad = sparse_matrix_mask(c_grad @ b^H, mask=a) + else: + b_grad = sparse_matrix_mask(a^H @ c_grad, mask=b) + */ + TORCH_CHECK( + grad_order == 0 || grad_order == 1, + ": grad_order not in [0, 1] at sparse_sparse_matmul_backward function"); + + // NOTE: _sparse_sparse_matmul returns a coalesced gradient, + // // hence there is no need in accumulating matches. + if (grad_order == 0) { + auto a_grad = _sparse_sparse_matmul(grad, b.conj().t()); + return sparse_mask_like_grad(a, a_grad, /*accumulate_matches=*/false); + } + auto b_grad = _sparse_sparse_matmul(a.conj().t(), grad); + return sparse_mask_like_grad(b, b_grad, /*accumulate_matches=*/false); +} + +Tensor renorm_backward( + const Tensor& grad, + const Tensor& self, + const Scalar& p, + int64_t dim, + const Scalar& maxnorm) { + auto n = self.dim(); + dim = c10::maybe_wrap_dim(dim, n); + auto reduce_dims = at::DimVector(n); + std::iota(reduce_dims.begin(), reduce_dims.end(), 0); + reduce_dims.erase(reduce_dims.begin() + dim); + + auto acc_type = + at::toAccumulateType(self.scalar_type(), self.device().type()); + auto norm = at::linalg_vector_norm( + self, p, reduce_dims, /*keepdim=*/true, /*dtype=*/acc_type); + + const auto real_acc_type = c10::toRealValueType(acc_type); + auto grad_output = (self.conj() * grad); + // vector_norm output is real, so grad_output must also be real + if (real_acc_type != acc_type) { + grad_output = at::real(grad_output); + } + grad_output = + grad_output.sum(reduce_dims, /*keepdim=*/true, /*dtype=*/real_acc_type); + auto nb = norm_backward( + std::move(grad_output), self, p, norm, reduce_dims, /*keepdim=*/true); + + auto invnorm = (norm + 1e-7).reciprocal(); + auto grad_norm = maxnorm * invnorm * (grad - invnorm * nb); + return at::where(norm > maxnorm, grad_norm.to(grad.scalar_type()), grad); +} + +Tensor renorm_jvp( + const Tensor& self_p, + const Tensor& self_t, + const Scalar& p, + int64_t dim, + const Scalar& maxnorm) { + auto self_sizes = self_p.sizes(); + dim = at::maybe_wrap_dim(dim, static_cast(self_sizes.size())); + + at::DimVector reduce_dims(self_sizes.size()); + std::iota(reduce_dims.begin(), reduce_dims.end(), 0); + reduce_dims.erase(reduce_dims.begin() + dim); + + // For cuda half, calculate norm in float precision then cast + // normalization factor to half + auto dtype = self_p.scalar_type(); + auto acc_type = at::toAccumulateType(dtype, /*is_cuda=*/true); + Tensor norm = [&self_p, &p, &reduce_dims, acc_type, dtype]() { + if (acc_type != dtype) { + return at::linalg_vector_norm( + self_p, + p.toDouble(), + reduce_dims, + /*keepdim=*/true, + /*dtype=*/acc_type); + } else { + return at::linalg_vector_norm( + self_p, + p.toDouble(), + reduce_dims, + /*keepdim=*/true); + } + }(); + + auto double_maxnorm = maxnorm.toDouble(); + auto invnorm = (norm + 1e-7).reciprocal(); + auto factor = invnorm * double_maxnorm; + + return where( + norm > double_maxnorm, + factor * + (self_t - + self_p * invnorm * + norm_jvp( + self_p, self_t, p, norm, reduce_dims, /*keepdim=*/true)), + self_t); +} + +Tensor repeat_backward( + Tensor grad, + c10::SymIntArrayRef repeats, + c10::SymIntArrayRef input_shape) { + auto find_iter = std::find( // NOLINT(modernize-use-ranges) + repeats.cbegin(), + repeats.cend(), + 0); + if (find_iter != repeats.cend()) { + return at::zeros_symint(input_shape, grad.options()); + } + const auto input_dims = input_shape.size(); + auto num_unsqueezed = grad.dim() - input_dims; + for ([[maybe_unused]] const auto i : c10::irange(num_unsqueezed)) { + grad = grad.sum(0, false); + } + + at::SymDimVector grad_size; + at::DimVector sum_dims; + for (const auto dim : c10::irange(input_dims)) { + const auto& repeat = repeats[dim + num_unsqueezed]; + // Reshape gradient (repeat > 1) + // Index: [..., dim , ...] [..., dim , dim+1 , ...] + // Shape: From [..., dimsize, ...] to [..., repeat, dimsize/repeat, ...] + // The gradient tensor at 'dim' is reshaped to 'repeat' times of input + // tensor. Then, sum up gradients over repeated tensors along 'dim', and + // reduce shape from 'repeat * dimsize/repeat' to 'dimsize/repeat' + // ('input_dimsize'). Example: + // Size(3, 2) Size(6, 2) + // [[v1_0, v1_1], + // [v1_2, v1_3], + // [[v0, v1], repeat(2, 1) [v1_4, v1_5], + // [v2, v3], -------------> [v2_0, v2_1], + // [v4, v5]] [v2_2, v2_3], + // [v2_4, v2_5]] + // + // input grad (3, 2) reshape (2, 3, 2) output grad (6, 2) + // [[[g1_0, g1_1], [[g1_0, g1_1], + // [g1_2, g1_3], [g1_2, g1_3], + // [[g1_0+g2_0, g1_1+g2_1], [g1_4, g1_5]], [g1_4, g1_5], + // [g1_2+g2_2, g1_3+g2_3], [g2_0, g2_1], [[g2_0, g2_1], + // [g1_4+g2_4, g1_5+g2_5]] [g2_2, g2_3], [g2_2, g2_3], + // [g2_4, g2_5]] [g2_4, g2_5]]] + // + // If gradient tensor is reshaped to [..., dimsize/repeat, repeat, ...] and + // then sum over 'dim+1'. The gradient for input is not correctly aligned + // with input. Example: + // input grad (3, 2) reshape (3, 2, 2) output grad (6, 2) + // [[[g1_0, g1_1], [[g1_0, g1_1], + // [g1_2, g1_3]], [g1_2, g1_3], + // [[g1_0+g1_2, g1_1+g1_3], [[g1_4, g1_5], [g1_4, g1_5], + // [g1_4+g2_0, g1_5+g2_1], [g2_0, g2_1]], [g2_0, g2_1], + // [g2_2+g2_4, g2_3+g2_5]] [[g2_2, g2_3], [g2_2, g2_3], + // [g2_4, g2_5]]] [g2_4, g2_5]] + if (repeat != 1) { + grad_size.push_back(repeat); + sum_dims.push_back(static_cast(grad_size.size() - 1)); + } + // Don't need to reshape gradient into (repeat, input_shape[dim]) (repeat == + // 1) + grad_size.push_back(input_shape[dim]); + } + // One-time Reshape & Sum + // Reshape gradient to grad_size: + // 1. If repeat equals to 1, append input size at that dimension, + // 2. If repeat is larger than 1, append both repeat and input size at that + // dimension. + // Sum over all "repeat" dimensions from sum_dims: + // Example: + // Input Size (2, 3, 4, 5) + // repeat [4, 1, 9, 3] + // output/grad Size (8, 3, 36, 15) + // grad_size [4, 2, 3, 9, 4, 3, 5] + // sum_dims [0, 3, 5] + + // When repeat 1 time over all original dimensions, the empty sum_dims will + // reduce the whole grad tensor into a scalar rather than keeping original + // dimensions. + if (!sum_dims.empty()) { + grad = grad.reshape_symint(grad_size); + grad = grad.sum(sum_dims); + } + return grad; +} + +// p1m == 1 - p +Tensor _fused_dropout_backward( + const Tensor& grad, + const Tensor& mask, + double p1m) { + if (grad.requires_grad()) { + // Use autograd-friendly backward if double backward is required + return grad * (mask.type_as(grad) * (1. / p1m)); + } else { + return at::_masked_scale(grad, mask, 1. / p1m); + } +} + +// scale == (1 / (1 - prob)) +Tensor infinitely_differentiable_native_dropout_backward( + const Tensor& grad, + const Tensor& mask, + double scale) { + return grad * (mask.type_as(grad) * scale); +} + +Tensor native_dropout_double_backward( + const Tensor& ggI, + const Tensor& grad, + const Tensor& mask, + double scale) { + return ggI.type_as(grad) * (mask.type_as(grad) * scale); +} + +Tensor evenly_distribute_backward( + const Tensor& grad, + const Tensor& input, + const Tensor& value) { + bool any_tensor_subclass_like = + areAnyTensorSubclassLike({grad, input, value}); + if (any_tensor_subclass_like || input.is_cuda()) { + const auto input_isnan = input.isnan(); + const auto value_isnan = value.isnan(); + const auto& input_and_value_isnan = any_tensor_subclass_like + ? input_isnan.logical_and(value_isnan) + : input_isnan.logical_and_(value_isnan); + const auto mask = (input == value).logical_or_(input_and_value_isnan); + return mask * (grad / mask.sum()); + } else { + auto mask = value.isnan().item() ? input.isnan() : input == value; + return grad.new_zeros(input.sizes(), input.options()) + .masked_fill_(mask, grad / mask.sum()); + } +} + +Tensor evenly_read_jvp( + const Tensor& fw_grad, + const Tensor& input, + const Tensor& value) { + auto mask = (input == value); + auto count = mask.sum(); + auto grad_output = fw_grad / count; + return at::sum(mask * grad_output); +} + +Tensor var_backward( + Tensor grad, + const Tensor& self, + at::OptionalIntArrayRef dim_opt, + const std::optional& correction_opt, + bool keepdim) { + const auto correction = correction_opt.value_or(1).toSymFloat(); + if (self.dim() == 0 || !dim_opt.has_value()) { + const auto dof = c10::SymFloat(self.sym_numel()) - correction; + if (dof <= 0) { + // when n == correction, 2 / (n - correction) is infinity + // when self == self.mean(), we return NaN because infinity * 0 = NaN + // otherwise, we return infinity because infinity * c = infinity, for all + // c > 0 + return grad * + at::where( + self == self.mean(), + std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity()); + } else { + return (c10::SymFloat(2.0) / dof) * grad * (self - self.mean()); + } + } + auto dim = dim_opt.value(); + if (!keepdim && self.dim() > 1) { + grad = unsqueeze_multiple(grad, dim, self.dim()); + } + const c10::SymFloat rnumel(_safe_size(self.sym_sizes(), dim)); + return (c10::SymFloat(2.0) / (rnumel - correction)) * grad * + (self - self.mean(dim, /*keepdim=*/true)); +} + +Tensor std_backward( + const Tensor& result, + const Tensor& grad, + const Tensor& self, + at::OptionalIntArrayRef dim, + const std::optional& correction_opt, + bool keepdim) { + auto grad_var = (grad / (result * 2)).masked_fill_(result == 0, 0); + return var_backward(std::move(grad_var), self, dim, correction_opt, keepdim); +} + +Tensor var_mean_backward( + const Tensor& gvar, + const Tensor& gmean, + const Tensor& self, + at::OptionalIntArrayRef dim_opt, + const std::optional& correction_opt, + bool keepdim) { + Tensor gself; + if (gvar.defined()) { + gself = var_backward(gvar, self, dim_opt, correction_opt, keepdim); + } + if (gmean.defined()) { + auto aux = mean_backward( + gmean, + self.sym_sizes(), + dim_opt.value_or(IntArrayRef({})), + self.sym_numel(), + keepdim); + gself = gself.defined() ? gself + aux : std::move(aux); + } + return gself; +} + +Tensor std_mean_backward( + const Tensor& gstd, + const Tensor& gmean, + const Tensor& self, + const Tensor& std, + at::OptionalIntArrayRef dim_opt, + const std::optional& correction_opt, + bool keepdim) { + Tensor gself; + if (gstd.defined()) { + gself = std_backward(std, gstd, self, dim_opt, correction_opt, keepdim); + } + if (gmean.defined()) { + auto aux = mean_backward( + gmean, + self.sym_sizes(), + dim_opt.value_or(IntArrayRef({})), + self.sym_numel(), + keepdim); + gself = gself.defined() ? gself + aux : std::move(aux); + } + return gself; +} + +Tensor cholesky_jvp(const Tensor& dA, const Tensor& L, bool upper) { + at::NoTF32Guard disable_tf32; + // Let A = LL^H + // dA = dLL^H + L(dL)^H + // L^{-1}dA(L^{-H}) = L^{-1}dL + (L^{-1}dL)^H + // = sym(L^{-1}dL) + // where sym(X) = X + X^H + // A short computation gives that the inverse of sym is given by + // \pi(X) = X.tril() - 0.5*diag(X) + // so + // dL = L\pi(L^{-1}dA(L^{-H})) + + // Precondition: dA is symmetric/Hermitian + auto L_ = upper ? L.mH() : L; + auto dL = at::linalg_solve_triangular(L_, dA, /*upper=*/false, /*left=*/true); + dL = at::linalg_solve_triangular(L_.mH(), dL, /*upper=*/true, /*left=*/false); + dL = dL.tril() - dL.diagonal(0, -2, -1).mul(0.5).diag_embed(); + dL = L_.matmul(dL); + return upper ? dL.mH() : std::move(dL); +} + +Tensor cholesky_backward(const Tensor& gL, bool upper, const Tensor& L) { + at::NoTF32Guard disable_tf32; + // From cholesky_jvp we have that + // dL = L\pi(L^{-1}dA(L^-H)) + // + // Let gL be the projection into the lower-triangular gradient wrt L. Taking + // adjoints we have gA = L^{-H}\pi^*((L^HgL).tril())L^{-1} where \pi^*(X) = + // 0.5 * (X + X^H - diag(X)) The only non-standard point of this derivation is + // noting that the adjoint to multiplying on the left by a lower triangular + // matrix L is multiplying by L^H and then projecting back to the lower + // triangular matrices (hence the .tril() projection) Note that the gradient + // is symmetric and not triangular. + auto L_ = upper ? L.mH() : L; + auto gL_ = upper ? gL.mH() : gL; + + // Nb. We don't need to compute gL_ = gL.tril() as + // tril(L^H gL) = tril(L^H (triu(gL, 1) + tril(gL))) + // = tril(L^H tril(gL)) + tril(L^H triu(gL, 1)) + // = tril(L^H tril(gL)) + // since tril(L^H triu(gL, 1)) = 0, as L^H triu(gL, 1) is upper triangular + auto gA = L_.mH().matmul(gL_).tril(); + // Equivalent to 0.5 * (gA + gA^H - diag(gA)) + gA = 0.5 * (gA + gA.tril(-1).mH()); + gA = at::linalg_solve_triangular(L_.mH(), gA, /*upper=*/true, /*left=*/true); + gA = at::linalg_solve_triangular(L_, gA, /*upper=*/false, /*left=*/false); + return gA; +} + +Tensor cholesky_inverse_backward( + const Tensor& grad, + const Tensor& L, + bool upper, + const Tensor& inverse) { + at::NoTF32Guard disable_tf32; + Tensor grad_L; + if (grad.defined()) { + Tensor common_term = grad + grad.mH(); + common_term = at::matmul(inverse, at::matmul(common_term, inverse)); + if (upper) { + grad_L = -at::matmul(L, common_term); + } else { + grad_L = -at::matmul(common_term, L); + } + } + + return grad_L; +} + +// If X = (L L^H)^{-1} with L lower-triangular with a real positive diagonal, +// then dX = K^H + K, where +// K = L^{-H} dL^{-1} [dL^{-1} = -L^{-1} dL L^{-1}] +// = -L^{-H} L^{-1} dL L^{-1} [L^{-H} L^{-1} = X] +// = -X dL L^{-1} [X = X^H = L^{-H} L^{-1} = L^{-1} L^{-H}] +// = -X dL X L^{H}. +// If X = (U^H U)^{-1} with U upper-triangular with a real positive diagonal, +// then K becomes +// K = -X dU^H X U +Tensor cholesky_inverse_jvp( + const Tensor& F, + const Tensor& dF, + const Tensor& X, + bool upper) { + at::NoTF32Guard disable_tf32; + const auto CF = upper ? F : F.mH(); + const auto dCF = upper ? dF.mH() : dF; + const auto partial_dX = -X.matmul(dCF).matmul(X).matmul(CF); + return partial_dX + partial_dX.mH(); +} + +// The formula for forward AD is adapted from +// +// Golub, Gene H., and Victor Pereyra. "The Differentiation of Pseudo-Inverses +// and Nonlinear Least Squares Problems Whose Variables Separate." SIAM Journal +// on Numerical Analysis 10(2). (1973). 413-432. doi: 10.1137/0710036 +// +// We present a short derivation below: +// Let Ap := pinv(A), then Ap is the unique matrix such that +// +// Ap A Ap = Ap [1] +// A Ap A = A [2] +// +// By differentiating [1] we get: +// +// dAp = dAp A Ap + Ap dA Ap + Ap A dAp [3] +// +// In the rhs of [3] the products involving dAp could be expressed as products +// of Ap^i, A^j, dA^k with i, j, k in {1, H}, where X^H = X.mH(). To prove that, +// note (A Ap)^H = A Ap and (Ap A)^H = Ap A, which could be shown by taking the +// product between the SVD decompositions of A and Ap. Consider the +// conjugate-transposed [2]: (A Ap A)^H = A^H (A Ap) = A^H. By differentiating +// it we get: dA^H A Ap + A^H dA Ap + A^H A dAp = dA^H. By multiplying from the +// left by Ap^H and using Ap^H A^H = (A Ap)^H = A Ap: Ap^H dA^H A Ap + A Ap dA +// Ap + A Ap A dAp = Ap^H dA^H. By multiplying from the left by Ap and by +// applying [1] and [2] repeatedly until impossible we get: Ap Ap^H dA^H A Ap + +// Ap dA Ap + Ap A dAp = Ap Ap^H dA^H. By rearranging the terms: +// +// Ap A dAp = -Ap dA Ap + Ap Ap^H dA^H (I - A Ap) [4], +// which is one of the summands in [3]. +// +// Similar, by differentiating the transpose-conjugated [2] written differently, +// i.e. (A Ap A)^H = Ap A A^H = A^H we will get an expression for dAp A Ap, +// which is +// +// dAp A Ap = -Ap dA Ap + (I - Ap A) dA^H Ap^H Ap [5]. +// +// By plugging in [4] and [5] into [3] we get the forward AD formula for pinv: +// +// dAp = -Ap dA Ap + (I - Ap A) dA^H Ap^H Ap + Ap Ap^H dA^H (I - A Ap). +Tensor pinv_jvp(const Tensor& A, const Tensor& pinvA, const Tensor& dA) { + at::NoTF32Guard disable_tf32; + auto m = A.size(-2); + auto n = A.size(-1); + auto dAh = dA.mH(); + auto pinvAh = pinvA.mH(); + // optimization to produce matrices of the smallest dimension + if (m <= n) { + auto K = pinvAh.matmul(dAh); + return pinvA.matmul(K - K.mH() - K.matmul(A.matmul(pinvA))) + + (dAh - pinvA.matmul(A.matmul(dAh))).matmul(pinvAh.matmul(pinvA)); + } else { + auto K = pinvA.matmul(dA); + auto Kh = K.mH(); + return (Kh - K - pinvA.matmul(A).matmul(Kh)).matmul(pinvA) + + (pinvA.matmul(pinvAh)).matmul(dAh - (dAh.matmul(A)).matmul(pinvA)); + } +} + +Tensor pinv_backward(const Tensor& grad, const Tensor& pinvA, const Tensor& A) { + at::NoTF32Guard disable_tf32; + auto m = A.sym_size(-2); + auto n = A.sym_size(-1); + auto pinvAh = pinvA.mH(); + auto gradh = grad.mH(); + // optimization to produce matrices of the smallest dimension + if (m <= n) { + auto K = gradh.matmul(pinvA); + auto KpinvAh = K.matmul(pinvAh); + return -(pinvA.matmul(K)).mH() + KpinvAh - + (A.matmul(pinvA)).matmul(KpinvAh) + + (pinvAh.matmul(pinvA)).matmul(gradh - K.matmul(A)); + } else { + auto K = pinvA.matmul(gradh); + auto pinvAhK = pinvAh.matmul(K); + return -(K.matmul(pinvA)).mH() + + (gradh - A.matmul(K)).matmul(pinvA).matmul(pinvAh) + pinvAhK - + pinvAhK.matmul(pinvA).matmul(A); + } +} + +Tensor chunk_backward_nested( + const std::vector& grads, + const Tensor& self, + int64_t chunks, + int64_t dim) { + TORCH_INTERNAL_ASSERT( + self.layout() == c10::kJagged, + "Nested Strided Tensor doesn't support chunk backward.") + dim = at::maybe_wrap_dim(dim, self.dim()); + Tensor ret = at::zeros_like(self); + std::vector rets = at::chunk(ret, chunks, dim); + for (const auto j : c10::irange(grads.size())) { + if (grads[j].defined()) { + rets[j].copy_(grads[j]); + } + } + return ret; +} + +Tensor split_with_sizes_backward( + const std::vector& grads, + c10::SymIntArrayRef split_sizes, + int64_t dim, + c10::SymIntArrayRef sizes, + const at::TensorOptions& options) { + dim = at::maybe_wrap_dim(dim, static_cast(sizes.size())); + + // it's possible some of the grads are not defined (represents tensors of all + // 0s). Since at::cat can't handle those, let's define them + std::vector grads_all_defined(grads.size()); + for (const auto j : c10::irange(grads.size())) { + if (grads[j].defined()) { + grads_all_defined[j] = grads[j]; + } else { + const auto& length = split_sizes[j]; + auto grad_size = sizes.vec(); + grad_size[dim] = length; + grads_all_defined[j] = at::zeros_symint(grad_size, options); + } + } + + auto ret = at::cat(grads_all_defined, dim); + return ret; +} + +Tensor _nested_split_with_sizes_backward( + const std::vector& grads, + c10::SymIntArrayRef split_sizes, + int64_t dim, + const Tensor& nt_sizes, + const at::TensorOptions& options) { + // add 1 to account for batch dim + dim = at::maybe_wrap_dim(dim, nt_sizes.size(1) + 1); + // it's possible some of the grads are not defined (represents tensors of all + // 0s). Since at::cat can't handle those, let's define them + std::vector grads_all_defined; + grads_all_defined.reserve(grads.size()); + for (int64_t i : c10::irange(static_cast(grads.size()))) { + if (grads[i].defined()) { + grads_all_defined.push_back(static_cast(grads[i])); + } else { + const auto& length = split_sizes[i].guard_int(__FILE__, __LINE__); + auto nt_split_size = nt_sizes.clone(); + auto nt_split_size_ptr = nt_split_size.data_ptr(); + for (int64_t j : c10::irange(nt_sizes.size(0))) { + // subtract 1 to account for batch dim + nt_split_size_ptr[j * nt_sizes.size(1) + (dim - 1)] = length; + } + Tensor zeros_buffer = at::zeros( + {at::native::get_numel_from_nested_size_tensor(nt_split_size)}, + options); + auto nt_split_grad = at::native::wrap_buffer(zeros_buffer, nt_split_size); + grads_all_defined.push_back(nt_split_grad); + } + } + + auto ret = at::cat(grads_all_defined, dim); + return ret; +} + +Tensor split_backward( + const std::vector& grads, + const c10::SymInt& split_size, + int64_t dim, + c10::SymIntArrayRef sym_sizes, + const at::TensorOptions& options) { + dim = at::maybe_wrap_dim(dim, static_cast(sym_sizes.size())); + const auto& dim_size = sym_sizes[dim]; + auto num_splits = grads.size(); + std::vector split_sizes(num_splits, split_size); + split_sizes[num_splits - 1] = split_size - + (split_size * c10::SymInt(static_cast(num_splits)) - dim_size); + return split_with_sizes_backward(grads, split_sizes, dim, sym_sizes, options); +} + +Tensor max_pool_double_backward( + const Tensor& grad, + const Tensor& indices, + int dim) { + AT_ASSERT(indices.dim() >= dim); + // handle non-empty inputs + if (indices.sym_numel() != 0) { + auto size = indices.sym_sizes().slice(0, indices.dim() - dim).vec(); + size.emplace_back(-1); + auto indices_view = indices.view_symint(size); + const auto memory_format = indices.suggest_memory_format(); + return grad.contiguous(memory_format) + .view_symint(size) + .gather(-1, indices_view) + .view_symint(indices.sym_sizes()); + } + // handle empty inputs + else { + return at::empty_like(indices, grad.options()); + } +} + +Tensor error_for_max_pool2d_double_backward() { // This is mps-only. + TORCH_CHECK( + false, + "max_pool2d with `return_indices=False` is not infinitely differentiable.", + " If you want to calculate higher order derivatives, e.g. second order,", + " set `return_indices=True`."); +} + +Tensor glu_double_backward( + const Tensor& grad, + const Tensor& grad_output, + const Tensor& input, + int64_t dim) { + auto& gO = grad_output; + auto input_size = input.size(dim) / 2; + auto first_half = input.narrow(dim, 0, input_size); + auto second_half = input.narrow(dim, input_size, input_size); + auto sig_second_half = second_half.sigmoid(); + auto one_sub_sig_second_half = 1 - sig_second_half; + auto sig_one_sub_sig = sig_second_half * one_sub_sig_second_half; + + auto ggI_first_half = grad.narrow(dim, 0, input_size); + auto ggI_second_half = grad.narrow(dim, input_size, input_size); + auto ggI_second_half_times_first_half = ggI_second_half * first_half; + + auto gI_first_half = ggI_second_half * gO * sig_one_sub_sig; + auto second_order_sh = sig_one_sub_sig * one_sub_sig_second_half - + sig_second_half * sig_one_sub_sig; + auto gI_second_half = + ggI_second_half_times_first_half * gO * second_order_sh + + ggI_first_half * gO * sig_one_sub_sig; + return at::cat({std::move(gI_first_half), std::move(gI_second_half)}, dim); +} + +Tensor glu_double_backward_grad_output( + const Tensor& grad, + const Tensor& input, + int64_t dim) { + if (dim < 0) + dim += input.dim(); + auto sizes = input.sizes().vec(); + sizes[dim] /= 2; + auto tmp = grad * glu_backward(at::ones(sizes, input.options()), input, dim); + return tmp.narrow(dim, 0, sizes[dim]) + + tmp.narrow(dim, sizes[dim], sizes[dim]); +} + +Tensor infinitely_differentiable_silu_backward( + const Tensor& grad_output, + const Tensor& input) { + const Tensor sigmoid = input.sigmoid(); + return grad_output * sigmoid * (1.0 + input * (1.0 - sigmoid)); +} + +Tensor infinitely_differentiable_mish_backward( + const Tensor& grad_output, + const Tensor& input) { + const Tensor sigmoid = input.sigmoid(); + const Tensor softplus = input.exp().log1p(); + const Tensor tanh_softplus = softplus.tanh(); + return grad_output * + (tanh_softplus + input * sigmoid * (1.0 - tanh_softplus * tanh_softplus)); +} + +Tensor infinitely_differentiable_logit_backward( + const Tensor& grad, + const Tensor& self, + std::optional eps) { + if (eps) { + const double lo = eps.value(); + const double hi = 1.0 - lo; + return at::where( + at::logical_and(self >= lo, self <= hi), + grad / (self * (1.0 - self)), + at::zeros({}, self.options())); + } else { + return at::where( + at::logical_and(self >= 0.0, self <= 1.0), + grad / (self * (1.0 - self)), + at::empty({}, self.options()) + .fill_(std::numeric_limits::quiet_NaN())); + } +} + +Tensor binary_cross_entropy_target_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& target, + const std::optional& weight, + int64_t reduction) { + auto grad_target = at::logit(self).neg_(); + + if (!areAnyTensorSubclassLike({grad})) { + grad_target.mul_(grad); + } else { + grad_target = grad_target * grad; + } + + if (isDefined(weight)) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + if (!isTensorSubclassLike(weight.value())) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + grad_target.mul_(weight.value()); + } else { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + grad_target = grad_target * weight.value(); + } + } + + if (reduction == at::Reduction::Mean) { + grad_target.div_(target.sym_numel()); + } + + return grad_target; +} + +Tensor binary_cross_entropy_double_backward_target( + const Tensor& grad, + const Tensor& grad_output, + const Tensor& self, + const Tensor& target, + const std::optional& weight, + int64_t reduction) { + auto res = -grad * grad_output; + + if (isDefined(weight)) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + res = isTensorSubclassLike(weight.value()) + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + ? res.mul(weight.value()) + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + : res.mul_(weight.value()); + } + + auto neg_self = 1 - self; + auto denom = + isTensorSubclassLike(self) ? neg_self.mul(self) : neg_self.mul_(self); + { + at::NoGradGuard guard; + // Default eps in binary_cross_entropy for ALL dtypes + // TODO: probably change this to a dtype-dependent value + double eps = 1e-12; + denom.clamp_min_(eps); + } + + res = isTensorSubclassLike(denom) ? res.div(denom) : res.div_(denom); + + if (reduction == at::Reduction::Mean) { + res.div_(target.sym_numel()); + } + + return res; +} + +Tensor binary_cross_entropy_with_logits_backward( + const Tensor& grad, + const Tensor& input, + const Tensor& target, + const std::optional& weight, + const std::optional& pos_weight, + int64_t reduction) { + // Trivial case + if (grad._is_zerotensor()) { + return at::_efficientzerotensor(input.sizes(), input.options()); + } + + // -w * [ pos * y * (1 -sigmoid(x)) - (1 - y) sigmoid(x)] * grad + + // If there are subclassed tensors use the out of place version + Tensor grad_input; + if (isDefined(pos_weight)) { + // pos_weight might need to be broadcasted, thus mul(target) is not inplace. + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + auto t = pos_weight->mul(target); + grad_input = at::areAnyTensorSubclassLike({input, target}) || + at::GradMode::is_enabled() + ? t.add(1).sub(target).mul(input.sigmoid()).sub(t) + : t.add(1).sub_(target).mul_(input.sigmoid()).sub_(t); + } else { + grad_input = at::areAnyTensorSubclassLike({input, target}) || + at::GradMode::is_enabled() + ? input.sigmoid().sub(target) + : input.sigmoid().sub_(target); + } + + if (at::isTensorSubclassLike(grad) || at::GradMode::is_enabled()) { + grad_input = grad_input.mul(grad); + } else { + grad_input.mul_(grad); + } + + if (isDefined(weight)) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + if (at::isTensorSubclassLike(*weight) || at::GradMode::is_enabled()) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + grad_input = grad_input.mul(*weight); + } else { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + grad_input.mul_(*weight); + } + } + + if (reduction == at::Reduction::Mean) { + grad_input.div_(input.sym_numel()); + } + + return grad_input; +} + +Tensor binary_cross_entropy_with_logits_target_backward( + const Tensor& grad_output, + const Tensor& self, + const Tensor& target, + const std::optional& weight, + const std::optional& pos_weight, + int64_t reduction) { + if (grad_output._is_zerotensor()) { + return at::_efficientzerotensor(target.sizes(), target.options()); + } + + Tensor grad_target; + if (isDefined(pos_weight)) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + if (areAnyTensorSubclassLike({*pos_weight, grad_output})) { + grad_target = at::log_sigmoid(-self) + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + .sub(at::log_sigmoid(self).mul(*pos_weight)) + .mul(grad_output); + } else { + grad_target = at::log_sigmoid(-self) + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + .sub_(at::log_sigmoid(self).mul_(*pos_weight)) + .mul_(grad_output); + } + } else { + grad_target = -self * grad_output; + } + + if (isDefined(weight)) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + if (at::isTensorSubclassLike(*weight)) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + grad_target = grad_target.mul(*weight); + } else { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + grad_target.mul_(*weight); + } + } + + if (reduction == at::Reduction::Mean) { + grad_target.div_(target.sym_numel()); + } + + return grad_target; +} + +Tensor log_sigmoid_double_backward(const Tensor& grad, const Tensor& input) { + auto z = input.sigmoid(); + return grad * (z - 1) * z; +} + +Tensor softmax_double_backward( + const Tensor& grad, + const Tensor& grad_output, + int dim, + const Tensor& output) { + return grad_output * grad - (output * grad_output).sum(dim, true) * grad - + grad_output * (output * grad).sum(dim, true); +} + +// NOTE: [How to write vmap-compatible backward formulas] +// +// See NOTE: [vmap-incompatible in-place operations] for what it means for an +// in-place operation to be incompatible with vmap. +// +// If an in-place operation used in a backward formula is vmap-incompatible, +// then as developers we have the following options: +// +// - If the in-place operation directly followed the creation of a tensor with +// a factory function like at::zeros(...), we should replace the factory with +// a corresponding grad.new_zeros(...) call. The grad.new_zeros(...) call +// propagates the batch dims to the resulting tensor. +// For example: +// Before: at::zeros(input.sizes(), grad.options()).copy_(grad) +// After: grad.new_zeros(input.sizes()).copy_(grad) +// +// - If the in-place operation followed some sequence of operations, if the +// we want to be able to vmap over the backward formula as-is (this is +// usually the case for simple (<15loc) backward formulas), then use +// areAnyTensorSubclassLike to guard the operation. For example: +// c = a * b +// Before: c.mul_(grad) +// After: c = !areAnyTensorSubclassLike({c, grad}) ? c.mul_(grad) : c * +// grad +// +// - If we don't want to vmap directly over the backward formula (e.g., if the +// backward formula is too complicated or has a lot of vmap-incompatible +// operations, then register the backward formula as an operator and +// eventually write a batching rule for it. + +Tensor binary_cross_entropy_double_backward( + const Tensor& grad_output, + const Tensor& grad, + const Tensor& input, + const Tensor& target, + const std::optional& weight, + int64_t reduction) { + auto eps = 1e-12; + auto inp_pl_eps = input + eps; + auto one_m_inp_pl_eps = 1 - input + eps; + // gradient wrt input + auto gI = (input * input - 2 * input * target + target) / + (inp_pl_eps.pow(2) * one_m_inp_pl_eps.pow(2)); + if (!areAnyTensorSubclassLike({gI, grad})) { + gI *= (grad * grad_output); + } else { + gI = gI * (grad * grad_output); + } + + if (isDefined(weight)) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + if (!isTensorSubclassLike(*weight)) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + gI *= *weight; + } else { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + gI = gI.mul(*weight); + } + } + if (reduction == at::Reduction::Mean) { + return gI / input.sym_numel(); + } + + return gI; +} + +Tensor binary_cross_entropy_double_backward_grad_output( + const Tensor& grad, + const Tensor& input, + const Tensor& target, + const std::optional& weight, + int64_t reduction) { + auto eps = 1e-12; + // gradient wrt grad_output + auto ggO = (input - target) / ((input + eps) * (1 - input + eps)); + if (!areAnyTensorSubclassLike({ggO, grad})) { + ggO *= grad; + } else { + ggO = ggO * grad; + } + + if (isDefined(weight)) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + if (!isTensorSubclassLike(*weight)) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + ggO *= *weight; + } else { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + ggO = ggO.mul(*weight); + } + } + if (reduction == at::Reduction::Mean) { + return ggO / input.sym_numel(); + } + return ggO; +} + +Tensor smooth_l1_loss_double_backward( + const Tensor& grad, + const Tensor& input, + const Tensor& target, + int64_t reduction, + double beta) { + // special case to protect against a divide-by-zero. + if (beta == 0) { + return at::zeros(grad.sizes(), grad.options()); + } + auto d = (input - target).abs(); + auto grad_input = grad * (d < beta).type_as(grad) / beta; + if (reduction == at::Reduction::Mean) { + grad_input /= input.sym_numel(); + } + return grad_input; +} + +Tensor huber_loss_double_backward( + const Tensor& grad, + const Tensor& input, + const Tensor& target, + int64_t reduction, + double delta) { + auto d = (input - target).abs(); + auto grad_input = grad * (d < delta); + if (reduction == at::Reduction::Mean) { + grad_input /= input.sym_numel(); + } + return grad_input; +} + +Tensor huber_loss_double_backward_grad_output( + const Tensor& grad, + const Tensor& grad_output, + const Tensor& input, + const Tensor& target, + int64_t reduction, + double delta) { + if (reduction == at::Reduction::None) { + return huber_loss_backward(grad, input, target, reduction, delta); + } + auto r = huber_loss_backward( + ones_like(grad_output), input, target, reduction, delta); + return (r * grad).sum(); +} + +Tensor mse_loss_double_backward( + const Tensor& grad, + const Tensor& input, + int64_t reduction) { + auto grad_input = 2 * grad; + if (reduction == at::Reduction::Mean) { + grad_input /= input.sym_numel(); + } + return grad_input; +} + +Tensor soft_margin_loss_double_backward( + const Tensor& grad, + const Tensor& input, + const Tensor& target, + int64_t reduction) { + auto z = (input * -target).exp(); + auto zplus1 = z + 1; + auto grad_input = grad * (target * target) * z / (zplus1 * zplus1); + if (reduction == at::Reduction::Mean) { + grad_input /= input.sym_numel(); + } + return grad_input; +} + +Tensor soft_margin_loss_double_backward_grad_output( + const Tensor& grad, + const Tensor& grad_output, + const Tensor& input, + const Tensor& target, + int64_t reduction) { + if (reduction == at::Reduction::None) { + return soft_margin_loss_backward(grad, input, target, reduction); + } + auto r = soft_margin_loss_backward( + ones_like(grad_output), input, target, reduction); + return (r * grad).sum(); +} + +Tensor softplus_double_backward( + const Tensor& grad, + const Tensor& input, + const Scalar& beta, + const Scalar& threshold) { + auto x = (input * beta); + return sigmoid_backward(grad, x.sigmoid()) * (x < threshold).type_as(grad) * + beta; +} + +// NOTE [ as_strided Backward and layout-aware/agnostic autograd ] +// +// `storage_offset` is ignored for simplicity in this note. If you just want the +// full algorithm without explanation, scroll down to bottom of this note. +// +// Implementing the backward of as_strided is tricky because you have to deal +// with mappings that map one memory location to multiple indices, i.e., the +// output tensor has multiple indices pointing to **overlapping** memory +// addresses. This can happen in all in all sorts of weird cases. For example, +// +// x = torch.randn(15) +// x.as_strided([3, 3], [1, 0]) # "expand" case +// x.as_strided([3, 3], [2, 1]) # "size too large" case +// x.as_strided([3, 2], [3, 6]) # res[2, 0] points to 2*3 + 0*6 = 6 +// # res[0, 1] points to 0*3 + 1*6 = 6 +// +// Here is the general strategy we apply in implementing as_strided backward: +// 0. ??? (optimization step. we will talk about this later) +// 1. Create some underlying flattened tensor as if it is the base tensor +// representing the contiguous memory storage for both input and output. +// 2. Use the output geometry to scatter (or index_add) the gradients into +// this storage tensor. +// 3. ??? (fix for input tensor with overlapping memory. we will talk about +// this later) +// 4. Return the as_strided view of the storage tensor using input geometry. +// +// In step (2), if the output tensor doesn't have overlapping memory, we can +// safely scatter (`storage.as_strided(output_geometry).copy_(grad)`); +// otherwise, we must use `index_add` as gradients at different indices may need +// to be summed to a single location. +// +// For example, in this case: +// +// x = torch.randn(3) +// y = x.as_strided([3, 3], [1, 0]) # "expand" case +// # size [ 3, 3] +// # stride [ 1, 0] +// y.backward() # step (1): contiguous storagte tensor `s` of size 3, which +// is large enough to be used as underlying storage +// for `x` and `y`. +// s = [ 0, 0, 0] +// # step (2): since `y` has overlapping memory, index_add grad +// into `s` basing on `y`'s geometry, i.e., +// s[i * y.stride(0) + j * y.stride(1)] += gy[i, j]. +// s = [ 3, 3, 3] +// # step (4): as_strided view `s` using `x`'s geometry +// s = [ 3, 3, 3] +// grad_input = s.as_strided(x.size(), x.stride()) +// = s.as_strided([3], [1]) +// = [ 3, 3, 3] +// +// This is exactly what we would get if using `expand`. However, here the input +// tensor doesn't have overlapping memory. If it does, we must add an extra step +// before (4). Considering this case: +// +// t = torch.randn(3) +// x = t.expand(3, 3) # input with overlapping memory +// # size [3, 3] +// # stride [0, 1] +// y = x.as_strided([1], [1]) # contiguous output +// # size [1] +// # stride [1] +// y.backward() # step (1): contiguous storage tensor `s` of size 3, which +// is large enough to be used as underlying storage +// for `x` and `y`. +// s = [ 0, 0, 0] +// # step (2): scatter grad into `s` basing on `y`'s geometry +// s = [ 1, 0, 0] +// # step (4): as_strided view `s` using `x`'s geometry +// s = [ 1, 0, 0] +// grad_input = s.as_strided([3, 3], [0, 1]) +// = s.as_strided([3, 3], [0, 1]) +// = [[ 1, 0, 0], +// [ 1, 0, 0], +// [ 1, 0, 0]] +// Is this result correct? +// +// `x.as_strided([1], [1])` call is obviously equivalent with +// `x[(0,) * x.dim()].view(1)` for any `x`. But autograd through the second +// gives gradient `[ [ 1, 0, 0], [ 0, 0, 0], [ 0, 0, 0]]`. For this specific +// case, indexing `x` at any index in first column is also equivalent, and +// yields a gradient of shape `[3 x 3]` containing eight 0's and one 1. There is +// an `x.size(1)`-times difference between these gradients computed from other +// PyTorch ops and the gradient we got from as_strided. +// +// You might conclude that the gradients from as_strided is wrong. However, +// let's first see why they are actually reasonable. Consider the pointwise +// perturbations by `delta` anywhere in the first column of `x`. It will lead to +// a `delta` change in the same memory location, and then `y` will change by +// `delta`. So one can say the gradient should be exactly 1 at the first column, +// as given by our above procedure. +// +// In the above computation of numerical gradients, they only match the +// analytical results because strides and memory locations are considered in the +// forward pass, i.e., this op (including both forward and backward) is +// layout-aware. +// +// However, in PyTorch, most (probably all) other ops (forward and backward) are +// layout-agnostic. E.g., +// +// t = torch.randn(1) +// x = t.expand(2) +// y = x.sum() +// y.backward() +// +// Layout-agnostic autograd (as it is currently in PyTorch) will give you +// +// gy = 1 +// gx = [ 1, 1] # SumBackward: torch.ones_like(x) +// gt = [ 2] # ExpandBackward: gx.sum() +// +// Note that `gx = [ 1, 1]`. However, if you perturb any value in `x` by `delta` +// (the other will also change by `delta`), `y` will change by `2 * delta`. So +// the gradients, if strides are taken into consideration, should be 2. +// +// Layout-aware autograd should give you +// +// gy = 1 +// gx = [ 2, 2] # Because the backward considers the fact that the input `x` +// # is already expanded. +// gt = [ 2] # Layout-aware backward of expand is just a slicing because +// # the previous backward should have already taken care of +// # strides and made sure that gradients are the same along the +// # expanded dimension. +// +// As shown above, these two types are not compatible. Therefore, we must either +// make as_strided layout-agnostic, or make all other ops layout-aware. +// +// It is difficult to support layout-aware autograd (at least in the current +// codebase structure), because it would mean +// 1. storing tensor geometries of every input tensor for backward +// 2. depending on input geometry, the gradient computed from backward change +// 3. ideally enforcing gradient of T to always have same strides as T +// (although these two methods only differ when it comes to overlapping memory) +// +// Therefore, we must formulate `as_strided` in a layout-agnostic way, i.e., +// giving the same output regardless of the input layout. We consider +// `input.stride()` as a separate independent fixed argument `input_stride`. +// Then, `as_strided(input, size, stride)` can be thought of as: +// 1. "Scatter" each value of `input` into a "storage" using storage location +// computed from the value's index in `input`, `input.size()` and +// `input_stride`, but if N values end up in the same location, the value +// is average of those N values (they will be the same value anyways). +// +// Formal description: +// Denote the set of all input indices that pointing to the same storage +// location `storage[n]` as `S(n)`, i.e., +// +// S(n) = { index : == n, index is valid given +// input.size() }, +// +// where `` is the dot product between `x` and `y`. +// +// Then, the process is: +// +// storage[n] = Avg { S(n) } +// +// Note that all values in `S(n)` are the same (they point to the same +// memory location anyways, so this step doesn't change anything, but +// effectively avoids having the dependency on the layout of `input`. +// I.e., the result holds fixed regardless of the layout of `input`, as +// long as `input_stride` is fixed. +// +// NOTE: for forward pass, we can equivalently simply select any one of +// `S(n)` as `storage[n]`. However, considering this as an average +// operation makes backward easier (so all values in set +// `{ grad_input[i] : i in S(n) }` are the same, and it can use the +// same geometry as input). +// 2. As usual, return the as_strided view of `storage` using required output +// `size` and `stride`. +// +// To backward through this layout-agnostic version, we simply add the following +// step: +// .... (scatter gradients into the storage tensor using output geometry) +// 3. For all storage location n, `storage[n] /= |S(n)|`. +// .... (return as_strided view of the storage tensor using input geometry) +// +// Finally, we note that these general operations are expensive, so we apply the +// following optimizations: +// Add step (0): For all output dimension `d` with output stride 0, sum the +// gradients along dimension `d` (don't keepdim), and remove +// dimension `d` from output size and stride. +// (An optimization for "expand" cases so we may avoid step (3)) +// Only apply step (3) when input tensor has overlapping memory. +// +// FULL ALGORITHM: +// 0. For all output dimension `d` with output stride 0, sum the gradients +// along dimension `d` (don't keepdim), and remove dimension `d` from +// output size and stride. +// 1. Create some underlying flattened tensor as if it is the base tensor +// representing the contiguous memory storage for both input and output. +// 2. Use the output geometry to scatter (or index_add) the gradients into +// this storage tensor `storage`. +// 3. If input tensor has overlapping memory, +// For all storage location `i`, `storage[i] /= N(i)`, where `N(i)` is the +// number of indices in input geometry pointing to the same storage +// location `i` (i.e., `|S(i)|` in equations above). +// 4. Return the as_strided view of the storage tensor using input geometry. +// +// See NOTE [ Detecting Memory Overlap Within A Strided Tensor ] on how to +// roughly detect overlapping memory. + +// NOTE [ Detecting Memory Overlap Within A Strided Tensor ] +// +// Checking memory overlap within a strided tensor is the special case of +// detecting memory overlap of two strided tensors, where the two tensors start +// at the same memory address. The later is HARD (see #8212). +// +// But even this special case isn't simple. This note describes a check for a +// even more constrained simple case where we can be certain that there is no +// overlap. +// +// The checking algorithm can be described as: +// 0. Return [ pass check ] if any dimension has size 0 +// 1. Ignore all dimensions that have size 1 +// 2. If no remaining dimensions, return [ pass check ] +// 3. Sort the remaining dimensions according to the strides decreasingly +// 4. Check that for each dimension k, +// +// stride[k] > \sum_{ i > k } (size[i] - 1) * stride[i] +// +// That is equivalent to, after reordering the dimensions so strides are +// in decreasing order, checking that stride of each dimension is larger +// than the maximum memory offset in a slice at that dimension. +// +// Obviously this check passes for contiguous tensors ( the dimensions will be +// already sorted with LHS = stride[0] = \prod size[i] being exactly 1 larger +// than RHS ). Similarly, the check passes for tensors contiguous in all but +// the last dimension, and LHS = stride[0] = stride[-1] * \prod size[i] being +// exactly stride[-1] larger than RHS. (*) +// +// We will show that these view operations, including all our view operations +// *except for* general as_strided and unfold, also preserve this invariant: +// +// alias: Obviously preserves +// +// expand: All changed dimensions are removed in step (1) +// +// view: Consider the input dimensions as grouped into consecutive +// dimension "blocks", where dimensions are contiguous in each one. +// one. view only works when the output dimensions can also be +// grouped into the same consecutive blocks of same ordering. +// +// NB: this means that the number of elements and stride of the +// last dimension in each block is the same in input and +// output. (**) +// +// Notation: +// Consider a single such block B, +// ... B_prev[-1]], [ B[0], ..., B[i], ..., B[k] = B[-1] ], [ +// B_next[0], ... +// start--^^^^ ^^^^^^^^^^^^--end +// Each B[i] denotes a dimension index such that B[i] = B[0] + i. +// +// We first show that in a tensor (i.e., input) satisfies the +// invariant, after sorting, the dimensions within each block +// still remain consecutive. (***) +// +// After removing dimensions of size 1, the dimensions within a +// block is already sorted by strides in descending order. So +// sorting all dimensions will not change the relative ordering +// among them. +// +// Assume that some block B is not consecutive after sorting, +// i.e., there exists a dimension d between B[0] and B[-1] in +// sorted order. +// +// By (*), we know that +// stride[B[0]] +// = \sum_{i > 0} (size[B[i]] - 1) * stride[B[i]] + +// stride[B[-1]] < \sum_{i > 0} (size[B[i]] - 1) * +// stride[B[i]] + stride[d] +// <= \sum_{i > 0} (size[B[i]] - 1) * stride[B[i]] + +// (size[d] - 1) * stride[d] +// <= \sum{j > B[0]} (size[j] - 1) * stride[j], +// +// where the first < comes from sorting and +// the second <= comes from the fact that dimension d +// exists after step (1) and +// thus must have size greater +// than 1 +// the third <= comes from the fact that each term in +// the sum is non-negative +// +// Then we have a countradiction as the invariant must not be +// satisfied at B[0]. So the original proposition is true. +// +// Now that we established the above claim (***), we consider the +// view operation as first sorting the dimensions (i.e., blocks), +// apply the original view (since it only cares dimensions being +// consecutive and contiguous within each block), and then undo +// the sort. +// +// Consider a single block B in the output, +// ... ], [ B[0], ..., B[i], ..., B[k] = B[-1] ], [ ... +// start--^^^^ ^^^^^^^^^^^^--end +// +// By (*), we know that for all i +// stride[i] = stride[B[-1]] + +// \sum_{j=i+1}^{k} (size[B[j]] - 1) * +// stride[B[j]] +// +// Then the invariant is obviously satisfied at every dimension +// in this block if it is satisfied at dimension B[-1]. It only +// remains to show that it is satisfied at the last dimension in +// each block. +// +// Since the same blocks are present in both input and output +// with the same ordering, we will abuse the notation in the +// following statements. +// +// By (*), we know that the following holds for both input and +// output, for any block B: +// \sum_{i > B[-1]} (size[i] - 1) * stride[i] +// = \sum_{block B' after B} \prod_{j in B'} size[B[j]] * +// stride[B'[-1]] = \sum_{block B' after B} numel(B') * +// stride[B'[-1]]. +// ^^^^^^^^^^^^^^^^^^^^^^^|^^^^^^^^^^^^^^^^^^^^^^^^^^ +// By (**), we know that, this quantity in the above equation +// remains the same in input and output. So both +// \sum_{i > B[-1]} (size[i] - 1) * stride[i] +// and +// stride[B[-1]] +// are the same in input and output. +// +// These two quantities are exactly the LHS and RHS of the +// invariant inequality. Since by assumption the invariant is +// satisfied in input at B[-1], it is also satisfied in output at +// B[-1]. This concludes the proof. +// +// squeeze: Special case of view +// +// unsqueeze: Special case of view +// +// slice: Consider slicing dimension i with step = k >= 1. +// +// Let stride' and size' be the output strides and sizes. We have +// +// stride'[i] = k * stride[i] +// size'[i] <= floor(size[i] / k) +// +// If size'[i] = 1, invariant is obviously satisfied as we are +// just removing a dimension (after step (1)). +// +// Assume size'[i] > 1. +// +// By assumption, the invariant is satisfied at every dimension +// in input. +// +// For any dimension j, if stride[j] > stride[i], we have +// stride'[j] = stride[j] +// > (size[i] - 1) * stride[i] +// = (size[i] / k * k - 1) * k * stride[i] / k +// = (size[i] / k - 1 / k) * stride'[i] +// >= (size'[i] - 1 / k) * stride'[i] +// >= stride'[i]. +// +// If stride[j] < stride[i], we have +// stride'[j] = stride[j] < stride[i] <= stride'[i]. +// +// So the sorting order remains unchanged after slice. +// +// Since +// (size'[i] - 1) * stride'[i] +// = (floor(size[i] / k) - 1) * k * stride[i] +// <= (size[i] / k - 1) * k * stride[i] +// = (size[i] - k) * stride[i] +// <= (size[i] - 1) * * stride[i], +// the term from this dimension i in the invariant inequality at +// other dimensions can only decrease after slice. So the +// invariant is preserved. +// +// narrow: Special case of slice +// +// select: narrow + squeeze +// +// permute: Sorting makes permutation of dimensions irrelevant +// +// transpose: Sorting makes swapping dimensions irrelevant +// +// diagonal: Effectively merging two dimensions i and j into a new +// dimension k s.t. +// stride'[k] = stride[i] + stride[j] +// size'[k] <= min(size[i], size[j]), +// where stride and size are on the input, and stride' and size' +// are on the output. +// +// Assuming that size[i] > 1 and size[j] > 1. If any has size 1, +// then this is unsqueeze on that dimension. +// +// WLOG, say stride[i] >= stride[j]. +// +// Each dimension d in input with stride[d] > stride[j] has +// stride'[d] = stride[d] +// > (size[i] - 1) * stride[i] + (size[j] - 1) * +// stride[j] +// >= stride[i] + stride[j] +// = stride[k]. +// So, considering the sorted dimensions, this is effectively +// removing i, and replacing j with k. +// +// For dimensions d with stride[i] < stride[d] < stride[j], the +// term from dimension i is removed in the invariant inequality. +// For dimensions d with stride[d] > stride[j], we have +// (size'[k] - 1) * stride'[k] +// <= (min(size[i], size[j]) - 1) * (stride[i] + stride[j]) +// <= (size[i] - 1) * stride[i] + (size[j] - 1) * stride[j], +// so the term from i and j in the invariant can only decrease. +// +// So this is generally relaxing the constraint, and thus it +// preserves it. + +// This implements steps (2)~(4) of the algorithm in +// NOTE [ Detecting Memory Overlap Within A Strided Tensor ] +// Helper for as_strided_backward +static bool _maybe_overlapping_memory( + c10::SymIntArrayRef sizes, + c10::SymIntArrayRef strides) { + if (!sizes.empty()) { + std::vector argsort(sizes.size()); + std::iota(argsort.begin(), argsort.end(), 0); + std::sort( // NOLINT(modernize-use-ranges) + argsort.begin(), + argsort.end(), + [&](std::size_t i, std::size_t j) { return strides[i] < strides[j]; }); + + c10::SymInt max_index_in_slice = 0; + for (auto i : argsort) { + const auto& stride_ = strides[i]; + if (stride_ <= max_index_in_slice) { + return true; + } + max_index_in_slice += stride_ * (sizes[i] - 1); + } + } + return false; +} + +// Returns the minimum storage size needed to contain a tensor of sizes, +// strides, and storage_offset Helper for as_strided_backward +static c10::SymInt _min_storage_size( + c10::SymIntArrayRef sizes, + c10::SymIntArrayRef strides, + c10::SymInt storage_offset) { + c10::SymInt storage_size = storage_offset + 1; + auto dim = sizes.size(); + for (const auto i : c10::irange(dim)) { + const auto& size_i = sizes[i]; + if (size_i == 0) { + return storage_offset; + } + storage_size += (size_i - 1) * strides[i]; + } + return storage_size; +} + +// See NOTE [ as_strided Backward and layout-aware/agnostic autograd ] for +// explanation +Tensor as_strided_backward( + Tensor grad, + const TensorGeometry& input_geometry, + c10::SymIntArrayRef sym_sizes, + c10::SymIntArrayRef sym_strides, + const std::optional& sym_storage_offset_) { + // For output geometry, + // check for size 0 dimensions, + // skip size 1 dimensions, + // reduce grad on expanded dims (stride=0, size>1) + // Step (0) for the algorithm in NOTE [ as_strided Backward and + // layout-aware/agnostic autograd ] Step (0)~(1) for the algorithm in NOTE [ + // Detecting Memory Overlap Within A Strided Tensor ] + // on output geometry + auto sym_storage_offset = + sym_storage_offset_.value_or(input_geometry.sym_storage_offset()); + auto odim = grad.dim(); + std::vector out_sizes_, out_strides_; + out_sizes_.reserve(odim); + out_strides_.reserve(odim); + for (int64_t i = odim - 1; i >= 0; i--) { + const auto& size_i = sym_sizes[i]; + const auto& stride_i = sym_strides[i]; + if (size_i == 0) { + return at::zeros_symint(input_geometry.sym_sizes(), grad.options()); + } else if (size_i == 1) { + grad = grad.squeeze(i); + } else if (stride_i == 0) { + grad = grad.sum(i, false); + } else { + out_sizes_.insert(out_sizes_.begin(), size_i); + out_strides_.insert(out_strides_.begin(), stride_i); + } + } + // Step (2)~(4) for the algorithm in NOTE [ Detecting Memory Overlap Within A + // Strided Tensor ] + // on output geometry + auto out_maybe_overlap = _maybe_overlapping_memory(out_sizes_, out_strides_); + + // For input geometry, + // check for size 0 dimensions, + // skip size 1 dimensions, + // Step (0)~(1) for the algorithm in NOTE [ Detecting Memory Overlap Within A + // Strided Tensor ] + // on input geometry + auto idim = input_geometry.dim(); + auto inp_sizes = input_geometry.sym_sizes(), + inp_strides = input_geometry.sym_strides(); + std::vector inp_sizes_, inp_strides_; + inp_sizes_.reserve(idim); + inp_strides_.reserve(idim); + for (int64_t i = idim - 1; i >= 0; i--) { + const auto& size_i = inp_sizes[i]; + const auto& stride_i = inp_strides[i]; + if (size_i == 0) { + return at::zeros_symint(input_geometry.sym_sizes(), grad.options()); + } else if (size_i != 1) { + inp_sizes_.insert(inp_sizes_.begin(), size_i); + inp_strides_.insert(inp_strides_.begin(), stride_i); + } + } + // Step (1)~(4) for the algorithm in NOTE [ Detecting Memory Overlap Within A + // Strided Tensor ] + // on input geometry + auto inp_maybe_overlap = _maybe_overlapping_memory(inp_sizes_, inp_strides_); + + // Rest of this function implements + // Step (1)~(4) for the algorithm in NOTE [ as_strided Backward and + // layout-aware/agnostic autograd ] + // TODO: Raise if not all output values are visible in input geometry. + // Technically speaking, if you treat those values as constants, not + // raising is fine, and mathematically correct. However, these values + // really are contained in some base tensor, and by treating them as + // constants we are ignoring this tight dependency. Therefore, it is + // more sensible to raise here. + + // Step (1): create underlying tensor as "storage" + auto shared_offset = + // TODO: symint-ify. Do we need a min() and max() for SymInts? + input_geometry.sym_storage_offset().min(sym_storage_offset); + auto inp_effective_offset = + input_geometry.sym_storage_offset() - shared_offset; + auto out_effective_offset = sym_storage_offset - shared_offset; + auto base_size1 = + _min_storage_size(inp_sizes_, inp_strides_, inp_effective_offset); + auto base_size2 = + _min_storage_size(out_sizes_, out_strides_, out_effective_offset); + auto base_size = base_size1.max(base_size2); + auto storage = grad.new_zeros_symint(c10::SymIntArrayRef(base_size)); + + // prepare indices tensor if we will do index_add_ later + std::optional flatten_full_indices; + if (inp_maybe_overlap || out_maybe_overlap) { + flatten_full_indices = + // TODO: should we symint-ify arange? Need SymScalar. + at::arange( + 0, + base_size.guard_int(__FILE__, __LINE__), + grad.options().dtype(at::kLong)); + } + + // Step (2): use output geometry to scatter gradients into storage + if (out_maybe_overlap) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + auto out_indices = flatten_full_indices->as_strided_symint( + out_sizes_, out_strides_, out_effective_offset); + storage.index_add_(0, out_indices.reshape(-1), grad.reshape(-1)); + } else { + // assume that new tensors have 0 storage offset + storage.as_strided_symint(out_sizes_, out_strides_, out_effective_offset) + .copy_(grad); + } + + // Step (3): if input tensor has overlapping memory, divide scattered gradient + // at storage[i] by the number of times i shows up in input geometry + if (inp_maybe_overlap) { + auto count = at::zeros_like(storage, LEGACY_CONTIGUOUS_MEMORY_FORMAT); + auto inp_indices = + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + flatten_full_indices + ->as_strided_symint(inp_sizes_, inp_strides_, inp_effective_offset) + .reshape(-1); + count.index_add_( + 0, inp_indices, at::ones({1}, grad.options()).expand_as(inp_indices)); + storage.div_(count); // this will give nan outside visible range + } + // Step (4): return as_strided view of the storage tensor with input geometry + return storage.as_strided_symint( + inp_sizes, inp_strides, inp_effective_offset); +} + +Tensor as_strided_scatter_backward( + const Tensor& grad, + const TensorGeometry& input_geometry, + const TensorGeometry& src_geometry, + c10::SymIntArrayRef sizes, + c10::SymIntArrayRef strides, + std::optional storage_offset) { + // Note [as_strided_scatter backward support] + // as_strided_scatter handling for autograd is a beast, and is non-trivial to + // implement for arbitrarily strided inputs. Most uses for as_strided with + // functionalization only care about the contiguous case anyway, So for now + // this is not implemented. When autograd is being used, we ban non-contiguous + // inputs. We can assume that the input was a contiguous tensor. Also, we'll + // take the perf hit and contiguify grad for now. + auto grad_ = grad.contiguous(); + auto grad_slice = grad_.as_strided_symint(sizes, strides, storage_offset); + auto result_buffer = grad_.new_zeros_symint(input_geometry.sym_sizes()); + auto result = result_buffer.as_strided_symint( + input_geometry.sym_sizes(), input_geometry.sym_strides()); + auto result_slice = result_buffer.as_strided_symint( + sizes, strides, std::move(storage_offset)); + result_slice.copy_(grad_slice); + return result; +} + +std::tuple atan2_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& other, + std::array output_mask) { + if (!grad.defined()) { + return std::tuple{Tensor(), Tensor()}; + } + auto denom = self * self + other * other; + auto recip = denom.reciprocal(); + if (at::areAnyTensorSubclassLike({self, other, denom, recip}) || + at::GradMode::is_enabled()) { + recip = recip.masked_fill(denom == 0, 0); + } else { + recip.masked_fill_(denom == 0, 0); + } + return std::tuple{ + output_mask[0] ? grad * other * recip : Tensor(), + output_mask[1] ? grad * -self * recip : Tensor()}; +} + +Tensor gelu_double_backward( + const Tensor& ggI, + const Tensor& gO, + const Tensor& input, + std::string_view approximate) { + // if (at::native::get_gelutype_enum(approximate) == + // at::native::GeluType::Tanh) { + if (approximate == "tanh") { + constexpr auto kBeta = M_SQRT2 * M_2_SQRTPI * 0.5; + constexpr auto kKappa = 0.044715; + + auto inner = kBeta * (input + kKappa * pow(input, 3)); + auto tanh_inner = tanh(inner); + auto sech_inner = 1 / cosh(inner); + + auto f = 0.5 * input; + auto g = 1 - tanh_inner * tanh_inner; + auto h = kBeta * (1 + 3 * kKappa * input * input); + + auto f_prime_gh = 0.5 * g * h; + + auto g_prime = (2 * sech_inner) * (-sech_inner * tanh_inner) * h; + auto g_prime_fh = f * h * g_prime; + + auto h_prime = 6 * kKappa * input * kBeta; + auto h_prime_fg = f * g * h_prime; + + // left_derivative = f_prime_gh + // right_derivative = f_prime_gh + g_prime_fh + h_prime_fg + // dgrad_dX = left_derivative + right_derivative + auto gI = ggI * gO * (2 * f_prime_gh + g_prime_fh + h_prime_fg); + return gI; + } else { + constexpr auto kBeta = M_2_SQRTPI * M_SQRT1_2 * 0.5; + auto input_sq = input * input; + auto pdf = kBeta * at::exp(-0.5 * input_sq); + auto dgrad_dInput = 2 * pdf - input_sq * pdf; + auto gI = ggI * gO * dgrad_dInput; + return gI; + } +} + +Tensor elu_double_backward( + const Tensor& grad, + const Tensor& grad_output, + const Scalar& alpha, + const Scalar& scale, + const Scalar& input_scale, + bool is_result, + const Tensor& self_or_result) { + if (is_result) { + return grad * grad_output * input_scale * + (self_or_result < 0).type_as(grad); + } else { + return at::elu_backward( + grad * grad_output * input_scale, + alpha, + scale, + input_scale, + is_result, + self_or_result) * + (self_or_result < 0).type_as(grad); + } +} + +Tensor slice_backward_wrapper( + const at::Tensor& grad, + const c10::SymIntArrayRef& input_sizes, + int64_t dim, + std::optional start, + std::optional end, + c10::SymInt step) { + auto start_val = start.has_value() ? start.value() : 0; + auto end_val = end.has_value() ? end.value() : INT64_MAX; + + return slice_backward_symint( + grad, + input_sizes, + dim, + std::move(start_val), + std::move(end_val), + std::move(step)); +} + +std::tuple linalg_svd_jvp( + const Tensor& dA, + const Tensor& U_, + const Tensor& S, + const Tensor& Vh_, + const bool full_matrices) { + at::NoTF32Guard disable_tf32; + // See svd_backward for the derivation + // With sym(X) = X + X^H, we implement + // dU = U (sym(dX S) / E + i Im(diag(dX)) / (2S)) + // if m > n + // dU = [dU for m == n] + (I_m - UU^H) dA V S^{-1} + // dS = Re(diag(dP)) + // dV = V (sym(S dX) / E - i Im(diag(dX)) / (2S)) + // if m < n + // dV = [dV for m == n] + (I_n - VV^H) (dA)^H U S^{-1} + // dVh = dV^H + // with dP = U^H dA V + // dX = dP - dS + // E_{jk} = S_k^2 - S_j^2 if j != k + // 1 otherwise + + // Checks compute_uv=true + TORCH_INTERNAL_ASSERT(U_.dim() >= 2 && Vh_.dim() >= 2); + + const auto is_complex = dA.is_complex(); + const auto m = dA.size(-2); + const auto n = dA.size(-1); + const auto k = S.size(-1); + + const auto U = full_matrices ? U_.narrow(-1, 0, k) : U_; + const auto Vh = full_matrices ? Vh_.narrow(-2, 0, k) : Vh_; + const auto V = Vh.mH(); + + // dP = U^H dA V + // U^H (dA V) is O(km(n + k)) + // (U^H dA) V is O(kn(m + k)) + // So prefer U^H (dA V) if m < n + auto dP = m < n ? at::matmul(U.mH(), at::matmul(dA, V)) + : at::matmul(at::matmul(U.mH(), dA), V); + + auto dS = + is_complex ? at::real(dP.diagonal(0, -2, -1)) : dP.diagonal(0, -2, -1); + + // dX = dP - dS + dP = dP - dS.diag_embed(); + + auto E = [&S] { + const auto S2 = S * S; + auto ret = S2.unsqueeze(-2) - S2.unsqueeze(-1); + // Any number a != 0 would, as we are just going to use it to compute 0 / a + // later on + ret.diagonal(0, -2, -1).fill_(1); + return ret; + }(); + + const auto sym = [](const Tensor& X) { return X + X.mH(); }; + + // diag(dP) / (2S) + auto diagdP2S = is_complex ? dP.diagonal(0, -2, -1).div(2. * S) : Tensor{}; + + // dU = U (sym(dP S) / E) + i Im(diag(dP)) / (2S) + auto dU = [&] { + auto dUaux = sym(dP * S.unsqueeze(-2)) / E; + if (is_complex) { + dUaux = dUaux + diagdP2S.diag_embed(); + } + return at::matmul(U, dUaux); + }(); + if (m > n) { + // dU += (I_m - UU^H) dA V S^{-1} + const auto dAVSinv = at::matmul(dA, V / S.unsqueeze(-2)); + dU = dU + dAVSinv - at::matmul(U, at::matmul(U.mH(), dAVSinv)); + + // To "fix" the full_matrices case (the full_matrices case should not be + // differentiable...) + if (full_matrices) { + auto shape = dU.sizes().vec(); + shape.end()[-1] = m - n; + dU = at::cat({dU, dU.new_zeros(shape)}, /*dim=*/-1); + } + } + + // dVh = -sym(S dP) / E + i Im(diag(dP)) / (2S) + // Perf: We negate the S as it's the smallest tensor in the equation + auto dVh = [&] { + auto dVhaux = sym(dP * (-S).unsqueeze(-1)) / E; + if (is_complex) { + dVhaux = dVhaux + diagdP2S.diag_embed(); + } + return at::matmul(dVhaux, Vh); + }(); + if (m < n) { + // dVh += S^{-1} U^H dA (I_n - VV^H) + const auto UHdASinv = at::matmul(U.mH() / S.unsqueeze(-1), dA); + dVh = dVh + UHdASinv - at::matmul(at::matmul(UHdASinv, V), Vh); + + // To "fix" the full_matrices case (the full_matrices case should not be + // differentiable...) + if (full_matrices) { + auto shape = dVh.sizes().vec(); + shape.end()[-2] = n - m; + dVh = at::cat({dVh, dVh.new_zeros(shape)}, /*dim=*/-2); + } + } + + return std::make_tuple(std::move(dU), std::move(dS), std::move(dVh)); +} + +Tensor svd_backward( + const Tensor& gU, + const Tensor& gS, + const Tensor& gVh, + const Tensor& U, + const Tensor& S, + const Tensor& Vh) { + at::NoTF32Guard disable_tf32; + // Throughout both the real and complex case we assume A has distinct singular + // values. Furthermore, if A is rectangular or complex, we assume it's + // full-rank. + // + // + // The real case (A \in R) + // See e.g. https://j-towns.github.io/papers/svd-derivative.pdf + // + // Denote by skew(X) = X - X^T, and by A o B the coordinatewise product, then + // if m == n + // gA = U [(skew(U^T gU) / E)S + S(skew(V^T gV) / E) + I o gS ]V^T + // where E_{jk} = S_k^2 - S_j^2 if j != k and 1 otherwise + // + // if m > n + // gA = [term in m == n] + (I_m - UU^T)gU S^{-1} V^T + // if m < n + // gA = [term in m == n] + U S^{-1} (gV)^T (I_n - VV^T) + // + // + // The complex case (A \in C) + // This one is trickier because the svd is not locally unique. + // Denote L = diag(e^{i\theta_k}), then we have that if A = USV^H, then (UL, + // S, VL) is another valid SVD decomposition of A as A = ULS(VL)^H = + // ULSL^{-1}V^H = USV^H, since L, S and L^{-1} commute, since they are all + // diagonal. + // + // Assume wlog that n >= k in what follows, as otherwise we could reason about + // A^H. Denote by St_k(C^n) = {A \in C^{n,k} | A^H A = I_k} the complex + // Stiefel manifold. What this invariance means is that the svd decomposition + // is not a map svd: C^{n x k} -> St_k(C^n) x R^n x St_k(C^k) (where St_k(C^k) + // is simply the unitary group U(k)) but a map svd: C^{n x k} -> M x R^n where + // M is the manifold given by quotienting St_k(C^n) x U(n) by the action (U, + // V) -> (UL, VL) with L as above. Note that M is a manifold, because the + // action is free and proper (as U(1)^k \iso (S^1)^k is compact). For this + // reason, pi : St_k(C^n) x U(n) -> M forms a principal bundle. + // + // To think about M, consider the case k = 1. Then, we have the bundle + // pi : St_1(C^n) x U(1) -> M + // now, St_1(C^n) are just vectors of norm 1 in C^n. That's exactly the sphere + // of dimension 2n-1 in C^n \iso R^{2n} S^{2n-1} = { z \in C^n | z^H z = 1}. + // Then, in this case, we're quotienting out U(1) completely, so we get that + // pi : S^{2n-1} x U(1) -> CP(n-1) + // where CP(n-1) is the complex projective space of dimension n-1. + // In other words, M is just the complex projective space, and pi is (pretty + // similar to) the usual principal bundle from S^{2n-1} to CP(n-1). The case k + // > 1 is the same, but requiring a linear independence condition between the + // vectors from the different S^{2n-1} or CP(n-1). + // + // Note that this is a U(1)^k-bundle. In plain words, this means that the + // fibres of this bundle, i.e. pi^{-1}(x) for x \in M are isomorphic to U(1) x + // ... x U(1). This is obvious as, if pi(U,V) = x, pi^{-1}(x) = {(U + // diag(e^{i\theta}), V diag(e^{i\theta})) | \theta \in R^k} + // = {(U diag(z), V diag(z)) | z \in U(1)^k} + // since U(1) = {z \in C | |z| = 1}. + // + // The big issue here is that M with its induced metric is not locally + // isometric to St_k(C^n) x U(k). [The why is rather technical, but you can + // see that the horizontal distribution is not involutive, and hence + // integrable due to Frobenius' theorem] What this means in plain words is + // that, no matter how we choose to return the U and V from the SVD, we won't + // be able to simply differentiate wrt. U and V and call it a day. An example + // of a case where we can do this is when performing an eigendecomposition on + // a real matrix that happens to have real eigendecomposition. In this case, + // even though you can rescale the eigenvectors by any real number, you can + // choose them of norm 1 and call it a day. In the eigenvector case, we are + // using that you can isometrically embed S^{n-1} into R^n. In the svd case, + // we need to work with the "quotient manifold" M explicitly, which is + // slightly more technically challenging. + // + // Since the columns of U and V are not uniquely defined, but are + // representatives of certain classes of equivalence which represent elements + // M, the user may not depend on the particular representative that we return + // from the SVD. In particular, if the loss function depends on U or V, it + // must be invariant under the transformation (U, V) -> (UL, VL) with L = + // diag(e^{i\theta})), for every \theta \in R^k. In more geometrical terms, + // this means that the loss function should be constant on the fibres, or, in + // other words, the gradient along the fibres should be zero. We may see this + // by checking that the gradients as element in the tangent space T_{(U, + // V)}(St(n,k) x U(k)) are normal to the fibres. Differentiating the map (U, + // V) -> (UL, VL), we see that the space tangent to the fibres is given by + // Vert_{(U, V)}(St(n,k) x U(k)) = { i[U, V]diag(\theta) | \theta in R^k} + // where [U, V] denotes the vertical concatenation of U and V to form an (n+k, + // k) matrix. Then, solving = 0 for two matrices + // S, T \in T_{(U, V)}(St(n,k) x U(k)) where = Re tr(A^H B) is the + // canonical (real) inner product in C^{n x k} we get that the function is + // invariant under action of U(1)^k iff Im(diag(U^H gU + V^H gV)) = 0 + // + // Using this in the derivation for the forward AD, one sees that, with the + // notation from those notes Using this and writing sym(X) = X + X^H, we get + // that the forward AD for SVD in the complex case is given by dU = U (sym(dX + // S) / E + i Im(diag(dX)) / (2S)) if m > n + // dU = [dU for m == n] + (I_m - UU^H) dA V S^{-1} + // dS = Re(diag(dP)) + // dV = V (sym(S dX) / E - i Im(diag(dX)) / (2S)) + // if m < n + // dV = [dV for m == n] + (I_n - VV^H) (dA)^H U S^{-1} + // dVh = dV^H + // with dP = U^H dA V + // dX = dP - dS + // E_{jk} = S_k^2 - S_j^2 if j != k + // 1 otherwise + // + // Similarly, writing skew(X) = X - X^H + // the adjoint wrt. the canonical metric is given by + // if m == n + // gA = U [((skew(U^H gU) / E) S + i Im(diag(U^H gU)) / S + S ((skew(V^H gV) + // / E)) + I o gS] V^H + // if m > n + // gA = [term in m == n] + (I_m - UU^H)gU S^{-1} V^H + // if m < n + // gA = [term in m == n] + U S^{-1} (gV)^H (I_n - VV^H) + // where we have used that Im(diag(U^H gU)) = - Im(diag(V^h gV)) to group the + // diagonal imaginary terms into one that just depends on U^H gU. + + // Checks compute_uv=true + TORCH_INTERNAL_ASSERT(U.dim() >= 2 && Vh.dim() >= 2); + + // Trivial case + if (!gS.defined() && !gU.defined() && !gVh.defined()) { + return {}; + } + + const auto m = U.sym_size(-2); + const auto n = Vh.sym_size(-1); + + // Optimisation for svdvals: gA = U @ diag(gS) @ Vh + if (!gU.defined() && !gVh.defined()) { + return m >= n ? at::matmul(U, gS.unsqueeze(-1) * Vh) + : at::matmul(U * gS.unsqueeze(-2), Vh); + } + // At this point, at least one of gU, gVh is defined + + const bool is_complex = U.is_complex(); + const auto skew = [](const Tensor& A) { return A - A.mH(); }; + const auto UhgU = gU.defined() ? skew(at::matmul(U.mH(), gU)) : Tensor{}; + const auto VhgV = gVh.defined() ? skew(at::matmul(Vh, gVh.mH())) : Tensor{}; + + // Check for the invariance of the loss function, i.e. + // Im(diag(U^H gU)) + Im(diag(V^H gV)) = 0 + if (is_complex) { + const auto imdiag_UhgU = + gU.defined() ? at::imag(UhgU.diagonal(0, -2, -1)) : at::zeros_like(S); + const auto imdiag_VhgV = + gVh.defined() ? at::imag(VhgV.diagonal(0, -2, -1)) : at::zeros_like(S); + // Rather lax atol and rtol, as we don't want false positives + TORCH_CHECK( + at::allclose(imdiag_UhgU, -imdiag_VhgV, /*rtol=*/1e-2, /*atol=*/1e-2), + "svd_backward: The singular vectors in the complex case are specified up to multiplication " + "by e^{i phi}. The specified loss function depends on this phase term, making " + "it ill-defined."); + } + + // gA = ((U^H gU) / E) S + S (((V^H gV) / E) + I o (gS + diag(U^H gU) / (2 * + // S)) + Tensor gA = [&] { + // ret holds everything but the diagonal of gA + auto ret = [&] { + const auto E = [&S] { + const auto S2 = S * S; + auto ret = S2.unsqueeze(-2) - S2.unsqueeze(-1); + // Any number a != 0 would, as we are just going to use it to compute 0 + // / a later on + ret.diagonal(0, -2, -1).fill_(1); + return ret; + }(); + + if (gU.defined()) { + if (gVh.defined()) { + return (UhgU * S.unsqueeze(-2) + S.unsqueeze(-1) * VhgV) / E; + } else { + return (UhgU / E) * S.unsqueeze(-2); + } + } else { // gVh.defined(); + return S.unsqueeze(-1) * (VhgV / E); + } + }(); + // Fill the diagonal + if (gS.defined()) { + ret = ret + gS.diag_embed(); + } + if (is_complex && gU.defined() && gVh.defined()) { + ret = ret + (UhgU.diagonal(0, -2, -1) / (2. * S)).diag_embed(); + } + return ret; + }(); + + if (m > n && gU.defined()) { + // gA = [UgA + (I_m - UU^H)gU S^{-1}]V^H + gA = at::matmul(U, gA); + const auto gUSinv = gU / S.unsqueeze(-2); + gA = gA + gUSinv - at::matmul(U, at::matmul(U.mH(), gUSinv)); + gA = at::matmul(gA, Vh); + } else if (m < n && gVh.defined()) { + // gA = U[gA V^H + S^{-1} (gV)^H (I_n - VV^H)] + gA = at::matmul(gA, Vh); + const auto SinvgVh = gVh / S.unsqueeze(-1); + gA = gA + SinvgVh - at::matmul(at::matmul(SinvgVh, Vh.mH()), Vh); + gA = at::matmul(U, gA); + } else { + // gA = U gA V^H + gA = m >= n ? at::matmul(U, at::matmul(gA, Vh)) + : at::matmul(at::matmul(U, gA), Vh); + } + + return gA; +} + +Tensor linalg_eig_backward( + const Tensor& gL, + const Tensor& gV, + const Tensor& L, + const Tensor& V, + const bool is_hermitian, + const bool symeig_eigenvectors) { + at::NoTF32Guard disable_tf32; + // https://arxiv.org/pdf/1701.00392.pdf Eq 4.77 + // For A = VLV^{-1}, denoting the gradients gA, gV and gL, we have + // gA = V^{-H}(diag_embed(gL) + (V^H gV -V^HV diag(real(V^H gV))) / E*)V^H + // Where: + // - E_{ij} = L_j - L_i if i != j + // 1 otherwise + // - diag_embed takes a vector into a diagonal matrix + // - diag zeroes out elements outside of the diagonal + + // Note: the term '-V^HV diag(real(V^H gV))' comes from the fact that the + // eigenvalue decomposition is returned with eigenvectors normalized to have + // norm one. + + // Note: The Hermitian case is a simplification of this formula using that + // V^{-1} = V^H and that L is real + + // This check just can be triggered in the backwards of torch.symeig + TORCH_CHECK( + symeig_eigenvectors, + "linalg_eig_backward: torch.symeig(A, eigenvectors=False) is not differentiable. ", + "Use torch.linalg.eigvalsh(A) instead."); + + // Trivial case + if (!gL.defined() && !gV.defined()) { + return {}; + } + + // Shortcut for linalg.eigvals/eigvalsh + // Compute V^-H gL V^H + if (!gV.defined()) { + if (is_hermitian) { + return at::matmul(V * gL.unsqueeze(-2), V.mH()); + } else { + return at::linalg_solve(V.mH(), gL.unsqueeze(-1) * V.mH()); + } + } + auto VhgV = at::matmul(V.mH(), gV); + const auto diag_VhgV = VhgV.diagonal(0, -2, -1); + + if (V.is_complex() && !at::isTensorSubclassLike(diag_VhgV)) { + // Check invariance of the loss function wrt the transformation + // V -> V * e^{i\phi} for an arbitrary phi in RR^n + const auto imdiag_VhgV = at::imag(diag_VhgV); + TORCH_CHECK( + at::allclose( + imdiag_VhgV, + at::zeros_like(imdiag_VhgV), + /*rtol=*/1e-2, + /*atol=*/1e-2), + is_hermitian ? "linalg_eigh_backward" : "linalg_eig_backward", + ": The eigenvectors in the complex case are specified up to multiplication ", + "by e^{i phi}. The specified loss function depends on this quantity, so it is ill-defined."); + } + + if (is_hermitian) { + // Project onto the tangent space at the identity of U(n), that is, the + // skew-Hermitian matrices + VhgV = 0.5 * (VhgV - VhgV.mH()); + } else { + // Project onto the tangent space at V^H V of complex matrices with columns + // of norm 1 + VhgV = VhgV - at::matmul(V.mH(), V * at::real(diag_VhgV).unsqueeze(-2)); + } + + auto gA = [&, VhgV = std::move(VhgV)] { + auto Econj = [&L] { + auto Lconj = L.conj(); + auto ret = Lconj.unsqueeze(-2) - Lconj.unsqueeze(-1); + ret.diagonal(0, -2, -1).fill_(1.); + return ret; + }(); + + auto ret = VhgV.div_(Econj); + + if (gL.defined()) { + // For CompositeCompliance, if `gL` is subclass but `ret` + // is a regular Tensor, then use out-of-place version of diagonal + // copy aka `diagonal_scatter`. + if (at::isTensorSubclassLike(gL)) { + ret = ret.diagonal_scatter(gL, 0, -2, -1); + } else { + ret.diagonal(0, -2, -1).copy_(gL); + } + } + return ret; + }(); + + // Conjugate by V^{-H} + if (is_hermitian) { + return at::matmul(V, at::matmul(gA, V.mH())); + } else { + return at::linalg_solve(V.mH(), at::matmul(gA, V.mH())); + } +} + +std::tuple linalg_eig_jvp( + const Tensor& dA, + const Tensor& L, + const Tensor& V, + const bool is_hermitian) { + at::NoTF32Guard disable_tf32; + // https://people.maths.ox.ac.uk/gilesm/files/NA-08-01.pdf + // see also https://arxiv.org/pdf/1701.00392.pdf Eqs. (4.60) and (4.63) + // Note that neither of the formulas in these pdfs are correct, as they do not + // assume that the eigenvectors are of unit norm. As such, they are missing + // the diagonal term in dV dL = diag(dP) dV = dX - V Re(diag V^H dX)) where dP + // = V^{-1} dA V dX = V ((dP - diag(dP)) / E) E_{ij} = L_j - L_i if i != j + // 1 otherwise + + // Precondition: if is_hermitian == true, then dA is Hermitian + const auto to_complex = [](const Tensor& A) { + return A.to(c10::toComplexType(A.scalar_type())); + }; + + const auto dP = is_hermitian + ? at::matmul(at::matmul(V.mH(), dA), V) + : at::linalg_solve(V, at::matmul(to_complex(dA), V)); + auto dL = is_hermitian && dA.is_complex() ? at::real(dP.diagonal(0, -2, -1)) + : dP.diagonal(0, -2, -1); + auto dV = [&dP, &V, &L, is_hermitian] { + auto dX = [&] { + auto ret = dP / (L.unsqueeze(-2) - L.unsqueeze(-1)); + ret.diagonal(0, -2, -1).zero_(); + ret = at::matmul(V, ret); + return ret; + }(); + + if (is_hermitian) { + return dX; + } else { + return dX - + V * + at::real(at::matmul(V.mH(), dX).diagonal(0, -2, -1)).unsqueeze(-2); + } + }(); + return std::make_pair(std::move(dL), std::move(dV)); +} + +Tensor linalg_lstsq_solution_jvp( + const Tensor& A, + const Tensor& B_, + const Tensor& dA, + const Tensor& dB_) { + at::NoTF32Guard disable_tf32; + const bool vector_case = at::native::linalg_solve_is_vector_rhs(A, B_); + const auto vector_to_matrix = [vector_case](const Tensor& X) { + return vector_case ? X.unsqueeze(-1) : X; + }; + const auto matrix_to_vector = [vector_case](const Tensor& X) { + return vector_case ? X.squeeze(-1) : X; + }; + auto B = vector_to_matrix(B_); + auto dB = vector_to_matrix(dB_); + auto pinvA = at::linalg_pinv(A); + auto dpinvA = pinv_jvp(A, pinvA, dA); + auto dX = matrix_to_vector(dpinvA.matmul(B) + pinvA.matmul(dB)); + return dX; +} + +Tensor linalg_lstsq_residuals_jvp( + const Tensor& A, + const Tensor& B_, + const Tensor& dA, + const Tensor& dB_, + const Tensor& X_, + const Tensor& L) { + at::NoTF32Guard disable_tf32; + if (L.numel() == 0) { + return L.clone(); + } + const bool vector_case = at::native::linalg_solve_is_vector_rhs(A, B_); + const auto vector_to_matrix = [vector_case](const Tensor& X) { + return vector_case ? X.unsqueeze(-1) : X; + }; + auto B = vector_to_matrix(B_); + auto dB = vector_to_matrix(dB_); + auto X = vector_to_matrix(X_); + auto r = A.matmul(X) - B; + auto dr = dA.matmul(X) - dB; + // Danskin's theorem lets us compute dL as if X did not depend on A and B + auto dL = 2 * at::real(r * dr.conj()).sum(-2); + return dL; +} + +std::tuple linalg_lstsq_backward( + const Tensor& gX_, + const Tensor& gL_, + const Tensor& A, + const Tensor& B_, + const Tensor& X_, + const std::array& grad_input_mask) { + at::NoTF32Guard disable_tf32; + auto A_requires_grad = grad_input_mask[0]; + auto B_requires_grad = grad_input_mask[1]; + if ((!gX_.defined() && !gL_.numel()) || // gL_ undefined or have shape [0] + (!A_requires_grad && !B_requires_grad)) { + return {}; + } + + const bool vector_case = at::native::linalg_solve_is_vector_rhs(A, B_); + const auto vector_to_matrix = [vector_case](const Tensor& X) { + return vector_case ? X.unsqueeze(-1) : X; + }; + const auto matrix_to_vector = [vector_case](const Tensor& X) { + return vector_case ? X.squeeze(-1) : X; + }; + + auto B = vector_to_matrix(B_); + Tensor A_grad_X, B_grad_X, A_grad, B_grad; + + if (gX_.defined()) { // Gradient from solution + auto gX = vector_to_matrix(gX_); + Tensor pinvA = at::linalg_pinv(A); + if (A_requires_grad) { + auto pinvA_grad = gX.matmul(B.mH()); + A_grad_X = pinv_backward(pinvA_grad, pinvA, A); + } + if (B_requires_grad) { + // Equivalent to + // B_grad = std::get<0>(at::linalg_lstsq(A.mH(), gX, rcond, driver)); + // but we avoid this approach as `gelsy` is non-deterministic + B_grad_X = matrix_to_vector(pinvA.mH().matmul(gX)); + } + } + + if (gL_.numel()) { // Gradient from residuals + auto X = vector_to_matrix(X_); + auto r = A.matmul(X) - B; + auto gL = gL_.unsqueeze(-2); + if (A_requires_grad) { + auto A_grad_L = 2 * (gL * r).matmul(X.mH()); + A_grad = A_grad_X.defined() ? A_grad_X + A_grad_L : A_grad_L; + } + if (B_requires_grad) { + auto B_grad_L = matrix_to_vector(-2 * gL * r); + B_grad = B_grad_X.defined() ? B_grad_X + B_grad_L : B_grad_L; + } + } else { // gX_.defined() == true + A_grad = A_grad_X; + B_grad = B_grad_X; + } + + return std::make_tuple(std::move(A_grad), std::move(B_grad)); +} + +std::tuple linalg_qr_jvp( + const Tensor& dA, + const Tensor& Q, + const Tensor& R, + const std::string_view mode) { + // dA = dQR + QdR + // + // Case m >= n + // We can put dQ in terms of dR + // dQ = dAR^{-1} - QdRR^{-1} + // Then we have + // Q^H dA R^{-1} = Q^HdQ + dRR^{-1} + // where Q^HdQ is skew Hermitian and dRR^{-1} is upper triangular + // Define sym(X) = X + X^H + // sym(dRR^{-1}) = sym(Q^H dA R^{-1}) + // and define syminv(X) = triu(X) - 0.5 * diag(X) the inverse of + // sym : Triu(k, diag \in \mathbb{R}) -> Her(k) to give + // dR = syminv(sym(Q^H dA R^{-1}))R + // + // Case m < n + // Put dR as a function of dQ + // dR = Q^H dA - Q^H dQ R + // Let X_1 be the main m x m submatrix of a matrix X \in C^{m x n} + // Q^H A_1 R_1^{-1} = Q^H dQ + dR_1 R_1^{-1} + // Define trilIm(X) = X.tril(-1) + i * Im diag(X) + // trilIm(Q^H dQ) = trilIm(Q^H A_1 R_1^{-1}) + // and define trilIminv(X) = X - X^H - i*Im diag(X). This is the inverse of + // trilIm : Skew_C(m) -> Tril(m, imaginary diag) + // Note that it is just the inverse when the inputs are skew-Hermitian, not + // necessarily when the inputs are arbitrary matrices. We then get dQ = Q + // trilImInv(trilIm(Q^H A_1 R_1^{-1})) + at::NoTF32Guard disable_tf32; + + auto [compute_q, reduced] = at::native::_parse_qr_mode(mode); + + TORCH_CHECK( + compute_q, + "The derivative of linalg.qr depends on Q, which is not computed when " + "mode='r'. Please use linalg.qr(A, mode='reduced') if you are " + "going to differentiate through linalg.qr."); + auto m = dA.size(-2); + auto n = dA.size(-1); + + TORCH_CHECK( + reduced || m <= n, + "The QR decomposition is not differentiable when " + "mode='complete' and nrows > ncols."); + if (m >= n) { + const auto sym = [](const Tensor& X) { return X + X.mH(); }; + const auto syminv = [](const Tensor& X) { + auto ret = X.triu(); + ret.diagonal(0, -2, -1).mul_(0.5); + return ret; + }; + auto dARinv = + at::linalg_solve_triangular(R, dA, /*upper=*/true, /*left=*/false); + auto dR = syminv(sym(Q.mH().matmul(dARinv))); + auto dQ = dARinv - Q.matmul(dR); + dR = dR.matmul(R); + return std::make_tuple(std::move(dQ), std::move(dR)); + } else { + const auto trilim = [](const Tensor& X) { + if (X.is_complex()) { + auto ret = X.tril(); + at::real(ret.diagonal(0, -2, -1)).zero_(); + return ret; + } else { + return X.tril(-1); + } + }; + const auto triliminv = [](const Tensor& X) { + if (X.is_complex()) { + auto ret = X - X.mH(); + ret.diagonal(0, -2, -1).mul_(0.5); + return ret; + } else { + return X - X.mT(); + } + }; + + auto QHdA = Q.mH().matmul(dA); + auto QHdA1Rinv = at::linalg_solve_triangular( + R.narrow(-1, 0, m), + QHdA.narrow(-1, 0, m), + /*upper=*/true, + /*left=*/false); + auto dQ = triliminv(trilim(QHdA1Rinv)); + auto dR = QHdA - dQ.matmul(R); + dQ = Q.matmul(dQ); + return std::make_tuple(std::move(dQ), std::move(dR)); + } +} + +Tensor linalg_qr_backward( + const Tensor& gQ, + const Tensor& gR, + const Tensor& Q, + const Tensor& R, + const std::string_view mode) { + // Nb. We won't be too formal below, as writing this proof formally is a pain + // We'll link here a formal writing of all this at some point in the future + // + // Case m >= n + // dQ = dAR^{-1} - Qsyminv(sym(Q^H dA R^{-1})) + // dR = syminv(sym(Q^H dA R^{-1}))R + // + // With the notation from the JVP formula, the only two computations that we + // need are syminv*(R) = 0.5 * (R.triu() + R.triu()^H - Re diag(R)) sym*(X) = + // 2 * X Using these, after a few simplifications we get that gA = (gQ + + // syminvadj(triu(gR R^H - Q^H gQ)))R^{-H} + // + // Case m < n + // dR = Q^H dA - Q^H dQ R + // dQ = Q trilImInv(trilIm(Q^H A_1 R_1^{-1})) + // + // In this case trilIm*(X) = X (it's the trivial embedding) + // while trilImInv*(X) = tril(Y) - 0.5 * diag(Y) + // with Y = X - X^H + // + // We also have that if X \in C^{m, n} an dpi(X) = X_1, + // projects X into its leading m x m submatrix, + // pi*(X) = cat(X, 0_{m,n-m}, dim=-1) + // + // Using this, we get that + // gA = QgR + pi*(Q trilImInv*(Q^H gQ - gR R^H)R_1^{-H}) + at::NoTF32Guard disable_tf32; + + auto [compute_q, reduced] = at::native::_parse_qr_mode(mode); + + TORCH_CHECK( + compute_q, + "The derivative of linalg.qr depends on Q, which is not computed when " + "mode='r'. Please use linalg.qr(A, mode='reduced') if you are " + "going to differentiate through linalg.qr."); + + auto m = Q.sym_size(-2); + auto n = R.sym_size(-1); + + TORCH_CHECK( + reduced || m <= n, + "The QR decomposition is not differentiable when " + "mode='complete' and nrows > ncols."); + + if (!gQ.defined() && !gR.defined()) { + return {}; + } + + Tensor gA; + if (gQ.defined()) { + if (gR.defined()) { + gA = gR.matmul(R.mH()) - Q.mH().matmul(gQ); + } else { + gA = -Q.mH().matmul(gQ); + } + } else { + gA = gR.matmul(R.mH()); + } + if (m >= n) { + const auto syminvadj = [](const Tensor& X) { + auto ret = X + X.mH(); + at::real(ret.diagonal(0, -2, -1)).mul_(0.5); + return ret; + }; + gA = Q.matmul(syminvadj(gA.triu())); + if (gQ.defined()) { + gA = gA + gQ; + } + gA = at::linalg_solve_triangular( + R.mH(), gA, /*upper*/ false, /*left*/ false); + return gA; + } else { + auto trilImInvAdjSkew = [](const Tensor& X) { + auto ret = (X - X.mH()).tril(); + if (X.is_complex()) { + at::imag(ret.diagonal(0, -2, -1)).mul_(0.5); + } + return ret; + }; + gA = Q.matmul(trilImInvAdjSkew(-gA)); + gA = at::linalg_solve_triangular( + R.narrow_symint(-1, 0, m).mH(), gA, /*upper*/ false, /*left*/ false); + auto shape = R.sym_sizes().vec(); + shape.end()[-1] = n - m; + gA = at::cat({gA, gA.new_zeros_symint(shape)}, /*dim=*/-1); + if (gR.defined()) { + gA = gA + Q.matmul(gR); + } + return gA; + } +} + +// Based on: +// +// Mathias, Roy. +// A Chain Rule for Matrix Functions and Applications. +// SIAM J. Matrix Anal. Appl. 17 (1996): 610-620. + +template +static Tensor differential_analytic_matrix_function( + const Tensor& self, + const Tensor& grad, + const func_t& matrix_function, + const bool adjoint // Choose between forward (adjoint=false) or backward AD + // (adjoint=true) +) { + // Given an analytic matrix function, this computes the differential (forward + // AD) or the adjoint of the differential (backward AD) + auto A = adjoint ? self.transpose(-2, -1).conj() : self; + auto meta_grad_sizes = A.sym_sizes().vec(); + meta_grad_sizes[A.dim() - 2] *= 2; + meta_grad_sizes[A.dim() - 1] *= 2; + + auto n = A.sym_size(-1); + Tensor meta_grad; + // For Composite Compliance, we can't copy a Subclass into a Regular Tensor, + // so we use out-of-place ops with equivalent output. + // NOTE: We can't use `new_zeros` directly as both `A` and `grad` can + // be Tensor Subclass and we don't want to make assumption about which + // one to choose for creating output buffer. + // eg. if both are BatchedTensor at different level. + if (areAnyTensorSubclassLike({A, grad})) { + meta_grad = at::cat( + {at::cat({A, grad}, -1), + at::cat({at::zeros_like(A), std::move(A)}, -1)}, + -2); + } else { + meta_grad = at::zeros_symint(meta_grad_sizes, grad.options()); + meta_grad.narrow_symint(-2, 0, n).narrow_symint(-1, 0, n).copy_(A); + meta_grad.narrow_symint(-2, n, n).narrow_symint(-1, n, n).copy_(A); + meta_grad.narrow_symint(-2, 0, n).narrow_symint(-1, n, n).copy_(grad); + } + + return matrix_function(meta_grad).narrow_symint(-2, 0, n).narrow_symint( + -1, n, n); +} + +Tensor linalg_matrix_exp_differential( + const Tensor& self, + const Tensor& grad, + bool adjoint) { + at::NoTF32Guard disable_tf32; + + return differential_analytic_matrix_function( + self, grad, at::linalg_matrix_exp, /* adjoint */ adjoint); +} + +// Differential of the symmetric/Hermitian matrix square root, used for both +// reverse-mode (grad = cotangent) and forward-mode (grad = input tangent). +// For A = Q diag(lambda) Q^H the Daleckii-Krein/Loewner derivative is +// T(E) = Q (W o (Q^H sym(E) Q)) Q^H with W_ij = 1 / (sqrt(l_i) + sqrt(l_j)), +// which for f = sqrt needs no separate diagonal case and no divided-difference +// cancellation handling. W is real symmetric and Q unitary, so T is +// self-adjoint w.r.t. the real trace inner product; hence the same operator +// serves the VJP and the JVP. Requires positive-definite input (the denominator +// vanishes at a zero eigenvalue). +Tensor linalg_matrix_sqrth_differential( + const Tensor& self, + const Tensor& grad) { + if (!grad.defined()) { + return {}; + } + at::NoTF32Guard disable_tf32; + auto [eigvals, eigvecs] = at::linalg_eigh(self); + auto sqrt_eigvals = eigvals.clamp_min(0).sqrt(); + auto denom = sqrt_eigvals.unsqueeze(-1) + sqrt_eigvals.unsqueeze(-2); + auto grad_sym = 0.5 * (grad + grad.mH()); + auto inner = + at::matmul(at::matmul(eigvecs.mH(), grad_sym), eigvecs).div(denom); + auto out = at::matmul(at::matmul(eigvecs, inner), eigvecs.mH()); + return 0.5 * (out + out.mH()); +} + +template +static Tensor masked_fmap( + const Tensor& mask, + const F1& f1, + const F2& f2, + const Tensor& t, + const Ts&... ts) { + // This function takes two functions f1 and f2 and a (variadic) list of + // tensors, and creates a new tensor of the same shape as the first element of + // the list of tensors by applying the function f1 to the tensors for which + // the mask is true and f2 to the tensors for which the mask is false This + // function is used when we have a formula that works for, say, all + // non-singular inputs and another one for when the inputs are singular. See + // for example det_backward + + // Precondition for the n == 0 case to make sense + TORCH_INTERNAL_ASSERT(t.sym_numel() != 0); + auto t_masked = t.index({mask}); + auto n = t_masked.sym_numel(); + if (n == t.sym_numel()) { + return f1(t, ts...); + } else if (n == 0) { + return f2(t, ts...); + } else { + // Equivalent to + // ret = torch.empty_like(t) + // ret[mask] = f1(t1[mask], ..., tn[mask]) + // ret[~mask] = f2(t1[~mask], ..., tn[~mask]) + auto not_mask = mask.logical_not(); + return at::empty_like(t) + .index_put_({mask}, f1(t_masked, ts.index({mask})...)) + .index_put_( + {not_mask}, f2(t.index({not_mask}), ts.index({not_mask})...)); + } +} + +Tensor linalg_det_jvp( + const Tensor& dA, + const Tensor& det, + const Tensor& LU, + const Tensor& pivots, + const bool use_A_T) { + // (d det)_A(E) = tr(A^{-1}E)*det + // We use that the determinant is C^1 to approximate the gradient of singular + // inputs Since we never differentiate over forward AD, we don't need to deal + // with further gradients, as we do in grad_backward + auto eps = at::native::_get_epsilon(c10::toRealValueType(LU.scalar_type())); + auto LU_ = + LU + at::diag_embed(at::where(LU.diagonal(0, -2, -1) == 0., eps, 0.)); + auto AinvE = + at::linalg_lu_solve(LU_, pivots, dA, /*left=*/true, /*adjoint=*/use_A_T); + return AinvE.diagonal(0, -2, -1).sum(-1) * det; +} + +Tensor linalg_det_backward( + const Tensor& grad, + const Tensor& det, + const Tensor& A, + const Tensor& LU, + const Tensor& pivots) { + at::NoTF32Guard disable_tf32; + if (!grad.defined()) { + return {}; + } + if (A.sym_numel() == 0) { + return at::zeros_like(A); + } + + // Special case handling for 1 x 1 matrix, to ensure mathematically correct. + // d(det)/dA = 1, so gradient = grad * ones_like(A) + // See #80761 + if (A.sym_size(-2) == 1 && A.sym_size(-1) == 1) { + // For batched 1x1 matrices, broadcast grad to match A's shape + return grad.unsqueeze(-1).unsqueeze(-1).expand_as(A); + } + + // The gradient G is the matrix solving + // A.mH G = det(A).conj() * grad * I + auto d_diag = grad * det.conj(); + // Optimisation, Make it F-transposed as it's what lu_solve expects + auto d = at::diag_embed(d_diag.unsqueeze(-1).expand_as(pivots)).mT(); + auto eps = at::native::_get_epsilon(c10::toRealValueType(LU.scalar_type())); + + // Optimisation if we are not going to compute higher-order gradients + if (!at::GradMode::is_enabled()) { + // The formula is given by the solution of AX = det.conj() * det * I when A + // is invertible det is C^1, so if it's not invertible, we can apply a + // perturbation to the LU decomposition and use the resulting matrix as a + // non-singular approximation + auto LU_ = + LU + at::diag_embed(at::where(LU.diagonal(0, -2, -1) == 0., eps, 0.)); + auto use_A_T = A.is_contiguous() && !A.is_complex(); + return at::linalg_lu_solve( + LU_, pivots, d, /*left=*/true, /*adjoint=*/!use_A_T); + } else { + // If we want to compute higher-order gradients, we need to recompute the + // LU decomposition so that autograd computes the correct gradients wrt + // to A (cf. solve_backward) + auto non_singular = + [](const Tensor& A, const Tensor& d, const Tensor& /*grad*/) { + return at::linalg_solve(A.mH(), d); + }; + + // The derivative may be then computed explicitly by noting that the + // gradient of the derivative of the determinant is given in terms of the + // adjugate of a matrix. The adjugate of a singular matrix may be computed + // as per https://nhigham.com/2020/06/16/what-is-the-adjugate-of-a-matrix/ + auto singular = [](const Tensor& A, + const Tensor& /*d*/, + const Tensor& grad) { + auto [U, S, Vh] = at::linalg_svd(A); + auto alpha = (at::linalg_det(U) * at::linalg_det(Vh)).conj() * grad; + auto D = prod_safe_zeros_backward(alpha.unsqueeze(-1), S, S.dim() - 1); + return (U * D.unsqueeze(-2)).matmul(Vh); + }; + + // We could use the singular formula for all inputs but we try to filter out + // some inputs via the masking, as computing an SVD is about 100 times + // slower than computing an lu_solve on GPU + // For tensor subclasses, we can't call masked_fmap as it calls + // index({mask}) which needs to call item to compute the number of elements + // in the result. + + if (areAnyTensorSubclassLike({A, d, grad})) { + return singular(A, d, grad); + } else { + return masked_fmap( + det.abs() < 100. * eps, singular, non_singular, A, d, grad); + } + } +} + +std::tuple slogdet_jvp( + const Tensor& LU, + const Tensor& pivots, + const Tensor& dA, + const Tensor& sign, + const bool use_A_T) { + // No need to handle the singular case separately as we do in det since + // this function is not differentiable on singular matrices + auto trAinvE = at::linalg_lu_solve(LU, pivots, dA, /*left*/ true, use_A_T) + .diagonal(0, -2, -1) + .sum(-1); + if (LU.is_complex()) { + auto i = c10::complex{0.0, 1.0}; + return std::make_tuple(at::imag(trAinvE) * (i * sign), at::real(trAinvE)); + } else { + return std::make_tuple( + at::_efficientzerotensor(sign.sizes(), sign.options()), + std::move(trAinvE)); + } +} + +Tensor slogdet_backward( + const Tensor& grad_sign, + const Tensor& grad_logabsdet, + const Tensor& A, + const Tensor& signdet, + const Tensor& LU, + const Tensor& pivots) { + // We compute the complex case, as the real case follows from it + // Forward AD + // d (logabsdet)_A(E) = Re(tr(A^{-1}E)) + // d (signdet)_A(E) = sgn * Im(tr(A^{-1}E)) * i + // So + // d (logabsdet)*_A(g) = gA^{-H} + // Now, to compute the adjoint of d(signdet), note that + // Re(z * Im(w)) = Re(-Re(z)iw) + // So, let g \in C, + // = Re(g.conj() * sgn * i * Im(A^{-1}E)) + // = Re(Re(g.conj() * sgn * i) * -i * A^{-1}E) + // = Re(Im(g.conj() * sgn) * i * A^{-1}E) + // = + // As such, + // (d slogabs)*_A(g_sign, g_abs) = (g_abs - g_sign.conj() * sgn) * A^{-H} + + if (!grad_sign.defined() && !grad_logabsdet.defined()) { + return {}; + } + + auto is_complex = A.is_complex(); + + // In the real case grad_sign is always zero + if (!is_complex && !grad_logabsdet.defined()) { + return {}; + } + + auto g = grad_logabsdet; + if (is_complex) { + if (grad_sign.defined()) { + auto i = c10::complex{0.0, 1.0}; + if (g.defined()) { + g = g - i * at::imag(grad_sign.conj() * signdet); + } else { + g = -i * at::imag(grad_sign.conj() * signdet); + } + } else { + // Cast to complex explicitly + g = g.to(A.scalar_type()); + } + } + + // No need to handle the singular case separately here (as we do in det) + // since this function is not differentiable on singular matrices + // Optimisation, Make it F-transposed as it's what lu_solve expects + auto d = at::diag_embed(g.unsqueeze(-1).expand_as(pivots)).mT(); + if (!at::GradMode::is_enabled()) { + auto use_A_T = A.is_contiguous() && !A.is_complex(); + return at::linalg_lu_solve( + LU, pivots, d, /*left=*/true, /*adjoint=*/!use_A_T); + } else { + // If we want to compute further gradients, we need to recompute the LU + // decomposition so that autograd computes the correct gradients wrt to A + // (cf. solve_backward) + return at::linalg_solve(A.mH(), d); + } +} + +// Reference: +// https://people.maths.ox.ac.uk/gilesm/files/NA-08-01.pdf +// Sec. 2.3.1 Matrix inverse product +std::tuple triangular_solve_backward( + const Tensor& grad_x, + const Tensor& grad_m, + const Tensor& b, + const Tensor& a, + const Tensor& x, + const bool upper, + const bool transpose, + const bool unitriangular, + std::array output_mask) { + at::NoTF32Guard disable_tf32; + Tensor grad_b, grad_a; + if (grad_x.defined() || grad_m.defined()) { + if (grad_x.defined()) { + grad_b = std::get<0>( + grad_x.triangular_solve(a.conj(), upper, !transpose, unitriangular)); + if (output_mask[1]) { + grad_a = + transpose ? -x.conj().matmul(grad_b.mT()) : -grad_b.matmul(x.mH()); + if (upper) { + grad_a = grad_a.triu((int)unitriangular); + } else { + grad_a = grad_a.tril(-((int)unitriangular)); + } + } + } + if (!grad_a.defined()) { + grad_a = at::zeros({1}, a.options()).expand_as(a); + } + if (!grad_b.defined()) { + grad_b = at::zeros({1}, b.options()).expand_as(b); + } + if (output_mask[1] && grad_m.defined()) { + grad_a = grad_a.add(grad_m); + } + } + return std::tuple{grad_b, grad_a}; +} + +Tensor triangular_solve_jvp( + const Tensor& X, + const Tensor& A, + const Tensor& dA, + const Tensor& dB, + const bool upper, + const bool transpose, + const bool unitriangular) { + return generic_solve_jvp( + [&](const Tensor& A, const Tensor& dB, const Tensor& dA_contrib) { + return std::get<0>(at::triangular_solve( + dB - dA_contrib, A, upper, transpose, unitriangular)); + }, + X, + A, + dA, + dB); +} + +Tensor linalg_solve_triangular_forward_AD( + const Tensor& A_t, + const Tensor& B_t, + const Tensor& A, + const Tensor& X, + const bool upper, + const bool left, + const bool unitriangular) { + at::NoTF32Guard disable_tf32; + // The forward AD formula (for left = true) is A^{-1}(B_t - A_tX) + // For the derivation see: + // [Note: Forward / Backward AD solve_triangular] + const Tensor proj_A_t = upper ? A_t.triu(static_cast(unitriangular)) + : A_t.tril(-static_cast(unitriangular)); + const Tensor X_t = + B_t - (left ? at::matmul(proj_A_t, X) : at::matmul(X, proj_A_t)); + return at::linalg_solve_triangular(A, X_t, upper, left, unitriangular); +} + +std::tuple linalg_solve_triangular_backward( + const Tensor& grad, + const Tensor& A, + const Tensor& X, + const bool upper, + const bool left, + const bool unitriangular, + std::array output_mask) { + at::NoTF32Guard disable_tf32; + const bool A_requires_grad = output_mask[0]; + const bool B_requires_grad = output_mask[1]; + // [Note: Forward / Backward AD solve_triangular] + // Assume left=true for simplicity. + // Remark: A solver computes A^{-1}B + // + // Forward AD: + // If f(A) = A^{-1}, differentiating the equation A^{-1}A = I_n gives + // (df)_A(E) = -A^{-1}EA^{-1} + // As such, if g(A,B) = A^{-1}B, + // (dg)_(A,B)(E_A, E_B) = -A^{-1}E_AA^{-1}B + A^{-1}E_B + // = A^{-1}(E_B - E_AX) + + // Backward AD: + // Denoting the gradients by G_A, G_B, we solve above to give + // G_B = A^{-H}G_X + // G_A = -A^{-H}G_XX^H = -G_B X^H + // + // Note that you don't need to store B for forward nor backward + // + // These formulas work for a general solver of linear equations. + // Let's prove now that when A is triangular, G_A is the projection onto the + // triangular matrices of the formula above, i.e. simply taking triu (resp. + // tril) in the formula above. This is because, since the triangular matrices + // form a vector space, the tangent space at any point is itself the space of + // triangular matrices. The result follows from a reasoning as that at the end + // of [Note: eigh backward] Something similar happens for `unitriangular`, + // only that int his case the tangent space is the set of lower-triangular + // matrices with zeros on the diagonal. + + if (!grad.defined() || (!A_requires_grad && !B_requires_grad)) { + return std::make_tuple(Tensor{}, Tensor{}); + } + // We always need to compute G_B + const Tensor A_H = A.mH(); + const Tensor G_B = + at::linalg_solve_triangular(A_H, grad, !upper, left, unitriangular); + + if (A_requires_grad) { + const Tensor X_H = X.mH(); + Tensor G_A = left ? -at::matmul(G_B, X_H) : -at::matmul(X_H, G_B); + G_A = upper ? G_A.triu(static_cast(unitriangular)) + : G_A.tril(-static_cast(unitriangular)); + return std::make_tuple(std::move(G_A), B_requires_grad ? G_B : Tensor{}); + } else { + return std::make_tuple(Tensor{}, G_B); + } +} + +std::tuple cholesky_solve_backward( + const Tensor& grad_x, + const Tensor& self, + const Tensor& input2, + const Tensor& result, + const bool upper, + std::array output_mask) { + at::NoTF32Guard disable_tf32; + Tensor grad_self, grad_input2; + if (grad_x.defined()) { + grad_self = grad_x.cholesky_solve(input2, /*upper=*/upper); + + if (output_mask[1]) { + Tensor common_term = at::matmul(grad_self, result.mH()); + common_term = common_term + common_term.mH(); + + if (upper) { + grad_input2 = -at::matmul(input2, common_term); + } else { + grad_input2 = -at::matmul(common_term, input2); + } + } + } + return std::tuple{grad_self, grad_input2}; +} + +Tensor cholesky_solve_jvp( + const Tensor& X, + const Tensor& U, + const Tensor& dU, + const Tensor& dB, + const bool upper) { + at::NoTF32Guard disable_tf32; + auto dK = upper ? dU.mH().matmul(U) : dU.matmul(U.mH()); + auto dA = dK + dK.mH(); + return generic_solve_jvp( + [&](const Tensor& A, const Tensor& dB, const Tensor& dA_contrib) { + return at::cholesky_solve(dB - dA_contrib, A, upper); + }, + X, + /*A=*/U, + dA, + dB); +} + +Tensor fft_c2r_backward( + const Tensor& grad, + IntArrayRef dim, + int64_t normalization) { + // Forward is C2R (onesided) + // Think of onesided C2R irfft as + // 1. fill the other half by conjugate symmetry + // 2. inverse C2C ifft + // 3. discard the complex dimension + // So backward is + // 1. R2C rfft (essentially add dummy complex dimension, and dft) + // 2. accumulate gradient by conjugate symmetry + // since rfft results follow conjugate symmetry, we only need to + // double some entries from onesided rfft results, i.e., the ones with + // their reflected indices also landing out of the onesided range. So + // consider the index of last dim: + // i. idx = 0. + // Reflected to (N - 0) % N = 0. Not doubled. + // ii 0 < idx < floor(N/2) (last). + // N > N - idx > ceil(N/2) + // Reflected to () + // iii. idx = floor(N/2) = N/2 (last) when N even. + // Reflected to (N - N/2) % N = N/2. Not doubled. + // iv. idx = floor(N/2) = (N-1)/2 (last) when N odd. + // Reflected to (N - (N-1)/2) % N = (N+1)/2. Doubled. + // Therefore, needs to double + // idx = 1, 2, ..., N/2 - 1 when N even + // idx = 1, 2, ..., (N-1)/2 when N odd + // that is + // idx = 1, 2, ..., N - (floor(N/2) + 1) + // = 1, 2, ..., N - onesided_length + auto gI = at::_fft_r2c(grad, dim, normalization, /*onesided=*/true); + + auto double_length = grad.sym_size(dim.back()) - gI.sym_size(dim.back()); + if (double_length > 0) { // also covers case when signal size is zero + gI.narrow_symint(dim.back(), 1, double_length).mul_(2); + } + return gI; +} + +Tensor fft_r2c_backward( + const Tensor& grad, + at::IntArrayRef dim, + int64_t normalization, + bool onesided, + const c10::SymInt& last_dim_size) { + if (!onesided) { + return at::real(at::_fft_c2c(grad, dim, normalization, /*forward=*/false)); + } + + // Forward is R2C (onesided) + // Think of onesided R2C rfft as + // 1. view as complex numbers (fill complex dim with zeros) + // 2. C2C fft + // 3. discard half of results + // So backward is + // 1. fill the other half with zeros (with `zero_grad_shape` below) + // (C2C ifft only take twosided inputs so we need to fill here) + // 2. inverse C2C ifft + // 3. discard the complex dim + auto half_sizes = grad.sym_sizes(); + std::vector new_grad_shape(half_sizes.begin(), half_sizes.end()); + const auto last_dim = + at::maybe_wrap_dim(dim.back(), static_cast(half_sizes.size())); + new_grad_shape[last_dim] = last_dim_size; + + const auto zero_length = last_dim_size - grad.sym_size(dim.back()); + auto complex_full_grad = + zero_length > 0 ? grad.new_zeros_symint(new_grad_shape) : grad; + if (zero_length > 0) { + complex_full_grad.slice_symint(last_dim, 0, half_sizes[last_dim]) + .copy_(grad); + } + return at::real( + at::_fft_c2c(complex_full_grad, dim, normalization, /*forward=*/false)); +} + +// Helper for batchnorm_double_backward +static Tensor sum_exclude_dim1(const Tensor& to_sum, bool keepdim = true) { + auto r = to_sum.sum(0, keepdim); + int64_t start_point_exclusive = keepdim ? 1 : 0; + for (int64_t dim = r.dim() - 1; dim > start_point_exclusive; dim--) { + r = r.sum(dim, keepdim); + } + return r; +} + +// Helper for batchnorm_double_backward +// similar to expand_as below, but doesn't do the expand_as; operates as if +// reductions were done with keepdim=True +static Tensor unsqueeze_dim1(const Tensor& src, const Tensor& target) { + auto src_expanded = src; + while (src_expanded.dim() < target.dim() - 1) { + src_expanded = src_expanded.unsqueeze(1); + } + if (src_expanded.dim() == target.dim() - 1) { + src_expanded = src_expanded.unsqueeze(0); + } + return src_expanded; +} + +// Helper for batchnorm_double_backward +// because gamma/ggG/ggB are 1-dimensional and represent dim==1, we can't +// do a straight expansion because it won't follow the broadcasting rules. +static Tensor expand_as_dim1(const Tensor& src, const Tensor& target) { + auto src_expanded = src; + while (src_expanded.dim() < target.dim() - 1) { + src_expanded = src_expanded.unsqueeze(1); + } + return src_expanded.expand_as(target); +} + +std::tuple batchnorm_double_backward( + const Tensor& input, + const std::optional& gamma, + const Tensor& ggI, + const Tensor& ggG, + const Tensor& ggB, + const Tensor& gO, + const std::optional& running_mean, + const std::optional& running_var, + bool training, + double eps, + const std::optional& save_mean, + const std::optional& save_invstd, + std::array output_mask) { + bool affine = isDefined(gamma); + // TODO: Do we have a ScalarOrTensor type? Would such a thing exist? + Tensor gamma_expanded; + Tensor ggG_expanded, ggB_expanded; + if (affine) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + gamma_expanded = expand_as_dim1(*gamma, input); + if (ggG.defined()) { + ggG_expanded = expand_as_dim1(ggG, input); + } + if (ggB.defined()) { + ggB_expanded = expand_as_dim1(ggB, input); + } + } else { + gamma_expanded = at::ones({}, input.options()); + } + + // define some terms we will reuse + auto M = input.size(0); + for (auto s : input.sizes().slice(2)) { + M *= s; + } + // for half inputs, save_mean, save_invstd are float (ideally, we would cast + // everything else, but not now) + auto mu = unsqueeze_dim1( + training ? toNonOptTensor(save_mean).to(input.scalar_type()) + : toNonOptTensor(running_mean), + input); + auto input_sub_mu = input - mu; + auto sigma2_eps_neg_1_2 = unsqueeze_dim1( + training ? toNonOptTensor(save_invstd).to(input.scalar_type()) + : toNonOptTensor(running_var).add(Scalar(eps)).pow(-0.5), + input); + auto sigma2_eps_neg_1 = sigma2_eps_neg_1_2.pow(2); + auto sigma2_eps_neg_3_2 = sigma2_eps_neg_1_2.pow(3); + + // calculate gI + auto input_mu_sigma2_neg_3_2 = input_sub_mu * sigma2_eps_neg_3_2; + auto gOinmu_sum = sum_exclude_dim1(gO * input_sub_mu); + auto gO_sum = sum_exclude_dim1(gO); + + Tensor gI; + if (ggI.defined() && training) { + auto ggI_sum = sum_exclude_dim1(ggI); + auto ggIinmu_sum = sum_exclude_dim1(ggI * input_sub_mu); + auto all_sub = ((ggI_sum * gO_sum).div_(M)) + .sub_(sum_exclude_dim1(gO * ggI)) + .add_((sigma2_eps_neg_1 * gOinmu_sum * ggIinmu_sum) + .mul_(3. / static_cast(M))); + auto gI_0t = (input_mu_sigma2_neg_3_2 * all_sub).div_(M); + auto gI_1t = + (ggIinmu_sum * sigma2_eps_neg_3_2).div_(M) * (gO_sum.div(M) - gO); + auto gI_2t = + (gOinmu_sum * sigma2_eps_neg_3_2).div_(M) * (ggI_sum.div(M) - ggI); + gI = gamma_expanded * (gI_0t.add_(gI_1t).add_(gI_2t)); + } + + // add contribution of gamma term to gI + Tensor gI_G_term; + if (affine && ggG.defined()) { + if (training) { + auto t0 = gO * sigma2_eps_neg_1_2; + auto t1 = (sigma2_eps_neg_1_2 * gO_sum).div_(-M); + auto t2 = (input_mu_sigma2_neg_3_2 * sum_exclude_dim1(gO * input_sub_mu)) + .div_(-M); + gI_G_term = ggG_expanded * (t0.add_(t1).add_(t2)); + gI = gI.defined() ? gI.add_(gI_G_term) : gI_G_term; + } else { + gI_G_term = ggG_expanded * sigma2_eps_neg_1_2 * gO; + gI = gI.defined() ? gI.add_(gI_G_term) : gI_G_term; + } + } + + // this is the first backward's grad_input + auto first_back_grad_input = [&](const Tensor& gO, + const Tensor& gamma) -> Tensor { + auto h0 = (gamma * sigma2_eps_neg_1_2).div_(M); + auto h1 = (M * gO) + .sub_(sum_exclude_dim1(gO)) + .sub_( + input_sub_mu.mul(sigma2_eps_neg_1) * + sum_exclude_dim1(gO * input_sub_mu)); + return h0 * h1; + }; + + // calculate gG + Tensor gG; + if (affine && ggI.defined()) { + if (training) { + // gG is just the first backwards with the gamma term removed (then shaped + // properly) + gG = ggI * + first_back_grad_input(gO, at::ones({}, sigma2_eps_neg_1_2.options())); + gG = sum_exclude_dim1(gG, false); + } else { + gG = sum_exclude_dim1(ggI * gO * sigma2_eps_neg_1_2, false); + } + } + + // calculate ggO + Tensor ggO; + // contribution of input term + if (ggI.defined()) { + if (training) { + ggO = first_back_grad_input(ggI, gamma_expanded); + } else { + ggO = ggI * sigma2_eps_neg_1_2 * gamma_expanded; + } + } + if (ggG.defined()) { + auto ggO_G_term = ggG_expanded * input_sub_mu * sigma2_eps_neg_1_2; + ggO = ggO.defined() ? ggO.add_(ggO_G_term) : ggO_G_term; + } + if (ggB.defined()) { + auto ggO_B_term = std::move(ggB_expanded); + ggO = ggO.defined() ? ggO.add_(ggO_B_term) : ggO_B_term; + } + + if (output_mask[1] && !gG.defined()) { + TORCH_INTERNAL_ASSERT( + affine, "gamma should always be defined when it requires grad"); + } + + return std::tuple{gI, gG, ggO}; +} + +std::tuple layer_norm_double_backward( + const Tensor& input_t, + const std::optional& gamma, + const Tensor& ggI, + const Tensor& ggG, + const Tensor& ggB, + const Tensor& gO_t, + const Tensor& save_mean_t, + const Tensor& save_invstd_t, + c10::SymIntArrayRef normalized_shape, + std::array output_mask) { + const auto normalized_ndim = normalized_shape.size(); + const auto input_shape = input_t.sym_sizes(); + const auto input_ndim = input_t.dim(); + const auto axis = input_ndim - normalized_ndim; + const c10::SymInt M = + c10::multiply_integers(input_shape.cbegin(), input_shape.cbegin() + axis); + const c10::SymInt N = + c10::multiply_integers(input_shape.cbegin() + axis, input_shape.cend()); + // printf("M: %ld, N: %ld", M, N); + + auto input = input_t.reshape_symint({M, N}); + auto gO = gO_t.reshape_symint({M, N}); + auto save_mean = save_mean_t.reshape_symint({M, c10::SymInt(1)}); + auto save_invstd = save_invstd_t.reshape_symint({M, c10::SymInt(1)}); + + bool affine = isDefined(gamma); + Tensor gamma_expanded; + Tensor ggG_expanded, ggB_expanded; + if (affine) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + gamma_expanded = gamma->reshape_symint({c10::SymInt(1), N}); + if (ggG.defined()) { + ggG_expanded = ggG.reshape_symint({c10::SymInt(1), N}); + } + if (ggB.defined()) { + ggB_expanded = ggB.reshape_symint({c10::SymInt(1), N}); + } + } else { + gamma_expanded = at::ones({1}, input.options()); + } + + Tensor ggI_expanded; + if (ggI.defined()) { + ggI_expanded = ggI.reshape_symint({M, N}); + } + + // for half inputs, save_mean, save_invstd are float + // (ideally, we would cast everything else, but not now) + auto mu = save_mean.to(input.scalar_type()); + auto input_sub_mu = input - mu; + auto sigma2_eps_neg_1_2 = save_invstd.to(input.scalar_type()); + auto sigma2_eps_neg_1 = sigma2_eps_neg_1_2.pow(2); + auto sigma2_eps_neg_3_2 = sigma2_eps_neg_1_2.pow(3); + + Tensor gI; + // calculate gI + auto input_mu_sigma2_neg_3_2 = input_sub_mu * sigma2_eps_neg_3_2; + + if (ggI.defined()) { + auto gxhat = gO * gamma_expanded; + auto gxhat_mu_sum = (gxhat * input_sub_mu).sum(1, true); + auto gxhat_sum = gxhat.sum(1, true); + + auto ggI_sum = ggI_expanded.sum(1, true); + auto ggI_mu_sum = (ggI_expanded * input_sub_mu).sum(1, true); + + auto three_over_n = c10::SymFloat(3.) / c10::SymFloat(N); + auto all_sub = ((ggI_sum * gxhat_sum).div_(N)) + .sub_((ggI_expanded * gxhat).sum(1, true)) + .add_((sigma2_eps_neg_1 * gxhat_mu_sum * ggI_mu_sum) + .mul_(three_over_n)); + auto gI_0t = (input_mu_sigma2_neg_3_2 * all_sub).div_(N); + auto gI_1t = + (ggI_mu_sum * sigma2_eps_neg_3_2).div_(N) * (gxhat_sum.div(N) - gxhat); + auto gI_2t = (gxhat_mu_sum * sigma2_eps_neg_3_2).div_(N) * + (ggI_sum.div(N) - ggI_expanded); + + gI = (gI_0t.add_(gI_1t).add_(gI_2t)); + } + + // add contribution of gamma term to gI + if (affine && ggG.defined()) { + auto t0 = gO * ggG_expanded * sigma2_eps_neg_1_2; + auto t1 = (sigma2_eps_neg_1_2 * (gO * ggG_expanded).sum(1, true)).div_(-N); + auto t2 = (input_mu_sigma2_neg_3_2 * + (gO * ggG_expanded * input_sub_mu).sum(1, true)) + .div_(-N); + auto gI_G_term = t0.add_(t1).add_(t2); + gI = gI.defined() ? gI.add_(gI_G_term) : gI_G_term; + } + + if (gI.defined()) { + // printf("=== computing gI\n"); + gI = gI.reshape_as(input_t); + } + + // this is the grad_input for the first backward function + auto first_bwd_fn_grad_input = [&](const Tensor& gO_local, + const Tensor& gamma_local) -> Tensor { + auto h0 = (gamma_local * sigma2_eps_neg_1_2).div_(N); + auto h1 = (gO_local.mul(N)) + .sub_(gO_local.sum(1, true)) + .sub_( + input_sub_mu.mul(sigma2_eps_neg_1) * + (gO_local * input_sub_mu).sum(1, true)); + return h0 * h1; + }; + + // calculate gG + Tensor gG; + if (affine && ggI.defined()) { + gG = first_bwd_fn_grad_input( + ggI_expanded, at::ones({}, sigma2_eps_neg_1_2.options())); + gG = (gO * gG).sum(0); + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + gG = gG.reshape_as(*gamma); + } + + // calculate ggO + Tensor ggO; + // contribution of input term + if (ggI.defined()) { + ggO = first_bwd_fn_grad_input(ggI_expanded, gamma_expanded); + } + if (ggG.defined()) { + auto ggO_G_term = ggG_expanded * input_sub_mu * sigma2_eps_neg_1_2; + ggO = ggO.defined() ? ggO.add_(ggO_G_term) : ggO_G_term; + } + if (ggB.defined()) { + auto ggO_B_term = std::move(ggB_expanded); + ggO = ggO.defined() ? ggO.add_(ggO_B_term) : ggO_B_term; + } + if (ggO.defined()) { + ggO = ggO.expand_symint({M, N}).reshape_as(input_t); + } + + if (output_mask[1] && !gG.defined()) { + TORCH_INTERNAL_ASSERT( + affine, "gamma should always be defined when it requires grad"); + } + + return std::tuple{gI, gG, ggO}; +} + +std::tuple infinitely_differentiable_native_rms_norm_backward( + const Tensor& dY, + const Tensor& drstd, + const Tensor& input, + IntArrayRef normalized_shape, + const Tensor& rstd, + const std::optional& weight_opt, + std::array grad_input_mask) { + c10::MaybeOwned weight_maybe_owned = + at::borrow_from_optional_tensor(weight_opt); + const Tensor& weight = *weight_maybe_owned; + + const auto input_shape = input.sizes(); + const auto input_ndim = input.dim(); + const auto normalized_ndim = static_cast(normalized_shape.size()); + const auto axis = input_ndim - normalized_ndim; + + int64_t N_rms = 1; + for (int i = 0; i < normalized_ndim; ++i) { + N_rms *= input_shape[axis + i]; + } + + Tensor dX; + Tensor dgamma; + + std::vector rstd_view_shape = rstd.sizes().vec(); + for (int i = 0; + i < std::max(static_cast(normalized_ndim - rstd.dim()), 0); + ++i) { + rstd_view_shape.push_back(1); + } + Tensor rstd_broadcast = rstd.view(rstd_view_shape); + Tensor rstd_pow3 = rstd_broadcast.pow(3); + Tensor grad_x_hat; + + if (dY.defined()) { + if (weight.defined()) { + grad_x_hat = dY * weight; + } else { + grad_x_hat = dY; + } + } + + if (grad_input_mask[0]) { + Tensor dX_from_dY_path; + Tensor dX_from_drstd_path; + + std::vector inner_sum_dims; + inner_sum_dims.reserve(normalized_ndim); + for (int i = 0; i < normalized_ndim; ++i) { + inner_sum_dims.push_back(axis + i); + } + + if (dY.defined() && grad_x_hat.defined()) { + Tensor sum_input_times_grad_x_hat = + sum(input * grad_x_hat, inner_sum_dims, /*keepdim=*/true); + dX_from_dY_path = rstd_broadcast * grad_x_hat - + (input * rstd_pow3 / static_cast(N_rms)) * + sum_input_times_grad_x_hat; + } + + if (drstd.defined()) { + Tensor drstd_broadcast = drstd.view(rstd_view_shape); + dX_from_drstd_path = + -(input * rstd_pow3 / static_cast(N_rms)) * drstd_broadcast; + } + + if (dX_from_dY_path.defined() && dX_from_drstd_path.defined()) { + dX = dX_from_dY_path + dX_from_drstd_path; + } else if (dX_from_dY_path.defined()) { + dX = dX_from_dY_path; + } else if (dX_from_drstd_path.defined()) { + dX = dX_from_drstd_path; + } + } + + if (grad_input_mask[1] && weight.defined()) { + if (dY.defined()) { + Tensor x_hat = input * rstd_broadcast; + Tensor dgamma_full_shape = dY * x_hat; + + if (axis > 0) { + std::vector outer_sum_dims; + outer_sum_dims.reserve(axis); + for (int i = 0; i < axis; ++i) { + outer_sum_dims.push_back(i); + } + dgamma = sum(dgamma_full_shape, outer_sum_dims, /*keepdim=*/false); + } else { + dgamma = dgamma_full_shape; + } + } + } + + return std::make_tuple(std::move(dX), std::move(dgamma)); +} + +std::tuple +infinitely_differentiable_native_group_norm_backward( + const Tensor& dY, + const Tensor& dmean, + const Tensor& drstd, + const Tensor& X, + const Tensor& mean, + const Tensor& rstd, + const std::optional& gamma, + const c10::SymInt& N, + const c10::SymInt& C, + const c10::SymInt& HxW, + int64_t group, + double eps, + std::array grad_input_mask) { + const int64_t G = group; + const auto D = C / G; + c10::SymFloat s = c10::SymFloat(1.0) / c10::SymFloat(D * HxW); + Tensor dX; + Tensor dgamma; + Tensor dbeta; + const Tensor X_tensor = X.reshape_symint({N, G, D, HxW}); + const Tensor mean_tensor = mean.reshape_symint({N, G, 1, 1}); + const Tensor rstd_tensor = rstd.reshape_symint({N, G, 1, 1}); + Tensor dY_tensor; + Tensor ds; + Tensor db; + if (dY.defined()) { + dY_tensor = dY.reshape_symint({N, G, D, HxW}); + ds = (dY_tensor * X_tensor).sum(3, /* keepdim */ true); + db = dY_tensor.sum(3, /* keepdim */ true); + } + if (grad_input_mask[0]) { + Tensor gamma_tensor; + if (isDefined(gamma)) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + gamma_tensor = gamma->reshape_symint({1, G, D, 1}); + } + const Tensor var = + ((rstd_tensor * rstd_tensor).reciprocal_() - eps).clamp_min(0); + const Tensor rstd_cube = rstd_tensor * rstd_tensor * rstd_tensor; + Tensor dvar; + if (drstd.defined()) { + dvar = -0.5 * rstd_cube * drstd.view_symint({N, G, 1, 1}); + } + if (dY.defined()) { + const Tensor a = + isDefined(gamma) ? rstd_tensor * gamma_tensor : rstd_tensor; + Tensor b = (isDefined(gamma) ? (ds * gamma_tensor) : ds) + .sum(2, /* keepdim */ true); + Tensor c = (isDefined(gamma) ? (db * gamma_tensor) : db) + .sum(2, /* keepdim */ true); + b = (c * mean_tensor - b) * rstd_cube * s; + c = -b * mean_tensor - c * rstd_tensor * std::move(s); + dX = a * dY_tensor + b * X_tensor + c; + if (dmean.defined() && drstd.defined()) { + dX = dX + + var_mean_backward( + dvar, + dmean.view_symint({N, G, 1, 1}), + X_tensor, + {2, 3}, + 0, + true); + } + dX = dX.reshape_as(X); + } else if (dmean.defined() && drstd.defined()) { + dX = var_mean_backward( + dvar, dmean.view_symint({N, G, 1, 1}), X_tensor, {2, 3}, 0, true) + .reshape_as(X); + } + } + if (grad_input_mask[1] && dY.defined()) { + dgamma = ((ds - db * mean_tensor) * rstd_tensor) + .sum(0) + .reshape_as(toNonOptTensor(gamma)); + } + if (grad_input_mask[2] && dY.defined()) { + dbeta = db.sum(0).reshape_as(toNonOptTensor(gamma)); + } + + return std::make_tuple(std::move(dX), std::move(dgamma), std::move(dbeta)); +} + +std::tuple _trilinear_backward( + const Tensor& grad_out, + const std::optional& i1, + const std::optional& i2, + const std::optional& i3, + IntArrayRef expand1, + IntArrayRef expand2, + IntArrayRef expand3, + IntArrayRef sumdim, + std::array grad_mask) { + Tensor grad_i1, grad_i2, grad_i3; + if (grad_out.defined()) { + if (grad_mask[0]) + grad_i1 = + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + at::_trilinear(grad_out, *i2, *i3, sumdim, expand2, expand3, expand1); + if (grad_mask[1]) + grad_i2 = + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + at::_trilinear(*i1, grad_out, *i3, expand1, sumdim, expand3, expand2); + if (grad_mask[2]) + grad_i3 = + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + at::_trilinear(*i1, *i2, grad_out, expand1, expand2, sumdim, expand3); + } + return std::tuple( + std::move(grad_i1), std::move(grad_i2), std::move(grad_i3)); +} + +Tensor log1p_backward(const Tensor& grad, const Tensor& self) { + // We must conditionally initialize this using to_dense if sparse, sparse + // addition is not supported without exact shape match + Tensor self_p1_conj; + if (self.layout() == c10::kSparse || self.layout() == c10::kSparseCsr || + self.layout() == c10::kSparseCsc || self.layout() == c10::kSparseBsr || + self.layout() == c10::kSparseBsc) { + // The warning only applies to the sparsity of self, dense grad is never + // materialized so if self is strided and grad is sparse nothing unexpected + // happens memory wise + TORCH_WARN( + "log1p_backward: received self with sparse layout, but backward requires materialization of a dense tensor with this shape"); + self_p1_conj = (self.to_dense() + 1).conj(); + } else { + // Although calling self.to_dense() would just return self when it has + // strided layout, that would breaks functorch tests. + self_p1_conj = (self + 1).conj(); + } + if (grad.layout() == c10::kSparse || grad.layout() == c10::kSparseCsr || + grad.layout() == c10::kSparseCsc || grad.layout() == c10::kSparseBsr || + grad.layout() == c10::kSparseBsc) { + // If grad is sparse we can't divide by the n-d (self + 1).conj(), so we + // must multiply by the recipricol, layout of grad is preserved which is + // important to gradcheck + return grad * self_p1_conj.reciprocal_(); + } + return grad / self_p1_conj; +} + +Tensor sinc_backward(const Tensor& grad, const Tensor& self) { + auto self_pi = self * M_PI; + auto self_squared_pi = self * self * M_PI; + auto out = grad * + ((self_pi * self_pi.cos() - self_pi.sin()) / self_squared_pi).conj(); + return at::where( + self_squared_pi == 0.0, at::scalar_tensor(0.0, grad.options()), out); +} + +// Because the backward of pad(input, pads) is just pad(grad_output, [-p for p +// in pads]) +Tensor constant_pad_nd_backward(const Tensor& grad, c10::SymIntArrayRef pad) { + auto negated_pad = pad.vec(); + std::transform( // NOLINT(modernize-use-ranges) + negated_pad.cbegin(), + negated_pad.cend(), + negated_pad.begin(), + // NOLINTNEXTLINE(modernize-use-transparent-functors) + std::negate()); + return at::constant_pad_nd_symint(grad, negated_pad, 0); +} + +Tensor embedding_dense_double_backward_symint( + const Tensor& grad, + const Tensor& indices, + const c10::SymInt& padding_idx) { + // since first backward takes care of scaling by frequency, + // we don't need to worry about it here. + auto gg_weight = grad.index_select(0, indices.reshape(-1)); + + // reshape gradient as per the shape of indices + auto size = indices.sizes().vec(); + size.push_back(-1); + + if (padding_idx >= 0) { + gg_weight.masked_fill_((indices == padding_idx).reshape({-1, 1}), 0); + } + return gg_weight.view(size); +} + +Tensor index_backward( + Tensor zeros_like_self, + const torch::List>& indices, + const Tensor& grad) { + return (areAnyTensorSubclassLike({zeros_like_self, grad}) || + areAnyOptionalTensorSubclassLike(indices)) + ? zeros_like_self.index_put(indices, grad, true) + : at::_index_put_impl_(zeros_like_self, indices, grad, true, true); +} + +Tensor _cudnn_ctc_loss_backward( + const Tensor& grad_out, + const Tensor& loss, + const Tensor& raw_grad, + bool zero_infinity) { + if (zero_infinity) { + return at::where( + loss.unsqueeze(0).unsqueeze(2) == 0, + at::zeros({}, raw_grad.options()), + raw_grad * grad_out.unsqueeze(0).unsqueeze(2)); + } else { + return raw_grad * grad_out.unsqueeze(0).unsqueeze(2); + } +} + +Tensor _miopen_ctc_loss_backward( + const Tensor& grad_out, + const Tensor& loss, + const Tensor& raw_grad, + bool zero_infinity) { + // MIOpen CTC Loss backward is identical to cuDNN + return _cudnn_ctc_loss_backward(grad_out, loss, raw_grad, zero_infinity); +} + +bool any_variable_defined(const variable_list& variables) { + for (const auto& variable : variables) { + if (variable.defined()) { + return true; + } + } + return false; +} + +// Derivations for the householder_product.backward method. +// +// Given a sequence of vectors v_1, ..., v_n and a sequence of scalars tau_1, +// ..., tau_k, the torch.linalg.householder_product computes the first n columns +// of the following product: Q = (I - tau_1 v_1 v_1^H) ... (I - tau_k v_k +// v_k^H). Let +// H_i(sigma) := I - sigma v_i v_i^H, so Q = (H_1(sigma_1) ... +// H_k(sigma_k)[:, :k]; H_i_minus = H_1(tau_1) ... H_{i - 1}(tau_{i - 1}), +// with H_1_minus := I; H_i_plus = H_{i + 1}(tau_{i + 1}) ... H_k(tau_k) +// with H_k_plus := I; +// +// Forward AD: +// dQ = sum_{i = 1}^k H_i_minus (-dtau_i v_i v_i^H - tau_i dv_i v_i^H - tau_i +// v_i dv_i^H) H_i_plus. +// +// Backward AD: +// Tr(Q_grad^H dQ) = sum_{i = 1}^k Tr(H_i_plus Q_grad^H H_i_minus (-dtau_i v_i +// v_i^H - tau_i dv_i v_i^H - tau_i v_i dv_i^H)). Let K_i := H_i_plus Q_grad^H +// H_i_minus, then the gradients are v_i_grad = (-tau_i v_i^H K_i)^H - tau_i K_i +// v_i, tau_i_grad = Tr(-v_i^H K_i v_i).conj(). NOTE: the algorithms ignores +// that only n columns of Q are observed, so there is no need in recomputing Q +// to full completion. +// +// Note that K_{i + 1} = H_{i + 1}^{-1} K_i H_i, so we can compute v_i_grad, +// tau_i_grad one by one by just efficiently updating K_i if that is possible. +// Multiplying with H_i from the right could be done with matrix-vector +// products, but what about the inverse H_{i + 1}^{-1} and does it even exist? +// Luckily, under some assumptions, H_{i + 1}^{-1} exists and admits a +// representation as H_i(sigma_i) for some sigma_i, so the left update is also +// could be done with matrix-vector and not matrix-matrix products. +// +// Let H(tau) := I - tau v v^H. +// H(tau) has eigenvalues 1 with multiplicity (m - 1) with eigenvectors +// orthogonal to v, and an eigenvalue (1 - tau ||v||^2) with the corresponding +// eigenvector v / ||v||. If (1 - tau ||v||^2) != 0, H(tau) is invertible. If (1 +// - tau ||v||^2) != 0, then with sigma defined as sigma := tau / (||v||^2 tau - +// 1) we get that H(tau) H(sigma) = H(sigma) H(tau) = I, so H(sigma) is the +// inverse of H(tau). +// +// WARNING: the algorithm below assumes that H_i(tau_i) are all invertible, so +// it expects that (1 - tau_i ||v_i||^2) != 0 for all i. +// We would like to point out that if there is H_i(tau_i) which is not +// invertible, the householder_product is still differentiable! We will not be +// able to compute K_i efficiently in such cases, however, as evaluating of each +// K_i will amount to calls to ORGQR to be able to compute H_i_plus. + +// This function computes either the product between +// (I - tau u v^H) and K (in-place or not) with `condition_with_I = true`, or +// between +// (-tau u v^H) and K (out-of-place only) with `condition_with_I = false`. +// Parameter `left` controls whether the matrix K is multiplied from the left or +// from the right. +// Additionally, when the computation is done in-place, we exploit that the +// first `k` coordinates of `u_full/v_full` are zeros. +static Tensor apply_simple_transformation( + const c10::SymInt& m, + const c10::SymInt& k, + const Tensor& u_full, + const Tensor& v_full, + const Tensor& t, + Tensor& K, + bool modify_K_in_place = true, + bool condition_with_I = true, + bool left = true) { + // we assume u_full is a vector of dimension (..., m, 1), t is a scalar of + // dimension (..., 1) + + // TODO: matrix-vector products in the code below are dispatched to + // matrix-matrix products. We either need to extend matmul to support batched + // matrix-vector products, or implement a batched variant of mv. We could + // enable mv for inputs which are not batched, but it is not done to eliminate + // the code duplication. + + // returns (I - t u v^H) K or -t u v^H K + if (left) { + if (modify_K_in_place) { + auto v = u_full.narrow_symint(-2, k, m - k); + auto u = v_full.narrow_symint(-2, k, m - k) + .mH() + .matmul(K.narrow_symint(-2, k, m - k)); + K.narrow_symint(-2, k, m - k).sub_((t.unsqueeze(-1) * v) * u); + return K; + } else { + auto transformation = (t.unsqueeze(-1) * u_full) * v_full.mH().matmul(K); + return condition_with_I ? K - transformation : -transformation; + } + } + // returns K (I - t u v^H) or -K t u v^H + else { + if (modify_K_in_place) { + auto v = u_full.narrow_symint(-2, k, m - k); + auto u = + K.narrow_symint(-1, k, m - k) + .matmul(t.unsqueeze(-1) * v_full.narrow_symint(-2, k, m - k)); + K.narrow_symint(-1, k, m - k).sub_(u * v.mH()); + return K; + } else { + auto transformation = K.matmul(t.unsqueeze(-1) * u_full) * v_full.mH(); + return condition_with_I ? K - transformation : -transformation; + } + } +} + +std::tuple householder_product_backward( + const Tensor& grad, + const Tensor& result, + const Tensor& input_, + const Tensor& tau, + const bool flip_order) { + // NOTE on `flip_order`: when flip_order is true, + // the algorithm below reverses the processing direction from + // range(k) to range(k - 1, -1, -1) in the main loop, and left/right + // Householder projection applications get flipped. + // The comments below about the algorithmic details assume flip_order = false. + if (!grad.defined() || input_.sym_numel() == 0 || tau.sym_numel() == 0) { + return std::tuple(Tensor(), Tensor()); + } + auto m = input_.sym_size(-2); + // guard_int is due to irange calls below + auto k = tau.sym_size(-1).guard_int(__FILE__, __LINE__); + + // forward operates only over the lower triangular part with the assumption + // that the diagonal of input is filled with 1s. + auto input = input_.tril(-1); + input.diagonal(0, -2, -1).fill_(1.0); + + // compute sigma such that + // H(sigma_i) == H(tau_i)^{-1}. + // If the input to householder_product comes from GEQRF, + // we will never encounter ||v_i||^2 tau_i == 1, so H(tau_i) will always be + // invertible. This follows from the documentation + // https://www.netlib.org/lapack/lug/node128.html, and tau always satisfying + // the condition |tau|^2 ||v||^2 == 2 * Re(tau). + auto input_first_k_cols = input.narrow(-1, 0, k); + auto input_first_k_cols_norm_squared = + (input_first_k_cols * input_first_k_cols.conj()).sum(-2); + auto sigma = tau / (tau * input_first_k_cols_norm_squared - 1.0); + + auto K = result.matmul(grad.mH()); + + // The algorithm updates K by multiplying it from the left/right with + // Householder reflectors. If only single backward is run, we modify K + // in-place and exploit triangularity of the input. With higher order + // derivatives we cannot rewrite the storage of K, hence we use much less + // efficient out-of-place methods. + // + // if only first-order derivative is expected, we can modify K in-place for + // better performance + bool modify_K_in_place = !at::GradMode::is_enabled(); + + // This method exploits that at k-th iteration vector v_k has only elements + // v_k[k:] which are non-zero. + auto update_grad = [&m]( + int64_t k, + const Tensor& v_full, + const Tensor& t, + const Tensor& K) -> std::tuple { + // v_full is a vector of dimension (..., m, 1), t is a scalar of dimension + // (..., 1) + auto v = v_full.narrow_symint(-2, k, m - k); + auto vHK = v.mH().matmul(K.narrow_symint(-2, k, m - k)); + auto Kv = K.narrow_symint(-1, k, m - k).matmul(v); + auto t_unsqueezed = t.unsqueeze(-1); + auto v_grad = (-t_unsqueezed * vHK).conj().squeeze(-2) - + (t_unsqueezed * Kv).squeeze(-1); + auto tau_grad = -(vHK.narrow_symint(-1, k, m - k).matmul(v)).conj(); + return std::make_tuple(v_grad.unsqueeze(-1), tau_grad.squeeze(-1)); + }; + + auto apply_householder_reflector = [m, modify_K_in_place]( + int64_t k, + const Tensor& v_full, + const Tensor& t, + Tensor& K, + bool left = true) -> Tensor { + return apply_simple_transformation( + m, + k, + v_full, + v_full, + t, + K, + modify_K_in_place, + /*condition_with_I=*/true, + left); + }; + + const auto flip_i = [flip_order, k](int64_t i) -> int64_t { + return !flip_order ? i : k - i - 1; + }; + const auto next_i = [flip_order](int64_t i) -> int64_t { + return !flip_order ? ++i : --i; + }; + const auto apply_left = !flip_order; + + // K <- H_0^{-1} @ K + const auto zero_idx = flip_i(0); + K = apply_householder_reflector( + zero_idx, + input.narrow(-1, zero_idx, 1), + sigma.narrow(-1, zero_idx, 1), + K, + /*left=*/apply_left); + + Tensor input_grad, tau_grad; + // For Composite Compliance, we can't copy a Subclass into a Regular Tensor, + // so we use out-of-place ops with equivalent output. + // NOTE: We can't use `new_zeros` directly as `input`, 'tau' or `grad` can + // be Tensor Subclass and we don't want to make assumption about which + // one to choose for creating output buffer. + // eg. if both are BatchedTensor at different level. + if (areAnyTensorSubclassLike({input, tau, K})) { + // k + 1 if input_grads hold a matrix of zeros for inactive parts of input. + auto input_grads = std::vector(k < input.sym_size(-1) ? k + 1 : k); + auto tau_grads = std::vector(k); + + for (const auto i_idx : c10::irange(k)) { + auto i = flip_i(i_idx); + // NOTE: narrow will unsqueeze(-1) + auto v_i = input.narrow(-1, i, 1); + auto t_i = tau.narrow(-1, i, 1); + + std::tie(input_grads[i], tau_grads[i]) = update_grad(i, v_i, t_i, K); + + // K <- H_{i + 1}^{-1} @ K @ H_i + if (i != flip_i(k - 1)) { + auto i_next = next_i(i); + auto v_i_next = input.narrow(-1, i_next, 1); + auto s_i_next = sigma.narrow(-1, i_next, 1); + K = apply_householder_reflector( + i_next, v_i_next, s_i_next, K, /*left=*/apply_left); + K = apply_householder_reflector(i, v_i, t_i, K, /*left=*/!apply_left); + } + } + + // Only first k columns are active in forward. + // zero gradients for the inactive input. + if (k < input.sym_size(-1)) { + auto zero_grad_shape = + at::SymDimVector(input_.sym_sizes().slice(0, input_.dim() - 1)); + zero_grad_shape.push_back(input.sym_size(-1) - k); + auto zero_grad = at::zeros_symint(zero_grad_shape, input_.options()); + input_grads[k] = zero_grad; + } + + input_grad = at::cat(input_grads, -1); + tau_grad = at::cat(tau_grads, -1); + } else { + input_grad = at::zeros_like(input_); + tau_grad = at::zeros_like(tau); + for (const auto i_idx : c10::irange(k)) { + auto i = flip_i(i_idx); + // NOTE: narrow will unsqueeze(-1) + auto v_i = input.narrow(-1, i, 1); + auto t_i = tau.narrow(-1, i, 1); + + auto [v_i_grad, tau_i_grad] = update_grad(i, v_i, t_i, K); + input_grad.select(-1, i).copy_(v_i_grad.squeeze(-1)); + tau_grad.select(-1, i).copy_(tau_i_grad.squeeze(-1)); + + // K <- H_{i + 1}^{-1} @ K @ H_i + if (i != flip_i(k - 1)) { + auto i_next = next_i(i); + auto v_i_next = input.narrow(-1, i_next, 1); + auto s_i_next = sigma.narrow(-1, i_next, 1); + K = apply_householder_reflector( + i_next, v_i_next, s_i_next, K, /*left=*/apply_left); + K = apply_householder_reflector(i, v_i, t_i, K, /*left=*/!apply_left); + } + } + } + + // forward operates only over the lower-triangular part of the input + // excluding the main diagonal, hence the gradient is also lower-triangular. + input_grad.tril_(-1); + + return std::make_tuple(std::move(input_grad), std::move(tau_grad)); +} + +// We refer to the derivations described above the method +// `apply_simple_transformation` +Tensor householder_product_jvp( + const Tensor& dV_, + const Tensor& dtau, + const Tensor& prod, + const Tensor& V_, + const Tensor& tau) { + auto m = V_.sym_size(-2); + auto k = tau.size(-1); + + // forward operates only over the lower triangular part with the assumption + // that the diagonal of input is filled with 1s. + auto V = V_.tril(-1); + V.diagonal(0, -2, -1).fill_(1.0); + auto dV = dV_.tril(-1); + + // compute sigma such that + // H(sigma_i) == H(tau_i)^{-1}. + // If the input to householder_product comes from GEQRF, + // we will never encounter ||v_i||^2 tau_i == 1, so H(tau_i) will always be + // invertible. This follows from the documentation + // https://www.netlib.org/lapack/lug/node128.html, and tau always satisfying + // the condition |tau|^2 ||v||^2 == 2 * Re(tau). + auto V_first_k_cols = V.narrow(-1, 0, k); + auto V_first_k_cols_norm_squared = + (V_first_k_cols * V_first_k_cols.conj()).sum(-2); + auto sigma = tau / (tau * V_first_k_cols_norm_squared - 1.0); + + auto apply_householder_reflector = [m](const Tensor& v_full, + const Tensor& t, + Tensor& K, + bool left = true) -> Tensor { + return apply_simple_transformation( + // setting `modify_K_in_place = true` causes CUDA memory leaks in OpInfo + // tests of forward AD for that reason we ignore `k` by passing zero + m, + /*k=*/0, + v_full, + v_full, + t, + K, + /*modify_K_in_place=*/false, + /*condition_with_I=*/true, + left); + }; + + // computes (-t u v^H) K + auto apply_simple_product = [m](const Tensor& u_full, + const Tensor& v_full, + const Tensor& t, + Tensor& K) -> Tensor { + return apply_simple_transformation( + // since ``modify_K_in_place = false`, we can ignore `k` and pass + // arbitrary value + m, + /*k=*/0, + u_full, + v_full, + t, + K, + /*modify_K_in_place=*/false, + /*condition_with_I=*/false, + /*left=*/true); + }; + + auto H_plus = prod.detach().clone(); + IntArrayRef batch_vector_shape(V.sizes().data(), V.dim() - 1); + auto H_minus = + at::diag_embed(at::ones({1}, V.options()).expand(batch_vector_shape)); + + auto dprod = at::zeros_like(prod); + for (const auto i : c10::irange(k)) { + auto v_i = V.narrow(-1, i, 1); + auto dv_i = dV.narrow(-1, i, 1); + auto tau_i = tau.narrow(-1, i, 1); + auto dtau_i = dtau.narrow(-1, i, 1); + auto sigma_i = sigma.narrow(-1, i, 1); + + H_plus = apply_householder_reflector(v_i, sigma_i, H_plus, /*left=*/true); + + // `H_minus_dH_i_H_plus` = H_1 * ... * H_{i-1} dH_i * H_{i+1} * ... + auto H_minus_dH_i_H_plus = H_minus.matmul( + apply_simple_product(v_i, v_i, dtau_i, H_plus) + + apply_simple_product(dv_i, v_i, tau_i, H_plus) + + apply_simple_product(v_i, dv_i, tau_i, H_plus)); + // For Composite Compliance, if `intermediate` is a Tensor-Subclass, + // we use out-of-place variant of add. + if (at::isTensorSubclassLike(H_minus_dH_i_H_plus)) { + dprod = dprod.add(H_minus_dH_i_H_plus); + } else { + dprod.add_(H_minus_dH_i_H_plus); + } + + H_minus = apply_householder_reflector(v_i, tau_i, H_minus, /*left=*/false); + } + + return dprod; +} + +std::tuple ormqr_backward( + const Tensor& grad, + const Tensor& result, + const Tensor& self, + const Tensor& tau, + const Tensor& other, + bool left, + bool transpose, + std::array grad_output_mask) { + Tensor self_grad, tau_grad, other_grad; + + if (!grad.defined()) { + return std::make_tuple( + std::move(self_grad), std::move(tau_grad), std::move(other_grad)); + } + + const auto self_requires_grad = grad_output_mask[0]; + const auto tau_requires_grad = grad_output_mask[1]; + const auto other_requires_grad = grad_output_mask[2]; + + if (other_requires_grad) { + other_grad = at::ormqr(self, tau, grad, left, !transpose); + } + if (self_requires_grad || tau_requires_grad) { + if (left ^ transpose) { + // Assume left = true and transpose = false. The case with + // left = false and transpose = true is very much similar with just + // transposed arguments passed into householder_product_backward. + // Ormqr computes B = H_1 * ... * H_k * A. + // The sensitivity wrt H_i is given by (see notes in + // householder_product_backward) Tr(H_i_plus B B_grad^H H_i_minus dH_i), + // so, since householder_product_backward respects `for i in range(k)`, we + // could reuse householder_product_backward with + // householder_product_backward.grad = grad and + // householder_product_backward.result = result. + const auto hpb_grad = !transpose ? grad : grad.mH(); + const auto hpb_result = !transpose ? result : result.mH(); + std::tie(self_grad, tau_grad) = + householder_product_backward(hpb_grad, hpb_result, self, tau); + } else { + // Assuming left = false and transpose = false. The case with + // left = true and transpose = true is very much similar with just + // transposed arguments passed into householder_product_backward. + // In this case Ormqr computes B = H_1 * ... * H_k * A and the sensitivity + // wrt H_i becomes Tr(H_i_plus B_grad^H B H_i_minus dH_k). + // We could see that the role of `grad` and `result` in + // householder_product_backward gets "swapped" and "transposed" and that + // in order to compute H_k_grad efficiently we would need to compute grads + // in reversed order (`for i in range(k - 1, -1, -1)`). Hence we reuse + // householder_product_backward with householder_product_backward.grad = + // result.mH, householder_product_backward.result = grad.mH, + // householder_product_backward.flip_order = true. + const auto hpb_grad = !transpose ? result.mH() : result; + const auto hpb_result = !transpose ? grad.mH() : grad; + std::tie(self_grad, tau_grad) = householder_product_backward( + hpb_grad, hpb_result, self, tau, /*flip_order=*/true); + } + } + + return std::make_tuple( + std::move(self_grad), std::move(tau_grad), std::move(other_grad)); +} + +std::tuple polar_backward( + const Tensor& grad, + const Tensor& result) { + Tensor grad_abs, grad_angle; + if (grad.defined()) { + auto grad_conj = grad.conj(); + grad_abs = at::real(grad_conj * at::sgn(result)); + auto result_mul_1_j = result * Scalar(c10::complex{0.0, 1.0}); + grad_angle = at::real(grad_conj * result_mul_1_j); + } + return std::make_tuple(std::move(grad_abs), std::move(grad_angle)); +} + +Tensor i1_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& result) { + return AT_DISPATCH_FLOATING_TYPES(self.scalar_type(), "i1_backward", [&]() { + // For x = 0, the correct gradient is 0.5, + // however due to floating point computation we get NaN. + // So we manually update gradient for x=0 + auto eps = std::numeric_limits::epsilon(); + auto self_is_not_tiny = self.abs() > eps; + + // Following `where` is needed as `where` computes gradients, + // even for the part which didn't affect the output. + // Look at https://github.com/pytorch/pytorch/issues/52248 + // Update if and when this is fixed. + auto safe_self = at::where( + self_is_not_tiny, self, at::scalar_tensor(eps, self.options())); + auto gradx = (safe_self.i0() - (result * safe_self.reciprocal())); + return grad * + at::where( + self_is_not_tiny, gradx, at::scalar_tensor(0.5, self.options())); + }); +} + +Tensor i1e_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& result) { + return AT_DISPATCH_FLOATING_TYPES(self.scalar_type(), "i1e_backward", [&]() { + // For x = 0, the correct gradient is 0.5, + // however due to floating point computation we get NaN. + // So we manually update gradient for x=0 + auto eps = std::numeric_limits::epsilon(); + auto self_is_not_tiny = self.abs() > eps; + + // Following `where` is needed as `where` computes gradients, + // even for the part which didn't affect the output. + // Look at https://github.com/pytorch/pytorch/issues/52248 + // Update if and when this is fixed. + auto safe_self = at::where( + self_is_not_tiny, self, at::scalar_tensor(eps, self.options())); + auto gradx = + (at::special_i0e(safe_self) - + result * (safe_self.sgn() + safe_self.reciprocal())); + return grad * + at::where( + self_is_not_tiny, gradx, at::scalar_tensor(0.5, self.options())); + }); +} + +// lu_solve is a map (LU, P, B) -> (PLU)^{-1} B, +// where LU = L + U - I and P is a permutation matrix, and is fixed. +// +// Let 1 = ones_like(LU), +// 1_U = 1.triu(), +// 1_L = 1.tril(-1) +// * := the Hadamard (element-wise) product +// +// Forward AD: +// +// Let X := U^{-1} L^{-1} P^T B be the output of the function. +// Also, the LU input of the function could be represented as +// LU = (L - I) + U. +// +// Differentiating LU = L + U - I produces: +// dLU = dL + dU. +// Noting that dL and dU are lower- and upper-triangular, respectively, +// and that the diagonal of L is never explicitly exposed, so +// diag(dL) = 0, it follows +// dL = dLU * 1_L, +// dU = dLU * 1_U. +// +// Differentiating X = U^{-1} L^{-1} P^T B produces: +// dX = dU^{-1} L^{-1} P^T B + U^{-1} dL^{-1} P^T B + U^{-1} L^{-1} P^T dB +// Note that for any invertible matrix A we have A A^{-1} = I, hence +// dA A^{-1} + A dA^{-1} = 0 => dA^{-1} = -A^{-1} dA A^{-1}. +// Inserting it back into the definition of dX gives: +// dX = -U^{-1} dU U^{-1} L^{-1} P^T B - U^{-1} L^{-1} dL L^{-1} P^T B + U^{-1} +// L^{-1} P^T dB dX = -U^{-1} dU X - U^{-1} L^{-1} dL U X + U^{-1} L^{-1} P^T dB +// +// Backward AD: +// +// Using the definition of dL, dU from above: +// Tr(L_grad^H dL) + Tr(U_grad^H dU) = Tr(L_grad^H (dLU * 1_L)) + Tr(U_grad^H +// (dLU * 1_U)) +// = [using Tr(A (B * C)) = Tr((A * B^T) C) +// = Tr((L_grad^H * 1_L^T) dLU) + Tr((U_grad^H +// * 1_U^T) dLU), +// hence +// LU_grad = L_grad * 1_L + U_grad * 1_U (!!!) +// +// Then, transposing the formula for dX above we get: +// B_grad = P L^{-H} U^{-H} X_grad = lu_solve(X_grad, LU_data, LU_pivots, +// /*adjoint=*/true) U_grad = -U^{-H} X_grad X^H L_grad = L^{-H} U_grad U^H +// After inserting U_grad and L_grad into (!!!) we get the value for LU_grad. + +Tensor linalg_lu_solve_LU( + const Tensor& gX, + const Tensor& LU, + const Tensor& pivots, + const Tensor& X, + const bool left, + const bool adjoint) { + // From linalg_lu_solve_jvp we have that: + // left = True, adjoint = True: A^HX = B + // left = True, adjoint = False: AX = B + // left = False, adjoint = True: AX^H = B^H + // left = False, adjoint = False: A^HX^H = B^H + // let op_1(A) = A^H or op_1(A) = A according to the list above + // same with op_2(X) and op_3(B) + // We have that letting S = lu_solve(LU, pivots, dB, left, adjoint) + // the JVP formula reads + // if left != adjoint: + // dX = op_2(-U^{-1}(dU + L^{-1}dL U)op_2(X)) + S + // else: + // dX = op_2(op_1(-op_3(X)^H P(LdUU^{-1} + dL)L^{-1} P^T)) + S + // So computing the adjoint of this operation we get that, using an auxiliary + // variable gR if left != adjoint: + // gR = U^{-H}op_2(-gX)op_2(X)^H + // gU = gR.triu() + // gL = (L^{-H} gR U^H).tril(-1) + // else: + // gR = -P^T op_3(X)op_1(op_2(gX))PL^{-H} + // gL = gR.tril(-1) + // gU = (L^H gR U^{-H}).triu() + // gLU = gL + gU + + at::NoTF32Guard disable_tf32; + auto [P, L, U] = at::lu_unpack( + LU, pivots, /*unpack_data=*/true, /*unpack_pivots=*/left == adjoint); + // TODO Optimise the order of the operations to avoid operating on large + // tensors unnecessarily + // The logic should be: if n < k == left then multiply the gX and X first + // (as it's done now) Otherwise multiply them last + if (left != adjoint) { + // gR = U^{-H}op_2(-gX)op_2(X)^H + auto gR = at::linalg_solve_triangular( + U.mH(), + -(left ? gX : gX.mH()).matmul(left ? X.mH() : X), + /*upper*/ false); + // gL = (L^{-H} gR U^H).tril(-1) + auto gL = at::linalg_solve_triangular( + L.mH(), + gR.matmul(U.mH()), + /*upper*/ true, + /*left*/ true, + /*unitriangular*/ true) + .tril(-1); + ; + return gL + gR.triu(); + } else { + // gR = -P^T op_3(X)op_1(op_2(gX))P + auto gR = + -P.mT().matmul(left ? X : X.mH()).matmul(left ? gX.mH() : gX).matmul(P); + // gR = gR L^{-H} + gR = at::linalg_solve_triangular( + L.mH(), gR, /*upper*/ true, /*left*/ false, /*unitriangular*/ true); + // gU = (L^H gR U^{-H}).triu() + auto gU = at::linalg_solve_triangular( + U.mH(), L.mH().matmul(gR), /*upper*/ false, /*left*/ false) + .triu(); + return gR.tril(-1) + gU; + } +} + +Tensor linalg_lu_solve_jvp( + const Tensor& X, + const Tensor& LU, + const Tensor& pivots, + const Tensor& dLU, + const Tensor& dB, + const bool left, + const bool adjoint) { + // We write the derivation in terms of some adjoint operations, as otherwise + // we would need to write down 4 different proofs with 4 different + // implementations and it'd be painful to derive and maintain Below, we just + // use that X -> X^H is linear, so it commutes with the derivative The + // derivation follows by differentiating op_1(PLU)op_2(X) = op_3(B) + + // left = True, adjoint = True: A^HX = B + // left = True, adjoint = False: AX = B + // left = False, adjoint = True: AX^H = B^H + // left = False, adjoint = False: A^HX^H = B^H + // let op_1(A) = A^H or op_1(A) = A according to the list above + // same with op_2(X) and op_3(B) + // We have that letting S = lu_solve(LU, pivots, dB, left, adjoint) + // the JVP formula reads + // dX = op_2(op_1(-U^{-1}(dUU^{-1} + L^{-1}dL)L^{-1} P^T)op_3(B)) + S + + at::NoTF32Guard disable_tf32; + auto S = at::linalg_lu_solve(LU, pivots, dB, left, adjoint); + if (left != adjoint) { + // We see that when left != adjoint, op_1(A) = A, and we can substitute + // A^{-1}op_3(B) by op_2(X) dX = op_2(-U^{-1}(dU + L^{-1}dL U)op_2(X)) + S + // Let R = -U^{-1}(dU + L^{-1}dL U) + auto R = at::linalg_solve_triangular( + LU, + dLU.tril(-1), + /*upper*/ false, + /*left*/ true, + /*unitriangular*/ true); + auto U = LU.triu(); + R = -at::linalg_solve_triangular( + U, dLU.triu() + R.matmul(U), /*upper*/ true); + // dX = op_2(R op_2(X)) + S + return (left ? R.matmul(X) : X.matmul(R.mH())) + S; + } else { + // We see that when left == adjoint, op_1(A) = A^H + // dX = op_2(op_1(-op_3(B)^H U^{-1}(dUU^{-1} + L^{-1}dL)L^{-1} P^T)) + S + // Now, note that whenever adjoint == left, we have that + // op_3(B)^H A^{-1} = op_3(X)^H + // We can then rewrite the formula above in terms of X as + // dX = op_2(op_1(-op_3(X)^H P(LdUU^{-1} + dL)L^{-1} P^T)) + S + auto [P, L, U] = at::lu_unpack(LU, pivots); + // Compute V = op_3(X)^H + auto V = left ? X.mH() : X; + // Compute the inner parens LdUU^{-1} + dL + auto R = at::linalg_solve_triangular( + U, L.matmul(dLU.triu()), /*upper*/ true, /*left*/ false) + + dLU.tril(-1); + // dX = op_2(op_1(-op_3(X)^H PRL^{-1} P^T)) + S + R = at::linalg_solve_triangular( + L, + -V.matmul(P).matmul(R), + /*upper*/ false, + /*left*/ false, + /*unitriangular*/ true) + .matmul(P.mT()); + // dX = op_2(R^H) + S + return (left ? R.mH() : R) + S; + } +} + +Tensor linalg_solve_jvp( + const Tensor& dA, + const Tensor& dB, + const Tensor& X, + const Tensor& LU, + const Tensor& pivots, + const bool left) { + at::NoTF32Guard disable_tf32; + // For left=True (left=False is analogous) + // dX = A^{-1}(dB - dAX) + + // [NumPy compat] Case where the rhs is a vector. + // We denote with an underscore vectors that have been converted to matrices + // by `unsqueeze(-1)` + const bool vector_case = at::native::linalg_solve_is_vector_rhs(LU, X); + const auto vector_to_matrix = [vector_case](const Tensor& X) { + return vector_case ? X.unsqueeze(-1) : X; + }; + const auto matrix_to_vector = [vector_case](const Tensor& X) { + return vector_case ? X.squeeze(-1) : X; + }; + + // This case is disallowed in the primal operation as A.shape = (*, 1, 1) + TORCH_INTERNAL_ASSERT(left || !vector_case); + + auto X_ = vector_to_matrix(X); + auto dB_ = vector_to_matrix(dB); + auto R_ = left ? dA.matmul(X_) : X_.matmul(dA); + auto dX_ = at::linalg_lu_solve(LU, pivots, dB_ - R_, left); + return matrix_to_vector(dX_); +} + +std::tuple linalg_solve_backward( + const Tensor& gX, + const Tensor& X, + const Tensor& A, + const Tensor& LU, + const Tensor& pivots, + const bool left, + const bool B_requires_grad) { + // for X = A^{-1}B + // gB = A^{-H}gX + // gA = -gB X^H + at::NoTF32Guard disable_tf32; + const auto A_requires_grad = A.requires_grad(); + if (!gX.defined() || (!A_requires_grad && !B_requires_grad)) { + return {}; + } + + // [NumPy compat] Case where the rhs is a vector. + // We denote with an underscore vectors that have been converted to matrices + // by `unsqueeze(-1)` + const bool vector_case = at::native::linalg_solve_is_vector_rhs(LU, X); + const auto vector_to_matrix = [vector_case](const Tensor& X) { + return vector_case ? X.unsqueeze(-1) : X; + }; + const auto matrix_to_vector = [vector_case](const Tensor& X) { + return vector_case ? X.squeeze(-1) : X; + }; + + // If the user is going to compute higher order gradients, then we need to + // recompute the LU and the pivots + Tensor gB_; + if (at::GradMode::is_enabled()) { + gB_ = at::linalg_solve(A.mH(), vector_to_matrix(gX), left); + } else { + gB_ = at::linalg_lu_solve( + LU, pivots, vector_to_matrix(gX), left, /*adjoint*/ true); + } + + Tensor gA_; + if (A_requires_grad) { + auto X_ = vector_to_matrix(X); + gA_ = left ? -gB_.matmul(X_.mH()) : -X_.mH().matmul(gB_); + } + return std::make_tuple( + A_requires_grad ? std::move(gA_) : Tensor{}, + B_requires_grad ? matrix_to_vector(gB_) : Tensor{}); +} + +Tensor solve_jvp( + const Tensor& X, + const Tensor& A, + const Tensor& dA, + const Tensor& dB) { + return generic_solve_jvp( + [](const Tensor& A, const Tensor& dB, const Tensor& dA_contrib) { + return at::linalg_solve(A, dB - dA_contrib); + }, + X, + A, + dA, + dB); +} + +Tensor lu_unpack_backward( + const Tensor& L_grad, + const Tensor& U_grad, + const c10::SymInt& m, + const c10::SymInt& n) { + if (!L_grad.defined() && !U_grad.defined()) { + return {}; + } + const auto k = std::min(m, n); + + // Getters for the principal and complementary part of the matrices + const auto get_L1 = [m, k](const Tensor& L) { + return m == k ? L.tril(-1) : L.narrow_symint(-2, 0, k).tril(-1); + }; + const auto get_L2 = [m, k](const Tensor& L) { + return L.narrow_symint(-2, k, m - k); + }; + const auto get_U1 = [n, k](const Tensor& U) { + return n == k ? U.triu() : U.narrow_symint(-1, 0, k).triu(); + }; + const auto get_U2 = [n, k](const Tensor& U) { + return U.narrow_symint(-1, k, n - k); + }; + + if (L_grad.defined()) { + if (U_grad.defined()) { + if (m == n) { + return L_grad.tril(-1) + U_grad.triu(); + } else { + auto A1_grad = get_L1(L_grad) + get_U1(U_grad); + auto A2_grad = m > n ? get_L2(L_grad) : get_U2(U_grad); + const auto dim = m > n ? -2 : -1; + return at::cat({std::move(A1_grad), std::move(A2_grad)}, /*dim=*/dim); + } + } else { + if (m >= n) { + return L_grad.tril(-1); + } else { + auto size = L_grad.sym_sizes().vec(); + size.end()[-1] = n - m; + return at::cat( + {L_grad.tril(-1), at::zeros_symint(size, L_grad.options())}, + /*dim=*/-1); + } + } + } else { + if (n >= m) { + return U_grad.triu(); + } else { + auto size = U_grad.sym_sizes().vec(); + size.end()[-2] = m - n; + return at::cat( + {U_grad.triu(), at::zeros_symint(size, U_grad.options())}, + /*dim=*/-2); + } + } +} + +Tensor cat_jvp(const at::ITensorListRef& tensors, int64_t dim) { + Tensor out_fw_grad; + + auto materialized = tensors.materialize(); + auto any_defined = false; + for (const Tensor& t : materialized) { + any_defined |= isFwGradDefined(t); + } + + if (any_defined) { + std::vector fw_grads; + + for (const Tensor& t : materialized) { + fw_grads.push_back( + isFwGradDefined(t) + ? t._fw_grad(/*level*/ 0) + : at::_efficientzerotensor(t.sizes(), t.options())); + } + + out_fw_grad = at::cat(fw_grads, dim); + } + + return out_fw_grad; +} + +Tensor block_diag_jvp(at::TensorList tensors) { + Tensor out_fw_grad; + + auto any_defined = false; + for (const auto& t : tensors) { + any_defined |= isFwGradDefined(t); + } + + if (any_defined) { + std::vector fw_grads; + fw_grads.reserve(tensors.size()); + + for (const auto& t : tensors) { + fw_grads.push_back( + isFwGradDefined(t) + ? t._fw_grad(/*level*/ 0) + : at::_efficientzerotensor(t.sizes(), t.options())); + } + + out_fw_grad = at::block_diag(fw_grads); + } + + return out_fw_grad; +} + +Tensor stack_jvp(at::TensorList tensors, int64_t dim) { + // Basically copy of cat_jvp above + // TODO: consolidate with the logic of cat_jvp + Tensor out_fw_grad; + + auto any_defined = false; + for (const auto& t : tensors) { + any_defined |= isFwGradDefined(t); + } + + if (any_defined) { + std::vector fw_grads; + + for (auto& t : tensors) { + fw_grads.push_back( + isFwGradDefined(t) + ? t._fw_grad(/*level*/ 0) + : at::_efficientzerotensor(t.sizes(), t.options())); + } + out_fw_grad = at::stack(fw_grads, dim); + } + return out_fw_grad; +} + +Tensor cumprod_jvp( + const Tensor& self_t, + const Tensor& self_p, + const Tensor& result, + int dim) { + // Generic formula when no 0. is involved + Tensor gradient = (self_t / self_p).cumsum(dim) * result; + + // Note that we have to use at::where below as we are removing nans + + if (self_p.dim() == 0) { + gradient.masked_fill_(self_p.eq(0), self_t); + return gradient; + } else { + // For input (a, 0, b, 0, c) with gradients (t0, t1, t2, t3, t4) + // The output of cumprod is (a, 0, 0, 0, 0) + // The gradient we want to compute is (t0, a*t1, a*b*t1, 0, 0) + // We do this by: + // Get a mask of all zeros (0, 1, 0, 1, 0) + auto mask_zeros = self_p.eq(0); + // Get a mask of the first zero for each dim (0, 1, 0, 0, 0) + auto mask_first_zero = mask_zeros.logical_and(mask_zeros.cumsum(dim).eq(1)); + + // Get the new grad value that should be used after any zero happened: + // (X, a*t1, a*b*t1, 0, 0) = cumprod((a, t1, b, 0, c)) + auto new_grad = at::where(mask_first_zero, self_t, self_p).cumprod(dim); + + // Get a mask of everything after the first zero: (0, 1, 1, 1, 1) + auto mask_after_first_zero = mask_first_zero.cumsum(dim); + + // Do the final replacement + return at::where( + mask_after_first_zero.to(ScalarType::Bool), new_grad, gradient); + } +} + +// Helper for {batch,layer,group}_norms below +// Computes the jvp for `1 / input.std(dims, keepdim)` +static Tensor _invstd_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& mean_p, + const Tensor& invstd_p, + IntArrayRef dims, + int64_t numel, + bool keepdim) { + Tensor invstd_t; + if (areAnyTensorSubclassLike({input_t, input_p, mean_p, invstd_p}) || + input_t._is_zerotensor()) { + invstd_t = -invstd_p.pow(3) * (input_t - input_t.mean(dims, true)) * + (input_p - mean_p); + } else { + invstd_t = input_t - input_t.mean(dims, true); + invstd_t *= input_p - mean_p; + invstd_t *= -invstd_p.pow(3); + } + invstd_t = invstd_t.sum(dims, keepdim); + invstd_t /= numel; + return invstd_t; +} + +// Helper for {batch,layer,group}_norms below only +// Computes the jvp for `(input - input.mean(dims)) * input.invstd(dims)` +static Tensor _norm_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& mean_p, + const Tensor& invstd_p, + IntArrayRef dims, + int64_t numel) { + auto invstd_t = + _invstd_jvp(input_p, input_t, mean_p, invstd_p, dims, numel, true); + Tensor result_t; + if (areAnyTensorSubclassLike({input_t, input_p, mean_p, invstd_p}) || + input_t._is_zerotensor()) { + result_t = (input_t - input_t.mean(dims, true)) * invstd_p + + (input_p - mean_p) * invstd_t; + } else { + result_t = input_t - input_t.mean(dims, true); + result_t *= invstd_p; + auto temp = input_p - mean_p; + temp *= invstd_t; + result_t += temp; + } + return result_t; +} + +// Helper for {batch,layer,group}_norms below only +// Computes the jvp for `input * weight + bias` where weight and bias may be +// undefined Possibly modifies the input inplace +static Tensor _affine_jvp( + const std::optional& input_p, + Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& bias_t) { + // We allow input_p to be optional because if weight_p isn't defined, + // it may be possible to avoid computing input_p + TORCH_INTERNAL_ASSERT(input_p.has_value() == weight_p.defined()); + if (weight_p.defined()) { + if (areAnyTensorSubclassLike( + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + {input_p.value(), input_t, weight_p, weight_t}) || + input_t._is_zerotensor() || weight_t._is_zerotensor()) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + input_t = input_t * weight_p + input_p.value() * weight_t; + } else { + input_t *= weight_p; + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + auto temp = input_p.value(); + temp *= weight_t; + input_t += temp; + } + } + if (bias_t.defined()) { + if (areAnyTensorSubclassLike({input_t, bias_t}) || + input_t._is_zerotensor()) { + input_t = input_t + bias_t; + } else { + input_t += bias_t; + } + } + return input_t; +} + +Tensor batch_norm_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& bias_p, + const Tensor& bias_t, + const std::optional& running_mean, + const std::optional& running_var, + const Tensor& saved_mean, + const Tensor& saved_invstd, + bool train, + double eps) { + auto dims = std::vector{}; + auto view_size = input_t.sizes().vec(); + int64_t numel = 1; + for (const auto dim : c10::irange(view_size.size())) { + if (dim != 1) { + numel *= input_t.size(static_cast(dim)); + view_size[dim] = 1; + dims.push_back(static_cast(dim)); + } + } + Tensor mean_p; + Tensor invstd_p; + Tensor result_t; + if (train) { + mean_p = saved_mean.view(view_size); + invstd_p = saved_invstd.view(view_size); + result_t = _norm_jvp(input_p, input_t, mean_p, invstd_p, dims, numel); + } else { + TORCH_INTERNAL_ASSERT( + running_mean.has_value() && running_var.has_value(), + "Expect running_mean and running_var to have value when train=false"); + TORCH_CHECK( + !running_mean.value()._fw_grad(/*level=*/0).defined() && + !running_var.value()._fw_grad(/*level=*/0).defined(), + "batch_norm is not differentiable wrt running_mean and running_var, they cannot have forward grad defined"); + mean_p = running_mean.value().view(view_size); + invstd_p = + (1 / at::sqrt(running_var.value() + at::Scalar(eps))).view(view_size); + result_t = input_t * invstd_p; + } + + std::optional result_p = weight_p.defined() + ? std::optional((input_p - mean_p) * invstd_p) + : std::nullopt; + return _affine_jvp( + result_p, + result_t, + weight_p.defined() ? weight_p.view(view_size) : weight_p, + weight_t.defined() ? weight_t.view(view_size) : weight_t, + bias_t.defined() ? bias_t.view(view_size) : bias_t); +} + +Tensor layer_norm_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& bias_p, + const Tensor& bias_t, + const Tensor& saved_mean, + const Tensor& saved_invstd, + c10::SymIntArrayRef normalized_shape) { + auto dims = std::vector{}; + auto view_size = input_t.sizes().vec(); + auto view_size_affine = input_t.sizes().vec(); + + int64_t numel = 1; + for (const auto i : c10::irange(view_size.size())) { + if (i < view_size.size() - normalized_shape.size()) { + view_size_affine[i] = 1; + } else { + numel *= input_t.size(static_cast(i)); + view_size[i] = 1; + dims.push_back(static_cast(i)); + } + } + auto mean_p = saved_mean.view(view_size); + auto invstd_p = saved_invstd.view(view_size); + auto result_t = _norm_jvp(input_p, input_t, mean_p, invstd_p, dims, numel); + + std::optional result_p = weight_p.defined() + ? std::optional((input_p - mean_p) * invstd_p) + : std::nullopt; + return _affine_jvp( + result_p, + result_t, + weight_p.defined() ? weight_p.view(view_size_affine) : weight_p, + weight_t.defined() ? weight_t.view(view_size_affine) : weight_t, + bias_t.defined() ? bias_t.view(view_size_affine) : bias_t); +} + +Tensor rms_norm_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& saved_rstd, + IntArrayRef normalized_shape) { + auto dims = std::vector{}; + auto view_size = input_t.sizes().vec(); + auto view_size_affine = input_t.sizes().vec(); + + int64_t numel = 1; + for (const auto i : c10::irange(view_size.size())) { + if (i < view_size.size() - normalized_shape.size()) { + view_size_affine[i] = 1; + } else { + numel *= input_t.size(static_cast(i)); + view_size[i] = 1; + dims.push_back(static_cast(i)); + } + } + + auto rstd_p = saved_rstd.view(view_size); + + Tensor rstd_t; + if (areAnyTensorSubclassLike({input_t, input_p, rstd_p}) || + input_t._is_zerotensor()) { + rstd_t = -rstd_p.pow(3) * input_t * input_p; + } else { + rstd_t = input_t * input_p; + rstd_t *= -rstd_p.pow(3); + } + rstd_t = rstd_t.sum(dims, true); + rstd_t /= numel; + + Tensor result_t; + if (areAnyTensorSubclassLike({input_t, input_p, rstd_p}) || + input_t._is_zerotensor()) { + result_t = input_t * rstd_p + input_p * rstd_t; + } else { + result_t = input_t * rstd_p; + auto temp = input_p * rstd_t; + result_t += temp; + } + + std::optional result_p = std::nullopt; + if (weight_p.defined()) { + result_p = std::optional(input_p * rstd_p); + } + + return _affine_jvp( + result_p, + result_t, + weight_p.defined() ? weight_p.view(view_size_affine) : weight_p, + weight_t.defined() ? weight_t.view(view_size_affine) : weight_t, + Tensor()); +} + +Tensor rms_norm_rstd_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& saved_rstd, + IntArrayRef normalized_shape) { + auto dims = std::vector{}; + auto view_size = input_t.sizes().vec(); + auto view_size_affine = input_t.sizes().vec(); + + int64_t numel = 1; + for (const auto i : c10::irange(view_size.size())) { + if (i < view_size.size() - normalized_shape.size()) { + view_size_affine[i] = 1; + } else { + numel *= input_t.size(static_cast(i)); + view_size[i] = 1; + dims.push_back(static_cast(i)); + } + } + + auto rstd_p = saved_rstd.view(view_size); + Tensor rstd_t; + if (areAnyTensorSubclassLike({input_t, input_p, rstd_p}) || + input_t._is_zerotensor()) { + rstd_t = -rstd_p.pow(3) * input_t * input_p; + } else { + rstd_t = input_t * input_p; + rstd_t *= -rstd_p.pow(3); + } + rstd_t = rstd_t.sum(dims, true); + rstd_t /= numel; + return rstd_t; +} + +Tensor group_norm_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& bias_p, + const Tensor& bias_t, + const Tensor& saved_mean, + const Tensor& saved_invstd, + int64_t groups) { + auto input_shape = input_p.sizes(); + int64_t N = input_p.size(0); + int64_t C = input_p.size(1); + + auto input_t_reshaped = input_t.reshape({1, N * groups, N ? -1 : 1}); + auto input_p_reshaped = input_p.reshape({1, N * groups, N ? -1 : 1}); + + auto result_t = batch_norm_jvp( + input_p_reshaped, + input_t_reshaped, + /*weight_p=*/{}, + /*weight_t=*/{}, + /*bias_p=*/{}, + /*bias_t=*/{}, + /*running_mean=*/{}, + /*running_var=*/{}, + saved_mean, + saved_invstd, + /*train=*/true, + /*eps=*/0) + .view(input_shape); + + std::optional result_p = std::nullopt; + if (weight_p.defined()) { + std::vector view_size(input_t_reshaped.dim(), 1); + view_size[1] = input_t_reshaped.size(1); + result_p = ((input_p_reshaped - saved_mean.view(view_size)) * + saved_invstd.view(view_size)) + .view(input_shape); + } + std::vector affine_param_shape(input_p.dim(), 1); + affine_param_shape[1] = C; + + return _affine_jvp( + result_p, + result_t, + weight_p.defined() ? weight_p.view(affine_param_shape) : weight_p, + weight_t.defined() ? weight_t.view(affine_param_shape) : weight_t, + bias_t.defined() ? bias_t.view(affine_param_shape) : bias_t); +} + +Tensor group_norm_mean_jvp( + const Tensor& input_t, + const Tensor& mean_p, + int64_t groups) { + int64_t N = input_t.size(0); + std::array view_shape = {1, N * groups, N ? -1 : 1}; + auto input_t_reshaped = input_t.reshape(view_shape); + return input_t_reshaped.mean({2}, false).view_as(mean_p); +} + +Tensor group_norm_invstd_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& mean_p, + const Tensor& invstd_p, + int64_t groups) { + int64_t N = input_p.size(0); + + std::vector view_shape = {1, N * groups, N ? -1 : 1}; + + auto input_t_reshaped = input_t.reshape(view_shape); + auto input_p_reshaped = input_p.reshape(view_shape); + + return _invstd_jvp( + input_t_reshaped, + input_p_reshaped, + mean_p.view(view_shape), + invstd_p.view(view_shape), + /*dims=*/{2}, + /*numel=*/input_t_reshaped.size(2), + /*keepdim=*/false) + .view_as(invstd_p); +} + +Tensor gather_with_keepdimed_indices( + const Tensor& input, + int64_t dim, + const Tensor& indices, + bool keepdim) { + auto full_indices = indices; + if (!keepdim) { + full_indices = indices.unsqueeze(dim); + } + auto out_fw_grad = at::gather(input, dim, full_indices); + if (!keepdim) { + out_fw_grad = out_fw_grad.squeeze(dim); + } + + return out_fw_grad; +} + +// Let A in \C^{m \times n}, then its pivoted LU decomposition is +// A = P L U, where P is a permutation matrix. +// +// Useful notation: +// Let o denote the elementwise, or Hadamard, product. +// k := min(m, n) +// 1 := ones(k, k), +// 1_U = 1.tril(); +// 1_L = 1 - 1_U (note the diagonal is zero) +// For a matrix A, A^H := A.mH() +// +// Below we derive the backward algorithm for the case when m <= n. +// The case m > n could be obtained using the same idea. +// Since we assume m <= n, the LU decomposition of A could be written as +// A = (A1 | A2) = P L (U1 | U2) where A1, U1 in \C^{m \times m}, A2, U2 in +// \C^{m, n - m} +// +// Forward AD: +// +// dA = P dL U + P L dU => [left-multiply P^T] +// (P^T dA1 | P^T dA2) = (dL U1 + L dU1 | dL U2 + L dU2) (*) +// From (*): +// P^T dA1 = dL U1 + L dU1 => [left-multiply by L^{-1}, right-multiply by +// U1^{-1}] L^{-1} P^T dA1 U1^{-1} = L^{-1} dL + dU1 U1^{-1} (**). Note, L is +// lower-triangular, and so is its inverse, hence L^{-1} dL is lower-triangular. +// Also, since the diagonal of L (all ones) is never exposed explicitly (packed +// representation), the diagonal of dL is zero, and hence diag(L^{-1} dL) = 0. +// Assuming that U1 is full-rank, similarly, dU1 U1^{-1} is upper-triangular. +// Combining these observations we conclude: +// +// L^{-1} dL = (L^{-1} P^T dA1 U1^{-1}) o 1_L, +// dU1 U1^{-1} = (L^{-1} P^T dA1 U1^{-1}) o 1_U. +// +// Hence, +// dL = L [(L^{-1} P^T dA1 U1^{-1}) o 1_L], +// dU1 = [(L^{-1} P^T dA1 U1^{-1}) o 1_U] U1. +// As for dU2, from (*) it follows +// P^T dA2 = dL U2 + L dU2 => +// dU2 = L^{-1} (P^T dA2 - dL U2). +// +// Backward AD: +// +// The following equality comes very handy: +// Tr(A (B o C)) = Tr((A o B^T) C) (!) +// or in other words, given that X -> B o X is a pointwise operation +// its Jacobian is diagonal, so its differential is self-adjoint +// = +// +// Tr(A_grad^H dA) = Tr(L_grad^H dL) + Tr(U_grad^H dU), then +// +// Tr(L_grad^H dL) = Tr(L_grad^H L [(L^{-1} P^T dA1 U1^{-1}) o 1_L] = [using +// (!)] +// = Tr((L_grad^H L o 1_L^T) L^{-1} P^T dA1 U1^{-1}) = [using +// the cyclic property of Tr] = Tr(U1^{-1} (L_grad^H L o 1_L^T) +// L^{-1} P^T dA1) +// +// Similar, using (!) and the cyclic property of the trace operator: +// Tr(U_grad^H dU) = Tr(U1_grad^H dU1) + Tr(U2_grad^H dU2) +// = Tr(U1^{-1} (U1 U1_grad^H o 1_U^T) L^{-1} P^T dA1) +// + Tr(U2_grad^H L^{-1} P^T dA2) +// - Tr(U1^{-1} (U2 U2_grad^H o 1_L^T) L^{-1} P^T dA1) +// +// By combining the matrices to the left from dA1 and dA2 and then applying +// conjugate transposition, we finally arrive at: +// +// A1_grad = P L^{-H} [L^H L_grad o 1_L + U1_grad U1^H o 1_U - U2_grad U2^H o +// 1_L] U1^{-H}, A2_grad = P L^{-H} U2_grad +Tensor linalg_lu_backward( + const Tensor& L_grad, + const Tensor& U_grad, + const Tensor& P, + const Tensor& L, + const Tensor& U, + const bool pivot) { + at::NoTF32Guard disable_tf32; + // Return early if there's nothing to do + if (!L_grad.defined() && !U_grad.defined()) { + return {}; + } + + // L.shape == (..., m, k) + // U.shape == (..., k, n) + auto m = L.sym_size(-2); + auto n = U.sym_size(-1); + auto k = std::min(m, n); + + if (m == n) { + // A_grad = P L^{-H} [L^H L_grad o 1_L + U_grad U^H o 1_U] U^{-H}, + auto A_grad = L_grad.defined() ? L.mH().matmul(L_grad).tril(-1) : Tensor{}; + if (U_grad.defined()) { + A_grad = A_grad.defined() ? A_grad + U_grad.matmul(U.mH()).triu() + : U_grad.matmul(U.mH()).triu(); + } + A_grad = at::linalg_solve_triangular( + U.mH(), + A_grad, + /*upper=*/false, + /*left=*/false); + A_grad = at::linalg_solve_triangular( + L.mH(), + A_grad, + /*upper=*/true, + /*left=*/true, + /*unitriangular=*/true); + + return pivot ? P.matmul(A_grad) : std::move(A_grad); + } else if (m < n) { + // Wide case + // A1_grad = P L^{-H} [U1_grad + (L^H L_grad o 1_L - U_grad U^H o 1_U) + // U1^{-H}) U^{-H}] A2_grad = P L^{-H} U2_grad + const auto get_U1 = [n, k](const Tensor& U) { + return n == k ? U : U.narrow_symint(-1, 0, k); + }; + const auto get_U2 = [n, k](const Tensor& U) { + return U.narrow_symint(-1, k, n - k); + }; + + auto A_grad = L_grad.defined() ? L.mH().matmul(L_grad) : Tensor{}; + if (U_grad.defined()) { + A_grad = A_grad.defined() ? A_grad - U_grad.triu().matmul(U.mH()) + : -U_grad.triu().matmul(U.mH()); + } + A_grad = at::linalg_solve_triangular( + get_U1(U).mH(), + A_grad.tril(-1), + /*upper=*/false, + /*left=*/false); + + if (U_grad.defined()) { + A_grad = + at::cat({A_grad + get_U1(U_grad).triu(), get_U2(U_grad)}, /*dim=*/-1); + } + + A_grad = at::linalg_solve_triangular( + L.mH(), + A_grad, + /*upper=*/true, + /*left=*/true, + /*unitriangular=*/true); + + if (!U_grad.defined()) { + A_grad = at::cat({A_grad, at::zeros_like(get_U2(U))}, /*dim=*/-1); + } + if (pivot) { + A_grad = P.matmul(A_grad); + } + return A_grad; + } else { + // Tall case + // A1_grad = P [L1_grad + L^{-H} (U_grad U^H o 1_U - L^H L_grad o + // 1_L)]U^{-H} A2_grad = P L2_grad U^{-H} + + const auto get_L1 = [m, k](const Tensor& L) { + return m == k ? L : L.narrow_symint(-2, 0, k); + }; + const auto get_L2 = [m, k](const Tensor& L) { + return L.narrow_symint(-2, k, m - k); + }; + + auto A_grad = U_grad.defined() ? U_grad.matmul(U.mH()) : Tensor{}; + if (L_grad.defined()) { + A_grad = A_grad.defined() ? A_grad - L.mH().matmul(L_grad.tril(-1)) + : -L.mH().matmul(L_grad.tril(-1)); + } + A_grad = at::linalg_solve_triangular( + get_L1(L).mH(), + A_grad.triu(), + /*upper=*/true, + /*left=*/true, + /*unitriangular=*/true); + + if (L_grad.defined()) { + A_grad = at::cat( + {A_grad + get_L1(L_grad).tril(-1), get_L2(L_grad)}, /*dim=*/-2); + } + + A_grad = at::linalg_solve_triangular( + U.mH(), + A_grad, + /*upper=*/false, + /*left=*/false); + + if (!L_grad.defined()) { + A_grad = at::cat({A_grad, at::zeros_like(get_L2(L))}, /*dim=*/-2); + } + if (pivot) { + A_grad = P.matmul(A_grad); + } + return A_grad; + } +} + +Tensor lu_factor_ex_backward( + const Tensor& grad, + const Tensor& LU, + const Tensor& pivs, + const bool pivot) { + auto [P, L, U] = + at::lu_unpack(LU, pivs, /*unpack_data=*/true, /*unpack_pivots*/ pivot); + + // L.shape == (..., m, k) + // U.shape == (..., k, n) + const auto m = LU.sym_size(-2); + const auto n = LU.sym_size(-1); + const auto k = std::min(m, n); + const auto L_grad = grad.narrow_symint(-1, 0, k); + const auto U_grad = grad.narrow_symint(-2, 0, k); + return linalg_lu_backward( + /*L_grad=*/L_grad, /*U_grad=*/U_grad, P, L, U, pivot); +} + +// This function is based on the forward AD derivations outlined +// in the description to the linalg_lu_backward function. +std::tuple linalg_lu_jvp( + const Tensor& dA, + const Tensor& P, + const Tensor& L, + const Tensor& U, + const bool pivot) { + at::NoTF32Guard disable_tf32; + + auto m = dA.size(-2); + auto n = dA.size(-1); + auto k = std::min(m, n); + + auto PdA = pivot ? P.mT().matmul(dA) : dA; + + // similar to the backward implementation, we also consider block structures + // such as: for a matrix A of size m x n we decompose it as A = (A1 | A2) with + // A1 of size m x m if m <= n and A = (A1^T | A2^T)^T with A1 of size n x n if + // m > n. + auto PdA1 = PdA.narrow(-2, 0, k).narrow(-1, 0, k); + auto L1 = L.narrow(-2, 0, k).narrow(-1, 0, k); + auto U1 = U.narrow(-2, 0, k).narrow(-1, 0, k); + + // We form using two triangular_solve the matrix, the second one in place + // dK = L1^{-1} PdA1 U2^{-1} + auto dK = at::linalg_solve_triangular( + L1, PdA1, /*upper=*/false, /*left=*/true, /*unitriangular*/ true); + + // TODO We should be able to do this in-place. At the moment it raises: + // RuntimeError: linalg_solve_triangular(): functions with out=... + // arguments don't support automatic differentiation, but one of the + // arguments requires grad. + + // at::linalg_solve_triangular_out(dK, U1, dK, /*upper=*/true, + // /*left=*/false); + dK = at::linalg_solve_triangular(U1, dK, /*upper=*/true, /*left=*/false); + + auto dL1 = L1.matmul(dK.tril(-1)); + auto dU1 = dK.triu().matmul(U1); + + if (m == n) { + return std::make_tuple(std::move(dL1), std::move(dU1)); + } else if (m < n) { + // we only need to update dU2 defined as + // dU2 := L1^{-1} PdA2 - dK.tril(-1) U2) + const auto PdA2 = PdA.narrow(-1, k, n - k); + const auto U2 = U.narrow(-1, k, n - k); + auto dU2 = + at::linalg_solve_triangular( + L1, PdA2, /*upper=*/false, /*left=*/true, /*unitriangular*/ true) - + dK.tril(-1).matmul(U2); + return std::make_tuple( + std::move(dL1), at::cat({std::move(dU1), std::move(dU2)}, /*dim=*/-1)); + } else { + // we only need to update dL2 defined as + // dL2 := PdA2 U^{-1} - L2 dK.triu() + const auto PdA2 = PdA.narrow(-2, k, m - k); + const auto L2 = L.narrow(-2, k, m - k); + auto dL2 = + at::linalg_solve_triangular(U1, PdA2, /*upper=*/true, /*left=*/false) - + L2.matmul(dK.triu()); + return std::make_tuple( + at::cat({std::move(dL1), std::move(dL2)}, /*dim=*/-2), std::move(dU1)); + } +} + +Tensor lu_factor_ex_jvp( + const Tensor& dA, + const Tensor& LU, + const Tensor& pivs, + const bool pivot) { + auto [P, L, U] = + at::lu_unpack(LU, pivs, /*unpack_data=*/true, /*unpack_pivots=*/pivot); + auto [dL, dU] = linalg_lu_jvp(dA, P, L, U, pivot); + + auto m = dA.size(-2); + auto n = dA.size(-1); + if (m >= n) { + dL.narrow(-2, 0, n).add_(dU); + return dL; + } else { + dU.narrow(-1, 0, m).add_(dL); + return dU; + } +} + +Tensor logsumexp_jvp( + const Tensor& self_p, + const Tensor& self_t, + IntArrayRef dim, + bool keepdim) { + // NB: for simplicity, we recompute some values that can be reused from + // forward + auto self_p_exp = [&self_p, &dim]() { + if (self_p.sym_numel() > 0) { + // Use only the real part for complex tensors + return (self_p - at::amax(at::real(self_p), dim, true)) + .exp(); // Use the exp-normalize trick + } else { + // amax fails if numel() == 0, in which case it doesn't matter anyway + return self_p.exp(); + } + }(); + + auto sumexp_p = self_p_exp.sum(dim, keepdim); + + // NB: it's OK for logsumexp_jvp to be reused for formulas like + // softmax/log_softmax + // that only have one differentiable input, because that means self_t are + // never zerotensors + TORCH_INTERNAL_ASSERT(!self_t._is_zerotensor()) + if (areAnyTensorSubclassLike({self_p, self_t})) { + auto result = (self_p_exp * self_t).sum(dim, keepdim); + result /= sumexp_p; + return result; + } else { + self_p_exp *= self_t; + auto sumexp_t = self_p_exp.sum(dim, keepdim); + return sumexp_t /= sumexp_p; + } +} + +Tensor safe_logsumexp_jvp( + const Tensor& self_p, + const Tensor& self_t, + IntArrayRef dim, + bool keepdim) { + auto lse_jvp = logsumexp_jvp(self_p, self_t, dim, keepdim); + const auto neg_inf = at::scalar_tensor( + -std::numeric_limits::infinity(), + at::TensorOptions().dtype(lse_jvp.dtype()).device(lse_jvp.device())); + const auto masked = self_p.eq(neg_inf); + const auto masked_rows = all(masked, dim, true); + const auto zero = at::scalar_tensor( + 0.0, at::TensorOptions().dtype(lse_jvp.dtype()).device(lse_jvp.device())); + return at::where(masked_rows, zero, lse_jvp); +} + +Tensor warn_backwards(const Tensor& grad_output) { + TORCH_WARN("Warn from backward"); + return grad_output; +} + +// This function only exists because cuDNN does not support bias gradient +// computation and it's not easy to slice a std::tuple to return only grad_input +// / grad_weight from convolution_backward. It will be removed when the +// cudnn_convolution and cudnn_convolution_transpose go away. +std::tuple _cudnn_convolution_backward( + const at::Tensor& self, + const at::Tensor& grad_output, + const at::Tensor& weight, + at::SymIntArrayRef padding, + at::SymIntArrayRef output_padding, + at::SymIntArrayRef stride, + at::SymIntArrayRef dilation, + bool transposed, + c10::SymInt groups, + ::std::array output_mask) { + if (!grad_output.defined()) { + return std::tuple(); + } + + // Just call the general backward and ignore the bias gradient part. + std::tuple grad_inputs = + at::convolution_backward_symint( + grad_output, + self, + weight, + std::nullopt, + stride, + padding, + dilation, + transposed, + output_padding, + std::move(groups), + {output_mask[0], output_mask[1], false}); + return std::make_tuple( + std::move(std::get<0>(grad_inputs)), std::move(std::get<1>(grad_inputs))); +} + +Tensor scatter_reduce_jvp( + const Tensor& self_p, + const Tensor& self_t, + int dim, + const Tensor& index, + const Tensor& src_p, + const Tensor& src_t, + std::string_view reduce, + bool include_self, + const Tensor& result) { + if (reduce == "sum" || reduce == "mean") { + // The function is linear + return at::scatter_reduce(self_t, dim, index, src_t, reduce, include_self); + // auto mask = x == restore_reduced_dims(result, dim, keepdim); + // return at::where(mask, dx, 0.).sum(dim, keepdim) / mask.sum(dim, + // keepdim); + } else if (reduce == "amin" || reduce == "amax") { + auto gather_result = at::gather(result, dim, index); + auto mask_self = self_p == result; + auto mask_src = src_p == gather_result; + auto masked_src_t = at::where(mask_src, src_t, 0.); + auto div = + mask_self.to(self_t.dtype()) + .scatter_reduce( + dim, index, mask_src.to(self_t.dtype()), "sum", include_self); + return at::where(mask_self, self_t, 0.) + .scatter_reduce(dim, index, masked_src_t, "sum", include_self) + .div(div); + } else { + // Not implemented + return Tensor{}; + } +} + +std::tuple scatter_reduce_backward( + const Tensor& grad, + const Tensor& self, + int dim, + const Tensor& index, + const Tensor& src, + std::string_view reduce, + bool include_self, + const Tensor& result) { + Tensor grad_self, grad_src; + + // FIXME: complex gradients not handled correctly + // For now this is ok as scatter_reduce isn't added to the whitelist + // in tools/autograd/gen_variable_type.py + + if (!grad.defined()) { + return std::make_tuple(std::move(grad_self), std::move(grad_src)); + } + + if (reduce == "sum") { + grad_self = grad; + grad_src = grad.gather(dim, index); + } else if (reduce == "prod") { + // Explicitly compute exclusive prod for elements in self/src that are 0 + Tensor masked_self = self.masked_fill(self == 0, 1); + Tensor masked_self_result = + masked_self.scatter_reduce(dim, index, src, reduce, include_self); + grad_self = grad * masked_self_result / masked_self; + Tensor src_zero = src == 0; + Tensor src_num_zeros = + zeros_like(self) + .scatter_add(dim, index, src_zero.to(self.dtype())) + .gather(dim, index); + Tensor src_single_zero = bitwise_and(src_zero, src_num_zeros == 1); + // For src positions with src_single_zero, grad * result.gather(dim,index) / + // src.masked_fill(src_zero, 1) would incorrectly propagate zeros as the + // gradient + Tensor masked_src = src.masked_fill(src_single_zero, 1); + Tensor masked_src_result = + self.scatter_reduce(dim, index, masked_src, reduce, include_self); + Tensor grad_src1 = where( + src_single_zero, + (grad * masked_src_result).gather(dim, index), + (grad * result).gather(dim, index) / src.masked_fill(src_zero, 1)); + // GradMode::is_enabled() - adding the autograd Node is a no-op if autograd + // is disabled; this also avoids having the item() call in the usual case. + if (GradMode::is_enabled() && (src_num_zeros > 1).any().item()) { + auto node = c10::make_intrusive( + "scatter_reduce(): Double backward is unsupported for src when >1 zeros in src are scattered to the same position in self", + /* num inputs */ 1); + auto result = node->apply({std::move(grad_src1)}); + grad_src = result[0]; + } else { + grad_src = grad_src1; + } + } else if (reduce == "mean") { + Tensor N = include_self ? ones_like(grad) : zeros_like(grad); + N = N.scatter_add(dim, index, ones_like(src)); + N.masked_fill_(N == 0, 1); + grad_self = grad / N; + Tensor N_src = N.gather(dim, index); + grad_src = grad.gather(dim, index) / N_src; + } else if (reduce == "amax" || reduce == "amin") { + // Evenly distribute gradient when there are multiple max/mins + Tensor value = result.gather(dim, index); + Tensor self_is_result = (self == result).to(self.scalar_type()); + Tensor src_is_result = (src == value).to(self.scalar_type()); + Tensor N_to_distribute = + self_is_result.scatter_add(dim, index, src_is_result); + Tensor grad_distributed = grad / N_to_distribute; + grad_self = (self == result) * grad_distributed; + grad_src = (src == value) * grad_distributed.gather(dim, index); + } else { + TORCH_CHECK( + false, + "Expected 'reduce' to be one of 'sum', 'prod', 'mean', 'amax', 'amin' but got ", + reduce, + "."); + } + + if (!include_self) { + grad_self = grad_self.scatter(dim, index, 0); + } + + return std::make_tuple(std::move(grad_self), std::move(grad_src)); +} + +Tensor _to_copy_backward( + const Tensor& grad_, + const c10::TensorOptions& self_options) { + // Handle R->C copies without raising a warning + const auto self_type = self_options.dtype().toScalarType(); + auto grad = c10::MaybeOwned::borrowed(grad_); + if (!c10::isComplexType(self_type) && grad->is_complex()) { + grad = c10::MaybeOwned::owned(at::real(grad_)); + } + + return grad->to(self_options, /*non_blocking=*/false, /*copy=*/false); +} + +std::tuple index_reduce_backward( + const Tensor& grad, + const Tensor& self, + int dim, + const Tensor& index, + const Tensor& source, + std::string_view reduce, + bool include_self, + const Tensor& result) { + Tensor grad_self, grad_src; + + // FIXME: index_add's backward formula has a special case for source.dim == 0 + // but this case seems to throw the error "IndexError: dimension specified as + // 0 but tensor has no dimensions" look into whether this case is reachable + // and should be covered here + + if (!grad.defined()) { + return std::make_tuple(std::move(grad_self), std::move(grad_src)); + } + + if (reduce == "prod") { + Tensor masked_self = self.masked_fill(self == 0, 1); + Tensor masked_self_result = + masked_self.index_reduce(dim, index, source, reduce, include_self); + grad_self = grad * masked_self_result / masked_self; + Tensor src_zero = source == 0; + Tensor src_num_zeros = zeros_like(self) + .index_add(dim, index, src_zero.to(self.dtype())) + .index_select(dim, index); + Tensor src_single_zero = bitwise_and(src_zero, src_num_zeros == 1); + // For src positions with src_single_zero, (grad * + // result).index_select(dim,index) / source.masked_fill(src_zero, 1) would + // incorrectly propagate zeros as the gradient + Tensor masked_src = source.masked_fill(src_single_zero, 1); + Tensor masked_src_result = + self.index_reduce(dim, index, masked_src, reduce, include_self); + Tensor grad_src1 = where( + src_single_zero, + (grad * masked_src_result).index_select(dim, index), + (grad * result).index_select(dim, index) / + source.masked_fill(src_zero, 1)); + // GradMode::is_enabled() - adding the autograd Node is a no-op if autograd + // is disabled this also avoids having the item() call in the usual case + if (GradMode::is_enabled() && (src_num_zeros > 1).any().item()) { + auto node = c10::make_intrusive( + "index_reduce(): Double backward is unsupported for source when >1 zeros in source are scattered to the same position in self", + /* num inputs */ 1); + auto result = node->apply({std::move(grad_src1)}); + grad_src = result[0]; + } else { + grad_src = grad_src1; + } + } else if (reduce == "mean") { + Tensor N = include_self ? ones_like(grad) : zeros_like(grad); + N = N.index_add(dim, index, ones_like(source)); + N.masked_fill_(N == 0, 1); + grad_self = grad / N; + Tensor N_src = N.index_select(dim, index); + grad_src = grad.index_select(dim, index) / N_src; + } else if (reduce == "amax" || reduce == "amin") { + Tensor value = result.index_select(dim, index); + Tensor self_is_result = (self == result).to(self.scalar_type()); + Tensor source_is_result = (source == value).to(self.scalar_type()); + Tensor N_to_distribute = + self_is_result.index_add(dim, index, source_is_result); + Tensor grad_distributed = grad / N_to_distribute; + grad_self = self_is_result * grad_distributed; + grad_src = source_is_result * grad_distributed.index_select(dim, index); + } else { + TORCH_CHECK( + false, + "Expected 'reduce' to be one of 'prod', 'amax', 'amin' or 'mean' but got ", + reduce, + "."); + } + + if (!include_self) { + grad_self = grad_self.index_fill(dim, index, 0); + } + + return std::make_tuple(std::move(grad_self), std::move(grad_src)); +} + +Tensor take_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& indices) { + Tensor grad_self = at::zeros_like(self); + // For Composite Compliance, + // if `grad` and `indices` are CCT but `grad_self` is not + // then we use the out-of-place variant of `put`. + if (areAnyTensorSubclassLike({grad, indices})) { + return grad_self.put(indices, grad, true); + } + return grad_self.put_(indices, grad, true); +} + +Tensor to_sparse_backward( + const Tensor& grad, + const c10::Layout self_layout, + const c10::OptionalArrayRef& self_blocksize) { + // Path for strided and nested + if (self_layout == c10::kStrided) { + return grad.to_dense(); + } else { + OptionalIntArrayRef blocksize = std::nullopt; + if (self_blocksize.has_value()) { + blocksize = c10::asIntArrayRefSlowOpt(*self_blocksize); + } + return grad.to_sparse(self_layout, blocksize); + } +} + +std::tuple +mkldnn_rnn_layer_differentiable_backward( + const Tensor& input, + const Tensor& weight0, + const Tensor& weight1, + const Tensor& weight2, + const Tensor& weight3, + const Tensor& hx_, + const Tensor& cx_tmp, + const Tensor& output, + const Tensor& hy_, + const Tensor& cy_, + const std::optional& grad_output_r_opt, + const std::optional& grad_hy_r_opt, + const std::optional& grad_cy_r_opt, + bool reverse, + int64_t mode, + int64_t hidden_size, + int64_t num_layers, + bool has_biases, + bool train, + bool bidirectional, + at::IntArrayRef batch_sizes, + bool batch_first, + const at::Tensor& workspace) { + const Tensor& grad_output_r = grad_output_r_opt.value_or(Tensor()); + const Tensor& grad_hy_r = grad_hy_r_opt.value_or(Tensor()); + const Tensor& grad_cy_r = grad_cy_r_opt.value_or(Tensor()); + if (!grad_output_r.defined() && !grad_hy_r.defined() && + !grad_cy_r.defined()) { + return std::make_tuple( + Tensor(), Tensor(), Tensor(), Tensor(), Tensor(), Tensor(), Tensor()); + } + auto grad_output = grad_output_r.defined() + ? grad_output_r.contiguous() + : at::zeros_like(output, LEGACY_CONTIGUOUS_MEMORY_FORMAT); + auto grad_hy = grad_hy_r.defined() + ? grad_hy_r.contiguous() + : at::zeros_like(hx_, LEGACY_CONTIGUOUS_MEMORY_FORMAT); + auto grad_cy = cx_tmp.defined() + ? (grad_cy_r.defined() + ? grad_cy_r.contiguous() + : at::zeros_like(cx_tmp, LEGACY_CONTIGUOUS_MEMORY_FORMAT)) + : grad_cy_r.contiguous(); + Tensor bias_ih, bias_hh; + if (has_biases) { + bias_ih = weight2; + bias_hh = weight3; + } else { + bias_ih = at::zeros( + {4 /* num_bias_gates of LSTM */ * hidden_size}, weight0.options()); + bias_hh = at::zeros( + {4 /* num_bias_gates of LSTM */ * hidden_size}, weight0.options()); + } + const auto& input_ = input; + auto hx_prev = hx_; + auto cx_prev = cx_tmp; + + // Re-calculate gates and hidden states during one layer, which will be used + // in backward. + int64_t seq_length = input.size(0); + std::vector> layer_gates( + seq_length); + std::vector> layer_states(seq_length + 1); + layer_states[0] = std::make_tuple(hx_, cx_tmp); + for (int64_t seq = 1; seq < seq_length + 1; seq++) { + auto hx = hx_prev; + auto cx = cx_prev; + auto x_index = reverse ? seq_length - seq : seq - 1; + auto gate = at::linear(input_[x_index], weight0, bias_ih) + .add_(at::linear(hx, weight1, bias_hh)); + auto chunked_gates = gate.unsafe_chunk(4, 1); + auto i = chunked_gates[0].sigmoid_(); + auto f = chunked_gates[1].sigmoid_(); + auto g = chunked_gates[2].tanh_(); + auto o = chunked_gates[3].sigmoid_(); + layer_gates[x_index] = std::make_tuple(i, f, g, o); + auto cy = (f * cx).add(i * g); + auto hy = o * cy.tanh(); + layer_states[seq] = std::make_tuple(hy, cy); + hx_prev = hy; + cx_prev = cy; + } + + Tensor dx, dWx, dWh, db, db_, dprev_h, dprev_c, dWh_, dWx_; + Tensor new_grad_hy, d1, dgp, dip, dfp, dop, do_, dg, df, di, da; + std::vector layer_dx(seq_length); + for (int64_t seq = seq_length - 1; seq >= 0; seq--) { + int64_t x_index = reverse ? seq_length - seq - 1 : seq; + const auto& [i, f, g, o] = layer_gates[x_index]; + const auto& cy = std::get<1>(layer_states[seq + 1]); + const auto& [hx, cx] = layer_states[seq]; + new_grad_hy = grad_output[x_index].add(grad_hy); + d1 = grad_cy.add(new_grad_hy * o * (1 - cy.tanh() * cy.tanh())); + dgp = d1 * i; + dip = d1 * g; + dprev_c = d1 * f; + dfp = d1 * cx; + dop = new_grad_hy * cy.tanh(); + do_ = dop * o * (1 - o); + dg = dgp * (1 - g * g); + df = dfp * f * (1 - f); + di = dip * i * (1 - i); + da = at::cat({di, df, dg, do_}, 1); + db_ = at::sum(da, 0); + dx = at::matmul(da, weight0); + dx = at::unsqueeze(dx, 0); + dprev_h = at::matmul(da, weight1); + dWx_ = at::matmul(da.transpose(0, 1), input_[x_index]); + dWh_ = at::matmul(da.transpose(0, 1), hx); + if (seq == seq_length - 1) { + db = db_; + dWx = dWx_; + dWh = dWh_; + } else { + db += db_; + dWx += dWx_; + dWh += dWh_; + } + layer_dx[x_index] = dx; + grad_hy = dprev_h; + grad_cy = dprev_c; + } + + auto cat_layer_dx = at::cat(layer_dx, 0); + return std::make_tuple( + std::move(cat_layer_dx), + std::move(dWx), + std::move(dWh), + db, + db, + std::move(dprev_h), + std::move(dprev_c)); +} + +Tensor values_backward(const Tensor& grad, const Tensor& self) { + Tensor grad_self; + if (grad.defined()) { + if (self.layout() == c10::kSparse) { + return at::_sparse_coo_tensor_unsafe_symint( + self.indices(), + grad, + self.sym_sizes(), + self.options(), + /*is_coalesced=*/true); + } else if (at::sparse_csr::is_sparse_compressed(self)) { + auto [compressed_indices, plain_indices] = + at::sparse_csr::getCompressedPlainIndices(self); + return at::_sparse_compressed_tensor_unsafe_symint( + compressed_indices, + plain_indices, + grad, + self.sym_sizes(), + self.options()); + } else { + TORCH_CHECK_NOT_IMPLEMENTED( + false, + "values backward with respect to self with layout ", + self.layout()); + } + } + return grad_self; +} + +// ==================== grid_sampler double backward ==================== + +namespace { + +// Bound integer tap coordinate per padding mode (for bicubic taps). +// Returns a kLong tensor in [0, size-1]. +static Tensor gs_bound_coord( + const Tensor& idx, + int64_t size, + GridSamplerPadding padding_mode, + bool align_corners) { + if (padding_mode != GridSamplerPadding::Reflection) { + return idx.clamp(0, size - 1); + } + // Reflection: mirrors the PyTorch reflect_coordinates convention. + if (size <= 1) { + return at::zeros_like(idx); + } + auto x = idx.to(at::kDouble); + double span{}; + double min_v{}; + if (align_corners) { + span = static_cast(size - 1); + min_v = 0.0; + } else { + span = static_cast(size); + min_v = -0.5; + } + // Native formula: in = idx - min_v (= idx + 0.5 for no-align_corners). + // For integer idx this is always a half-integer, so fold result + min_v is + // an exact integer — no rounding artifacts. + auto in = x - min_v; + auto in_abs = in.abs(); + auto even = at::fmod(at::floor(in_abs / span), 2.0) < 0.5; + auto rem = at::fmod(in_abs, span); + auto r = at::where(even, rem, span - rem); + return (r + min_v).round().clamp(0, size - 1).to(at::kLong); +} + +// Compute unnormalized source coordinate and per-position chain-rule multiplier +// d(source_coord)/d(grid_coord), accounting for unnormalization and padding. +static std::pair gs_compute_coords( + const Tensor& coord, + int64_t size, + GridSamplerPadding padding_mode, + bool align_corners) { + double unnorm_scale = + static_cast(align_corners ? size - 1 : size) / 2.0; + Tensor ix = align_corners ? (coord + 1) * unnorm_scale + : (coord + 1) * unnorm_scale - 0.5; + Tensor padding_grad; + if (padding_mode == GridSamplerPadding::Zeros) { + padding_grad = at::ones_like(ix); + } else if (padding_mode == GridSamplerPadding::Border) { + // clip_coordinates_set_grad: grad = 1 iff strictly inside (0, size-1) + padding_grad = + ((ix > 0) & (ix < static_cast(size - 1))).to(ix.dtype()); + ix = ix.clamp(0, size - 1); + } else { + TORCH_CHECK_NOT_IMPLEMENTED( + padding_mode == GridSamplerPadding::Reflection, + "Unknown padding mode: ", + static_cast(padding_mode)); + double twice_low = align_corners ? 0.0 : -1.0; + double twice_high = + static_cast(2 * size) - (align_corners ? 2.0 : 1.0); + if (twice_high <= twice_low) { + auto z = at::zeros_like(ix); + return {z, z}; + } + double min_v = twice_low / 2.0; + double span = (twice_high - twice_low) / 2.0; + auto in = ix - min_v; + auto in_abs = in.abs(); + auto reflect_sign = at::where(in >= 0, 1.0, -1.0); + auto even = at::fmod(at::floor(in_abs / span), 2.0) < 0.5; + auto flip_sign = at::where(even, 1.0, -1.0); + auto rem = in_abs % span; + auto ix_refl = at::where(even, rem + min_v, span - rem + min_v); + auto clip_grad = ((ix_refl > 0) & (ix_refl < static_cast(size - 1))) + .to(ix.dtype()); + padding_grad = reflect_sign * flip_sign * clip_grad; + ix = ix_refl.clamp(0, size - 1); + } + return {std::move(ix), padding_grad * unnorm_scale}; +} + +static Tensor gs_accum_sumprod_k(const Tensor& values, const Tensor& basis) { + return (values * basis.unsqueeze(1)).sum(-1); +} + +static Tensor gs_outer_last(const Tensor& a, const Tensor& b) { + auto shape = a.sizes().vec(); + shape.back() *= b.size(-1); + return (b.unsqueeze(-1) * a.unsqueeze(-2)).reshape(shape); +} + +// Multi-tap gather for 2D: h_idx/w_idx [N, Ho, Wo, K] → [N, C, Ho, Wo, K] +static Tensor gs_gather2d_multi( + const Tensor& input, + const Tensor& h_idx, + const Tensor& w_idx, + bool zeros_oob) { + auto N = input.size(0), C = input.size(1); + auto H = input.size(2), W = input.size(3); + auto out_H = h_idx.size(1), out_W = h_idx.size(2), K = h_idx.size(3); + auto flat = (h_idx.clamp(0, H - 1) * W + w_idx.clamp(0, W - 1)) + .reshape({N, 1, out_H * out_W * K}) + .expand({N, C, out_H * out_W * K}); + auto result = input.reshape({N, C, H * W}) + .gather(2, flat) + .reshape({N, C, out_H, out_W, K}); + if (zeros_oob) { + auto mask = (h_idx >= 0) & (h_idx < H) & (w_idx >= 0) & (w_idx < W); + result = result * mask.unsqueeze(1).to(result.dtype()); + } + return result; +} + +// Multi-tap scatter for 2D: values [N,C,Ho,Wo], weights [N,Ho,Wo,K] → [N,C,H,W] +static Tensor gs_scatter2d_multi( + const Tensor& values, + const Tensor& weights, + const Tensor& h_idx, + const Tensor& w_idx, + int64_t H, + int64_t W, + bool zeros_oob) { + auto N = values.size(0), C = values.size(1); + auto out_H = values.size(2), out_W = values.size(3); + auto K = h_idx.size(3); + auto flat = (h_idx.clamp(0, H - 1) * W + w_idx.clamp(0, W - 1)) + .reshape({N, 1, out_H * out_W * K}) + .expand({N, C, out_H * out_W * K}); + auto weighted = values.unsqueeze(-1) * weights.unsqueeze(1); + if (zeros_oob) { + auto mask = (h_idx >= 0) & (h_idx < H) & (w_idx >= 0) & (w_idx < W); + weighted = weighted * mask.unsqueeze(1).to(weighted.dtype()); + } + return at::zeros({N, C, H * W}, values.options()) + .scatter_add(2, flat, weighted.reshape({N, C, out_H * out_W * K})) + .reshape({N, C, H, W}); +} + +// Multi-tap bounded gather for bicubic 2D: h_idx/w_idx [N, Ho, Wo, K] → [N, C, +// Ho, Wo, K] +static Tensor gs_gather2d_bc_multi( + const Tensor& input, + const Tensor& h_idx, + const Tensor& w_idx, + GridSamplerPadding padding_mode, + bool align_corners) { + auto N = input.size(0), C = input.size(1); + auto H = input.size(2), W = input.size(3); + auto out_H = h_idx.size(1), out_W = h_idx.size(2), K = h_idx.size(3); + auto flat = (gs_bound_coord(h_idx, H, padding_mode, align_corners) * W + + gs_bound_coord(w_idx, W, padding_mode, align_corners)) + .reshape({N, 1, out_H * out_W * K}) + .expand({N, C, out_H * out_W * K}); + auto result = input.reshape({N, C, H * W}) + .gather(2, flat) + .reshape({N, C, out_H, out_W, K}); + if (padding_mode == GridSamplerPadding::Zeros) { + auto mask = (h_idx >= 0) & (h_idx < H) & (w_idx >= 0) & (w_idx < W); + result = result * mask.unsqueeze(1).to(result.dtype()); + } + return result; +} + +// Multi-tap bounded scatter for bicubic 2D: values [N,C,Ho,Wo], weights +// [N,Ho,Wo,K] → [N,C,H,W] +static Tensor gs_scatter2d_bc_multi( + const Tensor& values, + const Tensor& weights, + const Tensor& h_idx, + const Tensor& w_idx, + int64_t H, + int64_t W, + GridSamplerPadding padding_mode, + bool align_corners) { + auto N = values.size(0), C = values.size(1); + auto out_H = values.size(2), out_W = values.size(3); + auto K = h_idx.size(3); + auto flat = (gs_bound_coord(h_idx, H, padding_mode, align_corners) * W + + gs_bound_coord(w_idx, W, padding_mode, align_corners)) + .reshape({N, 1, out_H * out_W * K}) + .expand({N, C, out_H * out_W * K}); + auto weighted = values.unsqueeze(-1) * weights.unsqueeze(1); + if (padding_mode == GridSamplerPadding::Zeros) { + auto mask = (h_idx >= 0) & (h_idx < H) & (w_idx >= 0) & (w_idx < W); + weighted = weighted * mask.unsqueeze(1).to(weighted.dtype()); + } + return at::zeros({N, C, H * W}, values.options()) + .scatter_add(2, flat, weighted.reshape({N, C, out_H * out_W * K})) + .reshape({N, C, H, W}); +} + +// Multi-tap gather for 3D: d_idx/h_idx/w_idx [N, Do, Ho, Wo, K] → [N, C, Do, +// Ho, Wo, K] +static Tensor gs_gather3d_multi( + const Tensor& input, + const Tensor& d_idx, + const Tensor& h_idx, + const Tensor& w_idx, + bool zeros_oob) { + auto N = input.size(0), C = input.size(1); + auto D = input.size(2), H = input.size(3), W = input.size(4); + auto out_D = d_idx.size(1), out_H = d_idx.size(2), out_W = d_idx.size(3), + K = d_idx.size(4); + auto flat = ((d_idx.clamp(0, D - 1) * H + h_idx.clamp(0, H - 1)) * W + + w_idx.clamp(0, W - 1)) + .reshape({N, 1, out_D * out_H * out_W * K}) + .expand({N, C, out_D * out_H * out_W * K}); + auto result = input.reshape({N, C, D * H * W}) + .gather(2, flat) + .reshape({N, C, out_D, out_H, out_W, K}); + if (zeros_oob) { + auto mask = (d_idx >= 0) & (d_idx < D) & (h_idx >= 0) & (h_idx < H) & + (w_idx >= 0) & (w_idx < W); + result = result * mask.unsqueeze(1).to(result.dtype()); + } + return result; +} + +// Multi-tap scatter for 3D: values [N,C,Do,Ho,Wo], weights [N,Do,Ho,Wo,K] → +// [N,C,D,H,W] +static Tensor gs_scatter3d_multi( + const Tensor& values, + const Tensor& weights, + const Tensor& d_idx, + const Tensor& h_idx, + const Tensor& w_idx, + int64_t D, + int64_t H, + int64_t W, + bool zeros_oob) { + auto N = values.size(0), C = values.size(1); + auto out_D = values.size(2), out_H = values.size(3), out_W = values.size(4); + auto K = d_idx.size(4); + auto flat = ((d_idx.clamp(0, D - 1) * H + h_idx.clamp(0, H - 1)) * W + + w_idx.clamp(0, W - 1)) + .reshape({N, 1, out_D * out_H * out_W * K}) + .expand({N, C, out_D * out_H * out_W * K}); + auto weighted = values.unsqueeze(-1) * weights.unsqueeze(1); + if (zeros_oob) { + auto mask = (d_idx >= 0) & (d_idx < D) & (h_idx >= 0) & (h_idx < H) & + (w_idx >= 0) & (w_idx < W); + weighted = weighted * mask.unsqueeze(1).to(weighted.dtype()); + } + return at::zeros({N, C, D * H * W}, values.options()) + .scatter_add(2, flat, weighted.reshape({N, C, out_D * out_H * out_W * K})) + .reshape({N, C, D, H, W}); +} + +} // anonymous namespace + +std::tuple grid_sampler_2d_double_backward( + const Tensor& ggI, + const Tensor& ggGrid, + const Tensor& grad_output, + const Tensor& input, + const Tensor& grid, + int64_t interpolation_mode, + int64_t padding_mode, + bool align_corners, + std::array output_mask) { + Tensor d_grad_output, d_input, d_grid; + const auto interpolation = + static_cast(interpolation_mode); + const auto padding_mode_enum = static_cast(padding_mode); + + // ggI -> d_grad_output: gather ggI at grid positions = grid_sampler_2d(ggI, + // grid) + if (output_mask[0] && ggI.defined()) { + d_grad_output = at::grid_sampler_2d( + ggI, grid, interpolation_mode, padding_mode, align_corners); + } + // ggI -> d_grid: same structure as grad_grid but with ggI as "input" + if (output_mask[2] && ggI.defined()) { + d_grid = std::get<1>(at::grid_sampler_2d_backward( + grad_output, + ggI, + grid, + interpolation_mode, + padding_mode, + align_corners, + {false, true})); + } + // d_input from ggI is 0: grad_input has no dependence on input. + // For nearest, grad_grid = 0, so all ggGrid contributions vanish. + if (!ggGrid.defined() || interpolation == GridSamplerInterpolation::Nearest) { + return {std::move(d_grad_output), std::move(d_input), std::move(d_grid)}; + } + auto H = input.size(2), W = input.size(3); + + auto [ix, gix_mult] = gs_compute_coords( + grid.select(-1, 0), W, padding_mode_enum, align_corners); + auto [iy, giy_mult] = gs_compute_coords( + grid.select(-1, 1), H, padding_mode_enum, align_corners); + + auto x0 = at::floor(ix).to(at::kLong); + auto y0 = at::floor(iy).to(at::kLong); + auto fx = ix - x0.to(ix.dtype()); + auto fy = iy - y0.to(iy.dtype()); + + bool zeros_oob = (padding_mode_enum == GridSamplerPadding::Zeros); + auto ggG_x = ggGrid.select(-1, 0) * gix_mult; // (N, out_H, out_W) + auto ggG_y = ggGrid.select(-1, 1) * giy_mult; + + if (interpolation == GridSamplerInterpolation::Bilinear) { + auto x1 = x0 + 1; + auto y1 = y0 + 1; + // Stack all 4 tap indices: [N, Ho, Wo, 4] order: nw, ne, sw, se + auto h_stk = at::stack({y0, y0, y1, y1}, -1); + auto w_stk = at::stack({x0, x1, x0, x1}, -1); + + Tensor I; // [N, C, Ho, Wo, 4] + if (output_mask[0] || output_mask[1] || output_mask[2]) { + I = gs_gather2d_multi(input, h_stk, w_stk, zeros_oob); + } + + Tensor dw_dx, dw_dy; + if (output_mask[0] || output_mask[1]) { + // Derivative weights [N, Ho, Wo, 4]: d(w_k)/d(ix) and d(w_k)/d(iy) + auto omx = 1.0 - fx, omy = 1.0 - fy; + dw_dx = at::stack({-omy, omy, -fy, fy}, -1); + dw_dy = at::stack({-omx, -fx, omx, fx}, -1); + } + + if (output_mask[0]) { + auto sum_dx = gs_accum_sumprod_k(I, dw_dx); + auto sum_dy = gs_accum_sumprod_k(I, dw_dy); + auto contrib = sum_dx * ggG_x.unsqueeze(1); + contrib.addcmul_(sum_dy, ggG_y.unsqueeze(1)); + d_grad_output = d_grad_output.defined() ? d_grad_output + contrib + : std::move(contrib); + } + + if (output_mask[1]) { + // Combined per-tap weight [N, Ho, Wo, 4] + auto w_in = ggG_x.unsqueeze(-1) * dw_dx; + w_in.addcmul_(ggG_y.unsqueeze(-1), dw_dy); + d_input = + gs_scatter2d_multi(grad_output, w_in, h_stk, w_stk, H, W, zeros_oob); + } + + if (output_mask[2]) { + // cross second derivative: coeffs [nw,ne,sw,se] = [+1,-1,-1,+1] + auto I_nw = I.select(-1, 0); + auto I_ne = I.select(-1, 1); + auto I_sw = I.select(-1, 2); + auto I_se = I.select(-1, 3); + auto cross = (grad_output * (I_nw - I_ne - I_sw + I_se)).sum(1); + auto d_grid_x = gix_mult * ggG_y * cross; + auto d_grid_y = giy_mult * ggG_x * cross; + auto contrib = at::stack({std::move(d_grid_x), std::move(d_grid_y)}, -1); + d_grid = d_grid.defined() ? d_grid + contrib : std::move(contrib); + } + } else if (interpolation == GridSamplerInterpolation::Bicubic) { + // Native bicubic backward uses raw unnormalized coordinates with a constant + // gx_mult/gy_mult (the unnormalize scale only), applying padding + // exclusively when sampling each tap value. Using gs_compute_coords here + // would be wrong: for border/reflection modes it zeroes gix_mult at + // boundaries, making ggG_x/y = 0 where native still has nonzero + // sensitivity. + auto x_scale = static_cast(align_corners ? W - 1 : W) / 2.0; + auto y_scale = static_cast(align_corners ? H - 1 : H) / 2.0; + auto x_raw = align_corners ? (grid.select(-1, 0) + 1) * x_scale + : (grid.select(-1, 0) + 1) * x_scale - 0.5; + auto y_raw = align_corners ? (grid.select(-1, 1) + 1) * y_scale + : (grid.select(-1, 1) + 1) * y_scale - 0.5; + auto x0_bc = at::floor(x_raw).to(at::kLong); + auto y0_bc = at::floor(y_raw).to(at::kLong); + auto ggG_x_bc = ggGrid.select(-1, 0) * x_scale; + auto ggG_y_bc = ggGrid.select(-1, 1) * y_scale; + auto fx = x_raw - x0_bc.to(x_raw.dtype()); + auto fy = y_raw - y0_bc.to(y_raw.dtype()); + + // Cubic interpolation coefficients and derivatives w.r.t. fractional + // offset. For t in [0,1], the four corners are at offsets {-1, 0, 1, 2} + // from base. + constexpr double A = -0.75; + auto fx1 = fx + 1.0, fx2 = 1.0 - fx, fx3 = 2.0 - fx; + auto fy1 = fy + 1.0, fy2 = 1.0 - fy, fy3 = 2.0 - fy; + + auto cx_t = at::stack( + {(((A * fx1) - (5 * A)) * fx1 + (8 * A)) * fx1 - (4 * A), + (((A + 2) * fx) - (A + 3)) * fx.square() + 1, + (((A + 2) * fx2) - (A + 3)) * fx2.square() + 1, + (((A * fx3) - (5 * A)) * fx3 + (8 * A)) * fx3 - (4 * A)}, + -1); + auto dcx_t = at::stack( + {(((3 * A) * fx1) - (10 * A)) * fx1 + (8 * A), + (((3 * (A + 2)) * fx) - (2 * (A + 3))) * fx, + -((((3 * (A + 2)) * fx2) - (2 * (A + 3))) * fx2), + (((-3 * A) * fx3) + (10 * A)) * fx3 - (8 * A)}, + -1); + auto cy_t = at::stack( + {(((A * fy1) - (5 * A)) * fy1 + (8 * A)) * fy1 - (4 * A), + (((A + 2) * fy) - (A + 3)) * fy.square() + 1, + (((A + 2) * fy2) - (A + 3)) * fy2.square() + 1, + (((A * fy3) - (5 * A)) * fy3 + (8 * A)) * fy3 - (4 * A)}, + -1); + auto dcy_t = at::stack( + {(((3 * A) * fy1) - (10 * A)) * fy1 + (8 * A), + (((3 * (A + 2)) * fy) - (2 * (A + 3))) * fy, + -((((3 * (A + 2)) * fy2) - (2 * (A + 3))) * fy2), + (((-3 * A) * fy3) + (10 * A)) * fy3 - (8 * A)}, + -1); + + // Stack coefficients into [N, Ho, Wo, 4] tensors for vectorized outer + // products. Tap indices [N, Ho, Wo, 16]: flat index k=j*4+i → h=y0+(j-1), + // w=x0+(i-1) + auto offs = at::arange(-1, 3, x0_bc.options()); + auto xt = x0_bc.unsqueeze(-1) + offs; // [N, Ho, Wo, 4] + auto yt = y0_bc.unsqueeze(-1) + offs; + // Outer product: h_idx[...,j,i]=yt[j], w_idx[...,j,i]=xt[i] + auto w_idx_bc = xt.unsqueeze(-2) + .expand({-1, -1, -1, 4, 4}) + .reshape({-1, xt.size(1), xt.size(2), 16}); + auto h_idx_bc = yt.unsqueeze(-1) + .expand({-1, -1, -1, 4, 4}) + .reshape({-1, yt.size(1), yt.size(2), 16}); + + // Gather all 16 taps at once: [N, C, Ho, Wo, 16] + auto I_all = gs_gather2d_bc_multi( + input, h_idx_bc, w_idx_bc, padding_mode_enum, align_corners); + + Tensor B_dx, B_dy; + if (output_mask[0] || output_mask[1]) { + // 1D weight outer products → [N, Ho, Wo, 16] where k=j*4+i + // B_dx[k] = dcx[i]*cy[j], B_dy[k] = cx[i]*dcy[j] + B_dx = gs_outer_last(dcx_t, cy_t); + B_dy = gs_outer_last(cx_t, dcy_t); + } + + if (output_mask[0]) { + auto sum_dx = gs_accum_sumprod_k(I_all, B_dx); + auto sum_dy = gs_accum_sumprod_k(I_all, B_dy); + auto contrib = sum_dx * ggG_x_bc.unsqueeze(1); + contrib.addcmul_(sum_dy, ggG_y_bc.unsqueeze(1)); + d_grad_output = d_grad_output.defined() ? d_grad_output + contrib + : std::move(contrib); + } + + if (output_mask[1]) { + auto w_in = ggG_x_bc.unsqueeze(-1) * B_dx; + w_in.addcmul_(ggG_y_bc.unsqueeze(-1), B_dy); + d_input = gs_scatter2d_bc_multi( + grad_output, + w_in, + h_idx_bc, + w_idx_bc, + H, + W, + padding_mode_enum, + align_corners); + } + + // d_grid from ggGrid: non-zero for bicubic (second derivative of cubic + // weights). + if (output_mask[2]) { + auto d2cx_t = at::stack( + {((6 * A) * fx1) - (10 * A), + ((6 * (A + 2)) * fx) - (2 * (A + 3)), + ((6 * (A + 2)) * fx2) - (2 * (A + 3)), + ((6 * A) * fx3) - (10 * A)}, + -1); + auto d2cy_t = at::stack( + {((6 * A) * fy1) - (10 * A), + ((6 * (A + 2)) * fy) - (2 * (A + 3)), + ((6 * (A + 2)) * fy2) - (2 * (A + 3)), + ((6 * A) * fy3) - (10 * A)}, + -1); + Tensor dot_dx2, dot_dy2, dot_dxdy; + for (const auto j : c10::irange(4)) { + auto cy_j = cy_t.select(-1, j); + auto dcy_j = dcy_t.select(-1, j); + auto d2cy_j = d2cy_t.select(-1, j); + for (const auto i : c10::irange(4)) { + auto tap_dot = (grad_output * I_all.select(-1, j * 4 + i)).sum(1); + auto dx2_term = tap_dot * d2cx_t.select(-1, i) * cy_j; + auto dy2_term = tap_dot * cx_t.select(-1, i) * d2cy_j; + auto dxdy_term = tap_dot * dcx_t.select(-1, i) * dcy_j; + dot_dx2 = + dot_dx2.defined() ? dot_dx2.add_(dx2_term) : std::move(dx2_term); + dot_dy2 = + dot_dy2.defined() ? dot_dy2.add_(dy2_term) : std::move(dy2_term); + dot_dxdy = dot_dxdy.defined() ? dot_dxdy.add_(dxdy_term) + : std::move(dxdy_term); + } + } + auto ggrid_d_grid_x = ggG_x_bc * dot_dx2; + ggrid_d_grid_x.addcmul_(ggG_y_bc, dot_dxdy); + ggrid_d_grid_x.mul_(x_scale); + auto ggrid_d_grid_y = ggG_x_bc * dot_dxdy; + ggrid_d_grid_y.addcmul_(ggG_y_bc, dot_dy2); + ggrid_d_grid_y.mul_(y_scale); + auto ggrid_d_grid = + at::stack({std::move(ggrid_d_grid_x), std::move(ggrid_d_grid_y)}, -1); + d_grid = + d_grid.defined() ? d_grid + ggrid_d_grid : std::move(ggrid_d_grid); + } + } else { + TORCH_CHECK_NOT_IMPLEMENTED( + false, + "grid_sampler_2d_double_backward does not support interpolation mode ", + interpolation_mode); + } + return {std::move(d_grad_output), std::move(d_input), std::move(d_grid)}; +} + +std::tuple grid_sampler_3d_double_backward( + const Tensor& ggI, + const Tensor& ggGrid, + const Tensor& grad_output, + const Tensor& input, + const Tensor& grid, + int64_t interpolation_mode, + int64_t padding_mode, + bool align_corners, + std::array output_mask) { + Tensor d_grad_output, d_input, d_grid; + const auto interpolation = + static_cast(interpolation_mode); + const auto padding_mode_enum = static_cast(padding_mode); + + if (output_mask[0] && ggI.defined()) { + d_grad_output = at::grid_sampler_3d( + ggI, grid, interpolation_mode, padding_mode, align_corners); + } + if (output_mask[2] && ggI.defined()) { + d_grid = std::get<1>(at::grid_sampler_3d_backward( + grad_output, + ggI, + grid, + interpolation_mode, + padding_mode, + align_corners, + {false, true})); + } + if (!ggGrid.defined() || interpolation == GridSamplerInterpolation::Nearest) { + return {std::move(d_grad_output), std::move(d_input), std::move(d_grid)}; + } + TORCH_CHECK_NOT_IMPLEMENTED( + interpolation == GridSamplerInterpolation::Bilinear, + "grid_sampler_3d double backward not implemented for interpolation_mode=", + interpolation_mode); + + // Bilinear 3D: ggGrid contributions. + auto D = input.size(2), H = input.size(3), W = input.size(4); + + auto [ix, gix_mult] = gs_compute_coords( + grid.select(-1, 0), W, padding_mode_enum, align_corners); + auto [iy, giy_mult] = gs_compute_coords( + grid.select(-1, 1), H, padding_mode_enum, align_corners); + auto [iz, giz_mult] = gs_compute_coords( + grid.select(-1, 2), D, padding_mode_enum, align_corners); + + auto x0 = at::floor(ix).to(at::kLong); + auto y0 = at::floor(iy).to(at::kLong); + auto z0 = at::floor(iz).to(at::kLong); + auto x1 = x0 + 1, y1 = y0 + 1, z1 = z0 + 1; + auto fx = ix - x0.to(ix.dtype()); + auto fy = iy - y0.to(iy.dtype()); + auto fz = iz - z0.to(iz.dtype()); + + bool zeros_oob = (padding_mode_enum == GridSamplerPadding::Zeros); + auto ggG_x = ggGrid.select(-1, 0) * gix_mult; + auto ggG_y = ggGrid.select(-1, 1) * giy_mult; + auto ggG_z = ggGrid.select(-1, 2) * giz_mult; + + // Stack all 8 tap indices [N, Do, Ho, Wo, 8]: tnw,tne,tsw,tse,bnw,bne,bsw,bse + auto d_stk = at::stack({z0, z0, z0, z0, z1, z1, z1, z1}, -1); + auto h_stk = at::stack({y0, y0, y1, y1, y0, y0, y1, y1}, -1); + auto w_stk = at::stack({x0, x1, x0, x1, x0, x1, x0, x1}, -1); + + Tensor I; // [N, C, Do, Ho, Wo, 8] + if (output_mask[0] || output_mask[1] || output_mask[2]) { + I = gs_gather3d_multi(input, d_stk, h_stk, w_stk, zeros_oob); + } + + Tensor dw_dx, dw_dy, dw_dz; + if (output_mask[0] || output_mask[1]) { + // Trilinear derivative weights [N, Do, Ho, Wo, 8] + auto omx = 1.0 - fx, omy = 1.0 - fy, omz = 1.0 - fz; + dw_dx = at::stack( + {-omy * omz, + omy * omz, + -fy * omz, + fy * omz, + -omy * fz, + omy * fz, + -fy * fz, + fy * fz}, + -1); + dw_dy = at::stack( + {-omx * omz, + -fx * omz, + omx * omz, + fx * omz, + -omx * fz, + -fx * fz, + omx * fz, + fx * fz}, + -1); + dw_dz = at::stack( + {-omx * omy, + -fx * omy, + -omx * fy, + -fx * fy, + omx * omy, + fx * omy, + omx * fy, + fx * fy}, + -1); + } + + if (output_mask[0]) { + auto sum_dx = gs_accum_sumprod_k(I, dw_dx); + auto sum_dy = gs_accum_sumprod_k(I, dw_dy); + auto sum_dz = gs_accum_sumprod_k(I, dw_dz); + auto d_gO = sum_dx * ggG_x.unsqueeze(1); + d_gO.addcmul_(sum_dy, ggG_y.unsqueeze(1)); + d_gO.addcmul_(sum_dz, ggG_z.unsqueeze(1)); + d_grad_output = + d_grad_output.defined() ? d_grad_output + d_gO : std::move(d_gO); + } + + if (output_mask[1]) { + auto w_in = ggG_x.unsqueeze(-1) * dw_dx; + w_in.addcmul_(ggG_y.unsqueeze(-1), dw_dy); + w_in.addcmul_(ggG_z.unsqueeze(-1), dw_dz); + d_input = gs_scatter3d_multi( + grad_output, w_in, d_stk, h_stk, w_stk, D, H, W, zeros_oob); + } + + // ggGrid -> d_grid: cross second derivatives for 3D bilinear. + // d²output/(dix diy) = (1-fz)*(I_tnw-I_tne-I_tsw+I_tse) + fz*(I_bnw-...) + // d²output/(dix diz) = (1-fy)*(I_tnw-I_tne-I_bnw+I_bne) + fy*(I_tsw-...) + // d²output/(diy diz) = (1-fx)*(I_tnw-I_tsw-I_bnw+I_bsw) + fx*(I_tne-...) + if (output_mask[2]) { + auto fx_ = fx.unsqueeze(1), fy_ = fy.unsqueeze(1), fz_ = fz.unsqueeze(1); + auto I_tnw = I.select(-1, 0); + auto I_tne = I.select(-1, 1); + auto I_tsw = I.select(-1, 2); + auto I_tse = I.select(-1, 3); + auto I_bnw = I.select(-1, 4); + auto I_bne = I.select(-1, 5); + auto I_bsw = I.select(-1, 6); + auto I_bse = I.select(-1, 7); + auto top_xy = I_tnw - I_tne; + top_xy.sub_(I_tsw); + top_xy.add_(I_tse); + auto bottom_xy = I_bnw - I_bne; + bottom_xy.sub_(I_bsw); + bottom_xy.add_(I_bse); + auto d2_xy = (1 - fz_) * top_xy; + d2_xy.add_(fz_ * bottom_xy); + + auto top_xz = I_tnw - I_tne; + top_xz.sub_(I_bnw); + top_xz.add_(I_bne); + auto bottom_xz = I_tsw - I_tse; + bottom_xz.sub_(I_bsw); + bottom_xz.add_(I_bse); + auto d2_xz = (1 - fy_) * top_xz; + d2_xz.addcmul_(fy_, bottom_xz); + + auto top_yz = I_tnw - I_tsw; + top_yz.sub_(I_bnw); + top_yz.add_(I_bsw); + auto bottom_yz = I_tne - I_tse; + bottom_yz.sub_(I_bne); + bottom_yz.add_(I_bse); + auto d2_yz = (1 - fx_) * top_yz; + d2_yz.addcmul_(fx_, bottom_yz); + auto dot_xy = (grad_output * d2_xy).sum(1); + auto dot_xz = (grad_output * d2_xz).sum(1); + auto dot_yz = (grad_output * d2_yz).sum(1); + auto d_grid_x = ggG_y * dot_xy; + d_grid_x.addcmul_(ggG_z, dot_xz); + d_grid_x.mul_(gix_mult); + auto d_grid_y = ggG_x * dot_xy; + d_grid_y.addcmul_(ggG_z, dot_yz); + d_grid_y.mul_(giy_mult); + auto d_grid_z = ggG_x * dot_xz; + d_grid_z.addcmul_(ggG_y, dot_yz); + d_grid_z.mul_(giz_mult); + auto contrib = at::stack( + {std::move(d_grid_x), std::move(d_grid_y), std::move(d_grid_z)}, -1); + d_grid = d_grid.defined() ? d_grid + contrib : std::move(contrib); + } + return {std::move(d_grad_output), std::move(d_input), std::move(d_grid)}; +} + +} // namespace torch::autograd::generated::details diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/FunctionsManual.h b/western_ai/usa/pytorch/torch/csrc/autograd/FunctionsManual.h new file mode 100644 index 0000000000000000000000000000000000000000..024dc25a28a6eee094d42749bbccab28f4fd6ef1 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/FunctionsManual.h @@ -0,0 +1,1195 @@ +#pragma once + +// NB: Must be at the top of file to avoid including the deprecated "math.h". +// https://stackoverflow.com/questions/6563810/m-pi-works-with-math-h-but-not-with-cmath-in-visual-studio +#ifdef _MSC_VER +#ifndef _USE_MATH_DEFINES +#define _USE_MATH_DEFINES +#endif +#include +#endif + +#include +#include + +namespace torch::autograd::generated::details { + +extern const char* kCudnnDoubleBackwardMsg; + +// A simple way to imperatively compute index ranges for slots +// that have been flattened +struct TORCH_API IndexRangeGenerator { + IndexRange range(size_t range_size) { + i += range_size; + return {i - range_size, i}; + } + size_t size() { + return i; + } + + private: + size_t i = 0; +}; + +TORCH_API Tensor toNonOptFwGrad(const std::optional& t); +TORCH_API Tensor toNonOptPrimal(const std::optional& t); +TORCH_API Tensor toNonOptTensor(const std::optional& t); + +inline std::optional wrap_opt_if(const Tensor& t, const bool cond) { + using OptTensor = std::optional; + return cond ? OptTensor(t) : static_cast(std::nullopt); +} + +TORCH_API Tensor +apply_loss_reduction(const Tensor& unreduced, int64_t reduction); +TORCH_API bool any_variable_defined(const variable_list& variables); +TORCH_API void update_wrapped_number(Tensor& input, Tensor& output); +TORCH_API void copy_range( + variable_list& out, + IndexRange range, + const at::Tensor& t); +TORCH_API void copy_range( + variable_list& out, + IndexRange range, + at::ArrayRef t); +TORCH_API at::Tensor copysign_tensor_self_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& result); +TORCH_API at::Tensor not_implemented(const char* name, const char* reason = ""); +TORCH_API std::vector not_implemented_list( + const char* name, + const char* reason = ""); +at::Tensor handle_r_to_c(ScalarType self_st, Tensor gradient_result); +at::Tensor maybe_multiply(const at::Tensor& t, const at::Scalar& s); +int64_t _safe_size(IntArrayRef sizes, IntArrayRef dim); +Tensor restore_reduced_dims( + const Tensor& output, + IntArrayRef dims, + bool keepdim); +Tensor scale_grad_by_count( + const Tensor& grad, + const Tensor& mask, + IntArrayRef dims); +at::Tensor norm_backward( + const at::Tensor& grad, + const at::Tensor& self, + const std::optional& p_, + const at::Tensor& norm); +at::Tensor norm_backward( + at::Tensor grad, + const at::Tensor& self, + const std::optional& p_, + at::Tensor norm, + at::IntArrayRef dim, + bool keepdim); +Tensor norm_jvp( + const Tensor& self_p, + const Tensor& self_t, + const std::optional& p_, + Tensor norm, + IntArrayRef dim, + bool keepdim); +Tensor norm_jvp( + const Tensor& grad, + const Tensor& self, + const std::optional& p_, + Tensor norm); +Tensor _nested_from_padded_backward( + const Tensor& grad, + const Tensor& input, + const bool do_transform_0213); +std::tuple linear_double_backward( + const variable_list& grads, + const Tensor& self, + const Tensor& grad_output, + const Tensor& weight); +Tensor linalg_vector_norm_jvp( + const Tensor& self_p, + const Tensor& self_t, + const Scalar& scalar_ord, + Tensor norm, + const at::OptionalIntArrayRef& opt_dim, + bool keepdim); +at::Tensor linalg_vector_norm_backward( + at::Tensor grad, + const at::Tensor& self, + const at::Scalar& ord, + at::Tensor norm, + const at::OptionalIntArrayRef& opt_dim, + bool keepdim); +at::Tensor pow_backward( + at::Tensor grad, + const at::Tensor& self, + const at::Scalar& exponent_); +at::Tensor pow_backward_self( + const at::Tensor& grad, + const at::Tensor& self, + const at::Tensor& exponent); +at::Tensor pow_backward_exponent( + const at::Tensor& grad, + const at::Tensor& self, + const at::Tensor& exponent, + const at::Tensor& result); +at::Tensor pow_backward_exponent( + const at::Tensor& grad, + const at::Scalar& base, + const at::Tensor& exponent, + const at::Tensor& result); +at::Tensor angle_backward(const at::Tensor& grad, const at::Tensor& self); +template +at::Tensor mul_tensor_backward( + const Tensor& grad, + const T& other, + ScalarType self_st); +template +at::Tensor div_tensor_self_backward( + const Tensor& grad, + const T& other, + ScalarType self_st, + const std::optional& rounding_mode = std::nullopt); +at::Tensor div_tensor_other_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& other, + const std::optional& rounding_mode = std::nullopt); +at::Tensor mvlgamma_backward( + const at::Tensor& grad, + const at::Tensor& self, + int64_t p); +at::Tensor permute_backwards(const at::Tensor& grad, at::IntArrayRef fwd_dims); +at::Tensor rad2deg_backward(const at::Tensor& grad); +at::Tensor deg2rad_backward(const at::Tensor& grad); +at::Tensor unsqueeze_multiple( + const at::Tensor& t, + at::OptionalIntArrayRef opt_dim, + size_t n_dims); +at::Tensor sum_backward( + const at::Tensor& grad, + at::SymIntArrayRef sizes, + at::OptionalIntArrayRef opt_dims, + bool keepdim); +at::Tensor sum_backward( + const at::Tensor& grad, + c10::SymIntArrayRef sizes, + c10::IntArrayRef dims, + bool keepdim); +at::Tensor nansum_backward( + const at::Tensor& grad, + const at::Tensor& self, + at::OptionalIntArrayRef dims, + bool keepdim); +std::vector reverse_list(const at::IntArrayRef list); +std::vector reverse_list_symint(const c10::SymIntArrayRef list); +at::Tensor reverse_dim(const at::Tensor& t, int64_t dim); +at::Tensor prod_safe_zeros_backward( + const at::Tensor& grad, + const at::Tensor& inp, + int64_t dim); +at::Tensor prod_backward( + const at::Tensor& grad, + const at::Tensor& input, + const at::Tensor& result); +at::Tensor prod_backward( + at::Tensor grad, + const at::Tensor& input, + at::Tensor result, + int64_t dim, + bool keepdim); +at::Tensor solve_jvp( + const Tensor& X, + const Tensor& A, + const Tensor& dA, + const Tensor& dB); +at::Tensor solve_backward_self( + const at::Tensor& grad, + const at::Tensor& self, + const at::Tensor& A); +at::Tensor solve_backward_A( + const at::Tensor& grad, + const at::Tensor& self, + const at::Tensor& A, + const at::Tensor& solution); +at::Tensor cumsum_backward(const at::Tensor& grad, int64_t dim); +at::Tensor logsumexp_backward( + at::Tensor grad, + const at::Tensor& self, + at::Tensor result, + at::IntArrayRef dim, + bool keepdim); +at::Tensor logsumexp_jvp( + const at::Tensor& self_p, + const at::Tensor& self_t, + IntArrayRef dim, + bool keepdim); +at::Tensor safe_logsumexp_jvp( + const at::Tensor& self_p, + const at::Tensor& self_t, + IntArrayRef dim, + bool keepdim); +at::Tensor logcumsumexp_backward( + at::Tensor grad, + const at::Tensor& self, + const at::Tensor& result, + int64_t dim); +at::Tensor logcumsumexp_jvp( + const at::Tensor& self_p, + const at::Tensor& self_t, + int64_t dim); +at::Tensor unbind_backward(const variable_list& grads, int64_t dim); +at::Tensor unbind_backward_nested( + const variable_list& grads, + const Tensor& nt_sizes, + int64_t dim, + const at::TensorOptions& options); +at::Tensor unbind_backward_nested_jagged( + const variable_list& grads, + const Tensor& self, + int64_t dim); +at::Tensor unsqueeze_to(const at::Tensor& self, c10::SymIntArrayRef sym_sizes); +at::Tensor unsqueeze_to( + const at::Tensor& self, + int64_t dim, + c10::SymIntArrayRef sym_sizes); +at::Tensor unsqueeze_to( + const at::Tensor& self, + IntArrayRef dim, + c10::SymIntArrayRef sym_sizes); +std::vector cat_tensors_backward( + const at::Tensor& grad, + const std::vector>& sizes, + const std::vector& dtypes, + int64_t dim); +std::vector stack_tensors_backward( + const at::Tensor& grad, + int64_t dim, + const std::vector& dtypes); +std::vector block_diag_backward( + const at::Tensor& grad, + const std::vector>& sizes, + const std::vector& dtypes); +at::Tensor clamp_backward( + const at::Tensor& grad, + const at::Tensor& self, + const std::optional& min, + const std::optional& max); +at::Tensor clamp_backward( + const at::Tensor& grad, + const at::Tensor& self, + const at::Tensor& min, + const at::Tensor& max); +std::tuple clamp_backward_min_max( + const at::Tensor& grad, + const at::Tensor& self, + const at::Tensor& min, + const at::Tensor& max, + const std::array& /*grad_input_mask*/); +at::Tensor clamp_jvp( + const Tensor& self_p, + const Tensor& self_t, + const Tensor& min_p, + const Tensor& min_t, + const Tensor& max_p, + const Tensor& max_t); +at::SymIntArrayRef strides_or_error( + const Tensor& input, + std::string_view const& input_name); +at::Tensor mm_mat1_backward( + const Tensor& grad, + const Tensor& mat2, + at::SymIntArrayRef mat1_sizes, + at::SymIntArrayRef mat1_strides, + c10::Layout mat1_layout, + const Scalar& alpha); +at::Tensor mm_mat2_backward( + const at::Tensor& grad, + const at::Tensor& mat1, + at::SymIntArrayRef sizes, + at::SymIntArrayRef strides, + c10::Layout layout, + const at::Scalar& alpha); +at::Tensor _grouped_mm_mat1_backward( + const Tensor& grad, + const Tensor& mat2, + at::SymIntArrayRef mat1_sizes, + at::SymIntArrayRef mat1_strides, + c10::Layout mat1_layout, + std::optional offs, + const Scalar& alpha); +at::Tensor _grouped_mm_mat2_backward( + const at::Tensor& grad, + const at::Tensor& mat1, + at::SymIntArrayRef sizes, + at::SymIntArrayRef strides, + c10::Layout layout, + std::optional offs, + const at::Scalar& alpha); +at::Tensor mm_mat1_sparse_backward( + const at::Tensor& grad, + const at::Tensor& mat1, + const at::Tensor& mat2, + const at::Scalar& alpha); +std::tuple sparse_sampled_addmm_backward( + const Tensor& grad, + const Tensor& self, + const std::optional& mat1, + const std::optional& mat2, + const Scalar& alpha, + const Scalar& beta, + const std::array& grad_input_mask); +at::Tensor sparse_mask_backward( + const at::Tensor& grad, + const at::Tensor& mask, + c10::Layout self_layout); +at::Tensor sparse_sparse_matmul_backward( + const at::Tensor& grad, + const at::Tensor& mat1, + const at::Tensor& mat2, + int64_t grad_order); +at::Tensor renorm_backward( + const at::Tensor& grad, + const at::Tensor& self, + const at::Scalar& p, + int64_t dim, + const at::Scalar& maxnorm); +at::Tensor renorm_jvp( + const at::Tensor& self_p, + const at::Tensor& self_t, + const at::Scalar& p, + int64_t dim, + const at::Scalar& maxnorm); +at::Tensor repeat_backward( + at::Tensor grad, + at::SymIntArrayRef repeats, + at::SymIntArrayRef input_shape); +at::Tensor _fused_dropout_backward( + const at::Tensor& grad, + const at::Tensor& mask, + double p1m); +at::Tensor infinitely_differentiable_native_dropout_backward( + const at::Tensor& grad, + const at::Tensor& mask, + double scale); +at::Tensor native_dropout_double_backward( + const at::Tensor& ggI, + const at::Tensor& grad, + const at::Tensor& mask, + double scale); +at::Tensor evenly_distribute_backward( + const at::Tensor& grad, + const at::Tensor& input, + const at::Tensor& value); +Tensor sgn_backward(const Tensor& x, const Tensor& gx, const Tensor& sgn); +Tensor masked_fill_backward(const Tensor& grad, const Tensor& mask); +at::Tensor var_backward( + at::Tensor grad, + const at::Tensor& self, + at::OptionalIntArrayRef dim, + const std::optional& correction, + bool keepdim); +at::Tensor var_jvp( + const at::Tensor& self_t, + const at::Tensor& self_p, + const at::Tensor& result, + at::OptionalIntArrayRef dim_opt, + const std::optional& correction, + bool keepdim); +at::Tensor std_backward( + const at::Tensor& result, + const at::Tensor& grad, + const at::Tensor& self, + at::OptionalIntArrayRef dim, + const std::optional& correction, + bool keepdim); +Tensor mean_backward( + const Tensor& grad, + c10::SymIntArrayRef shape, + at::OptionalIntArrayRef opt_dim, + c10::SymInt numel, + bool keepdim); +Tensor var_mean_backward( + const Tensor& gvar, + const Tensor& gmean, + const Tensor& self, + at::OptionalIntArrayRef dim_opt, + const std::optional& correction, + bool keepdim); +Tensor std_mean_backward( + const Tensor& gstd, + const Tensor& gmean, + const Tensor& self, + const Tensor& std, + at::OptionalIntArrayRef dim_opt, + const std::optional& correction, + bool keepdim); +at::Tensor cholesky_backward( + const at::Tensor& grad, + bool upper, + const at::Tensor& L); +at::Tensor cholesky_jvp( + const at::Tensor& input_tangent, + const at::Tensor& L, + bool upper); +at::Tensor cholesky_inverse_backward( + const at::Tensor& grad, + const at::Tensor& L, + bool upper, + const at::Tensor& inverse); +at::Tensor cholesky_inverse_jvp( + const at::Tensor& F, + const at::Tensor& dF, + const at::Tensor& X, + bool upper); +Tensor pinv_jvp(const Tensor& A, const Tensor& pinvA, const Tensor& dA); +Tensor pinv_backward(const Tensor& grad, const Tensor& pinvA, const Tensor& A); +Tensor chunk_backward_nested( + const std::vector& grads, + const Tensor& self, + int64_t chunks, + int64_t dim); +at::Tensor split_with_sizes_backward( + const std::vector& grads, + c10::SymIntArrayRef split_sizes, + int64_t dim, + c10::SymIntArrayRef sizes, + const at::TensorOptions& options); +at::Tensor _nested_split_with_sizes_backward( + const std::vector& grads, + c10::SymIntArrayRef split_sizes, + int64_t dim, + const Tensor& nt_sizes, + const at::TensorOptions& options); +at::Tensor split_backward( + const std::vector& grads, + const c10::SymInt& split_size, + int64_t dim, + c10::SymIntArrayRef sizes, + const at::TensorOptions& options); +at::Tensor max_pool_double_backward( + const at::Tensor& grad, + const at::Tensor& indices, + int dim); +at::Tensor error_for_max_pool2d_double_backward(); +at::Tensor glu_double_backward( + const at::Tensor& grad, + const at::Tensor& grad_output, + const at::Tensor& input, + int64_t dim); +at::Tensor glu_double_backward_grad_output( + const at::Tensor& grad, + const at::Tensor& input, + int64_t dim); +at::Tensor infinitely_differentiable_silu_backward( + const at::Tensor& grad_output, + const at::Tensor& input); +at::Tensor infinitely_differentiable_mish_backward( + const at::Tensor& grad_output, + const at::Tensor& input); +Tensor infinitely_differentiable_logit_backward( + const Tensor& grad, + const Tensor& self, + std::optional eps); +Tensor binary_cross_entropy_target_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& target, + const std::optional& weight, + int64_t reduction); +Tensor binary_cross_entropy_double_backward_target( + const Tensor& grad, + const Tensor& grad_output, + const Tensor& self, + const Tensor& target, + const std::optional& weight, + int64_t reduction); +Tensor binary_cross_entropy_with_logits_backward( + const Tensor& grad, + const Tensor& input, + const Tensor& target, + const std::optional& weight_opt, + const std::optional& pos_weight_opt, + int64_t reduction); +at::Tensor binary_cross_entropy_with_logits_target_backward( + const at::Tensor& grad_output, + const at::Tensor& self, + const at::Tensor& target, + const std::optional& weight, + const std::optional& pos_weight, + int64_t reduction); +at::Tensor log_sigmoid_double_backward( + const at::Tensor& grad, + const at::Tensor& input); +at::Tensor softmax_double_backward( + const at::Tensor& grad, + const at::Tensor& grad_output, + int dim, + const at::Tensor& output); +at::Tensor binary_cross_entropy_double_backward( + const at::Tensor& grad_output, + const at::Tensor& grad, + const at::Tensor& input, + const at::Tensor& target, + const std::optional& weight, + int64_t reduction); +at::Tensor binary_cross_entropy_double_backward_grad_output( + const at::Tensor& grad, + const at::Tensor& input, + const at::Tensor& target, + const std::optional& weight, + int64_t reduction); +at::Tensor smooth_l1_loss_double_backward( + const at::Tensor& grad, + const at::Tensor& input, + const at::Tensor& target, + int64_t reduction, + double beta); +at::Tensor huber_loss_double_backward( + const at::Tensor& grad, + const at::Tensor& input, + const at::Tensor& target, + int64_t reduction, + double delta); +at::Tensor huber_loss_double_backward_grad_output( + const at::Tensor& grad, + const at::Tensor& grad_output, + const at::Tensor& input, + const at::Tensor& target, + int64_t reduction, + double delta); +at::Tensor mse_loss_double_backward( + const at::Tensor& grad, + const at::Tensor& input, + int64_t reduction); +at::Tensor soft_margin_loss_double_backward( + const at::Tensor& grad, + const at::Tensor& input, + const at::Tensor& target, + int64_t reduction); +at::Tensor soft_margin_loss_double_backward_grad_output( + const at::Tensor& grad, + const at::Tensor& grad_output, + const at::Tensor& input, + const at::Tensor& target, + int64_t reduction); +at::Tensor softplus_double_backward( + const at::Tensor& grad, + const at::Tensor& input, + const at::Scalar& beta, + const at::Scalar& threshold); +std::tuple slogdet_jvp( + const at::Tensor& LU, + const at::Tensor& pivots, + const at::Tensor& dA, + const at::Tensor& sign, + const bool use_A_T); +at::Tensor slogdet_backward( + const at::Tensor& grad_sign, + const at::Tensor& grad_logabsdet, + const at::Tensor& A, + const at::Tensor& signdet, + const at::Tensor& LU, + const at::Tensor& pivots); +at::Tensor log1p_backward(const at::Tensor& grad, const at::Tensor& self); +at::Tensor sinc_backward(const at::Tensor& grad, const at::Tensor& self); +at::Tensor sparse_constructor_values_backward( + const at::Tensor& sparse_grad_out, + const at::Tensor& indices); +at::Tensor embedding_dense_double_backward_symint( + const at::Tensor& grad, + const at::Tensor& indices, + const c10::SymInt& padding_idx); +at::Tensor index_backward( + at::Tensor zeros_like_self, + const torch::List>& indices, + const at::Tensor& grad); +at::Tensor _cudnn_ctc_loss_backward( + const at::Tensor& grad_out, + const at::Tensor& loss, + const at::Tensor& raw_grad, + bool zero_infinity); +at::Tensor _miopen_ctc_loss_backward( + const at::Tensor& grad_out, + const at::Tensor& loss, + const at::Tensor& raw_grad, + bool zero_infinity); +at::Tensor elu_double_backward( + const Tensor& grad, + const Tensor& grad_output, + const Scalar& alpha, + const Scalar& scale, + const Scalar& input_scale, + bool is_result, + const Tensor& self_or_result); + +Tensor svd_backward( + const Tensor& gU, + const Tensor& gS, + const Tensor& gVh, + const Tensor& U, + const Tensor& S, + const Tensor& Vh); + +std::tuple linalg_svd_jvp( + const Tensor& dA, + const Tensor& U, + const Tensor& S, + const Tensor& Vh, + const bool full_matrices); +Tensor slice_backward_wrapper( + const at::Tensor& grad, + const c10::SymIntArrayRef& input_sizes, + int64_t dim, + std::optional start, + std::optional end, + c10::SymInt step); +std::tuple linalg_eig_jvp( + const Tensor& dA, + const Tensor& L, + const Tensor& V, + const bool is_hermitian); +Tensor linalg_eig_backward( + const Tensor& gL, + const Tensor& gV, + const Tensor& L, + const Tensor& V, + const bool is_hermitian, + const bool symeig_eigenvectors = true); +Tensor linalg_lstsq_solution_jvp( + const Tensor& A, + const Tensor& B_, + const Tensor& dA, + const Tensor& dB_); +Tensor linalg_lstsq_residuals_jvp( + const Tensor& A, + const Tensor& B_, + const Tensor& dA, + const Tensor& dB_, + const Tensor& X_, + const Tensor& L); +std::tuple triangular_solve_backward( + const Tensor& grad_x, + const Tensor& grad_m, + const Tensor& b, + const Tensor& a, + const Tensor& x, + const bool upper, + const bool transpose, + const bool unitriangular, + std::array output_mask); +Tensor triangular_solve_jvp( + const Tensor& X, + const Tensor& A, + const Tensor& dA, + const Tensor& dB, + const bool upper, + const bool transpose, + const bool unitriangular); +Tensor linalg_solve_triangular_forward_AD( + const Tensor& A_t, + const Tensor& B_t, + const Tensor& A, + const Tensor& X, + const bool upper, + const bool left, + const bool unitriangular); +std::tuple linalg_solve_triangular_backward( + const Tensor& grad, + const Tensor& A, + const Tensor& X, + const bool upper, + const bool left, + const bool unitriangular, + std::array output_mask); +std::tuple _trilinear_backward( + const Tensor& grad_out, + const std::optional& i1, + const std::optional& i2, + const std::optional& i3, + IntArrayRef expand1, + IntArrayRef expand2, + IntArrayRef expand3, + IntArrayRef sumdim, + std::array grad_mask); +std::tuple linalg_qr_jvp( + const Tensor& dA, + const Tensor& Q, + const Tensor& R, + const std::string_view mode); +Tensor linalg_qr_backward( + const Tensor& gQ, + const Tensor& gR, + const Tensor& Q, + const Tensor& R, + const std::string_view mode); +Tensor linalg_matrix_exp_differential( + const Tensor& self, + const Tensor& grad, + bool adjoint); +Tensor linalg_matrix_sqrth_differential(const Tensor& self, const Tensor& grad); +std::tuple batchnorm_double_backward( + const Tensor& input, + const std::optional& gamma, + const Tensor& ggI, + const Tensor& ggG, + const Tensor& ggB, + const Tensor& gO, + const std::optional& running_mean, + const std::optional& running_var, + bool training, + double eps, + const std::optional& save_mean, + const std::optional& save_invstd, + std::array output_mask); +std::tuple grid_sampler_2d_double_backward( + const Tensor& ggI, + const Tensor& ggGrid, + const Tensor& grad_output, + const Tensor& input, + const Tensor& grid, + int64_t interpolation_mode, + int64_t padding_mode, + bool align_corners, + std::array output_mask); +std::tuple grid_sampler_3d_double_backward( + const Tensor& ggI, + const Tensor& ggGrid, + const Tensor& grad_output, + const Tensor& input, + const Tensor& grid, + int64_t interpolation_mode, + int64_t padding_mode, + bool align_corners, + std::array output_mask); +std::tuple _euclidean_dist_backward( + const Tensor& grad, + const Tensor& x1, + const Tensor& x2, + const Tensor& res); +Tensor fft_backward( + const Tensor& self, + const Tensor& grad, + int64_t signal_ndim, + bool complex_input, + bool complex_output, + bool inverse, + IntArrayRef checked_signal_sizes, + int64_t normalization, + bool onesided, + IntArrayRef output_sizes); +Tensor fft_r2c_backward( + const Tensor& grad, + at::IntArrayRef dim, + int64_t normalization, + bool onesided, + const c10::SymInt& last_dim_size); +Tensor fft_c2r_backward( + const Tensor& grad, + IntArrayRef dim, + int64_t normalization); +Tensor constant_pad_nd_backward(const Tensor& grad, c10::SymIntArrayRef pad); +std::tuple cholesky_solve_backward( + const Tensor& grad_x, + const Tensor& self, + const Tensor& input2, + const Tensor& result, + const bool upper, + std::array output_mask); +Tensor cholesky_solve_jvp( + const Tensor& X, + const Tensor& U, + const Tensor& dU, + const Tensor& dB, + const bool upper); +std::tuple +infinitely_differentiable_native_group_norm_backward( + const Tensor& dY, + const Tensor& dmean, + const Tensor& drstd, + const Tensor& X, + const Tensor& mean, + const Tensor& rstd, + const std::optional& gamma, + const c10::SymInt& N, + const c10::SymInt& C, + const c10::SymInt& HxW, + int64_t group, + double eps, + std::array grad_input_mask); +Tensor gelu_double_backward( + const Tensor& ggI, + const Tensor& gO, + const Tensor& input, + std::string_view approximate); +Tensor as_strided_backward( + Tensor grad, + const TensorGeometry& input_geometry, + c10::SymIntArrayRef sizes, + c10::SymIntArrayRef strides, + const std::optional& storage_offset_); +Tensor as_strided_scatter_backward( + const Tensor& grad, + const TensorGeometry& input_geometry, + const TensorGeometry& src_geometry, + c10::SymIntArrayRef sizes, + c10::SymIntArrayRef strides, + std::optional storage_offset); +std::tuple atan2_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& other, + std::array output_mask); +Tensor amaxamin_jvp( + const Tensor& x, + const Tensor& dx, + const Tensor& result, + IntArrayRef dim, + bool keepdim); +Tensor aminmax_backward( + const at::Tensor& self, + std::optional dim, + bool keepdim, + const at::Tensor& grad_min, + const at::Tensor& grad_max, + const at::Tensor& min, + const at::Tensor& max); +Tensor aminmax_jvp( + const Tensor& self_p, + const Tensor& self_t, + const Tensor& result, + std::optional dim, + bool keepdim); +std::tuple layer_norm_double_backward( + const Tensor& input, + const std::optional& gamma, + const Tensor& ggI, + const Tensor& ggG, + const Tensor& ggB, + const Tensor& gO, + const Tensor& save_mean, + const Tensor& save_invstd, + c10::SymIntArrayRef normalized_shape, + std::array output_mask); + +std::tuple infinitely_differentiable_native_rms_norm_backward( + const Tensor& dY, + const Tensor& drstd, + const Tensor& input, + IntArrayRef normalized_shape, + const Tensor& rstd, + const std::optional& weight_opt, + std::array grad_input_mask); + +std::tuple householder_product_backward( + const Tensor& grad, + const Tensor& result, + const Tensor& input, + const Tensor& tau, + const bool flip_order = false); +Tensor householder_product_jvp( + const Tensor& dV, + const Tensor& dtau, + const Tensor& prod, + const Tensor& V, + const Tensor& tau); +std::tuple ormqr_backward( + const Tensor& grad, + const Tensor& result, + const Tensor& self, + const Tensor& tau, + const Tensor& other, + bool left, + bool transpose, + std::array grad_output_mask); +std::tuple polar_backward( + const Tensor& grad, + const Tensor& result); +Tensor i1_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& result); +Tensor i1e_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& result); +Tensor linalg_lu_solve_LU( + const Tensor& grad, + const Tensor& LU, + const Tensor& pivots, + const Tensor& X, + const bool left, + const bool adjoint); +Tensor linalg_lu_solve_jvp( + const Tensor& X, + const Tensor& LU, + const Tensor& pivots, + const Tensor& dLU, + const Tensor& dB, + const bool left, + const bool adjoint); +std::tuple linalg_solve_backward( + const Tensor& gX, + const Tensor& X, + const Tensor& A, + const Tensor& LU, + const Tensor& pivots, + const bool left, + const bool B_requires_grad); +Tensor linalg_solve_jvp( + const Tensor& dA, + const Tensor& dB, + const Tensor& X, + const Tensor& LU, + const Tensor& pivots, + const bool left); +Tensor lu_unpack_backward( + const Tensor& L_grad, + const Tensor& U_grad, + const c10::SymInt& m, + const c10::SymInt& n); + +Tensor linalg_det_backward( + const Tensor& grad, + const Tensor& det, + const Tensor& A, + const Tensor& LU, + const Tensor& pivots); +Tensor linalg_det_jvp( + const Tensor& dA, + const Tensor& det, + const Tensor& LU, + const Tensor& pivots, + const bool use_A_T); +std::tuple linalg_lstsq_backward( + const Tensor& gX_, + const Tensor& gL, + const Tensor& A, + const Tensor& B_, + const Tensor& X_, + const std::array& grad_input_mask); +Tensor linalg_lu_backward( + const Tensor& L_grad, + const Tensor& U_grad, + const Tensor& P, + const Tensor& L, + const Tensor& U, + const bool pivot); + +std::tuple linalg_lu_jvp( + const Tensor& dA, + const Tensor& P, + const Tensor& L, + const Tensor& U, + const bool pivot); + +Tensor lu_factor_ex_backward( + const Tensor& grad, + const Tensor& LU, + const Tensor& pivs, + const bool pivot); +Tensor lu_factor_ex_jvp( + const Tensor& dX, + const Tensor& LU, + const Tensor& pivs, + const bool pivot); + +Tensor batch_norm_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& bias_p, + const Tensor& bias_t, + const std::optional& running_mean, + const std::optional& running_var, + const Tensor& saved_mean, + const Tensor& saved_invstd, + bool train, + double eps); + +Tensor layer_norm_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& bias_p, + const Tensor& bias_t, + const Tensor& saved_mean, + const Tensor& saved_invstd, + c10::SymIntArrayRef normalized_shape); + +Tensor rms_norm_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& saved_rstd, + IntArrayRef normalized_shape); + +Tensor rms_norm_rstd_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& saved_rstd, + IntArrayRef normalized_shape); + +Tensor group_norm_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& bias_p, + const Tensor& bias_t, + const Tensor& saved_mean, + const Tensor& saved_invstd, + int64_t groups); +Tensor group_norm_mean_jvp( + const Tensor& input_t, + const Tensor& mean_p, + int64_t groups); +Tensor group_norm_invstd_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& mean_p, + const Tensor& invstd_p, + int64_t groups); + +Tensor convolution_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& bias_p, + const Tensor& bias_t, + at::SymIntArrayRef stride, + at::SymIntArrayRef padding, + at::SymIntArrayRef dilation, + bool transposed, + at::SymIntArrayRef output_padding, + const c10::SymInt& groups); + +Tensor _convolution_jvp( + const Tensor& input_p, + const Tensor& input_t, + const Tensor& weight_p, + const Tensor& weight_t, + const Tensor& bias_p, + const Tensor& bias_t, + at::SymIntArrayRef stride, + at::SymIntArrayRef padding, + at::SymIntArrayRef dilation, + bool transposed, + at::SymIntArrayRef output_padding, + const c10::SymInt& groups, + bool benchmark, + bool deterministic, + bool cudnn_enabled, + bool allow_tf32); + +Tensor convolution_backward_jvp_grad_bias( + const Tensor& grad_out_t, + const Tensor& grad_bias); + +Tensor cat_jvp(const at::ITensorListRef& tensors, int64_t dim); +Tensor block_diag_jvp(at::TensorList tensors); +Tensor stack_jvp(at::TensorList tensors, int64_t dim); +Tensor cumprod_jvp( + const Tensor& self_t, + const Tensor& self_p, + const Tensor& result, + int dim); +Tensor gather_with_keepdimed_indices( + const Tensor& input, + int64_t dim, + const Tensor& indices, + bool keepdim); +Tensor evenly_read_jvp( + const Tensor& fw_grad, + const Tensor& input, + const Tensor& value); +Tensor warn_backwards(const Tensor& grad_output); + +std::tuple _cudnn_convolution_backward( + const at::Tensor& self, + const at::Tensor& grad_output, + const at::Tensor& weight, + at::SymIntArrayRef padding, + at::SymIntArrayRef output_padding, + at::SymIntArrayRef stride, + at::SymIntArrayRef dilation, + bool transposed, + c10::SymInt groups, + ::std::array output_mask); + +Tensor scatter_reduce_jvp( + const Tensor& self_p, + const Tensor& self_t, + int dim, + const Tensor& index, + const Tensor& src_p, + const Tensor& src_t, + std::string_view reduce, + bool include_self, + const Tensor& result); + +std::tuple scatter_reduce_backward( + const Tensor& grad, + const Tensor& self, + int dim, + const Tensor& index, + const Tensor& src, + std::string_view reduce, + bool include_self, + const Tensor& result); + +Tensor _to_copy_backward( + const Tensor& grad, + const c10::TensorOptions& self_options); + +std::tuple index_reduce_backward( + const Tensor& grad, + const Tensor& self, + int dim, + const Tensor& index, + const Tensor& source, + std::string_view reduce, + bool include_self, + const Tensor& result); + +Tensor take_backward( + const Tensor& grad, + const Tensor& self, + const Tensor& indices); + +Tensor to_sparse_backward( + const Tensor& grad, + const c10::Layout self_layout, + const c10::OptionalArrayRef& self_blocksize); + +std::tuple +mkldnn_rnn_layer_differentiable_backward( + const Tensor& input, + const Tensor& weight0, + const Tensor& weight1, + const Tensor& weight2, + const Tensor& weight3, + const Tensor& hx_, + const Tensor& cx_tmp, + const Tensor& output, + const Tensor& hy_, + const Tensor& cy_, + const std::optional& grad_output_r_opt, + const std::optional& grad_hy_r_opt, + const std::optional& grad_cy_r_opt, + bool reverse, + int64_t mode, + int64_t hidden_size, + int64_t num_layers, + bool has_biases, + bool train, + bool bidirectional, + at::IntArrayRef batch_sizes, + bool batch_first, + const at::Tensor& workspace); + +Tensor values_backward(const Tensor& grad, const Tensor& self); + +} // namespace torch::autograd::generated::details diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/InferenceMode.h b/western_ai/usa/pytorch/torch/csrc/autograd/InferenceMode.h new file mode 100644 index 0000000000000000000000000000000000000000..6c5516e20af354b2dacb023834d23d4dafe13be0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/InferenceMode.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +namespace torch::autograd { + +using InferenceMode = c10::InferenceMode; + +} diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/README.md b/western_ai/usa/pytorch/torch/csrc/autograd/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9c7d6dfdc419fe6fa90ee8f8db77d7c067af1cbb --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/README.md @@ -0,0 +1,33 @@ +## Autograd + +Autograd is a hotspot for PyTorch performance, so most of the heavy lifting is +implemented in C++. This implies that we have to do some shuffling between +Python and C++; and in general, we want data to be in a form that is convenient +to manipulate from C++. + +Our general model is that for any key data type that autograd manipulates, +there are two implementations: a C++ type and a Python object type. For +example, consider variables in autograd: we have both `Variable` in `variable.h` +(the C++ type) and `THPVariable` in `python_variable.h` (the Python type.) +(By the way, THP stands for TorcH Python, not to be confused with THPP, TorcH +C++). `Variable` contains the payload of a variable, while `THPVariable` just +contains a `shared_ptr` reference to `Variable`, as well as references to other +Python objects which the Python runtime needs to know about. A lot of +data accessor implementations in `python_variable.cpp` simply reach through +to the underlying `Variable` and return the appropriate value. + +The most complicated application of this principle is Function, which also +supports users implementing custom behavior in Python. We have the following +classes: + +* `Node` in `function.h`, the C++ type. +* `THPFunction` in `python_function.h`, the Python object type. In + `python_function.cpp`, you can see the boilerplate that tells the Python + interpreter about this object. +* `PyNode` in `python_function.h`, a subclass of `Node` which forwards + `apply` to a Python `THPFunction`. (NOT a Python object, despite its name!) + +Outside of `PyNode`, the C++ objects largely avoid referencing Python +objects (there are a few exceptions, like `pyobj` in `Variable`, and +`PyNode`, whose whole point is to let C++ call into Python). And `pyobj` +in `Node` to ensure uniqueness of the associated python wrapper (if it exists). diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/TraceTypeManual.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/TraceTypeManual.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6a0275e20b7e8acec8987b829e3b3e4d78730a02 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/TraceTypeManual.cpp @@ -0,0 +1,300 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace at; + +namespace torch::TraceType { + +namespace { + +Tensor& copy_(Tensor& self, const Tensor& src, bool non_blocking) { + jit::Value* output = nullptr; + if (torch::jit::tracer::isTracing()) { + const jit::tracer::TracingState& state = *jit::tracer::getTracingState(); + auto& graph = state.graph; + if (state.force_outplace && self.storage().use_count() <= 1) { + // if you have no views of self, then an in place copy is equivalent to + // making sure we expand src to the same size as self + jit::Node* node = graph->create(jit::aten::expand_as, /*num_outputs=*/1); + jit::tracer::addInputs(node, "src", src); + jit::tracer::addInputs(node, "self", self); + graph->insertNode(node); + output = node->output(); + } else { + output = graph->insert( + jit::aten::copy_, + {jit::tracer::getValueTrace(self), jit::tracer::getValueTrace(src)}); + jit::tracer::recordSourceLocation(output->node()); + } + jit::tracer::ensureUniqueIfOutOfPlaced( + "copy_ (possibly due to an assignment)", self); + } + + { + at::tracer::impl::NoTracerDispatchMode tracer_guard; + self.copy_(src, non_blocking); + } + + if (torch::jit::tracer::isTracing()) { + jit::tracer::setOutput(output, self); + } + return self; +} + +const Tensor& resize_( + const Tensor& self, + IntArrayRef size, + std::optional optional_memory_format) { + if (torch::jit::tracer::isTracing()) { + if (jit::tracer::ArgumentStash::hasIntArrayRef("size")) { + jit::tracer::ArgumentStash::popIntArrayRef("size"); + } + jit::tracer::warn("resize_", jit::tracer::WARN_RESIZE); + jit::tracer::delValueTrace(self); + } + + { + at::tracer::impl::NoTracerDispatchMode tracer_guard; + self.resize_(size, optional_memory_format); + } + return self; +} + +const Tensor& resize_as_( + const Tensor& self, + const Tensor& the_template, + std::optional optional_memory_format) { + if (torch::jit::tracer::isTracing()) { + jit::tracer::warn("resize_as_", jit::tracer::WARN_RESIZE); + jit::tracer::delValueTrace(self); + } + + { + at::tracer::impl::NoTracerDispatchMode tracer_guard; + self.resize_as_(the_template, optional_memory_format); + } + return self; +} + +Tensor detach(const Tensor& self) { + torch::jit::Node* node = nullptr; + if (jit::tracer::isTracing()) { + auto& graph = jit::tracer::getTracingState()->graph; + node = graph->create(jit::aten::detach, /*num_outputs=*/0); + jit::tracer::recordSourceLocation(node); + jit::tracer::addInputs(node, "self", self); + graph->insertNode(node); + } + + auto result = [&]() { + at::tracer::impl::NoTracerDispatchMode tracer_guard; + return self.detach(); + }(); + + if (jit::tracer::isTracing()) { + jit::tracer::addOutput(node, result); + } + return result; +} + +Tensor& detach_(Tensor& self) { + torch::jit::Node* node = nullptr; + if (jit::tracer::isTracing()) { + auto& graph = jit::tracer::getTracingState()->graph; + node = graph->create(jit::aten::detach, /*num_outputs=*/0); + jit::tracer::recordSourceLocation(node); + jit::tracer::addInputs(node, "self", self); + graph->insertNode(node); + jit::tracer::ensureUniqueIfOutOfPlaced("detach_", self); + } + + { + at::tracer::impl::NoTracerDispatchMode tracer_guard; + self.detach_(); + } + + if (jit::tracer::isTracing() && node) { + jit::tracer::addOutput(node, self); + } + return self; +} + +// Invariant: +// - Ops registered to DispatchKey::Tracer below must be included in +// `MANUAL_TRACER` in tools/autograd/gen_variable_type.py +TORCH_LIBRARY_IMPL(aten, Tracer, m) { + m.impl("resize_", resize_); + m.impl("resize_as_", resize_as_); + m.impl("detach", TORCH_FN(detach)); + m.impl("detach_", detach_); + m.impl("copy_", copy_); + + // Skip tracing for the following ops by registering fallthrough kernel + // explicitly. + m.impl("_backward", CppFunction::makeFallthrough()); + m.impl("set_data", CppFunction::makeFallthrough()); + m.impl("data", CppFunction::makeFallthrough()); + m.impl("is_leaf", CppFunction::makeFallthrough()); + m.impl("output_nr", CppFunction::makeFallthrough()); + m.impl("_version", CppFunction::makeFallthrough()); + m.impl("requires_grad_", CppFunction::makeFallthrough()); + m.impl("retain_grad", CppFunction::makeFallthrough()); + m.impl("_fw_primal", CppFunction::makeFallthrough()); + m.impl("_make_dual", CppFunction::makeFallthrough()); +} + +} // namespace + +} // namespace torch::TraceType + +namespace torch::jit { +static void general_trace_function( + const c10::OperatorHandle& op, + Stack* stack) { + const auto input_size = op.schema().arguments().size(); + const auto output_size = op.schema().returns().size(); + + Node* node = nullptr; + std::shared_ptr tracer_state; + + // trace the input before unwrapping, otherwise we may lose + // the input information + if (tracer::isTracing()) { + tracer_state = tracer::getTracingState(); + auto symbol = Symbol::fromQualString(op.schema().name()); + const auto& graph = tracer::getTracingState()->graph; + node = graph->create(symbol, 0); + tracer::recordSourceLocation(node); + const auto& args = op.schema().arguments(); + int i = 0; + for (auto iter = stack->end() - static_cast(input_size); + iter != stack->end(); + ++iter, ++i) { + // TODO we need to refactor graph APIs (e.g., addInputs) + // appropriately; after that, we can get rid of the giant if-else + // block we will clean this tech debt together in the following PRs + auto type = args[i].type(); + if (type->kind() == TypeKind::OptionalType) { + if (iter->isNone()) { + Value* none = graph->insertNode(graph->createNone())->output(); + node->addInput(none); + continue; + } else { + type = type->expectRef().getElementType(); + } + } + if (type->isSubtypeOf(*TensorType::get())) { + AT_ASSERT(iter->isTensor()); + tracer::addInputs(node, args[i].name().c_str(), iter->toTensor()); + } else if (type->kind() == TypeKind::FloatType) { + AT_ASSERT(iter->isDouble()); + tracer::addInputs(node, args[i].name().c_str(), iter->toDouble()); + } else if (type->kind() == TypeKind::IntType) { + AT_ASSERT(iter->isInt()); + tracer::addInputs(node, args[i].name().c_str(), iter->toInt()); + } else if (type->kind() == TypeKind::BoolType) { + AT_ASSERT(iter->isBool()); + tracer::addInputs(node, args[i].name().c_str(), iter->toBool()); + } else if (type->kind() == TypeKind::StringType) { + AT_ASSERT(iter->isString()); + tracer::addInputs(node, args[i].name().c_str(), iter->toStringView()); + } else if (type->kind() == TypeKind::NumberType) { + tracer::addInputs(node, args[i].name().c_str(), iter->toScalar()); + } else if (type->kind() == TypeKind::ListType) { + const auto& elem_type = type->expectRef().getElementType(); + if (elem_type->isSubtypeOf(*TensorType::get())) { + AT_ASSERT(iter->isTensorList()); + auto list = iter->toTensorVector(); + tracer::addInputs(node, args[i].name().c_str(), list); + } else if (auto class_type = elem_type->cast()) { + AT_ASSERT(iter->isList()); + auto list = iter->toList(); + std::vector> objects; + for (IValue iv : list) { + objects.emplace_back(std::move(iv).toObject()); + } + tracer::addInputs(node, args[i].name().c_str(), objects, class_type); + } else if (elem_type->kind() == TypeKind::FloatType) { + AT_ASSERT(iter->isDoubleList()); + // NB: now, tracer doesn't support tracing double list. We add + // special handling here, since in our case, we assume that all the + // doubles in the list are constants + auto value = iter->toDoubleVector(); + std::vector info(value.size()); + for (const auto value_index : c10::irange(value.size())) { + info[value_index] = graph->insertConstant(value[value_index]); + tracer::recordSourceLocation(info[value_index]->node()); + } + node->addInput( + graph->insertNode(graph->createList(FloatType::get(), info)) + ->output()); + } else if (elem_type->kind() == TypeKind::IntType) { + AT_ASSERT(iter->isIntList()); + tracer::addInputs( + node, + args[i].name().c_str(), + c10::IntArrayRef(iter->toIntVector())); + } else if (elem_type->kind() == TypeKind::BoolType) { + AT_ASSERT(iter->isBoolList()); + tracer::addInputs( + node, args[i].name().c_str(), iter->toBoolList().vec()); + } else { + TORCH_CHECK(false, "unsupported input list type: ", elem_type->str()); + } + } else if (iter->isObject()) { + tracer::addInputs(node, args[i].name().c_str(), iter->toObject()); + } else { + TORCH_CHECK(false, "unsupported input type: ", type->str()); + } + } + graph->insertNode(node); + + tracer::setTracingState(nullptr); + } + + op.callBoxed(stack); + + if (tracer_state) { + tracer::setTracingState(std::move(tracer_state)); + int i = 0; + for (auto iter = stack->end() - static_cast(output_size); + iter != stack->end(); + ++iter, ++i) { + const auto& type = op.schema().returns()[i].type(); + if (type->isSubtypeOf(*TensorType::get())) { + AT_ASSERT(iter->isTensor()); + tracer::addOutput(node, iter->toTensor()); + } else if (type->kind() == TypeKind::ListType) { + const auto& elem_type = type->expectRef().getElementType(); + if (elem_type->isSubtypeOf(*TensorType::get())) { + AT_ASSERT(iter->isTensorList()); + tracer::addOutput(node, iter->toTensorList()); + } else { + TORCH_CHECK( + false, "unsupported output list type: ", elem_type->str()); + } + } else if (type->kind() == TypeKind::ClassType) { + AT_ASSERT(iter->isObject()); + tracer::addOutput(node, iter->toObject()); + } else { + TORCH_CHECK( + false, + "unsupported output type: ", + type->str(), + ", from operator: ", + toString(op.operator_name())); + } + } + } +} +TORCH_LIBRARY_IMPL(_, Tracer, m) { + m.fallback(CppFunction::makeFromBoxedFunction<&general_trace_function>()); +} + +} // namespace torch::jit diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/VariableTypeManual.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/VariableTypeManual.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f4b3df73106151d7911a16e49ab38df56f5224e2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/VariableTypeManual.cpp @@ -0,0 +1,560 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace at; +using namespace torch::autograd::generated; +using torch::autograd::as_view; +using torch::autograd::CreationMeta; + +namespace torch { + +namespace autograd::VariableType { + +static std::vector allTypesForBackends( + at::ArrayRef backends) { + std::vector res; + res.reserve(backends.size()); + for (auto p : backends) { + for (const auto s : + c10::irange(static_cast(ScalarType::NumOptions))) { + auto& type = getDeprecatedTypeProperties( + static_cast(p), static_cast(s)); + res.emplace_back(&type); + } + } + return res; +} + +std::vector allCPUTypes() { + return allTypesForBackends({Backend::CPU, Backend::SparseCPU}); +} + +std::vector allCUDATypes() { + at::globalContext().lazyInitDevice(c10::DeviceType::CUDA); + return allTypesForBackends({Backend::CUDA, Backend::SparseCUDA}); +} + +std::vector allXPUTypes() { + return allTypesForBackends({Backend::XPU, Backend::SparseXPU}); +} + +std::vector allPrivateUser1Types() { + at::globalContext().lazyInitDevice(c10::DeviceType::PrivateUse1); + return allTypesForBackends( + {Backend::PrivateUse1, Backend::SparsePrivateUse1}); +} + +namespace { +const Variable& checked_cast_variable( + const Tensor& t, + const char* name, + int pos) { + if (!t.defined()) { + TORCH_CHECK( + false, + "Expected a proper Tensor but got None (or an undefined Tensor in C++) ", + "for argument #", + pos, + " '", + name, + "'"); + } + return t; +} + +Variable& checked_cast_variable(Tensor& t, const char* name, int pos) { + if (!t.defined()) { + TORCH_CHECK( + false, + "Expected a proper Tensor but got None (or an undefined Tensor in C++) ", + "for argument #", + pos, + " '", + name, + "'"); + } + return t; +} +} // namespace + +const Tensor& unpack(const Tensor& t, const char* name, int pos) { + return checked_cast_variable(t, name, pos); +} + +Tensor& unpack(Tensor& t, const char* name, int pos) { + return checked_cast_variable(t, name, pos); +} + +Tensor unpack_opt(const Tensor& t, const char* name, int pos) { + if (!t.defined()) { + return Tensor(); + } + return unpack(t, name, pos); +} + +std::vector unpack( + const at::ITensorListRef& tl, + const char* name, + int pos) { + std::vector ret; + ret.reserve(tl.size()); + for (const auto& t : tl) { + ret.push_back(t); + } + return ret; +} + +namespace { + +// Taken from codegened version +Tensor _fw_primal(c10::DispatchKeySet ks, const Tensor& self, int64_t level) { + auto& self_ = unpack(self, "self", 0); + c10::intrusive_ptr grad_fn; + if (compute_requires_grad(self)) { + grad_fn = c10::make_intrusive(); + grad_fn->set_next_edges(collect_next_edges(self)); + } + + auto result = ([&]() { + at::AutoDispatchBelowAutograd guard; + return at::redispatch::_fw_primal( + ks & c10::after_autograd_keyset, self_, level); + })(); + + if (grad_fn) { + set_history(flatten_tensor_args(result), grad_fn); + } + if (isFwGradDefined(self)) { + // Modified from original codegen + // We explicitly want to ignore the forward grad at the given level + TORCH_CHECK(level == 0, "Invalid level given to _fw_primal"); + // End modified from original codegen + } + return result; +} + +// NB: We need a manual variable type kernel so that set_fw_grad properly +// detects that _make_dual is not a forward-differentiable view +// +// This function can be used to create a dual Tensor that holds a tangent to +// compute forward mode gradients. Note that the dual Tensor's primal is a view +// of the given primal and the given tangent is used as-is. This function is +// backward differentiable. +Tensor _make_dual( + c10::DispatchKeySet ks, + const Tensor& primal, + const Tensor& tangent, + int64_t level) { + TORCH_CHECK( + !primal._fw_grad(level).defined(), + "Making a dual Tensor based on a Tensor that " + "already has a forward gradient at the same level ", + level, + " is not supported."); + auto& primal_ = unpack(primal, "primal", 0); + auto& tangent_ = unpack(tangent, "tangent", 0); + c10::intrusive_ptr grad_fn; + if (compute_requires_grad(primal_)) { + grad_fn = c10::make_intrusive(); + grad_fn->self_sym_sizes = primal_.sym_sizes().vec(); + grad_fn->set_next_edges(collect_next_edges(primal_)); + } + + auto result = ([&]() { + at::AutoDispatchBelowAutograd guard; + return at::redispatch::_make_dual( + ks & c10::after_autograd_keyset, primal_, tangent_, level); + })(); + + if (grad_fn) { + set_history(flatten_tensor_args(result), grad_fn); + } + + TORCH_CHECK(level == 0, "Invalid level given to _make_dual"); + result._set_fw_grad(tangent_, level, /* is_inplace_op */ false); + return result; +} + +// We don't have an outplace copy, so this can't be generated automatically +Tensor& copy_( + c10::DispatchKeySet ks, + Tensor& self, + const Tensor& src, + bool non_blocking) { + // TODO: once copy is exposed in Declarations.yaml we may be able to bind + // it automatically + auto& self_ = unpack(self, "self", 0); + auto& src_ = unpack(src, "src", 1); + c10::intrusive_ptr grad_fn; + auto requires_grad = compute_requires_grad(self, src); + requires_grad &= isDifferentiableType(self.scalar_type()); + check_inplace(self, requires_grad); + if (requires_grad) { + grad_fn = c10::make_intrusive(); + grad_fn->set_next_edges(collect_next_edges(self, src)); + grad_fn->src_options = src.options(); + } + { + at::AutoDispatchBelowAutograd mode; + at::redispatch::copy_( + ks & c10::after_autograd_keyset, self_, src_, non_blocking); + } + rebase_history(self, std::move(grad_fn)); + + if (isDifferentiableType(self.scalar_type()) && + (isFwGradDefined(self) || isFwGradDefined(src))) { + auto self_fw_grad = generated::details::toNonOptFwGrad(self); + auto src_fw_grad = generated::details::toNonOptFwGrad(src); + Tensor new_fw_grad; + if (self_fw_grad.defined()) { + if (src_fw_grad.defined()) { + new_fw_grad = self_fw_grad.copy_(src_fw_grad); + } else { + new_fw_grad = self_fw_grad.fill_(0); + } + } else { + if (!self.is_same_size(src_fw_grad)) { + new_fw_grad = src_fw_grad.broadcast_to(self.sizes()); + } else { + new_fw_grad = src_fw_grad.clone(); + } + } + self._set_fw_grad(new_fw_grad, /* level */ 0, /* is_inplace_op */ true); + } + + return self; +} + +const Tensor& resize_( + c10::DispatchKeySet ks, + const Tensor& self, + SymIntArrayRef size, + std::optional optional_memory_format) { + auto& self_ = unpack(self, "self", 0); + if (self.requires_grad()) { + TORCH_CHECK(false, "cannot resize variables that require grad"); + } + { + at::AutoDispatchBelowAutograd mode; + at::redispatch::resize__symint( + ks & c10::after_autograd_keyset, self_, size, optional_memory_format); + } + + if (self._fw_grad(/* level */ 0).defined()) { + TORCH_CHECK(false, "cannot resize variables that has a forward grad"); + } + + return self; +} + +const Tensor& resize_as_( + c10::DispatchKeySet ks, + const Tensor& self, + const Tensor& the_template, + std::optional optional_memory_format) { + auto& self_ = unpack(self, "self", 0); + auto& the_template_ = unpack(the_template, "the_template", 1); + if (self.requires_grad()) { + TORCH_CHECK(false, "cannot resize variables that require grad"); + } + { + at::AutoDispatchBelowAutograd mode; + at::redispatch::resize_as_( + ks & c10::after_autograd_keyset, + self_, + the_template_, + optional_memory_format); + } + + // Handle fw grad + if (self._fw_grad(/* level */ 0).defined()) { + TORCH_CHECK(false, "cannot resize variables that has a forward grad"); + } + + return self; +} + +Tensor detach(c10::DispatchKeySet ks, const Tensor& self) { + auto& self_ = unpack(self, "self", 0); + RECORD_FUNCTION("detach", std::vector({self})); + auto result = ([&]() { + at::AutoDispatchBelowAutograd guard; + return at::redispatch::detach(ks & c10::after_autograd_keyset, self_); + })(); + + // Detach the forward grads by not setting anything on the result + + return result; +} + +Tensor& detach_(c10::DispatchKeySet ks, Tensor& self) { + RECORD_FUNCTION("detach_", std::vector({self})); + if (self.is_view()) { + // See NOTE [ View + Inplace detection ] + TORCH_CHECK( + false, + "Can't detach views in-place. Use detach() instead. " + "If you are using DistributedDataParallel (DDP) for training, " + "and gradient_as_bucket_view is set as True, gradients are " + "views of DDP buckets, and hence detach_() cannot be called " + "on these gradients. To fix this error, please refer to the " + "Optimizer.zero_grad() function in torch/optim/optimizer.py " + "as the solution."); + } + // I think the choice here is conservative. In principle, doing + // an in-place detach should give us the ability to just clear + // the autograd meta. But this function ONLY resets requires_grad, + // grad_fn and output_nr; there's other metadata like debug name + // and hooks which aren't cleared. Is this function supposed to + // clear those too? I'm not too sure, so I'm leaving it be for now. + auto autograd_meta = impl::materialize_autograd_meta(self); + autograd_meta->set_requires_grad(false, self.unsafeGetTensorImpl()); + autograd_meta->grad_fn_.reset(); + autograd_meta->output_nr_ = 0; + autograd_meta->fw_grad_.reset(); + + return self; +} + +// Ops in the following registration list are registered as +// (1) CompositeImplicitAutograd kernels +// (2) Autograd kernels +// (3) CompositeExplicitAutograd kernels and additionally Autograd kernels +// The reason for (3) is that ops that also use dispatch (e.g. register +// CPU/CUDA/QuantizedCPU kernels) will skip picking up CompositeImplicitAutograd +// kernels for Autograd, so we register them to both CompositeExplicitAutograd +// and Autograd instead. See +// https://github.com/pytorch/pytorch/tree/master/aten/src/ATen/native#choosing-the-right-dispatch-keyword +// for more details. +// Invariant: +// - Ops registered to CompositeImplicitAutograd or CompositeExplicitAutograd +// below must match `MANUAL_BACKEND` set in tools/autograd/gen_variable_type.py. +// and they have manual_kernel_registration=True in native_functions.yaml. +// - Ops registered to DispatchKey::Autograd below must be included in +// `MANUAL_AUTOGRAD` in tools/autograd/gen_variable_type.py + +TORCH_LIBRARY_IMPL(aten, Autograd, m) { + m.impl( + "resize_", + torch::dispatch(DispatchKey::Autograd, TORCH_FN(VariableType::resize_))); + m.impl( + "resize_as_", + torch::dispatch( + DispatchKey::Autograd, TORCH_FN(VariableType::resize_as_))); + m.impl( + "detach", + torch::dispatch(DispatchKey::Autograd, TORCH_FN(VariableType::detach))); + m.impl( + "detach_", + torch::dispatch(DispatchKey::Autograd, TORCH_FN(VariableType::detach_))); + m.impl( + "copy_", + torch::dispatch(DispatchKey::Autograd, TORCH_FN(VariableType::copy_))); + m.impl( + "_fw_primal", + torch::dispatch( + DispatchKey::Autograd, TORCH_FN(VariableType::_fw_primal))); + m.impl( + "_make_dual", + torch::dispatch( + DispatchKey::Autograd, TORCH_FN(VariableType::_make_dual))); +} + +} // namespace +} // namespace autograd::VariableType + +namespace ADInplaceOrView { +#define CREATION_META_DEFINITION \ + InferenceMode::is_enabled() \ + ? CreationMeta::INFERENCE_MODE \ + : (at::GradMode::is_enabled() ? CreationMeta::DEFAULT \ + : CreationMeta::NO_GRAD_MODE) + +static Tensor& copy_( + c10::DispatchKeySet ks, + Tensor& self, + const Tensor& src, + bool non_blocking) { + { + at::AutoDispatchBelowADInplaceOrView guard; + at::redispatch::copy_( + ks & c10::after_ADInplaceOrView_keyset, self, src, non_blocking); + } + torch::autograd::increment_version(self); + return self; +} + +static const Tensor& resize_( + c10::DispatchKeySet ks, + const Tensor& self, + SymIntArrayRef size, + std::optional optional_memory_format) { + // Hold sizes to verify if we actually resize `self`. + // Explicitly copy data, since resizing can move original data + // and make references invalid. + auto org_size = self.sym_sizes().vec(); + { + at::AutoDispatchBelowADInplaceOrView guard; + at::redispatch::resize__symint( + ks & c10::after_ADInplaceOrView_keyset, + self, + size, + optional_memory_format); + } + // If `self` was resized, increment the version. + if (org_size != size) { + torch::autograd::increment_version(self); + } + return self; +} + +static const Tensor& resize_as_( + c10::DispatchKeySet ks, + const Tensor& self, + const Tensor& the_template, + std::optional optional_memory_format) { + // Hold sizes to verify if we actually resize `self`. + // Explicitly copy data, since resizing can move original data + // and make references invalid. + auto org_size = self.sym_sizes().vec(); + { + at::AutoDispatchBelowADInplaceOrView guard; + at::redispatch::resize_as_( + ks & c10::after_ADInplaceOrView_keyset, + self, + the_template, + optional_memory_format); + } + + // If `self` was resized, increment the version. + if (org_size != the_template.sym_sizes()) { + torch::autograd::increment_version(self); + } + return self; +} + +static Tensor detach(c10::DispatchKeySet ks, const Tensor& self) { + auto out = ([&]() { + at::AutoDispatchBelowADInplaceOrView guard; + return at::_ops::detach::redispatch( + ks & c10::after_ADInplaceOrView_keyset, self); + })(); + // NB: we can't make detach() a normal view operator because the + // codegen generates allow_tensor_metadata_change = True (and leaves + // is_fresh_tensor to the default setting of False) for them. In the + // future we should have an option for this in the codegen. + if (self.is_inference()) { + return out; + } + return ::torch::autograd::make_variable_non_differentiable_view( + self, + out, + /* allow_tensor_metadata_change */ false, + /* is_fresh_tensor */ true); +} + +static Tensor _fw_primal( + c10::DispatchKeySet ks, + const Tensor& self, + int64_t level) { + auto tmp = ([&]() { + at::AutoDispatchBelowADInplaceOrView guard; + return at::alias(self); + })(); + std::unique_ptr func(nullptr); + std::function rev_func = nullptr; + if (!self.unsafeGetTensorImpl()->support_as_strided()) { + func = std::make_unique(self.sym_sizes()); + rev_func = [=](const at::Tensor& input_view) -> at::Tensor { + TORCH_INTERNAL_ASSERT( + false, + "Reverse view_func for _fw_primal() is not currently supported"); + }; + } + auto result = as_view( + /* base */ self, + /* output */ tmp, + /* is_bw_differentiable */ true, + /* is_fw_differentiable */ false, + /* view_func */ std::move(func), + /* rev_view_func */ std::move(rev_func), + /* creation_meta */ CREATION_META_DEFINITION); + + return result; +} + +// NB: This does not redispatch any further +static Tensor _make_dual( + c10::DispatchKeySet ks, + const Tensor& primal, + const Tensor& tangent, + int64_t level) { + auto tmp = ([&]() { + at::AutoDispatchBelowADInplaceOrView guard; + return at::alias(primal); + })(); + std::unique_ptr func(nullptr); + std::function rev_func = nullptr; + if (!primal.unsafeGetTensorImpl()->support_as_strided()) { + func = std::make_unique(primal.sym_sizes()); + rev_func = [=](const at::Tensor& input_view) -> at::Tensor { + TORCH_INTERNAL_ASSERT( + false, + "Reverse view_func for _make_dual() is not currently supported"); + }; + } + auto result = as_view( + /* base */ primal, + /* output */ tmp, + /* is_bw_differentiable */ true, + /* is_fw_differentiable */ false, + /* view_func */ std::move(func), + /* rev_view_func */ std::move(rev_func), + /* creation_meta */ CREATION_META_DEFINITION); + + return result; +} + +namespace { +TORCH_LIBRARY_IMPL(aten, ADInplaceOrView, m) { + m.impl( + "copy_", + torch::dispatch( + DispatchKey::ADInplaceOrView, TORCH_FN(ADInplaceOrView::copy_))); + m.impl( + "detach", + torch::dispatch( + DispatchKey::ADInplaceOrView, TORCH_FN(ADInplaceOrView::detach))); + m.impl( + "resize_", + torch::dispatch( + DispatchKey::ADInplaceOrView, TORCH_FN(ADInplaceOrView::resize_))); + m.impl( + "resize_as_", + torch::dispatch( + DispatchKey::ADInplaceOrView, TORCH_FN(ADInplaceOrView::resize_as_))); + m.impl( + "_fw_primal", + torch::dispatch( + DispatchKey::ADInplaceOrView, TORCH_FN(ADInplaceOrView::_fw_primal))); + m.impl( + "_make_dual", + torch::dispatch( + DispatchKey::ADInplaceOrView, TORCH_FN(ADInplaceOrView::_make_dual))); +} +} // namespace +} // namespace ADInplaceOrView +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/VariableTypeUtils.h b/western_ai/usa/pytorch/torch/csrc/autograd/VariableTypeUtils.h new file mode 100644 index 0000000000000000000000000000000000000000..20534a1101c1fb63d0ff1723d22359196f9607c5 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/VariableTypeUtils.h @@ -0,0 +1,443 @@ +#pragma once + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +#ifdef Type +#undef Type +#endif +#endif + +namespace torch::autograd { +enum class can_mutate_inplace_result { + success, + non_default_backward_view, + view_of_leaf, + is_leaf, +}; + +// The requires_grad argument is used to know if the inplace operation needs +// gradient to be setup for it. +// In particular, we can have tensor.requires_grad() != requires_grad when +// writing a Tensor that requires gradients inplace into a Tensor that does not +// require gradients: a = torch.rand(2) b = torch.rand(2, requires_grad=True) +// a.copy_(b) +inline can_mutate_inplace_result can_mutate_inplace( + const at::Tensor& tensor, + bool requires_grad) { + if (!requires_grad || !GradMode::is_enabled()) { + return can_mutate_inplace_result::success; + } + auto diff_view_meta = impl::get_view_autograd_meta(tensor); + if (diff_view_meta && diff_view_meta->has_bw_view()) { + if (diff_view_meta->get_creation_meta() != CreationMeta::DEFAULT) { + return can_mutate_inplace_result::non_default_backward_view; + } + if (tensor.requires_grad() && tensor._base().is_leaf()) { + return can_mutate_inplace_result::view_of_leaf; + } + } + if (tensor.requires_grad() && tensor.is_leaf()) { + return can_mutate_inplace_result::is_leaf; + } + return can_mutate_inplace_result::success; +} + +inline void check_inplace(const at::Tensor& tensor, bool requires_grad) { + switch (can_mutate_inplace(tensor, requires_grad)) { + case can_mutate_inplace_result::success: + return; + case can_mutate_inplace_result::non_default_backward_view: { + return handle_view_on_rebase(impl::get_view_autograd_meta(tensor)); + } + case can_mutate_inplace_result::view_of_leaf: + TORCH_CHECK( + false, + "a view of a leaf Variable that requires grad is being used in an in-place operation."); + break; + + case can_mutate_inplace_result::is_leaf: + TORCH_CHECK( + false, + "a leaf Variable that requires grad is being used in an in-place operation."); + break; + } + TORCH_INTERNAL_ASSERT(false); +} + +inline void check_inplace(at::ITensorListRef tensors, bool requires_grad) { + for (const auto& tensor : tensors) { + check_inplace(tensor, requires_grad); + } +} + +inline void throw_error_out_requires_grad(const char* name) { + TORCH_CHECK( + false, + name, + "(): functions with out=... arguments don't support automatic differentiation, " + "but one of the arguments requires grad."); +} + +inline void throw_error_for_complex_autograd( + const at::Tensor& tensor, + const char* name) { + if (tensor.requires_grad()) { + TORCH_CHECK( + !tensor.is_complex(), + name, + " does not support automatic differentiation for outputs with complex dtype."); + } +} + +inline void throw_error_if_base_and_tensor_are_same( + const at::Tensor& base, + const at::Tensor& tensor) { + TORCH_CHECK( + base.unsafeGetTensorImpl() != tensor.unsafeGetTensorImpl(), + "View operation returned a tensor that is the same as the input base tensor. This " + "is no longer allowed; you must explicitly create a new tensor (e.g., using .detach()). " + "As a user, you could have made a mistake implementing __torch_dispatch__ or a Python " + "operator decomposition or meta registration; if that's not the case, please " + "report a bug to PyTorch or the backend you are using."); +} + +inline void throw_error_for_complex_autograd( + at::ITensorListRef tensorlist, + const char* name) { + for (const auto& tensor : tensorlist) { + throw_error_for_complex_autograd(tensor, name); + } +} + +// TODO: Blegh, bare references + +inline void rebase_history( + const Variable& var, + c10::intrusive_ptr grad_fn) { + if (grad_fn && var.defined()) { + grad_fn->add_input_metadata(var); + impl::rebase_history(var, {std::move(grad_fn), 0}); + } +} + +inline void rebase_history( + const std::vector& vars, + const c10::intrusive_ptr& grad_fn) { + if (grad_fn) { + for (auto& var : vars) { + if (var.defined()) { + auto output_nr = grad_fn->add_input_metadata(var); + impl::rebase_history(var, {grad_fn, output_nr}); + } else { + grad_fn->add_input_metadata(Node::undefined_input()); + } + } + } +} + +inline void increment_version(const at::Tensor& t) { + impl::bump_version(t); +} + +struct Flatten : IterArgs { + Flatten(variable_list& out) : out(out) {} + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + variable_list& out; + void operator()(const at::Tensor& x) { + out.emplace_back(x); + } + void operator()(const std::optional& x) { + if (x.has_value()) + out.emplace_back(x.value()); + } + void operator()(at::ArrayRef xs) { + out.insert(out.end(), xs.begin(), xs.end()); + } +}; + +template +inline variable_list flatten_tensor_args(Args&&... args) { + variable_list out; + out.reserve(count_tensors(std::forward(args)...)); + Flatten(out).apply(std::forward(args)...); + return out; // RVO +} + +// See NOTE [ Autograd View Variables ] for details. +inline at::Tensor as_view( + const at::Tensor& base, + const at::Tensor& tensor, + bool is_bw_differentiable, + bool is_fw_differentiable, + std::unique_ptr view_func = nullptr, + std::function rev_view_func = nullptr, + CreationMeta creation_meta = CreationMeta::DEFAULT, + bool allow_tensor_metadata_change = true) { + // Note [View of inference tensor] + // For inference tensor this code can only be hit outside InferenceMode + // since ADInplaceOrView is in the default_included_set. + // If Inplace and View were separate dispatch keys we can just put Inplace + // in the default_included_set, so that view ops on inference tensor doesn't + // have to go through as_view even outside InferenceMode. + if (base.is_inference()) + return tensor; + + auto diff_view_meta = torch::autograd::impl::get_view_autograd_meta(base); + + // To speed up the most common case, we specially handle when both the forward + // and backward view infos are the same, and so a single shared ViewInfo can + // be used for both of them. + if ((!diff_view_meta || diff_view_meta->shared_view_info()) && + is_bw_differentiable && is_fw_differentiable) { + throw_error_if_base_and_tensor_are_same(base, tensor); + if (diff_view_meta) { + creation_meta = propagate_creation_meta( + diff_view_meta->get_creation_meta(), creation_meta); + return make_variable_differentiable_view( + tensor, + diff_view_meta->get_backward_view().chain( + base, tensor, std::move(view_func), std::move(rev_view_func)), + std::nullopt, + /*shared_view_info*/ true, + creation_meta, + allow_tensor_metadata_change); + } else { + return make_variable_differentiable_view( + tensor, + ViewInfo(base, std::move(view_func), std::move(rev_view_func)), + std::nullopt, + /*shared_view_info*/ true, + creation_meta, + allow_tensor_metadata_change); + } + } + + // If they cannot be shared, create the required view infos + std::optional new_bw_info; + std::optional new_fw_info; + + if (is_bw_differentiable) { + auto bw_view_func = view_func ? view_func->clone_and_set() : nullptr; + if (diff_view_meta && diff_view_meta->has_bw_view()) { + const auto& base_bw_info = diff_view_meta->get_backward_view(); + new_bw_info = base_bw_info.chain( + base, tensor, std::move(bw_view_func), rev_view_func); + } else { + new_bw_info = ViewInfo(base, std::move(bw_view_func), rev_view_func); + } + } else { + TORCH_CHECK( + creation_meta == CreationMeta::DEFAULT, + "Non-backward differentiable views must have creation_meta=CreationMeta::DEFAULT"); + } + + if (is_fw_differentiable) { + // Check if base is a forward differentiable view + if (diff_view_meta && diff_view_meta->has_fw_view()) { + const auto& base_fw_info = diff_view_meta->get_forward_view(); + new_fw_info = base_fw_info.chain( + base, tensor, std::move(view_func), std::move(rev_view_func)); + } else { + new_fw_info = + ViewInfo(base, std::move(view_func), std::move(rev_view_func)); + } + } + + if (is_fw_differentiable || is_bw_differentiable) { + if (diff_view_meta && diff_view_meta->has_bw_view()) { + creation_meta = propagate_creation_meta( + diff_view_meta->get_creation_meta(), creation_meta); + } + throw_error_if_base_and_tensor_are_same(base, tensor); + return make_variable_differentiable_view( + tensor, + std::move(new_bw_info), + std::move(new_fw_info), + /*shared_view_info*/ false, + creation_meta, + allow_tensor_metadata_change); + } else { + return make_variable_non_differentiable_view( + base, tensor, allow_tensor_metadata_change); + } +} + +inline void check_no_requires_grad( + const at::Tensor& tensor, + const char* name, + const char* fn_name = "", + bool check_grad_mode = true) { + TORCH_CHECK( + !(tensor.defined() && tensor.requires_grad()) || + !(check_grad_mode && GradMode::is_enabled()), + "The function '", + fn_name, + "' is not differentiable with respect to argument '", + name, + "'. This input cannot have requires_grad True."); +} + +inline void check_no_requires_grad( + const std::optional& tensor, + const char* name, + const char* fn_name = "") { + if (tensor.has_value()) { + check_no_requires_grad(*tensor, name, fn_name); + } +} + +inline void check_no_requires_grad( + at::ITensorListRef tensors, + const char* name, + const char* fn_name = "") { + // GradMode check is expensive, so check it only once for TensorLists + if (!GradMode::is_enabled()) { + return; + } + for (auto& tensor : tensors) { + check_no_requires_grad(tensor, name, fn_name, /*check_grad_mode*/ false); + } +} + +inline void check_no_requires_grad( + const c10::List>& tensors, + const char* name, + const char* fn_name = "") { + // GradMode check is expensive, so check it only once for TensorLists + if (!GradMode::is_enabled()) { + return; + } + for (std::optional tensor : tensors) { + if (tensor.has_value()) { + check_no_requires_grad(*tensor, name, fn_name, /*check_grad_mode*/ false); + } + } +} + +// Assumed that saved tensor lists are never inplace outputs +inline std::vector make_saved_variable_list( + at::ITensorListRef tensors, + const bool is_output = false) { + return fmap(tensors, [&is_output](const at::Tensor& tensor) -> SavedVariable { + return SavedVariable{tensor, is_output /* is output */}; + }); +} + +// Assumed that saved tensor lists are never inplace outputs +inline std::vector make_saved_variable_list( + const c10::List>& tensors, + const bool is_output = false) { + return fmap( + tensors, + [&is_output](const std::optional& tensor) -> SavedVariable { + if (tensor.has_value()) { + return SavedVariable{*tensor, is_output /* is output */}; + } else { + return SavedVariable{at::Tensor(), is_output /* is output */}; + } + }); +} + +inline std::vector> to_args_sizes( + at::ITensorListRef tensors) { + std::vector> args_sizes(tensors.size()); + size_t i = 0; + for (const auto& t : tensors) { + args_sizes[i++] = t.sizes().vec(); + } + return args_sizes; +} + +inline std::vector> to_args_sizes_symint( + at::ITensorListRef tensors) { + std::vector> args_sizes(tensors.size()); + size_t i = 0; + for (const auto& t : tensors) { + args_sizes[i++] = t.sym_sizes().vec(); + } + return args_sizes; +} + +inline std::vector to_args_scalartypes( + at::ITensorListRef tensors) { + std::vector args_scalartypes(tensors.size()); + size_t i = 0; + for (const auto& t : tensors) { + args_scalartypes[i++] = t.scalar_type(); + } + return args_scalartypes; +} + +namespace impl { + +namespace { + +// If run_jit_decomposition were not a member function, we would be able +// to pass this as a template parameter to c10::Boxedkernel::makeFromFunction. +// However, member functions cannot be passed this way - instead we wrap our +// call in this functor so it can be passed to c10::BoxedKernel::makeFromFunctor +class WrapperFunctor final : public c10::OperatorKernel { + public: + WrapperFunctor(JitDecompInterface* impl) : impl_(impl) {} + + void operator()( + const c10::OperatorHandle& op, + c10::DispatchKeySet ks, + torch::jit::Stack* stack) { + impl_->run_jit_decomposition(op, stack); + } + JitDecompInterface* impl_; +}; + +} // namespace + +template +Return run_jit_decomposition_with_args_for_jvp( + std::string_view name, + const c10::OperatorHandle& opHandle, + c10::DispatchKeySet dispatchKeySet, + Args&&... args) { + // see NOTE: [Jit Decomposition Interface] + JitDecompInterface* impl = getJitDecompImpl(); + + TORCH_CHECK_NOT_IMPLEMENTED( + impl && impl->has_jit_decomposition(opHandle.schema()), + "Trying to use forward AD with ", + name, + " that does not support it because it has not been implemented yet.\nPlease file an issue " + "to PyTorch at https://github.com/pytorch/pytorch/issues/new?template=feature-request.yml " + "so that we can prioritize its implementation or submit a PR adding the implementation to " + "derivatives.yaml"); + + return c10::KernelFunction::makeFromBoxedKernel( + c10::BoxedKernel::makeFromFunctor( + std::make_unique(impl))) + .call( + opHandle, dispatchKeySet, std::forward(args)...); +} + +} // namespace impl + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/anomaly_mode.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/anomaly_mode.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0033635b870a3cb8896ec941110a41fd5ebb8110 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/anomaly_mode.cpp @@ -0,0 +1,78 @@ +#include +#include +#include +#include +#include + +namespace torch::autograd { + +bool AnomalyMode::_enabled = false; +bool AnomalyMode::_check_nan = true; + +namespace { +std::mutex& get_anomaly_guard_lock() { + static std::mutex anomaly_guard_lock{}; + return anomaly_guard_lock; +} + +uint32_t& get_anomaly_counter() { + static uint32_t counter = 0; + return counter; +} +} // namespace + +DetectAnomalyGuard::DetectAnomalyGuard(bool check_nan) { + TORCH_WARN_ONCE( + "This mode should be enabled only for debugging as the different tests will slow down your program execution."); + std::lock_guard lock(get_anomaly_guard_lock()); + uint32_t& counter = get_anomaly_counter(); + counter++; + this->prev_check_nan_ = AnomalyMode::should_check_nan(); + AnomalyMode::set_enabled(true, check_nan); +} + +DetectAnomalyGuard::~DetectAnomalyGuard() { + std::lock_guard lock(get_anomaly_guard_lock()); + uint32_t& counter = get_anomaly_counter(); + counter--; + AnomalyMode::set_enabled(counter > 0, this->prev_check_nan_); +} + +AnomalyMetadata::~AnomalyMetadata() = default; + +void AnomalyMetadata::store_stack() { + traceback_ = c10::get_backtrace(/* frames_to_skip */ 1); +} + +void AnomalyMetadata::print_stack(const std::string& current_node_name) { + TORCH_WARN( + "Error detected in ", + current_node_name, + ". ", + "Traceback of forward call that caused the error:\n", + traceback_); + + auto& cur_parent = parent_; + // if there is no "parent_" in metadata, then it means this metadata's node + // is the root and stop printing the traceback + while (cur_parent) { + auto parent_metadata = cur_parent->metadata(); + TORCH_WARN( + "\n\n", + "Previous calculation was induced by ", + cur_parent->name(), + ". " + "Traceback of forward call that induced the previous calculation:\n", + parent_metadata->traceback_); + // get the parent of this node, if this node is a root, pyparent is simply + // null + cur_parent = parent_metadata->parent_; + } +} + +void AnomalyMetadata::assign_parent( + const c10::intrusive_ptr& parent_node) { + parent_ = parent_node; +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/anomaly_mode.h b/western_ai/usa/pytorch/torch/csrc/autograd/anomaly_mode.h new file mode 100644 index 0000000000000000000000000000000000000000..81a494e627beba994e8a9bd9c64698993d564ef0 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/anomaly_mode.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include + +namespace torch::autograd { + +// forward declaration of Node from function.h +struct Node; + +struct TORCH_API AnomalyMode { + static bool is_enabled() { + return _enabled; + } + static bool should_check_nan() { + return _check_nan; + } + static void set_enabled(bool enabled, bool check_nan = true) { + _enabled = enabled; + _check_nan = check_nan; + } + + private: + static bool _enabled; + static bool _check_nan; +}; + +/// A RAII guard that enables Anomaly Detection Mode. +/// +/// Anomaly detection mode is useful for debugging problems happening +/// in the backward, such as unexpectedly modified tensors or NaNs +/// occurring in the backward. +/// +/// The enabling of anomaly mode is global - as soon as there is one +/// such guard, it is enabled for all computation and threads. It also +/// comes with a significant performance penalty. +/// +/// Example: +/// @code +/// auto x = torch::tensor({1.}, torch::requires_grad()); +/// { +/// torch::autograd::DetectAnomalyGuard detect_anomaly; +/// auto x = torch::tensor({5.0}, torch::requires_grad()); +/// auto y = x * x; +/// auto z = y * y; +/// y += 1; +/// z.backward(); +/// } +/// @endcode +class TORCH_API DetectAnomalyGuard { + public: + DetectAnomalyGuard(bool check_nan = true); + ~DetectAnomalyGuard(); + + private: + bool prev_check_nan_; +}; + +struct TORCH_API AnomalyMetadata { + virtual ~AnomalyMetadata(); + virtual void store_stack(); + virtual void print_stack(const std::string& current_node_name); + virtual void assign_parent(const c10::intrusive_ptr& parent_node); + + private: + std::string traceback_; + c10::intrusive_ptr parent_; +}; + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/autograd.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/autograd.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f97f4e09ac5a856e3b6fb2fdc713e0a4b9d1396f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/autograd.cpp @@ -0,0 +1,217 @@ +#include +#include + +#ifndef AT_PER_OPERATOR_HEADERS +#include +#else +#include +#endif + +#include +#include +#include +#include + +#include + +namespace torch::autograd { + +// NB: This code duplicates existing logic at torch/autograd/__init__.py and +// torch._C._EngineBase.run_backward in torch/csrc/autograd/python_engine.cpp +// This is a purely C++ API for Autograd without any dependencies on python +// it can be exposed in PyTorch C++ API and TorchScript. We will need to +// maintain the logic equality of this file and the python file together if one +// changes. +// TODO: Make the Python API above to just call this C++ API. +static variable_list _make_grads( + const variable_list& outputs, + const variable_list& grad_outputs) { + size_t num_tensors = outputs.size(); + size_t num_gradients = grad_outputs.size(); + variable_list new_grads; + new_grads.reserve(num_tensors); + if (grad_outputs.empty()) { + for (const Variable& output : outputs) { + if (output.requires_grad()) { + TORCH_CHECK( + output.numel() == 1, + "grad can be implicitly created only for scalar outputs"); + TORCH_CHECK( + c10::isFloatingType(output.scalar_type()), + "grad can be computed only for real scalar outputs but got ", + output.scalar_type()); + new_grads.emplace_back( + at::ones_like(output, LEGACY_CONTIGUOUS_MEMORY_FORMAT)); + } + } + } else { + TORCH_CHECK( + num_tensors == num_gradients, + "got ", + num_tensors, + " tensors and ", + num_gradients, + " gradients"); + for (const auto i : c10::irange(outputs.size())) { + const Variable& output = outputs[i]; + const Variable& grad_output = grad_outputs[i]; + if (!grad_output.defined()) { + if (output.requires_grad()) { + TORCH_CHECK( + output.numel() == 1, + "grad can be implicitly created only for scalar outputs"); + TORCH_CHECK( + c10::isFloatingType(output.scalar_type()), + "grad can be computed only for real scalar outputs but got ", + output.scalar_type()); + new_grads.emplace_back( + at::ones_like(output, LEGACY_CONTIGUOUS_MEMORY_FORMAT)); + } + } else { + TORCH_CHECK( + grad_output.is_complex() == output.is_complex(), + "For complex Tensors, both grad_output and output are required ", + "to have the same dtype. Mismatch in dtype: grad_output[", + grad_output, + "] has a dtype of ", + grad_output.scalar_type(), + " and output[", + output, + "] has a dtype of ", + output.scalar_type(), + "."); + // grad output is defined, just append to the new_grads + new_grads.emplace_back(grad_output); + } + } + } + return new_grads; +} +static variable_list run_backward( + const variable_list& outputs, + const variable_list& grad_outputs, + bool keep_graph, + bool create_graph, + const variable_list& inputs, + bool allow_unused, + bool accumulate_grad) { + size_t num_tensors = outputs.size(); + edge_list roots; + roots.reserve(num_tensors); + for (const auto i : c10::irange(num_tensors)) { + const Variable& output = outputs[i]; + auto gradient_edge = impl::gradient_edge(output); + TORCH_CHECK( + gradient_edge.function, + "element ", + i, + " of tensors does not require grad and does not have a grad_fn"); + roots.push_back(std::move(gradient_edge)); + } + + edge_list output_edges; + if (!inputs.empty()) { + size_t num_inputs = inputs.size(); + output_edges.reserve(num_inputs); + for (const auto i : c10::irange(num_inputs)) { + const Variable& input = inputs[i]; + const auto output_nr = input.output_nr(); + auto grad_fn = input.grad_fn(); + if (!grad_fn) { + grad_fn = impl::try_get_grad_accumulator(input); + } + if (accumulate_grad) { + input.retain_grad(); + } + TORCH_CHECK( + input.requires_grad(), + "element ", + i, + " of the input tensors does not require grad"); + if (!grad_fn) { + // See NOTE [ Autograd Unreachable Input ] for details + output_edges.emplace_back(c10::make_intrusive(), 0); + } else { + output_edges.emplace_back(grad_fn, output_nr); + } + } + } + + variable_list grad_inputs = Engine::get_default_engine().execute( + roots, + grad_outputs, + keep_graph, + create_graph, + accumulate_grad, + output_edges); + // check if grad_inputs contains None or not base on the allow_unused flag + if (!inputs.empty() && !allow_unused) { + size_t num_inputs = inputs.size(); + for (const auto i : c10::irange(num_inputs)) { + TORCH_CHECK( + grad_inputs[i].defined(), + "element ", + i, + "of the " + "differentiated Tensors appears to not have been used " + "in the graph. Set allow_unused=True if this is the " + "desired behavior."); + } + } + return grad_inputs; +} + +void backward( + const variable_list& tensors, + const variable_list& grad_tensors, + std::optional retain_graph, + bool create_graph, + const variable_list& inputs) { + variable_list gradients = _make_grads(tensors, grad_tensors); + if (!retain_graph) { + retain_graph = create_graph; + } + run_backward( + tensors, + gradients, + retain_graph.value(), + create_graph, + inputs, + /*allow_unused=*/true, + /*accumulate_grad=*/true); +} + +variable_list grad( + const variable_list& outputs, + const variable_list& inputs, + const variable_list& grad_outputs, + std::optional retain_graph, + bool create_graph, + bool allow_unused) { + variable_list gradients = _make_grads(outputs, grad_outputs); + if (!retain_graph) { + retain_graph = create_graph; + } + return run_backward( + outputs, + gradients, + retain_graph.value(), + create_graph, + inputs, + allow_unused, + /*accumulate_grad=*/false); +} + +namespace forward_ad { + +uint64_t enter_dual_level() { + return ForwardADLevel::get_next_idx(); +} + +void exit_dual_level(uint64_t level) { + ForwardADLevel::release_idx(level); +} + +} // namespace forward_ad + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/autograd.h b/western_ai/usa/pytorch/torch/csrc/autograd/autograd.h new file mode 100644 index 0000000000000000000000000000000000000000..bf6785e66f3535a84ffe46d6d443bb0351d7e9f2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/autograd.h @@ -0,0 +1,104 @@ +#pragma once + +#include + +namespace torch::autograd { + +/// Computes the sum of gradients of given tensors with respect to graph leaves. +/// +/// The graph is differentiated using the chain rule. If any of ``tensors`` +/// are non-scalar (i.e. their data has more than one element) and require +/// gradient, then the Jacobian-vector product would be computed, in this case +/// the function additionally requires specifying `grad_tensors`. It should be a +/// sequence of matching length, that contains the "vector" in the +/// Jacobian-vector product, usually the gradient of the differentiated function +/// w.r.t. corresponding tensors +/// (`torch::Tensor()` is an acceptable value for all tensors that don't need +/// gradient tensors). +/// +/// This function accumulates gradients in the leaves - you might need to zero +/// them before calling it. +/// +/// \param tensors Tensors of which the derivative will be computed. +/// \param grad_tensors The "vector" in the Jacobian-vector product, usually +/// gradients +/// w.r.t. each element of corresponding tensors. `torch::Tensor()` values +/// can be specified for scalar Tensors or ones that don't require grad. If +/// a `torch::Tensor()` value would be acceptable for all grad_tensors, then +/// this argument is optional. +/// \param retain_graph If `false`, the graph used to compute the grad will be +/// freed. +/// Note that in nearly all cases setting this option to `true` is not +/// needed and often can be worked around in a much more efficient way. +/// Defaults to the value of `create_graph`. +/// \param create_graph If `true`, graph of the derivative will be constructed, +/// allowing +/// to compute higher order derivative products. Defaults to `false`. +/// \param inputs Inputs w.r.t. which the gradient will be accumulated into +/// `at::Tensor::grad`. All other Tensors will be ignored. If not provided, +/// the gradient is accumulated into all the leaf Tensors that were used to +/// compute param `tensors`. +// When inputs are provided and a given input is not a leaf, +// the current implementation will call its grad_fn (even though it is not +// strictly needed to get these gradients). It is an implementation detail +// on which the user should not rely. See +// https://github.com/pytorch/pytorch/pull/60521#issuecomment-867061780 for +// more details. +TORCH_API void backward( + const variable_list& tensors, + const variable_list& grad_tensors = {}, + std::optional retain_graph = std::nullopt, + bool create_graph = false, + const variable_list& inputs = {}); + +/// Computes and returns the sum of gradients of outputs with respect to the +/// inputs. +/// +/// ``grad_outputs`` should be a sequence of length matching ``output`` +/// containing the "vector" in Jacobian-vector product, usually the pre-computed +/// gradients w.r.t. each of the outputs. If an output doesn't require_grad, +/// then the gradient can be ``torch::Tensor()``). +/// +/// \param outputs outputs of the differentiated function. +/// \param inputs Inputs w.r.t. which the gradient will be +/// returned (and not accumulated into ``at::Tensor::grad``). +/// \param grad_outputs The "vector" in the Jacobian-vector product. +/// Usually gradients w.r.t. each output. `torch::Tensor()` values can be +/// specified for scalar Tensors or ones that don't require grad. If a +/// `torch::Tensor()` value would be acceptable for all grad_tensors, then +/// this argument is optional. Default: `{}`. +/// \param retain_graph If ``false``, the graph used to compute the grad +/// will be freed. Note that in nearly all cases setting this option to +/// ``true`` is not needed and often can be worked around in a much more +/// efficient way. Defaults to the value of ``create_graph``. +/// \param create_graph If ``true``, graph of the derivative will +/// be constructed, allowing to compute higher order derivative products. +/// Default: ``false``. +/// \param allow_unused If ``false``, specifying inputs that were not +/// used when computing outputs (and therefore their grad is always zero) +/// is an error. Defaults to ``false``. +TORCH_API variable_list grad( + const variable_list& outputs, + const variable_list& inputs, + const variable_list& grad_outputs = {}, + std::optional retain_graph = std::nullopt, + bool create_graph = false, + bool allow_unused = false); + +namespace forward_ad { + +/// Creates a new dual level and returns its index. This level index should then +/// be used to call into the other functions below. This API supports entering a +/// new level before the previous one is exited. We call them nested forward AD +/// levels. These can be used to compute higher order derivatives. +TORCH_API uint64_t enter_dual_level(); + +/// Exits the given level. This will clear up all the gradients from this level +/// and all dual Tensors that had gradients for this level will become regular +/// Tensors again. This function can only be used to exit the innermost nesting +/// level and so exiting must happen in reverse order compared to the entering +/// that was done with the function above. +TORCH_API void exit_dual_level(uint64_t level); + +} // namespace forward_ad +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/autograd_meta.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/autograd_meta.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ed39d19d3e7864ea741538d66e9c36699ea19e34 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/autograd_meta.cpp @@ -0,0 +1,319 @@ +#define TORCH_ASSERT_ONLY_METHOD_OPERATORS +#include +#include +#include +#include + +#ifndef AT_PER_OPERATOR_HEADERS +#include +#else +#include +#include +#include +#endif + +namespace torch::autograd { + +using at::Tensor; + +// [Forward Grad View/inplace] +// It is important to us to allow view and inplace to work with dual Tensors. +// These operations should either compute the right gradient or raise a +// user-friendly error. + +// The basic case where all Tensors are dual Tensors is as follows: +// # Have: +// # foo is a dual Tensor that is not a view +// # bar is a dual Tensor of appropriate size (depending on cases) that is +// not a view +// +// # Case 1: no view +// foo.copy_(bar) +// +// # Case 2: with view, propagate from view to base +// view = foo[0] +// view.copy_(bar) +// +// # Case 3: with view, propagate from base to view +// view = foo[0] +// foo.copy_(bar) +// +// # In both cases, the forward grad of foo must be properly updated. +// # In the second and third cases, the forward grad of view must match +// # the one of foo for the subset they have in common. +// +// All these cases can be handled by the following layout constraint on the +// forward grad: +// - A Tensor and its forward grad (for all levels) must have the same +// metadata (size, stride +// conj/neg bit and storage offset). Storage offset must be in this metadata +// because of as_strided. conj/neg bit must be part of this metadata because +// of ops like `real`. +// - View operations must create a forward grad that is a view of the base's +// forward grad. +// - Inplace operations must modify the input's forward grad inplace. +// +// This layout constraint is ensured in the `set_fw_grad` function below + +// More complex cases arise when non-dual Tensor interact with dual Tensors. +// The two most important cases are: +// +// # Have: +// # foo is a regular Tensor that is not a view +// # bar is a dual Tensor of appropriate size (depending on cases) that is +// not a view +// +// # Case 4: Changes on the view must propagate to its base +// view = foo[0] +// # view is still a regular Tensor here +// view.copy_(bar) +// # Now both view and foo are dual Tensor with appropriate forward grad +// +// # Case 5: Changes on the base must propagate on all its views +// view = foo[0] +// # view is still a regular Tensor here +// base.copy_(bar) +// # Now both view and foo are dual Tensor with appropriate forward grad +// +// # NB there is a case 6 involving changes on a view propagating to other +// views # but it is fully described by the two others and is skipped in +// this discussion. +// +// Case 4 is handled by set_fw_grad by properly setting the forward grad of the +// base if needed. Case 5 is handled in fw_grad by reading the forward grad from +// the base if needed. + +namespace utils { + +// Enforcing that the metadata between the primal and tangent are same has two +// goals: +// - When properties of the primal are checked in composite op's to determine +// control flow, the code path decided upon is also reasonable for the tangent +// - Make sure that when the same as_strided is applied to both primal and +// and tangent, it behaves similarly. +// +// We do that by checking: +// 1) the storages have same properties: size and conj/neg-ness +// 2) the same indices refer to the same elements in storage +// (we are more strict than necessary here to satisfy the goal 1) +bool has_same_meta(const Variable& base, const Variable& other) { + if (!base.defined() || !other.defined()) { + return false; + } + // 1) The storages have the same properties + if (!at::_has_same_storage_numel(base, other)) { + return false; + } + if (base.is_conj() != other.is_conj() || base.is_neg() != other.is_neg()) { + return false; + } + + // Technically dim and size belong as part of (2), so we shouldn't really care + // if a zero-numel tensor violates these. But since these properties + // (unlike offset and strides) often determine control flow in composite ops + // it is useful to enforce that they match for primal and tangent here so + // nothing funny happens later (See goal 1). + if (base.dim() != other.dim()) { + return false; + } + for (const auto i : c10::irange(base.dim())) { + if (base.sym_sizes()[i] != other.sym_sizes()[i]) { + return false; + } + } + + // The check below will always be vacuously true for 0-element tensors + if (base.sym_numel() == 0 && other.sym_numel() == 0) { + return true; + } + + // 2) The same indices refer to the same elements in storage + if (base.sym_storage_offset() != other.sym_storage_offset()) { + return false; + } + + for (const auto i : c10::irange(base.dim())) { + if (base.sym_strides()[i] != other.sym_strides()[i] && + base.sym_sizes()[i] != 1 && base.sym_sizes()[i] != 0) { + return false; + } + } + return true; +} + +} // namespace utils + +// This function is will ensure that the fw_grad_ is properly a view of the base +// for inplace ops on Tensors that do not have forward grad originally. +void AutogradMeta::set_fw_grad( + const at::TensorBase& new_grad_base, + const at::TensorBase& self_base, + uint64_t level, + bool is_inplace_op) { + TORCH_CHECK( + !new_grad_base._fw_grad(level).defined(), + "Setting a forward grad that " + "itself has a forward gradient at the same level", + level, + " is not supported."); + TORCH_INTERNAL_ASSERT( + (new_grad_base.is_floating_point() || new_grad_base.is_complex()) && + (self_base.is_floating_point() || self_base.is_complex()), + "Expected both tensor and its forward grad to be floating point or complex"); + // Lazy initialization + { + std::lock_guard lock(mutex_); + if (!fw_grad_) { + fw_grad_ = std::make_shared(); + } + } + if (fw_grad_->contains(level)) { + // Setting the forward grad again is only allowed if it is a no-op. + // We do allow this case to simplify writing codegen for inplace ops. + TORCH_INTERNAL_ASSERT( + new_grad_base.defined(), + "Cannot set a forward grad that is an undefined Tensor. Use " + "_fw_primal(level) to get a new Tensor with this forward grad unset."); + + TORCH_INTERNAL_ASSERT( + is_inplace_op, + "Only inplace operations can re-set the forward grad of a Tensor that " + "already has one."); + + TORCH_INTERNAL_ASSERT( + fw_grad_->value(level).is_same(new_grad_base), + "Cannot set a value of a forward grad if it " + "already exists. Inplace operations should modify it inplace."); + } else { + // TODO(alband) remove this spurious version counter bump + Tensor new_grad(new_grad_base); + at::OptionalTensorRef self_ref(self_base); + const Tensor& self = *self_ref; + + TORCH_CHECK( + self.is_same_size(new_grad), + "Trying to set a forward gradient that has a different size than that " + "of the original Tensor, this is not supported. Tensor is of size ", + self.sizes(), + " while the given " + "forward gradient is of size ", + new_grad.sizes(), + "."); + + if (is_inplace_op && is_view_) { + auto this_view_meta = static_cast(this); + + // For inplace ops on a Tensor that does not already have a forward grad + // and is a view, we propagate the tangent to the base and ensure that the + // new_grad is a view of that base's tangent. This ensure that case 4 from + // [Forward Grad View/inplace] above works fine What happens in this long + // if statement is: + // - Check if the base already has a grad + // - If not, set a new fw_grad for it full of zeros + // - Take a view of the base's forward grad + // - Copy the given new_grad into this view + // - Use this view as the new new_grad + if (this_view_meta->has_fw_view()) { + auto& view_info = this_view_meta->get_forward_view(); + auto& base = view_info.base_; + + if (!base._fw_grad(level).defined()) { + // Enforce same meta here to make sure that the view op below is + // always valid + Tensor new_base_fw_grad; + if (utils::has_same_meta(new_grad, base) && + utils::has_same_meta(new_grad, self)) { + // TODO extend this special case to when the underlying storage of + // new_grad can be reused. + new_base_fw_grad = new_grad; + } else { + new_base_fw_grad = + at::_new_zeros_with_same_feature_meta(new_grad, base); + new_base_fw_grad._set_conj(base.is_conj()); + new_base_fw_grad._set_neg(base.is_neg()); + + // Update new_grad to be a view of the base + Tensor new_fw_grad_value; + if (view_info.has_view_fn()) { + new_fw_grad_value = view_info.view_fn()(new_base_fw_grad); + } else { + new_fw_grad_value = new_base_fw_grad.as_strided( + self.sizes(), self.strides(), self.storage_offset()); + } + + new_fw_grad_value.copy_(new_grad); + new_grad = new_fw_grad_value; + } + + base._set_fw_grad(new_base_fw_grad, level, /* is_inplace_op */ false); + } + } + } + + // Enforce the basic layout constraint + if (!utils::has_same_meta(new_grad, self)) { + if (is_view_) { + auto this_view_meta = static_cast(this); + TORCH_INTERNAL_ASSERT( + !this_view_meta->has_fw_view(), + "Expected the output of forward differentiable view operations to have the tangent have the same layout as primal") + } + auto res = at::_new_zeros_with_same_feature_meta(new_grad, self); + res._set_conj(self.is_conj()); + res._set_neg(self.is_neg()); + res.copy_(new_grad); + new_grad = res; + } + + fw_grad_->set_value(new_grad, level); + } +} + +const Variable& AutogradMeta::fw_grad( + uint64_t level, + const at::TensorBase& self) const { + // TLS that disables forward AD. + if (!c10::AutogradState::get_tls_state().get_fw_grad_mode()) { + return ForwardGrad::undef_grad(); + } + + // Ensure that concurrent fw_grad() "reads" are thread safe + std::lock_guard lock(mutex_); + + const auto& direct_fw_grad = + fw_grad_ ? fw_grad_->value(level) : ForwardGrad::undef_grad(); + + if (!direct_fw_grad.defined() && is_view_) { + // For view that don't have a forward grad, check if their base has one that + // has been defined by an inplace operation. + // This ensure that case 5 from [Forward Grad View/inplace] above works fine + auto const_view_meta = + static_cast(this); + // This is ok to do as we ONLY modify fw_grad_ and this field is properly + // locked in all methods + if (const_view_meta->has_fw_view()) { + const auto& view_info = const_view_meta->get_forward_view(); + const auto& base = view_info.base_; + + const auto& base_val = base._fw_grad(level); + if (base_val.defined()) { + // Lazy initialization of fw_grad_ + const_view_meta->fw_grad_ = std::make_shared(); + + Variable new_val; + if (view_info.has_view_fn()) { + new_val = view_info.view_fn()(base_val); + } else { + new_val = base_val.as_strided( + self.sizes(), self.strides(), self.storage_offset()); + } + + const_view_meta->fw_grad_->set_value(new_val, level); + return const_view_meta->fw_grad_->value(level); + } + } + } + return direct_fw_grad; +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/autograd_not_implemented_fallback.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/autograd_not_implemented_fallback.cpp new file mode 100644 index 0000000000000000000000000000000000000000..af010e8983afe2c667fe648f8c71030d45f3d445 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/autograd_not_implemented_fallback.cpp @@ -0,0 +1,729 @@ +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace torch::autograd { + +namespace { + +template +void _foreach_tensor( + F fn, + torch::jit::Stack* stack, + size_t stack_start, + size_t size) { + // Enumerate over tensors in a stack, including ones in TensorLists + int idx_tensor = 0; + for (const auto idx_arg : c10::irange(size)) { + auto& ivalue = (*stack)[stack_start + idx_arg]; + if (ivalue.isTensor()) { // true for optional tensor that has value + const auto& tensor = ivalue.toTensor(); + fn(idx_tensor, idx_arg, tensor); + idx_tensor++; + } else if (ivalue.isTensorList()) { + for (const auto& iv : ivalue.toListRef()) { + const auto& tensor = iv.toTensor(); + fn(idx_tensor, idx_arg, tensor); + idx_tensor++; + } + } + } +} + +[[maybe_unused]] +size_t expected_fresh_use_count(const at::Tensor& self) { + if (!self.defined()) { + // An UndefinedTensorImpl always has a use count of 0 + return 0; + } + if (self.unsafeGetTensorImpl()->pyobj_slot()->load_pyobj() != nullptr) { + // A TensorImpl with a Python object has a use count of 2 + return 2; + } + // A fresh TensorImpl (with no PyObject) has a use count of 1 + return 1; +} + +AutogradFallbackMode kAutogradFallbackMode = AutogradFallbackMode::Warn; + +} // namespace + +void setAutogradFallbackMode(AutogradFallbackMode mode) { + TORCH_CHECK(mode != AutogradFallbackMode::Error, "NYI: mode='error'"); + kAutogradFallbackMode = mode; +} + +AutogradFallbackMode getAutogradFallbackMode() { + return kAutogradFallbackMode; +} + +static void warnAutogradNotImplemented(const std::string& op_name) { + TORCH_WARN( + op_name, + ": an autograd kernel was not registered to the Autograd key(s) ", + "but we are trying to backprop through it. This may lead to silently incorrect behavior. ", + "This behavior is deprecated and will be removed in a future version of PyTorch. ", + "If your operator is differentiable, please ensure you have registered an " + "autograd kernel to the correct Autograd key (e.g. DispatchKey::Autograd, " + "DispatchKey::CompositeImplicitAutograd). If your operator is not " + "differentiable, or to squash this warning and use the previous behavior, " + "please register torch::CppFunction::makeFallthrough() to DispatchKey::Autograd."); +} + +struct WarnNotImplemented : public Node { + WarnNotImplemented( + std::string op_name, + size_t num_outputs, + edge_list&& next_edges) + : Node(std::move(next_edges)), + op_name(std::move(op_name)), + num_outputs(num_outputs) {} + + WarnNotImplemented(std::string op_name, size_t num_outputs) + : op_name(std::move(op_name)), num_outputs(num_outputs) {} + + variable_list apply(variable_list&& inputs) override; + + std::string op_name; + size_t num_outputs; +}; + +// NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved) +auto WarnNotImplemented::apply(variable_list&& inputs) -> variable_list { + auto inputsLocal = std::move(inputs); + warnAutogradNotImplemented(op_name); + std::vector output(num_outputs); + return output; +} + +static void basicAutogradNotImplementedFallbackImpl( + const c10::OperatorHandle& op, + c10::DispatchKeySet dispatch_keys, + torch::jit::Stack* stack) { + const auto& schema = op.schema(); + const auto& op_name = schema.operator_name().name; + const auto num_arguments = schema.arguments().size(); + const auto num_returns = schema.returns().size(); + const auto stack_start = stack->size() - num_arguments; + + if (getAutogradFallbackMode() == AutogradFallbackMode::Nothing) { + op.redispatchBoxed(dispatch_keys & c10::after_autograd_keyset, stack); + return; + } + TORCH_INTERNAL_ASSERT( + getAutogradFallbackMode() == AutogradFallbackMode::Warn); + + bool any_input_requires_grad = false; + _foreach_tensor( + [&](size_t _, size_t idx_arg, const at::Tensor& t) { + if (t.requires_grad()) { + any_input_requires_grad = true; + } + }, + stack, + stack_start, + num_arguments); + // Optimization: TLS access can be slow. So we only check if it necessary + // by putting it after the requires_grad checks. + any_input_requires_grad = any_input_requires_grad && GradMode::is_enabled(); + + c10::intrusive_ptr grad_fn; + if (any_input_requires_grad) { + // NB: It is standard to collect edges from all tensors + // (see generated/VariableTypeEverything.cpp for examples) + std::vector all_tensors_on_stack; + _foreach_tensor( + [&](size_t _, size_t idx_arg, const at::Tensor& t) { + all_tensors_on_stack.push_back(&t); + }, + stack, + stack_start, + num_arguments); + grad_fn = c10::make_intrusive( + op_name, all_tensors_on_stack.size()); + grad_fn->set_next_edges(collect_next_edges(all_tensors_on_stack)); + } + + op.redispatchBoxed(dispatch_keys & c10::after_autograd_keyset, stack); + + if (any_input_requires_grad) { + // NB: if the operator mutates any inputs in-place and does not return them + // as outputs, we are unable to lazily raise a warning. This is OK because + // we don't expect many existing operators to do this because of the amount + // of technical expertise necessary (you would need to manually register an + // autograd kernel without using autograd.Function) + _foreach_tensor( + [&](size_t _, size_t idx_ret, const at::Tensor& t) { + if (!isDifferentiableType(t.scalar_type())) { + return; + } + const bool is_mutable_output = + schema.is_aliasing({c10::SchemaArgType::output, idx_ret}) && + schema.is_mutable({c10::SchemaArgType::output, idx_ret}); + + // If the post-autograd implementation returns Tensors that require + // grad, then we install a hook that will warn during the backwards. + // + // NB: If the operation is inplace and the inputs were views, + // it is possible that the history was rebased and the hook will + // not warn in all places where it should. That is, the following + // won't warn: + // >>> x = torch.randn(3, 3, requires_grad=True) + // >>> z = x.clone() + // >>> w = z[0] + // >>> k = w[0] + // >>> y = op(k) + // >>> torch.autograd.grad(z.sum(), w) + if (t.requires_grad()) { + t.register_hook([op_name](const at::Tensor& grad) { + warnAutogradNotImplemented(op_name); + }); + // If history is rebased, then we will attempt to warn + // on the view's base. This will catch most cases (because + // users typically call .backward() and backprop through + // the entire program). + if (t.is_view() && is_mutable_output) { + const auto& base = t._base(); + if (base.requires_grad()) { + // Can only register_hook on tensors that require grad. + base.register_hook([op_name](const at::TensorBase& grad) { + warnAutogradNotImplemented(op_name); + }); + } + } + return; + } + + // If the post-autograd implementation returns any Tensors that + // don't require grad, then we install the WarnNotImplemented grad_fn. + // This grad_fn warns in backward and returns undefined tensor + // gradients. + // + // NOTE [autograd fallback and in-place operations] + // If the schema says the output is mutable, and the output + // is an input, and the input is a view Tensor, then... + // we're not sure if set_history is OK to do, so we just skip + // adding the grad_fn. Builtin operators do rebase_history here, + // but custom operators may have multiple Tensor(a!) returns, + // rebase_history assumes single Tensor(a!) return, and in general + // custom ops don't have a good in-place story. + if (!is_mutable_output) { + set_history(t, grad_fn); + } + }, + stack, + stack->size() - num_returns, + num_returns); + } +} + +torch::CppFunction basicAutogradNotImplementedFallback() { + return torch::CppFunction::makeFromBoxedFunction< + &basicAutogradNotImplementedFallbackImpl>(); +} + +void VariableHooks::basic_autograd_not_implemented_fallback( + const c10::OperatorHandle& op, + c10::DispatchKeySet dispatch_keys, + torch::jit::Stack* stack) const { + basicAutogradNotImplementedFallbackImpl(op, dispatch_keys, stack); +} + +static void autogradNotImplementedFallbackImpl( + const c10::OperatorHandle& op, + c10::DispatchKeySet dispatch_keys, + torch::jit::Stack* stack) { + // Mimics a subset of the logic of a VariableType NotImplemented kernel + // See gen_variable_type.py + const auto& schema = op.schema(); + const auto& op_name = schema.operator_name().name; + const auto num_arguments = schema.arguments().size(); + const auto num_returns = schema.returns().size(); + const auto stack_start = stack->size() - num_arguments; + const bool grad_mode = GradMode::is_enabled(); + std::vector tensors_requiring_grad_on_stack; + + // Keep track of which outputs are output of in-place modification + // so we can rebase_history if necessary + std::vector is_inplace_output(num_returns, false); + bool any_is_inplace_output = false; + std::vector is_aliased_output(num_returns, false); + std::optional aliased_output_idx; + + for (const auto i : c10::irange(num_returns)) { + if (schema.is_aliasing({c10::SchemaArgType::output, i})) { + if (schema.is_mutable({c10::SchemaArgType::output, i})) { + is_inplace_output[i] = true; + any_is_inplace_output = true; + } else { + TORCH_CHECK( + !aliased_output_idx.has_value(), + "Expected only a single output in the operator schema to have a non-write alias annotation (i.e., 'Tensor(a)'). " + "Non-composite functions where multiple outputs are aliased with inputs aren't supported." + "Please rewrite your function as a composite function."); + aliased_output_idx = i; + } + is_aliased_output[i] = true; + } + } + + int64_t aliased_input_idx = -1; + for (const auto i : c10::irange(num_arguments)) { + if (schema.is_aliasing({c10::SchemaArgType::input, i}) && + !schema.is_mutable({c10::SchemaArgType::input, i})) { + TORCH_CHECK( + aliased_input_idx == -1, + "Expected only a single input in the operator schema to have a non-write alias annotation (i.e., 'Tensor(a)'). " + "Non-composite functions where multiple inputs are aliased with outputs aren't supported. " + "Please rewrite your function as a composite function."); + aliased_input_idx = static_cast(i); + } + } + + size_t num_tensor_inputs = 0; // Only used for DEBUG-only checks + _foreach_tensor( + [&](size_t _, size_t idx_arg, const at::Tensor& t) { + if (grad_mode && t.requires_grad()) { + tensors_requiring_grad_on_stack.push_back(&t); + } + num_tensor_inputs++; + TORCH_CHECK_NOT_IMPLEMENTED( + !isFwGradDefined(t), + "Trying to use forward AD with ", + op_name, + " that does not support it."); + }, + stack, + stack_start, + num_arguments); + + const bool any_requires_grad = !tensors_requiring_grad_on_stack.empty(); + const bool has_out_arg = std::any_of( + schema.arguments().begin(), + schema.arguments().end(), + [](const c10::Argument& arg) { return arg.is_out(); }); + + _foreach_tensor( + [&](size_t _, size_t i, const at::Tensor& t) { + if (schema.is_mutable({c10::SchemaArgType::input, i})) { + if (has_out_arg) { + // Normally out argument overloads would not support any arguments + // that require grad. However, we loosen this check to maintain + // backward compatibility. + // See https://github.com/pytorch/pytorch/issues/120988 + if (can_mutate_inplace(t, any_requires_grad) != + can_mutate_inplace_result::success) { + throw_error_out_requires_grad(schema.name().c_str()); + } + } else { + check_inplace(t, any_requires_grad); + } + } + }, + stack, + stack_start, + num_arguments); + + c10::intrusive_ptr grad_fn; + if (any_requires_grad) { + grad_fn = c10::make_intrusive(op_name); + grad_fn->set_next_edges( + collect_next_edges(tensors_requiring_grad_on_stack)); + } + +#ifndef NDEBUG + // See NOTE [ TensorImpl and Storage Pointer Sanity Checks ] + auto stack_args_copy = std::vector( + stack->begin() + static_cast(stack_start), stack->end()); + std::vector> impl_saved; + impl_saved.reserve(num_tensor_inputs); + std::vector> storage_saved; + storage_saved.reserve(num_tensor_inputs); + _foreach_tensor( + [&](size_t idx, size_t _, const at::Tensor& t) { + storage_saved.push_back( + t.has_storage() ? std::optional(t.storage()) + : std::nullopt); + impl_saved.emplace_back(t.getIntrusivePtr()); + }, + &stack_args_copy, + 0, + num_arguments); +#endif + if (aliased_input_idx != -1 || any_is_inplace_output) { + at::AutoDispatchBelowAutograd guard; + op.redispatchBoxed(dispatch_keys & c10::after_autograd_keyset, stack); + } else { + // If neither in-place nor view + at::AutoDispatchBelowADInplaceOrView guard; + op.redispatchBoxed( + dispatch_keys & c10::after_ADInplaceOrView_keyset, stack); + } +#ifndef NDEBUG + _foreach_tensor( + [&](size_t idx_tensor, size_t _, const at::Tensor& t) { + // Skip next two for chunk_cat, see + // https://github.com/pytorch/pytorch/issues/130073 + if (storage_saved.at(idx_tensor).has_value() && + op_name != "aten::_chunk_cat") + TORCH_INTERNAL_ASSERT( + storage_saved.at(idx_tensor).value().is_alias_of(t.storage()), + op_name); + if (impl_saved.at(idx_tensor) && op_name != "aten::_chunk_cat") + TORCH_INTERNAL_ASSERT( + impl_saved.at(idx_tensor) == t.getIntrusivePtr(), op_name); + }, + &stack_args_copy, + 0, + num_arguments); + _foreach_tensor( + [&](size_t idx_tensor, size_t idx_ret, const at::Tensor& t) { + if (at::impl::tensor_has_dispatch(t) || + at::impl::dispatch_mode_enabled() || + // NJT components are expected to be reused; skip use_count() check + op_name.rfind("aten::_nested_get", 0) == 0) + return; + // Skip test_parallel_materialize + // For details see https://github.com/pytorch/pytorch/issues/130073 + if (op_name == "aten::_test_parallel_materialize" || + op_name == "aten::_test_optional_intlist" || + op_name == "aten::_test_optional_filled_intlist" || + op_name == "aten::_test_optional_floatlist") + return; + if (!is_inplace_output[idx_ret]) + TORCH_INTERNAL_ASSERT(t.use_count() == expected_fresh_use_count(t)); + // note(crcrpar): `_foreach_norm` returns a list of scalar Tensors and + // each Tensor shares a storage of a hidden, intermediate 1D Tensor + // created inside the CUDA implementation. This is because the + // reference implementation of nvidia/apex repo returns this 1D Tensor + // where each element represents the norm of corresponding input Tensor, + // here I want to return the same number of Tensors as the input + // TensorList, see https://github.com/pytorch/pytorch/issues/93940 + // Skip native_channel_shuffle as well as transformer_encoder + // For details see https://github.com/pytorch/pytorch/issues/130073 + if (!is_aliased_output[idx_ret] && t.has_storage() && + op_name != "aten::_foreach_norm" && + op_name != "aten::_transformer_encoder_layer_fwd" && + op_name != "aten::native_channel_shuffle" && + op_name != "aten::_sparse_semi_structured_tile") + TORCH_INTERNAL_ASSERT(t.storage().use_count() == 1); + }, + stack, + stack->size() - num_returns, + num_returns); + // There should be only a single base-view pair, make sure their storage is + // aliased. + if (aliased_input_idx != -1 && aliased_output_idx.has_value()) { + const c10::IValue& aliased_input_iv = stack_args_copy[aliased_input_idx]; + const c10::IValue& aliased_output_iv = + (*stack)[stack->size() - num_returns + *aliased_output_idx]; + TORCH_INTERNAL_ASSERT(aliased_input_iv.isTensor(), op_name); + TORCH_INTERNAL_ASSERT( + aliased_output_iv.isTensor() || aliased_output_iv.isTensorList(), + op_name); + const at::Tensor& aliased_input = aliased_input_iv.toTensor(); + if (aliased_input.has_storage()) { + if (aliased_output_iv.isTensor()) { + const at::Tensor& aliased_output = aliased_input_iv.toTensor(); + // for now, skip asserts for subclasses + // TODO: Fix the aliasing situation involving subclasses + if (!at::impl::dispatch_mode_enabled() && + !at::impl::tensor_has_dispatch(aliased_input) && + !at::impl::tensor_has_dispatch(aliased_output)) { + TORCH_INTERNAL_ASSERT( + aliased_input.storage().is_alias_of(aliased_output.storage()), + op_name); + } + } else { + const auto aliased_output_vec = aliased_output_iv.toTensorVector(); + for (const auto& aliased_output : aliased_output_vec) { + // for now, skip asserts for subclasses + // TODO: Fix the aliasing situation involving subclasses + if (!at::impl::dispatch_mode_enabled() && + !at::impl::tensor_has_dispatch(aliased_input) && + !at::impl::tensor_has_dispatch(aliased_output)) { + TORCH_INTERNAL_ASSERT( + aliased_input.storage().is_alias_of(aliased_output.storage()), + op_name); + } + } + } + } + } +#endif + + if (any_requires_grad) { + _foreach_tensor( + [&](size_t idx_tensor, size_t idx_ret, const at::Tensor& t) { + if (isDifferentiableType(t.scalar_type())) { + if (is_inplace_output[idx_ret]) { + rebase_history(t, grad_fn); + } else { + set_history(t, grad_fn); + } + } + }, + stack, + stack->size() - num_returns, + num_returns); + } +} + +torch::CppFunction autogradNotImplementedFallback() { + return torch::CppFunction::makeFromBoxedFunction< + &autogradNotImplementedFallbackImpl>(); +} + +struct GenericViewFunc : public ViewFunc { + GenericViewFunc( + torch::jit::Stack non_tensor_stack, + size_t aliased_input_idx_val, + c10::OperatorHandle op) + : non_tensor_stack_(non_tensor_stack), + aliased_input_idx_val_(aliased_input_idx_val), + op_(op) { + // This should report saved Tensors and SymInts. + // We already have an assert that ensure there are no Tensors here + // by making sure there is only one Tensor input. + // We also verify there are no SymInt here for now. + // Both can be lifted if the visit and clone logic get updated. + const auto& schema = op_.schema(); + for (const auto& arg : schema.arguments()) { + TORCH_CHECK( + arg.real_type()->kind() != c10::TypeKind::SymIntType, + "Custom ops that are views do not support SymInt. Please file an issue if you need it."); + for (const auto& ct : arg.real_type()->containedTypes()) { + TORCH_CHECK( + ct->kind() != c10::TypeKind::SymIntType, + "Custom ops that are views do not support SymInt. Please file an issue if you need it."); + } + } + } + + at::Tensor operator()(const at::Tensor& new_base) const override { + torch::jit::Stack local_stack = non_tensor_stack_; + local_stack.at(aliased_input_idx_val_) = c10::IValue(new_base); + + op_.callBoxed(local_stack); + auto& result = local_stack[local_stack.size() - 1]; + TORCH_CHECK( + result.isTensor(), + "ADInplaceOrView fallback view replay did not return a Tensor"); + return result.toTensor(); + } + + std::unique_ptr clone_and_set( + std::optional> /*unused*/ = std::nullopt, + std::optional> /*unused*/ = + std::nullopt) const override { + return std::make_unique( + non_tensor_stack_, aliased_input_idx_val_, op_); + } + + private: + torch::jit::Stack non_tensor_stack_; + size_t aliased_input_idx_val_; + c10::OperatorHandle op_; +}; + +static void autogradNotImplementedInplaceOrViewFallbackImpl( + const c10::OperatorHandle& op, + c10::DispatchKeySet dispatch_keys, + torch::jit::Stack* stack) { + // Mimics a subset of the logic from ADInplaceOrViewType kernel: + // - see gen_inplace_or_view_type.py + // - this should only be used with autogradNotImplementedFallback above + // - For more information see + // https://pytorch.org/tutorials/advanced/dispatcher + // + // NOTE [ Limitations of ADInplaceOrView boxed kernel ] + // + // This op should only be used with autogradNotImplementedFallback kernel + // because there is some logic we need specifically to enforce that even + // if we do in-place on view's created in this kernel, the proper "derivative + // is not implemented" error is still raised. + // + // Just like the codegened kernel, we try to enforce some things: + // - For views: we enforce that the view relationship is between the first + // input + // and the first output (which may be either Tensor or vec of Tensors + // - For inplace (TODO?): enforce that the same op cannot be both a view and + // inplace + // that is not allowed in the gen_inplace_or_view logic + const auto& schema = op.schema(); + const auto& op_name = schema.operator_name().name; + const auto num_arguments = schema.arguments().size(); + const auto num_returns = schema.returns().size(); + const auto stack_start = stack->size() - num_arguments; + + at::Tensor aliased_input; + + int64_t aliased_output_idx = -1; + for (const auto i : c10::irange(num_returns)) { + if (schema.is_aliasing({c10::SchemaArgType::output, i}) && + !schema.is_mutable({c10::SchemaArgType::output, i})) { + TORCH_CHECK( + aliased_output_idx == -1, + "Fallback ADInplaceOrView kernel expects only a single output in the operator schema to have a " + "non-write alias annotation (i.e., 'Tensor(a)'). " + "Non-composite functions where multiple outputs are aliased with inputs aren't supported." + "Please rewrite your function as a composite function."); + aliased_output_idx = static_cast(i); + } + } + + std::optional aliased_input_idx; + for (const auto i : c10::irange(num_arguments)) { + if (schema.is_aliasing({c10::SchemaArgType::input, i}) && + !schema.is_mutable({c10::SchemaArgType::input, i})) { + TORCH_CHECK( + !aliased_input_idx.has_value(), + "Fallback ADInplaceOrView kernel expects only a single input in the operator schema to have a " + "non-write alias annotation (i.e., 'Tensor(a)'). " + "Non-composite functions where multiple inputs are aliased with outputs aren't supported. " + "Please rewrite your function as a composite function."); + aliased_input_idx = i; + const c10::IValue& aliased_input_iv = + (*stack)[stack_start + i]; // get a reference to an ivalue on the + // stack + TORCH_CHECK(aliased_input_iv.isTensor()); + aliased_input = + aliased_input_iv.toTensor(); // TODO: Can we avoid saving this tensor + // and incurring the refcount bump? + } + } + // See NOTE [ Limitations of ADInplaceOrView boxed kernel ] above + TORCH_CHECK( + (!aliased_input_idx.has_value() && aliased_output_idx == -1) || + (aliased_input_idx.has_value() && aliased_input_idx.value() == 0 && + aliased_output_idx == 0), + "Fallback ADInplaceOrView kernel can only create view relationships between the first " + "input and the first output (the output can be a vector of tensors). Please change the " + "order of your operator's parameters so that this is the case."); + const bool is_view = aliased_input_idx.has_value(); + size_t aliased_input_idx_val = 0; + + // Save inputs before we redispatch down + torch::jit::Stack non_tensor_stack; + if (is_view) { + // Note that this won't be used if a TensorList is returned. + aliased_input_idx_val = aliased_input_idx.value(); + non_tensor_stack.reserve(num_arguments); + for (const auto i : c10::irange(num_arguments)) { + non_tensor_stack.push_back((*stack)[stack_start + i]); + } + } + + { + at::AutoDispatchBelowADInplaceOrView guard; + op.redispatchBoxed( + dispatch_keys & c10::after_ADInplaceOrView_keyset, stack); + } + + for (const auto i : c10::irange(num_returns)) { + if (schema.is_mutable({c10::SchemaArgType::output, i})) { + increment_version((*stack)[stack->size() - num_returns + i].toTensor()); + } + } + + if (is_view) { + c10::IValue& aliased_output_iv = + (*stack)[stack->size() - num_returns + aliased_output_idx]; + + // See NOTE [ View + Inplace detection ] for more details about this logic + // We always need this view_func because otherwise if we do in-place + // on this view, we would implicitly use AsStridedBackward instead + // of the NotImplemented node. For the cross-dtype/non-strided + // cases, we would create something like this anyway + auto error_msg = + ("Mutating the view " + op_name + + "which does not have a derivative implemented is forbidden."); + auto erroring_view_func = std::make_unique(error_msg); + + const auto erroring_rev_view_func = + [op_name = op_name](const at::Tensor&) -> at::Tensor { + TORCH_CHECK( + false, + "Accessing the reverse view for ", + op_name, + " which does not have a derivative implemented is forbidden."); + }; + + if (aliased_output_iv.isTensorList()) { + auto aliased_output = aliased_output_iv.toTensorVector(); + for (auto& sub_output : aliased_output) { + as_view( + /* base=*/aliased_input, + /* tensor=*/sub_output, + /* is_bw_differentiable=*/true, + /* is_fw_differentiable=*/true, + /* view_func=*/std::move(erroring_view_func), + /* rev_view_func=*/erroring_rev_view_func, + /* creation_meta=*/ + InferenceMode::is_enabled() + ? CreationMeta::INFERENCE_MODE + : (at::GradMode::is_enabled() ? CreationMeta::MULTI_OUTPUT_NODE + : CreationMeta::NO_GRAD_MODE)); + } + auto result = std::move(aliased_output); + stack->at(stack->size() - num_returns + aliased_output_idx) = result; + } else { + c10::IValue& aliased_output_iv = + (*stack)[stack->size() - num_returns + aliased_output_idx]; + TORCH_CHECK(aliased_output_iv.isTensor()); + TORCH_CHECK( + num_returns == 1, + "ADInplaceOrView fallback only support single output view functions"); + + // Remove the Tensor from the original stack + for (const auto i : c10::irange(num_arguments)) { + if (non_tensor_stack[i].isTensor()) { + TORCH_CHECK( + i == aliased_input_idx_val, + "Internal error in ADInplaceOrView fallback, unknown Tensor in the stack"); + non_tensor_stack[i] = {}; + } + } + + auto view_func = std::make_unique( + non_tensor_stack, aliased_input_idx_val, op); + + auto result = as_view( + /* base=*/aliased_input, + /* tensor=*/std::move(aliased_output_iv).toTensor(), + /* is_bw_differentiable=*/true, + /* is_fw_differentiable=*/true, + /* view_func=*/std::move(view_func), + /* rev_view_func=*/erroring_rev_view_func, + /* creation_meta=*/ + InferenceMode::is_enabled() + ? CreationMeta::INFERENCE_MODE + : (at::GradMode::is_enabled() ? CreationMeta::DEFAULT + : CreationMeta::NO_GRAD_MODE)); + stack->at(stack->size() - num_returns + aliased_output_idx) = + std::move(result); + } + } +} + +torch::CppFunction autogradNotImplementedInplaceOrViewFallback() { + return torch::CppFunction::makeFromBoxedFunction< + &autogradNotImplementedInplaceOrViewFallbackImpl>(); +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/autograd_not_implemented_fallback.h b/western_ai/usa/pytorch/torch/csrc/autograd/autograd_not_implemented_fallback.h new file mode 100644 index 0000000000000000000000000000000000000000..f406dde432ce8e37313026c654dfe5cd05a0d1df --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/autograd_not_implemented_fallback.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace torch::autograd { + +// Default DispatchKey::Autograd fallback for built-in operators. +// Can be registered for custom operators. +TORCH_API torch::CppFunction autogradNotImplementedFallback(); + +// Default DispatchKey::AdInplaceOrView fallback for built-in operators +// Can be registered for custom operators. +TORCH_API torch::CppFunction autogradNotImplementedInplaceOrViewFallback(); + +// Default DispatchKey::Autograd fallback for all other operators (i.e. custom +// operators) +TORCH_API torch::CppFunction basicAutogradNotImplementedFallback(); + +enum class AutogradFallbackMode { + Nothing, // Fallback is a redispatch + Warn, // Fallback raises a warning if backward is called + Error, // Fallback raises an error if backward is called +}; + +// Change the behavior of "basicAutogradNotImplementedFallback" +// In Python this is: +// - torch._C._set_autograd_fallback_mode(str) -> None +// - torch._C._get_autograd_fallback_mode() -> str +TORCH_API void setAutogradFallbackMode(AutogradFallbackMode mode); +TORCH_API AutogradFallbackMode getAutogradFallbackMode(); + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/cpp_hook.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/cpp_hook.cpp new file mode 100644 index 0000000000000000000000000000000000000000..941e80ddece4ce7d695caa6493a40398b701580a --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/cpp_hook.cpp @@ -0,0 +1,71 @@ +#include +#include +#include +#include + +#include + +namespace { +using torch::autograd::Variable; +void check_single_result( + const at::TensorBase& value, + const at::TensorBase& result, + const std::string& hook_name) { + TORCH_CHECK( + value.defined(), + "can't replace an empty gradient with a non-empty value"); + torch::autograd::check_variable_result(value, result, hook_name); +} +} // namespace + +namespace torch::autograd { + +CppFunctionTensorPreHook::CppFunctionTensorPreHook( + std::shared_ptr hooks, + size_t value_idx) + : hooks_(std::move(hooks)), value_idx_(value_idx) {} + +variable_list CppFunctionTensorPreHook::operator()( + const variable_list& values) { + auto value = values[value_idx_]; + for (const auto i : c10::irange(hooks_->size())) { + auto& hook = (*hooks_)[i]; + if (!hook) { + // hook was removed + continue; + } + auto res = hook(value); + if (!res.defined()) { + // Don't change gradient + continue; + } + check_single_result(value, res, std::to_string(i)); + value = std::move(res); + } + variable_list results(values); + results[value_idx_] = value; + return results; +} + +CppFunctionSingleTensorPreHook::CppFunctionSingleTensorPreHook( + std::function hook, + size_t value_idx) + : hook_(std::move(hook)), value_idx_(value_idx) {} + +variable_list CppFunctionSingleTensorPreHook::operator()( + const variable_list& values) { + const auto& value = values[value_idx_]; + auto res = hook_(value); + TORCH_INTERNAL_ASSERT( + !res.defined(), + "CppFunctionSingleTensorPreHook currently only supports hooks that don't return"); + variable_list results(values); + return results; +} + +void CppFunctionSingleTensorPreHook::compiled_args( + torch::dynamo::autograd::CompiledNodeArgs& args) const { + args.add_cpp_single_tensor_pre_hook(hook_, value_idx_); +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/cpp_hook.h b/western_ai/usa/pytorch/torch/csrc/autograd/cpp_hook.h new file mode 100644 index 0000000000000000000000000000000000000000..6b77940c9e3be007a6d22840efa01d2d3f6038cd --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/cpp_hook.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include +#include + +namespace torch::autograd { + +using hooks_list = + std::vector>; + +struct CppFunctionTensorPreHook : public FunctionPreHook { + CppFunctionTensorPreHook(std::shared_ptr hooks, size_t value_idx); + variable_list operator()(const variable_list& values) override; + + std::shared_ptr hooks_; + size_t value_idx_; +}; + +struct CppFunctionSingleTensorPreHook : public FunctionPreHook { + CppFunctionSingleTensorPreHook( + std::function hook, + size_t value_idx); + variable_list operator()(const variable_list& values) override; + + void compiled_args( + torch::dynamo::autograd::CompiledNodeArgs& args) const override; + + std::function hook_; + size_t value_idx_; +}; + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/custom_function.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/custom_function.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9a6ed3864a3672e028c400bc5cf371a35782e325 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/custom_function.cpp @@ -0,0 +1,626 @@ +#include +#include +#include +#include +#include + +#include + +namespace torch::autograd { + +// This function has two main goals: +// 1) Use the user-provided jvp function to populate the outputs' forward +// gradient 2) Perform error checking to ensure that view and inplace ops are +// properly handled +// +// For 1) we have to: +// - Create a variable_list of grad_inputs based on the function inputs +// - Call the user jvp function with these to get the grad_outputs +// - Set the forward grad field on each output based on these grad_outputs +// +// For 2) we want to check the following: +// - If an output is a view, then the generated forward grad must be a view as +// well and +// the output's base's forward grad must be the output's forward grad's base. +// - If an input was modified inplace (it must be an output as well) we make +// sure that its +// forward grad was also modified inplace and already present on the +// corresponding output. +static void _process_forward_mode_AD( + const variable_list& inputs, + std::unordered_map inputs_mapping, + const at::ArrayRef> raw_outputs, + const optional_variable_list& outputs, + const std::unordered_set& non_differentiable, + const std::unordered_set& dirty_inputs, + const _jvp_fn_t& jvp_user_function) { + // TODO handle multiple levels here + uint64_t level = 0; + + const auto num_inputs = inputs.size(); + const auto num_outputs = outputs.size(); + + // The tracking info below are used to perform the view and inplace checks. + // They are lazily initialized to reduce the cost of this function in the + // common case where the user is not using forward mode AD. + variable_list input_grads; + std::vector grad_versions; + std::vector grad_impls; + std::unordered_map inputs_bases; + + auto init_tracked_info = [&]() { + input_grads.resize(num_inputs); + grad_versions.resize(num_inputs); + grad_impls.resize(num_inputs); + + for (const auto i : c10::irange(num_inputs)) { + const auto& inp = inputs[i]; + if (inp.is_view() && impl::get_view_autograd_meta(inp)->has_fw_view()) { + inputs_bases.emplace( + impl::get_view_autograd_meta(inp) + ->get_forward_view() + .base_.unsafeGetTensorImpl(), + i); + } else { + inputs_bases.emplace(inp.unsafeGetTensorImpl(), i); + } + } + }; + + bool any_input_has_grad = false; + // Extract the input's forward gradients and record any info we will need + // later + for (const auto i : c10::irange(num_inputs)) { + const auto& inp = inputs[i]; + if (!inp.defined()) { + continue; + } + const auto& fw_grad = inp._fw_grad(level); + if (fw_grad.defined()) { + if (!any_input_has_grad) { + any_input_has_grad = true; + init_tracked_info(); + } + input_grads[i] = fw_grad; + grad_versions[i] = fw_grad._version(); + grad_impls[i] = fw_grad.unsafeGetTensorImpl(); + } + } + + // If no input has forward grad, nothing to do here + if (!any_input_has_grad) { + return; + } + + torch::autograd::variable_list forward_grads; + { + at::AutoFwGradMode fw_grad_mode(false); + forward_grads = jvp_user_function(inputs, std::move(input_grads)); + } + + const auto num_forward_grads = forward_grads.size(); + // contrary to backward mode, we don't allow returning too many gradients + TORCH_CHECK( + num_forward_grads == num_outputs, + "Function's jvp returned " + "an invalid number of forward gradients (expected ", + num_outputs, + " but got ", + num_forward_grads, + ")"); + + for (const auto i : c10::irange(num_outputs)) { + if (!raw_outputs[i].has_value()) { + continue; + } + const auto& out = + outputs[i].has_value() ? outputs[i].value() : at::Tensor(); + auto out_tensor_impl = raw_outputs[i].value().unsafeGetTensorImpl(); + bool is_differentiable = + (non_differentiable.count(out_tensor_impl) == 0 && + isDifferentiableType(raw_outputs[i].value().scalar_type())); + const auto& out_grad = forward_grads[i]; + if (!out.defined() || !is_differentiable) { + TORCH_CHECK( + !out_grad.defined(), + "Function's jvp returned a gradient at position ", + i, + ", but " + " the corresponding forward output is not a differentiable Tensor." + "You should return None at that position instead."); + continue; + } + + bool is_input = inputs_mapping.count(out_tensor_impl) > 0; + bool is_modified = dirty_inputs.count(out_tensor_impl) > 0; + + if (is_modified) { + TORCH_CHECK( + is_input, + "Only input Tensors should be given to ctx.mark_dirty(). If a Tensor is not an input, there" + " is no need to pass it to mark_dirty()."); + auto inp_idx = inputs_mapping[out_tensor_impl]; + if (grad_impls[inp_idx]) { + // If there was already a forward grad for that input + // Just make sure that it is modified inplace and returned as-is + TORCH_CHECK( + out_grad._version() != grad_versions[inp_idx], + "An inplace custom Function is not modifying the " + "forward mode gradients inplace. If the forward is modifying an input inplace, then the jvp " + "function must modify the corresponding gradient inplace.") + TORCH_CHECK( + out_grad.unsafeGetTensorImpl() == grad_impls[inp_idx], + "An inplace custom Function is not returning the " + "forward mode gradients as-is. If the forward is modifying an input inplace, then the jvp " + "function must modify the gradient inplace and return it as-is.") + } else { + // If that Tensor didn't have gradients already, set the newly returned + // one We could also use inputs[inp_idx] here as it is the same as out + out._set_fw_grad(out_grad, level, /* is_inplace_op */ true); + } + } else { + // At this point, outputs[i] cannot be one of the input (raw_outputs[i] + // might be but was changed by the backward code) + TORCH_INTERNAL_ASSERT( + inputs_mapping.count(out.unsafeGetTensorImpl()) == 0); + + if (out.is_view() && impl::get_view_autograd_meta(out)->has_fw_view()) { + // If the output is a view + const auto& out_view_info = + impl::get_view_autograd_meta(out)->get_forward_view(); + if (inputs_bases.count(out_view_info.base_.unsafeGetTensorImpl())) { + // And it is a view of an input (either that input is its base or they + // have a common base) + const auto matching_input_idx = + inputs_bases[out_view_info.base_.unsafeGetTensorImpl()]; + const auto& matching_input = inputs[matching_input_idx]; + + const auto& matching_input_grad = matching_input._fw_grad(level); + + // If the matching input has a forward grad, the user should have + // returned a view of that Tensor + if (matching_input_grad.defined()) { + TORCH_CHECK( + out_grad.is_view() && + impl::get_view_autograd_meta(out_grad)->has_fw_view(), + "A custom Function's forward is returning a view (or an input as-is) but the jvp is not " + "returning a view."); + const auto& out_grad_base = impl::get_view_autograd_meta(out_grad) + ->get_forward_view() + .base_; + if (matching_input_grad.is_view() && + impl::get_view_autograd_meta(matching_input_grad) + ->has_fw_view()) { + // If the matching input's grad is a view, ensure that the + // out_grad is a view of the same base + const auto& matching_input_grad_base = + impl::get_view_autograd_meta(matching_input_grad) + ->get_forward_view() + .base_; + TORCH_CHECK( + matching_input_grad_base.unsafeGetTensorImpl() == + out_grad_base.unsafeGetTensorImpl(), + "A custom Function is returning a view but the jvp is not returning a view of the same base as " + "the given grad input."); + } else { + // If the matching input's grad is not a view, then it must be the + // output gradient's base + TORCH_CHECK( + matching_input_grad.unsafeGetTensorImpl() == + out_grad_base.unsafeGetTensorImpl(), + "A custom Function is returning a view but the jvp is not returning a view of the given grad input."); + } + } else { + // We have a view op where the input didn't have a forward grad but + // the user returned one for the output To ensure that we maintain + // the view/inplace constraints, we consider this as an inplace op + // This case CANNOT happen in codegen as all view ops are mapping + // from one Tensor to one Tensor and so the output of the view + // cannot have a forward grad if the base does not. + out._set_fw_grad(out_grad, level, /* is_inplace_op */ true); + return; + } + } + } + + out._set_fw_grad(out_grad, level, /* is_inplace_op */ false); + } + } +} + +static at::Tensor _view_as_self_with_no_grad( + const at::Tensor& self, + const _view_as_self_fn_t& view_as_self_fn) { + // This is called below in _process_backward_mode_ad in two places: + // + // (1) An input has been returned, but it wasn't modified. Return it as a view + // so that we can attach a new grad_fn to the Variable. + // Run in no_grad mode to mimic the behavior of the forward. + // + // (2) Though it is not necessary for the purposes of attaching grad_fn, we + // also call this function when an output is non-differentiable (and does not + // require grad). to help custom forward AD UX more consistent. We'd like to + // uniformly say that returning an input as-is is treated as if + // `self.view_as(self)` were returned for that output. + // + // Alternatively, we could have not disabled forward grad while performing + // this view, but it would mean that the user defined jvp may be silently + // ignored. + at::AutoFwGradMode fw_grad_mode(false); + AutoGradMode grad_mode(false); + // We thread through this view_as_self_fn lambda so that in the case we are a + // Python custom function (rather than a cpp one), we can properly call the + // view_as from python so that torch function logic can still trigger. + return view_as_self_fn(self); +} + +static optional_variable_list _process_backward_mode_ad( + const std::unordered_map& inputs_mapping, + const std::unordered_set& non_differentiable, + const std::unordered_set& dirty_inputs, + const at::ArrayRef> raw_outputs, + const c10::intrusive_ptr& cdata, + const std::unordered_set& to_save_if_setup_context, + const _view_as_self_fn_t& view_as_self_fn, + bool pure_view) { + auto num_outputs = raw_outputs.size(); + +#ifndef STRIP_ERROR_MESSAGES + const char* error_msg_input_returned_as_is = + "An input that has been returned as-is as output is being saved for backward. " + "This is not supported if you override setup_context. You should return and " + "save a view of the input instead, e.g. with x.view_as(x) or setup ctx inside " + "the forward function itself."; +#endif + + // Sets the grad_fn and output_nr of an output Variable. + auto set_history = [&](Variable& var, + uint32_t output_nr, + bool is_input, + bool is_modified, + bool is_differentiable, + bool is_saved_and_setup_context) { + if (!is_differentiable) { + if (!var.requires_grad()) { + if (is_input && !is_modified) { + TORCH_CHECK( + !is_saved_and_setup_context, error_msg_input_returned_as_is) + var = _view_as_self_with_no_grad(var, view_as_self_fn); + } + return; + } + // Return detached aliases of inputs, instead of changing their + // requires_grad property. + if (is_input) { + var = var.detach(); + } else if (!var.is_view()) { + var.detach_(); + } + // If var is a view of one of the inputs of the custom autograd Function, + // we don't detach it in a no_grad block. This is so that we can mimic the + // behavior of returning a view from a no_grad block: + // x = torch.randn(3, requires_grad=True) + // with torch.no_grad(): + // y = x.view(-1) + // Here, `y` requires_grad (!). + } else if (is_modified) { + if (var.is_leaf() && var.requires_grad()) { + TORCH_CHECK( + false, + "a leaf Variable that requires grad has been used in an in-place operation."); + } + // No need to mark as modified Tensors that are not inputs. + if (!is_input) { + const char* mark_dirty_error_msg = + "ctx.mark_dirty() received a tensor that was not an input. " + "Only input Tensors that have been mutated should be passed to " + "ctx.mark_dirty()."; + // We reach this path in the view of intermediate case + TORCH_CHECK(!var.is_view(), mark_dirty_error_msg); + TORCH_WARN(mark_dirty_error_msg); + } + // If the input is a view, the rebase will need to rewrite the graph and + // this only works if we have a single output to this Function. + TORCH_CHECK( + !(var.is_view() && num_outputs > 1), + "If your Function modifies inplace an input that is a view" + " of another Tensor, your Function cannot return more than one Tensor. This is not supported" + " by the current autograd engine. You should either make sure the input is not a view (using" + " .clone() for example) or make your Function only return one Tensor (potentially splitting" + " it into two Functions: one doing the inplace that returns a single Tensor and a second one" + " that does the other operations). You can ask on the forum https://discuss.pytorch.org/ if" + " you need help to do this change."); + + // If the input was modified, transplant the grad_fn in the graph: + // grad_fn <- variable <- self ==> grad_fn <- self <- variable + var.mutable_grad().reset(); + impl::clear_hooks(var); + if (auto grad_acc_fn = impl::try_get_grad_accumulator(var)) { + auto& grad_acc = dynamic_cast(*grad_acc_fn); + grad_acc.variable.reset(); + } + // This repeats the mutation of leaf variables check already done above + check_inplace(var, true); + impl::rebase_history(var, {cdata, output_nr}); + } else if (is_input) { + TORCH_CHECK(!is_saved_and_setup_context, error_msg_input_returned_as_is) + var = _view_as_self_with_no_grad(var, view_as_self_fn); + impl::set_gradient_edge(var, {cdata, output_nr}); + } else { + impl::set_gradient_edge(var, {cdata, output_nr}); + } + }; + + optional_variable_list outputs; + std::unordered_set outputs_impl; // For dirty_inputs check + outputs.reserve(num_outputs); + int num_diff_outputs = 0; + + for (const auto i : c10::irange(num_outputs)) { + // We put a undefined_input placeholder for outputs that are not tensor and + // for when the output tensor is not differentiable (see below) + if (!raw_outputs[i].has_value()) { + if (cdata) { + auto output_nr = cdata->add_input_metadata(Node::undefined_input()); + AT_ASSERT(i == output_nr); + } + outputs.emplace_back(); + continue; + } + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + Variable var = raw_outputs[i].value(); + + auto out_tensor_impl = var.unsafeGetTensorImpl(); + bool is_input = inputs_mapping.count(out_tensor_impl) > 0; + bool is_modified = dirty_inputs.count(out_tensor_impl) > 0; + bool is_differentiable = cdata && + non_differentiable.count(out_tensor_impl) == 0 && + isDifferentiableType(var.scalar_type()); + bool is_saved_and_setup_context = + to_save_if_setup_context.count(out_tensor_impl) > 0; + + if (cdata) { + uint32_t output_nr = 0; + if (!is_differentiable) { + output_nr = cdata->add_input_metadata(Node::undefined_input()); + } else { + output_nr = cdata->add_input_metadata(var); + } + AT_ASSERT(i == output_nr); + } + set_history( + var, + i, + is_input, + is_modified, + is_differentiable, + is_saved_and_setup_context); + + // For deprecation cycle. Can be removed after 1.6. In the case where we + // detected a view in no grad mode during the forward, only warn the user + // (do not change the flag if we return and input that is a view as is). See + // NOTE [ View + Inplace detection ] for why we replace everything by a + // warning. + if (!(is_input && is_modified) && var.is_view()) { + // is_view() => diff_view_meta + auto diff_view_meta = impl::get_view_autograd_meta(var); + diff_view_meta->set_creation_meta( + pure_view ? CreationMeta::DEFAULT : CreationMeta::IN_CUSTOM_FUNCTION); + } + + if (is_differentiable) { + ++num_diff_outputs; + } + + outputs_impl.insert(out_tensor_impl); + outputs.emplace_back(var); + } + + // If multiple differentiable outputs are returned, we do not allow views to + // be modified inplace See NOTE [ View + Inplace detection ] for more details + if (num_diff_outputs > 1) { + for (auto& var : outputs) { + if (var.has_value()) { + auto diff_view_meta = impl::get_view_autograd_meta(var.value()); + if (diff_view_meta && diff_view_meta->has_bw_view()) { + diff_view_meta->set_creation_meta(CreationMeta::MULTI_OUTPUT_NODE); + } + } + } + } + + // All the modified Tensors must be returned as is for the rewrite to be + // valid. + for (auto& dirty_input : dirty_inputs) { + TORCH_CHECK( + outputs_impl.count(dirty_input) > 0, + "Some elements marked as dirty during the forward method were not returned as output. The" + " inputs that are modified inplace must all be outputs of the Function."); + } + + return outputs; +} + +optional_variable_list _wrap_outputs( + const variable_list& input_vars, + const std::unordered_set& non_differentiable, + const std::unordered_set& dirty_inputs, + const at::ArrayRef> raw_outputs, + const c10::intrusive_ptr& cdata, + const _jvp_fn_t& jvp_user_function, + const std::unordered_set& to_save_if_setup_context, + const _view_as_self_fn_t& view_as_self_fn, + bool pure_view) { + std::unordered_map inputs_mapping; + inputs_mapping.reserve(input_vars.size()); + for (const auto i : c10::irange(input_vars.size())) { + inputs_mapping.emplace(input_vars[i].unsafeGetTensorImpl(), i); + } + + // Limit pure views to 1-1 mapping as it is unclear if it is even + // possible to have a pure view for N-1 or 1-N. + TORCH_CHECK( + !pure_view || (input_vars.size() == 1 && raw_outputs.size() == 1), + "Pure view custom Function can only have one input Tensor and one output Tensor. Open an issue if you need to support more."); + + auto outputs = _process_backward_mode_ad( + inputs_mapping, + non_differentiable, + dirty_inputs, + raw_outputs, + cdata, + to_save_if_setup_context, + view_as_self_fn, + pure_view); + + // This must happen after the backward processing as we expect the + // computations happening here to track backward mode gradients. + _process_forward_mode_AD( + input_vars, + std::move(inputs_mapping), + raw_outputs, + outputs, + non_differentiable, + dirty_inputs, + jvp_user_function); + + return outputs; +} + +void check_variable_result( + const at::TensorBase& original, + const at::TensorBase& result, + const std::string& hook_name) { + TORCH_CHECK( + original.options().type_equal(result.options()), + "hook '", + hook_name, + "' has changed the type of value (was ", + original.toString(), + " got ", + result.toString(), + ")"); + + TORCH_CHECK( + original.is_cuda() == result.is_cuda(), + "hook '", + hook_name, + "' has changed the type of value (was ", + original.is_cuda() ? "CUDA tensor" : "CPU tensor", + " got ", + result.is_cuda() ? "CUDA tensor" : "CPU tensor", + ")"); + + TORCH_CHECK( + original.sym_sizes().vec() == result.sym_sizes().vec(), + "hook '", + hook_name, + "' has changed the size of value"); +} + +AutogradContext::AutogradContext(PackedArgs& packed_args) { + saved_data = packed_args.unpack_saved_data(); + saved_variables_override_ = packed_args.unpack(); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + materialize_grads_ = packed_args.unpack(); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + has_freed_buffers_ = packed_args.unpack(); + needs_input_grad_override_ = packed_args.unpack>(); +} + +void AutogradContext::save_for_backward(variable_list to_save) { + to_save_ = std::move(to_save); +} + +// The logic for handling saved variables here is the same as +// python_function.cpp See _save_variables() and unpack_saved_variables() +void AutogradContext::save_variables() { + saved_variables_.clear(); + auto ptr = grad_fn_.lock(); + + for (const auto& var : to_save_) { + // Allow empty variables to be saved + if (var.defined()) { + bool is_output = var.grad_fn().get() == ptr.get(); + saved_variables_.emplace_back(var, is_output); + } else { + saved_variables_.emplace_back(); + } + } + to_save_.clear(); +} + +variable_list AutogradContext::get_saved_variables() const { + TORCH_CHECK(!has_freed_buffers_, ERR_BACKWARD_TWICE); + if (saved_variables_override_.has_value()) { + return *saved_variables_override_; + } + variable_list saved; + saved.reserve(saved_variables_.size()); + auto ptr = grad_fn_.lock(); + TORCH_INTERNAL_ASSERT(ptr); + for (auto& var : saved_variables_) { + saved.push_back(var.unpack(ptr)); + } + return saved; +} + +bool AutogradContext::needs_input_grad(size_t output_edge_index) const { + if (needs_input_grad_override_.has_value()) { + return needs_input_grad_override_.value().at(output_edge_index); + } + auto ptr = grad_fn_.lock(); + TORCH_INTERNAL_ASSERT(ptr); + return ptr->task_should_compute_output(output_edge_index); +} + +bool AutogradContext::needs_input_grad( + std::initializer_list idxs) const { + if (needs_input_grad_override_.has_value()) { + return std::any_of(idxs.begin(), idxs.end(), [this](IndexRange range) { + bool result = false; + for (const auto i : c10::irange(range.first, range.second)) { + result |= needs_input_grad_override_.value().at(i); + } + return result; + }); + } + auto ptr = grad_fn_.lock(); + TORCH_INTERNAL_ASSERT(ptr); + return ptr->task_should_compute_output(idxs); +} + +void AutogradContext::mark_dirty(const variable_list& inputs) { + dirty_inputs_.clear(); + dirty_inputs_.reserve(inputs.size()); + for (auto& var : inputs) { + dirty_inputs_.insert(var.unsafeGetTensorImpl()); + } +} + +void AutogradContext::mark_non_differentiable(const variable_list& outputs) { + non_differentiable_.clear(); + non_differentiable_.reserve(outputs.size()); + for (auto& var : outputs) { + non_differentiable_.insert(var.unsafeGetTensorImpl()); + } +} + +void AutogradContext::set_materialize_grads(bool value) { + materialize_grads_ = value; +} + +const std::unordered_set& AutogradContext::get_and_bump_dirty() + const { + for (auto& var : dirty_inputs_) { + var->bump_version(); + } + return dirty_inputs_; +} + +const std::unordered_set& AutogradContext:: + get_non_differentiable() const { + return non_differentiable_; +} +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/custom_function.h b/western_ai/usa/pytorch/torch/csrc/autograd/custom_function.h new file mode 100644 index 0000000000000000000000000000000000000000..b7be9ff8e7fdb5b421f8624df7fff236c68266b6 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/custom_function.h @@ -0,0 +1,590 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace torch::autograd { + +using optional_variable_list = std::vector>; +using _jvp_fn_t = std::function; +using _view_as_self_fn_t = std::function; + +TORCH_API std::vector> _wrap_outputs( + const variable_list& input_vars, + const std::unordered_set& non_differentiable, + const std::unordered_set& dirty_inputs, + const at::ArrayRef> raw_outputs, + const c10::intrusive_ptr& cdata, + const _jvp_fn_t& jvp_user_function, + const std::unordered_set& to_save_if_setup_context, + const _view_as_self_fn_t& view_as_self_fn, + bool pure_view); + +TORCH_API void check_variable_result( + const at::TensorBase& original, + const at::TensorBase& result, + const std::string& hook_name); + +// Get the return type of the forward function of the custom Function class X +template +using forward_t = decltype(X::forward(nullptr, std::declval()...)); + +/// To use custom autograd operations, implement a Function subclass with +/// static forward and backward functions: +/// +/// `forward` can take as many arguments as you want and should return either a +/// variable list or a Variable. Use of any direct Variable arguments will be +/// registered in the graph but no vectors/sets or any other data structures +/// will be traversed. You can use std::optional as one of the arguments +/// and it will be registered as a variable in the graph if the argument has a +/// value. It should take a pointer to `torch::autograd::AutogradContext` as the +/// first argument. Variables can be saved in the `ctx` using +/// `ctx->save_for_backward` +/// (see `torch::autograd::AutogradContext::save_for_backward`) and other data +/// can be saved in the `ctx->saved_data` map +/// (see `torch::autograd::AutogradContext::saved_data`) +/// in the form of `` pairs. +/// +/// `backward` should take a pointer to `torch::autograd::AutogradContext` +/// and a variable list containing as many Variables as there were outputs from +/// `forward` as arguments. It should return as many Variables as there were +/// inputs with each of them containing the gradient w.r.t. its corresponding +/// input. Variables saved in `forward` can be accessed with +/// `ctx->get_saved_variables` (see +/// `torch::autograd::AutogradContext::get_saved_variables`) and other saved +/// data can be accessed from `ctx->saved_data`. +/// To enable compiled autograd support (torch.compile for backward) for your +/// custom autograd operation, you can set MyFunction::is_traceable +/// (see Function::istraceable notes below). +/// +/// For example: +/// ``` +/// class MyFunction : public Function { +/// public: +/// static constexpr bool is_traceable = true; +/// +/// static variable_list forward(AutogradContext *ctx, int n, Variable var) { +/// // Save data for backward in context +/// ctx->saved_data["n"] = n; +/// var.mul_(n); +/// // Mark var as modified by inplace operation +/// ctx->mark_dirty({var}); +/// return {var}; +/// } +/// +/// static variable_list backward(AutogradContext *ctx, variable_list +/// grad_output) { +/// // Use data saved in forward +/// auto n = ctx->saved_data["n"].toInt(); +/// return {grad_output[0]*n}; +/// } +/// }; +/// ``` +/// +/// To use `MyFunction`: +/// ``` +/// Variable x; +/// auto y = MyFunction::apply(6, x); +/// // Example backward call +/// y[0].sum().backward(); +/// ``` +template +struct TORCH_API Function { + // We need to use a different template parameter than T here because T will + // inherit from Function, and when Function is instantiated, T::forward + // is not declared yet. + // The enable_if check is to ensure that the user doesn't explicitly provide + // the parameter X. + template + static auto apply(Args&&... args) + -> std::enable_if_t, forward_t>; + + // This flag is for an experimental feature: compiled autograd. Not all + // built-in APIs are supported at the moment e.g. mark_dirty and + // mark_non_differentiable. Before setting this flag to enable tracing for + // your custom function , you need to ensure that the backward function is + // traceable i.e. any variables accessed in the backward other than the input + // arguments must be handled in a similar manner to built-ins in + // CppNode::compiled_args and CppNode::apply_with_saved. + static constexpr bool is_traceable = false; +}; + +/// Context to save information during `forward` that can be accessed in +/// `backward` in custom autograd operations (see `torch::autograd::Function` +/// for details). +struct TORCH_API AutogradContext { + AutogradContext() = default; + AutogradContext(const AutogradContext& other) = delete; + AutogradContext& operator=(const AutogradContext& other) = delete; + AutogradContext(AutogradContext&& other) = delete; + AutogradContext& operator=(AutogradContext&& other) = delete; + ~AutogradContext() = default; + + AutogradContext(PackedArgs& packed_args); + + /// Can be used to save non-variable data for `backward`. + ska::flat_hash_map saved_data; + + /// Saves the list of variables for a future call to `backward`. This + /// should be called at most once from inside of `forward`. + void save_for_backward(variable_list to_save); + /// Marks variables in the list as modified in an in-place operation. This + /// should be called at most once from inside of `forward` and all arguments + /// should be inputs. + void mark_dirty(const variable_list& inputs); + /// Marks outputs in the list as not requiring gradients. This should be + /// called at most once from inside of `forward` and all arguments should be + /// outputs. + void mark_non_differentiable(const variable_list& outputs); + // Sets whether undefined output grad tensors should be expanded to tensors + // full of zeros before calling backward function. Default value is true. + void set_materialize_grads(bool value); + + /// Get the list of variables that were saved in `forward` using + /// `save_for_backward()`. Before returning them to the user, a check is made + /// to ensure that they were not modified by any in-place operations. + variable_list get_saved_variables() const; + const std::unordered_set& get_and_bump_dirty() const; + const std::unordered_set& get_non_differentiable() const; + + /// Expose the Node's `task_should_compute_output` method to the cpp + /// custom autograd Function as `needs_input_grad`. + bool needs_input_grad(size_t output_edge_index) const; + bool needs_input_grad(std::initializer_list idxs) const; + + private: + std::unordered_set non_differentiable_; + std::unordered_set dirty_inputs_; + std::vector saved_variables_; + variable_list to_save_; + bool materialize_grads_{true}; + + // The CppNode in the autograd graph that owns this AutogradContext. We need a + // weak_ptr to avoid a refcycle. Since grad_fn_ owns this AutogradContext, it + // will always be alive when we want to use it. + c10::weak_intrusive_ptr grad_fn_{c10::intrusive_ptr()}; + bool has_freed_buffers_{false}; + + // Compiled autograd overrides saved_variables() and needs_input_grad(). + // We store the values we want to return here. + std::optional saved_variables_override_; + std::optional> needs_input_grad_override_; + + void save_variables(); + + template + friend struct CppNode; + template + friend variable_list CppNode_apply_functional( + variable_list&& inputs, + AutogradContext& ctx_, + const std::vector& is_variable_input_, + const std::vector& output_info_, + const std::string& name); +}; + +template +inline variable_list CppNode_apply_functional( + // NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved) + variable_list&& inputs, + AutogradContext& ctx_, + const std::vector& is_variable_input_, + const std::vector& output_info_, + const std::string& name) { + at::OptionalDeviceGuard _device_guard; + + auto num_inputs = inputs.size(); + variable_list backward_inputs; + backward_inputs.reserve(num_inputs); + for (const auto i : c10::irange(num_inputs)) { + if (inputs[i].defined() || !ctx_.materialize_grads_) { + backward_inputs.emplace_back(std::move(inputs[i])); + } else { + backward_inputs.emplace_back(output_info_[i].zeros(_device_guard)); + } + } + + auto outputs = T::backward(&ctx_, backward_inputs); + + const auto num_forward_inputs = + static_cast(is_variable_input_.size()); + auto num_outputs = static_cast(outputs.size()); + // Returning too many results is ok, but only as long as they're all + // undefined. Truncate the result vector in that case. + if (num_outputs > num_forward_inputs) { + bool all_undef = true; + for (const auto i : c10::irange(num_forward_inputs, num_outputs)) { + all_undef &= (!outputs[i].defined()); + } + if (all_undef) { + outputs.resize(num_forward_inputs); + num_outputs = num_forward_inputs; + } + } + + TORCH_CHECK( + num_outputs == num_forward_inputs, + "function ", + name, + " returned an incorrect number of gradients (expected ", + num_forward_inputs, + ", got ", + num_outputs, + ")"); + + variable_list results; + results.reserve(num_outputs); + for (const auto i : c10::irange(num_outputs)) { + if (!is_variable_input_[i]) { + TORCH_CHECK( + outputs[i].defined() == false, + "function ", + name, + " returned a gradient different that is defined at position ", + i + 1, + ", std the corresponding forward input was not a Variable"); + continue; + } + results.emplace_back(outputs[i]); + } + + return results; +} + +template +inline variable_list CppNode_apply_functional_ivalue( + const variable_list& inputs, + const std::vector& args) { + auto packed_args = PackedArgs(args); + auto ctx = AutogradContext(packed_args); + auto output_info = packed_args.unpack>(); + auto is_variable_input = packed_args.unpack>(); + auto name = packed_args.unpack(); + return CppNode_apply_functional( + variable_list(inputs), ctx, is_variable_input, output_info, name); +} + +// CppNode is the Node in the autograd graph that represents the user defined +// backward function for Function. Calls to CppNode::apply are forward to +// T::backward(). +template +struct CppNode : public Node { + variable_list apply(variable_list&& inputs) override; + AutogradContext ctx_; + std::vector is_variable_input_; + std::vector input_info_; + std::vector output_info_; + + void release_variables() override; + void release_resources() override; + + void set_ctx_grad_fn(c10::intrusive_ptr node); + void save_variables_to_ctx(); + + void compiled_args(CompiledNodeArgs& args) const override { + // although neither of the 2 methods below have uniqueness guarantees + // it is unlikely for them to collide at the same time + args.collect(static_cast(typeid(T).hash_code())); + args.collect(std::string(typeid(T).name())); + + args.collect(ctx_.saved_data); + TORCH_INTERNAL_ASSERT(ctx_.non_differentiable_.empty()); + TORCH_INTERNAL_ASSERT(ctx_.dirty_inputs_.empty()); + args.collect( + ctx_.saved_variables_, true); // always unpacked as output in eager + TORCH_INTERNAL_ASSERT(ctx_.to_save_.empty()); + args.collect(ctx_.materialize_grads_); + args.collect(ctx_.has_freed_buffers_); + args.collect(is_variable_input_); + args.collect(input_info_); + args.collect(output_info_); + } + + variable_list apply_with_saved( + const variable_list& inputs, + SwapSavedVariables& saved) override { + saved.before(ctx_.saved_data); + TORCH_INTERNAL_ASSERT(ctx_.non_differentiable_.empty()); + TORCH_INTERNAL_ASSERT(ctx_.dirty_inputs_.empty()); + saved.before(ctx_.saved_variables_); + TORCH_INTERNAL_ASSERT(ctx_.to_save_.empty()); + saved.before(ctx_.materialize_grads_); + saved.before(ctx_.has_freed_buffers_); + saved.before(input_info_); + saved.before(output_info_); + + PackedArgs packed_args; + packed_args.pack_saved_data(ctx_.saved_data); + variable_list saved_variables = ctx_.get_saved_variables(); + packed_args.pack(saved_variables); + packed_args.pack(ctx_.materialize_grads_); + packed_args.pack(ctx_.has_freed_buffers_); + + std::vector needs_input_grad; + { + auto ptr = ctx_.grad_fn_.lock(); + TORCH_INTERNAL_ASSERT(ptr); + for (const auto i : c10::irange(ptr->next_edges().size())) { + needs_input_grad.push_back(ptr->task_should_compute_output(i)); + } + } + packed_args.pack(needs_input_grad); + + packed_args.pack(output_info_); + packed_args.pack(is_variable_input_); + packed_args.pack(name()); + auto args = std::move(packed_args).vec(); + + auto output_metadata = torch::dynamo::autograd:: + IValuePacker>>::pack( + torch::dynamo::autograd::get_input_metadata(next_edges())); + + const auto& pyinterface = torch::dynamo::autograd::getPyCompilerInterface(); + + // Each time apply_with_saved is called, we bind a new function to Python. + // This is because the schema might be different on compiled autograd cache + // misses. An alternative is to pass the schema to Python so that it can be + // an input to a function, but the schema can't be put into an FX graph + // right now. + std::vector schema; + schema.reserve(args.size()); + for (const auto& ivalue : args) { + if (ivalue.isTensor()) { + schema.emplace_back(at::TensorType::get()); + } else { + schema.emplace_back(ivalue.type()); + } + } + static_assert( + std::is_same_v, bool>); + auto fn_name = pyinterface->bind_function( + saved.get_py_compiler(), + std::string(typeid(T).name()), + CppNode_apply_functional_ivalue, + schema, + /*is_custom_function*/ true, + /*is_traceable*/ T::is_traceable); + + auto results = pyinterface->call_function( + saved.get_py_compiler(), + "apply_functional", + fn_name, + inputs, + args, + output_metadata); + + saved.after(ctx_.saved_data); + TORCH_INTERNAL_ASSERT(ctx_.non_differentiable_.empty()); + TORCH_INTERNAL_ASSERT(ctx_.dirty_inputs_.empty()); + saved.after(ctx_.saved_variables_); + TORCH_INTERNAL_ASSERT(ctx_.to_save_.empty()); + saved.after(ctx_.materialize_grads_); + saved.after(ctx_.has_freed_buffers_); + saved.after(input_info_); + saved.after(output_info_); + return results; + } +}; + +struct ExtractVariables : IterArgs { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + std::vector& is_var_; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + variable_list& list_; + ExtractVariables(std::vector& is_var, variable_list& list) + : is_var_(is_var), list_(list) {} + void operator()(const std::optional& x) { + if (x.has_value() && x.value().defined()) { + is_var_.push_back(true); + list_.emplace_back(x.value()); + } else { + is_var_.push_back(false); + } + } + void operator()(const at::Tensor& x) { + is_var_.push_back(true); + list_.emplace_back(x); + } + void operator()(const at::TensorList& list) { + for (const at::Tensor& x : list) { + is_var_.push_back(true); + list_.emplace_back(x); + } + } + template + void operator()(const T& x) { + is_var_.push_back(false); + } +}; + +template +inline void extract_vars( + std::vector& is_var, + variable_list& list, + Args&&... args) { + ExtractVariables(is_var, list).apply(std::forward(args)...); +} + +template +std::enable_if_t, T> to_output_type( + std::vector>& output_list) { + variable_list result; + std::transform( + output_list.begin(), + output_list.end(), + std::back_inserter(result), + [](const std::optional& var) { return *var; }); + return result; +} + +template +std::enable_if_t, T> to_output_type( + std::vector>& output_list) { + return *output_list[0]; +} + +inline std::vector> to_optional(Variable& output) { + return std::vector>{output}; +} + +inline std::vector> to_optional(variable_list& output) { + std::vector> result; + std::transform( + output.begin(), + output.end(), + std::back_inserter(result), + [](const Variable& var) { return var; }); + return result; +} + +template +template +auto Function::apply(Args&&... args) + -> std::enable_if_t, forward_t> { + const auto& functorch_tls = at::functorch::functorchTLSAccessor(); + if (functorch_tls) { + // Function support for functorch is handled in Python. + // Here we are dealing with a (C++) Function, which is not supported. + // Let's raise an error instead of being silently incorrect. + functorch_tls->checkSupportsCppAutogradFunction(); + } + + auto node = c10::make_intrusive>(); + variable_list input_vars; + + const size_t num_inputs = sizeof...(Args); + input_vars.reserve(num_inputs); + node->is_variable_input_.reserve(num_inputs); + // TODO Add tracing here + extract_vars(node->is_variable_input_, input_vars, args...); + + bool is_executable = + GradMode::is_enabled() && any_variable_requires_grad(input_vars); + auto next_edges = + (is_executable ? collect_next_edges(input_vars) : edge_list()); + node->set_ctx_grad_fn(node); + node->set_next_edges(std::move(next_edges)); + node->clear_input_metadata(); + + node->input_info_.reserve(input_vars.size()); + for (auto& var : input_vars) { + node->input_info_.emplace_back(var); + } + + using forward_return_t = forward_t; + forward_return_t outputs; + { + AutoGradMode grad_mode(false); + outputs = T::forward(&node->ctx_, std::forward(args)...); + } + + _jvp_fn_t jvp_fn = [](const variable_list& inputs, + const variable_list& gI) -> variable_list { + TORCH_CHECK( + false, + "jvp is not implemented for the c++ API of custom Function yet.", + "Please open a feature request on GitHub if you need this."); + }; + + auto view_as_self_fn = [](const at::Tensor& x) -> at::Tensor { + return x.view_as(x); + }; + + auto wrapped_outputs = _wrap_outputs( + input_vars, + node->ctx_.get_non_differentiable(), + node->ctx_.get_and_bump_dirty(), + to_optional(outputs), + is_executable ? node : nullptr, + jvp_fn, + {}, + view_as_self_fn, + false); + + node->output_info_.reserve(wrapped_outputs.size()); + for (auto& output : wrapped_outputs) { + if (is_executable && output.has_value()) { + node->output_info_.emplace_back(output.value()); + } else if (is_executable) { + node->output_info_.emplace_back(); + } + } + + if (is_executable) { + node->save_variables_to_ctx(); + } + + // wrapped_outputs will be a variable_list so, convert it to the correct + // return type. Only Variable and variable_list are accepted as return types. + return to_output_type(wrapped_outputs); +} + +// The logic here is the same as PyNode::apply, so changes to it should be done +// in both the places +template +// NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved) +variable_list CppNode::apply(variable_list&& inputs) { + // Acquire lock to here protect thread safety on custom C++ Autograd Node + // This is needed for the custom Autograd Node since we don't know if the + // user defined Node will write to the shared data during backward. + // see Note [Thread Safety on Autograd Node] + std::lock_guard lock(mutex_); + return CppNode_apply_functional( + std::move(inputs), ctx_, is_variable_input_, output_info_, name()); +} + +template +void CppNode::release_variables() { + // lock to ensure thread safety, see [Thread Safety on Autograd Node] + std::lock_guard lock(mutex_); + ctx_.saved_variables_.clear(); + ctx_.has_freed_buffers_ = true; +} + +template +void CppNode::release_resources() { + Node::release_resources(); + + // AutogradContext deletes copy/move, so destroy and reconstruct in place. + ctx_.~AutogradContext(); + new (&ctx_) AutogradContext(); +} + +template +void CppNode::save_variables_to_ctx() { + ctx_.save_variables(); +} + +template +void CppNode::set_ctx_grad_fn(c10::intrusive_ptr node) { + ctx_.grad_fn_ = std::move(node); +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/edge.h b/western_ai/usa/pytorch/torch/csrc/autograd/edge.h new file mode 100644 index 0000000000000000000000000000000000000000..7b313b58138dd189480b3b7ea81b0fb47493c723 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/edge.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +#include +#include + +namespace torch::autograd { + +struct Node; + +/// Represents a particular input of a function. +struct Edge { + Edge() noexcept : function(nullptr), input_nr(0) {} + + Edge(c10::intrusive_ptr function_, uint32_t input_nr_) noexcept + : function(std::move(function_)), input_nr(input_nr_) {} + + /// Convenience method to test if an edge is valid. + bool is_valid() const noexcept { + return function != nullptr; + } + + // Required for use in associative containers. + bool operator==(const Edge& other) const noexcept { + return this->function == other.function && this->input_nr == other.input_nr; + } + + bool operator!=(const Edge& other) const noexcept { + return !(*this == other); + } + + /// The function this `Edge` points to. + c10::intrusive_ptr function; + + /// The identifier of a particular input to the function. + uint32_t input_nr; +}; +} // namespace torch::autograd + +// The idiomatic way of enabling use of a custom type as the key of hash +// containers in C++11. This method removes the requirement of having to pass +// a custom hasher to std::unordered_{map, set}. +// See http://en.cppreference.com/w/cpp/utility/hash for more information. +namespace std { +template <> +struct hash { + // These type aliases are required by the standard. + using argument_type = torch::autograd::Edge; + using return_type = size_t; + return_type operator()(const argument_type& edge) const noexcept { + return c10::get_hash(edge.function, edge.input_nr); + } +}; +} // namespace std diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/engine.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/engine.cpp new file mode 100644 index 0000000000000000000000000000000000000000..540f964bed0dca92ec679ed03f8fb65d2bf45b03 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/engine.cpp @@ -0,0 +1,1787 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef AT_PER_OPERATOR_HEADERS +#include +#else +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace torch::autograd { + +namespace { +static bool in_bad_autograd_fork = + false; // True for children forked after engine's thread pool init + +// Called in the forked child if engine's thread pool has already been +// initialized +static void forked_autograd_child() { + in_bad_autograd_fork = true; +} + +// Should be called before unsafe for forks (thread pool) calls +static void track_bad_autograd_forks() { +#if !defined(WIN32) + static auto result [[maybe_unused]] = + pthread_atfork(nullptr, nullptr, forked_autograd_child); +#endif +} + +inline bool should_run_in_cpu_ready_queue(c10::DeviceType device) { + if (device == c10::kCPU || device == c10::kMeta || device == c10::kLazy) { + return true; + } else { + return false; + } +} + +std::atomic the_compiled_autograd = nullptr; +#define COMPILED_AUTOGRAD_POISON \ + reinterpret_cast(1) +std::atomic num_threads_in_compiled_autograd; +struct CompiledAutogradThreadingDebugCheck { + CompiledAutogradThreadingDebugCheck() { + num_threads_in_compiled_autograd++; + } + ~CompiledAutogradThreadingDebugCheck() { + release(); + } + void release() { + if (std::exchange(incremented, false)) { + num_threads_in_compiled_autograd--; + } + } + + private: + bool incremented{true}; +}; + +} // namespace + +// Threads spawned by the engine are assigned a 'worker_device' specifying +// what device they process work for. This variable is initialized at: +// 1. thread creation time for CUDA, XLA device threads, as they are +// spinning threads waiting for works on their device. +// 2. before the graph task execution for CPU threads, as for each +// backward call we use the caller thread to drive engine execution. +// This is used when handling reentrant backwards calls; +// See Note [Reentrant backwards] +static thread_local int worker_device = NO_DEVICE; + +// This variable is true if ALL invocations in the stack of re-entrant engine +// invocations are imperative backwards. This special variable is needed for the +// gradient checkpointing feature only. +static thread_local bool checkpoint_valid = true; + +// Number of nested reentrant backwards calls currently on this thread +static thread_local int current_depth = 0; + +// For all device threads (i.e. CUDA, XLA), total_depth represents the total +// nested +// reentrant backwards depths over all device threads. +// For CPU devices, it is the total depth associated with the original backward +// call. +static thread_local int total_depth = 0; + +// The current GraphTask being executed by this thread. This helps +// queue_callback() to find the target GraphTask to append final callbacks. +C10_DEFINE_TLS_static(std::shared_ptr, tls_current_graph_task); +#define current_graph_task (tls_current_graph_task.get()) + +// Every autograd worker thread is associated with a ready queue, which +// specifies the stream of work of this thread to do. This shared_ptr is a +// thread_local pointer to each thread's ready_queue, and it should be +// initialized via the Engine::init_local_ready_queue() call in each +// corresponding thread before execution. +// +// The CUDA, XLA threads are shared among all invocations of backwards via +// device_ready_queues_, while the caller thread is dedicated to processing work +// for devices returning true in should_run_in_cpu_ready_queue (most notably the +// CPU device). So any given graph task maintains its own cpu_ready_queue_ where +// you should send work for it to be done. +// +// For reentrant backward calls, if we spawn new thread from the current thread +// because we reached the maximum depth, the new thread will just reuse the same +// ReadyQueue with the parent thread for performance improvement. +// see Note [Reentrant backwards] for more details. +C10_DEFINE_TLS_static(std::shared_ptr, tls_local_ready_queue); +#define local_ready_queue (tls_local_ready_queue.get()) + +// Note [Reentrant backwards] +// ~~~~~~~~~~~~~~~~~~~~~~~~~~ +// To understand the reentrant backwards problem, we have to notice two +// aspects of how the autograd engine is implemented today: +// +// 1. When you call Engine::execute(), you want to block until +// differentiation finishes so that you can get the final result variables +// of the backwards pass. +// +// 2. The engine operates by having a single worker thread per work queue, +// and every work queue is pinned to a specific device where the +// operation is executed. +// +// The problem is, suppose that you call backward() inside of a worker +// thread. By property (1), we're supposed to block until the nested task +// finishes. However, by property (2), this worker thread is on the +// hook for processing the tasks assigned to it; we better not block, +// because then all of our backward executions (including the one we +// just started) will deadlock! +// +// We maintain a pool of threads waiting for work to do +// When a reentrant backwards call occurs, the current thread blocks +// and a thread from the pool is woken up to complete the blocking tasks and an +// any other tasks that would have been assigned to that worker. If there are no +// threads available, a new thread is spawned. The new thread will continue +// processing tasks from the same ReadyQueue as the parent worker +// +// When the GraphTask is finished, the parent worker thread that is waiting on +// the task is notified and the current thread returns to the pool. + +// Note [Streaming backwards] +// ~~~~~~~~~~~~~~~~~~~~~~~~~~ +// On CUDA/privateuse1 devices the autograd engine's device operations are run +// on the same stream that ran them in forward. This requires automatically +// syncing the streams so that function A finishes producing its +// output before function B consumes it. +// +// This synchronization occurs when outputs are placed into input buffers. +// The functions corresponding to input buffer positions have metadata +// recording their streams from forward, and during backward this +// data is used to sync the producer's stream with the consumer's. +// +// When a CUDA/privateuse1 function is run either all its inputs were +// accumulated on the stream used to run the function OR the inputs are on +// different devices and the function is responsible for properly acquiring +// them. +// +// User-facing stream semantics of a backward() (or torch.autograd.grad()) +// call with respect to surrounding ops are the same as for any other call. +// See "Stream semantics of backward passes" on +// https://pytorch.org/docs/stable/notes/cuda.html +// +// Internally, backward() runs ops (including leaf nodes) on side threads. +// And streams are thread local. So GraphTask achieves the above semantics by +// 1. remembering the current streams on all active CUDA/privateuse1 devices +// in the user-facing thread (aka, the thread that called execute() to +// launch the GraphTask) +// 2. remembering the "leaf streams" (streams each backward leaf node ran on) +// 3. during exec_post_processing, for each leaf stream, sync the remembered +// current streams (on the leaf stream's device) with that +// leaf stream. + +int NodeTask::getReentrantDepth() const { + std::shared_ptr graph_task = base_.lock(); + if (graph_task) { + return graph_task->reentrant_depth_; + } else { + // The graph task is no longer valid indicating an error. As a result, we + // try to move this to the front of the queue to ensure the autograd + // engine threads pick up this error soon. + return std::numeric_limits::max(); + } +} + +CheckpointValidGuard::CheckpointValidGuard( + const std::shared_ptr& graph_task) + : prev_checkpoint_valid_state(checkpoint_valid) { + checkpoint_valid = + graph_task->can_checkpoint() && prev_checkpoint_valid_state; +} + +CheckpointValidGuard::~CheckpointValidGuard() { + checkpoint_valid = prev_checkpoint_valid_state; +} + +auto ReadyQueue::push(NodeTask item, bool incrementOutstandingTasks) -> void { + { + // Lock mutex for writing to heap_ + std::lock_guard lock(mutex_); + if (incrementOutstandingTasks) { + std::shared_ptr graph_task = item.base_.lock(); + TORCH_INTERNAL_ASSERT(graph_task, "GraphTask is no longer valid!"); + ++graph_task->outstanding_tasks_; + } + heap_.push(std::move(item)); + } + not_empty_.notify_one(); +} + +auto ReadyQueue::pushShutdownTask() -> void { + { + std::lock_guard lock(mutex_); + heap_.push(NodeTask({}, nullptr, InputBuffer(0), true)); + } + not_empty_.notify_one(); +} + +size_t ReadyQueue::size() const { + // Lock mutex for accesses to heap_ + std::unique_lock lock(mutex_); + return heap_.size(); +} + +auto ReadyQueue::pop() -> NodeTask { + // Lock mutex for accesses to heap_ + std::unique_lock lock(mutex_); + not_empty_.wait(lock, [this] { return !heap_.empty(); }); + auto task = std::move(const_cast(heap_.top())); + heap_.pop(); + return task; +} + +bool ReadyQueue::empty() const { + // Lock mutex for accesses to heap_ + std::unique_lock lock(mutex_); + return heap_.empty(); +} + +Engine::Engine() : non_reentrant_device_thread_count_(0) {} + +Engine::~Engine() { + stop(); +} + +// Send shutdown tasks to all device_ready_queues_ if no backward tasks are +// running Even though readyQueue should be empty, shutdown tasks have the +// highest priority +void Engine::stop() { + if (stopped_) { + return; + } + stopped_ = true; + // Under some conditions, autograd threads can hang on shutdown + // Do not wait for them to shutdown indefinitely but rely on timeout + auto wait_duration_str = + c10::utils::get_env("TORCH_AUTOGRAD_SHUTDOWN_WAIT_LIMIT"); + auto wait_duration = + wait_duration_str ? std::atof(wait_duration_str->c_str()) : 10.0; + bool noBackward = true; + for (auto& queue : device_ready_queues_) { + noBackward = noBackward && queue->empty(); + } + if (noBackward && wait_duration > 0.0f) { + for (auto& queue : device_ready_queues_) { + queue->pushShutdownTask(); + } + // Do not wait for termination of global threads on Windows + // Because CRT terminates DLL threads before calling + // global object destructors +#if !defined(_WIN32) || defined(C10_USE_MSVC_STATIC_RUNTIME) + + using namespace std::chrono_literals; + // Set a deadline for how long it is OK to wait device threads to shutdown + auto wait_deadline = + std::chrono::steady_clock::now() + wait_duration * 1.0s; + std::unique_lock lk(non_reentrant_device_thread_mutex_); + while (non_reentrant_device_thread_count_.load() != 0) { + if (non_reentrant_device_thread_condvar_.wait_until(lk, wait_deadline) == + std::cv_status::timeout) { + break; + } + } +#endif + } + // Otherwise threads are leaked +} + +void Engine::release_workers() { + std::unique_lock lk(non_reentrant_device_thread_mutex_); + non_reentrant_device_thread_count_.store(0); + non_reentrant_device_thread_condvar_.notify_one(); +} + +void Engine::increment_non_reentrant_thread_count() { + std::unique_lock lk(non_reentrant_device_thread_mutex_); + non_reentrant_device_thread_count_.fetch_add(1); + non_reentrant_device_thread_condvar_.notify_one(); +} + +void Engine::decrement_non_reentrant_thread_count() { + std::unique_lock lk(non_reentrant_device_thread_mutex_); + non_reentrant_device_thread_count_.fetch_sub(1); + non_reentrant_device_thread_condvar_.notify_one(); +} + +void Engine::thread_init( + int device, + const std::shared_ptr& ready_queue, + bool should_increment) { + // pthread_setname_np restricts the name to 16 characters including + // the null byte. + std::string thread_name = "pt_autograd_" + std::to_string(device); + c10::setThreadName(thread_name); + + c10::set_terminate_handler(); + if (should_increment) { + increment_non_reentrant_thread_count(); + } + + at::init_num_threads(); + + // Note [Allocating GPUs to autograd threads] + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // What's our strategy here? Originally, the autograd engine was written + // with only CUDA in mind. We allocate one thread to handle all CPU + // operations, and a thread per CUDA device. + // + // But what if we have OTHER devices? There are two plausible + // strategies: + // + // - We can allocate threads equal to max(num_cuda_devices, num_xla_devices, + // ...) and colocate cuda device 0 with xla device 0 + // - We can allocate threads equal to sum(num_cuda_devices, num_xla_devices, + // ...) keeping everyone separate. + // + // We don't have any good reason to prefer one or the other, so we've + // arbitrarily picked to colocate devices. Maybe the other approach is + // better. + worker_device = device; + + // initialize each device thread's thread local ready queue with the ready + // queue that is created before the thread initialization + init_local_ready_queue(ready_queue); + + std::shared_ptr graph_task = nullptr; + thread_main(graph_task); + if (should_increment) { + // Decrement the count during shutdown if we incremented earlier. + decrement_non_reentrant_thread_count(); + } +} + +GraphTaskGuard::GraphTaskGuard(std::shared_ptr graph_task) + : last_graph_task_(std::move(current_graph_task)) { + current_graph_task = std::move(graph_task); +} +GraphTaskGuard::~GraphTaskGuard() { + restore_current_graph_task(); +} + +void GraphTaskGuard::restore_current_graph_task() { + current_graph_task = std::move(last_graph_task_); +} + +// The current graph task's exec_info is being used to trim unnecessary edges +// during node evaluation, see `Node.task_should_compute_output()` function. +const std::unordered_map* +get_current_graph_task_exec_info() { + return current_graph_task ? ¤t_graph_task->exec_info_ : nullptr; +} + +const std::unordered_set* get_current_graph_task_nodes_in_graph() { + return current_graph_task ? ¤t_graph_task->nodes_in_graph_ : nullptr; +} + +int get_current_graph_task_id() { + return current_graph_task ? current_graph_task->id_ : -1; +} + +bool get_current_graph_task_keep_graph() { + return current_graph_task ? current_graph_task->keep_graph_ : true; +} + +void add_node_to_current_graph_task_exec_info(Node* fn) { + current_graph_task->exec_info_[fn].needed_ = true; +} + +// NB: The engine itself does not use the outputs of this function. +std::vector get_current_graph_task_execution_order() { + std::shared_ptr task = current_graph_task; + if (!task) { + return {}; + } + + // We could potentially check if there is only a single device here + // but explicitly require this context doesn't seem bad either + TORCH_CHECK( + !c10::AutogradState::get_tls_state().get_multithreading_enabled(), + "get_current_graph_task_execution_order expects the current backward to be " + "executed with multithreading disabled, e.g. by running:\n\n" + ">>> with torch.autograd.set_multithreading_enabled(False):\n" + "... torch.autograd.grad(...)\n"); + + const bool check_exec_info = !task->exec_info_.empty(); + std::vector out{}; + // Do a copy since we mutate it later + std::unordered_map dependencies = task->dependencies_; + + auto compare_seq_nr = [](Node* n1, Node* n2) { + return n1->sequence_nr() < n2->sequence_nr(); + }; + std::priority_queue, decltype(compare_seq_nr)> heap( + compare_seq_nr); + + for (Node* ptr : task->graph_roots_) { + heap.push(ptr); + } + + // Implementation notes: + // - We need count dependencies even though we have sequence_nr, because + // in the accumulate_grad case we cannot assume the outputs to have higher + // sequence_nr than the inputs + // - Don't need to check topological_nr because we have exec_info + while (!heap.empty()) { + Node* fn = heap.top(); + heap.pop(); + + out.push_back(fn); + for (const auto& edge : fn->next_edges()) { + Node* next_ptr = edge.function.get(); + if (!next_ptr) { + continue; + } + if (check_exec_info) { + auto it = task->exec_info_.find(next_ptr); + if (it == task->exec_info_.end() || !it->second.should_execute()) { + continue; + } + } + auto it = dependencies.find(edge.function.get()); + TORCH_INTERNAL_ASSERT(it != dependencies.end()); + if (--it->second == 0) { + dependencies.erase(it); + heap.push(next_ptr); + } + } + } + return out; +} + +// NOTE: graph_tasks do not necessarily form a stack. Imagine this +// case: +// +// +----> Eval1 +// Root +// +----> Eval2 +// +// Once Root is executed, both Eval1 and Eval2 are added to the ready queue. +// Next, Eval1 is run and this causes the worker to enter thread_main again. +// Then, it pops the next task from the queue, but at this point it is Eval2. +// It enters thread_main once again, but now with graph_task of Eval2, which is +// completely unrelated to that of Eval1 (it's not a recursive call). +// It's all ok and is handled right now, but it should be accounted for +// in case this code is to be changed. +// +// thread_main is used by: +// 1). autograd threads for devices (i.e. CUDA, XLA) +// 2). the caller/owning thread of the backward call on CPU (sync mode) +// 3). Renetrant backward that invoked by either 1) or 2) +// The exit conditions are different for the above three cases. +// For 1), we are spinning on running the thread_main on device autograd +// threads throughout the Engine lifetime, thread_main will get +// terminated during Engine destruction by pushing shutdown tasks +// For 2), the owning thread of the backward call drives the thread_main +// synchronously until the graph_task of that owning thread is +// completed and exit the thread_main to continue executing the +// result of caller's code. +// For 3), the reentrant backward that invokes +// thread_main, either from 1) or 2), will not spin and will exit as +// long as graph_task is completed and notify the owning thread as +// needed. +auto Engine::thread_main(const std::shared_ptr& graph_task) -> void { + // When graph_task is nullptr, this is a long running thread that processes + // tasks (ex: device threads). When graph_task is non-null (ex: reentrant + // backwards, user thread), this function is expected to exit once that + // graph_task complete. + + // local_ready_queue should already been initialized when we get into + // thread_main + TORCH_INTERNAL_ASSERT(local_ready_queue != nullptr); + while (graph_task == nullptr || !graph_task->future_result_->completed()) { + // local_graph_task represents the graph_task we retrieve from the queue. + // The outer graph_task represents the overall graph_task we need to execute + // for reentrant execution. + std::shared_ptr local_graph_task; + { + // Scope this block of execution since NodeTask is not needed after this + // block and can be deallocated (release any references to grad tensors + // as part of inputs_). + NodeTask task = local_ready_queue->pop(); + // This will only work if the worker is running a non backward task + // TODO Needs to be fixed this to work in all cases + if (task.isShutdownTask_) { + C10_LOG_API_USAGE_ONCE("torch.autograd.thread_shutdown"); + break; + } + + local_graph_task = task.base_.lock(); + if (!local_graph_task) { + // GraphTask for function is no longer valid, skipping further + // execution. + continue; + } + + set_device(worker_device); + + if (task.fn_ && !local_graph_task->has_error_.load()) { + // Set the ThreadLocalState before calling the function. + // NB: The ThreadLocalStateGuard doesn't set the grad_mode because + // GraphTask always saves ThreadLocalState without grad_mode. + at::ThreadLocalStateGuard tls_guard(local_graph_task->thread_locals_); + c10::WarningUtils::WarningHandlerGuard warnings_guard( + &local_graph_task->warning_handler_); + + try { + // The guard sets the thread_local current_graph_task on construction + // and restores it on exit. The current_graph_task variable helps + // queue_callback() to find the target GraphTask to append final + // callbacks. + GraphTaskGuard guard(local_graph_task); + NodeGuard ndguard(task.fn_); + { + RECORD_FUNCTION( + c10::str( + "autograd::engine::evaluate_function: ", + task.fn_.get()->name()), + c10::ArrayRef()); + evaluate_function( + local_graph_task, + task.fn_.get(), + task.inputs_, + local_graph_task->cpu_ready_queue_); + } + } catch (std::exception& e) { + // See Note [ Persisting PyErr state across autograd engine threads ] + thread_on_exception(local_graph_task, task.fn_, e); + } + } + } + + // Decrement the outstanding tasks. + --local_graph_task->outstanding_tasks_; + + // Check if we've completed execution. + if (local_graph_task->completed()) { + local_graph_task->mark_as_completed_and_run_post_processing(); + + auto base_owner = local_graph_task->owner_; + // The current worker thread finish the graph_task, but the owning thread + // of the graph_task might be sleeping on pop() if it does not have work. + // So we need to send a dummy function task to the owning thread just to + // ensure that it's not sleeping, so that we can exit the thread_main. + // If it has work, it might see that graph_task->outstanding_tasks_ == 0 + // before it gets to the task, but it's a no-op anyway. + // + // NB: This is not necessary if the current thread is the owning thread. + if (worker_device != base_owner) { + // Synchronize outstanding_tasks_ with queue mutex + std::atomic_thread_fence(std::memory_order_release); + ready_queue_by_index(local_graph_task->cpu_ready_queue_, base_owner) + ->push(NodeTask(local_graph_task, nullptr, InputBuffer(0))); + } + } + } +} + +// Reentrant call will reuse the graph_task's owner thread ready_queue for +// queueing tasks (NOTE: this is not true in the async_mode of the engine). +// While we can create separate ready queue for each new reentrant +// thread, but sharing the same cpu_ready_queue with parent thread is a +// performance improvement and cuda thread still have to do the same thing. +void Engine::reentrant_thread_init() { + c10::set_terminate_handler(); + at::init_num_threads(); + auto tp_shared = thread_pool_shared_; + while (true) { + std::unique_lock lk(tp_shared->mutex_); + ++thread_pool_shared_->num_workers_; + tp_shared->work_.wait( + lk, [&tp_shared] { return !tp_shared->graphtasks_queue_.empty(); }); + --thread_pool_shared_->num_workers_; + auto task = tp_shared->graphtasks_queue_.front(); + tp_shared->graphtasks_queue_.pop(); + lk.unlock(); + std::shared_ptr graph_task = task.lock(); + if (!graph_task) { + LOG(INFO) << "GraphTask has expired, skipping reentrant execution"; + continue; + } + set_device(graph_task->owner_); + // set the local_ready_queue to the ready queue on the graph_task->owner_ + // device + local_ready_queue = + ready_queue_by_index(graph_task->cpu_ready_queue_, graph_task->owner_); + total_depth = graph_task->reentrant_depth_; + thread_main(graph_task); + } +} + +void Engine::thread_on_exception( + const std::shared_ptr& graph_task, + const c10::intrusive_ptr& fn, + std::exception& e) { + graph_task->set_exception(std::current_exception(), fn); +} + +namespace { +std::atomic graph_task_id{0}; +} + +GraphTask::GraphTask( + bool keep_graph, + bool grad_mode, + int reentrant_depth, + std::shared_ptr cpu_ready_queue, + c10::SmallVector graph_roots, + bool exit_on_error) + : keep_graph_(keep_graph), + graph_roots_(std::move(graph_roots)), + + reentrant_depth_(reentrant_depth), + exit_on_error_(exit_on_error), + cpu_ready_queue_(std::move(cpu_ready_queue)), + future_result_(c10::make_intrusive( + c10::ListType::create(c10::TensorType::get()))), + id_(graph_task_id.fetch_add(1, std::memory_order_relaxed)) { + thread_locals_.set_grad_mode(grad_mode); +} + +bool GraphTask::completed() { + return outstanding_tasks_.load() == 0 || + (exit_on_error_ && has_error_.load()); +} + +void GraphTask::mark_as_completed_and_run_post_processing() { + // Allow only one thread one attempt to process this logic. + if (future_completed_.exchange(true)) { + // Future is already marked complete, or being marked as such. + // In case the marking complete is only in progress, we add a + // wait() to guarantee the future is marked complete on exit. + future_result_->wait(); + return; + } + + try { + // Run post processing, before marking the future as complete. + // Drop lock prior to completing, to avoid holding across callbacks. + std::unique_lock lock(mutex_); + + exec_post_processing(); + std::vector vars = std::move(captured_vars_); + + // Need to unlock before we call markCompleted to avoid holding locks + // when the callbacks are called. + lock.unlock(); + future_result_->markCompleted(vars); + } catch (std::exception&) { + future_result_->setErrorIfNeeded(std::current_exception()); + } +} + +void GraphTask::exec_post_processing() { + TORCH_CHECK( + not_ready_.empty(), "could not compute gradients for some functions"); + + // set the thread_local current_graph_task_ as more callbacks can be installed + // by existing final callbacks. + GraphTaskGuard guard(shared_from_this()); + // Lock mutex during each iteration for accessing final_callbacks.size() + // Unlocking is necessary, because the callback can register + // more callbacks (or they can be registered from other threads + // while it's waiting. + std::unique_lock cb_lock(final_callbacks_lock_); + + // caller_current_streams_ with nullopt entries removed + std::vector caller_current_streams_filtered; + + // See Note [Streaming backwards]. + // Syncs caller_current_stream with leaf streams, so final_callbacks may use + // any grad on its device's current stream. + if (!leaf_streams.empty()) { + for (const auto& leaf_stream : leaf_streams) { + // stash_current_cuda/privateuse1_streams() stashed streams for all device + // IDs that already had a CUDA/privateuse1 context before the GraphTask + // executed. For inactive devices, it stashed a std::nullopt. I don't + // expect GraphTask's backward pass ran leaf nodes on any new devices, so + // the stashed streams should be enough. If leaf_stream.device_index() + // happens to be for a new device, operator* on the std::nullopt should + // throw an error. + const auto& caller_current_stream = + caller_current_streams_[leaf_stream.device_index()]; + + if (caller_current_stream.has_value() && + caller_current_stream != leaf_stream) { + auto event = c10::Event{leaf_stream.device_type()}; + event.record(leaf_stream); + caller_current_stream->wait(event); + } + } + + caller_current_streams_filtered.reserve(caller_current_streams_.size()); + for (const auto& opt_stream : caller_current_streams_) { + if (opt_stream.has_value()) { + caller_current_streams_filtered.push_back(*opt_stream); + } + } + } + + { + // final_callbacks run on the per-device caller_current_streams (the ambient + // streams surrounding the user's call to backward()). This has two + // benefits: + // 1. caller_current_streams have been synced with leaf_streams, so + // callbacks may + // safely access any grad. + // 2. The callback's results can safely be used on (user-facing) + // caller_current_streams + // after backward(). + c10::MultiStreamGuard g(caller_current_streams_filtered); + + // Set the ThreadLocalState before calling the function. + // NB: The ThreadLocalStateGuard doesn't set the grad_mode because GraphTask + // always saves ThreadLocalState without grad_mode. + at::ThreadLocalStateGuard tls_guard(this->thread_locals_); + + // WARNING: Don't use a range-for loop here because more callbacks may be + // added in between callback calls, so iterators may become invalidated. + // NOLINTNEXTLINE(modernize-loop-convert) + for (size_t i = 0; i < final_callbacks_.size(); ++i) { + cb_lock.unlock(); + final_callbacks_[i](); + cb_lock.lock(); + } + } +} + +void GraphTask::set_exception_without_signal( + const c10::intrusive_ptr& fn) { + if (!has_error_.exchange(true)) { + if (AnomalyMode::is_enabled() && fn) { + fn->metadata()->print_stack(fn->name()); + } + } +} + +void GraphTask::set_exception( + std::exception_ptr eptr, + const c10::intrusive_ptr& fn) { + set_exception_without_signal(fn); + if (!future_completed_.exchange(true)) { + future_result_->setError(std::move(eptr)); + } +} + +static variable_list call_pre_hooks(Node& fn, variable_list inputs) { + for (const auto& hook : fn.pre_hooks()) { + inputs = (*hook)(inputs); + } + return inputs; +} + +static variable_list call_tensor_pre_hooks(Node& fn, variable_list inputs) { + for (const auto& hook : fn.tensor_pre_hooks()) { + inputs = (*hook)(inputs); + } + for (const auto& pair : fn.retains_grad_hooks()) { + inputs = (*pair.second)(inputs); + } + return inputs; +} + +static variable_list call_post_hooks( + Node& fn, + variable_list outputs, + const variable_list& inputs, + const bool had_post_hooks) { + for (const auto& hook : fn.post_hooks()) { + if (had_post_hooks) { + outputs = (*hook)(outputs, inputs); + } else { + variable_list null_inputs; + outputs = (*hook)(outputs, null_inputs); + } + } + return outputs; +} + +void set_device(int device) { + // NB: We MUST NOT construct the guard for device CPU, + // as in some settings we compile with cuda, but + // have lazy stubs for CUDA functionality (so actually + // attempting to setup a guard(CPU_DEVICE) will cause an + // error, because it will still query GetDevice). + // + // Don't use DeviceGuard here because its destructor may be called before the + // device is reset. This is fine because the device is thread local. + if (device != CPU_DEVICE) { + for (const auto i : c10::irange(static_cast( + c10::DeviceType::COMPILE_TIME_MAX_DEVICE_TYPES))) { + auto* impl = c10::impl::device_guard_impl_registry[i].load(); + if (impl && device < impl->deviceCount()) { + impl->setDevice(at::Device( + static_cast(i), + static_cast(device))); + } + } + } + worker_device = device; +} + +// validate_outputs has two overloads, one that accepts edge_list and one that +// accepts vector>. The former is stateful (it requires +// the autograd graph to actually use) and the latter is for functional +// autograd. (where we want to be able to take an autograd graph and then +// construct a FX graph out of it without specializing on the properties of the +// gradients). +// +// We do some templating to avoid dynamic allocations in the hot path (the eager +// autograd case). Otherwise, the problem is that we are given a vector +// and would need to materialize a vector> (or some +// other vector) to pass to a common helper function. The alternative is to use +// C++20's ranges which we don't have access to yet. + +// Given an Edge or optional, return the InputMetadata +template +const static InputMetadata& get_input_metadata(const T& thing); + +template <> +const InputMetadata& get_input_metadata>( + const std::optional& thing) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + return thing.value(); +} + +template <> +const InputMetadata& get_input_metadata(const Edge& thing) { + return thing.function->input_metadata(thing.input_nr); +} + +// Given an Edge or optional, return if there is an InputMetadata. +template +static bool has_input_metadata(const T& thing); + +template <> +bool has_input_metadata>( + const std::optional& thing) { + return thing.has_value(); +} + +template <> +bool has_input_metadata(const Edge& thing) { + return thing.is_valid(); +} + +std::vector> collect_input_metadata( + const edge_list& edges) { + std::vector> input_metadata; + for (const auto& edge : edges) { + if (!edge.is_valid()) { + input_metadata.emplace_back(std::nullopt); + continue; + } + input_metadata.emplace_back(edge.function->input_metadata(edge.input_nr)); + } + return input_metadata; +} + +// Given an vector or vector>, validate the +// outputs. This involves using the InputMetadata to check the outputs and also +// potentially calling .sum_to on the outputs. +template +static void validate_outputs_impl( + const std::vector& input_metadata_container, + variable_list& grads, + const std::function& format_error) { + if (grads.size() != input_metadata_container.size()) { + std::stringstream ss; + ss << "invalid number of gradients - expected "; + ss << input_metadata_container.size() << ", but got " << grads.size(); + TORCH_CHECK(false, format_error(std::move(ss).str())); + } + for (const auto i : c10::irange(grads.size())) { + if (!has_input_metadata(input_metadata_container[i])) { + continue; + } + const auto& metadata = get_input_metadata(input_metadata_container[i]); + auto& grad = grads[i]; + if (!grad.defined()) { + // FIXME: TestJit.test_ge_optimized fails this assertion. + // std::stringstream ss; + // ss << "undefined gradient at index " << i; + // TORCH_CHECK(false, format_error(ss.str())); + continue; + } + + grad = metadata.maybe_reduce(i, std::move(grad), format_error); + + bool input_is_complex = + isComplexType(c10::typeMetaToScalarType(metadata.options().dtype())); + bool grad_is_complex = isComplexType(grad.scalar_type()); + + TORCH_CHECK( + isFloatingType(grad.scalar_type()) || + (input_is_complex == grad_is_complex)); + + if (metadata.grad_dtype().has_value()) { + if (grad.scalar_type() != metadata.grad_dtype().value()) { + grad = grad.to(metadata.grad_dtype().value()); + } + if (grad.scalar_type() != metadata.grad_dtype().value()) { + std::stringstream ss; + ss << "invalid gradient at index " << i << " - expected dtype "; + ss << metadata.grad_dtype().value() << " but got " << grad.dtype(); + TORCH_CHECK(false, format_error(std::move(ss).str())); + } + } + if (grad.layout() != metadata.layout()) { + // TODO: Currently we only support (*, Sparse) combination for + // (tensor.layout(), tensor.grad.layout()) In future, there will be an + // opportunity to support more combinations of layouts if they are + // composable (example., operations like addition etc., are well defined + // between tensors of different layouts.), as well as all parts of + // autograd like AccumulateGrad correctly handle this. We allow grad to be + // Strided when metadata is SparseCsr + if (!grad.is_sparse() && + !(grad.layout() == at::kStrided && + (at::sparse_csr::is_sparse_compressed(metadata.layout()) || + metadata.layout() == at::kSparse))) { + std::stringstream ss; + ss << "invalid gradient at index " << i << " - expected layout "; + ss << metadata.layout() << " but got " << grad.layout(); + TORCH_CHECK(false, format_error(std::move(ss).str())); + } + } + + if (grad.device() != metadata.device()) { + if (grad.dim() == 0) { + grad = grad.to(metadata.device()); + } else { + // quick hack for: https://github.com/pytorch/pytorch/issues/65016 but + // should be eventually removed + if (!(metadata.is_tensor_subclass() || + grad.unsafeGetTensorImpl()->is_python_dispatch())) { + std::stringstream ss; + ss << "invalid gradient at index " << i << " - expected device "; + ss << metadata.device() << " but got " << grad.device(); + TORCH_CHECK(false, format_error(std::move(ss).str())); + } + } + } + // We should not build graph for Tensors that are not differentiable + TORCH_INTERNAL_ASSERT(isDifferentiableType(grad.scalar_type())); + } +} + +void validate_outputs( + const edge_list& edges, + variable_list& grads, + const std::function& format_error) { + return validate_outputs_impl(edges, grads, format_error); +} + +void validate_outputs( + const std::vector>& input_metadata, + variable_list& grads, + const std::function& format_error) { + return validate_outputs_impl(input_metadata, grads, format_error); +} + +static variable_list call_function( + std::shared_ptr& graph_task, + Node* func, + InputBuffer& inputBuffer) { + CheckpointValidGuard cpvguard(graph_task); + auto& fn = *func; + auto inputs = + call_tensor_pre_hooks(fn, InputBuffer::variables(std::move(inputBuffer))); + inputs = call_pre_hooks(fn, std::move(inputs)); + if (!graph_task->keep_graph_) { + fn.will_release_variables(); + } + + const auto has_post_hooks = !fn.post_hooks().empty(); + variable_list outputs; + + if (has_post_hooks) { + // In functions/accumulate_grad.cpp, there is some logic to check the + // conditions under which the incoming gradient can be stolen directly + // (which elides a deep copy) instead of cloned. One of these conditions + // is that the incoming gradient's refcount must be 1 (nothing else is + // referencing the same data). Stashing inputs_copy here bumps the + // refcount, so if post hooks are employed, it's actually still ok for + // accumulate_grad.cpp to steal the gradient if the refcount is 2. + // + // "new_grad.use_count() <= 1 + !post_hooks().empty()" in + // accumulate_grad.cpp accounts for this, but also creates a silent + // dependency between engine.cpp (ie, this particular engine + // implementation) and accumulate_grad.cpp. + // + // If you change the logic here, make sure it's compatible with + // accumulate_grad.cpp. + auto inputs_copy = inputs; + outputs = fn(std::move(inputs_copy)); + } else { + outputs = fn(std::move(inputs)); + } + + validate_outputs(fn.next_edges(), outputs, [&](const std::string& msg) { + std::ostringstream ss; + ss << "Function " << fn.name() << " returned an " << msg; + return ss.str(); + }); + + // NOLINTNEXTLINE(bugprone-use-after-move) + return call_post_hooks(fn, std::move(outputs), inputs, has_post_hooks); +} + +void Engine::evaluate_function( + std::shared_ptr& graph_task, + Node* func, + InputBuffer& inputs, + const std::shared_ptr& cpu_ready_queue) { + // The parent stream was cached on the InputBuffer by InputBuffer::add() + // as the consuming node's canonical stream (possibly overridden by the + // stale-capture path when a stale non-capturing node stream collides + // with a capturing producer). Reading the cached value here keeps the + // override decision in one place and avoids re-running the detection + // per node visit. For code paths where InputBuffer::add() was never + // called with an accelerator input (e.g. CPU-only backward), fall back + // to the node's canonical stream. See + // InputBuffer::opt_overridden_consumer_stream for the invariant. + auto opt_parent_stream = inputs.opt_overridden_consumer_stream.has_value() + ? inputs.opt_overridden_consumer_stream + : func->stream(); + + c10::OptionalStreamGuard parent_stream_guard{opt_parent_stream}; + + // Ensure that the incoming gradients are ready + for (size_t pos = 0; pos < inputs.ready_events.size(); ++pos) { + if (!inputs.buffer[pos].defined()) { + continue; + } + const auto device = inputs.buffer[pos].device(); + bool is_accelerator = at::accelerator::isAccelerator(device.type()); + if (!is_accelerator) { + continue; + } + auto& opt_ready_stream = inputs.ready_streams[pos]; + auto& opt_ready_event = inputs.ready_events[pos]; + TORCH_INTERNAL_ASSERT(opt_ready_stream && opt_parent_stream); + if (*opt_parent_stream != *opt_ready_stream) { + TORCH_INTERNAL_ASSERT(opt_ready_event); + opt_parent_stream->wait(opt_ready_event.value()); + } + } + + // If exec_info_ is not empty, we have to instrument the execution + auto& exec_info_ = graph_task->exec_info_; + if (!exec_info_.empty()) { + auto& fn_info = exec_info_.at(func); + variable_list new_inputs = inputs.buffer; + if (!fn_info.needed_) { + // We always want to call tensor pre-hooks, but want to avoid calling it + // twice. needed_ = True indicates that we will call tensor pre-hooks + // later. + // + // See NOTE [Hooks ordering] for more context. + new_inputs = call_tensor_pre_hooks( + *func, InputBuffer::variables(std::move(inputs))); + } + if (auto* capture_vec = fn_info.captures_.get()) { + // Lock mutex for writing to graph_task->captured_vars_. + std::lock_guard lock(graph_task->mutex_); + for (const auto& capture : *capture_vec) { + auto& captured_grad = graph_task->captured_vars_[capture.output_idx_]; + captured_grad = new_inputs[capture.input_idx_]; + // NOTE [Deprecated capture hooks] + for (const auto& hook : + capture.DO_NOT_USE_DEPRECATED_get_capture_hooks()) { + captured_grad = (*hook)(captured_grad); + } + if (opt_parent_stream) { + // No need to take graph_task->mutex_ here, we already hold it + graph_task->leaf_streams.emplace(*opt_parent_stream); + } + } + } + if (!fn_info.needed_) { + // Skip execution if we don't need to execute the function. + return; + } + } + + auto outputs = call_function(graph_task, func, inputs); + + auto& fn = *func; + if (!graph_task->keep_graph_) { + fn.release_variables(); + } + + auto num_outputs = outputs.size(); + if (num_outputs == 0) { // Note: doesn't acquire the mutex + // Records leaf stream (if applicable) + // See Note [Streaming backwards] + if (opt_parent_stream) { + std::lock_guard lock(graph_task->mutex_); + graph_task->leaf_streams.emplace(*opt_parent_stream); + } + return; + } + + if (AnomalyMode::is_enabled() && AnomalyMode::should_check_nan()) { + AutoGradMode grad_mode(false); + for (const auto i : c10::irange(num_outputs)) { + auto& output = outputs[i]; + at::OptionalDeviceGuard guard(device_of(output)); + TORCH_CHECK( + !output.defined() || !isnan(output)._is_any_true().item(), + "Function '", + fn.name(), + "' returned nan values in its ", + i, + "th output."); + } + } + + // Lock mutex for the accesses to GraphTask dependencies_, not_ready_ and + // cpu_ready_queue_ below + std::lock_guard lock(graph_task->mutex_); + for (const auto i : c10::irange(num_outputs)) { + auto& output = outputs[i]; + const auto& next = fn.next_edge(i); + + if (!next.is_valid()) + continue; + + // Check if the next function is ready to be computed + bool is_ready = false; + auto& dependencies = graph_task->dependencies_; + auto it = dependencies.find(next.function.get()); + + if (it == dependencies.end()) { + auto name = next.function->name(); + TORCH_CHECK(false, "dependency not found for ", name); + } else if (--it->second == 0) { + dependencies.erase(it); + is_ready = true; + } + + auto& not_ready = graph_task->not_ready_; + auto not_ready_it = not_ready.find(next.function.get()); + if (not_ready_it == not_ready.end()) { + // Skip functions that aren't supposed to be executed + if (!exec_info_.empty()) { + auto it = exec_info_.find(next.function.get()); + if (it == exec_info_.end() || !it->second.should_execute()) { + continue; + } + } + // No buffers have been allocated for the function + InputBuffer input_buffer(next.function->num_inputs()); + + input_buffer.add( + next.input_nr, + std::move(output), + opt_parent_stream, + next.function->stream(), + next.function.get()); + + if (is_ready) { + auto queue = ready_queue(cpu_ready_queue, next.function->device()); + queue->push( + NodeTask(graph_task, next.function, std::move(input_buffer))); + } else { + not_ready.emplace(next.function.get(), std::move(input_buffer)); + } + } else { + // The function already has a buffer + auto& input_buffer = not_ready_it->second; + + input_buffer.add( + next.input_nr, + std::move(output), + opt_parent_stream, + next.function->stream(), + next.function.get()); + if (is_ready) { + auto queue = ready_queue(cpu_ready_queue, next.function->device()); + queue->push( + NodeTask(graph_task, next.function, std::move(input_buffer))); + not_ready.erase(not_ready_it); + } + } + } +} + +static uint64_t compute_min_topological_nr(const edge_list& outputs) { + // Computes the minimum topological number among all the outputs + if (outputs.empty()) { + return 0; + } + auto min_topo_nr = std::numeric_limits::max(); + for (auto& output_edge : outputs) { + auto topo_nr = output_edge.function->topological_nr(); + min_topo_nr = (min_topo_nr < topo_nr) ? min_topo_nr : topo_nr; + } + return min_topo_nr; +} + +auto Engine::compute_dependencies( + Node* root, + GraphTask& task, + uint64_t min_topo_nr) -> void { + // Computes the number of dependencies for each function which requires grad + std::vector queue{root}; + bool will_use_accelerator = false; + + // Queue contains all nodes that will start propagating gradients. + // We no longer have to expand functions that don't require grad. + auto& dependencies = task.dependencies_; + while (!queue.empty()) { + auto fn = queue.back(); + queue.pop_back(); + if (fn->topological_nr() < min_topo_nr) { + continue; + } + if (!will_use_accelerator) { + will_use_accelerator = fn->stream().has_value(); + } + for (const auto& edge : fn->next_edges()) { + if (auto next_ptr = edge.function.get()) { + dependencies[next_ptr] += 1; + const bool was_inserted = task.nodes_in_graph_.insert(next_ptr).second; + if (was_inserted) + queue.push_back(next_ptr); + } + } + } + + if (will_use_accelerator) { + // Collects current streams for devices where this process has a + // context, so GraphTask::exec_post_processing can sync them with + // leaf_streams. + task.stash_current_streams(); + } +} + +auto Engine::execute( + const edge_list& root_edges, + const variable_list& inputs, + bool keep_graph, + bool create_graph, + bool accumulate_grad, + const edge_list& outputs) -> variable_list { + validate_outputs( + root_edges, + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + const_cast(inputs), + [](const std::string& msg) { return msg; }); + if (accumulate_grad && create_graph) { + TORCH_WARN_ONCE( + "Using backward() with create_graph=True will create a reference cycle " + "between the parameter and its gradient which can cause a memory leak. " + "We recommend using autograd.grad when creating the graph to avoid this. " + "If you have to use this function, make sure to reset the .grad fields of " + "your parameters to None after use to break the cycle and avoid the leak."); + } + + auto compiled_autograd = the_compiled_autograd.load(); + TORCH_INTERNAL_ASSERT(compiled_autograd != COMPILED_AUTOGRAD_POISON); + + // accumulate_grad is true if and only if the frontend call was to + // backward(), not grad(). grad() returns the sum of the gradients + // w.r.t. the inputs and thus needs the inputs to be present. + TORCH_INTERNAL_ASSERT( + accumulate_grad || !outputs.empty(), "grad requires non-empty inputs."); + + // A fresh first time Engine::execute call should start on the CPU device, + // initialize a new thread local ready queue on CPU or reuse the existing one + // (if there is one allocated already, i.e. consecutive backward calls, + // re-entrant backward calls), then memoize the local_ready_queue in GraphTask + init_local_ready_queue(); + bool not_reentrant_backward_call = worker_device == NO_DEVICE; + + // Store root nodes so we can traverse through the graph later + // e.g., for get_current_graph_task_execution_order + c10::SmallVector temp_roots{root_edges.size()}; + for (const auto i : c10::irange(root_edges.size())) { + temp_roots[i] = root_edges[i].function.get(); + } + + auto graph_task = std::make_shared( + /* keep_graph */ keep_graph, + /* grad_mode */ create_graph, + /* reentrant_depth */ not_reentrant_backward_call ? 0 : total_depth + 1, + /* cpu_ready_queue */ local_ready_queue, + /* graph_roots */ std::move(temp_roots)); + + // If we receive a single root, skip creating extra root node + bool skip_dummy_node = root_edges.size() == 1 && compiled_autograd == nullptr; + c10::intrusive_ptr graph_root; + if (skip_dummy_node) { + graph_root = root_edges.at(0).function; + } else { + graph_root = c10::make_intrusive(root_edges, inputs); + } + + auto min_topo_nr = compute_min_topological_nr(outputs); + // Now compute the dependencies for all executable functions + compute_dependencies(graph_root.get(), *graph_task, min_topo_nr); + + if (!outputs.empty()) { + graph_task->init_to_execute( + *graph_root, outputs, accumulate_grad, min_topo_nr); + } + + if (compiled_autograd != nullptr) { + TORCH_CHECK_NOT_IMPLEMENTED( + num_threads_in_compiled_autograd.load() == 0, + "Re-entrant into Compiled Autograd from a parent Compiled Autograd call is not yet supported. Consider disabling Compiled Autograd on the re-entrant call."); + // Allows us to assert no other threads are in backwards + CompiledAutogradThreadingDebugCheck _thread_check; + // see [Note: Compiled Autograd] + _thread_check.release(); + GraphTaskGuard guard(graph_task); + CheckpointValidGuard cpvguard(graph_task); + return (*compiled_autograd)( + graph_root, *graph_task, accumulate_grad, outputs); + } + + // Queue the root + if (skip_dummy_node) { + InputBuffer input_buffer(root_edges.at(0).function->num_inputs()); + auto input = inputs.at(0); + + const auto input_stream = InputMetadata(input).stream(); + auto opt_next_stream = root_edges.at(0).function->stream(); + input_buffer.add( + root_edges.at(0).input_nr, + std::move(input), + input_stream, + opt_next_stream, + root_edges.at(0).function.get()); + + execute_with_graph_task( + graph_task, std::move(graph_root), std::move(input_buffer)); + } else { + execute_with_graph_task( + graph_task, std::move(graph_root), InputBuffer(variable_list())); + } + // Avoid a refcount bump for the Future, since we check for refcount in + // DistEngine (see TORCH_INTERNAL_ASSERT(futureGrads.use_count() == 1) + // in dist_engine.cpp). + auto& fut = graph_task->future_result_; + fut->wait(); + graph_task->warning_handler_.replay_warnings(); + return fut->value().toTensorVector(); +} + +void Engine::initialize_device_threads_pool() { + TORCH_CHECK( + !in_bad_autograd_fork, + "Unable to handle autograd's threading in combination with fork-based multiprocessing. " + "See https://github.com/pytorch/pytorch/wiki/Autograd-and-Fork"); + // Ensures device_ready_queues_ are initialized only once + static bool start_device_threads_flag_ [[maybe_unused]] = [this]() { + this->start_device_threads(); + return true; + }(); +} + +c10::intrusive_ptr Engine::execute_with_graph_task( + const std::shared_ptr& graph_task, + c10::intrusive_ptr graph_root, + InputBuffer&& input_buffer) { + initialize_device_threads_pool(); + // Lock mutex for GraphTask. + std::unique_lock lock(graph_task->mutex_); + + auto queue = ready_queue(graph_task->cpu_ready_queue_, graph_root->device()); + + // worker_device == NO_DEVICE it's a CPU thread and it's trying to drive the + // autograd engine with corresponding GraphTask, and its NOT a re-entrant call + if (worker_device == NO_DEVICE) { + // We set the worker_device to CPU_DEVICE only if worker_device was + // previously NO_DEVICE. Setting it to CPU afterwards allow us to detect + // whether this is a re-entrant call or not. + set_device(CPU_DEVICE); + + // set the graph_task owner to the current device + graph_task->owner_ = worker_device; + + // Now that all the non-thread safe fields of the graph_task have been + // populated, we can enqueue it. + queue->push( + NodeTask(graph_task, std::move(graph_root), std::move(input_buffer))); + + // The owning thread start to drive the engine execution for any CPU task + // that was just pushed or will be added later from other worker threads + lock.unlock(); + thread_main(graph_task); + TORCH_INTERNAL_ASSERT(graph_task->future_result_->completed()); + // reset the worker_device after the completion of the graph_task, this is + // so that the initial state of the engine remains the same across every + // backward() or grad() call, we don't need to reset local_ready_queue as we + // could possibly reuse it for new backward calls. + worker_device = NO_DEVICE; + } else { + // If worker_device is any devices (i.e. CPU, CUDA): this is a re-entrant + // backward call from that device. + graph_task->owner_ = worker_device; + + // Now that all the non-thread safe fields of the graph_task have been + // populated, we can enqueue it. + queue->push( + NodeTask(graph_task, std::move(graph_root), std::move(input_buffer))); + + if (current_depth >= max_recursion_depth_) { + // See Note [Reentrant backwards] + // If reached the max depth, switch to a different thread + add_thread_pool_task(graph_task); + } else { + // Total depth needs to be updated only in this codepath, since it is + // not used in the block above (when we call add_thread_pool_task). + // In the codepath above, GraphTask.reentrant_depth_ is used to + // bootstrap total_depth in the other thread. + ++total_depth; + + // Get back to work while we wait for our new graph_task to + // complete! + ++current_depth; + lock.unlock(); + thread_main(graph_task); + --current_depth; + --total_depth; + + // The graph task should have completed and the associated future should + // be marked completed as well since 'thread_main' above is a call + // blocking an autograd engine thread. + TORCH_INTERNAL_ASSERT(graph_task->future_result_->completed()); + } + } + // graph_task_exec_post_processing is done when the Future is marked as + // completed in mark_as_completed_and_run_post_processing. + return graph_task->future_result_; +} + +// note that when python is present, this base engine will be overridden +// with a PythonEngine. Because this typically happens before get_default_engine +// is called, this base engine will never be created. +Engine& Engine::get_base_engine() { + static Engine engine; + return engine; +} + +static std::atomic engine_stub(Engine::get_base_engine); + +void set_default_engine_stub(EngineStub stub) { + engine_stub.store(stub); +} + +Engine& Engine::get_default_engine() { + return engine_stub.load()(); +} + +void Engine::set_compiled_autograd(Engine::compiled_autograd_fn fn) { + if (the_compiled_autograd.load() == fn) { + return; + } + auto prior = the_compiled_autograd.exchange(COMPILED_AUTOGRAD_POISON); + TORCH_CHECK( + prior != COMPILED_AUTOGRAD_POISON, + "compiled_autograd._enable() does not support multiple Python threads"); + the_compiled_autograd.store(fn); +} + +void Engine::queue_callback(std::function callback) { + TORCH_CHECK( + current_graph_task, + "Final callbacks can only be installed during backward pass."); + + std::lock_guard lock(current_graph_task->final_callbacks_lock_); + current_graph_task->final_callbacks_.emplace_back(std::move(callback)); +} + +bool Engine::is_checkpoint_valid() { + return checkpoint_valid; +} + +void Engine::init_local_ready_queue(std::shared_ptr ready_queue) { + if (ready_queue) { + // if ready_queue provided in the caller, use the caller's ready_queue to + // initialize local_ready_queue + local_ready_queue = std::move(ready_queue); + } else if (!local_ready_queue) { + // otherwise if local_ready_queue not allocated, allocate a new ready_queue + local_ready_queue = std::make_shared(); + } +} + +// CPU ready queue is per GraphTask, but CUDA device ready queues are shared +// across all graph tasks +auto Engine::ready_queue( + std::shared_ptr cpu_ready_queue, + at::Device device) -> std::shared_ptr { + bool multithreading_disabled = + !c10::AutogradState::get_tls_state().get_multithreading_enabled(); + if (multithreading_disabled || should_run_in_cpu_ready_queue(device.type())) { + // return the cpu ready queue passed in + TORCH_INTERNAL_ASSERT(cpu_ready_queue); + return cpu_ready_queue; + } else { + TORCH_INTERNAL_ASSERT( + 0 <= device.index() && + device.index() < + static_cast(device_ready_queues_.size())); + // See Note [Allocating GPUs to autograd threads] + return device_ready_queues_.at(device.index()); + } +} + +auto Engine::ready_queue_by_index( + std::shared_ptr cpu_ready_queue, + int device_index) -> std::shared_ptr { + if (device_index == CPU_DEVICE) { + // return the cpu ready queue passed in + TORCH_INTERNAL_ASSERT(cpu_ready_queue); + return cpu_ready_queue; + } else { + TORCH_INTERNAL_ASSERT( + 0 <= device_index && + device_index < + static_cast(device_ready_queues_.size())); + // See Note [Allocating GPUs to autograd threads] + // NB: This function would become obsolete if we truly allocated a CPU + // thread per device, rather than colocate. + return device_ready_queues_.at(device_index); + } +} + +auto Engine::start_device_threads() -> void { + // First always initialize the thread pool for re-entrant threads + thread_pool_shared_ = std::make_shared(); + + // Second, create special threads for each non-CPU device + // See Note [Allocating GPUs to autograd threads] + c10::DeviceIndex num_devices = 0; + for (const auto& impl_atomic : c10::impl::device_guard_impl_registry) { + auto* impl = impl_atomic.load(); + // Only record the number of devices for device that don't run on the + // cpu ready queue. + if (impl && !should_run_in_cpu_ready_queue(impl->type())) { + num_devices = std::max(num_devices, impl->deviceCount()); + } + } + + // If there are no device except cpu, no need to create worker threads + if (num_devices == 0) { + return; + } + + // Since we're about to create threads, forking is not possible anymore + track_bad_autograd_forks(); + + // allocate one thread for every GPU device (but colocate GPUs of different + // types), and pre-allocate the device_ready_queues_ to ensure safe reading on + // it. + device_ready_queues_ = std::vector>(num_devices); + for (auto& queue : device_ready_queues_) { + queue = std::make_shared(); + } + + for (const auto i : c10::irange(num_devices)) { + std::thread t(&Engine::thread_init, this, i, device_ready_queues_[i], true); + t.detach(); + } + // Wait for the threads to start + { + std::unique_lock lk(non_reentrant_device_thread_mutex_); + while (non_reentrant_device_thread_count_.load() != + static_cast(num_devices)) { + non_reentrant_device_thread_condvar_.wait(lk); + } + } +} + +void Engine::add_thread_pool_task(const std::weak_ptr& graph_task) { + std::unique_lock lck(thread_pool_shared_->mutex_); + // There may already be some items on the graphtasks_queue_ added by other + // threads but not enough workers to get to the new task that will be + // added + bool create_thread = + (thread_pool_shared_->num_workers_ <= + thread_pool_shared_->graphtasks_queue_.size()); + thread_pool_shared_->graphtasks_queue_.push(graph_task); + // Don't need to be holding the lock while actually creating the thread + lck.unlock(); + if (create_thread) { + // If we're creating a new thread, forking is not allowed anymore + track_bad_autograd_forks(); + std::thread t(&Engine::reentrant_thread_init, this); + t.detach(); + } + // This works even if new thread is created because wait() will test the + // predicate before waiting + thread_pool_shared_->work_.notify_one(); +} + +// Remembers current streams on all devices where a context has been created for +// This function assumes the accelerator device is available. +void GraphTask::stash_current_streams() { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + const auto accelerator = at::getAccelerator(true).value(); + const auto guard = c10::impl::VirtualGuardImpl{accelerator}; + auto num_devices = guard.deviceCount(); + caller_current_streams_.resize(num_devices); + if (num_devices > 0) { + for (c10::DeviceIndex idx = 0; idx < num_devices; idx++) { + if (at::globalContext().getAcceleratorHooksInterface().hasPrimaryContext( + idx)) { + caller_current_streams_[idx] = guard.getStream({accelerator, idx}); + } else { + caller_current_streams_[idx] = std::nullopt; + } + } + } +} + +void GraphTask::init_to_execute( + Node& graph_root, + const edge_list& outputs, + bool accumulate_grad, + uint64_t min_topo_nr) { + // Populates exec_info so nodes that should be executed have + // `exec_info[node].needed_ = true` Only nodes that have a path to any edge in + // `outputs` should be executed. The code below populates exec_info using + // recursion, but the actual code does this iteratively. Refer to the + // numbering to see how the actual code corresponds. A difference to note is + // that in the iterative version, when you are working with the current Node, + // you are responsible to update your parent's is_needed after all your + // children have been updated. + // + // is_needed = {fn: True for fn in outputs} # (0) + // seen = {} + // def compute_is_needed(fn): + // for next_edge in fn.next_edges: + // child_fn = next_edge.fn + // if child_fn in seen and is_needed[child_fn]: # (1) + // is_needed[fn] = true + // else: + // seen.add(child_fn) + // if compute_is_needed(child_fn): + // is_needed[fn] = true # (2) + // # (3) exit for-loop + // return is_needed[fn] + // compute_is_needed(graph_root) + // + // NB: you might be wondering why we don't populate `seen` with outputs. We + // cannot because in the case where two outputs lie on the same path, we still + // need to explore past the first output or we would miss the nodes that are + // required to compute the second output. + int output_idx = 0; + for (auto& output_edge : outputs) { + // (0) `is_needed` above corresponds to `exec_info_[fn].needed_` + Node* output = output_edge.function.get(); + auto& info = exec_info_[output]; + if (accumulate_grad) { + // if called through `.backward()` we directly set `needed_` for all the + // outputs to true + info.needed_ = true; + } else { + // otherwise it is `.grad()` and we set exec_info[fn].captures_ instead + // In terms of populating the rest of exec_info though, you can basically + // think of this as the same as setting `needed_` is true directly. + if (!info.captures_) { + info.captures_ = std::make_unique>(); + } + info.captures_->emplace_back(output_edge.input_nr, output_idx++); + } + } + captured_vars_.resize(output_idx); + + struct Frame { + Frame(Node* fn) : fn_(fn) {} + Node* fn_{}; + size_t next_next_fn_{}; + + Node* get_next_fn() { + const auto& next = fn_->next_edges(); + auto num_next = next.size(); + while (next_next_fn_ < num_next) { + auto fn = next[next_next_fn_++].function.get(); + if (fn) + return fn; + } + return nullptr; + } + }; + + auto nodeShouldExecute = [this](Node* fn) { + auto it = exec_info_.find(fn); + return it != exec_info_.end() && it->second.should_execute(); + }; + + std::vector stack; + std::unordered_set seen; + stack.emplace_back(&graph_root); + exec_info_.emplace(stack.back().fn_, ExecInfo()); + + while (!stack.empty()) { + auto& frame = stack.back(); + const auto fn = frame.fn_; + + Node* child_fn = nullptr; + while ((child_fn = frame.get_next_fn()) && !seen.emplace(child_fn).second) { + // (1) next child exists AND has already been seen + if (nodeShouldExecute(child_fn)) { + exec_info_[fn].needed_ = true; + } + } + + if (child_fn) { + // (2) next child exists but has not been seen + if (child_fn->topological_nr() < min_topo_nr) { + // child created before the first output means this child cannot have + // an edge to output + continue; + } + stack.emplace_back(child_fn); + } else { + // (3) no next child exists for `fn` means its `needed` has already been + // finalized. pop stack and update parent + stack.pop_back(); + if (nodeShouldExecute(fn) && !stack.empty()) { + exec_info_[stack.back().fn_].needed_ = true; + } + } + } +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/engine.h b/western_ai/usa/pytorch/torch/csrc/autograd/engine.h new file mode 100644 index 0000000000000000000000000000000000000000..9efc6d54cc60afeedc8a94585ee513ab7e0f0f01 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/engine.h @@ -0,0 +1,289 @@ +#pragma once + +// Engine implements backpropagation from output variables and their gradients +// to "root" variables (variables created by the user with requires_grad=True). + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace torch::autograd { +struct ReadyQueue; +} + +namespace torch::autograd { + +// Maximum reentrant backward depth before switching to a new thread +// This limit is based on the TSAN's deadlock detector, where it will +// fail if a program hold more than 65 locks in one thread at once. +// As we hold mutex in every of our custom C++ autograd Node, we would +// like to avoid TSAN complains on this when doing reentrant backwards +// For reference, see https://github.com/google/sanitizers/issues/950 +static constexpr int MAX_DEPTH = 60; + +void set_device(int device); +TORCH_API void validate_outputs( + const edge_list& edges, + variable_list& grads, + const std::function& format_error); +TORCH_API void validate_outputs( + const std::vector>& input_metadata, + variable_list& grads, + const std::function& format_error); +TORCH_API std::vector> collect_input_metadata( + const edge_list& edges); + +struct NodeTask { + std::weak_ptr base_; + c10::intrusive_ptr fn_; + // This buffer serves as an implicit "addition" node for all of the + // gradients flowing here. Once all the dependencies are finished, we + // use the contents of this buffer to run the function. + InputBuffer inputs_; + // When worker receives a task with isShutdownTask = true, it will immediately + // exit. The engine sends a shutdown task to every queue upon its destruction. + bool isShutdownTask_; + + int getReentrantDepth() const; + + NodeTask( + std::weak_ptr base, + c10::intrusive_ptr fn, + InputBuffer inputs, + bool isShutdownTask = false) + : base_(std::move(base)), + fn_(std::move(fn)), + inputs_(std::move(inputs)), + isShutdownTask_(isShutdownTask) {} +}; + +// Guard that sets and restores checkpoint_valid +class CheckpointValidGuard { + public: + explicit CheckpointValidGuard( + const std::shared_ptr& graph_task); + ~CheckpointValidGuard(); + + private: + bool prev_checkpoint_valid_state; +}; + +struct ReadyQueue { + private: + // Returns true when t2 should be (weakly) BEFORE t1 in the queue. + // Shutdown tasks are first and then empty NodeTask are next. + struct CompareNodeTaskTime { + bool operator()(NodeTask const& t1, NodeTask const& t2) { + // NOLINTNEXTLINE(bugprone-branch-clone) + if (t2.isShutdownTask_) { + return true; + } else if (!t1.fn_ || t1.isShutdownTask_) { + return false; + } else if (!t2.fn_) { + return true; + } else if (t1.getReentrantDepth() == t2.getReentrantDepth()) { + return t1.fn_->sequence_nr() < t2.fn_->sequence_nr(); + } else { + return t1.getReentrantDepth() < t2.getReentrantDepth(); + } + } + }; + + // To notify threads waiting on the ReadyQueue of available tasks on the heap_ + std::condition_variable not_empty_; + // To protect read and writes to heap_ + mutable std::mutex mutex_; + + std::priority_queue, CompareNodeTaskTime> + heap_; + + public: + // incrementOutstandingTasks indicates whether or not we should increment + // 'outstanding_tasks_' for the associated GraphTask. This should mostly + // always be true and is only set false in certain cases (see docs for + // DistEngine.execute_graph_task_until_ready_queue_empty) + void push(NodeTask item, bool incrementOutstandingTasks = true); + void pushShutdownTask(); + NodeTask pop(); + bool empty() const; + size_t size() const; +}; + +// A single instance of this struct should be created through the whole process +// lifetime. The worker thread creation logic and Engine's destructor rely on +// this. +struct TORCH_API Engine { + /// Returns a reference to a static `Engine` instance. + static Engine& get_default_engine(); + + static Engine& get_base_engine(); + + // compiled_autograd needs to live in a different .so file so that it + // can have python symbols, so we add a layer of indirection + // see [Note: Compiled Autograd] + typedef variable_list (*compiled_autograd_fn)( + const c10::intrusive_ptr& graph_root, + const GraphTask& graph_task, + bool accumulate_grad, + const edge_list& outputs); + static void set_compiled_autograd(compiled_autograd_fn fn); + + Engine(const Engine&) = delete; + Engine(Engine&&) = delete; + virtual ~Engine(); + + // Given a list of (Node, input number) pairs computes the value of the graph + // by following next_edge references. + virtual variable_list execute( + const edge_list& roots, + const variable_list& inputs, + bool keep_graph, + bool create_graph, + bool accumulate_grad, + const edge_list& outputs = {}); + + // Given a pre-populated GraphTask and GraphRoot, computes the backward pass + // for the graph. + // + // NB: This API should only be used by internal autograd specific + // machinery and shouldn't be exposed to users in anyway. + virtual c10::intrusive_ptr execute_with_graph_task( + const std::shared_ptr& graph_task, + c10::intrusive_ptr graph_root, + InputBuffer&& input_buffer); + + virtual std::unique_ptr make_anomaly_metadata() { + return std::make_unique(); + } + + virtual std::unique_ptr get_default_saved_variable_hooks() { + return nullptr; + } + + // We pass cpu_ready_queue to evaluate_function, so that it knows + // the correct ready queue to push to after a NodeTask is ready + void evaluate_function( + std::shared_ptr& graph_task, + Node* func, + InputBuffer& inputs, + const std::shared_ptr& cpu_ready_queue); + + void initialize_device_threads_pool(); + virtual void thread_on_exception( + const std::shared_ptr& graph_task, + const c10::intrusive_ptr& fn, + std::exception& e); + + void queue_callback(std::function callback); + + bool is_checkpoint_valid(); + + // Should be called after fork to notify that worker threads are gone + void release_workers(); + + // Must be called by subclass before destructing to avoid a data-race-on-vptr. + void stop(); + + // Initializes a device thread for the autograd engine. + virtual void thread_init( + int device, + const std::shared_ptr& ready_queue, + bool should_increment = true); + + protected: + Engine(); + void compute_dependencies(Node* root, GraphTask& task, uint64_t min_topo_nr); + + // initialize the thread local ready queue with the ready queue that is + // created elsewhere (i.e. thread_init, Engine::execute, etc), or create a new + // ready queue if ready_queue is not provided. + void init_local_ready_queue( + std::shared_ptr ready_queue = nullptr); + + std::shared_ptr ready_queue( + std::shared_ptr cpu_ready_queue, + at::Device device); + std::shared_ptr ready_queue_by_index( + std::shared_ptr cpu_ready_queue, + int device_index); + // start device threads (CUDA, XLA, etc.) in Engine, + // note that it does NOT start CPU thread. + void start_device_threads(); + void increment_non_reentrant_thread_count(); + void decrement_non_reentrant_thread_count(); + virtual void thread_main(const std::shared_ptr& task); + void reentrant_thread_init(); + void add_thread_pool_task(const std::weak_ptr& graph_task); + + // Safe to read device_ready_queues_ without synchronization after + // initialization + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::vector> device_ready_queues_; + + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::vector> final_callbacks_; + // To protect reads and writes to final_callbacks_ + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::mutex post_callbacks_lock_; + + // How many nested reentrant calls are allowed until a new thread is used + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + int max_recursion_depth_{MAX_DEPTH}; + + struct ThreadPoolShared { + // Data structures used by the threads for executing reentrant backwards + // tasks. See Note [Reentrant backwards] + // Number of available threads for processing new GraphTasks. + unsigned int num_workers_{0}; + // The threads will wait on work_ to be notified of GraphTasks + std::condition_variable work_; + // To protect reads and writes to graphtask_queue_ and num_workers_ + // and for synchronizing creating new threads when needed + std::mutex mutex_; + // Workers will process the GraphTasks added to this queue. A GraphTask is + // allocated inside Engine::execute and lives for the duration of execute + std::queue> graphtasks_queue_; + + ThreadPoolShared() = default; + }; + + // Temporary workaround until shutting down threads is done + // We need shared ownership of all these objects because the threads are + // leaked when Engine shuts down, so there may be threads waiting on work_ for + // the graphtasks_queue_ to be nonempty. + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes) + std::shared_ptr thread_pool_shared_; + + private: + // Number of non-reentrant threads + std::atomic non_reentrant_device_thread_count_; + // Destructor will wait for non-reentrant threads to finish + std::condition_variable non_reentrant_device_thread_condvar_; + std::mutex non_reentrant_device_thread_mutex_; + // stop() must be called before the destruction path goes down to the base + // class, in order to avoid a data-race-on-vptr. Use this boolean to guard + // whether stop() has already been called, so we can call this in every + // destructor of the class hierarchy. + bool stopped_{false}; +}; + +// allow python_engine to override the default engine when it loads +using EngineStub = Engine& (*)(); +TORCH_API void set_default_engine_stub(EngineStub stub); + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/forward_grad.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/forward_grad.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e99934a49ea81610114118bc24275c1512b42a85 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/forward_grad.cpp @@ -0,0 +1,78 @@ +#include + +namespace torch::autograd { + +namespace { +// See discussion in forward_grad.h for why these are global variables and not +// thread local + +std::mutex all_forward_levels_mutex_; +std::vector> all_forward_levels_; + +const static at::Tensor singleton_undefined_tensor; +} // namespace + +uint64_t ForwardADLevel::get_next_idx() { + std::lock_guard lock(all_forward_levels_mutex_); + auto next_idx = all_forward_levels_.size(); + TORCH_CHECK( + next_idx == 0, "Nested forward mode AD is not supported at the moment"); + all_forward_levels_.push_back(std::make_shared(next_idx)); + return next_idx; +} + +void ForwardADLevel::release_idx(uint64_t idx) { + std::unique_lock lock(all_forward_levels_mutex_); + TORCH_CHECK( + idx + 1 == all_forward_levels_.size(), + "Exiting a forward AD level that is not the " + "last that was created is not support. Ensure they are released in the reverse " + "order they were created."); + TORCH_INTERNAL_ASSERT(!all_forward_levels_.empty()); + // Keep the level alive until we have released the lock + auto lvl = std::move(all_forward_levels_.back()); + all_forward_levels_.pop_back(); + lock.unlock(); +} + +std::shared_ptr ForwardADLevel::get_by_idx(uint64_t idx) { + std::lock_guard lock(all_forward_levels_mutex_); + TORCH_CHECK( + idx < all_forward_levels_.size(), + "Trying to access a forward AD level with an invalid index. " + "This index was either not created or is already deleted."); + return all_forward_levels_[idx]; +} + +std::shared_ptr ForwardADLevel::try_get_by_idx(uint64_t idx) { + std::lock_guard lock(all_forward_levels_mutex_); + if (idx < all_forward_levels_.size()) { + return all_forward_levels_[idx]; + } else { + return nullptr; + } +} + +ForwardADLevel::~ForwardADLevel() { + std::lock_guard lock(mutex_); + auto it = grads_.begin(); + while (it != grads_.end()) { + // Warning this will lock *it mutex + // This is ok as this function is the *only* one to call back into another + // class's method. + (*it)->reset(idx_, /* update_level */ false); + it = grads_.erase(it); + } +} + +const at::Tensor& ForwardGrad::value(uint64_t level) const { + std::lock_guard lock(mutex_); + const auto& it = content_.find(level); + return it == content_.end() ? singleton_undefined_tensor : (*it).second; +} + +const at::Tensor& ForwardGrad::undef_grad() { + return singleton_undefined_tensor; +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/forward_grad.h b/western_ai/usa/pytorch/torch/csrc/autograd/forward_grad.h new file mode 100644 index 0000000000000000000000000000000000000000..52de12ab2e61e4b822179df014f393944fcb1e0d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/forward_grad.h @@ -0,0 +1,210 @@ +#pragma once + +#include +#include + +namespace torch::autograd { + +// [ Using ForwardGrad ] +// ForwardGrad needs to be a shared_ptr to satisfy constraints of its inner +// design. But this shared_ptr must be uniquely associated with the object that +// stores it (as of writing, either AutogradMeta or SavedVariable). This object +// is called the "owning object" in the discussions below. This owning object +// must call `ForwardGrad::clear()` when it is destroyed to ensure that the +// ForwardGrad is properly de-allocated. + +struct ForwardGrad; + +// This file contains two classes that are used to store forward AD gradients +// and ensure that they are scoped properly. Because forward AD runs +// concurrently with the evaluation of the function, we need a mechanism to +// separate different forward AD invocations and be able to compute the right +// gradients. We model such invocations as levels here. The particular scoping +// issue mentioned above has two main drivers: +// - Ensure that we can conveniently use forward AD within a high level API +// without +// leaking the forward AD states outside. +// - Ensure that we can keep the level that we expose to the user API simple +// (an integer +// that represents the nesting depth) while avoiding confusions when the +// level index is reused. + +// The important external APIs from this file are: +// - ForwardADLevel::get_next_idx() that can be used to enter a new level and +// get its index +// - ForwardADLevel::release_idx() that can be used to exit a given level. +// - ForwardGrad() can be used to store a given forward gradient that will +// handle the level +// tracking automatically. + +// The basic implementation strategy is as follows: +// Every tensor has a ForwardGrad, maintaining a map from levels to tangents. +// ForwardGrad is responsible for registering itself to the appropriate +// ForwardADLevel when a new tangent is added to it via ForwardGrad::set_value +// and to un-register itself from this same level if that tangent is removed via +// ForwardGrad::reset. The ForwardADLevel is created when a new level is entered +// via ForwardADLevel::get_next_idx. A reference to the new ForwardADLevel is +// stored into a global (for the whole process) vector that ensure it can be +// accessed via ForwardADLevel::get_by_idx. This reference is deleted when the +// index is released by the user when calling ForwardADLevel::release_idx. When +// it is destructed, the ForwardADLevel is responsible for clearing all the +// tangents for its level stored in all the ForwardGrad that registered with it. +// +// This process-wide level design, compared to a thread local one, allows us to +// use very simple user facing handle for the level (an int) while enabling +// cross-thread forward AD. The only required synchronization for the user is +// when entering and exiting the levels. Some discussion on alternative design +// is in https://github.com/pytorch/pytorch/pull/49097#discussion_r543716453 and +// can be refined in the future. + +// Correctness of concurrency: +// Each class uses its own lock when reading or modifying internal storages. +// This allows in particular to safely remove tangents from ForwardGrad when the +// ForwardADLevel is being exited. We ensure no deadlock by ensuring that a +// methods never calls into another class's method while the local class's lock +// is held except in one single case: calling from ForwardADLevel's destructor +// into ForwardGrad::reset with update_level=false. + +// The lifetime of these objects is as follows: +// The ForwardADLevel can be in three states: +// - Initialized: where one of its reference is held by the global vector +// and there may be more +// references held by temporary variables in ForwardGrad's methods. +// - About to be destructed: where "release_idx" has been called and the +// only reason for the +// ForwardADLevel not to be destructed right away is that some methods in +// ForwardGrad have owning reference to it. This is done so that a +// ForwardADLevel can never be destructed when a ForwardGrad is +// registered with it and in the process of adding something to its +// internal state. +// - Being destructed: Here the ForwardADLevel is not referenced anymore +// and can be safely reset +// all of the ForwardGrad. Note that we can have more than one reset +// being called here (which is ok) but we are guaranteed that there is at +// least one. +// The ForwardGrad is simpler as there is no intermediary state and no special +// destructor for. The logic to unregister it from the different ForwardADLevel +// is done when the owning object (AutogradMeta or SavedVariable) is being +// destroyed. + +// Other considered design: +// To avoid having the ForwardGrad::clear, we considered storing weak_ptr inside +// the ForwardADLevel. While this would work, it would mean that the set inside +// the ForwardADLevel would only grow unless we do an expensive linear scan to +// remove all the dangling weak pointers. Hence this approach was not used. + +// Data structures in this file are optimized for this maximum number of levels. +// The number of levels corresponds to the degree of the gradient being +// computed using forward AD and we don't expect more than second order +// gradients to be common. +#define EXPECTED_MAX_LEVEL 2 + +struct TORCH_API ForwardADLevel { + ForwardADLevel(uint64_t idx) : idx_(idx) {} + ~ForwardADLevel(); + + static uint64_t get_next_idx(); + static void release_idx(uint64_t idx); + static std::shared_ptr get_by_idx(uint64_t idx); + static std::shared_ptr try_get_by_idx(uint64_t idx); + + void erase(const std::shared_ptr& grad) { + std::lock_guard lock(mutex_); + grads_.erase(grad); + } + + void insert(const std::shared_ptr& grad) { + std::lock_guard lock(mutex_); + grads_.insert(grad); + } + + private: + std::unordered_set> grads_; + std::mutex mutex_; + uint64_t idx_; +}; + +struct TORCH_API ForwardGrad : std::enable_shared_from_this { + ForwardGrad() = default; + + // This function must only be called when AutogradMeta or SavedVariable is + // being destructed as it ensures that: + // - The only (potential) other references to this ForwardGrad are the + // different level it is registered to + // - No other thread will try to call `set_value` or `value` ever from now + // on + // - Any of the ForwardADLevel that this ForwardGrad is registered with + // might + // call `reset` at any point during this function + void clear() { + c10::SmallVector levels_idx; + + { + std::lock_guard lock(mutex_); + for (auto& c : content_) { + levels_idx.push_back(c.first); + } + } + + for (auto l_idx : levels_idx) { + // Use "try" version here as another thread might have deleted this + // level before we got here + // This is an owning reference as we want to keep the level alive + // until we successfully unregister ourselves + auto level = ForwardADLevel::try_get_by_idx(l_idx); + if (level) { + level->erase(shared_from_this()); + } + } + } + + void set_value(const at::Tensor& value, uint64_t level) { + // Owning reference to ensure the forward_level is not destroyed + // while we are updating our internal state + auto forward_level = ForwardADLevel::get_by_idx(level); + forward_level->insert(shared_from_this()); + + std::lock_guard lock(mutex_); + content_.insert({level, value}); + } + + // This function removes the tangent for a given level from this ForwardGrad + // Use the update_level flag to disable notifying the level about this reset + // This flag is most notably used by the ForwardADLevel destructor. + void reset(uint64_t level, bool update_level = true) { + if (update_level) { + ForwardADLevel::get_by_idx(level)->erase(shared_from_this()); + } + + std::unique_lock lock(mutex_); + const auto& it = content_.find(level); + TORCH_INTERNAL_ASSERT( + it != content_.end(), "Resetting a non-existent level."); + // Keep the Tensor alive until we have released the lock + // This is needed as we can be in a case where this function is called by + // ForwardADLevel destructor + auto t = (*it).second; + content_.erase(level); + lock.unlock(); + } + + const at::Tensor& value(uint64_t level) const; + + bool contains(uint64_t level) { + std::lock_guard lock(mutex_); + return content_.count(level) > 0; + } + + bool empty() const { + return content_.empty(); + } + + static const at::Tensor& undef_grad(); + + private: + // TODO(albanD): replace this with a SmallVector + std::unordered_map content_; + mutable std::mutex mutex_; +}; + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/function.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/function.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8531ff16e369b33222e29735a2095da85c53a83f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/function.cpp @@ -0,0 +1,137 @@ +#include + +#include +#include +#include + +#include + +#include +#include +#include + +#ifndef AT_PER_OPERATOR_HEADERS +#include +#else +#include +#endif + +namespace torch::autograd { + +// The current evaluating node. This is useful to assign the current node as a +// parent of new nodes created during the evaluation of this node in anomaly +// mode. +C10_DEFINE_TLS_static(c10::intrusive_ptr, tls_current_evaluating_node); +#define current_evaluating_node (tls_current_evaluating_node.get()) + +NodeGuard::NodeGuard(c10::intrusive_ptr node) + : last_evaluating_node_(std::move(current_evaluating_node)) { + current_evaluating_node = std::move(node); +} +NodeGuard::~NodeGuard() { + // restore the previous evaluating node + current_evaluating_node = std::move(last_evaluating_node_); +} + +c10::intrusive_ptr get_current_node() { + return current_evaluating_node; +} + +void Node::assign_parent() { + metadata()->assign_parent(current_evaluating_node); +} + +auto Node::name() const -> std::string { + return c10::demangle(typeid(*this).name()); +} + +auto Node::forward_op_name() const -> std::string { + auto n = name(); + // Strip "Backward" suffix to get the forward op name. + auto pos = n.rfind("Backward"); + if (pos == std::string::npos) { + return n; + } + // Verify everything after "Backward" is digits (e.g., "Backward0"). + auto suffix_start = pos + 8; + for (size_t i = suffix_start; i < n.size(); ++i) { + if (!std::isdigit(static_cast(n[i]))) { + return n; + } + } + // Keep the numeric suffix if it is not "0" (e.g., "AddBackward1" → "Add1"). + auto suffix = n.substr(suffix_start); + if (suffix == "0" || suffix.empty()) { + return n.substr(0, pos); + } + return n.substr(0, pos) + suffix; +} + +bool Node::task_should_compute_output(size_t output_edge_index) const { + TORCH_CHECK(output_edge_index < num_outputs(), "Index out of range"); + const auto& next = next_edges_[output_edge_index]; + if (next.is_valid()) { + const auto exec_info = get_current_graph_task_exec_info(); + if (exec_info && !exec_info->empty()) { + auto it = exec_info->find(next.function.get()); + if (it == exec_info->end() || !it->second.should_execute()) { + return false; + } + } + return true; + } + return false; +} + +AnomalyMetadata* Node::metadata() noexcept { + if (!anomaly_metadata_) { + anomaly_metadata_ = Engine::get_default_engine().make_anomaly_metadata(); + } + return anomaly_metadata_.get(); +} + +// Iteratively release child nodes to prevent stack overflow on deletion +// of deep computation graphs. See +// https://github.com/pytorch/pytorch/issues/5534 +static void gatherFunctions( + Node* func, + std::vector>& stack) { + func->release_variables(); + + for (auto& edge : func->next_edges()) { + if (edge.function.use_count() == 1) { + stack.emplace_back(std::move(edge.function)); + } else { + edge.function.reset(); + } + } +} + +static void releaseGraphIteratively(Node* node) { + std::vector> stack; + gatherFunctions(node, stack); + while (!stack.empty()) { + auto func = std::move(stack.back()); + stack.pop_back(); + gatherFunctions(func.get(), stack); + } +} + +void Node::release_resources() { + releaseGraphIteratively(this); + pre_hooks_.clear(); + post_hooks_.clear(); + tensor_pre_hooks_.clear(); + retains_grad_hooks_.clear(); + anomaly_metadata_.reset(); +} + +Node::~Node() { + releaseGraphIteratively(this); +} + +at::Tensor TypeAndSize::zeros() { + return at::zeros_symint(sym_sizes, options); +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/function.h b/western_ai/usa/pytorch/torch/csrc/autograd/function.h new file mode 100644 index 0000000000000000000000000000000000000000..f97acad3e16b76690ed5211b01688ecb7c0fad88 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/function.h @@ -0,0 +1,89 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace torch::autograd { + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// Associated Free Functions +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +namespace detail { +// Implementation of `collect_next_edges` (see below). +struct MakeNextFunctionList : IterArgs { + edge_list next_edges; + using IterArgs::operator(); + void operator()(const Variable& variable) { + if (variable.defined()) { + next_edges.emplace_back(impl::gradient_edge(variable)); + } else { + next_edges.emplace_back(); + } + } + void operator()(const Variable* variable) { + operator()(*variable); + } + void operator()(const std::optional& variable) { + if (variable.has_value()) { + operator()(*variable); + } else { + next_edges.emplace_back(); + } + } +}; +} // namespace detail + +/// Create an `Edge` between the given `variable` and the `function`, which is +/// assumed to be the gradient function of this variable (i.e. the function +/// through which this variable is backpropagated during the backward pass). +/// This sets the `grad_fn` property of the `variable`. This function assumes +/// that the `Variable` is a new input to the gradient function and its +/// `input_nr` thus equal to `function->num_inputs()`. Additionally, it +/// increments the `Node`'s number of inputs by one. Approximately +/// equivalent to `variable.set_gradient_edge(function, +/// function->add_input_metadata(variable.dispatch_type(), variable.sizes()))`. +/// If you don't want the `Node`'s `num_inputs` to be incremented, use +/// `set_gradient_edge` directly. +inline void create_gradient_edge( + Variable& variable, + c10::intrusive_ptr function) { + // Copy before move. + const auto input_nr = function->add_input_metadata(variable); + impl::set_gradient_edge(variable, {std::move(function), input_nr}); +} + +/// Return true if any of the variables in the list require a gradient. +inline bool any_variable_requires_grad(const variable_list& variables) { + return std::any_of( + variables.begin(), variables.end(), [](const Variable& variable) { + return variable.defined() && variable.requires_grad(); + }); +} + +/// Return the next edges of all the given variables, or tuples of variables. +template +edge_list collect_next_edges(Variables&&... variables) { + detail::MakeNextFunctionList make; + make.apply(std::forward(variables)...); + return std::move(make.next_edges); +} + +struct TypeAndSize { + TypeAndSize() = default; + /* implicit */ + TypeAndSize(const at::Tensor& t) + : sym_sizes(t.sym_sizes().vec()), options(t.options()) {} + + at::Tensor zeros(); + + std::vector sym_sizes; + at::TensorOptions options; +}; + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/function_hook.h b/western_ai/usa/pytorch/torch/csrc/autograd/function_hook.h new file mode 100644 index 0000000000000000000000000000000000000000..0d71ff172807783e6a736198485e6fa625dc73d8 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/function_hook.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include +#include + +namespace torch::dynamo::autograd { +class CompiledNodeArgs; +class SwapSavedVariables; +struct PackedArgs; +} // namespace torch::dynamo::autograd + +// A hook that's called on gradients + +namespace torch::autograd { + +using Variable = at::Tensor; +using variable_list = std::vector; + +struct TORCH_API FunctionPreHook { + virtual ~FunctionPreHook() = default; + virtual variable_list operator()(const variable_list& grads) = 0; + // only implemented for python hooks, registers hook with compiled autograd + virtual void compiled_args( + torch::dynamo::autograd::CompiledNodeArgs& args) const { + TORCH_CHECK_NOT_IMPLEMENTED( + false, + std::string("compiled_args nyi, see [Note: Compiled Autograd] ") + + typeid(*this).name()); + } +}; + +struct TORCH_API FunctionPostHook { + virtual ~FunctionPostHook() = default; + virtual variable_list operator()( + const variable_list& outputs /* grad_inputs */, + const variable_list& inputs /* grad_outputs */) = 0; + // only implemented for python hooks, registers hook with compiled autograd + virtual void compiled_args( + torch::dynamo::autograd::CompiledNodeArgs& args) const { + TORCH_CHECK_NOT_IMPLEMENTED( + false, + std::string("compiled_args nyi, see [Note: Compiled Autograd] ") + + typeid(*this).name()); + } +}; + +struct TORCH_API PostAccumulateGradHook { + virtual ~PostAccumulateGradHook() = default; + virtual void operator()(const Variable& tensor) = 0; + // only implemented for python hooks on nodes, registers hook with compiled + // autograd + virtual void compiled_args( + torch::dynamo::autograd::CompiledNodeArgs& args) const { + TORCH_CHECK_NOT_IMPLEMENTED( + false, + std::string("compiled_args nyi, see [Note: Compiled Autograd] ") + + typeid(*this).name()); + } + + virtual void apply_with_saved( + Variable& /*unused*/, + torch::dynamo::autograd::SwapSavedVariables& /*unused*/) { + TORCH_CHECK_NOT_IMPLEMENTED( + false, + std::string("compiled_args nyi, see [Note: Compiled Autograd] ") + + typeid(*this).name()); + } +}; + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/accumulate_grad.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/functions/accumulate_grad.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5c907dc7e8811f1ae4f55f9935306cdbeeb99855 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/accumulate_grad.cpp @@ -0,0 +1,194 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace torch::autograd { + +using torch::dynamo::autograd::IValuePacker; + +namespace { + +void AccumulateGrad_apply_impl( + variable_list&& grads, + at::Tensor& variable, + at::Tensor& variable_grad, + int64_t num_expected_refs, + const std::function& grad_update, + std::mutex* mutex = nullptr) { + check_input_variables("AccumulateGrad", grads, 1, 0); + + if (!grads[0].defined()) + return; + if (!variable.requires_grad()) + return; + + // std::move(grads[0]) to avoid bumping up refcount + at::Tensor new_grad = std::move(grads[0]); + + // Acquire lock to here protect thread safety on variable, this ensures + // AccumulateGrad does not race to shared variable from different threads + // when updating the gradients. We don't ensure thread safety on hooks + // and rely on user to provide thread safe hooks + // see Note [Thread Safety on Autograd Node] + // need to still lock for eager here + std::optional> lock; + if (mutex != nullptr) { + lock.emplace(*mutex); + } + + AccumulateGrad::accumulateGrad( + variable, variable_grad, new_grad, num_expected_refs, grad_update); +} + +variable_list AccumulateGrad_apply_functional_no_hooks_ivalue( + const variable_list& grads, + const ivalue_list& args) { + PackedArgs r(args); + auto variable = r.unpack(); + auto variable_grad = r.unpack(); + auto has_post_hooks = r.unpack(); + + // Functional Tensors insert an Error node to assert that backward is never + // called + if (variable.grad_fn() && + dynamic_cast(variable.grad_fn().get()) == nullptr) { + throw std::logic_error( + "leaf variable has been moved into the graph interior"); + } + + at::Tensor functional_grad; + AccumulateGrad_apply_impl( + variable_list(grads), + variable, + variable_grad, + 1 + has_post_hooks, + [&functional_grad](at::Tensor&& grad_update) { + functional_grad = std::move(grad_update); + }, + nullptr // no mutex needed since this is executed under a single thread + ); + if (!functional_grad.defined()) { + // In-place accumulation (Case 2.3) does not execute grad_update + functional_grad = std::move(variable_grad); + } + return {functional_grad}; +} +} // namespace + +// AccumulateGrad sets sequence_nr to the max value so it's always called +// ASAP during backwards. +AccumulateGrad::AccumulateGrad(Variable variable_) + : Node(/*sequence_nr=*/UINT64_MAX), variable(std::move(variable_)) { + add_input_metadata(variable); +} + +// NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved) +auto AccumulateGrad::apply(variable_list&& grads) -> variable_list { + if (variable.grad_fn()) { + throw std::logic_error( + "leaf variable has been moved into the graph interior"); + } + + at::Tensor& variable_grad = variable.mutable_grad(); + + // If the function has post hooks (for example, a DDP allreduce hook), + // call_function in Engine.cpp will temporarily bump the expected refcount + // by one, hence the addition of !post_hooks().empty() for + // 'num_expected_refs' in addition to the one reference that we're holding. + // 'num_expected_refs' is used to determine whether or not we should clone + // the grad or can steal the grad. + AccumulateGrad_apply_impl( + std::move(grads), + variable, + variable_grad, + 1 + !post_hooks().empty() /* num_expected_refs */, + [&variable_grad](at::Tensor&& grad_update) { + variable_grad = std::move(grad_update); + }, + &mutex_); + + auto& hook = tensor_post_acc_grad_hooks(); + if (hook != nullptr) { + (*hook)(variable); + } + + return variable_list(); +} + +void AccumulateGrad::compiled_args(CompiledNodeArgs& args) const { + if (args.cond(variable.defined() && variable.requires_grad())) { + args.collect(variable); + args.collect(variable.grad()); + } + args.collect(GradMode::is_enabled()); + const auto& hook = tensor_post_acc_grad_hooks(); + if (hook != nullptr) { + hook->compiled_args(args); + } +} + +variable_list AccumulateGrad::apply_with_saved( + const variable_list& grads, + SwapSavedVariables& saved) { + if (!(variable.defined() && variable.requires_grad()) || + !grads[0].defined()) { + return variable_list(); + } + TORCH_INTERNAL_ASSERT(!variable.grad_fn() && grads.size() == 1); + at::Tensor variable_copy = variable; + at::Tensor grad_copy = variable.grad(); + saved.before(variable_copy); + saved.before(grad_copy); + variable_copy.mutable_grad() = grad_copy; + + // name() includes namespace for historical reasons: + // torch::autograd::AccumulateGrad For Compiled Autograd, we just want the op + // name without the namespace + std::string name = "AccumulateGrad"; + + // Proxy a functional accumulate-grad call that takes the current grad + // explicitly and returns the updated grad. + static bool flag [[maybe_unused]] = [&]() { + std::vector schema = { + IValuePacker::packed_type(), + IValuePacker::packed_type(), + IValuePacker::packed_type()}; + const auto& interface = torch::dynamo::autograd::getPyCompilerInterface(); + interface->bind_function( + saved.get_py_compiler(), + name, + AccumulateGrad_apply_functional_no_hooks_ivalue, + schema); + return true; + }(); + + const auto& interface = torch::dynamo::autograd::getPyCompilerInterface(); + at::Tensor functional_grad = interface->call_accumulate_grad( + saved.get_py_compiler(), + variable_copy, + grad_copy, + grads[0], + !post_hooks().empty()); + variable_copy.mutable_grad() = functional_grad; + + auto& hook = tensor_post_acc_grad_hooks(); + if (hook != nullptr) { + hook->apply_with_saved(variable_copy, saved); + } + saved.after(variable_copy); + saved.after(grad_copy); + + return variable_list(); +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/accumulate_grad.h b/western_ai/usa/pytorch/torch/csrc/autograd/functions/accumulate_grad.h new file mode 100644 index 0000000000000000000000000000000000000000..084522a77994a61066afb1840c3230372b9d97ff --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/accumulate_grad.h @@ -0,0 +1,312 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef AT_PER_OPERATOR_HEADERS +#include +#else +#include +#endif + +#include + +namespace torch::autograd { + +#define CHECK_RESULT(RESULT, VAR) \ + if (c10::AutogradState::get_tls_state() \ + .get_grad_layout_enforcement_enabled() && \ + !(RESULT.is_sparse() || VAR.is_sparse() || RESULT.is_sparse_csr() || \ + VAR.is_sparse_csr())) { \ + if (!utils::obeys_layout_contract(RESULT, VAR)) { \ + TORCH_WARN_ONCE( \ + "grad and param do not obey the gradient layout contract. " \ + "This is not an error, but may impair performance.\n" \ + "grad.sizes() = ", \ + RESULT.sizes(), \ + ", strides() = ", \ + RESULT.strides(), \ + "\n", \ + "param.sizes() = ", \ + VAR.sizes(), \ + ", strides() = ", \ + VAR.strides()); \ + } \ + } + +struct TORCH_API AccumulateGrad : public Node { + explicit AccumulateGrad(Variable variable_); + + variable_list apply(variable_list&& grads) override; + + void release_resources() override { + variable.reset(); + Node::release_resources(); + } + + std::vector>& tensor_pre_hooks() noexcept + override { + // NB: Since the AccumulateGrad Node is only a weak ref from the Tensor, + // it can be destroyed even though the Tensor is still alive (contrary + // to all other Nodes). So we must lazily read the Tensor hooks here. + return impl::hooks(variable); + } + + std::unique_ptr& tensor_post_acc_grad_hooks() + const noexcept override { + // NB: Since the AccumulateGrad Node is only a weak ref from the Tensor, + // it can be destroyed even though the Tensor is still alive (contrary + // to all other Nodes). So we must lazily read the Tensor hooks here. + return impl::post_acc_grad_hooks(variable); + } + + // Note: Gradient Layout Contract + // + // AccumulateGrad tries to stash strided (non-sparse) grads with memory layout + // (strides) such that variables and grads interact efficiently in later + // optimizer kernels, and grads interact efficiently with c10d::Reducer.cpp. + // + // Specifically, AccumulateGrad tries to ensure the following + // (cf torch/csrc/autograd/utils/grad_layout_contract.h): + // (1) if variable.is_non_overlapping_and_dense(), the stashed grad's + // strides match variable. + // (2) else, stashed grad is rowmajor contiguous. + // If variable's grad does not exist (!variable_grad.defined()) + // AccumulateGrad steals new_grad if it's stealable and obeys the contract + // already, otherwise it deep copies new_grad into an obedient clone. + // + // If variable's grad already exists (variable_grad.defined()), new_grad must + // be added to variable_grad. If we aren't setting up for double backward + // (!GradMode::is_enabled()), AccumulateGrad performs "variable_grad += + // new_grad" in-place, which keeps variable_grad's layout. We assume (hope) + // variable_grad was created obeying (1) or (2) at some point in the past. + // + // If we are setting up for double backward, AccumulateGrad updates the grad + // out-of-place via "variable_grad + new_grad." TensorIterator operator+ + // decides result's layout. Typically TensorIterator matches strides of the + // first arg, so we once again assume (hope) variable_grad was originally + // created obeying (1) or (2). + // + // AccumulateGrad does not enforce the contract with 100% certainty. Examples: + // - If a user manually permutes a param or its grad, then runs a fwd+bwd, + // variable_grad += new_grad keeps variable_grad's layout without + // rechecking the contract. + // - If TensorIterator changes its corner cases about operator+'s result + // (for example, giving more or less priority to channels_last inputs, see + // https://github.com/pytorch/pytorch/pull/37968) the result may not obey. + // + // Fortunately, if a given grad doesn't satisfy (1) or (2), the penalty is + // degraded performance in Reducer.cpp or optimizer kernels, not death by + // assert or silently bad numerics. + + // Gradient Accumulation + // Given a variable with its current grad as variable_grad, accumulates + // new_grad into variable_grad if in place accumulation is possible. + // Otherwise, uses 'update_grad' to update the grad for the variable. + // + // Branch breakdown: + // - Case 1: Param has no existing grad + // - Case 1.1: Stealable dense new_grad + // . We aren't setting up for double-backward. + // . No other user-visible tensor references new_grad. + // . new_grad obeys the "Gradient Layout Contract", there has a special + // case, For MKLDNN tensor, which is a opaque tensor, assuming it obeys + // layout_contract. + // - Case 1.2: Stealable sparse new_grad + // . Can't detach sparse tensor (since metadata changes are not allowed + // after detach), so just create a new one for the grad which is a + // shallow copy. We need a shallow copy so that modifying the original + // grad tensor doesn't modify the grad we accumulate. + // . We only skip clone if indices and values themselves are contiguous + // for backward compatibility reasons. Since without this optimization, + // earlier we would clone the entire SparseTensor which cloned indices + // and values. For details see + // https://github.com/pytorch/pytorch/issues/34375. + // - Case 1.3: Cloning sparse/nested new_grad + // - Case 1.4: Cloning MKLDNN new_grad + // - Case 1.5: Deep copies new_grad according to the Gradient Layout + // Contract. + // - Case 2: Param has existing grad and grad mode is not enabled + // - This case is not strictly necessary, but it makes the first-order only + // case slightly more efficient. + // - Case 2.1: Sparse variable_grad + Dense new_grad + // . If `variable_grad` is sparse and `new_grad` is not sparse, their + // sum is not sparse, and we must change the TensorImpl type of + // `variable_grad` for it to store the result. However, changing the + // TensorImpl type of a tensor requires changing the tensor itself, and + // thus in this case we have to change the grad tensor. + // - Case 2.2: Vmap-incompatible + // . Ideally we'd perform an in-place operation to avoid changing + // the grad tensor. However, if that's impossible because the grads + // are vmap-incompatible (See NOTE: [vmap-incompatible in-place + // operations]), then we just add them out-of-place. + // - Case 2.3: In-place addition + // . In this case we can avoid changing the grad tensor. There are three + // scenarios when we'll hit this case: + // . `variable_grad` is sparse, and `new_grad` is sparse. + // . `variable_grad` is dense, and `new_grad` is sparse. + // . `variable_grad` is dense, and `new_grad` is dense. + // . `variable_grad` is mkldnn, and `new_grad` is mkldnn. + // + // In all of these four cases, `variable_grad += new_grad` is a + // valid operation which adds `new_grad` to `variable_grad` in + // place. `variable_grad` is thus still referring to the same tensor + // after the operation. + // . DistributedDataParallel(DDP) package relies on grad being + // mutated in place for saving peak memory usage. DDP will still + // work correctly if it is mutated out of place here, but DDP will + // maintain one extra copy of grad tensors in buffer and thus + // increase peak memory usage. + // - Case 3: Param has existing grad and grad mode is enabled + // - Case 3.1: Sparse variable_grad + Dense new_grad + // - Case 3.2: Not Sparse variable_grad + Dense new_grad + // + // variable: the variable whose grad we're accumulating. + // variable_grad: the current grad for the variable. + // new_grad: new grad we want to accumulate for the variable. + // num_expected_refs: the number of refs we expect to hold internally + // such that it is safe to avoid cloning the grad + // if use_count() of the grad is less than or equal + // to this value (in addition to post_hooks). + // update_grad: Function that is used to update grad for the variable. + // The argument to the function is a Tensor which + // is used to set a new value for the grad. + template + static void accumulateGrad( + const Variable& variable, + at::Tensor& variable_grad, + const at::Tensor& new_grad, + size_t num_expected_refs, + const T& update_grad) { + if (!variable_grad.defined()) { + if (!GradMode::is_enabled() && !new_grad.is_sparse() && + !new_grad.is_sparse_csr() && + !(variable.is_sparse_csr() && new_grad.layout() == at::kStrided) && + impl::is_tensor_stealable( + new_grad, + num_expected_refs + at::caching::is_cached_tensor(new_grad)) && + (new_grad.is_mkldnn() || + !c10::AutogradState::get_tls_state() + .get_grad_layout_enforcement_enabled() || + utils::obeys_layout_contract(new_grad, variable))) { + // See Case 1.1: Stealable dense new_grad + update_grad(new_grad.detach()); + } else if ( + !GradMode::is_enabled() && new_grad.is_sparse() && + new_grad._indices().is_contiguous() && + new_grad._values().is_contiguous() && + // Use count for indices and values should always be <=1 since the + // SparseTensor should be the only one holding a reference to these. + new_grad._indices().use_count() <= 1 && + new_grad._values().use_count() <= 1 && + impl::is_tensor_stealable(new_grad, num_expected_refs)) { + // Case 1.2: Stealable sparse new_grad + // No scenario where we expect this to be true currently + TORCH_INTERNAL_ASSERT_DEBUG_ONLY( + !at::caching::is_cached_tensor(new_grad._indices()) && + !at::caching::is_cached_tensor(new_grad._values()) && + !at::caching::is_cached_tensor(new_grad)); + + update_grad(at::_sparse_coo_tensor_unsafe( + new_grad._indices(), + new_grad._values(), + new_grad.sizes(), + new_grad.options())); + } else { + if (new_grad.is_sparse() || new_grad.is_sparse_csr() || + new_grad.is_nested()) { + // Case 1.3: Cloning sparse/nested new_grad + update_grad(new_grad.clone()); + } else { + if (new_grad.is_mkldnn()) { + // Case 1.4: Cloning MKLDNN new_grad + update_grad(new_grad.clone()); + } else { + // Case 1.5: Deep copies new_grad according to the "Gradient + // Layout Contract." + update_grad(utils::clone_obey_contract(new_grad, variable)); + } + } + } + } else if (!GradMode::is_enabled()) { + // Case 2: Param has existing grad and grad mode is not enabled + if (variable_grad.is_sparse() && !new_grad.is_sparse()) { + // Case 2.1: Sparse variable_grad + Dense new_grad + auto result = new_grad + variable_grad; + CHECK_RESULT(result, variable); + update_grad(std::move(result)); + } else if (!at::inplaceIsVmapCompatible(variable_grad, new_grad)) { + // Case 2.2: Vmap-incompatible + auto result = variable_grad + new_grad; + CHECK_RESULT(result, variable); + update_grad(std::move(result)); + } else { + // Case 2.3: In-place addition + variable_grad += new_grad; + CHECK_RESULT(variable_grad, variable); + // ^ We could enforce the contract more aggressively here by writing: + // if (variable_grad.is_sparse() || new_grad.is_sparse()) { + // variable_grad += new_grad; + // } else if (obeys_layout_contract(variable_grad, variable)) { + // variable_grad += new_grad; + // } else { + // result = at::empty_strided(variable.sizes(), variable.strides(), + // variable.options().memory_format(std::nullopt)); + // update_grad(at::native::add_out(result, variable_grad, + // new_grad, 1.0); + // } + // However, that accumulation is sometimes in place and sometimes not, + // which may break user code. + } + } else { + // Case 3: Param has existing grad and grad mode is enabled + at::Tensor result; + if (variable_grad.is_sparse() && !new_grad.is_sparse()) { + // Case 3.1: Sparse variable_grad + Dense new_grad + // CPU backend throws an error on sparse + dense, so + // prefer dense + sparse here. + result = new_grad + variable_grad; + } else { + // Case 3.2: Not Sparse variable_grad + Dense new_grad + // Assumes operator+ result typically matches strides of first arg, + // and hopes variable_grad was originally created obeying layout + // contract. + result = variable_grad + new_grad; + } + CHECK_RESULT(result, variable); + update_grad(std::move(result)); + // ^ We could enforce the contract more aggressively here by saying + // if (obeys_layout_contract(new_grad, variable)) { + // update_grad(new_grad + variable_grad); + // } else { + // update_grad(variable_grad + new_grad); + // } + // such that the stashed grad is likely to have the right strides if + // either variable_grad or new_grad already has the right strides. + // We could enforce the contract with certainty by saying + // auto result = variable_grad + new_grad (or vice versa), checking + // result's layout, and copying to an obedient clone if necessary before + // update_grad. The copy would require another gmem pass. We can't create + // empty result with the right layout then add_out into it with a single + // kernel, because GradMode is enabled in this branch, and add_out isn't + // differentiable. Maybe more trouble than it's worth. + } + } + + void compiled_args(CompiledNodeArgs& args) const override; + variable_list apply_with_saved( + const variable_list& inputs, + SwapSavedVariables& saved) override; + + Variable variable; +}; + +#undef CHECK_RESULT + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/basic_ops.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/functions/basic_ops.cpp new file mode 100644 index 0000000000000000000000000000000000000000..42c8c7ae46b72e7ff2206703b79d79521ec387a7 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/basic_ops.cpp @@ -0,0 +1,79 @@ +#include + +#include +#include +#include +#include + +#include + +#include +#include + +namespace torch::autograd { + +variable_list Error::apply(variable_list&& inputs) { + return static_cast(this)->apply(std::move(inputs)); +} + +variable_list Error::apply(variable_list&& inputs) const { + TORCH_CHECK(false, msg); +} + +void Error::compiled_args(CompiledNodeArgs& args) const { + // throw the error during collect, the graph won't get compiled + apply(variable_list()); +} + +variable_list Error::apply_with_saved( + const variable_list& inputs, + SwapSavedVariables& saved) { + TORCH_INTERNAL_ASSERT(false, "unreachable"); +} + +auto DelayedError::apply(variable_list&& inputs) -> variable_list { + tensor_list outputs; + outputs.reserve(inputs.size()); + for (auto& var : inputs) { + // FIXME: share version counters + outputs.emplace_back(var.defined() ? var.tensor_data() : at::Tensor()); + } + return wrap_outputs(inputs, std::move(outputs), [&](edge_list&& next_edges) { + return c10::make_intrusive(msg, std::move(next_edges)); + }); +} + +auto UndefinedGrad::apply(variable_list&& inputs) -> variable_list { + tensor_list outputs; + outputs.reserve(inputs.size()); + for (auto& var : inputs) { + outputs.emplace_back( + var.defined() ? var.clone().tensor_data() : at::Tensor()); + } + return wrap_outputs(inputs, std::move(outputs), [&](edge_list&& next_edges) { + return c10::make_intrusive(std::move(next_edges)); + }); +} + +auto UndefinedGradBackward::apply(variable_list&& output_grads) + -> variable_list { + return tensor_list(output_grads.size()); +} + +auto Identity::apply(variable_list&& grads) -> variable_list { + return std::move(grads); +} + +void GraphRoot::compiled_args(CompiledNodeArgs& args) const { + args.collect(outputs); +} +variable_list GraphRoot::apply_with_saved( + const variable_list& inputs, + SwapSavedVariables& saved) { + saved.before(outputs); + variable_list result(outputs); + saved.after(outputs); + return result; +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/basic_ops.h b/western_ai/usa/pytorch/torch/csrc/autograd/functions/basic_ops.h new file mode 100644 index 0000000000000000000000000000000000000000..352fac18dfefd8f675bc1864f70a99dd886b298c --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/basic_ops.h @@ -0,0 +1,118 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace torch::autograd { + +struct TORCH_API Error : public Node { + // The Error node should never actually be reached during backprop, so it + // doesn't need to increment the global sequence number counter. If it is to + // be executed, it should be executed asap and stop the execution, so we set + // sequence_nr to the max value. + Error(std::string msg, edge_list&& next_edges) + : Node(/*sequence_nr=*/UINT64_MAX, std::move(next_edges)), + msg(std::move(msg)) {} + + Error(std::string msg) + : Node(/*sequence_nr=*/UINT64_MAX), msg(std::move(msg)) {} + + variable_list apply(variable_list&& inputs) override; + variable_list apply(variable_list&& inputs) const; + + void compiled_args(CompiledNodeArgs& args) const override; + variable_list apply_with_saved( + const variable_list& inputs, + SwapSavedVariables& saved) override; + + std::string msg; +}; + +// We print grad_fn names in tensor printing. For functions with backward +// NYI, grad_fn= will be printed if we use Error, which is confusing. So +// special case with a new NotImplemented function here. +struct TORCH_API NotImplemented : public Error { + NotImplemented(const std::string& forward_fn, edge_list&& next_edges) + : Error( + "derivative for " + forward_fn + " is not implemented", + std::move(next_edges)) {} + + NotImplemented(const std::string& forward_fn) + : Error("derivative for " + forward_fn + " is not implemented") {} +}; + +// Identity in forward, Error in backward. Used to implement +// @once_differentiable +struct TORCH_API DelayedError : public Node { + DelayedError(std::string msg, int64_t num_inputs) : msg(std::move(msg)) { + for ([[maybe_unused]] const auto _ [[maybe_unused]] : + c10::irange(num_inputs)) { + add_input_metadata(Node::undefined_input()); + } + } + + variable_list apply(variable_list&& inputs) override; + variable_list apply(variable_list&& inputs) const; + + std::string msg; +}; + +struct TORCH_API UndefinedGrad : public Node { + UndefinedGrad() { + add_input_metadata(Node::undefined_input()); + } + + variable_list apply(variable_list&& inputs) override; + variable_list apply(variable_list&& inputs) const; +}; + +struct TORCH_API UndefinedGradBackward : public Node { + UndefinedGradBackward(edge_list&& next_edges) : Node(std::move(next_edges)) {} + + UndefinedGradBackward() = default; + + variable_list apply(variable_list&& inputs) override; + variable_list apply(variable_list&& inputs) const; + + void compiled_args(CompiledNodeArgs& args) const override {} + variable_list apply_with_saved( + const variable_list& inputs, + SwapSavedVariables& saved) override { + return apply(variable_list(inputs)); + } +}; + +struct TORCH_API GraphRoot : public Node { + GraphRoot(edge_list functions, variable_list inputs) + : Node(std::move(functions)), outputs(std::move(inputs)) { + // Ensures calls to stream() on a GraphRoot instance reflect current + // stream(s) on devices of root grad tensors at the time the instance is + // constructed. + for (const auto& t : outputs) { + add_input_metadata(t); + } + } + + variable_list apply(variable_list&& inputs) override { + return outputs; + } + + void compiled_args(CompiledNodeArgs& args) const override; + variable_list apply_with_saved( + const variable_list& inputs, + SwapSavedVariables& saved) override; + + variable_list outputs; +}; + +struct TORCH_API Identity : public Node { + variable_list apply(variable_list&& inputs) override; +}; + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/comm.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/functions/comm.cpp new file mode 100644 index 0000000000000000000000000000000000000000..dd67a11f32086cc0406d1e7e48f7c24c28c7b9a4 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/comm.cpp @@ -0,0 +1,138 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +namespace torch::autograd { +Scatter::Scatter( + std::vector devices, + std::optional> chunk_sizes, + int64_t dim, + std::optional>> streams, + bool unsqueeze_scalars) + : devices_(std::move(devices)), + chunk_sizes_(std::move(chunk_sizes)), + dim_(dim), + streams_(std::move(streams)), + unsqueeze_scalars_(unsqueeze_scalars) {} + +Scatter::~Scatter() = default; + +variable_list Scatter::apply(variable_list&& inputs) { + AT_ASSERT(inputs.size() == 1); + auto& input = inputs.front(); + + c10::intrusive_ptr grad_fn; + if (compute_requires_grad(input)) { + grad_fn = c10::make_intrusive( + /*destination_device=*/input.device(), dim_); + grad_fn->set_next_edges(collect_next_edges(input)); + } + + auto device_indices = fmap(devices_, [](const at::Device& device) -> int64_t { + return device.index(); + }); + auto tensors = + torch::cuda::scatter(input, device_indices, chunk_sizes_, dim_, streams_); + + std::vector variables; + variables.reserve(tensors.size()); + for (auto& tensor : tensors) { + AT_ASSERT(tensor.defined()); + if (unsqueeze_scalars_) { + AT_ASSERT(tensor.dim() == 1 && tensor.numel() == 1); + variables.push_back(tensor[0]); + } else { + variables.push_back(std::move(tensor)); + } + } + + if (grad_fn) { + set_history(variables, grad_fn); + } + + return variables; +} + +Gather::Gather(const at::Device& destination_device, int64_t dim) + : destination_device_(destination_device), dim_(dim) {} + +Gather::~Gather() = default; + +variable_list Gather::apply(variable_list&& inputs) { + bool all_are_zero_dim = true; + for (const auto& input : inputs) { + TORCH_CHECK( + input.is_cuda(), + "All inputs to Gather must be CUDA tensors, got ", + input.toString()); + if (input.dim() > 0) { + all_are_zero_dim = false; + } + } + + const bool unsqueeze_scalars = all_are_zero_dim && dim_ == 0; + if (unsqueeze_scalars) { + TORCH_WARN( + "Was asked to gather along dimension 0, but all " + "input tensors were scalars; will instead unsqueeze " + "and return a vector."); + } + + c10::intrusive_ptr grad_fn; + // compute this before moving variables from `inputs` + if (compute_requires_grad(inputs)) { + std::vector source_devices; + source_devices.reserve(inputs.size()); + std::vector input_sizes; + input_sizes.reserve(inputs.size()); + for (auto& input : inputs) { + source_devices.push_back(input.device()); + input_sizes.push_back(input.size(dim_)); + } + grad_fn = c10::make_intrusive( + std::move(source_devices), + std::move(input_sizes), + dim_, + /*streams=*/std::nullopt, + /*unsqueeze_scalars=*/unsqueeze_scalars); + grad_fn->set_next_edges(collect_next_edges(inputs)); + } + + std::vector tensors; + if (unsqueeze_scalars) { + tensors.reserve(inputs.size()); + for (auto& variable : inputs) { + tensors.push_back(variable.view(1)); + } + } else { + tensors = std::move(inputs); + } + + // Disable the autograd during the actual computation + // torch::cuda::gather does not return a view or change things inplace + // so no need for extra logic here + at::Tensor variable; + { + at::AutoDispatchBelowAutograd mode; + // This is special logic for torch::cuda::gather! + const auto destination_index = + destination_device_.is_cpu() ? -1 : destination_device_.index(); + variable = torch::cuda::gather(tensors, dim_, destination_index); + } + if (grad_fn) { + set_history(variable, grad_fn); + } + return {variable}; +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/comm.h b/western_ai/usa/pytorch/torch/csrc/autograd/functions/comm.h new file mode 100644 index 0000000000000000000000000000000000000000..6da4d4f63978c933b6e6abe4c4f760c8e5038e8f --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/comm.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace torch::autograd { + +struct TORCH_CUDA_CU_API Scatter : public Node { + explicit Scatter( + std::vector devices, + std::optional> chunk_sizes = std::nullopt, + int64_t dim = 0, + std::optional>> streams = + std::nullopt, + bool unsqueeze_scalars = false); + ~Scatter() override; + + variable_list apply(variable_list&& inputs) override; + + std::vector devices_; + std::optional> chunk_sizes_; + int64_t dim_; + std::optional>> streams_; + bool unsqueeze_scalars_; +}; + +struct TORCH_CUDA_CU_API Gather : public Node { + explicit Gather(const at::Device& destination_device, int64_t dim = 0); + ~Gather() override; + + variable_list apply(variable_list&& inputs) override; + + at::Device destination_device_; + int64_t dim_; +}; + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/init.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/functions/init.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a7ae608f74fefc561f2ab8bd5f80294ded2dd1ac --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/init.cpp @@ -0,0 +1,173 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef USE_DISTRIBUTED +#include +#endif +#include +#include +#include +#include + +#include + +using namespace torch::autograd; + +struct DelayedErrorCtor { + DelayedError* operator()(PyObject* args) { + TORCH_CHECK( + PyTuple_GET_SIZE(args) == 2, + "Requires two arguments, got ", + PyTuple_GET_SIZE(args)); + auto arg1 = PyTuple_GET_ITEM(args, 0); + TORCH_CHECK(THPUtils_checkString(arg1), "argument 'msg' must be a string"); + std::string msg = THPUtils_unpackString(arg1); + auto arg2 = PyTuple_GET_ITEM(args, 1); + TORCH_CHECK( + THPUtils_checkLong(arg2), "argument 'num_inputs' must be an int"); + auto num_inputs = THPUtils_unpackLong(arg2); + return new DelayedError(std::move(msg), num_inputs); + } +}; + +struct UndefinedGradCtor { + UndefinedGrad* operator()(PyObject* args) { + TORCH_CHECK( + PyTuple_GET_SIZE(args) == 0, + "Requires zero arguments, got ", + PyTuple_GET_SIZE(args)); + return new UndefinedGrad(); + } +}; + +struct NoCtor { + Node* operator()(PyObject* args) { + TORCH_CHECK(false, "Cannot construct"); + } +}; + +template +static void addClass( + PyObject* module, + PyTypeObject& type, + const char* name, + PyGetSetDef* function_properties = nullptr, + PyMethodDef* function_methods = nullptr) { + createForwardFunctionPyTypeObject( + type, name, function_properties, function_methods); + Py_INCREF(&type); + PyModule_AddObject(module, name, (PyObject*)&type); + registerCppFunction(typeid(C), &type); +} + +template < + typename T, + typename ValueT, + typename ParamsT, + ValueT ParamsT::* ptr, + typename ConvertArgT, + PyObject* (*Convert)(ConvertArgT)> +static PyObject* getTupleAttr(PyObject* obj, void* _unused) { + HANDLE_TH_ERRORS + THPCppFunction* self = (THPCppFunction*)obj; + auto& arr = ((T*)(self->cdata.get()))->*ptr; + auto num_elems = arr.size(); + THPObjectPtr py_tuple(PyTuple_New(num_elems)); + if (!py_tuple) + return nullptr; + for (const auto i : c10::irange(num_elems)) { + PyTuple_SET_ITEM(py_tuple.get(), i, Convert(arr[i])); + } + return py_tuple.release(); + END_HANDLE_TH_ERRORS +} + +template < + typename T, + typename ValueT, + typename ParamsT, + ValueT ParamsT::* ptr, + typename ConvertArgT, + PyObject* (*Convert)(ConvertArgT)> +static PyObject* getValueAttr(PyObject* obj, void* _unused) { + HANDLE_TH_ERRORS + THPCppFunction* self = (THPCppFunction*)obj; + auto& val = ((T*)(self->cdata.get()))->*ptr; + return Convert(val); + END_HANDLE_TH_ERRORS +} + +static PyObject* accumulateGradVar(PyObject* _self, void* _unused) { + THPCppFunction* self = (THPCppFunction*)_self; + auto grad_acc = (AccumulateGrad*)self->cdata.get(); + return THPVariable_Wrap(grad_acc->variable); +} + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) +static struct PyGetSetDef accumulate_grad_properties[] = { + THP_FUNCTION_DEFAULT_PROPERTIES, + {(char*)"variable", accumulateGradVar, nullptr, nullptr, nullptr}, + {nullptr}}; + +void THPAutograd_initFunctions() { + THPObjectPtr module(PyModule_New("torch._C._functions")); + if (!module) + throw python_error(); + + static PyTypeObject AccumulateGradClass; + addClass( + module, + AccumulateGradClass, + "AccumulateGrad", + accumulate_grad_properties); + + static PyTypeObject ErrorClass; + addClass(module, ErrorClass, "Error"); + + static PyTypeObject NotImplementedClass; + addClass( + module, NotImplementedClass, "NotImplemented"); + + static PyTypeObject DelayedErrorClass; + addClass( + module, DelayedErrorClass, "DelayedError"); + + static PyTypeObject UndefinedGradBackwardClass; + addClass( + module, UndefinedGradBackwardClass, "UndefinedGradBackward"); + + static PyTypeObject UndefinedGradClass; + addClass( + module, UndefinedGradClass, "UndefinedGrad"); + + static PyTypeObject CopyBackwardsClass; + addClass(module, CopyBackwardsClass, "CopyBackwards"); + +#ifdef USE_DISTRIBUTED + static PyTypeObject SendRpcBackwardClass; + addClass( + module, SendRpcBackwardClass, "SendRpcBackward"); +#endif + + static PyTypeObject CopySlicesClass; + addClass(module, CopySlicesClass, "CopySlices"); + + generated::initialize_autogenerated_functions(module); + + auto c_module = THPObjectPtr(PyImport_ImportModule("torch._C")); + if (!c_module) + throw python_error(); + + Py_INCREF(module.get()); + if (PyModule_AddObject(c_module, "_functions", module) < 0) { + Py_DECREF(module.get()); + throw python_error(); + } +} diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/pybind.h b/western_ai/usa/pytorch/torch/csrc/autograd/functions/pybind.h new file mode 100644 index 0000000000000000000000000000000000000000..036673d53f61e6c374bcf0b7d62560cb08ebb174 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/pybind.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +// NOLINTNEXTLINE(misc-unused-alias-decls) +namespace py = pybind11; + +namespace pybind11::detail {} // namespace pybind11::detail diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/tensor.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/functions/tensor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4171634345e1b1e683be04cd608ac7e0294938d1 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/tensor.cpp @@ -0,0 +1,289 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +namespace torch::autograd { + +using torch::dynamo::autograd::IValuePacker; + +static variable_list CopyBackwards_apply_functional( + variable_list&& grads, + std::array needs_input_grad, + const c10::TensorOptions& src_options) { + check_input_variables("CopyBackwards", grads, 1, -1, true); + auto& grad = std::move(grads)[0]; + variable_list grad_inputs(2); + if (grad.defined()) { + if (needs_input_grad[0]) { + grad_inputs[0] = at::zeros_like(grad, LEGACY_CONTIGUOUS_MEMORY_FORMAT); + } + if (needs_input_grad[1]) { + // Handle R->C copies without raising a warning + const auto src_type = src_options.dtype().toScalarType(); + if (!c10::isComplexType(src_type) && grad.is_complex()) { + grad = at::real(grad); + } + + at::DeviceGuard device_guard(src_options.device()); + grad_inputs[1] = grad.to(src_options); + } + } + return grad_inputs; +} + +static variable_list CopyBackwards_apply_functional_ivalue( + const variable_list& grads, + const ivalue_list& args) { + PackedArgs r(args); + auto needs_input_grad = r.unpack>(); + auto src_options = r.unpack(); + return CopyBackwards_apply_functional( + variable_list(grads), needs_input_grad, src_options); +} + +auto CopyBackwards::apply(variable_list&& grads) -> variable_list { + return CopyBackwards_apply_functional( + std::move(grads), + {task_should_compute_output(0), task_should_compute_output(1)}, + src_options); +} + +void CopyBackwards::compiled_args(CompiledNodeArgs& args) const { + args.collect(src_options); +} + +variable_list CopyBackwards::apply_with_saved( + const variable_list& inputs, + SwapSavedVariables& saved) { + saved.before(src_options); + + static bool flag [[maybe_unused]] = [&]() { + std::vector schema = { + IValuePacker>::packed_type(), + IValuePacker::packed_type()}; + const auto& interface = torch::dynamo::autograd::getPyCompilerInterface(); + interface->bind_function( + saved.get_py_compiler(), + name(), + CopyBackwards_apply_functional_ivalue, + schema); + return true; + }(); + + PackedArgs packed_args; + packed_args.pack>( + {task_should_compute_output(0), task_should_compute_output(1)}); + packed_args.pack(src_options); + + auto output_metadata = torch::dynamo::autograd:: + IValuePacker>>::pack( + torch::dynamo::autograd::get_input_metadata(next_edges())); + + const auto& interface = torch::dynamo::autograd::getPyCompilerInterface(); + auto result = interface->call_function( + saved.get_py_compiler(), + "apply_functional", + name(), + inputs, + std::move(packed_args).vec(), + output_metadata); + + saved.after(src_options); + return result; +} + +CopySlices::CopySlices( + const Variable& base_var, + at::TensorGeometry view_, + std::unique_ptr view_fn_, + c10::intrusive_ptr fn_) + : base(base_var), + view(std::move(view_)), + view_fn(std::move(view_fn_)), + fn(std::move(fn_)) { + // Take the next_edges of fn as our own, except for index 0 which goes + // to base instead of the view. + add_input_metadata(base_var); + const auto num_outputs = fn->num_outputs(); + next_edges_.reserve(num_outputs); + add_next_edge(impl::gradient_edge(base_var)); + for (const auto i : c10::irange(1, num_outputs)) { + add_next_edge(fn->next_edge(i)); + } +} + +void CopySlices::update_exec_info() { + // See Note [View + Inplace update for view tensor] For more details on this + // block Since the gradient edge for the 0th input is different between `this` + // and `fn`, make sure that the one from `fn` has the same metadata in the + // current GraphTask's exec_info as the one on `this`. + const auto exec_info = get_current_graph_task_exec_info(); + if (exec_info && !exec_info->empty()) { + const auto& fn_edge = fn->next_edge(0); + const auto& this_edge = this->next_edge(0); + TORCH_INTERNAL_ASSERT(fn_edge.is_valid() == this_edge.is_valid()); + if (fn_edge.is_valid()) { + const auto fn_next_node = fn_edge.function.get(); + auto it = exec_info->find(fn_next_node); + if (it == exec_info->end()) { + // Node is not in the exec_info already + if (task_should_compute_output(0)) { + // And we need gradient for the corresponding output + add_node_to_current_graph_task_exec_info(fn_next_node); + // There is no need to remove this after execution because we are + // guaranteed that this->next_edge(0) must be in the history of + // fn->next_edge(0) (we cannot easily assert this as it might be far + // away if there were many chained views). This means that, since + // fn->next_edge(0) was not needed (no exec_info entry for it), we + // know that nothing downstream of fn->next_edge(0) is needed either + // (otherwise the whole path from that Node to this->next_edge(0) + // would be needed as well). This means that no other Node will ever + // look at fn->next_edge(0) metadata and thus there is no need to + // clean them up. + } + } else { + TORCH_INTERNAL_ASSERT( + it->second.should_execute() == task_should_compute_output(0)); + } + } + } + + // Sanity check that the graph was never modified after the fact (it is + // read-only!) + TORCH_INTERNAL_ASSERT(num_outputs() == fn->num_outputs()); + for (const auto i : c10::irange(1, this->num_outputs())) { + TORCH_INTERNAL_ASSERT( + fn->next_edge(i).function.get() == this->next_edge(i).function.get()); + } +} + +// common code between apply/apply_with_saved +template +inline variable_list CopySlices::apply_impl( + variable_list&& inputs, + const T& call_fn) { + check_input_variables("CopySlices", inputs, 1, -1, true); + auto& grad = std::move(inputs)[0]; + if (!grad.defined()) { + return variable_list(num_outputs()); + } + + // Acquire lock to here protect thread safety on fn + // see Note [Thread Safety on Autograd Node] + std::lock_guard lock(mutex_); + + TORCH_CHECK(fn, ERR_BACKWARD_TWICE); + + auto result = + grad.new_empty_strided_symint(base.sym_sizes(), base.sym_strides()); + result.copy_(grad); + + at::Tensor grad_slice; + if (view_fn) { + grad_slice = (*view_fn)(result); + } else { + auto offset = view.sym_storage_offset() - base.sym_storage_offset(); + grad_slice = + result.as_strided_symint(view.sym_sizes(), view.sym_strides(), offset); + } + + update_exec_info(); + + // TODO: We clone grad_slice because we modify it below and "fn" might save + // it for the backward of res. We might be able to avoid the clone() if + // double-backprop is disabled. + auto res = call_fn({grad_slice.clone(at::MemoryFormat::Contiguous)}); + + variable_list grad_inputs(num_outputs()); + for (const auto i : c10::irange(res.size())) { + if (task_should_compute_output(i)) { + if (!res[i].defined()) { + // If the output is not defined, treat it as if it was a zero tensor. + // This can happen if users define a custom Function. + continue; + } + if (i == 0) { + grad_slice.copy_(res[i]); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.Move) + grad_inputs[i] = std::move(result); // NOLINT(bugprone-use-after-move) + } else { + grad_inputs[i] = std::move(res[i]); + } + } + } + + return grad_inputs; +} + +void CopySlices::release_variables() { + // Acquire lock to here protect thread safety on fn + std::lock_guard lock(mutex_); + fn = nullptr; +} + +void CopySlices::compiled_args(CompiledNodeArgs& args) const { + TORCH_CHECK(!view_fn, "view_fn not supported by compiled autograd") + TORCH_INTERNAL_ASSERT((bool)fn); + args.collect(base); + args.collect(view); + args.collect(fn); + fn->compiled_args(args); +} + +variable_list CopySlices::apply_with_saved( + const variable_list& grads, + SwapSavedVariables& saved) { + saved.before(base); + saved.before(view); + + auto results = variable_list(num_outputs()); + if (grads[0].defined()) { + TORCH_CHECK(fn, ERR_BACKWARD_TWICE); + update_exec_info(); + + std::vector needs_input_grad; + needs_input_grad.reserve(num_outputs()); + for (const auto i : c10::irange(num_outputs())) { + needs_input_grad.emplace_back(task_should_compute_output(i)); + } + // Not yet supported, also doesn't happen in typical eager mode execution + // (this only happens by default with torch-xla). + TORCH_INTERNAL_ASSERT(!view_fn); + const auto& interface = torch::dynamo::autograd::getPyCompilerInterface(); + variable_list stuff = interface->call_copy_slices_prologue( + saved.get_py_compiler(), grads, base, view); + TORCH_INTERNAL_ASSERT(stuff.size() == 3); + // These variables are named the same as in CopySlices::apply_impl. + // Follow along there. + const auto& result = stuff[0]; + const auto& grad_slice = stuff[1]; + const auto& grad_slice_clone = stuff[2]; + auto res = fn->apply_with_saved({grad_slice_clone}, saved); + results = interface->call_copy_slices_epilogue( + saved.get_py_compiler(), needs_input_grad, result, res, grad_slice); + } + + saved.after(base); + saved.after(view); + return results; +} + +auto CopySlices::apply(variable_list&& inputs1) -> variable_list { + return apply_impl(std::move(inputs1), [this](variable_list&& inputs2) { + return (*fn)(std::move(inputs2)); + }); +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/tensor.h b/western_ai/usa/pytorch/torch/csrc/autograd/functions/tensor.h new file mode 100644 index 0000000000000000000000000000000000000000..0ed11094022a3639a31d589ea0465638a69a2885 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/tensor.h @@ -0,0 +1,185 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace torch::autograd { + +struct TORCH_API CopyBackwards : public Node { + variable_list apply(variable_list&& grads) override; + void compiled_args(CompiledNodeArgs& args) const override; + variable_list apply_with_saved( + const variable_list& inputs, + SwapSavedVariables& saved) override; + + at::TensorOptions src_options; +}; + +// Note [View + Inplace update for base tensor] +// +// This note covers a few important topics related to view + inplace handling. +// - It explains what is the CopySlices Node and why we need it. +// - It explains the considerations on what is saved for backward in +// CopySlices. +// - It explains why we need to sometimes change the exec_info of the current +// backward +// +// What is CopySlices? +// ~~~~~~~~~~~~~~~~~~~ +// +// We support autograd with inplace mutation; e.g., if you write x.mul_(2) +// the autograd will work as if you now had multiple Tensors under the hood and +// you did +// x = t.clone() +// x0 = x +// x1 = x0 * 2 +// x = x1 +// As you can see here, after this operation, x.grad_fn now points to x1.grad_fn +// (the MulBackward node) and this node points to x's original grad_fn (which is +// also x0.grad_fn). It is important to keep in mind that after the inplace, +// there is no Tensor object that represents the x0 state anymore. But the graph +// for it is still around in autograd (in case x was used before being modified +// inplace). See Example 1 in +// https://docs.google.com/drawings/d/1-T5DyYfChMX1ONQkY-zU-hj_ayQ2zmA5CBOKDWqvEhE +// We call this rebasing the history of the Tensor. +// +// Now, a difficult situation is what happens if x is a differentiable view +// of a base b. +// b = t.clone() +// x = b.select(0, 0) +// x *= 2 +// With the same approach as above, this will become +// b = t.clone() +// x = b.select(0, 0) +// b0 = b +// x0 = x +// x1 = x0 * 2 +// b1 = b0.select_scatter(x1, 0, 0) +// x2 = b1.select(0, 0) +// x = x2 +// b = b1 +// As you can see here, not only we need to modify x's grad_fn, we also need to +// modify the one from b. We also need to ensure that the new grad_fn on x is +// linked to b's new grad_fn. The chain the select_scatter, multiplication and +// select is what CopySlices does, all wrapped into a single Node. +// +// See Example 1 in +// https://docs.google.com/drawings/d/1-T5DyYfChMX1ONQkY-zU-hj_ayQ2zmA5CBOKDWqvEhE +// +// What do we need to save in CopySlices to run backward? +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// +// We need to perform grad_view = fn(grad_view), but out-of-place. +// view_fn_ is an optional function saved in DifferentiableViewMeta +// from forward pass, so that we can recover we when as_strided is not +// supported. It preserves the invariants: +// view = view_fn_(base) +// grad_view = view_fn_(grad_base) +// +// When as_strided is supported (e.g. strided CPU/CUDA Tensors), view_fn_ +// is empty and we save TensorGeometry(view) instead. +// With the TensorGeometry information we can use `as_strided` call which +// is more efficient to recover views in backward. +// +// For example: +// view_1 = view_op_1(base) +// view_2 = view_op_2(view_1) +// ... +// view_n = view_op_n(view_n-1) +// view_n = inplace_op(view_n) +// +// In CPU/CUDA case where we support efficient as_strided implementation, +// grad_view_n can be calculated through 1 step. +// +// grad_view_n = grad_base.as_strided(view_sizes, view_strides, view_offset); +// +// But in XLA backend where we don't have full support of as_strided, +// it has to save a chained lambda function view_fn_, to exactly +// replay how the view was done in forward. +// +// view_fn_ = view_op_n(...(view_op_2(view_op_1()))) +// grad_view_n = view_fn_(grad_base) +// +// This chain view_fn_ works as long as forward view ops are implemented, +// e.g XLA simulates view without a real Storage behind Tensor, but it's less +// efficient than the as_strided one so we should be careful to only use it when +// necessary. +// +// - For CPU/CUDA we save TensorGeometry of both base and view tensors, +// That's all we need to pass into as_strided. +// E.g. int[] sizes, int[] strides, and int storage_offset. +// - For XLA we use view_fn_, which captures all forward view op arguments +// by **value**. +// E.g for at::narrow, int dim, int start, in length are saved. +// +// Theoretically we could also save Tensor `view` in CopySlices Node, but +// it's far more expensive than what we currently save. +// 1. We cannot afford keeping large tensors alive to recover views only. +// 2. There are inplace checks when Tensors are loaded back to make sure +// they haven't been changed (including size metadata). +// So saving metadata like TensorGeometry/view arguments is much better +// because it is minimal information needed to recover views, as well as it +// allows the user to modify the original Tensor without preventing the +// backward pass from running. +// +// Why do we manually change exec_info in the apply? +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// +// Using the same example as before, +// b = t.clone() +// x = b.select(0, 0) +// x *= y +// +// You can see the visualization at +// https://docs.google.com/drawings/d/1Bx-Hcz-zlIv7PabQqnPhUIVIs9F8WWi48svqMsAUMFs +// which contains the wrapped MulBackward Node and show what it links to. +// Since a backward can happen between any subset of the inputs (t and y) and +// outputs (o, x, b). It is possible to get into a state where CopySlices's 0th +// next function (CloneBackward) needs gradient but MulBackward's 0th next +// function (SelectBackward) is not. This happens if you do autograd.grad +// between x and t for example. +// In such a case, we do need to mark SelectBackward as requiring gradient such +// that, during the execution of MulBackward, we will actually compute gradient +// for the 0th input. +// +// All the other next functions are always shared (this is asserted in the apply +// code) and so nothing needs to be done for them. + +// See Note [View + Inplace update for view tensor] for what we do to view +// tensor when an in-place operation happens. +struct TORCH_API CopySlices : public Node { + CopySlices( + const Variable& base_var, + at::TensorGeometry view_, + std::unique_ptr view_fn_, + c10::intrusive_ptr fn_); + + // common code between apply/apply_with_saved + template + variable_list apply_impl(variable_list&& inputs, const T& call_fn); + + variable_list apply(variable_list&& inputs) override; + void release_variables() override; + void compiled_args(CompiledNodeArgs& args) const override; + variable_list apply_with_saved( + const variable_list& inputs, + SwapSavedVariables& saved) override; + void update_exec_info(); + + at::TensorGeometry base; + // view and view_fn are redundant and view_fn will be used if available. + // See Note [View + Inplace update for base tensor] for details. + at::TensorGeometry view; + std::unique_ptr view_fn; + c10::intrusive_ptr fn; +}; + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/utils.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/functions/utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5fff42edefed6b318692df48d7c48d6cd3243ec2 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/utils.cpp @@ -0,0 +1,74 @@ +#include +#include + +#include +#include +#include + +#include +#include + +namespace torch::autograd { + +variable_list wrap_outputs( + const variable_list& inputs, + // NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved) + tensor_list&& outputs, + const function_constructor& ctr) { + variable_list result; + result.reserve(outputs.size()); + if (!any_variable_requires_grad(inputs)) { + for (auto& output : outputs) { + if (output.defined()) { + result.push_back( + make_variable(std::move(output), /*requires_grad=*/false)); + } else { + result.emplace_back(); + } + } + } else { + auto grad_fn = + ctr(GradMode::is_enabled() ? collect_next_edges(inputs) : edge_list()); + for (auto& output : outputs) { + if (output.defined()) { + auto variable = + autograd::make_variable(std::move(output), /*requires_grad=*/false); + autograd::create_gradient_edge(variable, grad_fn); + result.push_back(std::move(variable)); + } else { + grad_fn->add_input_metadata(Node::undefined_input()); + result.emplace_back(); + } + } + } + return result; +} + +void check_input_variables( + const char* name, + const variable_list& inputs, + int args, + int required_args, + bool allow_undefined) { + if (required_args == -1) { + required_args = args; + } + TORCH_CHECK( + inputs.size() == static_cast(args), + name, + ": expected ", + args, + " arguments (got ", + inputs.size(), + ")"); + + for (const auto i : c10::irange(required_args)) { + TORCH_CHECK( + inputs[i].defined() || allow_undefined, + name, + ": expected Tensor at argument ", + i, + " (got None)"); + } +} +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/functions/utils.h b/western_ai/usa/pytorch/torch/csrc/autograd/functions/utils.h new file mode 100644 index 0000000000000000000000000000000000000000..e3d65f5269993e5305adf301f134c103f6711892 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/functions/utils.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace torch::autograd { + +using function_constructor = + std::function(edge_list&&)>; + +/** + * Wraps the tensor outputs in variables and creates the grad_fn and sets the + * grad_fn if necessary. + */ +TORCH_API variable_list wrap_outputs( + const variable_list& inputs, + tensor_list&& outputs, + const function_constructor& ctr); + +/// Checks that inputs contains exactly `args` items and that the first +/// `required_args` +/// items are not nullptr. If not specified, `required_args` defaults to `args`. +TORCH_API void check_input_variables( + const char* name, + const variable_list& inputs, + int args, + int required_args = -1, + bool allow_undefined = false); + +struct ComputeRequiresGrad : IterArgs { + bool out = false; + using IterArgs::operator(); + void operator()(const at::Tensor& tensor) { + const auto& var = static_cast(tensor); + if (var.defined() && var.requires_grad()) { + out = true; + } + } + void operator()(const std::optional& tensor) { + if (tensor.has_value()) { + (*this)(*tensor); + } + } + bool short_circuit() { + return out; + } +}; + +template +inline bool compute_requires_grad(Args&&... args) { + if (!GradMode::is_enabled()) { + return false; + } + return ComputeRequiresGrad().apply(std::forward(args)...).out; +} + +inline void set_history( + const at::Tensor& variable, + const c10::intrusive_ptr& grad_fn) { + TORCH_CHECK(grad_fn != nullptr); + if (variable.defined()) { + // If the codegen triggers this, you most likely want to add your newly + // added function to the DONT_REQUIRE_DERIVATIVE list in + // tools/autograd/gen_variable_type.py + TORCH_CHECK( + isDifferentiableType(variable.scalar_type()), + "Autograd not support dtype: ", + variable.scalar_type()); + auto output_nr = grad_fn->add_input_metadata(variable); + impl::set_gradient_edge(variable, {grad_fn, output_nr}); + } else { + grad_fn->add_input_metadata(Node::undefined_input()); + } +} + +inline void set_history( + const std::vector& variables, + const c10::intrusive_ptr& grad_fn) { + for (auto& variable : variables) { + set_history(variable, grad_fn); + } +} + +inline bool isFwGradDefined(const std::optional& t) { + return t.has_value() && t->defined() && t->_fw_grad(/*level */ 0).defined(); +} + +inline bool isFwGradDefinedTensorList(const at::ITensorListRef& variables) { + bool ret = false; + for (auto& variable : variables) { + ret |= isFwGradDefined(variable); + } + return ret; +} + +inline bool isFwGradDefinedTensorList( + const c10::List>& li) { + bool ret = false; + for (auto i : c10::irange(li.size())) { + auto t = li.get(i); + ret |= isFwGradDefined(t); + } + return ret; +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/grad_mode.h b/western_ai/usa/pytorch/torch/csrc/autograd/grad_mode.h new file mode 100644 index 0000000000000000000000000000000000000000..1935068fb8bb48aeaac06a9697ccc95f2d7efe74 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/grad_mode.h @@ -0,0 +1,11 @@ +#pragma once + +#include +#include + +namespace torch::autograd { + +using GradMode = at::GradMode; +using AutoGradMode = at::AutoGradMode; + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/graph_task.h b/western_ai/usa/pytorch/torch/csrc/autograd/graph_task.h new file mode 100644 index 0000000000000000000000000000000000000000..2fc42c04183c0e4f463cef63a3d609156dfac1ad --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/graph_task.h @@ -0,0 +1,231 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace torch::autograd { + +using edge_list = std::vector; +struct ReadyQueue; + +static constexpr int NO_DEVICE = -2; +static constexpr int CPU_DEVICE = -1; + +// GraphTask holds metadata needed for a single execution of backward() +struct GraphTask : std::enable_shared_from_this { + std::atomic outstanding_tasks_{0}; + // Indicates if an error occurred while executing any task. When this is + // true, it signals all threads to stop executing. + std::atomic_bool has_error_{false}; + std::atomic_bool future_completed_{false}; + // It is safe to read keep_graph_ without synchronization + bool keep_graph_; + + // To protect reads/writes to not_ready_, dependencies_, captured_vars_, + // has_error_, future_result_, cpu_ready_queue_, and leaf_streams. + std::mutex mutex_; + std::unordered_map not_ready_; + std::unordered_map dependencies_; + + // Records the nodes that are in the graph + std::unordered_set nodes_in_graph_; + c10::SmallVector graph_roots_; + // Note [Exec info] + // Exec info is created for each GraphTask, which allows filtering paths on + // the graph that are not needed. It has a bit complicated semantics. If it's + // empty, it means the task is run in a "default" mode, which means that all + // next_edges we encounter should get executed. If it's not empty, only + // functions that have an entry and this entry has needed == True should be + // executed. exec_info is only empty when the graph is executed via + // .backward() and the inputs parameter is not passed. Otherwise, when + // executed through .grad(), or when inputs arg is specified for .backward(), + // exec_info will be non-empty. + // + struct ExecInfo { + struct Capture { + Capture(const Capture&) = delete; + Capture(Capture&&) = default; + Capture& operator=(const Capture&) = delete; + Capture& operator=(Capture&&) = default; + ~Capture() = default; + + Capture(int input_idx, int output_idx) + : input_idx_(input_idx), output_idx_(output_idx) {} + int input_idx_; // within Node inputs + int output_idx_; // within the output vector of a GraphTask + + // This hook will be executed after a grad is captured. The captured + // grad will be replaced by the return value of the hook. + struct GradCaptureHook { + virtual ~GradCaptureHook() = default; + virtual at::Tensor operator()(const at::Tensor& grad) = 0; + }; + // NOTE [Deprecated capture hooks] + // + // The current status of capture hooks is that we continue to support + // the single usage of it by distributed in the dist_engine. If anyone + // else needs to use it for other purposes, they should file an issue. + // + // Capture hooks were originally created because there did not exist + // any way to register pre/post hooks to grad_fn in a way such that it + // would still be executed even if that is the grad_fn of a Tensor + // passed as input= of .grad. As far as I know, only dist_engine uses + // this hook. + // + // However, there are other alternatives today like tensor hooks that can + // replace the usage that originally motivated its creation. Also, + // Captures hooks are an outlier in terms of the types of hook that + // autograd offers in how it is registered and behaves, e.g. it is a hook + // registered not to the graph, but to a particular graph_task! This makes + // it a burden to maintain. + // + // It would be very nice to clean up/do a migration from pre/post + // hooks used in distributed to use tensor hooks, but for now we just + // mark this method as deprecated to prevent additional usage. + // + // If you still think you really need to capture hooks, please file an + // issue (and tag autograd). + const std::vector>& + DO_NOT_USE_DEPRECATED_get_capture_hooks() const { + return hooks_; + } + // See NOTE [deprecated capture hooks] + void DO_NOT_USE_DEPRECATED_register_capture_hook( + std::unique_ptr hook) { + hooks_.push_back(std::move(hook)); + } + + private: + // The hooks will be called one by one in the order as they were added. + // The input grad of a hook will be the output of its preceding hook. The + // first hook will take the captured grad as the input. The output of the + // last hook will replace the captured grad. + std::vector> hooks_; + }; + + bool should_execute() const { + return needed_ || captures_; + } + + bool needed_ = false; + std::unique_ptr> captures_; + }; + // exec_info_ is safe to read without synchronization + std::unordered_map exec_info_; + // Captured variables are grads captured that we return to the user. After + // execution of the GraphTask is completed, the captured_vars_ are moved + // out of the GraphTask and are no longer valid. + std::vector captured_vars_; + + // Note: this field is not ready to be used until the proper + // `thread_locals_.set_grad_mode()` call in the constructor. + at::ThreadLocalState thread_locals_; + + std::unordered_set leaf_streams; + + // Per-device current streams of the execute() that called this GraphTask. + // These will be synced with leaf_streams in exec_post_processing. + std::vector> caller_current_streams_; + + // Collects caller_current_streams_ for the accelerator device. + void stash_current_streams(); + + void init_to_execute( + Node& graph_root, + const edge_list& outputs, + bool accumulate_grad, + uint64_t min_topo_nr); + + // The value of worker_device in the thread that created this task. + // See Note [Reentrant backwards] + // Safe to read owner_ and reentrant_depth_ without synchronization + int owner_{NO_DEVICE}; + // The number of parent graph tasks for this graph task + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const int reentrant_depth_; + + bool can_checkpoint() const { + return exec_info_.empty(); + } + + // check if the GraphTask is completed or not + bool completed(); + // mark the graph task as completed and trigger post processing + void mark_as_completed_and_run_post_processing(); + + // Set an appropriate exception on this graph_task which was encountered while + // running the provided function. + void set_exception( + std::exception_ptr eptr, + const c10::intrusive_ptr& fn); + + // Set an appropriate exception on this graph_task which was encountered while + // running the provided function. But doesn't signal completion on + // 'future_result_' right away. The user needs to explicitly mark + // 'future_result_' completed with an appropriate exception. + void set_exception_without_signal(const c10::intrusive_ptr& fn); + + // Whether or not to stop execution for this GraphTask when an error is + // encountered. When set to true, this would cause Engine::execute() to throw + // an exception as soon as the autograd engine receives an exception. + bool exit_on_error_; + + // CPU threads are dedicated to processing CPU work for the backward they + // invoked. So any given graph task maintains its own cpu_ready_queue_ where + // you should send work for it to be done. We memoize the cpu_ready_queue_ per + // GraphTask so that we know which ready queue we should push to if we are on + // device thread (i.e. GPU) and but next NodeTask should be run on CPU. + std::shared_ptr cpu_ready_queue_; + + // Future representing the completion of the graph task. Notified when all + // tasks are done. + c10::intrusive_ptr future_result_; + + // Final callbacks installed during execution of this GraphTask + std::vector> final_callbacks_; + // To protect reads and writes to final_callbacks_. Intentionally not reusing + // mutex_ as the two are protecting different data structures. + std::mutex final_callbacks_lock_; + + utils::DelayWarningHandler warning_handler_; + + uint64_t id_; + + GraphTask( + bool keep_graph, + bool grad_mode, + int reentrant_depth, + std::shared_ptr cpu_ready_queue, + c10::SmallVector graph_roots, + bool exit_on_error = false); + + private: + // run GraphTask post processing + void exec_post_processing(); +}; + +// The guard that sets and restores current_graph_task. +class GraphTaskGuard { + public: + explicit GraphTaskGuard(std::shared_ptr graph_task); + ~GraphTaskGuard(); + + void restore_current_graph_task(); + + private: + std::shared_ptr last_graph_task_; +}; + +TORCH_API const std::unordered_map* +get_current_graph_task_exec_info(); +TORCH_API const std::unordered_set* +get_current_graph_task_nodes_in_graph(); +TORCH_API bool get_current_graph_task_keep_graph(); +TORCH_API std::vector get_current_graph_task_execution_order(); +TORCH_API int get_current_graph_task_id(); +void add_node_to_current_graph_task_exec_info(Node* fn); + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/init.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/init.cpp new file mode 100644 index 0000000000000000000000000000000000000000..47fa6d604bcdbed99281ccc18ee6fc9aa2d772c5 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/init.cpp @@ -0,0 +1,1851 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef USE_KINETO +#include +#include +#endif +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using torch::impl::py_context_manager; +using torch::impl::py_context_manager_DEPRECATED; + +namespace { + +struct DisableFuncTorch { + DisableFuncTorch() + : front_guard_(c10::DispatchKey::FuncTorchDynamicLayerFrontMode), + back_guard_(c10::DispatchKey::FuncTorchDynamicLayerBackMode) {} + c10::impl::ExcludeDispatchKeyGuard front_guard_; + c10::impl::ExcludeDispatchKeyGuard back_guard_; +}; + +struct DisableAutocast { + c10::impl::ExcludeDispatchKeyGuard guard_{c10::autocast_dispatch_keyset}; +}; + +struct EnableTorchFunction { + EnableTorchFunction() + : old_(at::impl::PythonTorchFunctionTLS::get_disabled_state()) { + at::impl::PythonTorchFunctionTLS::set_disabled_state( + at::impl::TorchFunctionDisabledState::ENABLED); + } + + EnableTorchFunction(const EnableTorchFunction& other) = delete; + EnableTorchFunction(EnableTorchFunction&& other) = delete; + EnableTorchFunction& operator=(const EnableTorchFunction& other) = delete; + EnableTorchFunction& operator=(EnableTorchFunction&& other) = delete; + + ~EnableTorchFunction() { + at::impl::PythonTorchFunctionTLS::set_disabled_state(old_); + } + at::impl::TorchFunctionDisabledState old_; +}; + +struct EnablePythonDispatcher { + EnablePythonDispatcher() : old_(c10::impl::PythonDispatcherTLS::get_state()) { + c10::impl::PythonDispatcherTLS::set_state(getPyInterpreter()); + } + EnablePythonDispatcher(const EnablePythonDispatcher& other) = delete; + EnablePythonDispatcher(EnablePythonDispatcher&& other) = delete; + EnablePythonDispatcher& operator=(const EnablePythonDispatcher& other) = + delete; + EnablePythonDispatcher& operator=(EnablePythonDispatcher&& other) = delete; + + ~EnablePythonDispatcher() { + c10::impl::PythonDispatcherTLS::set_state(old_); + } + c10::impl::PyInterpreter* old_; +}; + +struct EnablePreDispatch { + EnablePreDispatch() : guard_(c10::DispatchKey::PreDispatch) {} + c10::impl::IncludeDispatchKeyGuard guard_; +}; + +} // namespace + +PyObject* THPAutograd_initExtension(PyObject* _unused, PyObject* unused) { + using namespace torch::autograd::profiler; + using namespace torch::profiler::impl; + auto tensor_module = THPObjectPtr(PyImport_ImportModule("torch._tensor")); + if (!tensor_module) + return nullptr; + + // NOTE: "leaks" THPVariableClass + THPVariableClass = PyObject_GetAttrString(tensor_module, "Tensor"); + if (!THPVariableClass) + return nullptr; + + auto autograd_module = THPObjectPtr(PyImport_ImportModule("torch.autograd")); + if (!autograd_module) + return nullptr; + + // NOTE: "leaks" Function + THPFunctionClass = PyObject_GetAttrString(autograd_module, "Function"); + if (!THPFunctionClass) + return nullptr; + + // NOTE: "leaks" GradientEdge + auto autograd_graph_mod = + THPObjectPtr(PyImport_ImportModule("torch.autograd.graph")); + THPGradientEdgeClass = + PyObject_GetAttrString(autograd_graph_mod, "GradientEdge"); + if (!THPGradientEdgeClass) + return nullptr; + + auto torch_C_module = THPObjectPtr(PyImport_ImportModule("torch._C")); + if (!torch_C_module) + return nullptr; + auto _C_m = py::handle(torch_C_module).cast(); + auto m = _C_m.def_submodule("_autograd", "autograd bindings"); + + auto parameter_module = + THPObjectPtr(PyImport_ImportModule("torch.nn.parameter")); + if (!parameter_module) + return nullptr; + + // NOTE: "leaks" ParameterClass + ParameterClass = PyObject_GetAttrString(parameter_module, "Parameter"); + if (!ParameterClass) + return nullptr; + + py::class_(m, "TensorGeometry") + .def("sizes", &at::TensorGeometry::sizes) + .def("strides", &at::TensorGeometry::strides) + .def("storage_offset", &at::TensorGeometry::storage_offset); + + py::class_(m, "ProfilerEvent") + .def("kind", &LegacyEvent::kindStr) + .def("name", [](const LegacyEvent& e) { return e.name(); }) + .def("thread_id", &LegacyEvent::threadId) + .def("fwd_thread_id", &LegacyEvent::fwdThreadId) + .def("device", &LegacyEvent::device) + .def("cpu_elapsed_us", &LegacyEvent::cpuElapsedUs) + .def("cuda_elapsed_us", &LegacyEvent::cudaElapsedUs) + .def("has_cuda", &LegacyEvent::hasCuda) + .def("shapes", &LegacyEvent::shapes) + .def("cpu_memory_usage", &LegacyEvent::cpuMemoryUsage) + .def("cuda_memory_usage", &LegacyEvent::cudaMemoryUsage) + .def("handle", &LegacyEvent::handle) + .def("node_id", &LegacyEvent::nodeId) + .def("is_remote", &LegacyEvent::isRemote) + .def("sequence_nr", &LegacyEvent::sequenceNr) + .def("stack", &LegacyEvent::stack) + .def("scope", &LegacyEvent::scope) + .def("correlation_id", &LegacyEvent::correlationId) + .def("start_us", &LegacyEvent::cpuUs) + .def("flops", &LegacyEvent::flops) + .def("is_async", &LegacyEvent::isAsync); + + py::enum_(m, "DeviceType") + .value("CPU", c10::DeviceType::CPU) + .value("CUDA", c10::DeviceType::CUDA) + .value("MKLDNN", c10::DeviceType::MKLDNN) + .value("OPENGL", c10::DeviceType::OPENGL) + .value("OPENCL", c10::DeviceType::OPENCL) + .value("IDEEP", c10::DeviceType::IDEEP) + .value("HIP", c10::DeviceType::HIP) + .value("FPGA", c10::DeviceType::FPGA) + .value("MAIA", c10::DeviceType::MAIA) + .value("XLA", c10::DeviceType::XLA) + .value("Vulkan", c10::DeviceType::Vulkan) + .value("Metal", c10::DeviceType::Metal) + .value("XPU", c10::DeviceType::XPU) + .value("MPS", c10::DeviceType::MPS) + .value("MTIA", c10::DeviceType::MTIA) + .value("Meta", c10::DeviceType::Meta) + .value("HPU", c10::DeviceType::HPU) + .value("VE", c10::DeviceType::VE) + .value("Lazy", c10::DeviceType::Lazy) + .value("IPU", c10::DeviceType::IPU) + .value("PrivateUse1", c10::DeviceType::PrivateUse1); + + using torch::autograd::CreationMeta; + py::enum_(m, "CreationMeta") + .value("DEFAULT", CreationMeta::DEFAULT) + .value("IN_CUSTOM_FUNCTION", CreationMeta::IN_CUSTOM_FUNCTION) + .value("MULTI_OUTPUT_NODE", CreationMeta::MULTI_OUTPUT_NODE) + .value("NO_GRAD_MODE", CreationMeta::NO_GRAD_MODE) + .value("INFERENCE_MODE", CreationMeta::INFERENCE_MODE); + + py::class_(m, "_InputMetadata") + .def_property_readonly( + "dtype", + [](const torch::autograd::InputMetadata& m) { + PyObject* raw_obj = + (PyObject*)torch::getTHPDtype(m.dtype().toScalarType()); + return py::reinterpret_borrow(raw_obj); + }) + .def_property_readonly("device", &torch::autograd::InputMetadata::device) + .def_property_readonly( + "shape", &torch::autograd::InputMetadata::shape_as_dim_vector) + .def_property_readonly( + "is_nested_tensor", &torch::autograd::InputMetadata::is_nested_tensor) + .def_property_readonly( + "is_cpp_nested_tensor", + &torch::autograd::InputMetadata::is_cpp_nested_tensor); + + py::class_(m, "_KinetoEvent") + // name of the event + .def("name", [](const KinetoEvent& e) { return e.name(); }) + .def( + "overload_name", + [](const KinetoEvent& e) { return e.overload_name(); }) + // PyTorch thread id of the start callback + .def( + "start_thread_id", + [](const KinetoEvent& e) { return e.startThreadId(); }) + // PyTorch thread id of the end callback + .def( + "end_thread_id", [](const KinetoEvent& e) { return e.endThreadId(); }) + // for events of scope BACKWARD_FUNCTION - PyTorch thread id + // of the corresponding forward op + .def( + "fwd_thread_id", [](const KinetoEvent& e) { return e.fwdThreadId(); }) + // together with fwd_thread_id, used to uniquely identify + // the forward op + .def("sequence_nr", [](const KinetoEvent& e) { return e.sequenceNr(); }) + // absolute start time (since unix epoch) in ns + .def("start_ns", [](const KinetoEvent& e) { return e.startNs(); }) + // absolute end time (since unix epoch) in ns + .def("end_ns", [](const KinetoEvent& e) { return e.endNs(); }) + // duration in ns + .def("duration_ns", [](const KinetoEvent& e) { return e.durationNs(); }) + // used for correlation between high-level PyTorch events + // and low-level device events + .def( + "correlation_id", + [](const KinetoEvent& e) { return e.correlationId(); }) + // shapes of input tensors + .def("shapes", [](const KinetoEvent& e) { return e.shapes().vec(); }) + .def("dtypes", [](const KinetoEvent& e) { return e.dtypes().vec(); }) + .def( + "concrete_inputs", + [](const KinetoEvent& e) { + std::vector as_pyobj; + std::transform( + e.concreteInputs().begin(), + e.concreteInputs().end(), + std::back_inserter(as_pyobj), + [](const c10::IValue& val) { + return torch::jit::toPyObject(val); + }); + return as_pyobj; + }) + .def( + "kwinputs", + [](const KinetoEvent& e) { + std::unordered_map inputs; + for (const auto& [key, value] : e.kwinputs()) { + inputs[key] = torch::jit::toPyObject(value); + } + return inputs; + }) + // stack traces of the PyTorch CPU events + .def("stack", [](const KinetoEvent& e) { return e.stack().vec(); }) + // type of the RecordFunction that generated a PyTorch CPU event + // (op, torchscript function, user label, etc) + .def("scope", [](const KinetoEvent& e) { return e.scope(); }) + // device number, for CPU - process id + .def("device_index", [](const KinetoEvent& e) { return e.deviceIndex(); }) + // for CUDA - stream id, for CPU - start thread id + .def( + "device_resource_id", + [](const KinetoEvent& e) { return e.deviceResourceId(); }) + // device type + .def("device_type", [](const KinetoEvent& e) { return e.deviceType(); }) + // correlation id of a linked event + .def( + "linked_correlation_id", + [](const KinetoEvent& e) { return e.linkedCorrelationId(); }) + .def("flow_id", [](const KinetoEvent& e) { return e.flowId(); }) + .def("flow_type", [](const KinetoEvent& e) { return e.flowType(); }) + .def("flow_start", [](const KinetoEvent& e) { return e.flowStart(); }) + .def("external_id", [](const KinetoEvent& e) { return e.externalId(); }) + // compute flops + .def("flops", [](const KinetoEvent& e) { return e.flops(); }) + // Whether this is async event or not + .def("is_async", [](const KinetoEvent& e) { return e.isAsync(); }) + .def("cuda_elapsed_us", &KinetoEvent::cudaElapsedUs) + .def("privateuse1_elapsed_us", &KinetoEvent::privateuse1ElapsedUs) + .def( + "is_user_annotation", + [](const KinetoEvent& e) { + return e.activityType() == + (uint8_t)libkineto::ActivityType::USER_ANNOTATION || + e.activityType() == + (uint8_t)libkineto::ActivityType::GPU_USER_ANNOTATION; + }) + .def( + "is_python_function", + [](const KinetoEvent& e) { return e.isPythonFunction(); }) + .def("nbytes", [](const KinetoEvent& e) { return e.nBytes(); }) + // whether the event is hidden + .def( + "is_hidden_event", + [](const KinetoEvent& e) { return e.isHiddenEvent(); }) + // KinetoEvent metadata + .def( + "metadata_json", + [](const KinetoEvent& e) { return e.metadataJson(); }) + .def( + "activity_type", + [](const KinetoEvent& e) { + return libkineto::toString( + static_cast(e.activityType())); + }) + .def("extra_meta", [](const KinetoEvent& e) { return e.extraMeta(); }) + // Like shapes/strides, but also contains TensorList input shapes. + .def( + "structured_input_shapes", + [](const KinetoEvent& e) { + py::list result; + for (const auto& s : e.structuredInputShapes()) { + if (std::holds_alternative>(s)) { + result.append(std::get>(s)); + } else { + result.append(std::get>>(s)); + } + } + return result; + }) + .def( + "structured_input_strides", + [](const KinetoEvent& e) { + py::list result; + for (const auto& s : e.structuredInputStrides()) { + if (std::holds_alternative>(s)) { + result.append(std::get>(s)); + } else { + result.append(std::get>>(s)); + } + } + return result; + }) + .def("python_id", [](const KinetoEvent& e) { return e.pythonId(); }) + .def( + "python_parent_id", + [](const KinetoEvent& e) { return e.pythonParentId(); }) + .def("python_module_id", [](const KinetoEvent& e) { + return e.pythonModuleId(); + }); + + m.def("_soft_assert_raises", &setSoftAssertRaises); + m.def("_get_sequence_nr", &at::sequence_number::peek); + +#ifdef USE_KINETO + py::class_(m, "_ITraceActivity") + .def("name", &libkineto::ITraceActivity::name) + .def("timestamp", &libkineto::ITraceActivity::timestamp) + .def("duration", &libkineto::ITraceActivity::duration) + .def("device_id", &libkineto::ITraceActivity::deviceId) + .def("resource_id", &libkineto::ITraceActivity::resourceId) + .def("correlation_id", &libkineto::ITraceActivity::correlationId) + .def("flow_id", &libkineto::ITraceActivity::flowId) + .def("flow_type", &libkineto::ITraceActivity::flowType) + .def("flow_start", &libkineto::ITraceActivity::flowStart) + .def( + "type", + [](const libkineto::ITraceActivity& a) { + return libkineto::toString(a.type()); + }) + .def("metadata_json", &libkineto::ITraceActivity::metadataJson) + .def( + "linked_correlation_id", + [](const libkineto::ITraceActivity& a) -> int64_t { + auto* linked = a.linkedActivity(); + return linked ? linked->correlationId() : 0; + }) + .def( + "linked_activity", + [](const libkineto::ITraceActivity& a) + -> const libkineto::ITraceActivity* { + return a.linkedActivity(); + }, + py::return_value_policy::reference); +#endif + + py::class_(m, "_ProfilerResult") + .def("trace_start_ns", &ProfilerResult::trace_start_ns) + .def("events", &ProfilerResult::events) + .def("experimental_event_tree", &ProfilerResult::event_tree) +#ifdef USE_KINETO + .def("save", &ProfilerResult::save) + .def( + "trace_activities", + [](py::object self) { + auto& r = self.cast(); + auto* activities = r.traceActivities(); + if (!activities) { + return py::list(); + } + py::list result(activities->size()); + for (size_t i = 0; i < activities->size(); i++) { + // reference_internal ties each element's lifetime to self, + // preventing use-after-free if the list outlives the + // ProfilerResult. + result[i] = py::cast( + (*activities)[i], + py::return_value_policy::reference_internal, + self); + } + return result; + }) +#endif // USE_KINETO + ; + + m.def( + "_enable_profiler", + &enableProfiler, + py::arg("config"), + py::arg("activities"), + py::arg("scopes") = std::unordered_set()); + m.def( + "_disable_profiler", + disableProfiler, + py::call_guard()); + m.def( + "_prepare_profiler", + prepareProfiler, + py::arg("config"), + py::arg("activities"), + py::arg("activity_filter") = torch::autograd::profiler::ActivityFilter{}, + py::call_guard()); + m.def( + "_toggle_collection_dynamic", + toggleCollectionDynamic, + py::call_guard()); + m.def("_is_kineto_stopped", isKinetoStopped); + m.def("_add_metadata_json", addMetadataJson); // Only if `USE_KINETO` is set + m.def("_kineto_step", profilerStep); // Only if `USE_KINETO` is set + m.def("kineto_available", []() { return torch::profiler::kKinetoAvailable; }); + + // NOTICE: These record functions are not torch operators and may not show up + // in TorchScript tracing, FX transforms, or operator serialization. For these + // use cases, please use `torch.profiler.record_function`. + // Creates a new profiling scope using RecordFunction and invokes its starting + // callbacks. + m.def( + "_record_function_with_args_enter", + [](const std::string& name, const py::args& args) { + using torch::autograd::profiler::PythonRecordFunction; + auto python_rec = c10::make_intrusive( + at::RecordScope::USER_SCOPE); + auto* rec = &python_rec->record; + if (rec->isActive()) { + if (rec->needsInputs()) { + auto iv_inputs = std::vector(); + for (const auto& arg : args) { + iv_inputs.push_back(torch::jit::toTypeInferredIValue(arg)); + } + rec->before( + name, + c10::ArrayRef( + iv_inputs.data(), iv_inputs.size())); + } else { + rec->before(name); + } + } + return torch::jit::toPyObject(std::move(python_rec)); + }); + + // Ends the profiling scope created with record_function_with_param_enter. + m.def("_record_function_with_args_exit", [](const py::object& obj) { + using torch::autograd::profiler::PythonRecordFunction; + auto python_record = torch::jit::toCustomClass(obj); + + // We don't actually need to do anything with handle just need to persist + // the lifetime until now. + python_record->record.end(); + }); + + m.def("_supported_activities", []() { + std::set activities{ + torch::profiler::impl::ActivityType::CPU}; +#if defined(USE_KINETO) +#if defined(HAS_CUPTI) || defined(HAS_ROCTRACER) + if (at::getNumGPUs() > 0) { + activities.insert(torch::profiler::impl::ActivityType::CUDA); + } +#endif // defined(HAS_CUPTI) || defined(HAS_ROCTRACER) + if (at::hasXPU()) { + activities.insert(torch::profiler::impl::ActivityType::XPU); + } + if (at::hasMTIA()) { + activities.insert(torch::profiler::impl::ActivityType::MTIA); + } + if (at::hasHPU()) { + activities.insert(torch::profiler::impl::ActivityType::HPU); + } + if (c10::get_privateuse1_backend() != "privateuseone") { + activities.insert(torch::profiler::impl::ActivityType::PrivateUse1); + } +#endif // defined(USE_KINETO) + return activities; + }); + + m.def( + "_unsafe_set_version_counter", + [](const std::vector& tensors, + const std::vector& versions) { + auto tensors_len = tensors.size(); + auto versions_len = versions.size(); + TORCH_CHECK( + tensors_len == versions_len, + "tensors_len=", + tensors_len, + ", versions_len=", + versions_len); + for (const auto i : c10::irange(tensors_len)) { + auto vc = torch::autograd::impl::version_counter(tensors[i]); + vc.set_version(versions[i]); + } + }); + + m.def("_enable_profiler_legacy", enableProfilerLegacy); + py::class_(m, "_ProfilerDisableOptions") + .def(py::init()); + m.def( + "_disable_profiler_legacy", + disableProfilerLegacy, + py::arg("profiler_disable_options") = ProfilerDisableOptions()); + m.def("_profiler_enabled", profilerEnabled); + m.def("_profiler_type", torch::profiler::impl::profilerType); + m.def("_enable_record_function", [](bool enable) { + at::enableRecordFunction(enable); + }); + m.def("_set_empty_test_observer", [](bool is_global, double sampling_prob) { + auto cb = + at::RecordFunctionCallback(nullptr).needsInputs(true).samplingProb( + sampling_prob); + if (is_global) { + at::addGlobalCallback(cb); + } else { + at::addThreadLocalCallback(cb); + } + }); + m.def("_clear_callbacks", []() { at::clearCallbacks(); }); + m.def( + "_saved_tensors_hooks_is_enabled", + at::SavedTensorDefaultHooks::is_enabled); + m.def("_saved_tensors_hooks_enable", at::SavedTensorDefaultHooks::enable); + m.def( + "_saved_tensors_hooks_disable", + at::SavedTensorDefaultHooks::disable, + py::arg("error_message"), + py::arg("fail_if_non_empty") = true); + m.def( + "_saved_tensors_hooks_set_tracing", + at::SavedTensorDefaultHooks::set_tracing); + m.def( + "_saved_tensors_hooks_get_disabled_error_message", + at::SavedTensorDefaultHooks::get_disabled_error_message); + m.def( + "_push_saved_tensors_default_hooks", + [](py::function& pack_hook, py::function& unpack_hook) { + torch::autograd::PyDefaultSavedVariableHooks::push_hooks( + pack_hook, unpack_hook); + }); + m.def("_pop_saved_tensors_default_hooks", []() { + torch::autograd::PyDefaultSavedVariableHooks::pop_hooks(); + }); + m.def( + "_top_saved_tensors_default_hooks", + [](bool ignore_is_tracing) + -> std::optional> { + auto out = at::SavedTensorDefaultHooks::get_hooks(ignore_is_tracing); + + if (!out.has_value()) { + return std::nullopt; + } + + auto [pack_hook, unpack_hook] = *out; + // gil for destructor of pack_hook, unpack_hook that decrements + // reference + py::gil_scoped_acquire gil; + + return std::make_pair( + py::reinterpret_steal(pack_hook.release()), + py::reinterpret_steal(unpack_hook.release())); + } + + ); + + m.def("_get_creation_meta", [](const at::Tensor& t) { + auto* meta = torch::autograd::impl::get_view_autograd_meta(t); + TORCH_CHECK(meta != nullptr); + return meta->get_creation_meta(); + }); + + m.def( + "_set_creation_meta", + [](const at::Tensor& t, CreationMeta new_creation_meta) { + auto* meta = torch::autograd::impl::get_view_autograd_meta(t); + TORCH_CHECK(meta != nullptr); + meta->set_creation_meta(new_creation_meta); + }); + + m.def("_get_current_graph_task_keep_graph", []() { + return torch::autograd::get_current_graph_task_keep_graph(); + }); + + m.def( + "_get_data_attr", [](const at::Tensor& t) { return t.variable_data(); }); + + _C_m.def( + "_register_py_class_for_device", + [](const std::string& device, py::object python_type_class) { + auto cls = python_type_class.ptr(); + registerPythonTensorClass(device, cls); + }); + _C_m.def("_set_autograd_fallback_mode", [](const std::string& mode) { + if (mode == "nothing") { + torch::autograd::setAutogradFallbackMode( + torch::autograd::AutogradFallbackMode::Nothing); + return; + } + if (mode == "warn") { + torch::autograd::setAutogradFallbackMode( + torch::autograd::AutogradFallbackMode::Warn); + return; + } + if (mode == "error") { + torch::autograd::setAutogradFallbackMode( + torch::autograd::AutogradFallbackMode::Error); + return; + } + TORCH_INTERNAL_ASSERT(false, "Unsupported AutogradFallbackMode: ", mode); + }); + _C_m.def("_get_autograd_fallback_mode", []() { + auto mode = torch::autograd::getAutogradFallbackMode(); + switch (mode) { + case torch::autograd::AutogradFallbackMode::Nothing: + return "nothing"; + case torch::autograd::AutogradFallbackMode::Warn: + return "warn"; + case torch::autograd::AutogradFallbackMode::Error: + return "error"; + default: + TORCH_INTERNAL_ASSERT(false, "Unsupported AutogradFallbackMode"); + } + }); + + _C_m.def("_activate_gpu_trace", []() { activateGPUTrace(); }); + + py_context_manager_DEPRECATED( + _C_m, "_InferenceMode"); + py_context_manager( + _C_m, "_RestorePythonTLSSnapshot"); + + py_context_manager_DEPRECATED( + _C_m, "_DisableTorchDispatch"); + py_context_manager_DEPRECATED( + _C_m, "_EnableTorchFunction"); + py_context_manager_DEPRECATED( + _C_m, "_EnablePythonDispatcher"); + py_context_manager( + _C_m, "_DisablePythonDispatcher"); + py_context_manager(_C_m, "_EnablePreDispatch"); + py_context_manager_DEPRECATED(_C_m, "_DisableFuncTorch"); + py_context_manager(_C_m, "_DisableAutocast"); + py::class_(std::move(m), "SavedTensor") + .def(py::init([]() -> torch::autograd::SavedVariable { + TORCH_CHECK( + false, + "Trying to create a SavedTensor object from Python is forbidden."); + })) + .def( + "unpack", + [](const torch::autograd::SavedVariable& s) -> at::Tensor { + return s.unpack(); + }) + .def( + "register_hooks", + [](torch::autograd::SavedVariable& s, + py::function& pack_hook, + py::function& unpack_hook) { + // Because we use a py::object, pybind will increment the refcount + // of the hook functions for us + s.register_hooks( + std::make_unique( + pack_hook, unpack_hook)); + }) + .def_property_readonly( + "data", + [](const torch::autograd::SavedVariable& s) -> py::object { + if (s.has_hooks()) { + auto opt = s.retrieve_unpack_hook_data(); + TORCH_INTERNAL_ASSERT(opt.has_value()); + py::gil_scoped_acquire gil; + const auto& [_unpack_fn, data_obj] = *opt; + PyObject* raw = data_obj.ptr(getPyInterpreter()); + TORCH_INTERNAL_ASSERT(raw != nullptr); + return py::reinterpret_borrow(raw); + } else { + return py::cast(s.get_raw_data().value()); + } + }) + .def_property_readonly( + "unpack_hook", + [](const torch::autograd::SavedVariable& s) -> py::object { + auto opt = s.retrieve_unpack_hook_data(); + if (!opt.has_value()) { + return py::none(); + } + py::gil_scoped_acquire gil; + const auto& [unpack_safe, _unused_data] = *opt; + auto* unpack_ptr = unpack_safe.ptr(getPyInterpreter()); + return py::reinterpret_borrow(unpack_ptr); + }); + + m.def( + "_make_saved_tensor", + [](const at::Tensor& tensor, + bool is_output, + bool is_inplace_on_view) -> torch::autograd::SavedVariable { + return torch::autograd::SavedVariable( + tensor, is_output, is_inplace_on_view); + }, + py::arg("tensor"), + py::arg("is_output"), + py::arg("is_inplace_on_view") = false); + + torch::autograd::profiler::python_tracer::init(); + Py_RETURN_TRUE; +} + +namespace torch::autograd { + +static PyObject* set_autocast_enabled( + PyObject* _unused, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser( + {"set_autocast_enabled(std::string_view device_type, bool enabled)", + "set_autocast_enabled(bool enabled)"}); // this signature is deprecated. + ParsedArgs<2> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + // Set at::kCUDA as default value to prevent BC-breaking changes. + at::DeviceType device_type = at::kCUDA; + int enabled_id = 0; + if (r.idx == 0) { + device_type = at::Device(r.string(0)).type(); + enabled_id = 1; + } + auto enabled = r.toBool(enabled_id); + at::autocast::set_autocast_enabled(device_type, enabled); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_autocast_enabled( + PyObject* _unused, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser( + {"is_autocast_enabled(std::string_view device_type)", + "is_autocast_enabled()"}); // this signature is deprecated. + ParsedArgs<1> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + // Set at::kCUDA as default value to prevent BC-breaking changes. + at::DeviceType device_type = at::kCUDA; + if (r.idx == 0) { + device_type = at::Device(r.string(0)).type(); + } + if (at::autocast::is_autocast_enabled(device_type)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* get_autocast_dtype( + PyObject* _unused, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser( + {"get_autocast_dtype(std::string_view device_type)"}); + ParsedArgs<1> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + auto device_type = at::Device(r.string(0)).type(); + at::ScalarType current_dtype = at::autocast::get_autocast_dtype(device_type); + return utils::wrap(current_dtype); + END_HANDLE_TH_ERRORS +} + +static PyObject* set_autocast_dtype( + PyObject* _unused, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser( + {"set_autocast_dtype(std::string_view device_type, ScalarType dtype)"}); + ParsedArgs<2> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + auto device_type = at::Device(r.string(0)).type(); + auto dtype = r.scalartype(1); + at::autocast::set_autocast_dtype(device_type, dtype); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_any_autocast_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + if (at::autocast::is_autocast_enabled(at::kCPU) || + at::autocast::is_autocast_enabled(at::kCUDA) || + at::autocast::is_autocast_enabled(at::kXPU) || + at::autocast::is_autocast_enabled(at::kIPU) || + at::autocast::is_autocast_enabled(at::kXLA) || + at::autocast::is_autocast_enabled(at::kHPU) || + at::autocast::is_autocast_enabled(at::kMTIA) || + at::autocast::is_autocast_enabled(at::kPrivateUse1)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* is_autocast_available( + PyObject* _unused, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser( + {"_is_autocast_available(std::string_view device_type)"}); + ParsedArgs<1> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + auto device_type = at::Device(r.string(0)).type(); + if (at::autocast::is_autocast_available(device_type)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* set_autocast_cpu_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_CHECK_TYPE( + PyBool_Check(arg), + "enabled must be a bool (got ", + Py_TYPE(arg)->tp_name, + ")"); + TORCH_WARN_DEPRECATION( + "torch.set_autocast_cpu_enabled(enabled) is deprecated. Please use torch.set_autocast_enabled('cpu', enabled) instead.") + at::autocast::set_autocast_enabled(at::kCPU, Py_IsTrue(arg)); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_autocast_cpu_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_WARN_DEPRECATION( + "torch.is_autocast_cpu_enabled() is deprecated. Please use torch.is_autocast_enabled('cpu') instead.") + if (at::autocast::is_autocast_enabled(at::kCPU)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* set_autocast_ipu_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_CHECK_TYPE( + PyBool_Check(arg), + "enabled must be a bool (got ", + Py_TYPE(arg)->tp_name, + ")"); + TORCH_WARN_DEPRECATION( + "torch.set_autocast_ipu_enabled(enabled) is deprecated. Please use torch.set_autocast_enabled('ipu', enabled) instead.") + at::autocast::set_autocast_enabled(at::kIPU, Py_IsTrue(arg)); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_autocast_ipu_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_WARN_DEPRECATION( + "torch.is_autocast_ipu_enabled() is deprecated. Please use torch.is_autocast_enabled('ipu') instead.") + if (at::autocast::is_autocast_enabled(at::kIPU)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* set_autocast_xla_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_CHECK_TYPE( + PyBool_Check(arg), + "enabled must be a bool (got ", + Py_TYPE(arg)->tp_name, + ")"); + TORCH_WARN_DEPRECATION( + "torch.set_autocast_xla_enabled(enabled) is deprecated. Please use torch.set_autocast_enabled('xla', enabled) instead.") + at::autocast::set_autocast_enabled(at::kXLA, Py_IsTrue(arg)); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_autocast_xla_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_WARN_DEPRECATION( + "torch.is_autocast_xla_enabled() is deprecated. Please use torch.is_autocast_enabled('xla') instead.") + if (at::autocast::is_autocast_enabled(at::kXLA)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* set_autocast_gpu_dtype(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_CHECK_TYPE( + THPDtype_Check(arg), + "dtype must be a torch.dtype (got ", + Py_TYPE(arg)->tp_name, + ")"); + TORCH_WARN_DEPRECATION( + "torch.set_autocast_gpu_dtype(dtype) is deprecated. Please use torch.set_autocast_dtype('cuda', dtype) instead.") + at::ScalarType targetType = reinterpret_cast(arg)->scalar_type; + at::autocast::set_autocast_dtype(at::kCUDA, targetType); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* set_autocast_cpu_dtype(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_CHECK_TYPE( + THPDtype_Check(arg), + "dtype must be a torch.dtype (got ", + Py_TYPE(arg)->tp_name, + ")"); + TORCH_WARN_DEPRECATION( + "torch.set_autocast_cpu_dtype(dtype) is deprecated. Please use torch.set_autocast_dtype('cpu', dtype) instead.") + at::ScalarType targetType = reinterpret_cast(arg)->scalar_type; + at::autocast::set_autocast_dtype(at::kCPU, targetType); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* set_autocast_ipu_dtype(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_CHECK_TYPE( + THPDtype_Check(arg), + "dtype must be a torch.dtype (got ", + Py_TYPE(arg)->tp_name, + ")"); + TORCH_WARN_DEPRECATION( + "torch.set_autocast_ipu_dtype(dtype) is deprecated. Please use torch.set_autocast_dtype('ipu', dtype) instead.") + at::ScalarType targetType = reinterpret_cast(arg)->scalar_type; + at::autocast::set_autocast_dtype(at::kIPU, targetType); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* set_autocast_xla_dtype(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_CHECK_TYPE( + THPDtype_Check(arg), + "dtype must be a torch.dtype (got ", + Py_TYPE(arg)->tp_name, + ")"); + TORCH_WARN_DEPRECATION( + "torch.set_autocast_xla_dtype(dtype) is deprecated. Please use torch.set_autocast_dtype('xla', dtype) instead.") + at::ScalarType targetType = reinterpret_cast(arg)->scalar_type; + at::autocast::set_autocast_dtype(at::kXLA, targetType); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* get_autocast_gpu_dtype(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_WARN_DEPRECATION( + "torch.get_autocast_gpu_dtype() is deprecated. Please use torch.get_autocast_dtype('cuda') instead.") + at::ScalarType current_dtype = at::autocast::get_autocast_dtype(at::kCUDA); + return utils::wrap(current_dtype); + END_HANDLE_TH_ERRORS +} + +static PyObject* get_autocast_cpu_dtype(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_WARN_DEPRECATION( + "torch.get_autocast_cpu_dtype() is deprecated. Please use torch.get_autocast_dtype('cpu') instead.") + at::ScalarType current_dtype = at::autocast::get_autocast_dtype(at::kCPU); + return utils::wrap(current_dtype); + END_HANDLE_TH_ERRORS +} + +static PyObject* get_autocast_ipu_dtype(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_WARN_DEPRECATION( + "torch.get_autocast_ipu_dtype() is deprecated. Please use torch.get_autocast_dtype('ipu') instead.") + at::ScalarType current_dtype = at::autocast::get_autocast_dtype(at::kIPU); + return utils::wrap(current_dtype); + END_HANDLE_TH_ERRORS +} + +static PyObject* get_autocast_xla_dtype(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_WARN_DEPRECATION( + "torch.get_autocast_xla_dtype() is deprecated. Please use torch.get_autocast_dtype('xla') instead.") + at::ScalarType current_dtype = at::autocast::get_autocast_dtype(at::kXLA); + return utils::wrap(current_dtype); + END_HANDLE_TH_ERRORS +} + +static PyObject* clear_autocast_cache(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS { + pybind11::gil_scoped_release no_gil; + at::autocast::clear_cache(); + } + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* autocast_increment_nesting(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + return THPUtils_packInt64(at::autocast::increment_nesting()); + END_HANDLE_TH_ERRORS +} + +static PyObject* autocast_decrement_nesting(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + return THPUtils_packInt64(at::autocast::decrement_nesting()); + END_HANDLE_TH_ERRORS +} + +static PyObject* is_autocast_cache_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + if (at::autocast::is_autocast_cache_enabled()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* set_autocast_cache_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_CHECK_TYPE( + PyBool_Check(arg), + "enabled must be a bool (got ", + Py_TYPE(arg)->tp_name, + ")"); + at::autocast::set_autocast_cache_enabled(Py_IsTrue(arg)); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* set_grad_enabled( + PyObject* _unused, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser({ + "set_grad_enabled(bool enabled)", + }); + ParsedArgs<1> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + + if (at::impl::torch_function_mode_enabled()) { + auto torch_C_module = THPObjectPtr(PyImport_ImportModule("torch._C")); + return handle_torch_function( + r, args, kwargs, torch_C_module, "torch._C", "_set_grad_enabled"); + } + auto grad_enabled = r.toBool(0); + GradMode::set_enabled(grad_enabled); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_grad_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + if (GradMode::is_enabled()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* set_fwd_grad_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_CHECK_TYPE( + PyBool_Check(arg), + "enabled must be a bool (got ", + Py_TYPE(arg)->tp_name, + ")"); + c10::AutogradState::get_tls_state().set_fw_grad_mode(Py_IsTrue(arg)); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_fwd_grad_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + if (c10::AutogradState::get_tls_state().get_fw_grad_mode()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +template +static bool visit( + PyObject* o, + const std::function& visit_tensor) { + if (THPVariable_Check(o)) { + auto t = THPVariable_Unpack(o); + if (visit_tensor(t)) { + return true; + } + } else if (PyList_Check(o)) { + // Check that this List is TensorList + if constexpr (skip_tensors_in_non_tensorlist) { + for (const auto i : c10::irange(PyList_GET_SIZE(o))) { + if (!THPVariable_Check(PyList_GET_ITEM(o, i))) { + return false; + } + } + } + for (const auto i : c10::irange(PyList_GET_SIZE(o))) { + if (visit( + PyList_GET_ITEM(o, i), visit_tensor)) { + return true; + }; + } + } + return false; +} + +// Visiting of tensors in args and kwargs, +// only List container is visited. +// skip_tensors_in_non_tensorlist will skip any List with non-Tensor. +// Lambda returning true means short circuit, traversal stops after that. +template +static void visit_tensors( + PyObject* args, + PyObject* kwargs, + const std::function& visit_tensor) { + if (args && PyTuple_Check(args)) { + for (const auto i : c10::irange(PyTuple_GET_SIZE(args))) { + if (visit( + PyTuple_GET_ITEM(args, i), visit_tensor)) { + return; + } + } + } + if (kwargs && PyDict_Check(kwargs)) { + auto vals = THPObjectPtr{PyDict_Values(kwargs)}; + for (const auto i : c10::irange(PyList_Size(vals))) { + if (visit( + PyList_GetItem(vals, i), visit_tensor)) { + return; + } + } + } +} + +// Returns true if any of the args, kwargs tensor leaves have requires_grad. +// Only List[Tensor] container in args is supported. +static PyObject* any_requires_grad( + PyObject* _unused, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + bool has_requires_grad = false; + visit_tensors(args, kwargs, [&has_requires_grad](at::Tensor& t) { + if (t.requires_grad()) { + has_requires_grad = true; + return true; + } + return false; + }); + if (has_requires_grad) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; + END_HANDLE_TH_ERRORS +} + +// Checks aliasing constraint for custom ops: +// Returns true if any of outputs is alias to any of inputs or another output +// Args: +// args[0] - inputs args +// args[1] - inputs kwargs +// args[2] - outputs +// Only List container is supported. +// Tensors in Lists that has not only Tensor are checked. +static PyObject* any_output_is_alias_to_input_or_output( + PyObject* _unused, + PyObject* args) { + HANDLE_TH_ERRORS + PyObject* inps = PyTuple_GET_ITEM(args, 0); + PyObject* inps_kwargs = PyTuple_GET_ITEM(args, 1); + PyObject* outs = PyTuple_GET_ITEM(args, 2); + std::unordered_set s; + visit_tensors(inps, inps_kwargs, [&s](at::Tensor& t) { + if (!t.has_storage()) { + return false; + } + auto* cp = t.storage().unsafeGetStorageImpl(); + if (cp) { + s.insert(cp); + } + return false; + }); + bool ret = false; + visit_tensors(outs, nullptr, [&s, &ret](at::Tensor& t) { + if (!t.has_storage()) { + return false; + } + auto* cp = t.storage().unsafeGetStorageImpl(); + if (!cp) { + return false; + } + if (s.find(cp) != s.end()) { + ret = true; + return true; + } + s.insert(cp); + return false; + }); + if (ret) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; + END_HANDLE_TH_ERRORS +} + +static PyObject* set_multithreading_enabled( + PyObject* self, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser({ + "set_multithreading_enabled(bool enabled)", + }); + ParsedArgs<1> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + + if (at::impl::torch_function_mode_enabled()) { + auto torch_C_module = THPObjectPtr(PyImport_ImportModule("torch._C")); + return handle_torch_function( + r, + args, + kwargs, + torch_C_module, + "torch._C", + "_set_multithreading_enabled"); + } + auto multithreading_enabled = r.toBool(0); + c10::AutogradState::get_tls_state().set_multithreading_enabled( + multithreading_enabled); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_multithreading_enabled(PyObject* self, PyObject* args) { + HANDLE_TH_ERRORS + if (c10::AutogradState::get_tls_state().get_multithreading_enabled()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* set_view_replay_enabled( + PyObject* self, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser({ + "set_view_replay_enabled(bool enabled)", + }); + ParsedArgs<1> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + + if (at::impl::torch_function_mode_enabled()) { + auto torch_C_module = THPObjectPtr(PyImport_ImportModule("torch._C")); + return handle_torch_function( + r, + args, + kwargs, + torch_C_module, + "torch._C", + "_set_view_replay_enabled"); + } + auto view_replay_enabled = r.toBool(0); + c10::AutogradState::get_tls_state().set_view_replay_enabled( + view_replay_enabled); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_view_replay_enabled(PyObject* self, PyObject* args) { + HANDLE_TH_ERRORS + if (c10::AutogradState::get_tls_state().get_view_replay_enabled()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* set_grad_layout_enforcement_enabled( + PyObject* self, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser({ + "set_grad_layout_enforcement_enabled(bool enabled)", + }); + ParsedArgs<1> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + + if (at::impl::torch_function_mode_enabled()) { + auto torch_C_module = THPObjectPtr(PyImport_ImportModule("torch._C")); + return handle_torch_function( + r, + args, + kwargs, + torch_C_module, + "torch._C", + "_set_grad_layout_enforcement_enabled"); + } + auto enabled = r.toBool(0); + c10::AutogradState::get_tls_state().set_grad_layout_enforcement_enabled( + enabled); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_grad_layout_enforcement_enabled( + PyObject* self, + PyObject* args) { + HANDLE_TH_ERRORS + if (c10::AutogradState::get_tls_state() + .get_grad_layout_enforcement_enabled()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* set_graph_exec_group(PyObject* self, PyObject* obj) { + HANDLE_TH_ERRORS + if (Py_IsNone(obj)) { + c10::AutogradState::get_tls_state().set_graph_exec_group(std::nullopt); + } else { + Py_INCREF(obj); + c10::AutogradState::get_tls_state().set_graph_exec_group( + c10::SafePyObject(obj, getPyInterpreter())); + } + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* get_graph_exec_group(PyObject* self, PyObject* args) { + HANDLE_TH_ERRORS + const auto& group = + c10::AutogradState::get_tls_state().get_graph_exec_group(); + if (group.has_value()) { + PyObject* obj = group->ptr(getPyInterpreter()); + return Py_NewRef(obj); + } else { + Py_RETURN_NONE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* is_inference_mode_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + if (c10::InferenceMode::is_enabled()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* set_anomaly_mode_enabled( + PyObject* _unused, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser({ + "set_anomaly_enabled(bool enabled, bool check_nan=True)", + }); + ParsedArgs<2> parsed_args; + auto r = parser.parse(args, kwargs, parsed_args); + AnomalyMode::set_enabled(r.toBool(0), r.toBool(1)); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_anomaly_mode_enabled(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + if (AnomalyMode::is_enabled()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* is_anomaly_check_nan_enabled( + PyObject* _unused, + PyObject* arg) { + HANDLE_TH_ERRORS + if (AnomalyMode::should_check_nan()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* python_enter_dual_level(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + // It is unlikely that the depth of forward nesting will overflow int64_t so + // we just static cast here. + return utils::wrap(static_cast(forward_ad::enter_dual_level())); + END_HANDLE_TH_ERRORS +} + +static PyObject* python_exit_dual_level( + PyObject* _unused, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser({"exit_dual_level(int64_t level)"}); + + ParsedArgs<1> parsed_args; + auto _r = parser.parse(args, kwargs, parsed_args); + + auto idx = _r.toInt64(0); + // Make sure the given index is valid before casting it + TORCH_CHECK(idx >= 0, "Dual level must be a positive number."); + forward_ad::exit_dual_level(static_cast(idx)); + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* is_torch_function_mode_enabled( + PyObject* _unused, + PyObject* _unused2) { + HANDLE_TH_ERRORS + if (at::impl::torch_function_mode_enabled()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + END_HANDLE_TH_ERRORS +} + +static PyObject* push_on_torch_function_stack( + PyObject* _unused, + PyObject* arg) { + HANDLE_TH_ERRORS + if (!Py_IsNone(arg)) { + Py_INCREF(arg); + at::impl::PythonTorchFunctionTLS::push_onto_stack( + std::make_shared(arg, getPyInterpreter())); + } + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* pop_torch_function_stack( + PyObject* _unused, + PyObject* _unused2) { + HANDLE_TH_ERRORS + const auto& mode = at::impl::PythonTorchFunctionTLS::pop_stack(); + auto* r = mode->ptr(getPyInterpreter()); + return Py_NewRef(r); + END_HANDLE_TH_ERRORS +} + +static PyObject* get_function_stack_at( + PyObject* _unused, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser({"get_stack_at(int64_t level)"}); + + ParsedArgs<1> parsed_args; + auto _r = parser.parse(args, kwargs, parsed_args); + + auto idx = _r.toInt64(0); + const auto& mode = at::impl::PythonTorchFunctionTLS::get_stack_at(idx); + auto* r = mode->ptr(getPyInterpreter()); + return Py_NewRef(r); + END_HANDLE_TH_ERRORS +} + +static PyObject* len_torch_function_stack( + PyObject* _unused, + PyObject* _unused2) { + HANDLE_TH_ERRORS + const auto len = at::impl::PythonTorchFunctionTLS::stack_len(); + return utils::wrap(static_cast(len)); + END_HANDLE_TH_ERRORS +} + +static PyObject* push_on_torch_dispatch_stack( + PyObject* _unused, + PyObject* arg) { + HANDLE_TH_ERRORS + if (!Py_IsNone(arg)) { + using c10::impl::TorchDispatchModeKey; + // When we push a mode onto the mode stack, we need to + // check if it's an "infra" mode, by checking its _mode_key attribute. + std::optional mode_key; + py::object maybe_mode_key_obj = + PyObject_FastGetAttrString(arg, "_mode_key"); + if (maybe_mode_key_obj) { + mode_key = py::cast(maybe_mode_key_obj); + c10::impl::TorchDispatchModeTLS::set_mode( + std::make_shared( + arg, getPyInterpreter()), + mode_key.value()); + } else { + c10::impl::TorchDispatchModeTLS::push_non_infra_mode_onto_stack( + std::make_shared( + arg, getPyInterpreter())); + } + Py_INCREF(arg); + } + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* pop_torch_dispatch_stack( + PyObject* _unused, + PyObject* maybe_mode_key) { + HANDLE_TH_ERRORS + std::optional mode_key; + PyObject* r = nullptr; + // Keep the mode alive until after Py_INCREF to prevent use-after-free. + // When the shared_ptr is destroyed, ~SafePyObject will Py_DECREF, so we must + // Py_INCREF first to give the caller a valid reference. + std::shared_ptr mode; + if (!Py_IsNone(maybe_mode_key)) { + mode_key = py::cast(maybe_mode_key); + auto maybe_mode = + c10::impl::TorchDispatchModeTLS::unset_mode(mode_key.value()); + TORCH_CHECK( + maybe_mode.has_value(), + "Attempted to unset ", + c10::impl::to_string(mode_key.value()), + ", but there wasn't one active."); + mode = maybe_mode.value(); + } else { + mode = c10::impl::TorchDispatchModeTLS::pop_stack(); + } + r = mode->ptr(getPyInterpreter()); + // Increment refcount to give Python a reference. The SafePyObject destructor + // will decref when the shared_ptr is destroyed, so this balances out. + // Note: We cannot use release() here because the SafePyObject may be shared + // via ThreadLocalState copies, and release() would null out data_ causing + // other shared_ptr holders to see nullptr. + return Py_NewRef(r); + END_HANDLE_TH_ERRORS +} + +static PyObject* get_dispatch_stack_at( + PyObject* _unused, + PyObject* args, + PyObject* kwargs) { + HANDLE_TH_ERRORS + static PythonArgParser parser({"get_stack_at(int64_t level)"}); + + ParsedArgs<1> parsed_args; + auto _r = parser.parse(args, kwargs, parsed_args); + + auto idx = _r.toInt64(0); + const auto& mode = c10::impl::TorchDispatchModeTLS::get_stack_at(idx); + auto* r = mode->ptr(getPyInterpreter()); + return Py_NewRef(r); + END_HANDLE_TH_ERRORS +} + +static PyObject* set_dispatch_mode(PyObject* _unused, PyObject* mode) { + HANDLE_TH_ERRORS + TORCH_CHECK(!Py_IsNone(mode)); + + py::object maybe_mode_key_obj = PyObject_FastGetAttrString(mode, "_mode_key"); + TORCH_CHECK( + maybe_mode_key_obj, + "set_dispatch_mode() called with a mode that does not contain a _mode_key attribute!"); + auto mode_key = py::cast(maybe_mode_key_obj); + + Py_INCREF(mode); + c10::impl::TorchDispatchModeTLS::set_mode( + std::make_shared( + mode, getPyInterpreter()), + mode_key); + + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +static PyObject* get_dispatch_mode(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_CHECK(!Py_IsNone(arg)); + auto mode_key = py::cast(arg); + + auto maybe_mode = c10::impl::TorchDispatchModeTLS::get_mode(mode_key); + if (!maybe_mode.has_value()) { + Py_RETURN_NONE; + } + auto* r = maybe_mode.value()->ptr(getPyInterpreter()); + return Py_NewRef(r); + END_HANDLE_TH_ERRORS +} + +static PyObject* unset_dispatch_mode(PyObject* _unused, PyObject* arg) { + HANDLE_TH_ERRORS + TORCH_CHECK(!Py_IsNone(arg)); + auto mode_key = py::cast(arg); + + const auto maybe_mode = c10::impl::TorchDispatchModeTLS::unset_mode(mode_key); + if (!maybe_mode.has_value()) { + Py_RETURN_NONE; + } + auto* r = maybe_mode.value()->ptr(getPyInterpreter()); + return Py_NewRef(r); + END_HANDLE_TH_ERRORS +} + +static PyObject* len_torch_dispatch_stack(PyObject* _unused, PyObject* args) { + HANDLE_TH_ERRORS + const auto len = c10::impl::TorchDispatchModeTLS::stack_len(); + return utils::wrap(static_cast(len)); + END_HANDLE_TH_ERRORS +} + +static PyObject* THPModule_increment_version( + PyObject* _unused, + PyObject* tensor_list) { + HANDLE_TH_ERRORS + auto iterator = THPObjectPtr(PyObject_GetIter(tensor_list)); + TORCH_CHECK(iterator, "increment_version expect a Iterable[Tensor] as input"); + auto item = THPObjectPtr(PyIter_Next(iterator)); + while (item) { + TORCH_CHECK( + THPVariable_Check(item), + "increment_version expects each element of the iterable to be a tensor"); + auto t = THPVariable_Unpack(item); + if (!t.is_inference()) { + torch::autograd::increment_version(t); + } + item = THPObjectPtr(PyIter_Next(iterator)); + } + Py_RETURN_NONE; + END_HANDLE_TH_ERRORS +} + +// autograd methods on torch._C +// NOLINTNEXTLINE(*array*) +static PyMethodDef methods[] = { + {"_set_grad_enabled", + castPyCFunctionWithKeywords(set_grad_enabled), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"is_grad_enabled", is_grad_enabled, METH_NOARGS, nullptr}, + {"_set_fwd_grad_enabled", set_fwd_grad_enabled, METH_O, nullptr}, + {"_any_requires_grad", + castPyCFunctionWithKeywords(any_requires_grad), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"_any_output_is_alias_to_input_or_output", + any_output_is_alias_to_input_or_output, + METH_VARARGS, + nullptr}, + {"_is_fwd_grad_enabled", is_fwd_grad_enabled, METH_NOARGS, nullptr}, + {"is_inference_mode_enabled", + is_inference_mode_enabled, + METH_NOARGS, + nullptr}, + {"set_autocast_enabled", + castPyCFunctionWithKeywords(set_autocast_enabled), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"is_autocast_enabled", + castPyCFunctionWithKeywords(is_autocast_enabled), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"set_autocast_dtype", + castPyCFunctionWithKeywords(set_autocast_dtype), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"get_autocast_dtype", + castPyCFunctionWithKeywords(get_autocast_dtype), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"_is_any_autocast_enabled", is_any_autocast_enabled, METH_NOARGS, nullptr}, + {"_is_autocast_available", + castPyCFunctionWithKeywords(is_autocast_available), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"clear_autocast_cache", clear_autocast_cache, METH_NOARGS, nullptr}, + {"set_autocast_cpu_enabled", set_autocast_cpu_enabled, METH_O, nullptr}, + {"is_autocast_cpu_enabled", is_autocast_cpu_enabled, METH_NOARGS, nullptr}, + {"set_autocast_cpu_dtype", set_autocast_cpu_dtype, METH_O, nullptr}, + {"get_autocast_cpu_dtype", get_autocast_cpu_dtype, METH_NOARGS, nullptr}, + {"set_autocast_gpu_dtype", set_autocast_gpu_dtype, METH_O, nullptr}, + {"get_autocast_gpu_dtype", get_autocast_gpu_dtype, METH_NOARGS, nullptr}, + {"set_autocast_xla_enabled", set_autocast_xla_enabled, METH_O, nullptr}, + {"is_autocast_xla_enabled", is_autocast_xla_enabled, METH_NOARGS, nullptr}, + {"set_autocast_xla_dtype", set_autocast_xla_dtype, METH_O, nullptr}, + {"get_autocast_xla_dtype", get_autocast_xla_dtype, METH_NOARGS, nullptr}, + {"set_autocast_ipu_enabled", set_autocast_ipu_enabled, METH_O, nullptr}, + {"is_autocast_ipu_enabled", is_autocast_ipu_enabled, METH_NOARGS, nullptr}, + {"set_autocast_ipu_dtype", set_autocast_ipu_dtype, METH_O, nullptr}, + {"get_autocast_ipu_dtype", get_autocast_ipu_dtype, METH_NOARGS, nullptr}, + {"autocast_increment_nesting", + autocast_increment_nesting, + METH_NOARGS, + nullptr}, + {"autocast_decrement_nesting", + autocast_decrement_nesting, + METH_NOARGS, + nullptr}, + {"is_autocast_cache_enabled", + is_autocast_cache_enabled, + METH_NOARGS, + nullptr}, + {"set_autocast_cache_enabled", set_autocast_cache_enabled, METH_O, nullptr}, + {"_increment_version", THPModule_increment_version, METH_O, nullptr}, + {"set_anomaly_enabled", + castPyCFunctionWithKeywords(set_anomaly_mode_enabled), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"is_anomaly_enabled", is_anomaly_mode_enabled, METH_NOARGS, nullptr}, + {"is_anomaly_check_nan_enabled", + is_anomaly_check_nan_enabled, + METH_NOARGS, + nullptr}, + {"_is_multithreading_enabled", + is_multithreading_enabled, + METH_NOARGS, + nullptr}, + {"_set_multithreading_enabled", + castPyCFunctionWithKeywords(set_multithreading_enabled), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"_is_view_replay_enabled", is_view_replay_enabled, METH_NOARGS, nullptr}, + {"_set_view_replay_enabled", + castPyCFunctionWithKeywords(set_view_replay_enabled), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"_is_grad_layout_enforcement_enabled", + is_grad_layout_enforcement_enabled, + METH_NOARGS, + nullptr}, + {"_set_grad_layout_enforcement_enabled", + castPyCFunctionWithKeywords(set_grad_layout_enforcement_enabled), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"_set_graph_exec_group", set_graph_exec_group, METH_O, nullptr}, + {"_get_graph_exec_group", get_graph_exec_group, METH_NOARGS, nullptr}, + {"_enter_dual_level", python_enter_dual_level, METH_NOARGS, nullptr}, + {"_exit_dual_level", + castPyCFunctionWithKeywords(python_exit_dual_level), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"_is_torch_function_mode_enabled", + is_torch_function_mode_enabled, + METH_NOARGS, + nullptr}, + {"_push_on_torch_function_stack", + push_on_torch_function_stack, + METH_O, + nullptr}, + {"_pop_torch_function_stack", + pop_torch_function_stack, + METH_NOARGS, + nullptr}, + {"_get_function_stack_at", + castPyCFunctionWithKeywords(get_function_stack_at), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"_len_torch_function_stack", + len_torch_function_stack, + METH_NOARGS, + nullptr}, + {"_push_on_torch_dispatch_stack", + push_on_torch_dispatch_stack, + METH_O, + nullptr}, + {"_pop_torch_dispatch_stack", pop_torch_dispatch_stack, METH_O, nullptr}, + {"_get_dispatch_stack_at", + castPyCFunctionWithKeywords(get_dispatch_stack_at), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"_len_torch_dispatch_stack", + len_torch_dispatch_stack, + METH_NOARGS, + nullptr}, + {"_set_dispatch_mode", set_dispatch_mode, METH_O, nullptr}, + {"_get_dispatch_mode", get_dispatch_mode, METH_O, nullptr}, + {"_unset_dispatch_mode", unset_dispatch_mode, METH_O, nullptr}, + + {nullptr, nullptr, 0, nullptr}}; + +PyMethodDef* python_functions() { + return methods; +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/input_buffer.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/input_buffer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9ba572ea16fb07574ec836960beedff3c1309b49 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/input_buffer.cpp @@ -0,0 +1,399 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace torch::autograd { + +std::optional maybe_override_stale_capture_stream( + const std::optional& node_stream, + const std::optional& capturing_stream, + const std::string& node_name) { + TORCH_INTERNAL_ASSERT( + node_stream.has_value() && !node_stream->is_capturing(), + "maybe_override_stale_capture_stream: node_stream must be set and " + "non-capturing"); + TORCH_INTERNAL_ASSERT( + capturing_stream.has_value() && capturing_stream->is_capturing(), + "maybe_override_stale_capture_stream: capturing_stream must be set " + "and capturing"); + if (at::globalContext().overrideStaleCaptureStream()) { + return capturing_stream; + } + if (node_stream->id() == 0) { + TORCH_CHECK( + false, + "During CUDA graph capture, autograd node '", + node_name, + "' has a stale reference to the default stream (stream 0) from " + "warmup. This will invalidate the capture because " + "cudaStreamWaitEvent on the default stream pulls a non-capturing " + "stream into the graph.\n\n" + "To fix, either:\n" + " (a) Run warmup on the same stream that capture will use, or\n" + " (b) Delete references to the loss / autograd graph (e.g. " + "`del loss`) before capture, or\n" + " (c) Call torch.autograd.graph." + "set_override_stale_capture_stream(True) to automatically " + "redirect stale nodes to the capturing stream."); + } + // Non-default stale stream: leave unchanged. The user may have joined it + // into the capture (e.g. via `capture_stream.wait_stream(stale_stream)`) + // in which case the capture will succeed; otherwise the CUDA runtime will + // fail the capture with a downstream error. Users who want an automatic + // redirect can opt in via set_override_stale_capture_stream(True). + return node_stream; +} + +namespace { +// look what you made me do >.< +// Divergent paths for per-Impl stream recording that leak implementation +// details of the impls should not be needed here. +// See https://github.com/pytorch/pytorch/issues/60306 +// TODO: clean this up when https://github.com/pytorch/pytorch/issues/60306 is +// improved +void record_stream_any_impl(Variable& var, const c10::Stream& stream) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + + if (stream.device_index() != var.device().index()) { + return; + } + + const auto guard = c10::impl::VirtualGuardImpl(device_of(var).value().type()); + + if (C10_UNLIKELY(at::isBatchedTensor(var))) { + auto* impl = at::maybeGetBatchedImpl(var); + if (impl) { + guard.recordDataPtrOnStream(impl->value().storage().data_ptr(), stream); + } else { + TORCH_INTERNAL_ASSERT(false, "Expected batched tensor"); + } + } else { + switch (var.layout()) { + case c10::kSparseCsr: + case c10::kSparseCsc: + case c10::kSparseBsr: + case c10::kSparseBsc: { + auto* impl = at::sparse_csr::get_sparse_csr_impl(var); + guard.recordDataPtrOnStream( + impl->values().storage().data_ptr(), stream); + guard.recordDataPtrOnStream( + impl->compressed_indices().storage().data_ptr(), stream); + guard.recordDataPtrOnStream( + impl->plain_indices().storage().data_ptr(), stream); + break; + } + case c10::kSparse: { + auto* impl = at::sparse::get_sparse_impl(var); + guard.recordDataPtrOnStream( + impl->values().storage().data_ptr(), stream); + guard.recordDataPtrOnStream( + impl->indices().storage().data_ptr(), stream); + break; + } + case c10::kStrided: + guard.recordDataPtrOnStream(var.storage().data_ptr(), stream); + break; + default: + TORCH_INTERNAL_ASSERT( + false, "Unknown layout in record_stream_any_impl"); + } + } +} + +bool can_accumulate_inplace(const Variable& v) { + return ( + // `v` is a "vanilla" Tensor + !(at::isTensorSubclassLike(v) || v._is_zerotensor() || v.is_nested()) && + + // with a favorable memory layout + v.is_non_overlapping_and_dense() && + + // and we hold the last reference + impl::is_tensor_stealable(v, 1 + at::caching::is_cached_tensor(v)) && + v.has_storage() && v.storage().use_count() == 1); +} +} // anonymous namespace + +static void accumulate( + std::vector& buffer, + const size_t pos, + Variable&& var) { + TORCH_INTERNAL_ASSERT(pos < buffer.size()); + auto& old_var = buffer[pos]; + // If we hold the last reference to `old_var` AND its storage we will try to + // repurpose it to store the output. (Or, if `old_var` is sparse then `var` + // becomes the candidate output Tensor.) We only do this if: + // 1) GradMode is disabled since Autograd has special handling for inplace + // mutation which we don't want to trigger. + // + // 2) We hold the last reference. + // (Both `.use_count` and `.storage().use_count()` are one) + // + // 3) The candidate tensor is a contiguous, non-overlapping, dense, and + // otherwise stock standard Tensor. + // + // 4) The candidate is mutable. Currently only ZeroTensors are immutable. + // + // 5) The other Tensor is not a Tensor subclass (except sparse), since + // it's hard to predict the semantics of arbitrary subclass behavior. + + // NOLINTNEXTLINE(bugprone-branch-clone) + if (at::GradMode::is_enabled()) { + buffer[pos] = old_var + var; + } else if ( + // ATen doesn't route sparse additions correctly... + old_var.is_sparse() || old_var.is_sparse_csr()) { + if (can_accumulate_inplace(var)) { + buffer[pos] = var.add_(old_var); + } else { + buffer[pos] = var + old_var; + } + } else if ( + can_accumulate_inplace(old_var) && !at::isTensorSubclassLike(var)) { + buffer[pos] = old_var.add_(var); + } else { + buffer[pos] = old_var + var; + } +} + +// Note: [Stream sync contract when dealing with multi-deviced-ness] +// +// An operator can deal with multiple devices, e.g. if it does a device +// transfer, etc. However, for the purpose of stream synchronization, the engine +// is only aware of single canonical device/stream for each autograd Node. +// +// For the proper synchronization, the Node author should make sure of the +// following: +// +// 1) A node consuming a gradient should wait on the canonical stream before +// using it. +// 2) A node producing a gradient should have it ready on the canonical +// stream during node execution. +// + +// Note: [Autograd Producer-Consumer Stream Syncs] +// +// The producer-consumer stream syncs are partially handled in this method +// and partially handled in the engine prior to the consumer's execution. +// The logic here is mainly responsible for handling the synchronization needed +// for accumulation and recording the event that the consumer should wait on +// later. The corresponding wait and record_stream happens in the engine. +// +// First producer +// ============== +// There are several things we need to do upon seeing the first producer: +// 1) Determine the accumulation stream (which may or may not be used): +// case A) var's device matches consumer node's canonical device +// (The producer node's canonical device may or may not match) +// -> accumulator stream = consumer stream +// case B) var's device matches producer node's canonical device +// and does not match consumer node's canonical device +// -> accumulator stream = producer stream +// case C) var device matches neither +// -> accumulator stream = var device's current stream +// See Note [Stream sync contract when dealing with +// multi-deviced-ness] +// 2) Because we are the first producer, there's no accumulation necessary. +// Just move var into the buffer. +// 3) Update the ready_events and streams for the current position.** +// ready_events are events you need to wait for to ensure the corresponding +// buffers are ready. The events are updated as we accumulate into the +// buffer. +// +// Nth producer +// ============ +// 1) Synchronize for accumulation. Accumulation operates on both the new +// incoming gradient and the existing gradient in the buffer. +// (i) wait stream and (ii) record stream to make sure both are ready to be +// used on the accumulation stream. +// 2) Accumulate on the accumulation stream +// 3) Update the ready event and stream for the current position.** +// +// **As an optimization, we avoid creating and recording an event if we +// know that we won't need to wait on it, saving on the order of microseconds. +// +void InputBuffer::add( + size_t pos, + Variable&& var, + const std::optional& opt_producer_stream_, + const std::optional& opt_consumer_stream_, + Node* fn) { + TORCH_INTERNAL_ASSERT(pos < buffer.size()); + + if (!var.defined()) { + return; + } + const auto device = var.device(); + const auto device_type = device.type(); + bool is_accelerator = at::accelerator::isAccelerator(device.type()); + // + // Non-accelerator case + // + if (!is_accelerator) { + if (!buffer[pos].defined()) { + buffer[pos] = std::move(var); + } else { + c10::OptionalDeviceGuard device_guard{device}; + accumulate(buffer, pos, std::move(var)); + } + return; + } + // Handle the case where var is on an accelerator but producer node has no + // canonical stream, e.g. this can happen if forward is DtoH + const std::optional& opt_producer_stream = + (opt_producer_stream_.has_value() + ? opt_producer_stream_ + : std::optional( + at::accelerator::getCurrentStream(device.index()))); + + // opt_consumer_stream is always non-null when is_accelerator is true + // when InputBuffer is used in the engine. InputBuffer is also called + // elsewhere however! (e.g. other engine implementations) + // + // If a prior add() on this InputBuffer applied the stale-capture + // override, reuse the cached stream so every producer sees the same + // consumer stream. Otherwise honor the caller-provided value, falling + // back to the device's current stream. + std::optional opt_consumer_stream; + if (opt_overridden_consumer_stream.has_value()) { + opt_consumer_stream = opt_overridden_consumer_stream; + } else if (opt_consumer_stream_.has_value()) { + opt_consumer_stream = opt_consumer_stream_; + } else { + opt_consumer_stream = at::accelerator::getCurrentStream(device.index()); + } + + TORCH_INTERNAL_ASSERT(opt_consumer_stream && opt_producer_stream); + + // Handle producer/consumer stream mismatch. Two independent cases: + // 1. Producer is capturing but the consumer holds a stale non-capturing + // stream (e.g. from warmup). Depending on + // `set_override_stale_capture_stream` the helper either errors out + // (default stream), overrides the consumer stream with the producer's + // capturing stream, or leaves it unchanged. A successful override + // resolves the mismatch for case 2 and is cached on the InputBuffer + // so every producer and Engine::evaluate_function see it. + // 2. Pre-existing AccumulateGrad-stream-mismatch warning from + // PR #166136. Fires only when the mismatch still exists after case 1. + bool stream_mismatch = opt_consumer_stream->id() != opt_producer_stream->id(); + if (stream_mismatch && opt_producer_stream->is_capturing() && + !opt_consumer_stream->is_capturing()) { + auto resolved = maybe_override_stale_capture_stream( + opt_consumer_stream, opt_producer_stream, fn->name()); + if (resolved != opt_consumer_stream) { + // Override was applied. Cache so subsequent add() calls and + // Engine::evaluate_function consult the same consumer stream. + opt_consumer_stream = resolved; + opt_overridden_consumer_stream = opt_consumer_stream; + stream_mismatch = false; + } + } + if (stream_mismatch && dynamic_cast(fn) && + at::globalContext().warnOnAccumulateGradStreamMismatch()) { + TORCH_WARN_ONCE( + "The AccumulateGrad node's stream does not match the stream of the " + "node that produced the incoming gradient. This may incur unnecessary " + "synchronization and break CUDA graph capture if the AccumulateGrad " + "node's stream is the default stream. This mismatch is caused by an " + "AccumulateGrad node created prior to the current iteration being " + "kept alive. This can happen if the autograd graph is still being " + "kept alive by tensors such as the loss, or if you are using DDP, " + "which will stash a reference to the node. To resolve the mismatch, " + "delete all references to the autograd graph or ensure that DDP " + "initialization is performed under the same stream as subsequent " + "forwards. If the mismatch is intentional, you can use " + "torch.autograd.graph.set_warn_on_accumulate_grad_stream_mismatch(" + "False) to suppress this warning."); + } + // See Note: [Autograd Producer-Consumer Stream Syncs] + if (!opt_accum_streams[pos].has_value()) { + // [ First producer ] + TORCH_INTERNAL_ASSERT(!buffer[pos].defined()); + // 1) + if (opt_consumer_stream->device() == device) { + // Case A + opt_accum_streams[pos] = opt_consumer_stream; + if (*opt_consumer_stream != *opt_producer_stream) { + // We will end up doing record_stream on the accumulation stream + // (which is the consumer stream) later, but we also need to do + // it here in case we don't end up accumulating. + record_stream_any_impl(var, *opt_consumer_stream); + } + } else if (opt_producer_stream->device() == device) { + // Case B + opt_accum_streams[pos] = opt_producer_stream; + } else { + // Case C + opt_accum_streams[pos] = + at::accelerator::getCurrentStream(device.index()); + } + // 2) + buffer[pos] = std::move(var); + // 3) + auto& opt_accum_stream = opt_accum_streams[pos]; + TORCH_INTERNAL_ASSERT(opt_accum_stream.has_value()); + if (*opt_consumer_stream != *opt_producer_stream || + *opt_accum_stream != *opt_producer_stream) { + // Either the consumer or accum stream waits for the producer + // stream depending on whether accumulation is needed. + auto event = c10::Event{device_type}; + event.record(*opt_producer_stream); + ready_events[pos] = std::move(event); + } + ready_streams[pos] = opt_producer_stream; + } else { + // [ Nth producer ] + auto accum_stream = opt_accum_streams[pos]; + auto& ready_event = ready_events[pos]; + auto& ready_stream = ready_streams[pos]; + TORCH_INTERNAL_ASSERT(accum_stream && ready_stream); + // 1) + if (*accum_stream != *opt_producer_stream) { + auto event = c10::Event{device_type}; + event.record(*opt_producer_stream); + accum_stream->wait(event); + record_stream_any_impl(var, *accum_stream); + } + if (*accum_stream != *ready_stream) { + TORCH_INTERNAL_ASSERT(ready_event); + accum_stream->wait(*ready_event); + // This is redundant for case A, but needed for case C + record_stream_any_impl(buffer[pos], *accum_stream); + } + // 2) + c10::OptionalStreamGuard stream_guard{accum_stream}; + accumulate(buffer, pos, std::move(var)); + // 3) + if (*opt_consumer_stream != *accum_stream) { + // Only the consumer stream needs to wait for this event + auto event = c10::Event{device_type}; + event.record(*accum_stream); + ready_events[pos] = std::move(event); + } + ready_streams[pos] = accum_stream; + } +} + +auto InputBuffer::variables(InputBuffer&& g) -> std::vector { + std::vector result = std::move(g.buffer); + return result; +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/input_buffer.h b/western_ai/usa/pytorch/torch/csrc/autograd/input_buffer.h new file mode 100644 index 0000000000000000000000000000000000000000..fa61efddee0ce0314eb8d89f9ccd7ee6b10ad50d --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/input_buffer.h @@ -0,0 +1,88 @@ +#pragma once + +// The InputBuffer class accumulates a list of Variables for use by a +// function. It implements logic to avoid modifying the passed +// values in-place (adding an input twice will accumulate the result). +// This behaviour is needed and used only in backward graphs. + +#include +#include + +#include +#include +#include + +namespace torch::autograd { + +// Detect and handle stale non-capturing stream references during CUDA graph +// capture. Autograd nodes snapshot the current stream at construction time. +// If a node was created on a non-capturing stream (e.g. during warmup) but +// backward runs under capture on a different stream, the engine would issue +// cudaStreamWaitEvent on the stale stream, pulling it into the capture. +// +// Three possible outcomes, depending on global state: +// 1. If overrideStaleCaptureStream() is true, returns capturing_stream +// (the caller should use it in place of node_stream). +// 2. If node_stream refers to the default stream (id == 0), throws a +// c10::Error with actionable guidance; this case always invalidates +// the capture, so failing fast is better than the opaque CUDA error. +// 3. Otherwise, returns node_stream unchanged. +// +// Preconditions (asserted): node_stream is set and non-capturing; +// capturing_stream is set and capturing. Intended as an internal helper, +// not part of the stable C++ ABI. +std::optional maybe_override_stale_capture_stream( + const std::optional& node_stream, + const std::optional& capturing_stream, + const std::string& node_name); + +struct InputBuffer { + explicit InputBuffer(size_t size) + : buffer(size), + opt_accum_streams(size), + ready_events(size), + ready_streams(size) {} + InputBuffer(const InputBuffer& other) = delete; + InputBuffer(InputBuffer&& other) = default; + explicit InputBuffer(variable_list&& inputs) : buffer(std::move(inputs)) {} + InputBuffer& operator=(InputBuffer&& other) = default; + + // Accumulates the variable at a specified index. + // The optional CUDA streams determine which stream the accumulation + // is run on and how the addition is synchronized. + TORCH_API void add( + size_t pos, + Variable&& var, + const std::optional& opt_producer_stream, + const std::optional& opt_consumer_stream, + Node* fn); + + Variable operator[](size_t pos) { + return buffer[pos]; + } + + // Returns the inputs as a list of variables. Destroys given InputBuffer. + static std::vector variables(InputBuffer&& g); + + std::vector buffer; + // The stream used for accumulation when a variable is used multiple times. + std::vector> opt_accum_streams; + // The events you need to wait for to ensure the corresponding buffers + // are ready. The events are updated as we accumulate into the buffer. + std::vector> ready_events; + // The streams corresponding to the events above. This is only used to + // check if more synchronization is needed or not. + std::vector> ready_streams; + // The consumer stream after the stale-capture override fires. Set by + // InputBuffer::add() the first time a producer triggers the override + // (i.e. only when set_override_stale_capture_stream is true and a stale + // non-capturing consumer is detected against a capturing producer). + // Subsequent add() calls reuse this so all producers feeding this buffer + // converge on the overridden stream, and Engine::evaluate_function uses + // it as the parent stream for the node. When no override has fired this + // is nullopt and Engine::evaluate_function falls back to the node's + // func->stream(). + std::optional opt_overridden_consumer_stream; +}; + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/input_metadata.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/input_metadata.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4024aa15dd7a011cc8d2eb0d2c132d85181c5234 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/input_metadata.cpp @@ -0,0 +1,207 @@ +#include + +// TODO: we may be able to move some imports from input_metadata.h to here, but +// it seems that function.h transitively depends on some of them. + +namespace torch::autograd { + +namespace { + +MetadataShape compute_variant_shape(const at::Tensor& input) { + if (input.is_nested() && !input.unsafeGetTensorImpl()->is_python_dispatch()) { + auto nested_size = input._nested_tensor_size(); + return MetadataShape{std::in_place_type, nested_size}; + } + return MetadataShape{std::in_place_type, input.sym_sizes()}; +} + +bool is_python_dispatch(const at::Tensor& tensor) { + return tensor.unsafeGetTensorImpl()->is_python_dispatch(); +} + +bool is_cpp_nested_tensor(const at::Tensor& tensor) { + return tensor.is_nested() && !is_python_dispatch(tensor); +} + +} // namespace + +InputMetadata::InputMetadata( + const at::TensorOptions& options, + MetadataShape input_shape, + bool is_tensor_subclass, + bool is_nested, + std::optional grad_dtype) + : options_{options}, + shape_{std::move(input_shape)}, + is_tensor_subclass_{is_tensor_subclass}, + is_nested_{is_nested}, + was_default_constructed_{false}, + grad_dtype_{grad_dtype} { + auto device_ = options.device(); + stream_ = c10::impl::getDeviceGuardImpl(device_.type())->getStream(device_); +} + +InputMetadata::InputMetadata(const at::Tensor& t) + : InputMetadata( + t.options(), + compute_variant_shape(t), + is_python_dispatch(t), + t.is_nested(), + t.grad_dtype()) {} + +at::Tensor InputMetadata::zeros_like() const { + TORCH_CHECK( + !is_nested_, "Zeros is not currently supported for nested tensors.") + return at::zeros_symint(shape_as_dim_vector(), options_); +} + +at::Tensor InputMetadata::maybe_reduce( + const size_t i, + at::Tensor grad, + const std::function& format_error) const { + auto fail = [&]() { + const auto message = incompatible_shape_error_message(i, grad); + TORCH_CHECK(false, format_error(message.str())); + }; + + // Nested tensor makes my brain explode, so I've just hard-coded the logic + // for this case, at risk of code duplication. This logic does NOT do the + // careful oblivious logic as seen below + if (is_nested_ || is_cpp_nested_tensor() || grad.is_nested() || + ::torch::autograd::is_cpp_nested_tensor(grad)) { + if (!is_same_shape(grad)) { + if (is_expandable_to_shape(grad)) { + return reduce_grad(grad); + } else { + fail(); + } + } else { + return grad; + } + } + + auto shape = shape_as_dim_vector(); + auto desired = grad.sym_sizes(); + + size_t ndim = shape.size(); + size_t target_dim = desired.size(); + if (ndim > target_dim) { + fail(); + } + bool needs_reduce = false; + for (const auto i : c10::irange(ndim)) { + const auto& size = shape[ndim - i - 1]; + const auto& target = desired[target_dim - i - 1]; + // The conditions here are written carefully so that we are able to + // infer deferred runtime asserts + if (TORCH_GUARD_OR_FALSE(size.sym_eq(1))) { + // NB: we could short circuit this once needs_reduce is true but there's + // no point since the reduction function will guard on this anyway + if (!c10::guard_or_false(size.sym_eq(target), __FILE__, __LINE__)) { + needs_reduce = true; + } + } else { + if (!size.sym_eq(target).expect_true(__FILE__, __LINE__)) { + fail(); + } + } + } + if (ndim != target_dim) { + needs_reduce = true; + } + + if (needs_reduce) { + return reduce_grad(grad); + } else { + return grad; + } +} + +bool InputMetadata::is_same_shape(const at::Tensor& grad) const { + if (!is_nestedness_same(grad)) { + return false; + } + if (is_cpp_nested_tensor()) { + return grad._nested_tensor_size().is_same_size(shape_as_tensor()); + } + return grad.sym_sizes().equals(shape_as_dim_vector()); +} + +bool InputMetadata::is_expandable_to_shape(const at::Tensor& grad) const { + if (!maybe_expandable_to(grad)) { + return false; + } + return at::is_expandable_to(shape_as_dim_vector(), grad.sym_sizes()); +} + +at::Tensor InputMetadata::reduce_grad(at::Tensor& grad) const { + // reduce_grad should only be called if is_expandable_to_shape returns true. + TORCH_INTERNAL_ASSERT(maybe_expandable_to(grad)); + return at::sum_to(std::move(grad), shape_as_dim_vector()); +} + +std::stringstream InputMetadata::incompatible_shape_error_message( + const size_t index, + const at::Tensor& grad) const { + std::stringstream ss{}; + ss << "invalid gradient at index " << index << " - got "; + if (::torch::autograd::is_cpp_nested_tensor(grad)) { + ss << grad._nested_tensor_size(); + } else { + ss << grad.sym_sizes(); + } + ss << " but expected shape compatible with "; + if (is_cpp_nested_tensor()) { + ss << shape_as_tensor(); + } else { + ss << shape_as_dim_vector(); + } + return ss; +} + +bool InputMetadata::is_cpp_nested_tensor() const { + bool ret = std::holds_alternative(shape_); + TORCH_INTERNAL_ASSERT(ret == (is_nested_ && !is_tensor_subclass_)) + return ret; +} + +c10::SymIntArrayRef InputMetadata::shape_as_dim_vector() const { + const auto& dim_shape = std::get(shape_); + return c10::SymIntArrayRef(dim_shape.data(), dim_shape.size()); +} + +// Danger: not thread safe, caller must protect with lock +SymIntSmallVec& InputMetadata::mutable_shape_as_dim_vector() { + return std::get(shape_); +} + +bool InputMetadata::is_nestedness_same(const at::Tensor& grad) const { + return ( + grad.is_nested() == is_nested_ && + ::torch::autograd::is_cpp_nested_tensor(grad) == is_cpp_nested_tensor()); +} + +at::Tensor InputMetadata::shape_as_tensor() const { + return std::get(shape_); +} + +bool InputMetadata::maybe_expandable_to(const at::Tensor& grad) const { + // This is the initial step to determine whether or not the tensor represented + // by input_metadata is expandable to grad based on is-nestedness information + // alone. If this function returns true, then is_expandable_to_shape will be + // called. We support the following 3 types of expansion: + bool grad_is_nested = grad.is_nested(); + if (!is_nested_ && !grad_is_nested) { + // Normal case (no NestedTensors are involved) + // (1) plain Tensor -> plain Tensor + return true; + } else { + // (2) python NT -> python NT + // (3) plain Tensor -> python NT + return ( + grad_is_nested && is_python_dispatch(grad) && + (!is_nested_ || is_tensor_subclass_)); + } +} + +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/input_metadata.h b/western_ai/usa/pytorch/torch/csrc/autograd/input_metadata.h new file mode 100644 index 0000000000000000000000000000000000000000..a4cba561752e9bb3f845bc6e195ec5b4a7f53567 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/input_metadata.h @@ -0,0 +1,132 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef AT_PER_OPERATOR_HEADERS +#include +#else +#include +#endif + +namespace torch::autograd { + +using SymIntSmallVec = c10::SmallVector; +using MetadataShape = std::variant; + +/** + * Records TensorOptions, shape of the tensor, whether or not the Python + * dispatch key is set (tensor subclass), and, where applicable, the stream the + * corresponding operation took place on. + * + * If is_valid() is false, then the corresponding input is not used and may be + * an undefined tensor. + */ +struct TORCH_API InputMetadata { + InputMetadata() = default; + InputMetadata( + const at::TensorOptions& options, + MetadataShape input_shape, + bool is_tensor_subclass, + bool is_nested, + std::optional grad_dtype); + InputMetadata(const at::Tensor& t); + + const at::TensorOptions& options() const { + return options_; + } + + caffe2::TypeMeta dtype() const { + return options_.dtype(); + } + + at::Device device() const { + return options_.device(); + } + + at::Layout layout() const { + return options_.layout(); + } + + c10::Stream stream() const { + return stream_; + } + + bool is_tensor_subclass() const { + return is_tensor_subclass_; + } + + at::Tensor zeros_like() const; + + bool is_same_shape(const at::Tensor& grad) const; + + bool is_expandable_to_shape(const at::Tensor& grad) const; + + at::Tensor reduce_grad(at::Tensor& grad) const; + + at::Tensor maybe_reduce( + const size_t index, + at::Tensor grad, + const std::function& format_error) const; + + std::stringstream incompatible_shape_error_message( + const size_t index, + const at::Tensor& grad) const; + + bool was_default_constructed() const { + return was_default_constructed_; + } + + bool is_cpp_nested_tensor() const; + + bool is_nested_tensor() const { + return is_nested_; + } + + c10::SymIntArrayRef shape_as_dim_vector() const; + + // Danger: not thread safe, caller must protect with lock + SymIntSmallVec& mutable_shape_as_dim_vector(); + + std::optional grad_dtype() const { + TORCH_INTERNAL_ASSERT(!was_default_constructed_); + return grad_dtype_; + } + + void set_grad_dtype(const std::optional& grad_dtype) { + TORCH_INTERNAL_ASSERT(!was_default_constructed_); + grad_dtype_ = grad_dtype; + } + + private: + at::Tensor shape_as_tensor() const; + bool is_nestedness_same(const at::Tensor& grad) const; + bool maybe_expandable_to(const at::Tensor& grad) const; + + // NB: The engine does not use the dtype from the options, but rather the + // grad_dtype_ field to validate grad_output dtype. + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const at::TensorOptions options_; + MetadataShape shape_; + c10::Stream stream_ = c10::Stream(c10::Stream::Default::DEFAULT, device()); + bool is_tensor_subclass_ = false; + bool is_nested_ = false; + bool was_default_constructed_ = true; + + // The grad_dtype_ field is the dtype that the engine expects the grad to be. + // When nullopt, grad_dtype_ is allowed to be any dtype. + // This field is mutated if THPVariable_set_grad_dtype is called + // and the AccumulateGrad has already been created. + std::optional grad_dtype_; +}; +} // namespace torch::autograd diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/jit_decomp_interface.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/jit_decomp_interface.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0e27f96354f0b3eca10dd89aa2db7ff8283ab390 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/jit_decomp_interface.cpp @@ -0,0 +1,17 @@ +#include + +namespace torch::autograd::impl { + +namespace { +JitDecompInterface* impl = nullptr; +} + +void setJitDecompImpl(JitDecompInterface* impl_) { + impl = impl_; +} + +JitDecompInterface* getJitDecompImpl() { + return impl; +} + +} // namespace torch::autograd::impl diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/jit_decomp_interface.h b/western_ai/usa/pytorch/torch/csrc/autograd/jit_decomp_interface.h new file mode 100644 index 0000000000000000000000000000000000000000..fa050b5e5806ceec1a42eafb5c55e502b7622362 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/jit_decomp_interface.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include + +// NOTE: [Jit Decomposition Interface] +// +// For some context of why we need this at all, see NOTE: [forward-mode AD +// decompositions mechanism] +// +// Introducing that mechanism from the NOTE is problematic because: +// - it relies on TorchScript, so now VariableTypeX.cpp depends on TorchScript. +// - there exist internal builds like lite_trainer, which depend on VariableType +// but do not depend on TorchScript. +// +// For internal builds like lite_trainer builds to pass, and for OSS builds that +// do depend on TorchScript to still support the forward AD decomp mechanism, we +// implement a PImpl pattern to avoid a static dependency in favor of a dynamic +// one +// - during static initialization time, if the library is built with TorchScript +// setJitDecompImpl is called in decomposition_registry.cpp setting a global +// ptr to the impl +// - when the program is run,if getJitDecompImpl returns a non null ptr, we can +// carry on normally, otherwise we gracefully error out +// +// For extra context, see VariableHooksInterface.h, where a similar technique +// is used + +namespace torch::autograd::impl { + +struct TORCH_API JitDecompInterface { + virtual ~JitDecompInterface() = default; + virtual bool has_jit_decomposition( + const c10::FunctionSchema& schema) const = 0; + virtual void run_jit_decomposition( + const c10::OperatorHandle& op, + jit::Stack* stack) const = 0; +}; + +TORCH_API void setJitDecompImpl(JitDecompInterface* impl); +TORCH_API JitDecompInterface* getJitDecompImpl(); + +struct TORCH_API JitDecompRegisterer{explicit JitDecompRegisterer( + JitDecompInterface * impl){setJitDecompImpl(impl); +} // namespace torch::autograd::impl +} +; + +} // namespace torch::autograd::impl diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/node.h b/western_ai/usa/pytorch/torch/csrc/autograd/node.h new file mode 100644 index 0000000000000000000000000000000000000000..7f018656a29187a79d6a635c96d3ab079ba53496 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/node.h @@ -0,0 +1,725 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace torch::autograd { + +struct Edge; +struct FunctionPostHook; +struct FunctionPreHook; +struct Node; +class SavedVariable; + +using Variable = at::Tensor; +using tensor_list = std::vector; +using variable_list = std::vector; +using edge_list = std::vector; +using saved_variable_list = std::vector; +using ivalue_list = std::vector; +using functional_apply_t = std::function< + variable_list(const variable_list&, const std::vector&)>; +using IndexRange = std::pair; +using torch::dynamo::autograd::CompiledNodeArgs; +using torch::dynamo::autograd::PackedArgs; +using torch::dynamo::autograd::SwapSavedVariables; + +// Guard that sets and restores the evaluating node +class NodeGuard { + public: + explicit NodeGuard(c10::intrusive_ptr node); + ~NodeGuard(); + + private: + c10::intrusive_ptr last_evaluating_node_; +}; + +// Return the Node currently being evaluated (if any) +// This is only set during the backward pass while a Node is being +// executed. +TORCH_API c10::intrusive_ptr get_current_node(); + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// Node +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// A `Node` is an abstract class that represents an operation taking zero +// or more input `Variable`s and producing zero or more output `Variable`s. All +// functions in PyTorch's autograd machinery derive from this class and +// override its `apply` method. Instances of such subclasses will then be +// invocable via the call operator. +// +// Nodes in the Autograd Graph +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// When viewing the autograd system as a graph, `Node`s are the vertices or +// nodes, connected to each other via (directed) `Edge`s, which themselves are +// represented via (`Node`, input_nr) pairs. `Variable`s are the outputs to +// and inputs of `Node`s, and travel between these edges during execution +// of the graph. When two or more `Edge`s (from different sources) point at the +// same input to a `Node`, the values produced along all of these edges are +// implicitly summed prior to being forwarded to the target `Node`. +// +// Hierarchy +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// Subclasses usually represent differentiable functions as well as their +// gradient operators. Note, however, that due to the very general definition +// of a `Node` taking *zero* or more inputs and producing *zero* or more +// outputs, uses of `Node`s are flexible and extend beyond purely +// mathematical operations. For example, the `AccumulateGrad` function is a +// *sink*: it takes one input, but produces no outputs, instead accumulating +// the input as a side effect. At the other extreme, the `GraphRoot` function +// receives no inputs from other functions, but produces multiple outputs. +// +// Interface +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// The most important method on `Node` is the call operator, which takes in +// a list of variables and produces a list of variables. The precise size of +// these lists can be determined with `num_inputs()` and `num_outputs()`. +// `Node`s are stitched together via their `next_edge` interface, which let +// you manipulate the set of outgoing edges of a `Node`. You can add an +// edge with `add_next_edge()`, retrieve an edge with `next_edge(index)` and +// iterate over them via the `next_edges()` method. Other methods exist for +// integration with the JIT and other parts of PyTorch. Every `Node` has a +// *sequence number* that increases monotonically in the order of `Node` +// construction. It can be retrieved via the `sequence_nr()` method. Note that +// this sequence number is *thread local*. This means that when `Node`s +// `A`, `B` and `C` are created consecutively in the same thread, their +// sequence numbers will be ordered `A` < `B` < `C`. If, however, `A` and `B` +// are created in one thread and `C` is created in a new thread, there are *no +// guarantees* w.r.t. the ordering of `C` relative to `A` or `B`. +// See NOTE [ Sequence Number] for more details on the usages of sequence +// number. +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +struct TORCH_API Node : c10::intrusive_ptr_target { + public: + /// Construct a new `Node` with the given `next_edges` + explicit Node(uint64_t sequence_nr, edge_list&& next_edges = edge_list()) + : sequence_nr_(sequence_nr), next_edges_(std::move(next_edges)) { + for (const Edge& edge : next_edges_) { + update_topological_nr(edge); + } + + if (AnomalyMode::is_enabled()) { + metadata()->store_stack(); + + // If anomaly mode is enabled and graph is constructed, then assign the + // currently evaluating node as the parent of this node. + // A parent is a Node where this Node is created. + // We are tracking the parents to track multiple backward operations. + assign_parent(); + } + + // Store the thread_id of the forward operator. + // See NOTE [ Sequence Numbers ] + thread_id_ = at::RecordFunction::currentThreadId(); + } + + explicit Node(edge_list&& next_edges = edge_list()) + : Node( + /*sequence_nr=*/at::sequence_number::get_and_increment(), + std::move(next_edges)) {} + + /// Nodes are neither copyable nor moveable. + Node(const Node& other) = delete; + Node(Node&& other) = delete; + Node& operator=(const Node& other) = delete; + Node& operator=(Node&& other) = delete; + ~Node() override; + void release_resources() override; + + c10::intrusive_ptr getptr() { + return c10::intrusive_ptr::unsafe_reclaim_from_nonowning(this); + } + /// Evaluates the function on the given inputs and returns the result of the + /// function call. + variable_list operator()(variable_list&& inputs) { +#ifdef USE_ROCM + // Keep track of backward pass for rocblas. + at::ROCmBackwardPassGuard in_backward; +#endif + + auto step_callbacks = + at::getStepCallbacksUnlessEmpty(at::RecordScope::BACKWARD_FUNCTION); + if (C10_UNLIKELY(step_callbacks.has_value())) { + at::RecordFunction guard(std::move(*step_callbacks)); + // Using sequence number and thread id to correlate with + // the forward pass function + guard.setForwardThreadId(thread_id_); + if (guard.needsInputs()) { + std::vector inputs_vec(inputs.begin(), inputs.end()); + guard.before( + name(), + c10::ArrayRef( + inputs_vec.data(), inputs_vec.size()), + static_cast(sequence_nr())); + } else { + guard.before(name(), static_cast(sequence_nr())); + } + return apply(std::move(inputs)); + } else { + return apply(std::move(inputs)); + } + } + + // Graph Connectivity API + //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + // Inputs. NOTE: inputs of the grad_fn correspond to Tensor outputs of the + // forward function. + + // Marker for expected undefined input + struct undefined_input {}; + + /// Adds the type and shape metadata for a new input. Returns the index of + /// of the new input. + uint32_t add_input_metadata( + const at::TensorOptions& options, + c10::SymIntArrayRef shape, + bool is_tensor_subclass, + bool is_nested, + std::optional grad_dtype) noexcept { + uint32_t input_nr = input_metadata_.size(); + input_metadata_.emplace_back( + options, + MetadataShape{std::in_place_type, shape}, + is_tensor_subclass, + is_nested, + grad_dtype); + return input_nr; + } + + uint32_t add_input_metadata(const at::Tensor& t) noexcept { + uint32_t input_nr = input_metadata_.size(); + input_metadata_.emplace_back(t); + return input_nr; + } + + /// Adds a placeholder for an input that will not be used. + uint32_t add_input_metadata(undefined_input u) noexcept { + uint32_t input_nr = input_metadata_.size(); + input_metadata_.emplace_back(); + return input_nr; + } + + uint32_t num_inputs() const noexcept { + return input_metadata_.size(); + } + + const InputMetadata& input_metadata(size_t index) const { + return input_metadata_[index]; + } + + // Danger: not thread safe, caller must protect with lock + InputMetadata& mutable_input_metadata(size_t index) { + return input_metadata_[index]; + } + + /** + * Note: Function Streams + * A function's stream (for a given device type) is the stream of the first + * element of its input buffer on a device of that type. + * + * If all elements are on the same device they MUST share a stream. If + * elements are on different devices (across multiple GPUs, for example) + * they may have different streams. + */ + std::optional stream() { + auto opt_device_type = at::getAccelerator(); + if (!opt_device_type.has_value()) { + return std::nullopt; + } + for (const auto& metadata : input_metadata_) { + if (metadata.device().type() == opt_device_type.value()) + return metadata.stream(); + } + + return std::nullopt; + } + + // Used by the engine to determine what device thread to run on + at::Device device() { + // Since we pick the first non-CPU tensor, this won't work with + // mixed device-type operations (e.g., an op that is both CUDA + // and XLA). This is *incredibly* unlikely, so we don't worry + // about it. + for (const auto& metadata : input_metadata_) { + auto device = metadata.device(); + if (device.type() != at::kCPU) { + return device; + } + } + // Only report to the CPU thread if there really were no tensors + // from other devices. + return at::kCPU; + } + + void clear_input_metadata() { + input_metadata_.clear(); + } + + // Outputs ("Next Edges") + + void update_topological_nr(const Edge& edge) { + TORCH_INTERNAL_ASSERT( + !has_parent_, + "Cannot update a node's topological_nr after it already has a parent." + " If we allow this, we can no longer guarantee that a parent's" + " topo_nr is always greater than those of all its children") + Node* node = edge.function.get(); + if (node) { + auto topo_nr = node->topological_nr(); + if (topological_nr_ <= topo_nr) { + topological_nr_ = topo_nr + 1; + } + } + } + + void set_next_edge(size_t index, Edge edge) { + update_topological_nr(edge); + next_edges_[index] = std::move(edge); + } + + void add_next_edge(Edge edge) { + update_topological_nr(edge); + next_edges_.emplace_back(std::move(edge)); + } + + void set_next_edges(edge_list&& next_edges) { + next_edges_ = std::move(next_edges); + for (const auto& next_edge : next_edges_) { + update_topological_nr(next_edge); + } + } + + const Edge& next_edge(size_t index) const noexcept { + return next_edges_[index]; + } + + const edge_list& next_edges() const noexcept { + return next_edges_; + } + + edge_list& next_edges() noexcept { + return next_edges_; + } + + uint32_t num_outputs() const noexcept { + return next_edges_.size(); + } + + // Miscellaneous Methods + //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + /// NOTE [ Sequence Number] + /// + /// The sequence_nr has two main usages in autograd: + /// + /// 1) Helps determine the node's execution priority in the engine. + /// All else being equal, nodes with higher priority numbers are executed + /// first. Thus, nodes corresponding to ops executed later are the first to + /// be executed in the backward pass. One caveat is that we prioritize + /// AccumulateGrad nodes by explicitly setting its sequence_nr to be + /// UINT64_MAX. + /// 2) The sequence number of this `Node` is paired with thread_id it was + /// created in + /// as a unique identifier by the profiler to annotate recorded events. + /// The purpose of this is to help users (and possibly programs) + /// interpreting the profiler's output to correlate backward nodes with its + /// forward ops. We need both sequence_nr and thread_id to identify a node + /// because sequence_nr is thread_local, i.e., starts counting up from zero + /// in a new thread + uint64_t sequence_nr() const noexcept { + return sequence_nr_; + } + + void set_sequence_nr(uint64_t sequence_nr) { + sequence_nr_ = sequence_nr; + } + + // NOTE [ Topological Number ] + // + // topological_nr is used to prune branches in the DAG during autograd + // discovery as maintaining topological_nr helps us check in O(1) if there + // does NOT exist a directed path between two nodes. + // + // The topological order number of this `Node` representing the length of the + // longest possible path from this Node to any leaf node. If you are leaf + // node, aka AccumulateGrad, this will be zero. This value has the property + // that For every pair of nodes X, Y in G, existence of a directed path from X + // to Y implies topo_nr(X) > topo_nr(Y). The converse is not true, however, so + // we cannot prove existence of a path from X to Y, only non-existence. + // + // One assumption we make when using topo_nr is that once a node + // has been used, i.e., has a parent node, its own topo_nr does not change + // we have added some checks with the `has_parent_` field to enforce this. + // + // What NOT to do: + // + // 1) 2 -> 1 -> 0 In this diagram we label nodes with their + // topo_nr. + // 2 -> 1 -> 0 We have two simple graphs that can each + // arise from + // `t.exp().exp()`, for example. + // 2) 2 -> 1 -> 0 + // / + // 2 -> 1 -> 0 We add 2 as a next edge to 1 even though 1 + // already + // has a parent. + // 3) 2 -> 1 -> 0 + // / + // 2 -> 3 -> 0 2 < 3, yet there exists a path from 2 to 3! + // + uint64_t topological_nr() const noexcept { + has_parent_ = true; + return topological_nr_; + } + + // assigning a node as a parent to this node + void assign_parent(); + + /// Id of the thread that created Node + uint64_t thread_id() const noexcept { + return thread_id_; + } + + /// Returns the name of the dynamic type of the function, for debugging. + virtual std::string name() const; + + /// Returns the name of the corresponding forward op by stripping the + /// "Backward" suffix from name(), if present. + std::string forward_op_name() const; + + /// The difference between functions `should_compute_output` and + /// `task_should_compute_output`: + /// - `should_compute_output` should only be used during graph construction + /// and takes into account only requires_grad information + /// - `task_should_compute_output` should only be called during the backward + /// pass (unless called directly through grad_fn) and takes into account the + /// current graph task. Specifically, the autograd engine trims unnecessary + /// edges when `inputs` are specified, and during backward untrimmed nodes + /// left on the graph can/should check `task_should_compute_output` to see if + /// any outgoing edges have been trimmed by the engine. If that is the case, + /// gradient computation wrt those edges can be omitted. + /// + /// Returns true if the particular output edge is active, and that particular + /// output of this function should be computed. + bool should_compute_output(size_t output_edge_index) const { + TORCH_CHECK(output_edge_index < num_outputs(), "Index out of range"); + return next_edges_[output_edge_index].is_valid(); + } + + /// Returns true if any of the output edges in any of the ranges are active. + bool should_compute_output(std::initializer_list idxs) const { + return std::any_of(idxs.begin(), idxs.end(), [this](IndexRange range) { + for (const auto i : c10::irange(range.first, range.second)) { + if (should_compute_output(i)) + return true; + } + return false; + }); + } + + /// Same as the above `should_compute_output` function but will also + /// check whether this edge is needed within the current graph task. + /// Implemented out of line to avoid including graph_task.h. + bool task_should_compute_output(size_t output_edge_index) const; + + /// Returns true if any of the output edges in any of the ranges are active + /// and should be computed in the current graph task. + bool task_should_compute_output( + std::initializer_list idxs) const { + return std::any_of(idxs.begin(), idxs.end(), [this](IndexRange range) { + for (const auto i : c10::irange(range.first, range.second)) { + if (task_should_compute_output(i)) + return true; + } + return false; + }); + } + + /// Returns the PyObjectSlot for thread-safe Python object access. + c10::impl::PyObjectSlot* pyobj_slot() noexcept { + return &pyobj_slot_; + } + const c10::impl::PyObjectSlot* pyobj_slot() const noexcept { + return &pyobj_slot_; + } + + void incref_pyobject() const noexcept override { + pyobj_slot_.incref(); + } + void decref_pyobject() const noexcept override { + pyobj_slot_.decref(); + } + bool try_incref_pyobject() const noexcept override { + return pyobj_slot_.try_incref(); + } + + /// Returns the anomaly metadata stored for this `Node`. + /// If none exist, creates a new empty one. + AnomalyMetadata* metadata() noexcept; + + // Hook API + //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + uintptr_t add_post_hook(std::unique_ptr&& post_hook) { + post_hooks_.emplace_back(std::move(post_hook)); + // Use the raw pointer as the unique key to identify this hook. This key + // can then be used in del_post_hook(key) to remove this hook. + return reinterpret_cast(post_hooks_.back().get()); + } + + const std::vector>& post_hooks() + const noexcept { + return post_hooks_; + } + + // delete a post hook matching the key + bool del_post_hook(const uintptr_t& key) { + for (auto it = post_hooks_.begin(); it != post_hooks_.end(); ++it) { + if (key == reinterpret_cast(it->get())) { + post_hooks_.erase(it); + return true; + } + } + return false; + } + + std::vector>& post_hooks() noexcept { + return post_hooks_; + } + + void add_pre_hook(std::unique_ptr&& pre_hook) { + pre_hooks_.emplace_back(std::move(pre_hook)); + } + + void add_tensor_pre_hook(std::unique_ptr&& pre_hook) { + tensor_pre_hooks_.emplace_back(std::move(pre_hook)); + } + + void add_retains_grad_hook( + std::unique_ptr&& pre_hook, + size_t output_idx) { + retains_grad_hooks_[output_idx] = std::move(pre_hook); + } + + std::unique_ptr pop_retains_grad_hook(size_t output_idx) { + auto ret = std::move(retains_grad_hooks_[output_idx]); + retains_grad_hooks_.erase(output_idx); + return ret; + } + + const std::vector>& pre_hooks() + const noexcept { + return pre_hooks_; + } + + std::vector>& pre_hooks() noexcept { + return pre_hooks_; + } + + virtual std::vector>& + tensor_pre_hooks() noexcept { + return tensor_pre_hooks_; + } + + virtual std::unique_ptr& tensor_post_acc_grad_hooks() + const noexcept { + static std::unique_ptr empty = nullptr; + return empty; + } + + std::unordered_map>& + retains_grad_hooks() noexcept { + return retains_grad_hooks_; + } + + // Customization Points for Subclasses + //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + /// Releases saved variables if the operation won't be reused. + virtual void release_variables() {} + + /// Called before an apply if `release_variables()` is going to be called. + /// Allows larger ops like `InterpreterAutogradFunction` to incrementally + /// release variables as they run. + virtual void will_release_variables() {} + + /// Returns true if this function is traceable. An op is traceable if all + /// operations happening within `apply()` are performed on autograd + /// `Variables` (i.e. apply mostly instantiates and applies other functions). + virtual bool is_traceable() { + return false; + } + + /// A `Node` is said to pass state transparently to backward, if the + /// state consists only of (Saved)Variables and only non-variable objects + /// that parameterize the operation in some way that defines the graph + /// structure AND the backward function is traceable. In particular, + /// parametrization MUST NOT depend on the data of any `Variable`. + /// TODO: it might be possible to handle cases where backward is + /// non-traceable but state passing could be considered transparent. This + /// will probably depend on saved_variable_list being mutable. + /// NOTE: this value matters only if is_traceable() returns false. + virtual bool passes_state_transparently() { + return false; + } + + // see [Note: Compiled Autograd] + // Used by compiled autograd to + // 1) Extract tensors/symint args + // 2) Collect node information for specialization and caching + // Implementations in subclasses should call args.collect() with all node + // attrs. These functions are only called during backward. + virtual void compiled_args(CompiledNodeArgs& args) const { + TORCH_CHECK_NOT_IMPLEMENTED( + false, std::string("compiled_args not implemented: ") + name()); + } + + // Used by compiled autograd to call apply() with different saved tensors + // Implementations should call saved.before() on all attrs, then apply(), then + // saved.after() on all attrs in the same order. + virtual variable_list apply_with_saved( + const variable_list& inputs, + SwapSavedVariables& saved) { + TORCH_CHECK_NOT_IMPLEMENTED( + false, std::string("apply_with_saved not implemented: ") + name()); + } + + // If this node is the AOTBackward node produced by torch.compile. + // Compiled Autograd special-cases on this information. + virtual bool is_aot_backward() const { + return false; + } + + protected: + /// Performs the `Node`'s actual operation. + virtual variable_list apply(variable_list&& inputs) = 0; + + /// Calls `apply()`, but instruments it with tracing machinery. + variable_list traced_apply(variable_list inputs); + + // Sequence number used to correlate backward nodes with forward ops in the + // profiler and provide determinism in the engine. + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + uint64_t sequence_nr_; + + // See NOTE [ Topological Number ] + uint64_t topological_nr_ = 0; + + // Tracks whether this node has been added as the next_edge of another node + // via set_next_edge(s), which always calls topological_nr() of all its + // children See NOTE [ Topological Number ] for why we need this. + mutable bool has_parent_ = false; + + // Id of the thread that created the instance + uint64_t thread_id_ = 0; + + // Note [Thread Safety on Autograd Node] + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // Autograd Engine let the owning thread which calls Engine::execute to drive + // the GraphTask execution, there might be cases that part of the GraphTask is + // shared across different `backward()` or `grad()` calls, i.e. fork new + // threads in the middle of the forward and call `backward()` separately from + // different threads. We need to protect the thread safety on NodeTask to + // prevent data racing on shared variables read/write. + // + // NB: This is only needed for Autograd Nodes that runs on CPU, technically + // "CUDA", "XLA" nodes don't need locking because device threads are always + // single threaded. + // + // Here we add a thread mutex to help protect the Node's thread safety, so + // that different threads cannot race the shared data when executing the same + // NodeTask from multiple CPU threads. It IS the user/developer responsibility + // to take advantage of this mutex to protect the thread safety of their + // autograd Node. The general strategy of thread safety on autograd Node: + // + // 1. User should lock the mutex during Node::release_variables() if the Node + // needs + // to release the variables on the fly, this serve the purpose that when we + // release saved_variables from one thread, no other threads can release + // the saved variables concurrently. call the Node::apply(), + // 2. User should lock the mutex during Node::apply(), this is to ensure Node + // that + // writing to the shared variable are not racing across threads (i.e. + // AccumulateGrad and custom C++ Autograd Node if writing to shared + // variables ) + // 3. item 2 and item 3 should work together so that when we release saved + // variables + // from one thread, no other threads can call Node::apply(), this ensures + // the variable references from other threads aren't dangling. + // 4. if the Node don't release any variables and no shared data read/write in + // the Node + // i.e. purely functional, user don't need to lock the mutex + // + // This way we could protect the thread safety on Autograd Node, but we could + // still not protect the thread safety on Node pre/post C++ hooks (python + // hooks are automatically thread safe), we rely on the user to write thread + // safe C++ hooks if they want the hook to be correctly applied in + // multithreading environment. + std::mutex mutex_; + + edge_list next_edges_; + c10::impl::PyObjectSlot pyobj_slot_; + std::unique_ptr anomaly_metadata_ = nullptr; + + // NOTE [Hooks ordering] + // We have 3 separate fields for pre hooks registered to the autograd nodes + // because the conditions under which they execute are different, and we + // want more fine-grained control over the order in which different types + // of hooks are executed. + // - pre_hooks are only executed when the node itself is executed + // - tensor_pre_hook is executed as long as the engine traverses over it + // even if that node won't be executed. + // - retains_grad_hook are like tensor_pre_hooks except they are always + // ordered after all other tensor pre hooks + std::vector> pre_hooks_; + std::vector> tensor_pre_hooks_; + std::unordered_map> + retains_grad_hooks_; + std::vector> post_hooks_; + at::SmallVector input_metadata_; +}; + +/// See Node::is_traceable() for definition. +struct TraceableFunction : public Node { + using Node::Node; + bool is_traceable() final { + return true; + } +}; + +} // namespace torch::autograd + +// Enable PyObject support for intrusive_ptr so that the refcount +// machinery calls incref_pyobject/decref_pyobject on transitions. +namespace c10::detail { +#ifndef C10_MOBILE +template +struct TargetTraits< + T, + std::enable_if_t< + std::is_base_of_v>>> { + static constexpr bool can_have_pyobject = true; +}; +#endif +} // namespace c10::detail diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/profiler.h b/western_ai/usa/pytorch/torch/csrc/autograd/profiler.h new file mode 100644 index 0000000000000000000000000000000000000000..0bee2efbcbf215e65d3a058588b19cffd7b944f7 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/profiler.h @@ -0,0 +1,4 @@ +#pragma once + +#include +#include diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/profiler_kineto.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/profiler_kineto.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8d75dd371ecf1ed5dbdcefe7395f976f351f05f9 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/profiler_kineto.cpp @@ -0,0 +1,1703 @@ +#include +#include +#define TORCH_ASSERT_ONLY_METHOD_OPERATORS +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifdef USE_KINETO +#include +#include +#include +#include + +#ifndef _MSC_VER +// TODO: TO be removed, once this properly works from libkineto +// Literal copy-n-paste from third_party/kineto/libkineto/src/WeakSymbols.cpp +extern "C" { +// This function is needed to avoid superfluous dependency on GNU OpenMP library +// when cuPTI is linked statically For more details see +// https://github.com/pytorch/pytorch/issues/51026 +__attribute__((weak)) int acc_get_device_type(); +__attribute__((weak)) int acc_get_device_type() { + TORCH_CHECK( + false, + "Dummy implementation of acc_get_device_type is not supposed to be called!"); +} +} // extern "C" +#endif // _MSC_VER +#endif // USE_KINETO + +namespace torch { +namespace autograd::profiler { + +namespace { +inline int64_t getTimeNs() { +#ifdef USE_KINETO + return libkineto::timeSinceEpoch(std::chrono::system_clock::now()); +#else + return c10::getTime(); +#endif // USE_KINETO +} + +using torch::profiler::impl::ActiveProfilerType; +using torch::profiler::impl::EventType; +using torch::profiler::impl::ExtraFields; +using torch::profiler::impl::get_record_concrete_inputs_enabled; +using torch::profiler::impl::ivalueListToStr; +using torch::profiler::impl::ivalueToStr; +using torch::profiler::impl::op_input_t; +using torch::profiler::impl::ProfilerStateBase; +using torch::profiler::impl::PyExtraFieldsBase; +using torch::profiler::impl::Result; +using torch::profiler::impl::shape; +using torch::profiler::impl::shapesToStr; +using torch::profiler::impl::stacksToStr; +using torch::profiler::impl::strListToStr; +using torch::profiler::impl::TensorMetadata; +using torch::profiler::impl::variantShapesToStr; + +// Helper function to check if ProfilerState is a Kineto-compatible state +inline bool isKinetoCompatibleState(ProfilerState state) { + return state == ProfilerState::KINETO || + state == ProfilerState::KINETO_GPU_FALLBACK || + state == ProfilerState::KINETO_PRIVATEUSE1_FALLBACK || + state == ProfilerState::KINETO_PRIVATEUSE1; +} + +// Helper function to check if ProfilerState is valid for disabling profiler +inline bool isValidDisableState(ProfilerState state) { + return isKinetoCompatibleState(state) || + state == ProfilerState::KINETO_ONDEMAND || state == ProfilerState::NVTX || + state == ProfilerState::ITT || state == ProfilerState::PRIVATEUSE1; +} + +// Helper function to check if ProfilerState uses an external tracer +// (NVTX/ITT/PRIVATEUSE1 - these use their own tracing callbacks, not Kineto) +inline bool isExternalTracerState(ProfilerState state) { + return state == ProfilerState::NVTX || state == ProfilerState::ITT || + state == ProfilerState::PRIVATEUSE1; +} + +inline bool hasRequestedDeviceActivity( + const std::set& activities) { + return activities.contains(ActivityType::CUDA) || + activities.contains(ActivityType::XPU) || + activities.contains(ActivityType::MTIA) || + activities.contains(ActivityType::HPU) || + activities.contains(ActivityType::PrivateUse1); +} + +struct OpArgData { + bool hasData; + std::vector shapes; + std::vector dtypes; + std::vector concreteInputs; + std::vector> shapesForKinetoEvent; + std::vector strides; +}; + +auto parseArgData( + const std::vector& input_shapes, + const std::vector& concreteInputs) { + if (input_shapes.empty()) { + return OpArgData{.hasData = false}; + } + + std::vector shapes(input_shapes.size()); + std::vector strides(input_shapes.size()); + std::vector> shapesForKinetoEvent(input_shapes.size()); + + std::vector dtypes(input_shapes.size()); + std::vector concrete_inputs_list; + + for (const auto& i : c10::irange(input_shapes.size())) { + std::visit( + c10::overloaded( + [&](const TensorMetadata& t) { + shapes[i] = t.sizes_; + shapesForKinetoEvent[i] = t.sizes_; + dtypes[i] = std::string(scalarTypeToTypeMeta(t.dtype_).name()); + strides[i] = t.strides_; + }, + [&](const std::vector& l) { + std::vector> shape; + shape.reserve(l.size()); + std::vector> stride; + stride.reserve(l.size()); + for (const auto& t : l) { + shape.emplace_back(t.sizes_); + stride.emplace_back(t.strides_); + } + shapes[i] = shape; + strides[i] = stride; + dtypes[i] = "TensorList"; + }, + [&](const c10::IValue&) { dtypes[i] = "Scalar"; }, + [&](const auto&) {}), + input_shapes[i]); + } + + // If we recorded concrete inputs, then parse them + if (input_shapes.size() == concreteInputs.size() && !concreteInputs.empty()) { + concrete_inputs_list.resize(input_shapes.size()); + + for (const auto& i : c10::irange(input_shapes.size())) { + std::visit( + c10::overloaded( + [&](const c10::IValue& val) { concrete_inputs_list[i] = val; }, + [&](const auto&) {}), + input_shapes[i]); + std::visit( + c10::overloaded( + [&](const c10::IValue& val) { + concrete_inputs_list[i] = val; + dtypes[i] = "ScalarList"; + }, + [&](const auto&) {}), + concreteInputs[i]); + } + } + + return OpArgData{ + .hasData = true, + .shapes = shapes, + .dtypes = dtypes, + .concreteInputs = concrete_inputs_list, + .shapesForKinetoEvent = shapesForKinetoEvent, + .strides = strides}; +} + +struct MetadataBase { + /* implicit */ MetadataBase(const std::shared_ptr& result) + : kinetoActivity_{result->kineto_activity_} { + if (std::holds_alternative>( + result->extra_fields_)) { + // In order to add metadata we have to downcast from + // `libkineto::ITraceActivity` to `libkineto::GenericTraceActivity`. We + // know that all activities provided by PyTorch are of the correct type, + // however Kineto profilers can (and do) add events that inherit directly + // from ITraceActivity. As a result, any Result which was constructed from + // an event that Kineto provided is unsafe to cast. + if (!(SOFT_ASSERT(!hasKinetoActivity()))) { + result->kineto_activity_ = nullptr; + } + kinetoActivity_ = result->kineto_activity_; + } + } + + void addMetadata( + const std::string& key, + const std::string& value, + bool quote = false) { + if (kinetoActivity_ && !value.empty() && value != "\"\"") { + torch::profiler::impl::kineto::addMetadata( + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + const_cast( + kinetoActivity_), + key, + value, + quote); + } + } + + bool hasKinetoActivity() const { + return kinetoActivity_ != nullptr; + } + + private: + const torch::profiler::impl::kineto::activity_t* kinetoActivity_{nullptr}; +}; + +struct AddTensorboardFields : public MetadataBase { + AddTensorboardFields( + const std::shared_ptr& result, + KinetoEvent& kineto_event) + : MetadataBase(result) { + result->visit(*this); + const auto module_hierarchy = kineto_event.moduleHierarchy(); + addMetadata("Module Hierarchy", stacksToStr(module_hierarchy.vec(), ".")); + addMetadata("Call stack", stacksToStr(kineto_event.stack().vec(), ";")); + + result->visit_if_base([&, this](const auto& i) -> void { + this->addMetadata("Python id", std::to_string(i.id_)); + + std::optional parent_id; + std::shared_ptr parent = result->parent_.lock(); + while (parent && !parent_id.has_value()) { + parent->visit_if_base( + [&](const auto& j) { parent_id = std::to_string(j.id_); }); + parent = parent->parent_.lock(); + } + this->addMetadata("Python parent id", parent_id.value_or("null")); + if (i.caller_.line_no_ > 0) { + this->addMetadata( + "CallFrom", + fmt::format("{}:{}", i.caller_.filename_.str(), i.caller_.line_no_), + /*quote=*/true); + } + }); + } + + void operator()(const ExtraFields& py_call) { + if (py_call.module_.has_value()) { + addMetadata("Python module id", std::to_string(py_call.module_->id_)); + } + } + + template + void operator()(const T& /*unused*/) {} +}; + +struct AddGenericMetadata : public MetadataBase { + AddGenericMetadata( + std::shared_ptr& result, + const torch::profiler::impl::ProfilerConfig* config) + : MetadataBase(result), config_(config) { + result->visit(*this); + if (config->experimental_config.verbose) { + result->visit_if_base( + [&, this](const auto& i) -> void { + this->addMetadata("Python thread", std::to_string(i.python_tid_)); + }); + } + } + + void operator()(ExtraFields& op_event) { + const auto arg_data = + parseArgData(op_event.inputs_, op_event.concrete_inputs_); + + if (arg_data.hasData) { + if (get_record_concrete_inputs_enabled()) { + addMetadata("Input Dims", variantShapesToStr(arg_data.shapes)); + } else { + addMetadata("Input Dims", shapesToStr(arg_data.shapesForKinetoEvent)); + } + addMetadata("Input Strides", variantShapesToStr(arg_data.strides)); + addMetadata("Input type", strListToStr(arg_data.dtypes)); + if (!arg_data.concreteInputs.empty()) { + addMetadata( + "Concrete Inputs", ivalueListToStr(arg_data.concreteInputs)); + } + } + + // Add metadata for kwinputs if exist + for (const auto& [key, val] : op_event.kwinputs_) { + if (key == "stream" && !val.isInt()) { + LOG(WARNING) << "Inputted stream is not an int for op: " + << op_event.name_ << " skipping"; + continue; + } + + // Until needed, let's limit the kwargs to only ints, doubles, strings, + // bools, and list of strings + bool isValidType = + val.isInt() || val.isDouble() || val.isString() || val.isBool(); + bool isStringList = false; + + if (!isValidType && val.isList()) { + // Check if it's a list of strings + auto list = val.toListRef(); + isStringList = std::ranges::all_of( + list, [](const c10::IValue& item) { return item.isString(); }); + } + + if (!isValidType && !isStringList) { + LOG(WARNING) + << "Inputted kwarg: " << key + << " is not an int, double, string, bool, or list of strings for op: " + << op_event.name_ << " skipping"; + continue; + } + + if (isStringList) { + // For list of strings, use ivalueListToStr + auto list = val.toListRef(); + std::vector stringList(list.begin(), list.end()); + addMetadata(key, ivalueListToStr(stringList)); + } else { + bool isString = val.isString(); + addMetadata(key, ivalueToStr(val, isString)); + } + } + // Add extra metadata if any + for (const auto& [key, val] : op_event.extra_meta_) { + addMetadata(key, val); + } + + if (config_ && !config_->experimental_config.performance_events.empty()) { + auto& event_names = config_->experimental_config.performance_events; + for (const auto i : c10::irange(op_event.perf_event_counters_->size())) { + addMetadata( + event_names[i], + std::to_string((*op_event.perf_event_counters_)[i])); + } + } + + // add information about an associated forward op, if a sequence number + // is available (e.g. during training) + if (op_event.sequence_number_ >= 0) { + addMetadata("Fwd thread id", std::to_string(op_event.forward_tid_)); + addMetadata("Sequence number", std::to_string(op_event.sequence_number_)); + } + addMetadata( + "Record function id", std::to_string(op_event.record_function_id_)); + } + + void operator()(ExtraFields& backend_event) { + if (!backend_event.backend_.empty()) { + addMetadata("Backend", "\"" + backend_event.backend_ + "\""); + } + } + + void operator()(const ExtraFields& alloc) { + addMetadata("Device Type", std::to_string((int8_t)alloc.device_type_)); + addMetadata("Device Id", std::to_string(alloc.device_index_)); + addMetadata("Addr", std::to_string(reinterpret_cast(alloc.ptr_))); + addMetadata("Bytes", std::to_string(alloc.alloc_size_)); + addMetadata("Total Allocated", std::to_string(alloc.total_allocated_)); + addMetadata("Total Reserved", std::to_string(alloc.total_reserved_)); + } + + void operator()(const ExtraFields& alloc) { + addMetadata("Device Type", std::to_string((int8_t)alloc.device_type_)); + addMetadata("Device Id", std::to_string(alloc.device_index_)); + addMetadata("Bytes", std::to_string(alloc.alloc_size_)); + addMetadata("Total Allocated", std::to_string(alloc.total_allocated_)); + addMetadata("Total Reserved", std::to_string(alloc.total_reserved_)); + } + + template + void operator()(const T& /*unused*/) {} + + private: + /* To get names of the performance events */ + const torch::profiler::impl::ProfilerConfig* config_; +}; + +// Lightweight metadata pass for trace_only mode: annotates Kineto activities +// with the same metadata as materializeOpEvents but without creating +// KinetoEvent wrappers or building eventTree. +void addTraceMetadata( + std::vector>& events, + const torch::profiler::impl::ProfilerConfig& config, + int64_t trace_end_ns) { + for (auto& e : events) { + // Unfinished events automatically have end time set to trace end time + if (!e->finished_) { + e->visit(c10::overloaded( + [trace_end_ns](ExtraFields& i) { + i.end_time_ns_ = trace_end_ns; + }, + [](auto&) {})); + } + + if (!e->kineto_activity_) { + continue; + } + AddGenericMetadata add_generic(e, &config); + + // Subset of AddTensorboardFields that doesn't require KinetoEvent or + // parent chain (no python_stack_, no Python parent id). + MetadataBase tb(e); + e->visit(c10::overloaded( + [&](const ExtraFields& i) { + tb.addMetadata("Module Hierarchy", stacksToStr(i.jit_modules_, ".")); + tb.addMetadata("Call stack", stacksToStr(i.jit_stack_, ";")); + }, + [&](const ExtraFields& i) { + tb.addMetadata("Module Hierarchy", stacksToStr(i.jit_modules_, ".")); + tb.addMetadata("Call stack", stacksToStr(i.jit_stack_, ";")); + }, + [](const auto&) {})); + e->visit_if_base([&](const auto& i) { + tb.addMetadata("Python id", std::to_string(i.id_)); + }); + e->visit(c10::overloaded( + [&](const ExtraFields& py_call) { + if (py_call.module_.has_value()) { + tb.addMetadata( + "Python module id", std::to_string(py_call.module_->id_)); + } + }, + [](const auto&) {})); + + e->kineto_activity_ = nullptr; + } +} + +struct KinetoThreadLocalState : public ProfilerStateBase { + explicit KinetoThreadLocalState( + const ProfilerConfig& config, + std::set activities) + : ProfilerStateBase(config), + startTime(getTimeNs()), + recordQueue(config, std::move(activities)) {} + ~KinetoThreadLocalState() override = default; + + static std::shared_ptr getGlobal() { + std::shared_ptr state = ProfilerStateBase::getGlobal(); + TORCH_INTERNAL_ASSERT_DEBUG_ONLY( + state == nullptr || + state->profilerType() == ActiveProfilerType::KINETO); + return std::static_pointer_cast(std::move(state)); + } + + static KinetoThreadLocalState* getTLS() { + ProfilerStateBase* state = ProfilerStateBase::getTLS(); + TORCH_INTERNAL_ASSERT_DEBUG_ONLY( + state == nullptr || + state->profilerType() == ActiveProfilerType::KINETO); + return static_cast(state); + } + + ActiveProfilerType profilerType() override { + return ActiveProfilerType::KINETO; + } + + void reportVulkanEventToProfiler(torch::profiler::impl::vulkan_id_t id) { + if (!config_.disabled()) { + recordQueue.getSubqueue()->emplace_vulkan_event( + c10::getApproximateTime(), id); + } + } + + void reportMemoryUsage( + void* ptr, + int64_t alloc_size, + size_t total_allocated, + size_t total_reserved, + c10::Device device) override { + if (config_.profile_memory && !config_.disabled()) { + recordQueue.getSubqueue()->emplace_allocation_event( + c10::getApproximateTime(), + ptr, + alloc_size, + total_allocated, + total_reserved, + device.type(), + device.index()); + } + } + + void reportOutOfMemory( + int64_t alloc_size, + size_t total_allocated, + size_t total_reserved, + c10::Device device) override { + if (config_.profile_memory && !config_.disabled()) { + recordQueue.getSubqueue()->emplace_ooms_event( + c10::getApproximateTime(), + alloc_size, + total_allocated, + total_reserved, + device.type(), + device.index()); + } + } + + void setEventPostProcessingCallback(post_process_t&& cb) { + eventPostProcessCb = std::move(cb); + } + + void pausePython() { + recordQueue.stop(); + } + + void resumePython() { + recordQueue.restart(); + } + + std::unique_ptr + finalizeTrace() { + auto end_time = getTimeNs(); + recordQueue.stop(); + + std::lock_guard guard(state_mutex_); + auto converter = clockConverter.makeConverter(); +#ifdef USE_KINETO + libkineto::get_time_converter() = converter; +#endif + auto records_and_trace = + recordQueue.getRecords(std::move(converter), startTime, end_time); + + if (config_.experimental_config.trace_only) { + addTraceMetadata(records_and_trace.first, config_, end_time); + } else { + materializeOpEvents(records_and_trace.first, end_time); + } + + return std::move(records_and_trace.second); + } + + template + void invokeCallback(T& t) { + if (eventPostProcessCb) { + eventPostProcessCb(t.debug_handle_, t.jit_stack_, t.jit_modules_); + } + } + + void materializeOpEvents( + std::vector>& events, + int64_t trace_end_ns) { + for (auto& e : events) { + if (e->parent_.expired() && e->deviceType() == c10::DeviceType::CPU) { + eventTree.push_back(e); + } + + // Unfinished events automatically have end time set to trace end time + if (!e->finished_) { + e->visit(c10::overloaded( + [trace_end_ns](ExtraFields& i) { + i.end_time_ns_ = trace_end_ns; + }, + [](auto&) {})); + } + + e->visit(c10::overloaded( + [this](ExtraFields& i) { invokeCallback(i); }, + [this](ExtraFields& i) { invokeCallback(i); }, + [](auto&) {})); + + kinetoEvents.emplace_back(e, config_.experimental_config.verbose); + AddTensorboardFields add_tb(e, kinetoEvents.back()); + AddGenericMetadata add_generic(e, &config_); + + // It is not safe to use the activity after post processing. + e->kineto_activity_ = nullptr; + } + } + + uint64_t startTime; + c10::ApproximateClockToUnixTimeConverter clockConverter; + torch::profiler::impl::RecordQueue recordQueue; + std::vector kinetoEvents; + std::vector eventTree; + // Optional, if event post-processing is enabled. + post_process_t eventPostProcessCb; +}; + +// Coordinates one global RecordFunction callback session (KINETO_ONDEMAND, or +// KINETO with profile_all_threads) against teardown. Those callbacks fire on +// every thread and touch the profiler state while disableProfiler() finalizes +// and frees it on another thread. +// +// While the session is active, each callback invocation brackets only its own +// short critical section - getGlobal() plus the RecordQueue access - with +// enter()/exit(). Teardown calls drain(), which closes the session to new +// callbacks and blocks until the in-flight count reaches zero, so finalize +// never races a live critical section. +// +// The enter()/isActive() re-check is a handshake: a callback that joined before +// teardown is always waited for, while one arriving after bails without +// touching the state. +// +// Bracketing only the critical section (not the op body) is what keeps drain() +// from deadlocking against the GIL, but it lets an op enter under one session +// and exit after that session is torn down (its RecordQueue and event freed). A +// per-session generation (see generation()), stamped on each callback's +// ObserverContext, closes that gap: onFunctionExitGlobal drops a straddling +// exit on generation mismatch instead of dereferencing the freed event. +// +// The drain counter is not a std::latch: a std::latch fixes its count at +// construction, cannot be incremented afterward, and is single-use. Here the +// number of in-flight callbacks is unknown up front, grows dynamically as ops +// dispatch, and the session must be re-armed for every enable/disable cycle, +// none of which std::latch supports. The counter stays lock-free on the per-op +// hot path; the mutex/cv are touched only on the teardown handoff. +// +// Because we coordinate across threads that can call it at any arbitrary time, +// this object must have static lifetime. +class GlobalCallbackSession { + public: + GlobalCallbackSession() = default; + ~GlobalCallbackSession() = default; + + // Single, static instance only: neither copyable nor movable. + GlobalCallbackSession(const GlobalCallbackSession&) = delete; + GlobalCallbackSession& operator=(const GlobalCallbackSession&) = delete; + GlobalCallbackSession(GlobalCallbackSession&&) = delete; + GlobalCallbackSession& operator=(GlobalCallbackSession&&) = delete; + + // Open the session (at enableProfiler time) so callbacks begin participating. + // On a true enable a fresh RecordQueue was just installed, so bump the + // session generation first: callbacks in the new session stamp the new value + // and a straddling exit from the prior session detects the mismatch. The + // mid-session dynamic collection toggle re-arms the same RecordQueue and + // passes false; bumping there would wrongly drop end-events for ops that + // straddle the toggle. + void activate(bool new_session) { + if (new_session) { + generation_.fetch_add(1); + } + active_.store(true); + } + + // Session generation, stamped into each callback's ObserverContext at enter + // and re-checked at exit. A mismatch means the stamping session was torn down + // (its RecordQueue and event freed), so the exit drops itself. + uint64_t generation() const { + return generation_.load(); + } + + // Whether the session is open; that is, teardown has not begun. + bool isActive() const { + return active_.load(); + } + + // Register an in-flight callback, paired with exit(). Callers must re-check + // isActive() after enter() to close the race with a concurrent drain(). + void enter() { + in_flight_.fetch_add(1); + } + + // Deregister an in-flight callback. When this is the last one and the session + // has been closed (drain() in progress), wake the waiting teardown thread. + // During normal profiling this is a single lock-free decrement. + void exit() { + if (in_flight_.fetch_sub(1) == 1 && !active_.load()) { + std::lock_guard lock(mutex_); + cv_.notify_all(); + } + } + + // Close the session to new callbacks and block until all in-flight ones have + // exited; returns true once drained (also returns true if the session was + // never opened; it remained purely thread-local profiling). Closing is fused + // with the wait, so the session cannot be closed without waiting out + // in-flight callbacks. + // + // Returns false if the drain does not complete within kDrainTimeout. A normal + // drain takes microseconds (the in-flight window is just a callback's + // getGlobal() + RecordQueue critical section, never an op body), so a + // multi-second wait means a callback's thread was killed or wedged inside + // that critical section. The caller must then leave the state installed: + // freeing it under a live callback would reintroduce the use-after-free this + // session guards against. + bool drain() { + if (!active_.exchange(false)) { + return true; + } + std::unique_lock lock(mutex_); + return cv_.wait_for( + lock, kDrainTimeout, [this] { return in_flight_.load() == 0; }); + } + + private: + // Backstop so disableProfiler() can never hang on a wedged or dead callback + // thread. Far longer than any real drain, which completes in microseconds. + static constexpr std::chrono::seconds kDrainTimeout{30}; + + std::atomic active_{false}; + std::atomic in_flight_{0}; + std::atomic generation_{0}; + std::mutex mutex_; + std::condition_variable cv_; // guarded by mutex_ +}; + +GlobalCallbackSession global_callback_session; + +std::unique_ptr onFunctionEnterGlobal( + const at::RecordFunction& fn) { + // Fast bail once teardown has begun, before taking an in-flight ref, so the + // drain in disableProfiler() always converges. + if (!global_callback_session.isActive()) { + return nullptr; + } + + global_callback_session.enter(); + + // Release the ref when this function returns (success or early bail), so the + // in-flight window is just this enter's getGlobal() + begin_op() critical + // section. + auto in_flight_guard = + c10::make_scope_exit([] { global_callback_session.exit(); }); + + // Re-check after the increment. This is the handshake that makes teardown + // safe: if disableProfiler() cleared active concurrently, its drain is + // guaranteed to observe our increment and wait if and only if we still + // observe active true here. + if (!global_callback_session.isActive()) { + return nullptr; + } + + std::shared_ptr state_ptr = + KinetoThreadLocalState::getGlobal(); + if (!state_ptr) { + return nullptr; + } + auto ctx = state_ptr->recordQueue.getSubqueue()->begin_op(fn); + if (ctx) { + // Stamp the current session so a straddling exit (begin_op here, exit after + // this session is torn down) drops itself on mismatch instead of writing + // through a freed event_. + ctx->session_generation_ = global_callback_session.generation(); + } + return ctx; +} + +std::unique_ptr onFunctionEnterTLS( + const at::RecordFunction& fn) { + KinetoThreadLocalState* state_ptr = KinetoThreadLocalState::getTLS(); + if (!state_ptr) { + return nullptr; + } + return state_ptr->recordQueue.getSubqueue()->begin_op(fn); +} + +void onFunctionExitImpl( + KinetoThreadLocalState& state, + const at::RecordFunction& fn, + at::ObserverContext* ctx_ptr) { + const auto& config = state.config(); + auto* kineto_ctx_ptr = + static_cast(ctx_ptr); + TORCH_INTERNAL_ASSERT(kineto_ctx_ptr != nullptr); + kineto_ctx_ptr->event_->end_time_ = c10::getApproximateTime(); + if (!config.experimental_config.performance_events.empty()) { + state.recordQueue.getSubqueue()->disable_perf_profiler( + *kineto_ctx_ptr->event_->counters_); + } + kineto_ctx_ptr->event_->basic_fields_.end_tid_ = + at::RecordFunction::currentThreadId(); + if (fn.isNcclMeta()) { + auto& extra_meta = *(kineto_ctx_ptr->event_->extra_nccl_meta_); + // Record only the outputs in this exit callback of the record function + torch::profiler::impl::SaveNcclMetaConfig ncclMetaConfig{ + true, false, false, true}; + auto additional_nccl_meta = + torch::profiler::impl::saveNcclMeta(fn, ncclMetaConfig); + extra_meta.insert(additional_nccl_meta.begin(), additional_nccl_meta.end()); + } + if (config.state == ProfilerState::KINETO_GPU_FALLBACK) { + try { + auto fallback = kineto_ctx_ptr->fallback_; + TORCH_INTERNAL_ASSERT(fallback != nullptr); + torch::profiler::impl::cudaStubs()->record( + nullptr, &fallback->device_event_end_, nullptr); + } catch (const std::exception& e) { + LOG(WARNING) << "Failed to record CUDA event. " << e.what(); + } + } else if (config.state == ProfilerState::KINETO_PRIVATEUSE1_FALLBACK) { + auto fallback = kineto_ctx_ptr->fallback_; + TORCH_INTERNAL_ASSERT(fallback != nullptr); + torch::profiler::impl::privateuse1Stubs()->record( + nullptr, &fallback->device_event_end_, nullptr); + } +} + +// Pop the external correlation id that begin_op pushed for this op, if any. +// Must run on every onFunctionExit path, including the teardown and +// stale-session early exits that skip event finalization: the correlation stack +// lives in the device profiling backend (per thread) and is not reset across +// profiler sessions, so a skipped pop leaks. Safe on those paths because it +// touches only that stack, never the possibly-freed event_. +void maybePopCorrelationId( + const at::RecordFunction& fn, + at::ObserverContext* ctx_ptr) { + auto* kineto_ctx = + static_cast(ctx_ptr); + if (kineto_ctx == nullptr || !kineto_ctx->pushed_correlation_id_) { + return; + } + if (fn.scope() == at::RecordScope::USER_SCOPE) { + torch::profiler::impl::kineto::popUserCorrelationId(); + } else { + torch::profiler::impl::kineto::popCorrelationId(); + } +} + +void onFunctionExitGlobal( + const at::RecordFunction& fn, + at::ObserverContext* ctx_ptr) { + if (ctx_ptr == nullptr) { + // Enter bailed (teardown in progress, or no state): no ref was taken and + // there is nothing to finalize. + return; + } + + // Balance the correlation id pushed at begin_op on every path below, + // including the teardown and stale-session early exits that skip event + // finalization. + auto correlation_guard = + c10::make_scope_exit([&] { maybePopCorrelationId(fn, ctx_ptr); }); + + // Take this exit's own in-flight ref, paired here because + // onFunctionEnterGlobal no longer hands one off. It covers only this exit's + // getGlobal() + RecordQueue write, never the op body, so the drain cannot + // block on it. + global_callback_session.enter(); + auto in_flight_guard = + c10::make_scope_exit([] { global_callback_session.exit(); }); + + // Re-check after the increment, mirroring onFunctionEnterGlobal: if teardown + // began concurrently the queue may already be finalized/freed, so skip the + // write (the event keeps its start and gets no end). + if (!global_callback_session.isActive()) { + return; + } + + std::shared_ptr state_ptr = + KinetoThreadLocalState::getGlobal(); + if (!state_ptr) { + return; + } + + // If the session generations don't match, that means this op entered under an + // earlier session that has since been torn down, freeing its RecordQueue and + // event_. In that case, we don't need to do any exit cleanup. + auto* kineto_ctx = + static_cast(ctx_ptr); + if (kineto_ctx->session_generation_ != global_callback_session.generation()) { + return; + } + + onFunctionExitImpl(*state_ptr, fn, ctx_ptr); +} + +void onFunctionExitTLS( + const at::RecordFunction& fn, + at::ObserverContext* ctx_ptr) { + // Balance the correlation id pushed at begin_op even when the TLS state is + // gone by exit (early return below), for the same reason as + // onFunctionExitGlobal. + auto correlation_guard = + c10::make_scope_exit([&] { maybePopCorrelationId(fn, ctx_ptr); }); + KinetoThreadLocalState* state_ptr = KinetoThreadLocalState::getTLS(); + if (!state_ptr) { + return; + } + onFunctionExitImpl(*state_ptr, fn, ctx_ptr); +} + +void pushGlobalProfilingCallbacks( + const std::unordered_set& scopes, + bool new_session) { + std::shared_ptr state_ptr = + KinetoThreadLocalState::getGlobal(); + TORCH_INTERNAL_ASSERT(state_ptr, "Expected profiler state set"); + auto recordFunctionCallback = + at::RecordFunctionCallback(onFunctionEnterGlobal, onFunctionExitGlobal) + .needsInputs(state_ptr->config().report_input_shapes) + .scopes(scopes); + + // Arm the drain gate before the global callback and fire on any thread. If + // this a new profiling session, also bump the session generation. + // disableProfiler() relies on this to know it must drain in-flight callbacks. + global_callback_session.activate(new_session); + + state_ptr->setCallbackHandle(at::addGlobalCallback(recordFunctionCallback)); +} + +void pushTLSProfilingCallbacks( + const std::unordered_set& scopes) { + KinetoThreadLocalState* state_ptr = KinetoThreadLocalState::getTLS(); + TORCH_INTERNAL_ASSERT(state_ptr, "Expected profiler state set"); + auto recordFunctionCallback = + at::RecordFunctionCallback(onFunctionEnterTLS, onFunctionExitTLS) + .needsInputs(state_ptr->config().report_input_shapes) + .scopes(scopes); + state_ptr->setCallbackHandle( + at::addThreadLocalCallback(recordFunctionCallback)); +} + +struct ProfilerStateInfo { + std::shared_ptr state_ptr; + std::unordered_set scopes; +}; +std::shared_ptr profiler_state_info_ptr{nullptr}; + +} // namespace + +void reportBackendEventToActiveKinetoProfiler( + const int64_t start_time_us, + const int64_t end_time_us, + const int64_t debug_handle, + const at::RecordScope scope, + const std::string& event_name, + const std::string& backend_name) { + TORCH_INTERNAL_ASSERT( + KinetoThreadLocalState::getGlobal() == nullptr, + "On-demand profiling does not support post processing callback"); + + KinetoThreadLocalState* state_ptr = KinetoThreadLocalState::getTLS(); + if (!state_ptr) { + return; + } + + state_ptr->recordQueue.getSubqueue()->emplace_backend_event( + start_time_us, + end_time_us, + debug_handle, + scope, + event_name, + backend_name); + + /* no support for input shapes now? + if (config.report_input_shapes) { + ctx_ptr->shapes = inputSizes(fn); + ctx_ptr->dtypes = inputTypes(fn); + } + */ +} + +void prepareProfiler( + const torch::profiler::impl::ProfilerConfig& config, + const std::set& activities, + const ActivityFilter& activity_filter) { + if (config.state == ProfilerState::NVTX || + config.state == ProfilerState::ITT) { + return; + } + + // Forward registered PrivateUse1 profiler factory to Kineto. + // Only for KINETO_PRIVATEUSE1 state where backend provides its own + // IActivityProfiler. +#ifdef USE_KINETO + if (config.state == ProfilerState::KINETO_PRIVATEUSE1) { + torch::profiler::impl::PrivateUse1ProfilerRegistry::instance() + .onKinetoInit(); + } +#endif // USE_KINETO + + TORCH_CHECK( + isKinetoCompatibleState(config.state), + "Supported only in Kineto profiler"); + torch::profiler::impl::kineto::prepareTrace( + /*cpuOnly=*/!hasRequestedDeviceActivity(activities), + activities, + config.experimental_config, + config.trace_id, + activity_filter); + + if (!config.experimental_config.performance_events.empty()) { + /* For now only CPU activity is supported */ + TORCH_CHECK( + activities.count(torch::autograd::profiler::ActivityType::CPU), + "Cannot run cpu hardware profiler without CPU activities, please only use CPU activity type"); + /* + * Sending a warning and passing the non-standard event to the backend + * Backend can abort if the event is not supported. + * TODO Should we gracefully drop the invalid event if we have at least one + * valid? + */ + auto is_standard_event = [](const std::string& event) -> bool { + for (auto e : torch::profiler::ProfilerPerfEvents) { + if (!std::strcmp(event.c_str(), e)) { + return true; + } + } + return false; + }; + + for (const auto& e : config.experimental_config.performance_events) { + if (!is_standard_event(e)) { + TORCH_WARN("Forwarding a non-standard CPU performance event : ", e); + } + } + } +} + +static void toggleTorchOpCollectionDynamic(bool enable) { + std::shared_ptr global_state = + ProfilerStateBase::getGlobal(); + if (global_state) { + if (enable) { + auto scopes = profiler_state_info_ptr->scopes; + // Mid-session re-arm on the same RecordQueue: do not bump the generation. + pushGlobalProfilingCallbacks(scopes, /*new_session=*/false); + } else { + global_state->removeCallback(); + } + } else { + ProfilerStateBase* tls_state = ProfilerStateBase::getTLS(); + TORCH_CHECK(tls_state); + if (enable) { + auto scopes = profiler_state_info_ptr->scopes; + pushTLSProfilingCallbacks(scopes); + } else { + tls_state->removeCallback(); + } + } +} + +// Set this function to be unused as profiler implementation needs more +// refactoring to support Python ops collection dynamic toggling +#ifdef _MSC_VER +#define UNUSED +#else +#define UNUSED __attribute__((unused)) +#endif +static UNUSED void togglePythonCollectionDynamic(bool enable) { + std::shared_ptr global_state = + KinetoThreadLocalState::getGlobal(); + if (global_state) { + if (enable) { + global_state->resumePython(); + } else { + global_state->pausePython(); + } + } else { + KinetoThreadLocalState* tls_state = KinetoThreadLocalState::getTLS(); + TORCH_CHECK(tls_state); + if (enable) { + tls_state->resumePython(); + } else { + tls_state->pausePython(); + } + } +} + +static void toggleCPUCollectionDynamic(bool enable) { + toggleTorchOpCollectionDynamic(enable); + // For now we only support Torch Op collection dynamic toggling as + // implementing Python ops would require not only string parsing to get rid of + // the toggling events as well as other unfinished events as well as changes + // in stack logic + // togglePythonCollectionDynamic(enable); +} + +void toggleCollectionDynamic( + const bool enable, + const std::set& activities) { + if (activities.contains(torch::autograd::profiler::ActivityType::CPU) && + (!activities.contains(torch::autograd::profiler::ActivityType::CUDA) || + !activities.contains(torch::autograd::profiler::ActivityType::XPU))) { + LOG(WARNING) + << "Toggling CPU activity with GPU activity on may result in traces with GPU events on arbitrary tracks"; + } else if ( + (activities.contains(torch::autograd::profiler::ActivityType::CUDA) || + activities.contains(torch::autograd::profiler::ActivityType::XPU)) && + !activities.contains(torch::autograd::profiler::ActivityType::CPU)) { + LOG(WARNING) + << "Toggling GPU activity with CPU activity on may result in traces with incorrect correlation between CPU and GPU events"; + } + for (auto act : activities) { + if (act == torch::autograd::profiler::ActivityType::CUDA || + act == torch::autograd::profiler::ActivityType::XPU) { + torch::profiler::impl::kineto::toggleCollectionDynamic(enable); + } else if (act == torch::autograd::profiler::ActivityType::CPU) { + toggleCPUCollectionDynamic(enable); + } else { + LOG(WARNING) + << "Dynamic toggle is only supported for CPU/GPU activity, skipping toggling of " + << actToString(act); + continue; + } + } +} + +void enableProfilerWithEventPostProcess( + const torch::profiler::impl::ProfilerConfig& config, + const std::set& activities, + post_process_t&& cb, + const std::unordered_set& scopes) { + TORCH_CHECK( + config.state != ProfilerState::NVTX, + "NVTX does not support post processing callback."); + TORCH_CHECK( + config.state != ProfilerState::ITT, + "ITT does not support post processing callback."); + TORCH_INTERNAL_ASSERT( + KinetoThreadLocalState::getGlobal() == nullptr, + "On-demand profiling does not support post processing callback"); + + enableProfiler(config, activities, scopes); + if (config.pushGlobalCallbacks()) { + std::shared_ptr state_ptr = + KinetoThreadLocalState::getGlobal(); + state_ptr->setEventPostProcessingCallback(std::move(cb)); + } else { + KinetoThreadLocalState* state_ptr = KinetoThreadLocalState::getTLS(); + state_ptr->setEventPostProcessingCallback(std::move(cb)); + } +} + +void enableProfiler( + const torch::profiler::impl::ProfilerConfig& config, + const std::set& activities, + const std::unordered_set& scopes) { + const auto has_cpu = activities.count(ActivityType::CPU); + bool already_enabled = config.pushGlobalCallbacks() + ? KinetoThreadLocalState::getGlobal() != nullptr + : KinetoThreadLocalState::getTLS() != nullptr; + TORCH_CHECK( + !already_enabled, + "Profiler is already enabled", + (config.global() ? "." : " on this thread.")); + + // Handle external tracer states - these use their own tracing callbacks + if (isExternalTracerState(config.state)) { + switch (config.state) { + case ProfilerState::NVTX: + torch::profiler::impl::pushNVTXCallbacks(config, scopes); + break; + case ProfilerState::ITT: + torch::profiler::impl::pushITTCallbacks(config, scopes); + break; + case ProfilerState::PRIVATEUSE1: + torch::profiler::impl::pushPRIVATEUSE1CallbacksStub(config, scopes); + break; + default: + break; + } + return; + } + + TORCH_CHECK(isKinetoCompatibleState(config.state) || config.global()); + TORCH_CHECK(!activities.empty(), "No activities specified."); + TORCH_INTERNAL_ASSERT( + has_cpu || !config.global(), + "Ondemand profiling must enable CPU tracing"); + + auto state_ptr = std::make_shared(config, activities); + KinetoThreadLocalState::push(state_ptr); + + if (has_cpu) { + config.pushGlobalCallbacks() + ? pushGlobalProfilingCallbacks(scopes, /*new_session=*/true) + : pushTLSProfilingCallbacks(scopes); + } + + if (!config.global()) { + torch::profiler::impl::kineto::startTrace(); + } + + if (has_cpu) { + auto state_info_ptr = std::make_shared(); + state_info_ptr->state_ptr = state_ptr; + state_info_ptr->scopes = scopes; + profiler_state_info_ptr = state_info_ptr; + } +} + +bool isProfilerEnabledInMainThread() { + return profiler_state_info_ptr != nullptr; +} + +void enableProfilerInChildThread() { + auto state_info_ptr = profiler_state_info_ptr; + TORCH_CHECK(state_info_ptr, "Profiler is not enabled in main thread."); + TORCH_CHECK( + KinetoThreadLocalState::getTLS() == nullptr, + "Profiler is already enabled in this thread."); + + KinetoThreadLocalState::push(state_info_ptr->state_ptr); + pushTLSProfilingCallbacks(state_info_ptr->scopes); +} + +void disableProfilerInChildThread() { + auto state_ptr = ProfilerStateBase::pop(); + TORCH_CHECK( + state_ptr, + "Can't disable Kineto profiler when it's not running in this thread"); + state_ptr->removeCallback(); +} + +std::unique_ptr disableProfiler() { + // releasing to inform child threads to stop profiling + profiler_state_info_ptr = nullptr; + + // If global callbacks were installed (KINETO_ONDEMAND, or KINETO with + // profile_all_threads), they may be running on other threads right now. Wait + // for in-flight ones to finish before popping and finalizing the state, else + // a worker thread can mutate the record queue while finalizeTrace() reads + // and frees it. + if (!global_callback_session.drain()) { + // The drain timed out: an in-flight global callback is wedged or dead, so + // we cannot finalize (reading the queue would race the live callback and + // reintroduce the use-after-free). We still pop the state and remove the + // global callback so a later enableProfiler() is not rejected as "already + // enabled". Safe with callbacks in flight: the shared_ptr keepalive keeps + // the state alive for one still using it, and removeCallback() is versioned + // (running invocations use per-thread snapshots). The session trace is + // lost. + LOG(ERROR) + << "disableProfiler timed out draining in-flight global RecordFunction " + "callbacks; abandoning this trace. A callback thread is likely " + "wedged or was killed mid-callback."; + if (auto state_ptr = ProfilerStateBase::pop()) { + state_ptr->removeCallback(); + } + return std::make_unique(); + } + + auto state_ptr = ProfilerStateBase::pop(); + if (!state_ptr) { + LOG(WARNING) + << "disableProfiler called but no active profiling session found. " + << "This can happen if profiling was cancelled during warmup."; + return std::make_unique(); + } + const auto& config = state_ptr->config(); + TORCH_CHECK( + isValidDisableState(config.state), + "Can't disable Kineto profiler: config is not in a valid disable state"); + + state_ptr->removeCallback(); + + // Traces are converged via libkineto automatically for ondemand flow + if (config.global()) { + (void)std::static_pointer_cast(state_ptr) + ->finalizeTrace(); + return std::make_unique(); + } + + // Shared among NVTX, PRIVATEUSE1, KINETO, KINETO_GPU_FALLBACK, + // KINETO_PRIVATEUSE1_FALLBACK + std::unique_ptr result; + if (config.state == ProfilerState::NVTX || + config.state == ProfilerState::PRIVATEUSE1) { + result = std::make_unique(); + } + + if (isKinetoCompatibleState(config.state)) { + auto kineto_state_ptr = + std::static_pointer_cast(state_ptr); + auto trace = kineto_state_ptr->finalizeTrace(); + result = std::make_unique( + kineto_state_ptr->startTime, + std::move(kineto_state_ptr->kinetoEvents), + std::move(trace), + std::move(kineto_state_ptr->eventTree)); + } + + return result; +} +namespace tracer = torch::profiler::impl::python_tracer; +static std::unique_ptr memory_tracer; +void startMemoryProfile() { + if (memory_tracer == nullptr) { + memory_tracer = tracer::PythonMemoryTracerBase::make(); + } + memory_tracer->start(); +} + +void stopMemoryProfile() { + memory_tracer->stop(); +} + +void exportMemoryProfile(const std::string& filename) { + memory_tracer->export_memory_history(filename); +} + +KinetoEvent::KinetoEvent( + const std::shared_ptr& result, + const bool verbose) + : result_{result} { + TORCH_INTERNAL_ASSERT(result != nullptr); + + if (verbose) { + // Populate Python stack + auto parent = result_->parent_.lock(); + while (parent != nullptr) { + parent->visit_if_base( + [&](const auto&) { python_stack_.push_back(parent->name()); }); + parent = parent->parent_.lock(); + } + } + + result->visit_if_base>([&](const auto& op) { + auto arg_data = parseArgData(op.inputs_, op.concrete_inputs_); + shapes_ = std::move(arg_data.shapesForKinetoEvent); + structured_input_shapes_ = std::move(arg_data.shapes); + structured_input_strides_ = std::move(arg_data.strides); + dtypes_ = std::move(arg_data.dtypes); + concrete_inputs_ = std::move(arg_data.concreteInputs); + kwinputs_ = std::move(op.kwinputs_); + }); +} + +bool KinetoEvent::isPythonFunction() const { + bool out{false}; + result_->visit_if_base([&](const auto&) { out = true; }); + return out; +} + +int64_t KinetoEvent::pythonId() const { + int64_t out{-1}; + result_->visit_if_base( + [&](const auto& i) { out = static_cast(i.id_); }); + return out; +} + +int64_t KinetoEvent::pythonParentId() const { + int64_t out{-1}; + // Walk the python parent pointers up to find the next event of type + // PyExtraFieldsBase + result_->visit_if_base([&](const auto&) { + auto parent = result_->parent_.lock(); + while (parent) { + parent->visit_if_base( + [&](const auto& j) { out = static_cast(j.id_); }); + if (out >= 0) { + break; + } + parent = parent->parent_.lock(); + } + }); + return out; +} + +int64_t KinetoEvent::pythonModuleId() const { + int64_t out{-1}; + // Returns the module id for PyCall events (python function calls to + // nn.Module) + result_->visit(c10::overloaded( + [&](const ExtraFields& py_call) { + if (py_call.module_.has_value()) { + out = static_cast(py_call.module_->id_); + } + }, + [](const auto&) {})); + return out; +} + +bool KinetoEvent::hasShapes() const { + return !shapes_.empty(); +} + +const c10::ArrayRef> KinetoEvent::shapes() const { + return shapes_; +} + +const c10::ArrayRef KinetoEvent:: + structuredInputShapes() const { + return structured_input_shapes_; +} + +const c10::ArrayRef KinetoEvent:: + structuredInputStrides() const { + return structured_input_strides_; +} + +bool KinetoEvent::hasTypes() const { + return !dtypes_.empty(); +} + +const c10::ArrayRef KinetoEvent::dtypes() const { + return dtypes_; +} + +bool KinetoEvent::hasConcreteInputs() const { + return !concrete_inputs_.empty(); +} + +const c10::ArrayRef KinetoEvent::concreteInputs() const { + return concrete_inputs_; +} + +bool KinetoEvent::hasKwinputs() const { + return !kwinputs_.empty(); +} + +bool KinetoEvent::isHiddenEvent() const { + return result_ && result_->hidden_; +} + +const std::unordered_map KinetoEvent::kwinputs() + const { + return kwinputs_; +} + +const c10::ArrayRef KinetoEvent::stack() const { + auto get = [&](const auto& i) -> auto& { + return !i.jit_stack_.empty() ? i.jit_stack_ : python_stack_; + }; + + auto const& extra_fields = result_->extra_fields_; + if (auto p = std::get_if>(&extra_fields)) { + return get(*p); + } + if (auto p = std::get_if>(&extra_fields)) { + return get(*p); + } + return python_stack_; +} + +const c10::ArrayRef KinetoEvent::moduleHierarchy() const { + auto const& extra_fields = result_->extra_fields_; + if (auto p = std::get_if>(&extra_fields)) { + return p->jit_modules_; + } + if (auto p = std::get_if>(&extra_fields)) { + return p->jit_modules_; + } + return {}; +} + +uint64_t KinetoEvent::endNs() const { + return result_->endTimeNS(); +} + +uint64_t KinetoEvent::durationNs() const { + return (result_->endTimeNS() - result_->start_time_ns_); +} + +int64_t KinetoEvent::debugHandle() const { + return result_->visit(c10::overloaded( + [](const ExtraFields& i) { return i.debug_handle_; }, + [](const ExtraFields& i) { return i.debug_handle_; }, + [](const auto&) -> int64_t { return -1; })); +} + +int KinetoEvent::deviceIndex() const { + return result_->visit(c10::overloaded( + [](const ExtraFields& i) { + return static_cast(i.device_index_); + }, + [](const ExtraFields& i) { + return static_cast(i.device_index_); + }, + [&](const auto&) { + return static_cast(result_->kineto_info_.device); + })); +} + +bool KinetoEvent::hasStack() const { + return !stack().empty(); +} + +int64_t KinetoEvent::cudaElapsedUs() const { + auto cuda_event_start = fallbackStart(); + auto cuda_event_end = fallbackEnd(); + if (!cuda_event_start || !cuda_event_end) { + return -1; + } + try { + return (int64_t)torch::profiler::impl::cudaStubs()->elapsed( + &cuda_event_start, &cuda_event_end); + } catch (std::exception& e) { + LOG(WARNING) << "Failed to measure time between two CUDA events. " + << e.what(); + } + return -1; +} + +int64_t KinetoEvent::privateuse1ElapsedUs() const { + auto privateuse1_event_start = fallbackStart(); + auto privateuse1_event_end = fallbackEnd(); + if (!privateuse1_event_start || !privateuse1_event_end) { + return -1; + } + return (int64_t)torch::profiler::impl::privateuse1Stubs()->elapsed( + &privateuse1_event_start, &privateuse1_event_end); +} + +void KinetoEvent::getPerfEventCounters(std::vector& in) const { + return result_->visit(c10::overloaded( + [&in](const ExtraFields& e) -> void { + const size_t n = e.perf_event_counters_->size(); + // should be rare + if (in.size() < n) { + in.resize(n, 0); + } + for (size_t i = 0; i < n; ++i) { + in[i] = (*e.perf_event_counters_)[i]; + } + }, + [](const auto&) -> void { return; })); +} + +std::string KinetoEvent::metadataJson() const { + return result_->visit(c10::overloaded( + [](const ExtraFields& op) -> std::string { + return op.metadata_json_; + }, + [](const ExtraFields& op) -> std::string { + return op.metadata_json_; + }, + [](const auto&) -> std::string { return std::string(""); })); +} + +int64_t KinetoEvent::externalId() const { + // Mirrors libkineto::ChromeTraceLogger::handleActivity() "External id" logic. + // libkineto::ChromeTraceLogger checks op.linkedActivity() != nullptr; here we + // check linkedCorrelationId() > 0, which is equivalent because PyTorch + // correlation IDs are monotonically increasing from 1 (a valid linked + // activity always has a non-zero correlation ID). + uint64_t linked = linkedCorrelationId(); + if (linked > 0) { + return static_cast(linked); + } + + // Orphaned GPU activities (no linked CPU op) in these types should not get + // an External id, to avoid incorrect cross-linking in trace viewers. + auto type = static_cast(activityType()); + if (type != libkineto::ActivityType::GPU_MEMCPY && + type != libkineto::ActivityType::GPU_MEMSET && + type != libkineto::ActivityType::CONCURRENT_KERNEL && + type != libkineto::ActivityType::CUDA_RUNTIME && + type != libkineto::ActivityType::CUDA_DRIVER && + type != libkineto::ActivityType::PRIVATEUSE1_RUNTIME && + type != libkineto::ActivityType::PRIVATEUSE1_DRIVER) { + return static_cast(result_->visit(c10::overloaded( + [](const ExtraFields& e) -> uint64_t { + return e.correlation_id_; + }, + [](const ExtraFields& e) -> uint64_t { + return e.correlation_id_; + }, + [](const auto&) -> uint64_t { return 0; }))); + } + + return 0; +} + +#define FORWARD_FROM_RESULT(method_name, result_expr) \ + decltype(std::declval().method_name()) \ + KinetoEvent::method_name() const { \ + return static_cast().method_name())>( \ + result_->result_expr); \ + } + +FORWARD_FROM_RESULT(startThreadId, start_tid_) +FORWARD_FROM_RESULT(endThreadId, endTID()) +FORWARD_FROM_RESULT(activityType, kinetoType()) +FORWARD_FROM_RESULT(name, name()) +FORWARD_FROM_RESULT(overload_name, overload_name()) +FORWARD_FROM_RESULT(deviceType, deviceType()) +FORWARD_FROM_RESULT(startNs, start_time_ns_) +FORWARD_FROM_RESULT(correlationId, correlationID()) +FORWARD_FROM_RESULT(deviceResourceId, kineto_info_.resource) +#undef FORWARD_FROM_RESULT + +// Most of the fields in `KinetoEvent` only make sense for a single event type. +// (Generally TorchOp.) For all other types they simply return the default +// value. This macro provides a succinct way of expressing this behavior. +#define TYPED_ATTR_WITH_DEFAULT( \ + event_type, method_name, expression, default_value) \ + decltype(std::declval().method_name()) \ + KinetoEvent::method_name() const { \ + using out_t = decltype(std::declval().method_name()); \ + return result_->visit(c10::overloaded( \ + [](const ExtraFields& e) -> out_t { \ + return expression; \ + }, \ + [](const auto&) -> out_t { return default_value; })); \ + } + +#define TYPED_ATTR(event_type, method_name, expression) \ + TYPED_ATTR_WITH_DEFAULT(event_type, method_name, expression, {}) + +TYPED_ATTR_WITH_DEFAULT(TorchOp, sequenceNr, e.sequence_number_, -1) +TYPED_ATTR(TorchOp, fwdThreadId, e.sequence_number_ >= 0 ? e.forward_tid_ : 0) +TYPED_ATTR(TorchOp, scope, static_cast(e.scope_)) +TYPED_ATTR(TorchOp, hasModuleHierarchy, !e.jit_modules_.empty()) +TYPED_ATTR(TorchOp, isAsync, e.is_async_) + +extra_meta_t KinetoEvent::extraMeta() const { + extra_meta_t out; + result_->visit(c10::overloaded( + [&](const ExtraFields& e) { out = e.extra_meta_; }, + [&](const ExtraFields& e) { out = e.extra_meta_; }, + [](const auto&) {})); + return out; +} + +TYPED_ATTR(TorchOp, fallbackStart, e.device_fallback_.device_event_start_) +TYPED_ATTR(TorchOp, fallbackEnd, e.device_fallback_.device_event_end_) +TYPED_ATTR( + TorchOp, + flops, + !e.extra_args_.empty() + ? torch::profiler::impl::computeFlops(e.name_, e.extra_args_) + : 0) +TYPED_ATTR(Backend, backend, e.backend_) +TYPED_ATTR(Allocation, nBytes, e.alloc_size_) +TYPED_ATTR(Kineto, linkedCorrelationId, [&]() { + const auto linked = e.linked_activity_.lock(); + return linked ? linked->correlationID() : 0; +}()) +#undef TYPED_ATTR +#undef TYPED_ATTR_WITH_DEFAULT + +// Flow fields exist on both TorchOp and Kineto event types. +uint32_t KinetoEvent::flowId() const { + return result_->visit(c10::overloaded( + [](const ExtraFields& e) { return e.flow.id; }, + [](const ExtraFields& e) { return e.flow.id; }, + [](const auto&) -> uint32_t { return 0; })); +} +uint32_t KinetoEvent::flowType() const { + return result_->visit(c10::overloaded( + [](const ExtraFields& e) { return e.flow.type; }, + [](const ExtraFields& e) { return e.flow.type; }, + [](const auto&) -> uint32_t { return 0; })); +} +bool KinetoEvent::flowStart() const { + return result_->visit(c10::overloaded( + [](const ExtraFields& e) { + return static_cast(e.flow.start); + }, + [](const ExtraFields& e) { + return static_cast(e.flow.start); + }, + [](const auto&) { return false; })); +} + +ProfilerResult::ProfilerResult( + uint64_t start_time, + std::vector events, + std::unique_ptr&& + trace, + std::vector&& event_tree) + : trace_start_ns_(start_time), + events_(std::move(events)), + trace_(std::move(trace)), + event_tree_(std::move(event_tree)) {} +ProfilerResult::ProfilerResult() = default; +ProfilerResult::~ProfilerResult() = default; + +void ProfilerResult::save(const std::string& path) { + trace_->save(path); +} + +#ifdef USE_KINETO +const std::vector* ProfilerResult:: + traceActivities() { + if (trace_) { + return trace_->get()->activities(); + } + return nullptr; +} +#endif + +} // namespace autograd::profiler + +namespace profiler::impl { +void _reportVulkanEventToProfiler(vulkan_id_t id) { + ::torch::autograd::profiler::KinetoThreadLocalState* state_ptr = + ::torch::autograd::profiler::KinetoThreadLocalState::getTLS(); + if (state_ptr) { + state_ptr->reportVulkanEventToProfiler(id); + } +} +} // namespace profiler::impl + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/profiler_kineto.h b/western_ai/usa/pytorch/torch/csrc/autograd/profiler_kineto.h new file mode 100644 index 0000000000000000000000000000000000000000..4de48c1e224c01dacee4d56761f165731395ada9 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/profiler_kineto.h @@ -0,0 +1,250 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace libkineto { +struct ITraceActivity; +} + +namespace torch { + +namespace profiler::impl { +struct Result; +namespace kineto { +struct ActivityTraceWrapper; +} // namespace kineto +} // namespace profiler::impl + +namespace autograd::profiler { +using experimental_event_t = std::shared_ptr; +using extra_meta_t = std::unordered_map; + +struct TORCH_API KinetoEvent { + KinetoEvent( + const std::shared_ptr& /*result*/, + const bool verbose); + + uint64_t startThreadId() const; + uint64_t endThreadId() const; + uint8_t activityType() const; + uint64_t fwdThreadId() const; + bool hasShapes() const; + const c10::ArrayRef> shapes() const; + bool hasTypes() const; + const c10::ArrayRef dtypes() const; + bool hasConcreteInputs() const; + const c10::ArrayRef concreteInputs() const; + bool hasKwinputs() const; + bool isHiddenEvent() const; + const std::unordered_map kwinputs() const; + uint64_t flops() const; + int64_t sequenceNr() const; + bool hasStack() const; + const c10::ArrayRef stack() const; + uint8_t scope() const; + bool hasModuleHierarchy() const; + const c10::ArrayRef moduleHierarchy() const; + int64_t debugHandle() const; + std::string name() const; + std::string overload_name() const; + c10::DeviceType deviceType() const; + int deviceIndex() const; + int64_t nBytes() const; + uint64_t startNs() const; + uint64_t endNs() const; + uint64_t durationNs() const; + bool isAsync() const; + uint64_t correlationId() const; + uint64_t linkedCorrelationId() const; + uint32_t flowId() const; + uint32_t flowType() const; + bool flowStart() const; + int64_t externalId() const; + int64_t deviceResourceId() const; + std::string backend() const; + bool isPythonFunction() const; + int64_t cudaElapsedUs() const; + int64_t privateuse1ElapsedUs() const; + void getPerfEventCounters(torch::profiler::perf_counters_t& /*in*/) const; + extra_meta_t extraMeta() const; + std::string metadataJson() const; + + const c10::ArrayRef structuredInputShapes() + const; + const c10::ArrayRef structuredInputStrides() + const; + int64_t pythonId() const; + int64_t pythonParentId() const; + int64_t pythonModuleId() const; + + private: + torch::profiler::impl::ProfilerVoidEventStub fallbackStart() const; + torch::profiler::impl::ProfilerVoidEventStub fallbackEnd() const; + + std::shared_ptr result_; + std::vector python_stack_; + + // Copy fields from result so we can return ArrayRefs. + std::vector> shapes_; + std::vector dtypes_; + std::vector concrete_inputs_; + std::unordered_map kwinputs_; + std::vector structured_input_shapes_; + std::vector structured_input_strides_; +}; + +// Consolidating events returned directly from Kineto +// with events manually created by us (e.g. start/stop marks, +// memory allocation events) +struct TORCH_API ProfilerResult { + ProfilerResult(); + ProfilerResult( + uint64_t start_time, + std::vector events, + std::unique_ptr&& + trace, + std::vector&& event_tree); + ~ProfilerResult(); + + uint64_t trace_start_ns() const { + return trace_start_ns_; + } + + const std::vector& events() const { + return events_; + } + + const std::vector& event_tree() const { + return event_tree_; + } + + void save(const std::string& path); +#ifdef USE_KINETO + const std::vector* traceActivities(); +#endif + + private: + uint64_t trace_start_ns_ = 0; + std::vector events_; + std::unique_ptr trace_; + std::vector event_tree_; +}; + +/* + * This API is used by backends to record latency of events that + * happened in the backend but were not visible to pytorch runtime. + * For example, if part of the model is lowered to a dsp backend, then + * the execution of that part of the model is delegated to the backend. + * When backend finishes execution it has an option to provide profiling + * information (latency only at the moment) corresponding to different operators + * that were executed in the backend. + * When such events are recorded by backend using this API, the event + * records will be collected by active kineto profiler. If no kineto profiler + * is active then the event is ignored. + * This provides us with a way to generate all the profiling information + * for a model regardless of where model (or part of it) executed. + * @param start_time_us: start time in us of the event + * @param end_time_us: end time in us of the event + * @param debug_handle: debug handle to correlate this event/op with + * model level module/source information + * @param scope: scope of the event, e.g. LITE_INTERPRETER, RECORD_FN etc. + * @param event_name: name of the event, e.g. op name + * @param backend_name: name of the backend where the event took place. + */ +TORCH_API void reportBackendEventToActiveKinetoProfiler( + const int64_t start_time_us, + const int64_t end_time_us, + const int64_t debug_handle, + const at::RecordScope scope, + const std::string& event_name, + const std::string& backend_name); + +TORCH_API void enableProfiler( + const torch::profiler::impl::ProfilerConfig& config, + const std::set& activities, + const std::unordered_set& scopes = {}); + +/* + * Same as enableProfiler but with callback to do post-processing of + * KinetoEvents. + * enableProfilerWithEventPostProcess enables profiler to capture + * specified activities, with specified RecordFunction scope, if any. + * Additionally, it takes a functor that does in-place post processing of + * events, e.g. populate stack trace or module hierarchy information lazily + * using debug_handle. + * Example usage is with lite interpreter that has recording scope of + * LITE_INTERPRETER. In this case lite interpreter runtime, records debug + * handles in RecordFunction, along with other information. Debug handles are + * eventually passed down to KinetoEvent and recorded as part of the event. + * KinetoEdgeCPUProfiler, in torch/csrc/jit/mobile/profiler_edge.cpp, enables + * profiler using post-processing callback, via + * enableProfilerWithEventPostProcess, that takes these debug handles and + * generates stack trace and module hierarchy information, once profiling is + * done. + */ +using post_process_t = std::function&, + /*jit_modules */ std::vector&)>; +TORCH_API void enableProfilerWithEventPostProcess( + const torch::profiler::impl::ProfilerConfig& config, + const std::set& activities, + post_process_t&& cb, + const std::unordered_set& scopes = {}); + +TORCH_API std::unique_ptr disableProfiler(); + +using ActivityFilter = std::unordered_map< + torch::profiler::impl::ActivityType, + std::unordered_set>; +TORCH_API void prepareProfiler( + const torch::profiler::impl::ProfilerConfig& config, + const std::set& activities, + const ActivityFilter& activity_filter = {}); + +TORCH_API void toggleCollectionDynamic( + const bool enable, + const std::set& activities); + +TORCH_API void startMemoryProfile(); +TORCH_API void stopMemoryProfile(); +TORCH_API void exportMemoryProfile(const std::string& path); + +/** + * When a C++ thread really has no control over how the profiler was enabled, + * for example, by some unreachable Python code, it can call these functions + * to test/join/unjoin itself into the collection set of a profiler, if any. + * Without calling these functions, the symptom may be "not seeing GPU events + * from some child C++ threads". This is an example on how to use them, + * + * using namespace torch::autograd::profiler; + * bool enabled = isProfilerEnabledInMainThread(); + * if (enabled != saved_enabled_state) { + * if (enabled) { + * enableProfilerInChildThread(); + * } else { + * disableProfilerInChildThread(); + * } + * saved_enabled_state = enabled; + * } + */ +TORCH_API bool isProfilerEnabledInMainThread(); +TORCH_API void enableProfilerInChildThread(); +TORCH_API void disableProfilerInChildThread(); + +} // namespace autograd::profiler + +namespace profiler::impl { + +// Experimental. +TORCH_API void _reportVulkanEventToProfiler(vulkan_id_t id); + +} // namespace profiler::impl + +} // namespace torch diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/profiler_legacy.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/profiler_legacy.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cb3abf601d6f76e6a4109a0dbef3129e94b699cd --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/profiler_legacy.cpp @@ -0,0 +1,678 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +namespace torch::autograd::profiler { + +// We decompose the profiler logic into the following components: +// +// ThreadLocalDebugInfo: +// +// ThreadLocalDebugInfo is a thread local mapping from slots into +// the debug information structs. +// ThreadLocalDebugInfo is automatically propagated across thread +// boundaries, including the cases of: +// - launching async jobs with at::launch +// - executing JIT continuations +// - moving from the forward threads into autograd (backward) threads +// +// Entries in ThreadLocalDebugInfo are managed by DebugInfoGuard +// which can be used to add or overwrite an entry in the thread local +// mapping. A corresponding entry is removed when the guard is destroyed, +// potentially revealing the previously set value for the same slot. +// +// For the async tasks, slots previously set in the main thread before +// launching of an async task are shared and visible in the async task. +// +// On the other hand, any adding or overwriting of the mapping by the +// async task is not visible to the main thread and any modification +// (including removal of the entries) in the main thread is not visible +// to the async task if it happens after launching the task. +// +// We use ThreadLocalDebugInfo (slot PROFILER_STATE) to store profiler config, +// as well as a list of events that happen during profiling. +// An instance of ThreadLocalDebugInfo is created each time we enter +// profiler (i.e. enter profiling context manager/call enableConfig) and +// uniquely identifies a profiling run. +// +// We automatically propagate ThreadLocalDebugInfo into async tasks, +// as well as across JIT continuations and autograd thread, so all +// the operations that happen between profiling start and end +// (not necessarily within the same thread) are recorded. +// Unless the profiling slot is overwritten as in the case of nested +// profiling ranges (in this case events for the subrange are handled +// by the nested profiler) +// +// When we exit a profiling range (either by exiting profiling context +// manager or by calling disableProfiler), we remove the previously set +// profiling entry for the given thread local mapping, and consolidate +// events in the profiling result +// +// +// ThreadLocalState: +// +// ThreadLocalState takes a 'snapshot' of thread local variables +// using provided getters. It is used together with ThreadLocalStateGuard +// to transfer the snapshot across thread boundary and set the thread local +// values as in the parent task. +// +// Profiler uses ThreadLocalState to propagate profiler's thread local state. +// ThreadLocalState also automatically propagates profiler callbacks. +// +// +// at::RecordFunction and observers +// +// Profiler uses observers mechanism to add a pair of thread local callbacks +// that are executed on a number of predetermined ranges, including: +// - c10/ATen ops +// - TorchScript functions/methods +// - user defined named ranges (see `record_function` python context manager) +// +// Profiler setups a pair of callbacks that record profiling events and save +// them into the thread local profiler struct (ThreadLocalDebugInfo, +// PROFILER_STATE slot) +// +// +// Thus, the overall logic is: +// +// enableProfiler: +// - checks that profiler is not enabled (otherwise throws) +// - pushes new ThreadLocalDebugInfo (slot PROFILER_STATE) as the profiler +// config for the current thread +// - pushes profiling callbacks for the current thread +// +// disableProfiler: +// - pops PROFILER_STATE slot from the current ThreadLocalDebugInfo and +// consolidates events +// - removes profiling callbacks +// +// ThreadLocalState: +// - propagates ThreadLocalDebugInfo across threads +// - propagates profiler callbacks across threads +// +// Profiler callbacks: +// - get the current profiling state (PROFILER slot in ThreadLocalDebugInfo) +// - save profiling events into the profiling state +// + +namespace { +using torch::profiler::impl::ActiveProfilerType; +using torch::profiler::impl::ProfilerStateBase; + +struct ProfilerLegacyThreadLocalState : public ProfilerStateBase { + explicit ProfilerLegacyThreadLocalState( + const torch::profiler::impl::ProfilerConfig& config) + : ProfilerStateBase(config), remoteProfiledEvents_{std::nullopt} {} + ~ProfilerLegacyThreadLocalState() override = default; + + static ProfilerLegacyThreadLocalState* getTLS() { + ProfilerStateBase* tls = ProfilerStateBase::getTLS(); + TORCH_INTERNAL_ASSERT_DEBUG_ONLY( + tls == nullptr || tls->profilerType() == ActiveProfilerType::LEGACY); + return static_cast(tls); + } + + thread_event_lists consolidate(); + + void mark(std::string name, bool include_cuda = true); + + void setOrAddRemoteProfiledEvents( + std::vector&& remoteProfiledEvents); + + void pushRange( + const at::RecordFunction& fn, + const bool record_cuda, + std::vector>&& shapes = {}); + + void popRange(const at::RecordFunction& fn, const bool record_cuda); + + void reportMemoryUsage( + void* /* unused */, + int64_t alloc_size, + size_t /* total_allocated, unused for legacy */, + size_t /* total_reserved, unused for legacy */, + c10::Device device) override; + + ActiveProfilerType profilerType() override { + return ActiveProfilerType::LEGACY; + } + + void leakHandle() { + handle_ = 0; + } + + protected: + RangeEventList& getEventList( + std::optional thread_id = std::nullopt); + + std::mutex state_mutex_; + std::unordered_map> + event_lists_map_; + + std::optional>> remoteProfiledEvents_; +}; + +thread_event_lists ProfilerLegacyThreadLocalState::consolidate() { + std::lock_guard g(state_mutex_); + thread_event_lists result; + for (auto& kv : event_lists_map_) { + auto& list = kv.second; + result.emplace_back(list->consolidate()); + } + // Consolidate remote events if applicable as well. + if (remoteProfiledEvents_) { + result.insert( + result.end(), + std::make_move_iterator(remoteProfiledEvents_->begin()), + std::make_move_iterator(remoteProfiledEvents_->end())); + } + return result; +} + +void ProfilerLegacyThreadLocalState::mark(std::string name, bool include_cuda) { + if (config_.disabled()) { + return; + } + if (config_.state == torch::profiler::impl::ProfilerState::NVTX) { + torch::profiler::impl::cudaStubs()->mark(name.c_str()); + } else { + LegacyEvent evt( + EventKind::Mark, + at::StringView(std::move(name)), + at::RecordFunction::currentThreadId(), + include_cuda && + config_.state == torch::profiler::impl::ProfilerState::CUDA); + evt.setNodeId(at::RecordFunction::getDefaultNodeId()); + getEventList().record(std::move(evt)); + } +} + +void ProfilerLegacyThreadLocalState::setOrAddRemoteProfiledEvents( + std::vector&& remoteProfiledEvents) { + // Lock to serialize access from multiple callback threads. + std::lock_guard guard(state_mutex_); + if (remoteProfiledEvents_) { + (*remoteProfiledEvents_).emplace_back(remoteProfiledEvents); + } else { + remoteProfiledEvents_ = {std::move(remoteProfiledEvents)}; + } +} + +void ProfilerLegacyThreadLocalState::pushRange( + const at::RecordFunction& fn, + const bool record_cuda, + std::vector>&& shapes) { + if (config_.disabled()) { + return; + } + if (config_.state == torch::profiler::impl::ProfilerState::NVTX) { + torch::profiler::impl::cudaStubs()->rangePush( + torch::profiler::impl::getNvtxStr(fn.name(), fn.seqNr(), shapes) + .c_str()); + } else { + LegacyEvent evt( + EventKind::PushRange, + at::StringView(std::string(fn.name())), + at::RecordFunction::currentThreadId(), + record_cuda, + fn.handle(), + std::move(shapes), + at::RecordFunction::getDefaultNodeId(), + fn.isAsync()); + evt.setSequenceNr(fn.seqNr()); + evt.setFwdThreadId(fn.forwardThreadId()); + evt.setScope((uint8_t)fn.scope()); + if (config_.with_flops) { + evt.setExtraArgs(torch::profiler::impl::saveExtraArgs(fn)); + evt.setFlops(torch::profiler::impl::computeFlops( + std::string(fn.name()), evt.extraArgs())); + } + +// TODO: will unify the two macros BUILD_LITE_INTERPRETER and C10_MOBILE soon. +#if !defined BUILD_LITE_INTERPRETER && !defined C10_MOBILE + // backward nodes source range corresponds to the forward node + // TODO: consider using C++ stack trace + if (config_.with_stack && + fn.scope() != at::RecordScope::BACKWARD_FUNCTION) { + auto cs = + torch::profiler::impl::prepareCallstack(jit::currentCallstack()); + if (cs.empty()) { + cs = torch::profiler::impl::prepareCallstack( + jit::tracer::pythonCallstack()); + } + evt.setStack(callstackStr(cs)); + } +#endif + getEventList().record(std::move(evt)); + } +} + +void ProfilerLegacyThreadLocalState::popRange( + const at::RecordFunction& fn, + const bool record_cuda) { + if (config_.disabled()) { + return; + } + if (config_.state == torch::profiler::impl::ProfilerState::NVTX) { + torch::profiler::impl::cudaStubs()->rangePop(); + } else { + // In some cases RecordFunction (and popRange) may be + // called on a different thread than pushRange + // As a convention, we put the async pop on the original + // thread and save current thread id in pop event + LegacyEvent evt( + EventKind::PopRange, + at::StringView(""), + at::RecordFunction::currentThreadId(), + record_cuda, + fn.handle()); + evt.setNodeId(at::RecordFunction::getDefaultNodeId()); + getEventList(fn.threadId()).record(std::move(evt)); + } +} + +void ProfilerLegacyThreadLocalState::reportMemoryUsage( + void* /* unused */, + int64_t alloc_size, + size_t /* total_allocated, unused for legacy */, + size_t /* total_reserved, unused for legacy */, + c10::Device device) { + if (config_.profile_memory && !config_.disabled()) { + uint64_t thread_id = at::RecordFunction::currentThreadId(); + LegacyEvent evt( + EventKind::MemoryAlloc, + at::StringView(""), + thread_id, + config_.state == torch::profiler::impl::ProfilerState::CUDA); + evt.updateMemoryStats(alloc_size, device); + getEventList(thread_id).record(std::move(evt)); + } +} + +RangeEventList& ProfilerLegacyThreadLocalState::getEventList( + std::optional thread_id) { + if (!thread_id.has_value()) { + thread_id = at::RecordFunction::currentThreadId(); + } + RangeEventList* list_ptr = nullptr; + std::lock_guard guard(state_mutex_); + auto it = event_lists_map_.find(thread_id.value()); + if (it != event_lists_map_.end()) { + list_ptr = it->second.get(); + } else { + auto event_list = std::make_shared(); + event_lists_map_[thread_id.value()] = event_list; + list_ptr = event_list.get(); + } + return *list_ptr; +} + +enum EventIValueIdx { + KIND = 0, + NAME, + THREAD_ID, + HANDLE, + NODE_ID, + CPU_MEM_USAGE, + CPU_NS, + CUDA_RECORDED, + CUDA_MEM_USAGE, + CUDA_DEVICE, + CUDA_US, + SHAPES, + NUM_EVENT_IVALUE_IDX // must be last in list +}; + +const std::unordered_set disable_cuda_profiling = { + "aten::view", + "aten::t", + "aten::transpose", + "aten::stride", + "aten::empty", + "aten::empty_like", + "aten::empty_strided", + "aten::as_strided", + "aten::expand", + "aten::resize_", + "aten::squeeze", + "aten::unsqueeze", + "aten::slice", + "aten::_unsafe_view", + "aten::size"}; + +void pushProfilingCallbacksLegacy() { + auto registration_state_ptr = ProfilerLegacyThreadLocalState::getTLS(); + TORCH_INTERNAL_ASSERT(registration_state_ptr, "Expected profiler state set"); + auto handle = at::addThreadLocalCallback( + at::RecordFunctionCallback( + [](const at::RecordFunction& fn) + -> std::unique_ptr { + auto state_ptr = ProfilerLegacyThreadLocalState::getTLS(); + if (!state_ptr || state_ptr->config().disabled()) { + return nullptr; + } + bool record_cuda = state_ptr->config().state == + torch::profiler::impl::ProfilerState::CUDA; + if (record_cuda && + disable_cuda_profiling.find(fn.name()) != + disable_cuda_profiling.end()) { + record_cuda = false; + } + + if (state_ptr->config().report_input_shapes) { + auto sizes = torch::profiler::impl::inputSizes(fn); + state_ptr->pushRange(fn, record_cuda, std::move(sizes)); + } else { + state_ptr->pushRange(fn, record_cuda); + } + + return nullptr; + }, + [](const at::RecordFunction& fn, at::ObserverContext*) { + auto state_ptr = ProfilerLegacyThreadLocalState::getTLS(); + if (!state_ptr || state_ptr->config().disabled()) { + return; + } + bool record_cuda = state_ptr->config().state == + torch::profiler::impl::ProfilerState::CUDA; + if (record_cuda && + disable_cuda_profiling.find(fn.name()) != + disable_cuda_profiling.end()) { + record_cuda = false; + } + state_ptr->popRange(fn, record_cuda); + }) + .needsInputs(registration_state_ptr->config().report_input_shapes) + .needsIds(true)); + registration_state_ptr->setCallbackHandle(handle); +} + +} // namespace + +void enableProfilerLegacy( + const torch::profiler::impl::ProfilerConfig& new_config) { + TORCH_CHECK( + new_config.state != torch::profiler::impl::ProfilerState::NVTX || + torch::profiler::impl::cudaStubs()->enabled(), + "Can't use NVTX profiler - PyTorch was compiled without CUDA"); + + TORCH_CHECK(new_config.state != torch::profiler::impl::ProfilerState::KINETO); + + auto state_ptr = ProfilerLegacyThreadLocalState::getTLS(); + TORCH_CHECK(!state_ptr, "Profiler is already enabled on this thread"); + auto state = std::make_shared(new_config); + c10::ThreadLocalDebugInfo::_push(c10::DebugInfoKind::PROFILER_STATE, state); + + pushProfilingCallbacksLegacy(); + + state->mark("__start_profile", false); +} + +thread_event_lists disableProfilerLegacy( + std::optional profilerDisableOptions) { + auto cleanupTLSState = + profilerDisableOptions ? profilerDisableOptions->cleanupTLSState : true; + auto consolidate = + profilerDisableOptions ? profilerDisableOptions->consolidate : true; + // all the DebugInfoBase objects are scope based and supposed to use + // DebugInfoGuard + std::shared_ptr state; + if (cleanupTLSState) { + state = c10::ThreadLocalDebugInfo::_pop(c10::DebugInfoKind::PROFILER_STATE); + } else { + state = + c10::ThreadLocalDebugInfo::_peek(c10::DebugInfoKind::PROFILER_STATE); + } + + auto state_ptr = static_cast(state.get()); + TORCH_CHECK( + state_ptr && !state_ptr->config().disabled(), + "Can't disable profiler when it's not running"); + + cleanupTLSState ? state_ptr->removeCallback() : state_ptr->leakHandle(); + if (!consolidate || + state_ptr->config().state == torch::profiler::impl::ProfilerState::NVTX) { + return thread_event_lists(); + } + + state_ptr->mark("__stop_profile", false); + // Note that this will erase the underlying events. + return state_ptr->consolidate(); +} + +void addEventList(std::vector&& profiledEvents) { + auto state_ptr = ProfilerLegacyThreadLocalState::getTLS(); + TORCH_CHECK(state_ptr, "Profiler must be enabled."); + state_ptr->setOrAddRemoteProfiledEvents(std::move(profiledEvents)); +} + +void LegacyEvent::record(bool record_cuda) { + if (record_cuda) { + torch::profiler::impl::cudaStubs()->record(&device_, &cuda_event, &cpu_ns_); + return; + } + cpu_ns_ = c10::getTime(); +} + +/* static */ LegacyEvent LegacyEvent::fromIValue( + const at::IValue& eventIValue) { + TORCH_INTERNAL_ASSERT( + eventIValue.isList(), + "Expected IValue to contain type c10::impl::GenericList"); + auto ivalues = eventIValue.toList(); + TORCH_INTERNAL_ASSERT( + ivalues.size() >= NUM_EVENT_IVALUE_IDX, + "Expected at least ", + NUM_EVENT_IVALUE_IDX, + " elements to reconstruct LegacyEvent."); + + // Reconstruct input shapes from ivalues. + const auto& shapeListIValue = ivalues.get(EventIValueIdx::SHAPES); + TORCH_INTERNAL_ASSERT( + shapeListIValue.isList(), + "Expected profiler shapes IValue to contain type c10::impl::GenericList."); + + auto shapeList = shapeListIValue.toList(); + std::vector> shapes; + shapes.reserve(shapeList.size()); + for (const auto i : c10::irange(shapeList.size())) { + std::vector s; + const auto& shapeIValue = shapeList.get(i); + TORCH_INTERNAL_ASSERT( + shapeIValue.isList(), + "Expected each profiler shape element to contain shapes of type c10::impl::GenericList.") + auto curShapesList = shapeIValue.toList(); + s.reserve(curShapesList.size()); + for (const auto j : c10::irange(curShapesList.size())) { + s.emplace_back(curShapesList.get(j).toInt()); + } + shapes.emplace_back(s); + } + + LegacyEvent evt( + static_cast( + ivalues.get(EventIValueIdx::KIND).toInt()), // EventKind + at::StringView(ivalues.get(EventIValueIdx::NAME).toStringRef()), // name + ivalues.get(EventIValueIdx::THREAD_ID).toInt(), // thread_id + static_cast( + ivalues.get(EventIValueIdx::HANDLE).toDouble()), // handle + std::move(shapes), // input shapes + ivalues.get(EventIValueIdx::NODE_ID).toInt(), // node id + true, // is remote + ivalues.get(EventIValueIdx::CPU_MEM_USAGE).toInt(), // cpu_mem_usage + ivalues.get(EventIValueIdx::CPU_NS).toInt(), // cpu_ns + ivalues.get(EventIValueIdx::CUDA_RECORDED).toBool(), // was cuda recorded + ivalues.get(EventIValueIdx::CUDA_MEM_USAGE).toInt(), // cuda memory usage + c10::DeviceIndex( + ivalues.get(EventIValueIdx::CUDA_DEVICE).toInt()), // device + static_cast( + ivalues.get(EventIValueIdx::CUDA_US).toInt()) // cuda_us + ); + return evt; +} + +at::IValue LegacyEvent::toIValue() const { + c10::impl::GenericList eventIValueList(at::AnyType::get()); + eventIValueList.reserve(NUM_EVENT_IVALUE_IDX); + eventIValueList.emplace_back(static_cast(kind_)); + eventIValueList.emplace_back(std::string(name_.str())); + eventIValueList.emplace_back(static_cast(thread_id_)); + eventIValueList.emplace_back(static_cast(handle_)); + eventIValueList.emplace_back(node_id_); + eventIValueList.emplace_back(cpu_memory_usage_); + eventIValueList.emplace_back(cpu_ns_); + // CUDA event information + bool cuda_profiling_enabled = hasCuda(); + eventIValueList.emplace_back(cuda_profiling_enabled); + eventIValueList.emplace_back(static_cast(cuda_memory_usage_)); + eventIValueList.emplace_back(device_); + eventIValueList.emplace_back(cuda_us_); + // Shapes + c10::impl::GenericList shapesList = + c10::impl::GenericList(at::ListType::create(at::IntType::get())); + shapesList.reserve(shapes_.size()); + for (const auto& shape : shapes_) { + c10::impl::GenericList s = c10::impl::GenericList(at::IntType::get()); + s.reserve(shape.size()); + for (const auto& k : shape) { + s.emplace_back(k); + } + shapesList.emplace_back(s); + } + eventIValueList.emplace_back(shapesList); + return at::IValue(eventIValueList); +} + +double LegacyEvent::cudaElapsedUs(const LegacyEvent& e) const { + TORCH_CHECK(e.hasCuda() && hasCuda(), "Events were not recorded for CUDA"); + TORCH_CHECK( + e.device() == device(), + c10::str( + "Events are not on the same device: ", e.device(), " vs ", device())); + if (isRemote() && e.isRemote()) { + // validate that cuda_us_ has been set properly. + TORCH_INTERNAL_ASSERT(cuda_us_ >= 0 && e.cuda_us_ >= 0); + return static_cast(e.cuda_us_ - cuda_us_); + } + return torch::profiler::impl::cudaStubs()->elapsed( + &cuda_event, &e.cuda_event); +} + +static const at::jit::CodeTemplate event_template(R"( +{ + "name": "${name}", + "ph": "X", + "ts": ${ts}, + "dur": ${dur}, + "tid": ${tid}, + "pid": "CPU Functions", + "args": {} +})"); + +void writeProfilerEventsToStream( + std::ostream& out, + const std::vector& events) { + TORCH_CHECK(out, "Could not open file"); + LegacyEvent* profiler_start = nullptr; + for (LegacyEvent* e : events) { + if (0 == strcmp(e->name(), "__start_profile")) { + profiler_start = e; + break; + } + } + TORCH_CHECK(profiler_start, "Could not find __start_profile mark"); + + struct PairHash { + size_t operator()( + std::pair p) const noexcept { + return std::hash()(p.first) ^ + std::hash()(p.second); + } + }; + std::unordered_map< + std::pair, + LegacyEvent*, + PairHash> + events_map; + out << "[\n"; + bool first = true; + for (LegacyEvent* evt : events) { + if (evt->kindStr() == "push") { + events_map[std::make_pair(evt->handle(), evt->nodeId())] = evt; + } else if (evt->kindStr() == "pop") { + if (!first) { + out << ",\n"; + } + first = false; + auto it = events_map.find(std::make_pair(evt->handle(), evt->nodeId())); + TORCH_CHECK(it != events_map.end(), "Unmatched pop event"); + LegacyEvent* evt_start = it->second; + events_map.erase(it); + + at::jit::TemplateEnv env; + env.s("name", evt_start->name()); + env.d("ts", profiler_start->cpuElapsedUs(*evt_start)); + env.d("dur", evt_start->cpuElapsedUs(*evt)); + env.d("tid", evt_start->threadId()); + out << event_template.format(env); + } + } + out << "]\n"; +} + +RecordProfile::RecordProfile(std::ostream& out) : out_(out) { + init(); +} + +RecordProfile::RecordProfile(const std::string& filename) + : file_(std::make_unique(filename)), out_(*file_) { + init(); +} + +void RecordProfile::init() { + enableProfilerLegacy(torch::profiler::impl::ProfilerConfig( + torch::profiler::impl::ProfilerState::CPU)); +} + +RecordProfile::~RecordProfile() { + try { + thread_event_lists event_lists = disableProfilerLegacy(); + std::vector events; + for (auto& l : event_lists) { + for (auto& e : l) { + events.push_back(&e); + } + } + processEvents(events); + } catch (const std::exception& e) { + LOG(ERROR) << e.what() << '\n'; + } catch (...) { + LOG(ERROR) << "Unknown error" << '\n'; + } +} + +void RecordProfile::processEvents(const std::vector& events) { + writeProfilerEventsToStream(out_, events); +} + +} // namespace torch::autograd::profiler diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/profiler_legacy.h b/western_ai/usa/pytorch/torch/csrc/autograd/profiler_legacy.h new file mode 100644 index 0000000000000000000000000000000000000000..673d0feb9872472f2fcce3524bc50ee542de38f7 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/profiler_legacy.h @@ -0,0 +1,402 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace torch::autograd::profiler { + +enum class C10_API_ENUM EventKind : uint16_t { + Mark, + PushRange, + PopRange, + MemoryAlloc, +}; + +// To be deprecated, once we switch to Kineto profiling +struct TORCH_API LegacyEvent { + LegacyEvent( + EventKind kind, + at::StringView name, + uint16_t thread_id, + bool record_cuda, + at::RecordFunctionHandle handle = 0, + std::vector>&& shapes = {}, + int64_t node_id = -1, + bool is_async = false) + : name_(std::move(name)), + kind_(kind), + thread_id_(thread_id), + handle_(handle), + shapes_(std::move(shapes)), + node_id_(node_id), + is_async_(is_async) { + record(record_cuda); + } + + // Constructor to be used in conjunction with LegacyEvent::fromIValue. + LegacyEvent( + EventKind kind, + at::StringView name, + uint16_t thread_id, + at::RecordFunctionHandle handle, + std::vector>&& shapes, + int64_t node_id, + bool is_remote, + int64_t cpu_memory_usage, + int64_t cpu_ns, + bool cuda_recorded, + int64_t cuda_memory_usage = 0, + c10::DeviceIndex device = -1, + double cuda_us = -1) + : cpu_ns_(cpu_ns), + name_(std::move(name)), + kind_(kind), + thread_id_(thread_id), + handle_(handle), + shapes_(std::move(shapes)), + cpu_memory_usage_(cpu_memory_usage), + cuda_memory_usage_(cuda_memory_usage), + device_(device), + node_id_(node_id), + is_remote_(is_remote), + cuda_us_(static_cast(cuda_us)) { + // Sanity check values that were deserialized + TORCH_INTERNAL_ASSERT(cpu_ns_ > 0); + if (cuda_recorded) { + TORCH_INTERNAL_ASSERT(device_ >= 0); + TORCH_INTERNAL_ASSERT(cuda_us_ >= 0); + } + } + + // Returns IValues corresponding to event structure, to be used for + // serialization. + at::IValue toIValue() const; + + // Reconstructs an event from IValues given by toIValue. + static LegacyEvent fromIValue(const at::IValue& eventIValue); + + void record(bool record_cuda); + + std::string kindStr() const { + switch (kind_) { + case EventKind::Mark: + return "mark"; + case EventKind::PushRange: + return "push"; + case EventKind::PopRange: + return "pop"; + case EventKind::MemoryAlloc: + return "memory_alloc"; + default: + TORCH_CHECK(false, "unknown event kind"); + } + } + + EventKind kind() const { + return kind_; + } + + const char* name() const { + return name_.str(); + } + + uint64_t threadId() const { + return thread_id_; + } + + std::vector> shapes() const { + return shapes_; + } + + double cpuElapsedUs(const LegacyEvent& e) const { + return static_cast(e.cpu_ns_ - cpu_ns_) / 1000.0; + } + + void setCpuUs(int64_t cpu_us) { + cpu_ns_ = cpu_us * 1000; + } + + double cpuUs() const { + return static_cast(cpu_ns_) / 1000.0; + } + + double cudaElapsedUs(const LegacyEvent& e) const; + + bool hasCuda() const { + return cuda_event != nullptr || (isRemote() && device_ != -1); + } + + c10::DeviceIndex device() const { + return device_; + } + + void updateMemoryStats(int64_t alloc_size, c10::Device device) { + if (device.is_cuda() || device.type() == c10::DeviceType::HIP) { + cuda_memory_usage_ = alloc_size; + } else if ( + device.is_cpu() || device.type() == c10::DeviceType::MKLDNN || + device.type() == c10::DeviceType::IDEEP) { + cpu_memory_usage_ = alloc_size; + } else { + LOG(WARNING) << "Unsupported memory profiling device: " << device; + } + } + + int64_t cpuMemoryUsage() const { + return cpu_memory_usage_; + } + + int64_t cudaMemoryUsage() const { + return cuda_memory_usage_; + } + + at::RecordFunctionHandle handle() const { + return handle_; + } + + // Node ID corresponding to this event. + int64_t nodeId() const { + return node_id_; + } + + // Set Node ID on this event. + void setNodeId(int64_t node_id) { + node_id_ = node_id; + } + + void setName(at::StringView newName_) { + name_ = std::move(newName_); + } + + bool isRemote() const { + return is_remote_; + } + + void setCudaUs(int64_t cuda_us) { + cuda_us_ = cuda_us; + } + + void setSequenceNr(int64_t sequence_nr) { + sequence_nr_ = sequence_nr; + } + + int64_t sequenceNr() const { + return sequence_nr_; + } + + void setCorrelationId(uint64_t correlation_id) { + correlation_id_ = correlation_id; + } + + uint64_t correlationId() const { + return correlation_id_; + } + + const std::vector& stack() const { + return stack_; + } + + void setStack(const std::vector& stack) { + stack_ = stack; + } + + uint64_t fwdThreadId() const { + return fwd_thread_id_; + } + + void setFwdThreadId(uint64_t fwd_thread_id) { + fwd_thread_id_ = fwd_thread_id; + } + + uint8_t scope() const { + return scope_; + } + + void setScope(uint8_t scope) { + scope_ = scope; + } + + const std::unordered_map& extraArgs() const { + return extra_args_; + } + + void setExtraArgs(std::unordered_map&& save_args) { + extra_args_ = std::move(save_args); + } + + uint64_t flops() { + return flops_; + } + + bool isAsync() { + return is_async_; + } + + void setFlops(uint64_t flops) { + flops_ = flops; + } + + private: + // signed to allow for negative intervals, initialized for safety. + int64_t cpu_ns_ = 0; + at::StringView name_; + EventKind kind_; + uint64_t thread_id_; + uint64_t fwd_thread_id_{0}; + at::RecordFunctionHandle handle_{0}; + std::vector> shapes_; + int64_t cpu_memory_usage_ = 0; + int64_t cuda_memory_usage_ = 0; + c10::DeviceIndex device_ = -1; + torch::profiler::impl::ProfilerVoidEventStub cuda_event = nullptr; + int64_t node_id_ = 0; + bool is_remote_ = false; + int64_t cuda_us_ = -1; + int64_t sequence_nr_ = -1; + bool is_async_ = false; + + std::vector stack_; + uint8_t scope_{0}; + uint64_t correlation_id_{0}; + // Extra arguments for computing op flops + std::unordered_map extra_args_; + uint64_t flops_ = 0; +}; + +// a linked-list of fixed sized vectors, to avoid +// a std::vector resize from taking a large amount of time inside +// a profiling event +struct RangeEventList { + RangeEventList() { + events_.reserve(kReservedCapacity); + } + + template + void record(Args&&... args) { + std::lock_guard guard(mutex_); + events_.emplace_back(std::forward(args)...); + } + + std::vector consolidate() { + std::lock_guard lock(mutex_); + std::vector result; + result.insert( + result.begin(), + std::make_move_iterator(events_.begin()), + std::make_move_iterator(events_.end())); + events_.erase(events_.begin(), events_.end()); + return result; + } + + size_t size() { + std::lock_guard lock(mutex_); + return events_.size(); + } + + private: + // This mutex is used to serialize access when different threads are writing + // to the same instance of RangeEventList. + std::mutex mutex_; + std::vector events_; + + static const size_t kReservedCapacity = 1024; +}; + +// A struct to control settings of disableProfiler options. +struct TORCH_API ProfilerDisableOptions { + ProfilerDisableOptions() = default; + ProfilerDisableOptions(bool shouldCleanupTLSState, bool shouldConsolidate) + : cleanupTLSState(shouldCleanupTLSState), + consolidate(shouldConsolidate) {} + // Whether we should clean up profiler states that are thread local, such as + // ThreadLocalDebugInfo and thread local RecordFunction callbacks. + bool cleanupTLSState = true; + // Whether we should consolidate all currently recorded profiled events. If + // false, will not consolidate and other threads can continue to write to the + // event lists. + bool consolidate = true; +}; + +// NOTE: profiler mode is thread local, with automatic propagation +// across thread boundary (e.g. at::launch tasks) +TORCH_API void enableProfilerLegacy( + const torch::profiler::impl::ProfilerConfig& /*new_config*/); +using thread_event_lists = std::vector>; +TORCH_API thread_event_lists disableProfilerLegacy( + std::optional profilerDisableOptions = + std::nullopt); + +// adds profiledEvents to the current thread local recorded events. Each event +// will be marked with node ID given by fromNodeId. +TORCH_API void addEventList(std::vector&& profiledEvents); +// Writes profiled events to a stream. +TORCH_API void writeProfilerEventsToStream( + std::ostream& out, + const std::vector& events); + +// Usage: +// { +// RecordProfile guard("filename.trace"); +// // code you want to profile +// } +// Then open filename.trace in chrome://tracing +struct TORCH_API RecordProfile { + RecordProfile(std::ostream& out); + RecordProfile(const std::string& filename); + + ~RecordProfile(); + + private: + void init(); + std::unique_ptr file_; + std::ostream& out_; + void processEvents(const std::vector& events); +}; + +// A guard that enables the legacy profiler, taking in an optional callback to +// process the results Usage: +// { +// TLSLegacyProfilerGuard g([](thread_event_lists profilerResults) { +// // process profilerResults +// }); +// Code to profile +// } +struct TORCH_API TLSLegacyProfilerGuard { + explicit TLSLegacyProfilerGuard( + const torch::profiler::impl::ProfilerConfig& cfg, + std::optional> + resultCallback = std::nullopt, + std::optional profilerDisableOptions = + std::nullopt) + : cb_(std::move(resultCallback)), + profilerDisableOptions_(profilerDisableOptions) { + enableProfilerLegacy(cfg); + } + ~TLSLegacyProfilerGuard() { + thread_event_lists event_lists = + disableProfilerLegacy(profilerDisableOptions_); + if (cb_) { + try { + (*cb_)(event_lists); + } catch (const std::exception& e) { + LOG(ERROR) << "Got error processing profiler events: " << e.what(); + } + } + } + + private: + std::optional> cb_; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const std::optional profilerDisableOptions_; +}; + +} // namespace torch::autograd::profiler diff --git a/western_ai/usa/pytorch/torch/csrc/autograd/profiler_python.cpp b/western_ai/usa/pytorch/torch/csrc/autograd/profiler_python.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4414299dc50939df2f68be8652cb16713c006962 --- /dev/null +++ b/western_ai/usa/pytorch/torch/csrc/autograd/profiler_python.cpp @@ -0,0 +1,1644 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; + +namespace torch::profiler::impl { +namespace { +enum CallType { PyCall = 0, PyModuleCall, PyCCall, PyOptimizerCall }; +static constexpr size_t CallTypeSize = 4; +using no_ephemeral_t = std::tuple<>; +static constexpr uint64_t NoTID = std::numeric_limits::max(); + +// ============================================================================ +// == Miscellaneous structs and utils ========================================= +// ============================================================================ +struct CodeLocation { + CodeLocation() = default; + explicit CodeLocation(PyFrameObject* frame) + : line_number_{PyFrame_GetLineNumber(frame)} { + auto code = THPCodeObjectPtr(PyFrame_GetCode(frame)); + filename_ = THPUtils_unpackStringView(code->co_filename).data(); + name_ = THPUtils_unpackStringView(code->co_name).data(); + } + + bool operator==(const CodeLocation& other) const { + return filename_ == other.filename_ && name_ == other.name_ && + line_number_ == other.line_number_; + } + + const char* filename_{nullptr}; + const char* name_{nullptr}; + int line_number_{0}; +}; + +template +PyCodeObject* getCode(); + +template <> +PyCodeObject* getCode() { + static auto module_call_code = []() { + pybind11::gil_scoped_acquire gil; + auto res = py::module::import("torch.nn") + .attr("Module") + .attr("__call__") + .attr("__code__") + .ptr(); + TORCH_INTERNAL_ASSERT(PyCode_Check(res)); + return (PyCodeObject*)res; + }(); + return module_call_code; +} + +template <> +PyCodeObject* getCode() { + static auto optimizer_step_code = []() { + pybind11::gil_scoped_acquire gil; + auto res = py::module::import("torch.optim") + .attr("Optimizer") + .attr("_optimizer_step_code") + .attr("__code__") + .ptr(); + TORCH_INTERNAL_ASSERT(PyCode_Check(res)); + return (PyCodeObject*)res; + }(); + return optimizer_step_code; +} + +} // namespace +} // namespace torch::profiler::impl + +template <> +struct std::hash { + size_t operator()(const torch::profiler::impl::CodeLocation& x) { + return c10::get_hash(x.filename_, x.name_, x.line_number_); + } +}; + +namespace torch::profiler::impl { +namespace { +// ============================================================================ +// == CallTypeHelper: Tools for generic programming on specializations. ======= +// ============================================================================ +template