Instructions to use luclima/mythos-models with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- llama.cpp
How to use luclima/mythos-models with llama.cpp:
Install (macOS, Linux)
curl -LsSf https://llama.app/install.sh | sh # Start a local OpenAI-compatible server with a web UI: llama serve -hf luclima/mythos-models:Q4_K_M # Run inference directly in the terminal: llama cli -hf luclima/mythos-models:Q4_K_M
Install from WinGet (Windows)
winget install llama.cpp # Start a local OpenAI-compatible server with a web UI: llama serve -hf luclima/mythos-models:Q4_K_M # Run inference directly in the terminal: llama cli -hf luclima/mythos-models:Q4_K_M
Build from source code
git clone https://github.com/ggml-org/llama.cpp.git cd llama.cpp cmake -B build cmake --build build -j --target llama-app # Start a local OpenAI-compatible server with a web UI: ./build/bin/llama serve -hf luclima/mythos-models:Q4_K_M # Run inference directly in the terminal: ./build/bin/llama cli -hf luclima/mythos-models:Q4_K_M
Use Docker
docker model run hf.co/luclima/mythos-models:Q4_K_M
- Llama new
- LM Studio
- Jan
- Ollama
How to use luclima/mythos-models with Ollama:
ollama run hf.co/luclima/mythos-models:Q4_K_M
- Unsloth Desktop
- Docker Model Runner
How to use luclima/mythos-models with Docker Model Runner:
docker model run hf.co/luclima/mythos-models:Q4_K_M
- Lemonade
How to use luclima/mythos-models with Lemonade:
Pull the model
# Download Lemonade from https://lemonade-server.ai/ lemonade pull luclima/mythos-models:Q4_K_M
Run and chat with the model
lemonade run user.mythos-models-Q4_K_M
List all available models
lemonade list
- Atomic Chat
Upload folder using huggingface_hub (part 39)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- western_ai/usa/pytorch/torch/CMakeLists.txt +568 -0
- western_ai/usa/pytorch/torch/autograd/_functions/replace.vim +3 -0
- western_ai/usa/pytorch/torch/autograd/_functions/tensor.py +72 -0
- western_ai/usa/pytorch/torch/autograd/_functions/utils.py +26 -0
- western_ai/usa/pytorch/torch/autograd/anomaly_mode.py +123 -0
- western_ai/usa/pytorch/torch/autograd/forward_ad.py +241 -0
- western_ai/usa/pytorch/torch/autograd/function.py +898 -0
- western_ai/usa/pytorch/torch/autograd/functional.py +1212 -0
- western_ai/usa/pytorch/torch/autograd/grad_mode.py +489 -0
- western_ai/usa/pytorch/torch/autograd/gradcheck.py +2324 -0
- western_ai/usa/pytorch/torch/autograd/graph.py +989 -0
- western_ai/usa/pytorch/torch/autograd/profiler.py +1799 -0
- western_ai/usa/pytorch/torch/autograd/profiler_legacy.py +327 -0
- western_ai/usa/pytorch/torch/autograd/profiler_util.py +1617 -0
- western_ai/usa/pytorch/torch/autograd/variable.py +15 -0
- western_ai/usa/pytorch/torch/backends/__init__.py +140 -0
- western_ai/usa/pytorch/torch/backends/_coreml/__init__.py +0 -0
- western_ai/usa/pytorch/torch/backends/_coreml/preprocess.py +154 -0
- western_ai/usa/pytorch/torch/backends/_nnapi/__init__.py +0 -0
- western_ai/usa/pytorch/torch/backends/_nnapi/prepare.py +208 -0
- western_ai/usa/pytorch/torch/backends/_nnapi/serializer.py +0 -0
- western_ai/usa/pytorch/torch/backends/cpu/__init__.py +21 -0
- western_ai/usa/pytorch/torch/backends/cuda/__init__.py +711 -0
- western_ai/usa/pytorch/torch/backends/cudnn/__init__.py +261 -0
- western_ai/usa/pytorch/torch/backends/cudnn/rnn.py +128 -0
- western_ai/usa/pytorch/torch/backends/cusparselt/__init__.py +57 -0
- western_ai/usa/pytorch/torch/backends/kleidiai/__init__.py +7 -0
- western_ai/usa/pytorch/torch/backends/mha/__init__.py +25 -0
- western_ai/usa/pytorch/torch/backends/miopen/__init__.py +50 -0
- western_ai/usa/pytorch/torch/backends/mkl/__init__.py +59 -0
- western_ai/usa/pytorch/torch/backends/mkldnn/__init__.py +139 -0
- western_ai/usa/pytorch/torch/backends/mps/__init__.py +60 -0
- western_ai/usa/pytorch/torch/backends/nnpack/__init__.py +32 -0
- western_ai/usa/pytorch/torch/backends/openmp/__init__.py +7 -0
- western_ai/usa/pytorch/torch/backends/opt_einsum/__init__.py +117 -0
- western_ai/usa/pytorch/torch/backends/python_native/__init__.py +387 -0
- western_ai/usa/pytorch/torch/backends/quantized/__init__.py +65 -0
- western_ai/usa/pytorch/torch/backends/xeon/__init__.py +0 -0
- western_ai/usa/pytorch/torch/backends/xeon/run_cpu.py +957 -0
- western_ai/usa/pytorch/torch/backends/xnnpack/__init__.py +29 -0
- western_ai/usa/pytorch/torch/compiler/__init__.py +984 -0
- western_ai/usa/pytorch/torch/compiler/_cache.py +342 -0
- western_ai/usa/pytorch/torch/compiler/config.py +344 -0
- western_ai/usa/pytorch/torch/contrib/__init__.py +0 -0
- western_ai/usa/pytorch/torch/contrib/_tensorboard_vis.py +152 -0
- western_ai/usa/pytorch/torch/cpu/__init__.py +252 -0
- western_ai/usa/pytorch/torch/cpu/amp/__init__.py +3 -0
- western_ai/usa/pytorch/torch/cpu/amp/autocast_mode.py +71 -0
- western_ai/usa/pytorch/torch/cpu/amp/grad_scaler.py +35 -0
- western_ai/usa/pytorch/torch/csrc/CudaIPCTypes.cpp +273 -0
western_ai/usa/pytorch/torch/CMakeLists.txt
ADDED
|
@@ -0,0 +1,568 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# This file used to build libtorch.so.
|
| 2 |
+
# Now it only builds the Torch python bindings.
|
| 3 |
+
|
| 4 |
+
if(NOT CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO)
|
| 5 |
+
cmake_minimum_required(VERSION 3.27 FATAL_ERROR)
|
| 6 |
+
project(torch CXX C)
|
| 7 |
+
find_package(torch REQUIRED)
|
| 8 |
+
option(USE_CUDA "Use CUDA" ON)
|
| 9 |
+
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
| 10 |
+
endif()
|
| 11 |
+
|
| 12 |
+
if(NOT BUILD_PYTHON)
|
| 13 |
+
return()
|
| 14 |
+
endif()
|
| 15 |
+
|
| 16 |
+
set(TORCH_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
|
| 17 |
+
set(TORCH_ROOT "${TORCH_SRC_DIR}/..")
|
| 18 |
+
|
| 19 |
+
if(NOT TORCH_INSTALL_LIB_DIR)
|
| 20 |
+
set(TORCH_INSTALL_LIB_DIR lib)
|
| 21 |
+
endif()
|
| 22 |
+
|
| 23 |
+
if(MSVC)
|
| 24 |
+
set(LIBSHM_SUBDIR libshm_windows)
|
| 25 |
+
else()
|
| 26 |
+
set(LIBSHM_SUBDIR libshm)
|
| 27 |
+
endif()
|
| 28 |
+
|
| 29 |
+
set(LIBSHM_SRCDIR ${TORCH_SRC_DIR}/lib/${LIBSHM_SUBDIR})
|
| 30 |
+
add_subdirectory(${LIBSHM_SRCDIR})
|
| 31 |
+
|
| 32 |
+
# Generate files
|
| 33 |
+
set(TOOLS_PATH "${TORCH_ROOT}/tools")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
set(TORCH_PYTHON_SRCS
|
| 37 |
+
${GENERATED_CXX_PYTHON}
|
| 38 |
+
)
|
| 39 |
+
append_filelist("libtorch_python_core_sources" TORCH_PYTHON_SRCS)
|
| 40 |
+
|
| 41 |
+
# NB: This has to match the condition under which the JIT test directory
|
| 42 |
+
# is included (at the time of writing that's in caffe2/CMakeLists.txt).
|
| 43 |
+
if(BUILD_TEST)
|
| 44 |
+
add_definitions(-DBUILDING_TESTS)
|
| 45 |
+
list(APPEND TORCH_PYTHON_SRCS
|
| 46 |
+
${TORCH_ROOT}/test/cpp/jit/torch_python_test.cpp
|
| 47 |
+
)
|
| 48 |
+
endif()
|
| 49 |
+
|
| 50 |
+
set(TORCH_PYTHON_INCLUDE_DIRECTORIES
|
| 51 |
+
${PYTHON_INCLUDE_DIR}
|
| 52 |
+
|
| 53 |
+
${TORCH_ROOT}
|
| 54 |
+
${TORCH_ROOT}/aten/src
|
| 55 |
+
|
| 56 |
+
${CMAKE_BINARY_DIR}
|
| 57 |
+
${CMAKE_BINARY_DIR}/aten/src
|
| 58 |
+
${CMAKE_BINARY_DIR}/caffe2/aten/src
|
| 59 |
+
${CMAKE_BINARY_DIR}/third_party
|
| 60 |
+
${CMAKE_BINARY_DIR}/third_party/onnx
|
| 61 |
+
|
| 62 |
+
${TORCH_ROOT}/third_party/valgrind-headers
|
| 63 |
+
|
| 64 |
+
${TORCH_ROOT}/third_party/gloo
|
| 65 |
+
${TORCH_ROOT}/third_party/onnx
|
| 66 |
+
${TORCH_ROOT}/third_party/flatbuffers/include
|
| 67 |
+
${TORCH_ROOT}/third_party/kineto/libkineto/include
|
| 68 |
+
${TORCH_ROOT}/third_party/cpp-httplib
|
| 69 |
+
${TORCH_ROOT}/third_party/nlohmann/include
|
| 70 |
+
|
| 71 |
+
${TORCH_SRC_DIR}/csrc
|
| 72 |
+
${TORCH_SRC_DIR}/csrc/api/include
|
| 73 |
+
${TORCH_SRC_DIR}/lib
|
| 74 |
+
${TORCH_SRC_DIR}/standalone
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
list(APPEND TORCH_PYTHON_INCLUDE_DIRECTORIES ${LIBSHM_SRCDIR})
|
| 78 |
+
|
| 79 |
+
set(TORCH_PYTHON_LINK_LIBRARIES
|
| 80 |
+
Python::Module
|
| 81 |
+
pybind::pybind11
|
| 82 |
+
httplib
|
| 83 |
+
nlohmann
|
| 84 |
+
moodycamel
|
| 85 |
+
shm
|
| 86 |
+
fmt::fmt-header-only
|
| 87 |
+
ATEN_CPU_FILES_GEN_LIB)
|
| 88 |
+
|
| 89 |
+
if(USE_ASAN AND TARGET Sanitizer::address)
|
| 90 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::address)
|
| 91 |
+
endif()
|
| 92 |
+
# Skip UBSan on ROCm: see comment in cmake/Dependencies.cmake (UBSan + ASAN
|
| 93 |
+
# on ROCm Clang produces misaligned ASAN global metadata that aborts at
|
| 94 |
+
# startup).
|
| 95 |
+
if(USE_ASAN AND TARGET Sanitizer::undefined AND NOT USE_ROCM)
|
| 96 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::undefined)
|
| 97 |
+
endif()
|
| 98 |
+
if(USE_LSAN AND TARGET Sanitizer::leak)
|
| 99 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::leak)
|
| 100 |
+
endif()
|
| 101 |
+
if(USE_TSAN AND TARGET Sanitizer::thread)
|
| 102 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::thread)
|
| 103 |
+
endif()
|
| 104 |
+
|
| 105 |
+
set(TORCH_PYTHON_COMPILE_DEFINITIONS)
|
| 106 |
+
|
| 107 |
+
set(TORCH_PYTHON_COMPILE_OPTIONS)
|
| 108 |
+
|
| 109 |
+
set(TORCH_PYTHON_LINK_FLAGS "")
|
| 110 |
+
|
| 111 |
+
if(MSVC)
|
| 112 |
+
string(APPEND TORCH_PYTHON_LINK_FLAGS " /NODEFAULTLIB:LIBCMT.LIB")
|
| 113 |
+
# Python headers use #pragma comment(lib, "pythonXX") which makes MSVC search
|
| 114 |
+
# for the library by bare name. Add the directory so the linker can find it.
|
| 115 |
+
foreach(_pylib IN LISTS Python_LIBRARIES)
|
| 116 |
+
if(_pylib MATCHES "python[0-9].*\\.lib$")
|
| 117 |
+
get_filename_component(_pylib_dir "${_pylib}" DIRECTORY)
|
| 118 |
+
string(APPEND TORCH_PYTHON_LINK_FLAGS " /LIBPATH:\"${_pylib_dir}\"")
|
| 119 |
+
break()
|
| 120 |
+
endif()
|
| 121 |
+
endforeach()
|
| 122 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES ${Python_LIBRARIES} onnx_library)
|
| 123 |
+
if(NOT CMAKE_BUILD_TYPE MATCHES "Release")
|
| 124 |
+
string(APPEND TORCH_PYTHON_LINK_FLAGS " /DEBUG:FULL")
|
| 125 |
+
endif()
|
| 126 |
+
elseif(APPLE)
|
| 127 |
+
string(APPEND TORCH_PYTHON_LINK_FLAGS " -undefined dynamic_lookup")
|
| 128 |
+
else()
|
| 129 |
+
list(APPEND TORCH_PYTHON_COMPILE_OPTIONS
|
| 130 |
+
-fno-strict-aliasing
|
| 131 |
+
-Wno-strict-aliasing)
|
| 132 |
+
endif()
|
| 133 |
+
|
| 134 |
+
if(USE_ITT)
|
| 135 |
+
list(APPEND TORCH_PYTHON_SRCS
|
| 136 |
+
${TORCH_SRC_DIR}/csrc/itt.cpp
|
| 137 |
+
)
|
| 138 |
+
list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_ITT)
|
| 139 |
+
endif()
|
| 140 |
+
|
| 141 |
+
if(USE_CUDA)
|
| 142 |
+
include(${TORCH_ROOT}/cmake/public/cuda.cmake)
|
| 143 |
+
append_filelist("libtorch_python_cuda_core_sources" TORCH_PYTHON_SRCS)
|
| 144 |
+
|
| 145 |
+
list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUDA)
|
| 146 |
+
if(USE_CUDNN)
|
| 147 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::cudnn)
|
| 148 |
+
list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUDNN)
|
| 149 |
+
endif()
|
| 150 |
+
if(USE_CUSPARSELT)
|
| 151 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::cusparselt)
|
| 152 |
+
list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUSPARSELT)
|
| 153 |
+
endif()
|
| 154 |
+
if(USE_CUFILE)
|
| 155 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::cufile)
|
| 156 |
+
list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUFILE)
|
| 157 |
+
endif()
|
| 158 |
+
|
| 159 |
+
if(TARGET torch::nvtx3)
|
| 160 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::nvtx3)
|
| 161 |
+
else()
|
| 162 |
+
if(TARGET torch::nvtoolsext)
|
| 163 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::nvtoolsext)
|
| 164 |
+
endif()
|
| 165 |
+
endif()
|
| 166 |
+
endif()
|
| 167 |
+
|
| 168 |
+
if(USE_ROCM)
|
| 169 |
+
append_filelist("libtorch_python_cuda_core_sources" TORCH_PYTHON_SRCS)
|
| 170 |
+
|
| 171 |
+
list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS
|
| 172 |
+
USE_ROCM
|
| 173 |
+
__HIP_PLATFORM_AMD__
|
| 174 |
+
)
|
| 175 |
+
if(NOT WIN32)
|
| 176 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES ${ROCM_ROCTX_LIB})
|
| 177 |
+
endif()
|
| 178 |
+
endif()
|
| 179 |
+
|
| 180 |
+
if(USE_XPU)
|
| 181 |
+
include(${TORCH_ROOT}/cmake/public/xpu.cmake)
|
| 182 |
+
append_filelist("libtorch_python_xpu_sources" TORCH_PYTHON_SRCS)
|
| 183 |
+
list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_XPU)
|
| 184 |
+
endif()
|
| 185 |
+
|
| 186 |
+
if(USE_CUDNN OR USE_ROCM)
|
| 187 |
+
list(APPEND TORCH_PYTHON_SRCS
|
| 188 |
+
${TORCH_SRC_DIR}/csrc/cuda/shared/cudnn.cpp
|
| 189 |
+
)
|
| 190 |
+
if(USE_STATIC_CUDNN)
|
| 191 |
+
set_source_files_properties(
|
| 192 |
+
${TORCH_SRC_DIR}/csrc/cuda/shared/cudnn.cpp
|
| 193 |
+
PROPERTIES COMPILE_DEFINITIONS "USE_STATIC_CUDNN"
|
| 194 |
+
)
|
| 195 |
+
endif()
|
| 196 |
+
endif()
|
| 197 |
+
|
| 198 |
+
if(USE_CUSPARSELT)
|
| 199 |
+
list(APPEND TORCH_PYTHON_SRCS ${TORCH_SRC_DIR}/csrc/cuda/shared/cusparselt.cpp)
|
| 200 |
+
endif()
|
| 201 |
+
|
| 202 |
+
if(USE_MPS)
|
| 203 |
+
list(APPEND TORCH_PYTHON_SRCS ${MPS_PYTHON_SRCS})
|
| 204 |
+
endif()
|
| 205 |
+
|
| 206 |
+
if(USE_VALGRIND AND NOT WIN32)
|
| 207 |
+
list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_VALGRIND)
|
| 208 |
+
endif()
|
| 209 |
+
|
| 210 |
+
add_custom_target(torch_python_stubs DEPENDS
|
| 211 |
+
"${TORCH_SRC_DIR}/_C/__init__.pyi"
|
| 212 |
+
"${TORCH_SRC_DIR}/_C/_VariableFunctions.pyi"
|
| 213 |
+
"${TORCH_SRC_DIR}/nn/functional.pyi"
|
| 214 |
+
"${TORCH_SRC_DIR}/utils/data/datapipes/datapipe.pyi"
|
| 215 |
+
)
|
| 216 |
+
|
| 217 |
+
file(GLOB_RECURSE torchgen_python CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/torchgen/*.py")
|
| 218 |
+
file(GLOB_RECURSE autograd_python CONFIGURE_DEPENDS "${TOOLS_PATH}/autograd/*.py")
|
| 219 |
+
file(GLOB_RECURSE pyi_python CONFIGURE_DEPENDS "${TOOLS_PATH}/pyi/*.py")
|
| 220 |
+
add_custom_command(
|
| 221 |
+
OUTPUT
|
| 222 |
+
"${TORCH_SRC_DIR}/_C/__init__.pyi"
|
| 223 |
+
"${TORCH_SRC_DIR}/_C/_VariableFunctions.pyi"
|
| 224 |
+
"${TORCH_SRC_DIR}/nn/functional.pyi"
|
| 225 |
+
COMMAND
|
| 226 |
+
"${Python_EXECUTABLE}" -mtools.pyi.gen_pyi
|
| 227 |
+
--native-functions-path "aten/src/ATen/native/native_functions.yaml"
|
| 228 |
+
--tags-path "aten/src/ATen/native/tags.yaml"
|
| 229 |
+
--deprecated-functions-path "tools/autograd/deprecated.yaml"
|
| 230 |
+
DEPENDS
|
| 231 |
+
"${TORCH_SRC_DIR}/_C/__init__.pyi.in"
|
| 232 |
+
"${TORCH_SRC_DIR}/_C/_VariableFunctions.pyi.in"
|
| 233 |
+
"${TORCH_SRC_DIR}/nn/functional.pyi.in"
|
| 234 |
+
"${TORCH_ROOT}/aten/src/ATen/native/native_functions.yaml"
|
| 235 |
+
"${TORCH_ROOT}/aten/src/ATen/native/tags.yaml"
|
| 236 |
+
"${TORCH_ROOT}/tools/autograd/deprecated.yaml"
|
| 237 |
+
"${TORCH_ROOT}/torch/_torch_docs.py"
|
| 238 |
+
"${TORCH_ROOT}/torch/_tensor_docs.py"
|
| 239 |
+
${pyi_python}
|
| 240 |
+
${autograd_python}
|
| 241 |
+
${torchgen_python}
|
| 242 |
+
WORKING_DIRECTORY
|
| 243 |
+
"${TORCH_ROOT}"
|
| 244 |
+
)
|
| 245 |
+
file(GLOB_RECURSE datapipe_files CONFIGURE_DEPENDS "${TORCH_SRC_DIR}/utils/data/datapipes/*.py")
|
| 246 |
+
add_custom_command(
|
| 247 |
+
OUTPUT
|
| 248 |
+
"${TORCH_SRC_DIR}/utils/data/datapipes/datapipe.pyi"
|
| 249 |
+
COMMAND
|
| 250 |
+
${CMAKE_COMMAND} -E env --modify PYTHONPATH=path_list_prepend:"${TORCH_ROOT}" --
|
| 251 |
+
"${Python_EXECUTABLE}" ${TORCH_SRC_DIR}/utils/data/datapipes/gen_pyi.py
|
| 252 |
+
DEPENDS
|
| 253 |
+
"${TORCH_SRC_DIR}/utils/data/datapipes/datapipe.pyi.in"
|
| 254 |
+
${datapipe_files}
|
| 255 |
+
WORKING_DIRECTORY
|
| 256 |
+
"${TORCH_ROOT}"
|
| 257 |
+
)
|
| 258 |
+
if(USE_DISTRIBUTED)
|
| 259 |
+
if(WIN32)
|
| 260 |
+
append_filelist("libtorch_python_distributed_core_sources" TORCH_PYTHON_SRCS)
|
| 261 |
+
else()
|
| 262 |
+
append_filelist("libtorch_python_distributed_sources" TORCH_PYTHON_SRCS)
|
| 263 |
+
endif()
|
| 264 |
+
# Disable certain warnings for GCC-9.X
|
| 265 |
+
if(CMAKE_COMPILER_IS_GNUCXX)
|
| 266 |
+
set_source_files_properties(${TORCH_SRC_DIR}/csrc/distributed/autograd/init.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type")
|
| 267 |
+
set_source_files_properties(${TORCH_SRC_DIR}/csrc/distributed/rpc/testing/init.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type")
|
| 268 |
+
set_source_files_properties(${TORCH_SRC_DIR}/csrc/distributed/c10d/init.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type")
|
| 269 |
+
endif()
|
| 270 |
+
# NCCL is a private dependency of libtorch, but libtorch_python includes
|
| 271 |
+
# some private headers of libtorch, which in turn include NCCL. As a hacky
|
| 272 |
+
# alternative to making NCCL a public dependency of libtorch, we make it
|
| 273 |
+
# a private dependency of libtorch_python as well.
|
| 274 |
+
if(USE_NCCL)
|
| 275 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES __caffe2_nccl)
|
| 276 |
+
endif()
|
| 277 |
+
# Same for MPI.
|
| 278 |
+
if(USE_MPI)
|
| 279 |
+
list(APPEND TORCH_PYTHON_LINK_LIBRARIES MPI::MPI_CXX)
|
| 280 |
+
endif()
|
| 281 |
+
list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_C10D)
|
| 282 |
+
|
| 283 |
+
endif()
|
| 284 |
+
|
| 285 |
+
if(USE_NCCL AND NOT WIN32)
|
| 286 |
+
list(APPEND TORCH_PYTHON_SRCS
|
| 287 |
+
${TORCH_SRC_DIR}/csrc/cuda/python_nccl.cpp)
|
| 288 |
+
list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_NCCL)
|
| 289 |
+
endif()
|
| 290 |
+
|
| 291 |
+
# coreml
|
| 292 |
+
if(USE_COREML_DELEGATE)
|
| 293 |
+
list(APPEND TORCH_PYTHON_SRCS ${TORCH_SRC_DIR}/csrc/jit/backends/coreml/cpp/backend.cpp)
|
| 294 |
+
list(APPEND TORCH_PYTHON_SRCS ${TORCH_SRC_DIR}/csrc/jit/backends/coreml/cpp/preprocess.cpp)
|
| 295 |
+
endif()
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
add_library(torch_python SHARED ${TORCH_PYTHON_SRCS})
|
| 299 |
+
torch_compile_options(torch_python) # see cmake/public/utils.cmake
|
| 300 |
+
if(APPLE)
|
| 301 |
+
target_compile_options(torch_python PRIVATE
|
| 302 |
+
$<$<COMPILE_LANGUAGE:CXX>: -fvisibility=default>)
|
| 303 |
+
endif()
|
| 304 |
+
|
| 305 |
+
if(CAFFE2_USE_MKL AND BUILD_LIBTORCHLESS)
|
| 306 |
+
|
| 307 |
+
# Use the RPATH of the linked libraries
|
| 308 |
+
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
|
| 309 |
+
# we need to explicitly link caffe2::mkl in order to have the
|
| 310 |
+
# correct RPATH in torch_python for the split build
|
| 311 |
+
target_link_libraries(torch_python PRIVATE caffe2::mkl)
|
| 312 |
+
endif()
|
| 313 |
+
|
| 314 |
+
add_dependencies(torch_python onnx_proto)
|
| 315 |
+
# Avoid numpy for the DEPLOY build
|
| 316 |
+
if(USE_NUMPY)
|
| 317 |
+
target_link_libraries(torch_python PRIVATE Python::NumPy)
|
| 318 |
+
target_compile_definitions(torch_python PRIVATE USE_NUMPY)
|
| 319 |
+
endif()
|
| 320 |
+
|
| 321 |
+
if(USE_CUFILE AND NOT USE_ROCM)
|
| 322 |
+
target_compile_definitions(torch_python PRIVATE USE_CUFILE)
|
| 323 |
+
endif()
|
| 324 |
+
|
| 325 |
+
if(BUILD_LAZY_TS_BACKEND)
|
| 326 |
+
target_compile_definitions(torch_python PRIVATE BUILD_LAZY_TS_BACKEND)
|
| 327 |
+
endif()
|
| 328 |
+
|
| 329 |
+
if(HAVE_SOVERSION)
|
| 330 |
+
set_target_properties(torch_python PROPERTIES
|
| 331 |
+
VERSION ${TORCH_VERSION} SOVERSION ${TORCH_SOVERSION})
|
| 332 |
+
endif()
|
| 333 |
+
|
| 334 |
+
# in case of the split build we need to add compile definitions
|
| 335 |
+
if(BUILD_LIBTORCHLESS)
|
| 336 |
+
|
| 337 |
+
if(USE_UCC AND USE_C10D_UCC)
|
| 338 |
+
target_compile_definitions(torch_python PRIVATE USE_C10D_UCC)
|
| 339 |
+
endif()
|
| 340 |
+
|
| 341 |
+
if(USE_NCCL AND USE_C10D_NCCL)
|
| 342 |
+
target_compile_definitions(torch_python PRIVATE USE_C10D_NCCL)
|
| 343 |
+
endif()
|
| 344 |
+
|
| 345 |
+
if(USE_DISTRIBUTED)
|
| 346 |
+
target_compile_definitions(torch_python PRIVATE USE_DISTRIBUTED)
|
| 347 |
+
endif()
|
| 348 |
+
|
| 349 |
+
if(USE_MPI AND USE_C10D_MPI)
|
| 350 |
+
target_compile_definitions(torch_python PRIVATE USE_C10D_MPI)
|
| 351 |
+
endif()
|
| 352 |
+
|
| 353 |
+
if(USE_GLOO AND USE_C10D_GLOO)
|
| 354 |
+
target_compile_definitions(torch_python PRIVATE USE_C10D_GLOO)
|
| 355 |
+
endif()
|
| 356 |
+
|
| 357 |
+
if(NOT WIN32)
|
| 358 |
+
target_compile_definitions(torch_python PRIVATE USE_RPC)
|
| 359 |
+
endif()
|
| 360 |
+
|
| 361 |
+
if(USE_TENSORPIPE)
|
| 362 |
+
target_compile_definitions(torch_python PRIVATE USE_TENSORPIPE)
|
| 363 |
+
endif()
|
| 364 |
+
|
| 365 |
+
set(EXPERIMENTAL_SINGLE_THREAD_POOL "0" CACHE STRING
|
| 366 |
+
"Experimental option to use a single thread pool for inter- and intra-op parallelism")
|
| 367 |
+
if("${EXPERIMENTAL_SINGLE_THREAD_POOL}")
|
| 368 |
+
target_compile_definitions(torch_python PRIVATE "-DAT_EXPERIMENTAL_SINGLE_THREAD_POOL=1")
|
| 369 |
+
endif()
|
| 370 |
+
|
| 371 |
+
endif()
|
| 372 |
+
|
| 373 |
+
add_dependencies(torch_python torch_python_stubs)
|
| 374 |
+
add_dependencies(torch_python flatbuffers)
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
if(USE_PRECOMPILED_HEADERS)
|
| 378 |
+
target_precompile_headers(torch_python PRIVATE
|
| 379 |
+
"$<$<COMPILE_LANGUAGE:CXX>:ATen/ATen.h>")
|
| 380 |
+
endif()
|
| 381 |
+
|
| 382 |
+
# Required workaround for generated sources
|
| 383 |
+
# See https://samthursfield.wordpress.com/2015/11/21/cmake-dependencies-between-targets-and-files-and-custom-commands/#custom-commands-in-different-directories
|
| 384 |
+
add_dependencies(torch_python generate-torch-sources)
|
| 385 |
+
set_source_files_properties(
|
| 386 |
+
${GENERATED_CXX_PYTHON}
|
| 387 |
+
PROPERTIES GENERATED TRUE
|
| 388 |
+
)
|
| 389 |
+
|
| 390 |
+
# Disable certain warnings for GCC-9.X
|
| 391 |
+
if(CMAKE_COMPILER_IS_GNUCXX)
|
| 392 |
+
set_source_files_properties(${TORCH_SRC_DIR}/csrc/Module.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type")
|
| 393 |
+
set_source_files_properties(${TORCH_SRC_DIR}/csrc/autograd/python_variable.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type")
|
| 394 |
+
endif()
|
| 395 |
+
|
| 396 |
+
# Preserve CUDA_GENCODE flags
|
| 397 |
+
if(USE_CUDA)
|
| 398 |
+
torch_cuda_get_nvcc_gencode_flag(_ARCH_FLAGS)
|
| 399 |
+
set_source_files_properties(${TORCH_SRC_DIR}/csrc/cuda/Module.cpp PROPERTIES COMPILE_FLAGS "-DCUDA_ARCH_FLAGS=\"${_ARCH_FLAGS_readable}\"")
|
| 400 |
+
endif()
|
| 401 |
+
|
| 402 |
+
# Preserve HIP arch flags
|
| 403 |
+
if(USE_ROCM)
|
| 404 |
+
string(REPLACE ";" " " PYTORCH_ROCM_ARCH_readable "${PYTORCH_ROCM_ARCH}")
|
| 405 |
+
set_source_files_properties(${TORCH_SRC_DIR}/csrc/cuda/Module.cpp PROPERTIES COMPILE_FLAGS "-DCUDA_ARCH_FLAGS=\"${PYTORCH_ROCM_ARCH_readable}\"")
|
| 406 |
+
endif()
|
| 407 |
+
|
| 408 |
+
# Preserve XPU arch flags
|
| 409 |
+
if(USE_XPU)
|
| 410 |
+
string(REPLACE "," " " _ARCH_FLAGS_readable "${TORCH_XPU_ARCH_LIST}")
|
| 411 |
+
set_source_files_properties(${TORCH_SRC_DIR}/csrc/xpu/Module.cpp PROPERTIES COMPILE_FLAGS "-DXPU_ARCH_FLAGS=\"${_ARCH_FLAGS_readable}\"")
|
| 412 |
+
endif()
|
| 413 |
+
|
| 414 |
+
target_compile_definitions(torch_python PRIVATE "-DTHP_BUILD_MAIN_LIB")
|
| 415 |
+
|
| 416 |
+
target_link_libraries(torch_python PRIVATE ${TORCH_LIB} ${TORCH_PYTHON_LINK_LIBRARIES})
|
| 417 |
+
|
| 418 |
+
target_compile_definitions(torch_python PRIVATE ${TORCH_PYTHON_COMPILE_DEFINITIONS})
|
| 419 |
+
|
| 420 |
+
target_compile_options(torch_python PRIVATE ${TORCH_PYTHON_COMPILE_OPTIONS})
|
| 421 |
+
|
| 422 |
+
target_include_directories(torch_python PUBLIC ${TORCH_PYTHON_INCLUDE_DIRECTORIES})
|
| 423 |
+
|
| 424 |
+
if(USE_UCC)
|
| 425 |
+
target_link_libraries(torch_python PRIVATE __caffe2_ucc)
|
| 426 |
+
target_compile_definitions(torch_python PRIVATE USE_UCC)
|
| 427 |
+
endif()
|
| 428 |
+
|
| 429 |
+
if(BUILD_ONEDNN_GRAPH)
|
| 430 |
+
target_compile_definitions(torch_python PRIVATE "-DBUILD_ONEDNN_GRAPH")
|
| 431 |
+
if(NOT BUILD_LIBTORCHLESS)
|
| 432 |
+
target_compile_definitions(torch_cpu PRIVATE "-DBUILD_ONEDNN_GRAPH")
|
| 433 |
+
endif()
|
| 434 |
+
endif()
|
| 435 |
+
|
| 436 |
+
if(${CMAKE_BUILD_TYPE} STREQUAL "RelWithAssert")
|
| 437 |
+
# Workaround numerous decret-without-a-gil warnings from JIT
|
| 438 |
+
# see https://github.com/pytorch/pytorch/issues/130073
|
| 439 |
+
target_compile_definitions(torch_python PRIVATE "-DPYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF")
|
| 440 |
+
endif()
|
| 441 |
+
|
| 442 |
+
if(NOT TORCH_PYTHON_LINK_FLAGS STREQUAL "")
|
| 443 |
+
set_target_properties(torch_python PROPERTIES LINK_FLAGS "${TORCH_PYTHON_LINK_FLAGS}")
|
| 444 |
+
endif()
|
| 445 |
+
|
| 446 |
+
install(TARGETS torch_python DESTINATION "${TORCH_INSTALL_LIB_DIR}")
|
| 447 |
+
|
| 448 |
+
# Build the torch._C Python extension module from the thin stub that
|
| 449 |
+
# forwards to torch_python. This was historically built by setuptools;
|
| 450 |
+
# building it here lets both setuptools and scikit-build-core use the
|
| 451 |
+
# same code path.
|
| 452 |
+
if(BUILD_PYTHON AND NOT BUILD_LIBTORCH_WHL)
|
| 453 |
+
Python_add_library(_C MODULE WITH_SOABI "${TORCH_SRC_DIR}/csrc/stub.c")
|
| 454 |
+
target_link_libraries(_C PRIVATE torch_python)
|
| 455 |
+
set_target_properties(_C PROPERTIES
|
| 456 |
+
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/torch"
|
| 457 |
+
# libtorch_python.so is installed to lib/ relative to _C, so use
|
| 458 |
+
# $ORIGIN/lib as the install RPATH (the global default is just $ORIGIN).
|
| 459 |
+
INSTALL_RPATH "${_rpath_portable_origin}/lib"
|
| 460 |
+
)
|
| 461 |
+
if(NOT WIN32)
|
| 462 |
+
set_target_properties(_C PROPERTIES
|
| 463 |
+
C_VISIBILITY_PRESET "default"
|
| 464 |
+
)
|
| 465 |
+
endif()
|
| 466 |
+
# Install at the install prefix root, which maps to torch/ in the wheel
|
| 467 |
+
# via wheel.install-dir = "torch". The import library (Windows _C.lib) goes
|
| 468 |
+
# alongside the other import libs in torch/lib/, matching the layout that
|
| 469 |
+
# setup.py used before _C moved to CMake.
|
| 470 |
+
install(TARGETS _C
|
| 471 |
+
RUNTIME DESTINATION "."
|
| 472 |
+
LIBRARY DESTINATION "."
|
| 473 |
+
ARCHIVE DESTINATION "lib"
|
| 474 |
+
)
|
| 475 |
+
|
| 476 |
+
# ---[ NCCL EP extension (torch._nccl_ep)
|
| 477 |
+
# Optional standalone extension holding the NCCL EP bindings, imported lazily
|
| 478 |
+
# by torch/distributed/_token_switch.py so libtorch_cuda never references
|
| 479 |
+
# ncclEp*. How it links libnccl_ep follows how torch links NCCL:
|
| 480 |
+
# - USE_SYSTEM_NCCL=OFF: __caffe2_nccl_ep is libnccl_ep.a, statically linked
|
| 481 |
+
# here -- self-contained (nccl* bind to torch_cuda's embedded NCCL; JIT
|
| 482 |
+
# headers baked from the submodule build dir; no nccl4py).
|
| 483 |
+
# - USE_SYSTEM_NCCL=ON: __caffe2_nccl_ep is libnccl_ep.so, NEEDED-linked here
|
| 484 |
+
# and resolved from the nccl4py wheel at runtime (RPATH to nccl/ep/lib; JIT
|
| 485 |
+
# headers also from nccl4py, so nothing is baked).
|
| 486 |
+
if(USE_NCCL_EP)
|
| 487 |
+
Python_add_library(_nccl_ep MODULE WITH_SOABI
|
| 488 |
+
"${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/nccl_ep.cu"
|
| 489 |
+
"${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/nccl_ep_pybind.cpp")
|
| 490 |
+
target_compile_definitions(_nccl_ep PRIVATE USE_NCCL_EP USE_C10D_NCCL)
|
| 491 |
+
# torch_python: pybind + Tensor/ProcessGroup casters; torch_cuda:
|
| 492 |
+
# ProcessGroupNCCL::getCommPtr, getNcclDataType, CUDA stream, embedded NCCL;
|
| 493 |
+
# __caffe2_nccl_ep: libnccl_ep (static .a or shared .so) + nccl_ep headers.
|
| 494 |
+
target_link_libraries(_nccl_ep PRIVATE
|
| 495 |
+
torch_python torch_cuda __caffe2_nccl_ep pybind::pybind11)
|
| 496 |
+
set_target_properties(_nccl_ep PROPERTIES
|
| 497 |
+
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/torch")
|
| 498 |
+
if(USE_SYSTEM_NCCL)
|
| 499 |
+
# Dynamic: resolve nccl4py's libnccl_ep.so (site-packages/nccl/ep/lib, one
|
| 500 |
+
# dir up from torch/ via $ORIGIN/..) at import.
|
| 501 |
+
set_target_properties(_nccl_ep PROPERTIES
|
| 502 |
+
INSTALL_RPATH "${_rpath_portable_origin}/lib;${_rpath_portable_origin}/../nccl/ep/lib")
|
| 503 |
+
else()
|
| 504 |
+
# Static: only torch's own libs needed; bake the in-tree JIT header dir so
|
| 505 |
+
# the extension self-configures NCCL_EP_HOME / NCCL_HOME at import.
|
| 506 |
+
target_compile_definitions(_nccl_ep PRIVATE
|
| 507 |
+
"NCCL_EP_JIT_HOME=\"${NCCL_EP_JIT_HOME}\"")
|
| 508 |
+
set_target_properties(_nccl_ep PROPERTIES
|
| 509 |
+
INSTALL_RPATH "${_rpath_portable_origin}/lib")
|
| 510 |
+
endif()
|
| 511 |
+
if(NOT WIN32)
|
| 512 |
+
set_target_properties(_nccl_ep PROPERTIES C_VISIBILITY_PRESET "default")
|
| 513 |
+
endif()
|
| 514 |
+
install(TARGETS _nccl_ep RUNTIME DESTINATION "." LIBRARY DESTINATION ".")
|
| 515 |
+
endif()
|
| 516 |
+
endif()
|
| 517 |
+
|
| 518 |
+
# Generate torch/version.py from the appropriate CMake cache variables.
|
| 519 |
+
if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
|
| 520 |
+
set(TORCH_VERSION_DEBUG 1)
|
| 521 |
+
else()
|
| 522 |
+
set(TORCH_VERSION_DEBUG 0)
|
| 523 |
+
endif()
|
| 524 |
+
|
| 525 |
+
set(CUDA_VERSION "")
|
| 526 |
+
if(CUDAToolkit_VERSION_MAJOR)
|
| 527 |
+
set(CUDA_VERSION "${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR}")
|
| 528 |
+
endif()
|
| 529 |
+
|
| 530 |
+
add_custom_target(
|
| 531 |
+
gen_torch_version ALL
|
| 532 |
+
"${Python_EXECUTABLE}" -m tools.generate_torch_version
|
| 533 |
+
--is-debug=${TORCH_VERSION_DEBUG}
|
| 534 |
+
--cuda-version=${CUDA_VERSION}
|
| 535 |
+
--hip-version=${HIP_VERSION_CLEAN}
|
| 536 |
+
--rocm-version=${ROCM_VERSION_DEV}
|
| 537 |
+
--xpu-version=${SYCL_COMPILER_VERSION}
|
| 538 |
+
BYPRODUCTS ${TORCH_SRC_DIR}/version.py
|
| 539 |
+
COMMENT "Regenerating version file..."
|
| 540 |
+
WORKING_DIRECTORY ${TORCH_ROOT}
|
| 541 |
+
)
|
| 542 |
+
add_dependencies(torch_python gen_torch_version)
|
| 543 |
+
# version.py is gitignored (generated), so scikit-build-core won't include it
|
| 544 |
+
# via Python package collection. Install it explicitly via cmake instead.
|
| 545 |
+
install(FILES "${TORCH_SRC_DIR}/version.py" DESTINATION ".")
|
| 546 |
+
|
| 547 |
+
# Skip building this library under MacOS, since it is currently failing to build on Mac
|
| 548 |
+
# Github issue #61930
|
| 549 |
+
if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
|
| 550 |
+
# Add Android Nnapi delegate library
|
| 551 |
+
add_library(nnapi_backend SHARED
|
| 552 |
+
${TORCH_SRC_DIR}/csrc/jit/backends/nnapi/nnapi_backend_lib.cpp
|
| 553 |
+
${TORCH_SRC_DIR}/csrc/jit/backends/nnapi/nnapi_backend_preprocess.cpp
|
| 554 |
+
)
|
| 555 |
+
# Pybind11 requires explicit linking of the torch_python library
|
| 556 |
+
if(BUILD_LIBTORCHLESS)
|
| 557 |
+
target_link_libraries(nnapi_backend PRIVATE ${TORCH_LIB})
|
| 558 |
+
else()
|
| 559 |
+
target_link_libraries(nnapi_backend PRIVATE torch)
|
| 560 |
+
endif()
|
| 561 |
+
target_link_libraries(nnapi_backend PRIVATE torch_python pybind::pybind11 fmt::fmt-header-only)
|
| 562 |
+
if(MSVC AND TORCH_PYTHON_LINK_FLAGS)
|
| 563 |
+
set_target_properties(nnapi_backend PROPERTIES LINK_FLAGS "${TORCH_PYTHON_LINK_FLAGS}")
|
| 564 |
+
endif()
|
| 565 |
+
endif()
|
| 566 |
+
|
| 567 |
+
set(TORCH_PYTHON_COMPILE_OPTIONS ${TORCH_PYTHON_COMPILE_OPTIONS} PARENT_SCOPE)
|
| 568 |
+
set(TORCH_PYTHON_LINK_FLAGS ${TORCH_PYTHON_LINK_FLAGS} PARENT_SCOPE)
|
western_ai/usa/pytorch/torch/autograd/_functions/replace.vim
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
%s/self/ctx/g
|
| 2 |
+
%s/\s\+def forward/ @staticmethod\r def forward/g
|
| 3 |
+
%s/\s\+def backward/ @staticmethod\r @once_differentiable\r def backward/g
|
western_ai/usa/pytorch/torch/autograd/_functions/tensor.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import operator
|
| 3 |
+
from functools import reduce
|
| 4 |
+
from typing_extensions import deprecated
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
import torch._utils
|
| 8 |
+
from torch.autograd.function import Function
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class Type(Function):
|
| 12 |
+
@staticmethod
|
| 13 |
+
@deprecated(
|
| 14 |
+
"`torch.autograd._functions.Type` is deprecated as of PyTorch 2.1, "
|
| 15 |
+
"please use `torch.tensor.to(dtype=dtype)` instead.",
|
| 16 |
+
category=FutureWarning,
|
| 17 |
+
)
|
| 18 |
+
# pyrefly: ignore [bad-override]
|
| 19 |
+
def forward(ctx, i, dest_type):
|
| 20 |
+
ctx.input_type = type(i)
|
| 21 |
+
ctx.input_device = -1 if not i.is_cuda else i.get_device()
|
| 22 |
+
return i.type(dest_type)
|
| 23 |
+
|
| 24 |
+
@staticmethod
|
| 25 |
+
# pyrefly: ignore [bad-override]
|
| 26 |
+
def backward(ctx, grad_output):
|
| 27 |
+
if ctx.input_device == -1:
|
| 28 |
+
return grad_output.type(ctx.input_type), None
|
| 29 |
+
else:
|
| 30 |
+
with torch.accelerator.device_index(ctx.input_device):
|
| 31 |
+
return grad_output.type(ctx.input_type), None
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# TODO: deprecate this
|
| 35 |
+
class Resize(Function):
|
| 36 |
+
@staticmethod
|
| 37 |
+
# pyrefly: ignore [bad-override]
|
| 38 |
+
def forward(ctx, tensor, sizes):
|
| 39 |
+
ctx.sizes = sizes
|
| 40 |
+
ctx.numel = reduce(operator.mul, sizes, 1)
|
| 41 |
+
if tensor.numel() != ctx.numel:
|
| 42 |
+
raise RuntimeError(
|
| 43 |
+
(
|
| 44 |
+
"requested resize to {} ({} elements in total), "
|
| 45 |
+
"but the given tensor has a size of {} ({} elements). "
|
| 46 |
+
"autograd's resize can only change the shape of a given "
|
| 47 |
+
"tensor, while preserving the number of elements. "
|
| 48 |
+
).format(
|
| 49 |
+
"x".join(map(str, sizes)),
|
| 50 |
+
ctx.numel,
|
| 51 |
+
"x".join(map(str, tensor.size())),
|
| 52 |
+
tensor.numel(),
|
| 53 |
+
)
|
| 54 |
+
)
|
| 55 |
+
ctx.input_sizes = tensor.size()
|
| 56 |
+
if tensor.is_quantized:
|
| 57 |
+
tensor.copy_(tensor)
|
| 58 |
+
return tensor.contiguous().view(*sizes)
|
| 59 |
+
if tensor.is_contiguous():
|
| 60 |
+
result = tensor.new(tensor).contiguous().view(*sizes)
|
| 61 |
+
return result
|
| 62 |
+
else:
|
| 63 |
+
return tensor.contiguous().view(*sizes)
|
| 64 |
+
|
| 65 |
+
@staticmethod
|
| 66 |
+
# pyrefly: ignore [bad-override]
|
| 67 |
+
def backward(ctx, grad_output):
|
| 68 |
+
if grad_output.numel() != ctx.numel:
|
| 69 |
+
raise AssertionError(
|
| 70 |
+
f"Expected grad_output to have {ctx.numel} elements, but got {grad_output.numel()}"
|
| 71 |
+
)
|
| 72 |
+
return grad_output.contiguous().view(ctx.input_sizes), None
|
western_ai/usa/pytorch/torch/autograd/_functions/utils.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def maybe_view(tensor, size, check_same_size=True):
|
| 5 |
+
if check_same_size and tensor.size() == size:
|
| 6 |
+
return tensor
|
| 7 |
+
return tensor.contiguous().view(size)
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def maybe_unexpand(tensor, old_size, check_same_size=True):
|
| 11 |
+
if check_same_size and tensor.size() == old_size:
|
| 12 |
+
return tensor
|
| 13 |
+
num_unsqueezed = tensor.dim() - len(old_size)
|
| 14 |
+
expanded_dims = [
|
| 15 |
+
dim
|
| 16 |
+
for dim, (expanded, original) in enumerate(
|
| 17 |
+
zip(tensor.size()[num_unsqueezed:], old_size)
|
| 18 |
+
)
|
| 19 |
+
if expanded != original
|
| 20 |
+
]
|
| 21 |
+
|
| 22 |
+
for _ in range(num_unsqueezed):
|
| 23 |
+
tensor = tensor.sum(0, keepdim=False)
|
| 24 |
+
for dim in expanded_dims:
|
| 25 |
+
tensor = tensor.sum(dim, keepdim=True)
|
| 26 |
+
return tensor
|
western_ai/usa/pytorch/torch/autograd/anomaly_mode.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
r"""Autograd anomaly mode."""
|
| 3 |
+
|
| 4 |
+
import warnings
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
__all__ = ["detect_anomaly", "set_detect_anomaly"]
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class detect_anomaly:
|
| 13 |
+
r"""Context-manager that enable anomaly detection for the autograd engine.
|
| 14 |
+
|
| 15 |
+
This does two things:
|
| 16 |
+
|
| 17 |
+
- Running the forward pass with detection enabled will allow the backward
|
| 18 |
+
pass to print the traceback of the forward operation that created the failing
|
| 19 |
+
backward function.
|
| 20 |
+
- If ``check_nan`` is ``True``, any backward computation that generate "nan"
|
| 21 |
+
value will raise an error. Default ``True``.
|
| 22 |
+
|
| 23 |
+
.. warning::
|
| 24 |
+
This mode should be enabled only for debugging as the different tests
|
| 25 |
+
will slow down your program execution.
|
| 26 |
+
|
| 27 |
+
Example:
|
| 28 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_ANOMALY)
|
| 29 |
+
>>> import torch
|
| 30 |
+
>>> from torch import autograd
|
| 31 |
+
>>> class MyFunc(autograd.Function):
|
| 32 |
+
... @staticmethod
|
| 33 |
+
... def forward(ctx, inp):
|
| 34 |
+
... return inp.clone()
|
| 35 |
+
...
|
| 36 |
+
... @staticmethod
|
| 37 |
+
... def backward(ctx, gO):
|
| 38 |
+
... # Error during the backward pass
|
| 39 |
+
... raise RuntimeError("Some error in backward")
|
| 40 |
+
... return gO.clone()
|
| 41 |
+
>>> def run_fn(a):
|
| 42 |
+
... out = MyFunc.apply(a)
|
| 43 |
+
... return out.sum()
|
| 44 |
+
>>> inp = torch.rand(10, 10, requires_grad=True)
|
| 45 |
+
>>> out = run_fn(inp)
|
| 46 |
+
>>> out.backward()
|
| 47 |
+
Traceback (most recent call last):
|
| 48 |
+
File "<stdin>", line 1, in <module>
|
| 49 |
+
File "/your/pytorch/install/torch/_tensor.py", line 93, in backward
|
| 50 |
+
torch.autograd.backward(self, gradient, retain_graph, create_graph)
|
| 51 |
+
File "/your/pytorch/install/torch/autograd/__init__.py", line 90, in backward
|
| 52 |
+
allow_unreachable=True) # allow_unreachable flag
|
| 53 |
+
File "/your/pytorch/install/torch/autograd/function.py", line 76, in apply
|
| 54 |
+
return self._forward_cls.backward(self, *args)
|
| 55 |
+
File "<stdin>", line 8, in backward
|
| 56 |
+
RuntimeError: Some error in backward
|
| 57 |
+
>>> with autograd.detect_anomaly():
|
| 58 |
+
... inp = torch.rand(10, 10, requires_grad=True)
|
| 59 |
+
... out = run_fn(inp)
|
| 60 |
+
... out.backward()
|
| 61 |
+
Traceback of forward call that caused the error:
|
| 62 |
+
File "tmp.py", line 53, in <module>
|
| 63 |
+
out = run_fn(inp)
|
| 64 |
+
File "tmp.py", line 44, in run_fn
|
| 65 |
+
out = MyFunc.apply(a)
|
| 66 |
+
Traceback (most recent call last):
|
| 67 |
+
File "<stdin>", line 4, in <module>
|
| 68 |
+
File "/your/pytorch/install/torch/_tensor.py", line 93, in backward
|
| 69 |
+
torch.autograd.backward(self, gradient, retain_graph, create_graph)
|
| 70 |
+
File "/your/pytorch/install/torch/autograd/__init__.py", line 90, in backward
|
| 71 |
+
allow_unreachable=True) # allow_unreachable flag
|
| 72 |
+
File "/your/pytorch/install/torch/autograd/function.py", line 76, in apply
|
| 73 |
+
return self._forward_cls.backward(self, *args)
|
| 74 |
+
File "<stdin>", line 8, in backward
|
| 75 |
+
RuntimeError: Some error in backward
|
| 76 |
+
|
| 77 |
+
"""
|
| 78 |
+
|
| 79 |
+
def __init__(self, check_nan=True) -> None:
|
| 80 |
+
self.prev = torch.is_anomaly_enabled()
|
| 81 |
+
self.check_nan = check_nan
|
| 82 |
+
self.prev_check_nan = torch.is_anomaly_check_nan_enabled()
|
| 83 |
+
warnings.warn(
|
| 84 |
+
"Anomaly Detection has been enabled. "
|
| 85 |
+
"This mode will increase the runtime "
|
| 86 |
+
"and should only be enabled for debugging.",
|
| 87 |
+
stacklevel=2,
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
def __enter__(self) -> None:
|
| 91 |
+
torch.set_anomaly_enabled(True, self.check_nan)
|
| 92 |
+
|
| 93 |
+
def __exit__(self, *args: object) -> None:
|
| 94 |
+
torch.set_anomaly_enabled(self.prev, self.prev_check_nan)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
class set_detect_anomaly:
|
| 98 |
+
r"""Context-manager that sets the anomaly detection for the autograd engine on or off.
|
| 99 |
+
|
| 100 |
+
``set_detect_anomaly`` will enable or disable the autograd anomaly detection
|
| 101 |
+
based on its argument :attr:`mode`.
|
| 102 |
+
It can be used as a context-manager or as a function.
|
| 103 |
+
|
| 104 |
+
See ``detect_anomaly`` above for details of the anomaly detection behaviour.
|
| 105 |
+
|
| 106 |
+
Args:
|
| 107 |
+
mode (bool): Flag whether to enable anomaly detection (``True``),
|
| 108 |
+
or disable (``False``).
|
| 109 |
+
check_nan (bool): Flag whether to raise an error when the backward
|
| 110 |
+
generate "nan"
|
| 111 |
+
|
| 112 |
+
"""
|
| 113 |
+
|
| 114 |
+
def __init__(self, mode: bool, check_nan: bool = True) -> None:
|
| 115 |
+
self.prev = torch.is_anomaly_enabled()
|
| 116 |
+
self.prev_check_nan = torch.is_anomaly_check_nan_enabled()
|
| 117 |
+
torch.set_anomaly_enabled(mode, check_nan)
|
| 118 |
+
|
| 119 |
+
def __enter__(self) -> None:
|
| 120 |
+
pass
|
| 121 |
+
|
| 122 |
+
def __exit__(self, *args: object) -> None:
|
| 123 |
+
torch.set_anomaly_enabled(self.prev, self.prev_check_nan)
|
western_ai/usa/pytorch/torch/autograd/forward_ad.py
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import os
|
| 3 |
+
from typing import Any, NamedTuple
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
from .grad_mode import _DecoratorContextManager
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
__all__ = [
|
| 11 |
+
"UnpackedDualTensor",
|
| 12 |
+
"enter_dual_level",
|
| 13 |
+
"exit_dual_level",
|
| 14 |
+
"make_dual",
|
| 15 |
+
"unpack_dual",
|
| 16 |
+
"dual_level",
|
| 17 |
+
]
|
| 18 |
+
|
| 19 |
+
# Global variable used to make the python API simpler to use
|
| 20 |
+
_current_level = -1
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def enter_dual_level():
|
| 24 |
+
r"""Enter a new forward grad level.
|
| 25 |
+
|
| 26 |
+
This level can be used to make and unpack dual Tensors to compute
|
| 27 |
+
forward gradients.
|
| 28 |
+
|
| 29 |
+
This function also updates the current level that is used by default
|
| 30 |
+
by the other functions in this API.
|
| 31 |
+
"""
|
| 32 |
+
from torch._functorch.predispatch import (
|
| 33 |
+
_enter_dual_level as _predispatch_enter_dual_level,
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
global _current_level
|
| 37 |
+
new_level = _predispatch_enter_dual_level()
|
| 38 |
+
if new_level != _current_level + 1:
|
| 39 |
+
raise RuntimeError(
|
| 40 |
+
"Entering a new forward AD level but the current level "
|
| 41 |
+
"is not valid. Make sure you did not modify it directly."
|
| 42 |
+
)
|
| 43 |
+
_current_level = new_level
|
| 44 |
+
return new_level
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def exit_dual_level(*, level=None):
|
| 48 |
+
r"""Exit a forward grad level.
|
| 49 |
+
|
| 50 |
+
This function deletes all the gradients associated with this
|
| 51 |
+
level. Only deleting the latest entered level is allowed.
|
| 52 |
+
|
| 53 |
+
This function also updates the current level that is used by default
|
| 54 |
+
by the other functions in this API.
|
| 55 |
+
"""
|
| 56 |
+
global _current_level
|
| 57 |
+
if level is None:
|
| 58 |
+
level = _current_level
|
| 59 |
+
if level != _current_level:
|
| 60 |
+
raise RuntimeError(
|
| 61 |
+
"Trying to exit a forward AD level that was not the last one "
|
| 62 |
+
"that was created. This is not supported."
|
| 63 |
+
)
|
| 64 |
+
from torch._functorch.predispatch import (
|
| 65 |
+
_exit_dual_level as _predispatch_exit_dual_level,
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
_predispatch_exit_dual_level(level=level)
|
| 69 |
+
_current_level = level - 1
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _maybe_load_decompositions():
|
| 73 |
+
if os.environ.get("PYTORCH_JIT", "1") == "1" and __debug__:
|
| 74 |
+
from torch._decomp import decompositions_for_jvp # noqa: F401
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def make_dual(tensor, tangent, *, level=None):
|
| 78 |
+
r"""Associate a tensor value with its tangent to create a "dual tensor" for forward AD gradient computation.
|
| 79 |
+
|
| 80 |
+
The result is a new tensor aliased to :attr:`tensor` with :attr:`tangent` embedded
|
| 81 |
+
as an attribute as-is if it has the same storage layout or copied otherwise.
|
| 82 |
+
The tangent attribute can be recovered with :func:`unpack_dual`.
|
| 83 |
+
|
| 84 |
+
This function is backward differentiable.
|
| 85 |
+
|
| 86 |
+
Given a function `f` whose jacobian is `J`, it allows one to compute the Jacobian-vector product (`jvp`)
|
| 87 |
+
between `J` and a given vector `v` as follows.
|
| 88 |
+
|
| 89 |
+
Example::
|
| 90 |
+
|
| 91 |
+
>>> # xdoctest: +SKIP("Undefined variables")
|
| 92 |
+
>>> with dual_level():
|
| 93 |
+
... inp = make_dual(x, v)
|
| 94 |
+
... out = f(inp)
|
| 95 |
+
... y, jvp = unpack_dual(out)
|
| 96 |
+
|
| 97 |
+
Please see the `forward-mode AD tutorial <https://pytorch.org/tutorials/intermediate/forward_ad_usage.html>`__
|
| 98 |
+
for detailed steps on how to use this API.
|
| 99 |
+
|
| 100 |
+
"""
|
| 101 |
+
# See NOTE: [forward-mode AD decompositions mechanism]
|
| 102 |
+
#
|
| 103 |
+
# Import from torch._decomp import decompositions_for_jvp to register
|
| 104 |
+
# decompositions for jvp to the jit registry
|
| 105 |
+
#
|
| 106 |
+
# FIXME: We specify that __debug__ must be True because
|
| 107 |
+
# if python is run with -OO or -O flags (i.e., __debug__ is False), we encounter the
|
| 108 |
+
# following error:
|
| 109 |
+
#
|
| 110 |
+
# Return value was annotated as having type Tuple[NoneType, NoneType] but is actually of
|
| 111 |
+
# type Tuple[Tensor, Tensor]:
|
| 112 |
+
# File ".../torch/_decomp/__init__.py", line 1585
|
| 113 |
+
# else:
|
| 114 |
+
# buffer = z
|
| 115 |
+
# return min - torch.log1p(z), buffer
|
| 116 |
+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE
|
| 117 |
+
_maybe_load_decompositions()
|
| 118 |
+
|
| 119 |
+
if level is None:
|
| 120 |
+
level = _current_level
|
| 121 |
+
|
| 122 |
+
if level < 0:
|
| 123 |
+
raise RuntimeError(
|
| 124 |
+
"Trying to create a dual Tensor for forward AD but no level "
|
| 125 |
+
"exists, make sure to enter_dual_level() first."
|
| 126 |
+
)
|
| 127 |
+
if not (tensor.is_floating_point() or tensor.is_complex()):
|
| 128 |
+
raise ValueError(
|
| 129 |
+
f"Expected primal to be floating point or complex, but got: {tensor.dtype}"
|
| 130 |
+
)
|
| 131 |
+
if not (tangent.is_floating_point() or tangent.is_complex()):
|
| 132 |
+
raise ValueError(
|
| 133 |
+
f"Expected tangent to be floating point or complex, but got: {tangent.dtype}"
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
from torch._functorch.predispatch import _make_dual as _predispatch_make_dual
|
| 137 |
+
|
| 138 |
+
return _predispatch_make_dual(tensor, tangent, level=level)
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
class UnpackedDualTensor(NamedTuple):
|
| 142 |
+
r"""Namedtuple returned by :func:`unpack_dual` containing the primal and tangent components of the dual tensor.
|
| 143 |
+
|
| 144 |
+
See :func:`unpack_dual` for more details.
|
| 145 |
+
"""
|
| 146 |
+
|
| 147 |
+
primal: torch.Tensor
|
| 148 |
+
tangent: torch.Tensor | None
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def unpack_dual(tensor, *, level=None):
|
| 152 |
+
r"""Unpack a "dual tensor" to get both its Tensor value and its forward AD gradient.
|
| 153 |
+
|
| 154 |
+
The result is a namedtuple ``(primal, tangent)`` where ``primal`` is a view of
|
| 155 |
+
:attr:`tensor`'s primal and ``tangent`` is :attr:`tensor`'s tangent as-is.
|
| 156 |
+
Neither of these tensors can be dual tensor of level :attr:`level`.
|
| 157 |
+
|
| 158 |
+
This function is backward differentiable.
|
| 159 |
+
|
| 160 |
+
Example::
|
| 161 |
+
|
| 162 |
+
>>> # xdoctest: +SKIP("Undefined variables")
|
| 163 |
+
>>> with dual_level():
|
| 164 |
+
... inp = make_dual(x, x_t)
|
| 165 |
+
... out = f(inp)
|
| 166 |
+
... y, jvp = unpack_dual(out)
|
| 167 |
+
... jvp = unpack_dual(out).tangent
|
| 168 |
+
|
| 169 |
+
Please see the `forward-mode AD tutorial <https://pytorch.org/tutorials/intermediate/forward_ad_usage.html>`__
|
| 170 |
+
for detailed steps on how to use this API.
|
| 171 |
+
"""
|
| 172 |
+
if level is None:
|
| 173 |
+
level = _current_level
|
| 174 |
+
|
| 175 |
+
if level < 0:
|
| 176 |
+
return UnpackedDualTensor(tensor, None)
|
| 177 |
+
|
| 178 |
+
from torch._functorch.predispatch import _unpack_dual as _predispatch_unpack_dual
|
| 179 |
+
|
| 180 |
+
primal, dual = _predispatch_unpack_dual(tensor, level=level)
|
| 181 |
+
|
| 182 |
+
return UnpackedDualTensor(primal, dual)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
class dual_level(_DecoratorContextManager):
|
| 186 |
+
r"""Context-manager for forward AD, where all forward AD computation must occur within the ``dual_level`` context.
|
| 187 |
+
|
| 188 |
+
.. Note::
|
| 189 |
+
|
| 190 |
+
The ``dual_level`` context appropriately enters and exit the dual level to
|
| 191 |
+
controls the current forward AD level, which is used by default by the other
|
| 192 |
+
functions in this API.
|
| 193 |
+
|
| 194 |
+
We currently don't plan to support nested ``dual_level`` contexts, however, so
|
| 195 |
+
only a single forward AD level is supported. To compute higher-order
|
| 196 |
+
forward grads, one can use :func:`torch.func.jvp`.
|
| 197 |
+
|
| 198 |
+
Example::
|
| 199 |
+
|
| 200 |
+
>>> # xdoctest: +SKIP("Undefined variables")
|
| 201 |
+
>>> x = torch.tensor([1])
|
| 202 |
+
>>> x_t = torch.tensor([1])
|
| 203 |
+
>>> with dual_level():
|
| 204 |
+
... inp = make_dual(x, x_t)
|
| 205 |
+
... # Do computations with inp
|
| 206 |
+
... out = your_fn(inp)
|
| 207 |
+
... _, grad = unpack_dual(out)
|
| 208 |
+
>>> grad is None
|
| 209 |
+
False
|
| 210 |
+
>>> # After exiting the level, the grad is deleted
|
| 211 |
+
>>> _, grad_after = unpack_dual(out)
|
| 212 |
+
>>> grad is None
|
| 213 |
+
True
|
| 214 |
+
|
| 215 |
+
Please see the `forward-mode AD tutorial <https://pytorch.org/tutorials/intermediate/forward_ad_usage.html>`__
|
| 216 |
+
for detailed steps on how to use this API.
|
| 217 |
+
"""
|
| 218 |
+
|
| 219 |
+
def __enter__(self):
|
| 220 |
+
return enter_dual_level()
|
| 221 |
+
|
| 222 |
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
| 223 |
+
exit_dual_level()
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
# Private helper functions
|
| 227 |
+
_is_fwd_grad_enabled = torch._C._is_fwd_grad_enabled
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
# Private helper function to enable or disable fwd grad.
|
| 231 |
+
# If you're a user and want to use this, please file an issue to discuss the use case.
|
| 232 |
+
class _set_fwd_grad_enabled(_DecoratorContextManager):
|
| 233 |
+
def __init__(self, mode: bool) -> None:
|
| 234 |
+
self.prev = _is_fwd_grad_enabled()
|
| 235 |
+
torch._C._set_fwd_grad_enabled(mode)
|
| 236 |
+
|
| 237 |
+
def __enter__(self) -> None:
|
| 238 |
+
pass
|
| 239 |
+
|
| 240 |
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
| 241 |
+
torch._C._set_fwd_grad_enabled(self.prev)
|
western_ai/usa/pytorch/torch/autograd/function.py
ADDED
|
@@ -0,0 +1,898 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import functools
|
| 3 |
+
import inspect
|
| 4 |
+
import itertools
|
| 5 |
+
import warnings
|
| 6 |
+
from collections import OrderedDict
|
| 7 |
+
from collections.abc import Callable
|
| 8 |
+
from typing import Any, Concatenate, TypeVar
|
| 9 |
+
from typing_extensions import deprecated, ParamSpec
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
import torch._C as _C
|
| 13 |
+
import torch._functorch as _functorch
|
| 14 |
+
import torch.utils.hooks as hooks
|
| 15 |
+
from torch._C import _functions
|
| 16 |
+
from torch._functorch.autograd_function import custom_function_call
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
__all__ = [
|
| 20 |
+
"FunctionCtx",
|
| 21 |
+
"BackwardCFunction",
|
| 22 |
+
"FunctionMeta",
|
| 23 |
+
"Function",
|
| 24 |
+
"once_differentiable",
|
| 25 |
+
"InplaceFunction",
|
| 26 |
+
"NestedIOFunction",
|
| 27 |
+
]
|
| 28 |
+
|
| 29 |
+
# Unique id provider for each class inheriting from Function
|
| 30 |
+
# This is incremented in FunctionMeta during class definition
|
| 31 |
+
AUTOGRAD_FUNCTION_COUNTER = itertools.count()
|
| 32 |
+
|
| 33 |
+
_T = TypeVar("_T")
|
| 34 |
+
_R = TypeVar("_R")
|
| 35 |
+
_P = ParamSpec("_P")
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
# Formerly known as: _ContextMethodMixin
|
| 39 |
+
class FunctionCtx:
|
| 40 |
+
def save_for_backward(self, *tensors: torch.Tensor):
|
| 41 |
+
r"""Save given tensors for a future call to :func:`~Function.backward`.
|
| 42 |
+
|
| 43 |
+
``save_for_backward`` should be called at most once, in either the
|
| 44 |
+
:func:`setup_context` or :func:`forward` methods, and only with tensors.
|
| 45 |
+
|
| 46 |
+
All tensors intended to be used in the backward pass should be saved
|
| 47 |
+
with ``save_for_backward`` (as opposed to directly on ``ctx``) to prevent
|
| 48 |
+
incorrect gradients and memory leaks, and enable the application of saved
|
| 49 |
+
tensor hooks. See :class:`torch.autograd.graph.saved_tensors_hooks`.
|
| 50 |
+
See :ref:`extending-autograd` for more details.
|
| 51 |
+
|
| 52 |
+
Note that if intermediary tensors, tensors that are neither inputs
|
| 53 |
+
nor outputs of :func:`forward`, are saved for backward, your custom Function
|
| 54 |
+
may not support double backward.
|
| 55 |
+
Custom Functions that do not support double backward should decorate their
|
| 56 |
+
:func:`backward` method with ``@once_differentiable`` so that performing
|
| 57 |
+
double backward raises an error. If you'd like to support double backward,
|
| 58 |
+
you can either recompute intermediaries based on the inputs during backward
|
| 59 |
+
or return the intermediaries as the outputs of the custom Function. See the
|
| 60 |
+
`double backward tutorial <https://pytorch.org/tutorials/intermediate/custom_function_double_backward_tutorial.html>`_
|
| 61 |
+
for more details.
|
| 62 |
+
|
| 63 |
+
In :func:`backward`, saved tensors can be accessed through the :attr:`saved_tensors`
|
| 64 |
+
attribute. Before returning them to the user, a check is made to ensure
|
| 65 |
+
they weren't used in any in-place operation that modified their content.
|
| 66 |
+
|
| 67 |
+
Arguments can also be ``None``. This is a no-op.
|
| 68 |
+
|
| 69 |
+
See :ref:`extending-autograd` for more details on how to use this method.
|
| 70 |
+
|
| 71 |
+
Example::
|
| 72 |
+
|
| 73 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 74 |
+
>>> class Func(Function):
|
| 75 |
+
>>> @staticmethod
|
| 76 |
+
>>> def forward(ctx, x: torch.Tensor, y: torch.Tensor, z: int):
|
| 77 |
+
>>> w = x * z
|
| 78 |
+
>>> out = x * y + y * z + w * y
|
| 79 |
+
>>> ctx.save_for_backward(x, y, w, out)
|
| 80 |
+
>>> ctx.z = z # z is not a tensor
|
| 81 |
+
>>> return out
|
| 82 |
+
>>>
|
| 83 |
+
>>> @staticmethod
|
| 84 |
+
>>> @once_differentiable
|
| 85 |
+
>>> def backward(ctx, grad_out):
|
| 86 |
+
>>> x, y, w, out = ctx.saved_tensors
|
| 87 |
+
>>> z = ctx.z
|
| 88 |
+
>>> gx = grad_out * (y + y * z)
|
| 89 |
+
>>> gy = grad_out * (x + z + w)
|
| 90 |
+
>>> gz = None
|
| 91 |
+
>>> return gx, gy, gz
|
| 92 |
+
>>>
|
| 93 |
+
>>> a = torch.tensor(1., requires_grad=True, dtype=torch.double)
|
| 94 |
+
>>> b = torch.tensor(2., requires_grad=True, dtype=torch.double)
|
| 95 |
+
>>> c = 4
|
| 96 |
+
>>> d = Func.apply(a, b, c)
|
| 97 |
+
|
| 98 |
+
"""
|
| 99 |
+
self.to_save = tensors
|
| 100 |
+
|
| 101 |
+
def save_for_forward(self, *tensors: torch.Tensor):
|
| 102 |
+
r"""Save given tensors for a future call to :func:`~Function.jvp`.
|
| 103 |
+
|
| 104 |
+
``save_for_forward`` should be called at most once, in either the
|
| 105 |
+
:func:`setup_context` or :func:`forward` methods, and all arguments
|
| 106 |
+
should be tensors.
|
| 107 |
+
|
| 108 |
+
In :func:`jvp`, saved objects can be accessed through the :attr:`saved_tensors`
|
| 109 |
+
attribute.
|
| 110 |
+
|
| 111 |
+
Arguments can also be ``None``. This is a no-op.
|
| 112 |
+
|
| 113 |
+
See :ref:`extending-autograd` for more details on how to use this method.
|
| 114 |
+
|
| 115 |
+
Example::
|
| 116 |
+
|
| 117 |
+
>>> # xdoctest: +SKIP
|
| 118 |
+
>>> class Func(torch.autograd.Function):
|
| 119 |
+
>>> @staticmethod
|
| 120 |
+
>>> def forward(ctx, x: torch.Tensor, y: torch.Tensor, z: int):
|
| 121 |
+
>>> ctx.save_for_backward(x, y)
|
| 122 |
+
>>> ctx.save_for_forward(x, y)
|
| 123 |
+
>>> ctx.z = z
|
| 124 |
+
>>> return x * y * z
|
| 125 |
+
>>>
|
| 126 |
+
>>> @staticmethod
|
| 127 |
+
>>> def jvp(ctx, x_t, y_t, _):
|
| 128 |
+
>>> x, y = ctx.saved_tensors
|
| 129 |
+
>>> z = ctx.z
|
| 130 |
+
>>> return z * (y * x_t + x * y_t)
|
| 131 |
+
>>>
|
| 132 |
+
>>> @staticmethod
|
| 133 |
+
>>> def vjp(ctx, grad_out):
|
| 134 |
+
>>> x, y = ctx.saved_tensors
|
| 135 |
+
>>> z = ctx.z
|
| 136 |
+
>>> return z * grad_out * y, z * grad_out * x, None
|
| 137 |
+
>>>
|
| 138 |
+
>>> a = torch.tensor(1., requires_grad=True, dtype=torch.double)
|
| 139 |
+
>>> t = torch.tensor(1., dtype=torch.double)
|
| 140 |
+
>>> b = torch.tensor(2., requires_grad=True, dtype=torch.double)
|
| 141 |
+
>>> c = 4
|
| 142 |
+
>>>
|
| 143 |
+
>>> with fwAD.dual_level():
|
| 144 |
+
>>> a_dual = fwAD.make_dual(a, t)
|
| 145 |
+
>>> d = Func.apply(a_dual, b, c)
|
| 146 |
+
|
| 147 |
+
"""
|
| 148 |
+
for tensor in tensors:
|
| 149 |
+
if not (isinstance(tensor, torch.Tensor) or tensor is None):
|
| 150 |
+
raise AssertionError(
|
| 151 |
+
"save_for_forward expects all arguments to be tensors; you should "
|
| 152 |
+
"save non-tensors as attributes on ctx."
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
self.saved_for_forward = tensors
|
| 156 |
+
|
| 157 |
+
def mark_dirty(self, *args: torch.Tensor):
|
| 158 |
+
r"""Mark given tensors as modified in an in-place operation.
|
| 159 |
+
|
| 160 |
+
This should be called at most once, in either the :func:`setup_context`
|
| 161 |
+
or :func:`forward` methods, and all arguments should be inputs.
|
| 162 |
+
|
| 163 |
+
Every tensor that's been modified in-place in a call to :func:`forward`
|
| 164 |
+
should be given to this function, to ensure correctness of our checks.
|
| 165 |
+
It doesn't matter whether the function is called before or after
|
| 166 |
+
modification.
|
| 167 |
+
|
| 168 |
+
Examples::
|
| 169 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 170 |
+
>>> class Inplace(Function):
|
| 171 |
+
>>> @staticmethod
|
| 172 |
+
>>> def forward(ctx, x):
|
| 173 |
+
>>> x_npy = x.numpy() # x_npy shares storage with x
|
| 174 |
+
>>> x_npy += 1
|
| 175 |
+
>>> ctx.mark_dirty(x)
|
| 176 |
+
>>> return x
|
| 177 |
+
>>>
|
| 178 |
+
>>> @staticmethod
|
| 179 |
+
>>> @once_differentiable
|
| 180 |
+
>>> def backward(ctx, grad_output):
|
| 181 |
+
>>> return grad_output
|
| 182 |
+
>>>
|
| 183 |
+
>>> a = torch.tensor(1., requires_grad=True, dtype=torch.double).clone()
|
| 184 |
+
>>> b = a * a
|
| 185 |
+
>>> Inplace.apply(a) # This would lead to wrong gradients!
|
| 186 |
+
>>> # but the engine would not know unless we mark_dirty
|
| 187 |
+
>>> # xdoctest: +SKIP
|
| 188 |
+
>>> b.backward() # RuntimeError: one of the variables needed for gradient
|
| 189 |
+
>>> # computation has been modified by an inplace operation
|
| 190 |
+
|
| 191 |
+
"""
|
| 192 |
+
self.dirty_tensors = args
|
| 193 |
+
|
| 194 |
+
@deprecated(
|
| 195 |
+
"`mark_shared_storage` is deprecated. "
|
| 196 |
+
"Tensors with shared storages are automatically tracked. "
|
| 197 |
+
"Note that calls to `set_()` are not tracked",
|
| 198 |
+
category=FutureWarning,
|
| 199 |
+
)
|
| 200 |
+
def mark_shared_storage(self, *pairs):
|
| 201 |
+
pass
|
| 202 |
+
|
| 203 |
+
def mark_non_differentiable(self, *args: torch.Tensor):
|
| 204 |
+
r"""Mark outputs as non-differentiable.
|
| 205 |
+
|
| 206 |
+
This should be called at most once, in either the :func:`setup_context`
|
| 207 |
+
or :func:`forward` methods, and all arguments should be tensor outputs.
|
| 208 |
+
|
| 209 |
+
This will mark outputs as not requiring gradients, increasing the
|
| 210 |
+
efficiency of backward computation. You still need to accept a gradient
|
| 211 |
+
for each output in :meth:`~Function.backward`, but it's always going to
|
| 212 |
+
be a zero tensor with the same shape as the shape of a corresponding
|
| 213 |
+
output.
|
| 214 |
+
|
| 215 |
+
This is used e.g. for indices returned from a sort. See example::
|
| 216 |
+
>>> class Func(Function):
|
| 217 |
+
>>> @staticmethod
|
| 218 |
+
>>> def forward(ctx, x):
|
| 219 |
+
>>> sorted, idx = x.sort()
|
| 220 |
+
>>> ctx.mark_non_differentiable(idx)
|
| 221 |
+
>>> ctx.save_for_backward(x, idx)
|
| 222 |
+
>>> return sorted, idx
|
| 223 |
+
>>>
|
| 224 |
+
>>> @staticmethod
|
| 225 |
+
>>> @once_differentiable
|
| 226 |
+
>>> def backward(ctx, g1, g2): # still need to accept g2
|
| 227 |
+
>>> x, idx = ctx.saved_tensors
|
| 228 |
+
>>> grad_input = torch.zeros_like(x)
|
| 229 |
+
>>> grad_input.index_add_(0, idx, g1)
|
| 230 |
+
>>> return grad_input
|
| 231 |
+
|
| 232 |
+
"""
|
| 233 |
+
self.non_differentiable = args
|
| 234 |
+
|
| 235 |
+
def set_materialize_grads(self, value: bool):
|
| 236 |
+
r"""Set whether to materialize grad tensors. Default is ``True``.
|
| 237 |
+
|
| 238 |
+
This should be called only from either the :func:`setup_context` or
|
| 239 |
+
:func:`forward` methods.
|
| 240 |
+
|
| 241 |
+
If ``True``, undefined grad tensors will be expanded to tensors full of zeros
|
| 242 |
+
prior to calling the :func:`backward` and :func:`jvp` methods.
|
| 243 |
+
|
| 244 |
+
Example::
|
| 245 |
+
|
| 246 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 247 |
+
>>> class SimpleFunc(Function):
|
| 248 |
+
>>> @staticmethod
|
| 249 |
+
>>> def forward(ctx, x):
|
| 250 |
+
>>> return x.clone(), x.clone()
|
| 251 |
+
>>>
|
| 252 |
+
>>> @staticmethod
|
| 253 |
+
>>> @once_differentiable
|
| 254 |
+
>>> def backward(ctx, g1, g2):
|
| 255 |
+
>>> return g1 + g2 # No check for None necessary
|
| 256 |
+
>>>
|
| 257 |
+
>>> # We modify SimpleFunc to handle non-materialized grad outputs
|
| 258 |
+
>>> class Func(Function):
|
| 259 |
+
>>> @staticmethod
|
| 260 |
+
>>> def forward(ctx, x):
|
| 261 |
+
>>> ctx.set_materialize_grads(False)
|
| 262 |
+
>>> ctx.save_for_backward(x)
|
| 263 |
+
>>> return x.clone(), x.clone()
|
| 264 |
+
>>>
|
| 265 |
+
>>> @staticmethod
|
| 266 |
+
>>> @once_differentiable
|
| 267 |
+
>>> def backward(ctx, g1, g2):
|
| 268 |
+
>>> x, = ctx.saved_tensors
|
| 269 |
+
>>> grad_input = torch.zeros_like(x)
|
| 270 |
+
>>> if g1 is not None: # We must check for None now
|
| 271 |
+
>>> grad_input += g1
|
| 272 |
+
>>> if g2 is not None:
|
| 273 |
+
>>> grad_input += g2
|
| 274 |
+
>>> return grad_input
|
| 275 |
+
>>>
|
| 276 |
+
>>> a = torch.tensor(1., requires_grad=True)
|
| 277 |
+
>>> b, _ = Func.apply(a) # induces g2 to be undefined
|
| 278 |
+
|
| 279 |
+
"""
|
| 280 |
+
self.materialize_grads = value
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
# DO NOT USE: This is only defined to be able to load old serialized models
|
| 284 |
+
_ContextMethodMixin = FunctionCtx
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
class _HookMixin:
|
| 288 |
+
@staticmethod
|
| 289 |
+
def _register_hook(backward_hooks, hook):
|
| 290 |
+
if backward_hooks is None:
|
| 291 |
+
backward_hooks = OrderedDict()
|
| 292 |
+
handle = hooks.RemovableHandle(backward_hooks)
|
| 293 |
+
backward_hooks[handle.id] = hook
|
| 294 |
+
return backward_hooks, handle
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
class BackwardCFunction(_C._FunctionBase, FunctionCtx, _HookMixin):
|
| 298 |
+
r"""
|
| 299 |
+
This class is used for internal autograd work. Do not use.
|
| 300 |
+
"""
|
| 301 |
+
|
| 302 |
+
def _get_user_fn(self):
|
| 303 |
+
backward_fn = self._forward_cls.backward # type: ignore[attr-defined]
|
| 304 |
+
vjp_fn = self._forward_cls.vjp # type: ignore[attr-defined]
|
| 305 |
+
if backward_fn is not Function.backward and vjp_fn is not Function.vjp:
|
| 306 |
+
raise RuntimeError(
|
| 307 |
+
"Implementing both 'backward' and 'vjp' for a custom "
|
| 308 |
+
"Function is not allowed. You should only implement one "
|
| 309 |
+
"of them."
|
| 310 |
+
)
|
| 311 |
+
return vjp_fn if vjp_fn is not Function.vjp else backward_fn
|
| 312 |
+
|
| 313 |
+
def apply(self, *args):
|
| 314 |
+
r"""
|
| 315 |
+
Apply method used when executing this Node during the backward.
|
| 316 |
+
|
| 317 |
+
Called by the autograd engine (non-boxed path) and by direct
|
| 318 |
+
grad_fn.apply() calls. When boxed_grads_call is True, boxes
|
| 319 |
+
grads into a mutable list before calling user's backward.
|
| 320 |
+
"""
|
| 321 |
+
user_fn = self._get_user_fn()
|
| 322 |
+
fwd_cls = self._forward_cls # type: ignore[attr-defined] # pyrefly: ignore[missing-attribute]
|
| 323 |
+
if getattr(fwd_cls, "boxed_grads_call", False):
|
| 324 |
+
args = (list(args),)
|
| 325 |
+
return user_fn(self, *args)
|
| 326 |
+
|
| 327 |
+
def apply_boxed(self, *args):
|
| 328 |
+
r"""
|
| 329 |
+
Apply method called by the autograd engine when boxed_grads_call
|
| 330 |
+
is True. Grads arrive as a single mutable list argument, allowing
|
| 331 |
+
backward to free individual grads mid-execution.
|
| 332 |
+
"""
|
| 333 |
+
return self._get_user_fn()(self, *args)
|
| 334 |
+
|
| 335 |
+
def apply_jvp(self, *args):
|
| 336 |
+
r"""
|
| 337 |
+
Apply method used when executing forward mode AD during the forward
|
| 338 |
+
"""
|
| 339 |
+
# _forward_cls is defined by derived class
|
| 340 |
+
return self._forward_cls.jvp(self, *args) # type: ignore[attr-defined]
|
| 341 |
+
|
| 342 |
+
def _compiled_autograd_key(self):
|
| 343 |
+
return self._forward_cls._compiled_autograd_key(self) # type: ignore[attr-defined]
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
class FunctionMeta(type):
|
| 347 |
+
"""Function metaclass.
|
| 348 |
+
|
| 349 |
+
This metaclass sets up the following properties:
|
| 350 |
+
_backward_cls: The Function class corresponding to the differentiated
|
| 351 |
+
version of this function (which is generated on the fly by this
|
| 352 |
+
metaclass).
|
| 353 |
+
"""
|
| 354 |
+
|
| 355 |
+
def __init__(cls, name, bases, attrs):
|
| 356 |
+
backward_fn = type(
|
| 357 |
+
name + "Backward", (BackwardCFunction,), {"_forward_cls": cls}
|
| 358 |
+
)
|
| 359 |
+
backward_fn._autograd_function_id = next(AUTOGRAD_FUNCTION_COUNTER) # type: ignore[attr-defined]
|
| 360 |
+
cls._backward_cls = backward_fn
|
| 361 |
+
|
| 362 |
+
super().__init__(name, bases, attrs)
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
class _SingleLevelFunction(
|
| 366 |
+
_C._FunctionBase, FunctionCtx, _HookMixin, metaclass=FunctionMeta
|
| 367 |
+
):
|
| 368 |
+
@staticmethod
|
| 369 |
+
def forward(*args: Any, **kwargs: Any) -> Any:
|
| 370 |
+
r"""Define the forward of the custom autograd Function.
|
| 371 |
+
|
| 372 |
+
This function is to be overridden by all subclasses.
|
| 373 |
+
There are two ways to define forward:
|
| 374 |
+
|
| 375 |
+
Usage 1 (Combined forward and ctx)::
|
| 376 |
+
|
| 377 |
+
@staticmethod
|
| 378 |
+
def forward(ctx: Any, *args: Any, **kwargs: Any) -> Any:
|
| 379 |
+
pass
|
| 380 |
+
|
| 381 |
+
- It must accept a context ctx as the first argument, followed by any
|
| 382 |
+
number of arguments (tensors or other types).
|
| 383 |
+
- See :ref:`combining-forward-context` for more details
|
| 384 |
+
|
| 385 |
+
Usage 2 (Separate forward and ctx)::
|
| 386 |
+
|
| 387 |
+
@staticmethod
|
| 388 |
+
def forward(*args: Any, **kwargs: Any) -> Any:
|
| 389 |
+
pass
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
@staticmethod
|
| 393 |
+
def setup_context(ctx: Any, inputs: Tuple[Any, ...], output: Any) -> None:
|
| 394 |
+
pass
|
| 395 |
+
|
| 396 |
+
- The forward no longer accepts a ctx argument.
|
| 397 |
+
- Instead, you must also override the :meth:`torch.autograd.Function.setup_context`
|
| 398 |
+
staticmethod to handle setting up the ``ctx`` object.
|
| 399 |
+
``output`` is the output of the forward, ``inputs`` are a Tuple of inputs
|
| 400 |
+
to the forward.
|
| 401 |
+
- See :ref:`extending-autograd` for more details
|
| 402 |
+
|
| 403 |
+
The context can be used to store arbitrary data that can be then
|
| 404 |
+
retrieved during the backward pass. Tensors should not be stored
|
| 405 |
+
directly on `ctx` (though this is not currently enforced for
|
| 406 |
+
backward compatibility). Instead, tensors should be saved either with
|
| 407 |
+
:func:`ctx.save_for_backward` if they are intended to be used in
|
| 408 |
+
``backward`` (equivalently, ``vjp``) or :func:`ctx.save_for_forward`
|
| 409 |
+
if they are intended to be used for in ``jvp``.
|
| 410 |
+
"""
|
| 411 |
+
raise NotImplementedError(
|
| 412 |
+
"You must implement the forward function for custom autograd.Function."
|
| 413 |
+
)
|
| 414 |
+
|
| 415 |
+
@staticmethod
|
| 416 |
+
def setup_context(ctx: Any, inputs: tuple[Any, ...], output: Any) -> Any:
|
| 417 |
+
r"""There are two ways to define the forward pass of an autograd.Function.
|
| 418 |
+
|
| 419 |
+
Either:
|
| 420 |
+
|
| 421 |
+
1. Override forward with the signature ``forward(ctx, *args, **kwargs)``.
|
| 422 |
+
``setup_context`` is not overridden. Setting up the ctx for backward
|
| 423 |
+
happens inside the ``forward``.
|
| 424 |
+
2. Override forward with the signature ``forward(*args, **kwargs)`` and
|
| 425 |
+
override ``setup_context``. Setting up the ctx for backward happens
|
| 426 |
+
inside ``setup_context`` (as opposed to inside the ``forward``)
|
| 427 |
+
|
| 428 |
+
See :meth:`torch.autograd.Function.forward` and :ref:`extending-autograd` for more details.
|
| 429 |
+
"""
|
| 430 |
+
raise NotImplementedError("setup_context is not implemented.")
|
| 431 |
+
|
| 432 |
+
@staticmethod
|
| 433 |
+
def backward(ctx: Any, *grad_outputs: Any) -> Any:
|
| 434 |
+
r"""Define a formula for differentiating the operation with backward mode automatic differentiation.
|
| 435 |
+
|
| 436 |
+
This function is to be overridden by all subclasses.
|
| 437 |
+
(Defining this function is equivalent to defining the ``vjp`` function.)
|
| 438 |
+
|
| 439 |
+
It must accept a context :attr:`ctx` as the first argument, followed by
|
| 440 |
+
as many outputs as the :func:`forward` returned (None will be passed in
|
| 441 |
+
for non tensor outputs of the forward function),
|
| 442 |
+
and it should return as many tensors, as there were inputs to
|
| 443 |
+
:func:`forward`. Each argument is the gradient w.r.t the given output,
|
| 444 |
+
and each returned value should be the gradient w.r.t. the
|
| 445 |
+
corresponding input. If an input is not a Tensor or is a Tensor not
|
| 446 |
+
requiring grads, you can just pass None as a gradient for that input.
|
| 447 |
+
|
| 448 |
+
The context can be used to retrieve tensors saved during the forward
|
| 449 |
+
pass. It also has an attribute :attr:`ctx.needs_input_grad` as a tuple
|
| 450 |
+
of booleans representing whether each input needs gradient. E.g.,
|
| 451 |
+
:func:`backward` will have ``ctx.needs_input_grad[0] = True`` if the
|
| 452 |
+
first input to :func:`forward` needs gradient computed w.r.t. the
|
| 453 |
+
output.
|
| 454 |
+
"""
|
| 455 |
+
raise NotImplementedError(
|
| 456 |
+
"You must implement either the backward or vjp method for "
|
| 457 |
+
"your custom autograd.Function to use it with backward "
|
| 458 |
+
"mode AD."
|
| 459 |
+
)
|
| 460 |
+
|
| 461 |
+
# vjp and backward are alias of each other
|
| 462 |
+
vjp = backward
|
| 463 |
+
|
| 464 |
+
"""
|
| 465 |
+
Bool that specifies if PyTorch should clear saved tensors after the first
|
| 466 |
+
access to ``ctx.saved_tensors``. When set to True, accessing saved_tensors
|
| 467 |
+
clears the internal references, allowing the tensors to be cleared as soon
|
| 468 |
+
as the Tensor returned by saved_tensors is deleted.
|
| 469 |
+
|
| 470 |
+
This is useful for reducing memory pressure in backward passes when you
|
| 471 |
+
only need to access saved tensors once.
|
| 472 |
+
|
| 473 |
+
Default is False.
|
| 474 |
+
"""
|
| 475 |
+
clear_saved_tensors_on_access = False
|
| 476 |
+
|
| 477 |
+
"""
|
| 478 |
+
Bool that specifies if backward should receive grads as a single mutable
|
| 479 |
+
list argument instead of individual args in an immutable tuple. This allows
|
| 480 |
+
backward to free individual grads mid-execution by removing them from the
|
| 481 |
+
list, reducing peak memory.
|
| 482 |
+
|
| 483 |
+
When True, ``backward(ctx, grads)`` receives a single list instead of
|
| 484 |
+
``backward(ctx, *grads)``.
|
| 485 |
+
|
| 486 |
+
Default is False.
|
| 487 |
+
"""
|
| 488 |
+
boxed_grads_call = False
|
| 489 |
+
|
| 490 |
+
@staticmethod
|
| 491 |
+
def jvp(ctx: Any, *grad_inputs: Any) -> Any:
|
| 492 |
+
r"""Define a formula for differentiating the operation with forward mode automatic differentiation.
|
| 493 |
+
|
| 494 |
+
This function is to be overridden by all subclasses.
|
| 495 |
+
It must accept a context :attr:`ctx` as the first argument, followed by
|
| 496 |
+
as many inputs as the :func:`forward` got (None will be passed in
|
| 497 |
+
for non tensor inputs of the forward function),
|
| 498 |
+
and it should return as many tensors as there were outputs to
|
| 499 |
+
:func:`forward`. Each argument is the gradient w.r.t the given input,
|
| 500 |
+
and each returned value should be the gradient w.r.t. the
|
| 501 |
+
corresponding output. If an output is not a Tensor or the function is not
|
| 502 |
+
differentiable with respect to that output, you can just pass None as a
|
| 503 |
+
gradient for that input.
|
| 504 |
+
|
| 505 |
+
You can use the :attr:`ctx` object to pass any value from the forward to this
|
| 506 |
+
functions.
|
| 507 |
+
"""
|
| 508 |
+
raise NotImplementedError(
|
| 509 |
+
"You must implement the jvp function for custom "
|
| 510 |
+
"autograd.Function to use it with forward mode AD."
|
| 511 |
+
)
|
| 512 |
+
|
| 513 |
+
|
| 514 |
+
class Function(_SingleLevelFunction):
|
| 515 |
+
r"""Base class to create custom `autograd.Function`.
|
| 516 |
+
|
| 517 |
+
To create a custom `autograd.Function`, subclass this class and implement
|
| 518 |
+
the :meth:`forward` and :meth:`backward` static methods. Then, to use your custom
|
| 519 |
+
op in the forward pass, call the class method ``apply``. Do not call
|
| 520 |
+
:meth:`forward` directly.
|
| 521 |
+
|
| 522 |
+
To ensure correctness and best performance, make sure you are calling the
|
| 523 |
+
correct methods on ``ctx`` and validating your backward function using
|
| 524 |
+
:func:`torch.autograd.gradcheck`.
|
| 525 |
+
|
| 526 |
+
See :ref:`extending-autograd` for more details on how to use this class.
|
| 527 |
+
|
| 528 |
+
Examples::
|
| 529 |
+
|
| 530 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 531 |
+
>>> class Exp(Function):
|
| 532 |
+
>>> @staticmethod
|
| 533 |
+
>>> def forward(ctx, i):
|
| 534 |
+
>>> result = i.exp()
|
| 535 |
+
>>> ctx.save_for_backward(result)
|
| 536 |
+
>>> return result
|
| 537 |
+
>>>
|
| 538 |
+
>>> @staticmethod
|
| 539 |
+
>>> def backward(ctx, grad_output):
|
| 540 |
+
>>> result, = ctx.saved_tensors
|
| 541 |
+
>>> return grad_output * result
|
| 542 |
+
>>>
|
| 543 |
+
>>> # Use it by calling the apply method:
|
| 544 |
+
>>> # xdoctest: +SKIP
|
| 545 |
+
>>> output = Exp.apply(input)
|
| 546 |
+
"""
|
| 547 |
+
|
| 548 |
+
def __init__(self, *args, **kwargs):
|
| 549 |
+
warnings.warn(
|
| 550 |
+
f"{self.__class__} should not be instantiated. Methods on autograd functions "
|
| 551 |
+
"are all static, so you should invoke them on the class itself. "
|
| 552 |
+
"Instantiating an autograd function will raise an "
|
| 553 |
+
"error in a future version of PyTorch.",
|
| 554 |
+
DeprecationWarning,
|
| 555 |
+
stacklevel=2,
|
| 556 |
+
)
|
| 557 |
+
|
| 558 |
+
def __call__(self, *args, **kwargs):
|
| 559 |
+
raise RuntimeError(
|
| 560 |
+
"Legacy autograd function with non-static forward method is deprecated. "
|
| 561 |
+
"Please use new-style autograd function with static forward method. "
|
| 562 |
+
"(Example: https://pytorch.org/docs/stable/autograd.html#torch.autograd.Function)"
|
| 563 |
+
)
|
| 564 |
+
|
| 565 |
+
"""
|
| 566 |
+
Bool that specifies if PyTorch should attempt to autogenerate
|
| 567 |
+
:func:`torch.vmap` support for this autograd.Function. You may set this to
|
| 568 |
+
True only if this autograd.Function's forward, backward, and jvp (if they
|
| 569 |
+
exist) are written using PyTorch operations; otherwise, please override
|
| 570 |
+
:meth:`torch.autograd.Function.vmap` to add support for :func:`torch.vmap`.
|
| 571 |
+
|
| 572 |
+
Please see :ref:`func-autograd-function` for more details.
|
| 573 |
+
"""
|
| 574 |
+
generate_vmap_rule = False
|
| 575 |
+
|
| 576 |
+
@staticmethod
|
| 577 |
+
def vmap(info, in_dims, *args):
|
| 578 |
+
r"""Define the behavior for this autograd.Function underneath :func:`torch.vmap`.
|
| 579 |
+
|
| 580 |
+
For a :func:`torch.autograd.Function` to support
|
| 581 |
+
:func:`torch.vmap`, you must either override this static method, or set
|
| 582 |
+
``generate_vmap_rule`` to ``True`` (you may not do both).
|
| 583 |
+
|
| 584 |
+
If you choose to override this staticmethod: it must accept
|
| 585 |
+
|
| 586 |
+
- an ``info`` object as the first argument. ``info.batch_size``
|
| 587 |
+
specifies the size of the dimension being vmapped over,
|
| 588 |
+
while ``info.randomness`` is the randomness option passed to
|
| 589 |
+
:func:`torch.vmap`.
|
| 590 |
+
- an ``in_dims`` tuple as the second argument.
|
| 591 |
+
For each arg in ``args``, ``in_dims`` has a corresponding
|
| 592 |
+
``Optional[int]``. It is ``None`` if the arg is not a Tensor or if
|
| 593 |
+
the arg is not being vmapped over, otherwise, it is an integer
|
| 594 |
+
specifying what dimension of the Tensor is being vmapped over.
|
| 595 |
+
- ``*args``, which is the same as the args to :meth:`~Function.forward`.
|
| 596 |
+
|
| 597 |
+
The return of the vmap staticmethod is a tuple of ``(output, out_dims)``.
|
| 598 |
+
Similar to ``in_dims``, ``out_dims`` should be of the same structure as
|
| 599 |
+
``output`` and contain one ``out_dim`` per output that specifies if the
|
| 600 |
+
output has the vmapped dimension and what index it is in.
|
| 601 |
+
|
| 602 |
+
Please see :ref:`func-autograd-function` for more details.
|
| 603 |
+
"""
|
| 604 |
+
raise NotImplementedError(
|
| 605 |
+
"To use autograd.Function with vmap, you must either override the "
|
| 606 |
+
"vmap staticmethod or set generate_vmap_rule=True."
|
| 607 |
+
)
|
| 608 |
+
|
| 609 |
+
@classmethod
|
| 610 |
+
def apply(cls, *args, **kwargs):
|
| 611 |
+
def bind_default_args(func, *args, **kwargs):
|
| 612 |
+
signature = inspect.signature(func)
|
| 613 |
+
bound_args = signature.bind(*args, **kwargs)
|
| 614 |
+
bound_args.apply_defaults()
|
| 615 |
+
|
| 616 |
+
return bound_args.args, bound_args.kwargs
|
| 617 |
+
|
| 618 |
+
is_setup_ctx_defined = _is_setup_context_defined(cls.setup_context)
|
| 619 |
+
if is_setup_ctx_defined:
|
| 620 |
+
args, kwargs = bind_default_args(cls.forward, *args, **kwargs)
|
| 621 |
+
|
| 622 |
+
if not torch._C._are_functorch_transforms_active():
|
| 623 |
+
# See NOTE: [functorch vjp and autograd interaction]
|
| 624 |
+
args = _functorch.utils.unwrap_dead_wrappers(args)
|
| 625 |
+
return super().apply(*args, **kwargs) # type: ignore[misc]
|
| 626 |
+
|
| 627 |
+
if not is_setup_ctx_defined:
|
| 628 |
+
raise RuntimeError(
|
| 629 |
+
"In order to use an autograd.Function with functorch transforms "
|
| 630 |
+
"(vmap, grad, jvp, jacrev, ...), it must override the setup_context "
|
| 631 |
+
"staticmethod. For more details, please see "
|
| 632 |
+
"https://pytorch.org/docs/main/notes/extending.func.html"
|
| 633 |
+
)
|
| 634 |
+
|
| 635 |
+
return custom_function_call(cls, *args, **kwargs)
|
| 636 |
+
|
| 637 |
+
@staticmethod
|
| 638 |
+
def _compiled_autograd_key(ctx):
|
| 639 |
+
return (ctx._autograd_function_id,)
|
| 640 |
+
|
| 641 |
+
|
| 642 |
+
def _is_setup_context_defined(fn):
|
| 643 |
+
return fn != _SingleLevelFunction.setup_context
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
def once_differentiable(
|
| 647 |
+
fn: Callable[Concatenate[_T, _P], _R],
|
| 648 |
+
) -> Callable[Concatenate[_T, _P], _R]:
|
| 649 |
+
@functools.wraps(fn)
|
| 650 |
+
def wrapper(ctx: _T, *args: _P.args, **kwargs: _P.kwargs) -> _R:
|
| 651 |
+
with torch.no_grad():
|
| 652 |
+
outputs = fn(ctx, *args, **kwargs)
|
| 653 |
+
|
| 654 |
+
if not torch.is_grad_enabled():
|
| 655 |
+
return outputs
|
| 656 |
+
|
| 657 |
+
# If any of the inputs have requires_grad=True, we force the outputs
|
| 658 |
+
# to have requires_grad=True but point to a grad_fn which throws an
|
| 659 |
+
# error message during (double) back-propagation.
|
| 660 |
+
# XXX: this is only an approximation of requires_grad - there's no way
|
| 661 |
+
# to figure out if fn didn't use ctx.saved_tensors and as a result
|
| 662 |
+
# some Tensors might require grad, even if no args do.
|
| 663 |
+
# Unfortunately, this leads to unexpected error messages ("no nodes
|
| 664 |
+
# require computing gradients"), but I don't have a better idea.
|
| 665 |
+
# These functions would raise an error in backward anyway.
|
| 666 |
+
requires_grad = any(
|
| 667 |
+
isinstance(arg, torch.Tensor) and arg.requires_grad for arg in args
|
| 668 |
+
)
|
| 669 |
+
if not requires_grad:
|
| 670 |
+
return outputs
|
| 671 |
+
|
| 672 |
+
if not isinstance(outputs, tuple):
|
| 673 |
+
outputs_ = (outputs,)
|
| 674 |
+
else:
|
| 675 |
+
outputs_ = outputs
|
| 676 |
+
|
| 677 |
+
err_fn = _functions.DelayedError(
|
| 678 |
+
b"trying to differentiate twice a function that was marked "
|
| 679 |
+
b"with @once_differentiable",
|
| 680 |
+
len(outputs_),
|
| 681 |
+
)
|
| 682 |
+
|
| 683 |
+
# Create aliases of each output that has requires_grad=True. We need
|
| 684 |
+
# at least one of the inputs to err_fn to require grad so that the
|
| 685 |
+
# output will have a grad_fn.
|
| 686 |
+
def fake_requires_grad(var):
|
| 687 |
+
if var is not None:
|
| 688 |
+
var = var.detach()
|
| 689 |
+
var.requires_grad = True
|
| 690 |
+
return var
|
| 691 |
+
|
| 692 |
+
return err_fn(*[fake_requires_grad(v) for v in outputs_]) # type: ignore[return-value]
|
| 693 |
+
|
| 694 |
+
return wrapper
|
| 695 |
+
|
| 696 |
+
|
| 697 |
+
class InplaceFunction(Function):
|
| 698 |
+
r"""
|
| 699 |
+
This class is here only for backward compatibility reasons.
|
| 700 |
+
Use :class:`Function` instead of this for any new use case.
|
| 701 |
+
"""
|
| 702 |
+
|
| 703 |
+
def __init__(self, inplace=False):
|
| 704 |
+
super().__init__()
|
| 705 |
+
self.inplace = inplace
|
| 706 |
+
|
| 707 |
+
|
| 708 |
+
def _nested_map(condition, fn, condition_msg=None):
|
| 709 |
+
def _map(obj):
|
| 710 |
+
if condition(obj):
|
| 711 |
+
return fn(obj)
|
| 712 |
+
elif obj is None:
|
| 713 |
+
return None
|
| 714 |
+
elif isinstance(obj, (list, tuple)):
|
| 715 |
+
mapped = (_map(x) for x in obj)
|
| 716 |
+
if hasattr(obj, "_fields"):
|
| 717 |
+
# obj is namedtuple
|
| 718 |
+
return type(obj)(*mapped)
|
| 719 |
+
return type(obj)(mapped)
|
| 720 |
+
elif isinstance(obj, dict):
|
| 721 |
+
return {x: _map(obj[x]) for x in obj}
|
| 722 |
+
else:
|
| 723 |
+
raise ValueError(
|
| 724 |
+
"Auto nesting doesn't know how to process "
|
| 725 |
+
"an input object of type "
|
| 726 |
+
+ torch.typename(obj)
|
| 727 |
+
+ (
|
| 728 |
+
". Accepted types: " + condition_msg + ", or lists/tuples of them"
|
| 729 |
+
if condition_msg
|
| 730 |
+
else ""
|
| 731 |
+
)
|
| 732 |
+
)
|
| 733 |
+
|
| 734 |
+
return _map
|
| 735 |
+
|
| 736 |
+
|
| 737 |
+
def _jit_unwrap_structured(obj):
|
| 738 |
+
if hasattr(obj, "_jit_unwrap"):
|
| 739 |
+
return obj._jit_unwrap()
|
| 740 |
+
return obj
|
| 741 |
+
|
| 742 |
+
|
| 743 |
+
def _iter_filter(condition, allow_unknown=False, condition_msg=None, conversion=None):
|
| 744 |
+
def _iter(obj):
|
| 745 |
+
if conversion is not None:
|
| 746 |
+
obj = conversion(obj)
|
| 747 |
+
if condition(obj):
|
| 748 |
+
yield obj
|
| 749 |
+
elif obj is None:
|
| 750 |
+
return
|
| 751 |
+
elif isinstance(obj, (list, tuple)):
|
| 752 |
+
for o in obj:
|
| 753 |
+
yield from _iter(o)
|
| 754 |
+
elif isinstance(obj, dict):
|
| 755 |
+
# We only accept primitive key types, so we needn't inspect them
|
| 756 |
+
for o in obj.values():
|
| 757 |
+
yield from _iter(o)
|
| 758 |
+
elif allow_unknown:
|
| 759 |
+
yield obj
|
| 760 |
+
else:
|
| 761 |
+
raise ValueError(
|
| 762 |
+
"Auto nesting doesn't know how to process "
|
| 763 |
+
"an input object of type "
|
| 764 |
+
+ torch.typename(obj)
|
| 765 |
+
+ (
|
| 766 |
+
". Accepted types: " + condition_msg + ", or lists/tuples of them"
|
| 767 |
+
if condition_msg
|
| 768 |
+
else ""
|
| 769 |
+
)
|
| 770 |
+
)
|
| 771 |
+
|
| 772 |
+
return _iter
|
| 773 |
+
|
| 774 |
+
|
| 775 |
+
def _unflatten(input, proto):
|
| 776 |
+
# unflatten a list or tuple input into a nested list/tuple structure
|
| 777 |
+
# specified by proto
|
| 778 |
+
def unflatten_helper(input, proto):
|
| 779 |
+
res: list[torch.Tensor | None] = []
|
| 780 |
+
if hasattr(proto, "_jit_wrap"):
|
| 781 |
+
return proto._jit_wrap(input)
|
| 782 |
+
if not isinstance(proto, (list, tuple)):
|
| 783 |
+
return input[0], input[1:]
|
| 784 |
+
for e in proto:
|
| 785 |
+
if e is None:
|
| 786 |
+
res.append(e)
|
| 787 |
+
else:
|
| 788 |
+
res_e, input = unflatten_helper(input, e)
|
| 789 |
+
res.append(res_e)
|
| 790 |
+
return type(proto)(res), input
|
| 791 |
+
|
| 792 |
+
return unflatten_helper(input, proto)[0]
|
| 793 |
+
|
| 794 |
+
|
| 795 |
+
_iter_jit_values = _iter_filter(
|
| 796 |
+
lambda o: o is None or isinstance(o, torch._C.Value),
|
| 797 |
+
condition_msg="jit's Values or None",
|
| 798 |
+
)
|
| 799 |
+
_iter_tensors = _iter_filter(
|
| 800 |
+
lambda x: isinstance(x, torch.Tensor),
|
| 801 |
+
condition_msg="Tensors",
|
| 802 |
+
conversion=_jit_unwrap_structured,
|
| 803 |
+
)
|
| 804 |
+
_iter_tensors_permissive = _iter_filter(
|
| 805 |
+
lambda x: isinstance(x, torch.Tensor),
|
| 806 |
+
allow_unknown=True,
|
| 807 |
+
condition_msg="Tensors (permissive)",
|
| 808 |
+
)
|
| 809 |
+
_iter_None_tensors = _iter_filter(
|
| 810 |
+
lambda o: o is None or isinstance(o, torch.Tensor), condition_msg="Tensors or None"
|
| 811 |
+
)
|
| 812 |
+
_map_tensor_data = _nested_map(
|
| 813 |
+
lambda x: isinstance(x, torch.Tensor), lambda o: o.data, condition_msg="Tensors"
|
| 814 |
+
)
|
| 815 |
+
|
| 816 |
+
|
| 817 |
+
class NestedIOFunction(Function):
|
| 818 |
+
r"""
|
| 819 |
+
This class is here only for backward compatibility reasons.
|
| 820 |
+
Use :class:`Function` instead of this for any new use case.
|
| 821 |
+
"""
|
| 822 |
+
|
| 823 |
+
# The 'type: ignore' statements are needed here because these functions are declared as '@staticmethod' in the
|
| 824 |
+
# superclass (Function) but are instance methods here, which mypy reports as incompatible.
|
| 825 |
+
|
| 826 |
+
def _do_forward(self, *input):
|
| 827 |
+
self._nested_input = input
|
| 828 |
+
flat_input = tuple(_iter_tensors(input))
|
| 829 |
+
flat_output = super()._do_forward(*flat_input) # type: ignore[misc]
|
| 830 |
+
nested_tensors = _unflatten(flat_output, self._nested_output)
|
| 831 |
+
return nested_tensors
|
| 832 |
+
|
| 833 |
+
def _do_backward(self, gradients, retain_variables):
|
| 834 |
+
self.retain_variables = retain_variables
|
| 835 |
+
result = super()._do_backward(gradients, retain_variables) # type: ignore[misc]
|
| 836 |
+
if not retain_variables:
|
| 837 |
+
del self._nested_output
|
| 838 |
+
del self._to_save_nested
|
| 839 |
+
return result
|
| 840 |
+
|
| 841 |
+
def backward(self, *gradients: Any) -> Any: # type: ignore[override]
|
| 842 |
+
r"""
|
| 843 |
+
Shared backward utility.
|
| 844 |
+
"""
|
| 845 |
+
nested_gradients = _unflatten(gradients, self._nested_output)
|
| 846 |
+
result = self.backward_extended(*nested_gradients) # type: ignore[func-returns-value]
|
| 847 |
+
return tuple(_iter_None_tensors(result))
|
| 848 |
+
|
| 849 |
+
__call__ = _do_forward
|
| 850 |
+
|
| 851 |
+
def forward(self, *args: Any) -> Any: # type: ignore[override]
|
| 852 |
+
r"""
|
| 853 |
+
Shared forward utility.
|
| 854 |
+
"""
|
| 855 |
+
nested_tensors = _map_tensor_data(self._nested_input)
|
| 856 |
+
result = self.forward_extended(*nested_tensors) # type: ignore[func-returns-value]
|
| 857 |
+
del self._nested_input
|
| 858 |
+
self._nested_output = result
|
| 859 |
+
return tuple(_iter_tensors(result))
|
| 860 |
+
|
| 861 |
+
def save_for_backward(self, *args: Any) -> None:
|
| 862 |
+
r"""
|
| 863 |
+
See :meth:`Function.save_for_backward`.
|
| 864 |
+
"""
|
| 865 |
+
self.to_save = tuple(_iter_tensors(args))
|
| 866 |
+
self._to_save_nested = args
|
| 867 |
+
|
| 868 |
+
@property
|
| 869 |
+
def saved_tensors(self): # type: ignore[override]
|
| 870 |
+
r"""
|
| 871 |
+
See :meth:`Function.saved_tensors`.
|
| 872 |
+
"""
|
| 873 |
+
flat_tensors = super().saved_tensors # type: ignore[misc]
|
| 874 |
+
return _unflatten(flat_tensors, self._to_save_nested)
|
| 875 |
+
|
| 876 |
+
def mark_dirty(self, *args: Any, **kwargs: Any) -> None:
|
| 877 |
+
r"""
|
| 878 |
+
See :meth:`Function.mark_dirty`.
|
| 879 |
+
"""
|
| 880 |
+
self.dirty_tensors = tuple(_iter_tensors((args, kwargs)))
|
| 881 |
+
|
| 882 |
+
def mark_non_differentiable(self, *args: Any, **kwargs: Any) -> None:
|
| 883 |
+
r"""
|
| 884 |
+
See :meth:`Function.mark_non_differentiable`.
|
| 885 |
+
"""
|
| 886 |
+
self.non_differentiable = tuple(_iter_tensors((args, kwargs)))
|
| 887 |
+
|
| 888 |
+
def forward_extended(self, *input: Any) -> None:
|
| 889 |
+
r"""
|
| 890 |
+
User defined forward.
|
| 891 |
+
"""
|
| 892 |
+
raise NotImplementedError
|
| 893 |
+
|
| 894 |
+
def backward_extended(self, *grad_output: Any) -> None:
|
| 895 |
+
r"""
|
| 896 |
+
User defined backward.
|
| 897 |
+
"""
|
| 898 |
+
raise NotImplementedError
|
western_ai/usa/pytorch/torch/autograd/functional.py
ADDED
|
@@ -0,0 +1,1212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from torch._vmap_internals import _vmap
|
| 5 |
+
|
| 6 |
+
from . import forward_ad as fwAD
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
__all__ = ["vjp", "jvp", "jacobian", "hessian", "hvp", "vhp"]
|
| 10 |
+
|
| 11 |
+
# Utility functions
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _as_tuple_nocheck(x):
|
| 15 |
+
if isinstance(x, tuple):
|
| 16 |
+
return x
|
| 17 |
+
elif isinstance(x, list):
|
| 18 |
+
return tuple(x)
|
| 19 |
+
else:
|
| 20 |
+
return (x,)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _as_tuple(inp, arg_name=None, fn_name=None):
|
| 24 |
+
# Ensures that inp is a tuple of Tensors
|
| 25 |
+
# Returns whether or not the original inp was a tuple and the tupled version of the input
|
| 26 |
+
if arg_name is None and fn_name is None:
|
| 27 |
+
return _as_tuple_nocheck(inp)
|
| 28 |
+
|
| 29 |
+
is_inp_tuple = True
|
| 30 |
+
if not isinstance(inp, tuple):
|
| 31 |
+
inp = (inp,)
|
| 32 |
+
is_inp_tuple = False
|
| 33 |
+
|
| 34 |
+
for i, el in enumerate(inp):
|
| 35 |
+
if not isinstance(el, torch.Tensor):
|
| 36 |
+
if is_inp_tuple:
|
| 37 |
+
raise TypeError(
|
| 38 |
+
f"The {arg_name} given to {fn_name} must be either a Tensor or a tuple of Tensors but the"
|
| 39 |
+
f" value at index {i} has type {type(el)}."
|
| 40 |
+
)
|
| 41 |
+
else:
|
| 42 |
+
raise TypeError(
|
| 43 |
+
f"The {arg_name} given to {fn_name} must be either a Tensor or a tuple of Tensors but the"
|
| 44 |
+
f" given {arg_name} has type {type(el)}."
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
return is_inp_tuple, inp
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _tuple_postprocess(res, to_unpack):
|
| 51 |
+
# Unpacks a potentially nested tuple of Tensors
|
| 52 |
+
# to_unpack should be a single boolean or a tuple of two booleans.
|
| 53 |
+
# It is used to:
|
| 54 |
+
# - invert _as_tuple when res should match the inp given to _as_tuple
|
| 55 |
+
# - optionally remove nesting of two tuples created by multiple calls to _as_tuple
|
| 56 |
+
if isinstance(to_unpack, tuple):
|
| 57 |
+
if len(to_unpack) != 2:
|
| 58 |
+
raise AssertionError("Expected to_unpack tuple to have exactly 2 elements")
|
| 59 |
+
if not to_unpack[1]:
|
| 60 |
+
res = tuple(el[0] for el in res)
|
| 61 |
+
if not to_unpack[0]:
|
| 62 |
+
res = res[0]
|
| 63 |
+
else:
|
| 64 |
+
if not to_unpack:
|
| 65 |
+
res = res[0]
|
| 66 |
+
return res
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _grad_preprocess(inputs, create_graph, need_graph):
|
| 70 |
+
# Preprocess the inputs to make sure they require gradient
|
| 71 |
+
# inputs is a tuple of Tensors to preprocess
|
| 72 |
+
# create_graph specifies if the user wants gradients to flow back to the Tensors in inputs
|
| 73 |
+
# need_graph specifies if we internally want gradients to flow back to the Tensors in res
|
| 74 |
+
# Note that we *always* create a new Tensor object to be able to see the difference between
|
| 75 |
+
# inputs given as arguments and the same Tensors automatically captured by the user function.
|
| 76 |
+
# Check this issue for more details on how that can happen: https://github.com/pytorch/pytorch/issues/32576
|
| 77 |
+
res = []
|
| 78 |
+
for inp in inputs:
|
| 79 |
+
if create_graph and inp.requires_grad:
|
| 80 |
+
# Create at least a new Tensor object in a differentiable way
|
| 81 |
+
if not inp.is_sparse:
|
| 82 |
+
# Use .view_as() to get a shallow copy
|
| 83 |
+
res.append(inp.view_as(inp))
|
| 84 |
+
else:
|
| 85 |
+
# We cannot use view for sparse Tensors so we clone
|
| 86 |
+
res.append(inp.clone())
|
| 87 |
+
else:
|
| 88 |
+
res.append(inp.detach().requires_grad_(need_graph))
|
| 89 |
+
return tuple(res)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _grad_postprocess(inputs, create_graph):
|
| 93 |
+
# Postprocess the generated Tensors to avoid returning Tensors with history when the user did not
|
| 94 |
+
# request it.
|
| 95 |
+
if isinstance(inputs[0], torch.Tensor):
|
| 96 |
+
if not create_graph:
|
| 97 |
+
return tuple(inp.detach() for inp in inputs)
|
| 98 |
+
else:
|
| 99 |
+
return inputs
|
| 100 |
+
else:
|
| 101 |
+
return tuple(_grad_postprocess(inp, create_graph) for inp in inputs)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def _validate_v(v, other, is_other_tuple):
|
| 105 |
+
# This assumes that other is the correct shape, and v should match
|
| 106 |
+
# Both are assumed to be tuples of Tensors
|
| 107 |
+
if len(other) != len(v):
|
| 108 |
+
if is_other_tuple:
|
| 109 |
+
raise RuntimeError(
|
| 110 |
+
f"v is a tuple of invalid length: should be {len(other)} but got {len(v)}."
|
| 111 |
+
)
|
| 112 |
+
else:
|
| 113 |
+
raise RuntimeError("The given v should contain a single Tensor.")
|
| 114 |
+
|
| 115 |
+
for idx, (el_v, el_other) in enumerate(zip(v, other)):
|
| 116 |
+
if el_v.size() != el_other.size():
|
| 117 |
+
prepend = ""
|
| 118 |
+
if is_other_tuple:
|
| 119 |
+
prepend = f"Entry {idx} in "
|
| 120 |
+
raise RuntimeError(
|
| 121 |
+
f"{prepend}v has invalid size: should be {el_other.size()} but got {el_v.size()}."
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def _check_requires_grad(inputs, input_type, strict):
|
| 126 |
+
# Used to make all the necessary checks to raise nice errors in strict mode.
|
| 127 |
+
if not strict:
|
| 128 |
+
return
|
| 129 |
+
|
| 130 |
+
if input_type not in ["outputs", "grad_inputs", "jacobian", "hessian"]:
|
| 131 |
+
raise RuntimeError("Invalid input_type to _check_requires_grad")
|
| 132 |
+
for i, inp in enumerate(inputs):
|
| 133 |
+
if inp is None:
|
| 134 |
+
# This can only be reached for grad_inputs.
|
| 135 |
+
raise RuntimeError(
|
| 136 |
+
f"The output of the user-provided function is independent of input {i}."
|
| 137 |
+
" This is not allowed in strict mode."
|
| 138 |
+
)
|
| 139 |
+
if not inp.requires_grad:
|
| 140 |
+
if input_type == "hessian":
|
| 141 |
+
raise RuntimeError(
|
| 142 |
+
f"The hessian of the user-provided function with respect to input {i}"
|
| 143 |
+
" is independent of the input. This is not allowed in strict mode."
|
| 144 |
+
" You should ensure that your function is thrice differentiable and that"
|
| 145 |
+
" the hessian depends on the inputs."
|
| 146 |
+
)
|
| 147 |
+
elif input_type == "jacobian":
|
| 148 |
+
raise RuntimeError(
|
| 149 |
+
"While computing the hessian, found that the jacobian of the user-provided"
|
| 150 |
+
f" function with respect to input {i} is independent of the input. This is not"
|
| 151 |
+
" allowed in strict mode. You should ensure that your function is twice"
|
| 152 |
+
" differentiable and that the jacobian depends on the inputs (this would be"
|
| 153 |
+
" violated by a linear function for example)."
|
| 154 |
+
)
|
| 155 |
+
elif input_type == "grad_inputs":
|
| 156 |
+
raise RuntimeError(
|
| 157 |
+
f"The gradient with respect to input {i} is independent of the inputs of the"
|
| 158 |
+
" user-provided function. This is not allowed in strict mode."
|
| 159 |
+
)
|
| 160 |
+
else:
|
| 161 |
+
raise RuntimeError(
|
| 162 |
+
f"Output {i} of the user-provided function does not require gradients."
|
| 163 |
+
" The outputs must be computed in a differentiable manner from the input"
|
| 164 |
+
" when running in strict mode."
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def _autograd_grad(
|
| 169 |
+
outputs,
|
| 170 |
+
inputs,
|
| 171 |
+
grad_outputs=None,
|
| 172 |
+
create_graph=False,
|
| 173 |
+
retain_graph=None,
|
| 174 |
+
is_grads_batched=False,
|
| 175 |
+
):
|
| 176 |
+
# Version of autograd.grad that accepts `None` in outputs and do not compute gradients for them.
|
| 177 |
+
# This has the extra constraint that inputs has to be a tuple
|
| 178 |
+
if not isinstance(outputs, tuple):
|
| 179 |
+
raise AssertionError("Expected outputs to be a tuple")
|
| 180 |
+
if grad_outputs is None:
|
| 181 |
+
grad_outputs = (None,) * len(outputs)
|
| 182 |
+
if not isinstance(grad_outputs, tuple):
|
| 183 |
+
raise AssertionError("Expected grad_outputs to be a tuple")
|
| 184 |
+
if len(outputs) != len(grad_outputs):
|
| 185 |
+
raise AssertionError(
|
| 186 |
+
f"Expected outputs and grad_outputs to have the same length, "
|
| 187 |
+
f"but got {len(outputs)} and {len(grad_outputs)}"
|
| 188 |
+
)
|
| 189 |
+
|
| 190 |
+
new_outputs: tuple[torch.Tensor, ...] = ()
|
| 191 |
+
new_grad_outputs: tuple[torch.Tensor, ...] = ()
|
| 192 |
+
for out, grad_out in zip(outputs, grad_outputs):
|
| 193 |
+
if out is not None and out.requires_grad:
|
| 194 |
+
new_outputs += (out,)
|
| 195 |
+
# pyrefly: ignore [bad-assignment]
|
| 196 |
+
new_grad_outputs += (grad_out,)
|
| 197 |
+
|
| 198 |
+
if len(new_outputs) == 0:
|
| 199 |
+
# No differentiable output, we don't need to call the autograd engine
|
| 200 |
+
return (None,) * len(inputs)
|
| 201 |
+
else:
|
| 202 |
+
return torch.autograd.grad(
|
| 203 |
+
new_outputs,
|
| 204 |
+
inputs,
|
| 205 |
+
new_grad_outputs,
|
| 206 |
+
allow_unused=True,
|
| 207 |
+
create_graph=create_graph,
|
| 208 |
+
retain_graph=retain_graph,
|
| 209 |
+
is_grads_batched=is_grads_batched,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def _fill_in_zeros(grads, refs, strict, create_graph, stage):
|
| 214 |
+
# Used to detect None in the grads and depending on the flags, either replace them
|
| 215 |
+
# with Tensors full of 0s of the appropriate size based on the refs or raise an error.
|
| 216 |
+
# strict and create graph allow us to detect when it is appropriate to raise an error
|
| 217 |
+
# stage gives us information of which backward call we consider to give good error message
|
| 218 |
+
if stage not in ["back", "back_trick", "double_back", "double_back_trick"]:
|
| 219 |
+
raise RuntimeError(f"Invalid stage argument '{stage}' to _fill_in_zeros")
|
| 220 |
+
|
| 221 |
+
res: tuple[torch.Tensor, ...] = ()
|
| 222 |
+
for i, grads_i in enumerate(grads):
|
| 223 |
+
if grads_i is None:
|
| 224 |
+
if strict:
|
| 225 |
+
if stage == "back":
|
| 226 |
+
raise RuntimeError(
|
| 227 |
+
"The output of the user-provided function is independent of "
|
| 228 |
+
f"input {i}. This is not allowed in strict mode."
|
| 229 |
+
)
|
| 230 |
+
elif stage == "back_trick":
|
| 231 |
+
raise RuntimeError(
|
| 232 |
+
f"The gradient with respect to the input is independent of entry {i}"
|
| 233 |
+
" in the grad_outputs when using the double backward trick to compute"
|
| 234 |
+
" forward mode gradients. This is not allowed in strict mode."
|
| 235 |
+
)
|
| 236 |
+
elif stage == "double_back":
|
| 237 |
+
raise RuntimeError(
|
| 238 |
+
"The jacobian of the user-provided function is independent of "
|
| 239 |
+
f"input {i}. This is not allowed in strict mode."
|
| 240 |
+
)
|
| 241 |
+
else:
|
| 242 |
+
raise RuntimeError(
|
| 243 |
+
"The hessian of the user-provided function is independent of "
|
| 244 |
+
f"entry {i} in the grad_jacobian. This is not allowed in strict "
|
| 245 |
+
"mode as it prevents from using the double backward trick to "
|
| 246 |
+
"replace forward mode AD."
|
| 247 |
+
)
|
| 248 |
+
|
| 249 |
+
grads_i = torch.zeros_like(refs[i])
|
| 250 |
+
else:
|
| 251 |
+
if strict and create_graph and not grads_i.requires_grad:
|
| 252 |
+
if "double" not in stage:
|
| 253 |
+
raise RuntimeError(
|
| 254 |
+
"The jacobian of the user-provided function is independent of "
|
| 255 |
+
f"input {i}. This is not allowed in strict mode when create_graph=True."
|
| 256 |
+
)
|
| 257 |
+
else:
|
| 258 |
+
raise RuntimeError(
|
| 259 |
+
"The hessian of the user-provided function is independent of "
|
| 260 |
+
f"input {i}. This is not allowed in strict mode when create_graph=True."
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
res += (grads_i,)
|
| 264 |
+
|
| 265 |
+
return res
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
# Public API
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def vjp(func, inputs, v=None, create_graph=False, strict=False):
|
| 272 |
+
r"""Compute the dot product between a vector ``v`` and the Jacobian of the given function at the point given by the inputs.
|
| 273 |
+
|
| 274 |
+
Args:
|
| 275 |
+
func (function): a Python function that takes Tensor inputs and returns
|
| 276 |
+
a tuple of Tensors or a Tensor.
|
| 277 |
+
inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
|
| 278 |
+
v (tuple of Tensors or Tensor): The vector for which the vector
|
| 279 |
+
Jacobian product is computed. Must be the same size as the output
|
| 280 |
+
of ``func``. This argument is optional when the output of ``func``
|
| 281 |
+
contains a single element and (if it is not provided) will be set
|
| 282 |
+
as a Tensor containing a single ``1``.
|
| 283 |
+
create_graph (bool, optional): If ``True``, both the output and result
|
| 284 |
+
will be computed in a differentiable way. Note that when ``strict``
|
| 285 |
+
is ``False``, the result can not require gradients or be
|
| 286 |
+
disconnected from the inputs. Defaults to ``False``.
|
| 287 |
+
strict (bool, optional): If ``True``, an error will be raised when we
|
| 288 |
+
detect that there exists an input such that all the outputs are
|
| 289 |
+
independent of it. If ``False``, we return a Tensor of zeros as the
|
| 290 |
+
vjp for said inputs, which is the expected mathematical value.
|
| 291 |
+
Defaults to ``False``.
|
| 292 |
+
|
| 293 |
+
Returns:
|
| 294 |
+
output (tuple): tuple with:
|
| 295 |
+
func_output (tuple of Tensors or Tensor): output of ``func(inputs)``
|
| 296 |
+
|
| 297 |
+
vjp (tuple of Tensors or Tensor): result of the dot product with
|
| 298 |
+
the same shape as the inputs.
|
| 299 |
+
|
| 300 |
+
Example:
|
| 301 |
+
|
| 302 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 303 |
+
>>> def exp_reducer(x):
|
| 304 |
+
... return x.exp().sum(dim=1)
|
| 305 |
+
>>> inputs = torch.rand(4, 4)
|
| 306 |
+
>>> v = torch.ones(4)
|
| 307 |
+
>>> # xdoctest: +IGNORE_WANT("non-deterministic")
|
| 308 |
+
>>> vjp(exp_reducer, inputs, v)
|
| 309 |
+
(tensor([5.7817, 7.2458, 5.7830, 6.7782]),
|
| 310 |
+
tensor([[1.4458, 1.3962, 1.3042, 1.6354],
|
| 311 |
+
[2.1288, 1.0652, 1.5483, 2.5035],
|
| 312 |
+
[2.2046, 1.1292, 1.1432, 1.3059],
|
| 313 |
+
[1.3225, 1.6652, 1.7753, 2.0152]]))
|
| 314 |
+
|
| 315 |
+
>>> vjp(exp_reducer, inputs, v, create_graph=True)
|
| 316 |
+
(tensor([5.7817, 7.2458, 5.7830, 6.7782], grad_fn=<SumBackward1>),
|
| 317 |
+
tensor([[1.4458, 1.3962, 1.3042, 1.6354],
|
| 318 |
+
[2.1288, 1.0652, 1.5483, 2.5035],
|
| 319 |
+
[2.2046, 1.1292, 1.1432, 1.3059],
|
| 320 |
+
[1.3225, 1.6652, 1.7753, 2.0152]], grad_fn=<MulBackward0>))
|
| 321 |
+
|
| 322 |
+
>>> def adder(x, y):
|
| 323 |
+
... return 2 * x + 3 * y
|
| 324 |
+
>>> inputs = (torch.rand(2), torch.rand(2))
|
| 325 |
+
>>> v = torch.ones(2)
|
| 326 |
+
>>> vjp(adder, inputs, v)
|
| 327 |
+
(tensor([2.4225, 2.3340]),
|
| 328 |
+
(tensor([2., 2.]), tensor([3., 3.])))
|
| 329 |
+
"""
|
| 330 |
+
with torch.enable_grad():
|
| 331 |
+
is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "vjp")
|
| 332 |
+
inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True)
|
| 333 |
+
|
| 334 |
+
outputs = func(*inputs)
|
| 335 |
+
is_outputs_tuple, outputs = _as_tuple(
|
| 336 |
+
outputs, "outputs of the user-provided function", "vjp"
|
| 337 |
+
)
|
| 338 |
+
_check_requires_grad(outputs, "outputs", strict=strict)
|
| 339 |
+
|
| 340 |
+
if v is not None:
|
| 341 |
+
_, v = _as_tuple(v, "v", "vjp")
|
| 342 |
+
v = _grad_preprocess(v, create_graph=create_graph, need_graph=False)
|
| 343 |
+
_validate_v(v, outputs, is_outputs_tuple)
|
| 344 |
+
else:
|
| 345 |
+
if len(outputs) != 1 or outputs[0].nelement() != 1:
|
| 346 |
+
raise RuntimeError(
|
| 347 |
+
"The vector v can only be None if the "
|
| 348 |
+
"user-provided function returns "
|
| 349 |
+
"a single Tensor with a single element."
|
| 350 |
+
)
|
| 351 |
+
|
| 352 |
+
enable_grad = True if create_graph else torch.is_grad_enabled()
|
| 353 |
+
with torch.set_grad_enabled(enable_grad):
|
| 354 |
+
grad_res = _autograd_grad(outputs, inputs, v, create_graph=create_graph)
|
| 355 |
+
vjp = _fill_in_zeros(grad_res, inputs, strict, create_graph, "back")
|
| 356 |
+
|
| 357 |
+
# Cleanup objects and return them to the user
|
| 358 |
+
outputs = _grad_postprocess(outputs, create_graph)
|
| 359 |
+
vjp = _grad_postprocess(vjp, create_graph)
|
| 360 |
+
|
| 361 |
+
return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess(
|
| 362 |
+
vjp, is_inputs_tuple
|
| 363 |
+
)
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
def jvp(func, inputs, v=None, create_graph=False, strict=False):
|
| 367 |
+
r"""Compute the dot product between the Jacobian of the given function at the point given by the inputs and a vector ``v``.
|
| 368 |
+
|
| 369 |
+
Args:
|
| 370 |
+
func (function): a Python function that takes Tensor inputs and returns
|
| 371 |
+
a tuple of Tensors or a Tensor.
|
| 372 |
+
inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
|
| 373 |
+
v (tuple of Tensors or Tensor): The vector for which the Jacobian
|
| 374 |
+
vector product is computed. Must be the same size as the input of
|
| 375 |
+
``func``. This argument is optional when the input to ``func``
|
| 376 |
+
contains a single element and (if it is not provided) will be set
|
| 377 |
+
as a Tensor containing a single ``1``.
|
| 378 |
+
create_graph (bool, optional): If ``True``, both the output and result
|
| 379 |
+
will be computed in a differentiable way. Note that when ``strict``
|
| 380 |
+
is ``False``, the result can not require gradients or be
|
| 381 |
+
disconnected from the inputs. Defaults to ``False``.
|
| 382 |
+
strict (bool, optional): If ``True``, an error will be raised when we
|
| 383 |
+
detect that there exists an input such that all the outputs are
|
| 384 |
+
independent of it. If ``False``, we return a Tensor of zeros as the
|
| 385 |
+
jvp for said inputs, which is the expected mathematical value.
|
| 386 |
+
Defaults to ``False``.
|
| 387 |
+
|
| 388 |
+
Returns:
|
| 389 |
+
output (tuple): tuple with:
|
| 390 |
+
func_output (tuple of Tensors or Tensor): output of ``func(inputs)``
|
| 391 |
+
|
| 392 |
+
jvp (tuple of Tensors or Tensor): result of the dot product with
|
| 393 |
+
the same shape as the output.
|
| 394 |
+
|
| 395 |
+
Note:
|
| 396 |
+
``autograd.functional.jvp`` computes the jvp by using the backward of
|
| 397 |
+
the backward (sometimes called the double backwards trick). This is not
|
| 398 |
+
the most performant way of computing the jvp. Please consider using
|
| 399 |
+
:func:`torch.func.jvp` or the
|
| 400 |
+
:ref:`low-level forward-mode AD API <forward-mode-ad>` instead.
|
| 401 |
+
|
| 402 |
+
Example:
|
| 403 |
+
|
| 404 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 405 |
+
>>> def exp_reducer(x):
|
| 406 |
+
... return x.exp().sum(dim=1)
|
| 407 |
+
>>> inputs = torch.rand(4, 4)
|
| 408 |
+
>>> v = torch.ones(4, 4)
|
| 409 |
+
>>> # xdoctest: +IGNORE_WANT("non-deterministic")
|
| 410 |
+
>>> jvp(exp_reducer, inputs, v)
|
| 411 |
+
(tensor([6.3090, 4.6742, 7.9114, 8.2106]),
|
| 412 |
+
tensor([6.3090, 4.6742, 7.9114, 8.2106]))
|
| 413 |
+
|
| 414 |
+
>>> jvp(exp_reducer, inputs, v, create_graph=True)
|
| 415 |
+
(tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=<SumBackward1>),
|
| 416 |
+
tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=<SqueezeBackward1>))
|
| 417 |
+
|
| 418 |
+
>>> def adder(x, y):
|
| 419 |
+
... return 2 * x + 3 * y
|
| 420 |
+
>>> inputs = (torch.rand(2), torch.rand(2))
|
| 421 |
+
>>> v = (torch.ones(2), torch.ones(2))
|
| 422 |
+
>>> jvp(adder, inputs, v)
|
| 423 |
+
(tensor([2.2399, 2.5005]),
|
| 424 |
+
tensor([5., 5.]))
|
| 425 |
+
|
| 426 |
+
"""
|
| 427 |
+
with torch.enable_grad():
|
| 428 |
+
is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jvp")
|
| 429 |
+
inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True)
|
| 430 |
+
|
| 431 |
+
if v is not None:
|
| 432 |
+
_, v = _as_tuple(v, "v", "jvp")
|
| 433 |
+
v = _grad_preprocess(v, create_graph=create_graph, need_graph=False)
|
| 434 |
+
_validate_v(v, inputs, is_inputs_tuple)
|
| 435 |
+
else:
|
| 436 |
+
if len(inputs) != 1 or inputs[0].nelement() != 1:
|
| 437 |
+
raise RuntimeError(
|
| 438 |
+
"The vector v can only be None if the input to "
|
| 439 |
+
"the user-provided function is a single Tensor "
|
| 440 |
+
"with a single element."
|
| 441 |
+
)
|
| 442 |
+
|
| 443 |
+
outputs = func(*inputs)
|
| 444 |
+
is_outputs_tuple, outputs = _as_tuple(
|
| 445 |
+
outputs, "outputs of the user-provided function", "jvp"
|
| 446 |
+
)
|
| 447 |
+
_check_requires_grad(outputs, "outputs", strict=strict)
|
| 448 |
+
# The backward is linear so the value of grad_outputs is not important as
|
| 449 |
+
# it won't appear in the double backward graph. We only need to ensure that
|
| 450 |
+
# it does not contain inf or nan.
|
| 451 |
+
grad_outputs = tuple(
|
| 452 |
+
torch.zeros_like(out, requires_grad=True) for out in outputs
|
| 453 |
+
)
|
| 454 |
+
|
| 455 |
+
grad_inputs = _autograd_grad(outputs, inputs, grad_outputs, create_graph=True)
|
| 456 |
+
_check_requires_grad(grad_inputs, "grad_inputs", strict=strict)
|
| 457 |
+
|
| 458 |
+
if create_graph:
|
| 459 |
+
with torch.enable_grad():
|
| 460 |
+
grad_res = _autograd_grad(
|
| 461 |
+
grad_inputs, grad_outputs, v, create_graph=create_graph
|
| 462 |
+
)
|
| 463 |
+
jvp = _fill_in_zeros(grad_res, outputs, strict, create_graph, "back_trick")
|
| 464 |
+
else:
|
| 465 |
+
grad_res = _autograd_grad(
|
| 466 |
+
grad_inputs, grad_outputs, v, create_graph=create_graph
|
| 467 |
+
)
|
| 468 |
+
jvp = _fill_in_zeros(grad_res, outputs, strict, create_graph, "back_trick")
|
| 469 |
+
|
| 470 |
+
# Cleanup objects and return them to the user
|
| 471 |
+
outputs = _grad_postprocess(outputs, create_graph)
|
| 472 |
+
jvp = _grad_postprocess(jvp, create_graph)
|
| 473 |
+
|
| 474 |
+
return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess(
|
| 475 |
+
jvp, is_outputs_tuple
|
| 476 |
+
)
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
def _construct_standard_basis_for(
|
| 480 |
+
tensors: tuple[torch.Tensor, ...], tensor_numels: tuple[int, ...]
|
| 481 |
+
) -> tuple[torch.Tensor, ...]:
|
| 482 |
+
# This function:
|
| 483 |
+
# - constructs a N=sum(tensor_numels) standard basis. i.e. an NxN identity matrix.
|
| 484 |
+
# - Splits the identity matrix into chunks with each chunk size determined by `tensor_numels`.
|
| 485 |
+
# - Each chunk corresponds to one tensor. The chunk has the same dtype and
|
| 486 |
+
# device as the tensor
|
| 487 |
+
#
|
| 488 |
+
# For example, with tensor_numels = [1, 2, 1], this function returns:
|
| 489 |
+
# ( tensor([[1], tensor([[0, 0], tensor([[0],
|
| 490 |
+
# [0], [1, 0], [0],
|
| 491 |
+
# [0], [0, 1], [0],
|
| 492 |
+
# [0]]) , [0, 0]]) , [1]]) )
|
| 493 |
+
#
|
| 494 |
+
# Precondition: tensor_numels == tuple(tensor.numel() for tensor in tensors)
|
| 495 |
+
# Precondition: tensors always has at least one element.
|
| 496 |
+
#
|
| 497 |
+
# See NOTE: [Computing jacobian with vmap and grad for multiple tensors]
|
| 498 |
+
# for context behind this function. All the pre-conditions are guarded for
|
| 499 |
+
# in torch.autograd.functional.jacobian.
|
| 500 |
+
if len(tensors) != len(tensor_numels):
|
| 501 |
+
raise AssertionError(
|
| 502 |
+
f"Expected tensors and tensor_numels to have the same length, "
|
| 503 |
+
f"but got {len(tensors)} and {len(tensor_numels)}"
|
| 504 |
+
)
|
| 505 |
+
if len(tensors) == 0:
|
| 506 |
+
raise AssertionError("Expected at least one tensor")
|
| 507 |
+
total_numel = sum(tensor_numels)
|
| 508 |
+
chunks = tuple(
|
| 509 |
+
tensor.new_zeros(total_numel, tensor_numel)
|
| 510 |
+
for tensor, tensor_numel in zip(tensors, tensor_numels)
|
| 511 |
+
)
|
| 512 |
+
diag_start_idx = 0
|
| 513 |
+
for chunk, numel in zip(chunks, tensor_numels):
|
| 514 |
+
chunk.diagonal(diag_start_idx).fill_(1)
|
| 515 |
+
diag_start_idx -= numel
|
| 516 |
+
return chunks
|
| 517 |
+
|
| 518 |
+
|
| 519 |
+
def _jacfwd(func, inputs, strict=False, vectorize=False):
|
| 520 |
+
if strict:
|
| 521 |
+
raise RuntimeError(
|
| 522 |
+
"torch.autograd.functional.jacobian: `strict=True` "
|
| 523 |
+
'and `strategy="forward-mode"` are not supported together (yet). '
|
| 524 |
+
"Please either set `strict=False` or "
|
| 525 |
+
'`strategy="reverse-mode"`.'
|
| 526 |
+
)
|
| 527 |
+
is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jacobian")
|
| 528 |
+
output_info = []
|
| 529 |
+
|
| 530 |
+
if vectorize:
|
| 531 |
+
# See NOTE: [Computing jacobian with vmap and grad for multiple outputs]
|
| 532 |
+
input_numels = tuple(input.numel() for input in inputs)
|
| 533 |
+
|
| 534 |
+
# Step 1: Prepare tangents
|
| 535 |
+
tangents = _construct_standard_basis_for(inputs, input_numels)
|
| 536 |
+
|
| 537 |
+
# Step 2: Compute vmap over computation with dual tensors
|
| 538 |
+
def jvp(tangents):
|
| 539 |
+
with fwAD.dual_level():
|
| 540 |
+
dual_inputs = tuple(
|
| 541 |
+
fwAD.make_dual(input, tangent.view_as(input))
|
| 542 |
+
for input, tangent in zip(inputs, tangents)
|
| 543 |
+
)
|
| 544 |
+
_is_outputs_tuple, dual_outputs = _as_tuple(
|
| 545 |
+
func(*dual_inputs), "outputs"
|
| 546 |
+
)
|
| 547 |
+
output_info.append(_is_outputs_tuple)
|
| 548 |
+
jv = []
|
| 549 |
+
primal_outs = []
|
| 550 |
+
for dual_out in dual_outputs:
|
| 551 |
+
primal, tangent = fwAD.unpack_dual(dual_out)
|
| 552 |
+
primal_outs.append(primal)
|
| 553 |
+
if tangent is not None:
|
| 554 |
+
jv.append(tangent)
|
| 555 |
+
else:
|
| 556 |
+
jv.append(torch.zeros_like(primal))
|
| 557 |
+
output_info.append(primal_outs)
|
| 558 |
+
return tuple(jv)
|
| 559 |
+
|
| 560 |
+
outputs_before_split = _vmap(jvp)(tangents)
|
| 561 |
+
is_outputs_tuple, outputs = output_info
|
| 562 |
+
# Step 3: for each of the output tangents, split along dim 0
|
| 563 |
+
jacobian_input_output = []
|
| 564 |
+
for jac_output_i, output_i in zip(outputs_before_split, outputs):
|
| 565 |
+
jacobian_output_i_output = []
|
| 566 |
+
for jac, input_j in zip(jac_output_i.split(input_numels, dim=0), inputs):
|
| 567 |
+
# We need to transpose the Jacobian because in forward AD, the
|
| 568 |
+
# batch dimension represents that of the inputs
|
| 569 |
+
jacobian_input_i_output_j = jac.permute(*range(1, jac.ndim), 0).reshape(
|
| 570 |
+
(*output_i.shape, *input_j.shape)
|
| 571 |
+
)
|
| 572 |
+
|
| 573 |
+
jacobian_output_i_output.append(jacobian_input_i_output_j)
|
| 574 |
+
jacobian_input_output.append(jacobian_output_i_output)
|
| 575 |
+
|
| 576 |
+
# Omit [Step 4] because everything is already transposed w/ forward AD
|
| 577 |
+
return _tuple_postprocess(
|
| 578 |
+
jacobian_input_output, (is_outputs_tuple, is_inputs_tuple)
|
| 579 |
+
)
|
| 580 |
+
else:
|
| 581 |
+
raise NotImplementedError(
|
| 582 |
+
"Computing Jacobian using forward-AD or forward-over-reverse Hessian is"
|
| 583 |
+
"only implemented for `vectorize=True`."
|
| 584 |
+
)
|
| 585 |
+
|
| 586 |
+
|
| 587 |
+
def jacobian(
|
| 588 |
+
func,
|
| 589 |
+
inputs,
|
| 590 |
+
create_graph=False,
|
| 591 |
+
strict=False,
|
| 592 |
+
vectorize=False,
|
| 593 |
+
strategy="reverse-mode",
|
| 594 |
+
):
|
| 595 |
+
r"""Compute the Jacobian of a given function.
|
| 596 |
+
|
| 597 |
+
Args:
|
| 598 |
+
func (function): a Python function that takes Tensor inputs and returns
|
| 599 |
+
a tuple of Tensors or a Tensor.
|
| 600 |
+
inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
|
| 601 |
+
create_graph (bool, optional): If ``True``, the Jacobian will be
|
| 602 |
+
computed in a differentiable manner. Note that when ``strict`` is
|
| 603 |
+
``False``, the result can not require gradients or be disconnected
|
| 604 |
+
from the inputs. Defaults to ``False``.
|
| 605 |
+
strict (bool, optional): If ``True``, an error will be raised when we
|
| 606 |
+
detect that there exists an input such that all the outputs are
|
| 607 |
+
independent of it. If ``False``, we return a Tensor of zeros as the
|
| 608 |
+
jacobian for said inputs, which is the expected mathematical value.
|
| 609 |
+
Defaults to ``False``.
|
| 610 |
+
vectorize (bool, optional): This feature is experimental.
|
| 611 |
+
Please consider using :func:`torch.func.jacrev` or
|
| 612 |
+
:func:`torch.func.jacfwd` instead if you are looking for something
|
| 613 |
+
less experimental and more performant.
|
| 614 |
+
When computing the jacobian, usually we invoke
|
| 615 |
+
``autograd.grad`` once per row of the jacobian. If this flag is
|
| 616 |
+
``True``, we perform only a single ``autograd.grad`` call with
|
| 617 |
+
``batched_grad=True`` which uses the vmap prototype feature.
|
| 618 |
+
Though this should lead to performance improvements in many cases,
|
| 619 |
+
because this feature is still experimental, there may be performance
|
| 620 |
+
cliffs. See :func:`torch.autograd.grad`'s ``batched_grad`` parameter for
|
| 621 |
+
more information.
|
| 622 |
+
strategy (str, optional): Set to ``"forward-mode"`` or ``"reverse-mode"`` to
|
| 623 |
+
determine whether the Jacobian will be computed with forward or reverse
|
| 624 |
+
mode AD. Currently, ``"forward-mode"`` requires ``vectorized=True``.
|
| 625 |
+
Defaults to ``"reverse-mode"``. If ``func`` has more outputs than
|
| 626 |
+
inputs, ``"forward-mode"`` tends to be more performant. Otherwise,
|
| 627 |
+
prefer to use ``"reverse-mode"``.
|
| 628 |
+
|
| 629 |
+
Returns:
|
| 630 |
+
Jacobian (Tensor or nested tuple of Tensors): if there is a single
|
| 631 |
+
input and output, this will be a single Tensor containing the
|
| 632 |
+
Jacobian for the linearized inputs and output. If one of the two is
|
| 633 |
+
a tuple, then the Jacobian will be a tuple of Tensors. If both of
|
| 634 |
+
them are tuples, then the Jacobian will be a tuple of tuple of
|
| 635 |
+
Tensors where ``Jacobian[i][j]`` will contain the Jacobian of the
|
| 636 |
+
``i``\th output and ``j``\th input and will have as size the
|
| 637 |
+
concatenation of the sizes of the corresponding output and the
|
| 638 |
+
corresponding input and will have same dtype and device as the
|
| 639 |
+
corresponding input. If strategy is ``forward-mode``, the dtype will be
|
| 640 |
+
that of the output; otherwise, the input.
|
| 641 |
+
|
| 642 |
+
Example:
|
| 643 |
+
|
| 644 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 645 |
+
>>> def exp_reducer(x):
|
| 646 |
+
... return x.exp().sum(dim=1)
|
| 647 |
+
>>> inputs = torch.rand(2, 2)
|
| 648 |
+
>>> # xdoctest: +IGNORE_WANT("non-deterministic")
|
| 649 |
+
>>> jacobian(exp_reducer, inputs)
|
| 650 |
+
tensor([[[1.4917, 2.4352],
|
| 651 |
+
[0.0000, 0.0000]],
|
| 652 |
+
[[0.0000, 0.0000],
|
| 653 |
+
[2.4369, 2.3799]]])
|
| 654 |
+
|
| 655 |
+
>>> jacobian(exp_reducer, inputs, create_graph=True)
|
| 656 |
+
tensor([[[1.4917, 2.4352],
|
| 657 |
+
[0.0000, 0.0000]],
|
| 658 |
+
[[0.0000, 0.0000],
|
| 659 |
+
[2.4369, 2.3799]]], grad_fn=<ViewBackward>)
|
| 660 |
+
|
| 661 |
+
>>> def exp_adder(x, y):
|
| 662 |
+
... return 2 * x.exp() + 3 * y
|
| 663 |
+
>>> inputs = (torch.rand(2), torch.rand(2))
|
| 664 |
+
>>> jacobian(exp_adder, inputs)
|
| 665 |
+
(tensor([[2.8052, 0.0000],
|
| 666 |
+
[0.0000, 3.3963]]),
|
| 667 |
+
tensor([[3., 0.],
|
| 668 |
+
[0., 3.]]))
|
| 669 |
+
|
| 670 |
+
>>> def linear_model(x):
|
| 671 |
+
... W = torch.tensor([[2.0, -1.0], [0.0, 1.0]])
|
| 672 |
+
... b = torch.tensor([1.0, 0.5])
|
| 673 |
+
... return x @ W.T + b
|
| 674 |
+
|
| 675 |
+
>>> x = torch.randn(4, 2, requires_grad=True)
|
| 676 |
+
>>> jac = jacobian(linear_model, x, vectorize=True)
|
| 677 |
+
>>> jac.shape
|
| 678 |
+
torch.Size([4, 2, 4, 2])
|
| 679 |
+
"""
|
| 680 |
+
if strategy not in ("forward-mode", "reverse-mode"):
|
| 681 |
+
raise AssertionError(
|
| 682 |
+
'Expected strategy to be either "forward-mode" or "reverse-mode". Hint: If your '
|
| 683 |
+
'function has more outputs than inputs, "forward-mode" tends to be more performant. '
|
| 684 |
+
'Otherwise, prefer to use "reverse-mode".'
|
| 685 |
+
)
|
| 686 |
+
if strategy == "forward-mode":
|
| 687 |
+
if create_graph:
|
| 688 |
+
raise NotImplementedError(
|
| 689 |
+
"torch.autograd.functional.jacobian: `create_graph=True` "
|
| 690 |
+
'and `strategy="forward-mode"` are not supported together (yet). '
|
| 691 |
+
"Please either set `create_graph=False` or "
|
| 692 |
+
'`strategy="reverse-mode"`.'
|
| 693 |
+
)
|
| 694 |
+
return _jacfwd(func, inputs, strict, vectorize)
|
| 695 |
+
|
| 696 |
+
with torch.enable_grad():
|
| 697 |
+
is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jacobian")
|
| 698 |
+
inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True)
|
| 699 |
+
|
| 700 |
+
outputs = func(*inputs)
|
| 701 |
+
is_outputs_tuple, outputs = _as_tuple(
|
| 702 |
+
outputs, "outputs of the user-provided function", "jacobian"
|
| 703 |
+
)
|
| 704 |
+
_check_requires_grad(outputs, "outputs", strict=strict)
|
| 705 |
+
|
| 706 |
+
if vectorize:
|
| 707 |
+
if strict:
|
| 708 |
+
raise RuntimeError(
|
| 709 |
+
"torch.autograd.functional.jacobian: `strict=True` "
|
| 710 |
+
"and `vectorized=True` are not supported together. "
|
| 711 |
+
"Please either set `strict=False` or "
|
| 712 |
+
"`vectorize=False`."
|
| 713 |
+
)
|
| 714 |
+
# NOTE: [Computing jacobian with vmap and grad for multiple outputs]
|
| 715 |
+
#
|
| 716 |
+
# Let's consider f(x) = (x**2, x.sum()) and let x = torch.randn(3).
|
| 717 |
+
# It turns out we can compute the jacobian of this function with a single
|
| 718 |
+
# call to autograd.grad by using vmap over the correct grad_outputs.
|
| 719 |
+
#
|
| 720 |
+
# Firstly, one way to compute the jacobian is to stack x**2 and x.sum()
|
| 721 |
+
# into a 4D vector. E.g., use g(x) = torch.stack([x**2, x.sum()])
|
| 722 |
+
#
|
| 723 |
+
# To get the first row of the jacobian, we call
|
| 724 |
+
# >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([1, 0, 0, 0]))
|
| 725 |
+
# To get the 2nd row of the jacobian, we call
|
| 726 |
+
# >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([0, 1, 0, 0]))
|
| 727 |
+
# and so on.
|
| 728 |
+
#
|
| 729 |
+
# Using vmap, we can vectorize all 4 of these computations into one by
|
| 730 |
+
# passing the standard basis for R^4 as the grad_output.
|
| 731 |
+
# vmap(partial(autograd.grad, g(x), x))(torch.eye(4)).
|
| 732 |
+
#
|
| 733 |
+
# Now, how do we compute the jacobian *without stacking the output*?
|
| 734 |
+
# We can just split the standard basis across the outputs. So to
|
| 735 |
+
# compute the jacobian of f(x), we'd use
|
| 736 |
+
# >>> autograd.grad(f(x), x, grad_outputs=_construct_standard_basis_for(...))
|
| 737 |
+
# The grad_outputs looks like the following:
|
| 738 |
+
# ( torch.tensor([[1, 0, 0],
|
| 739 |
+
# [0, 1, 0],
|
| 740 |
+
# [0, 0, 1],
|
| 741 |
+
# [0, 0, 0]]),
|
| 742 |
+
# torch.tensor([[0],
|
| 743 |
+
# [0],
|
| 744 |
+
# [0],
|
| 745 |
+
# [1]]) )
|
| 746 |
+
#
|
| 747 |
+
# But we're not done yet!
|
| 748 |
+
# >>> vmap(partial(autograd.grad(f(x), x, grad_outputs=...)))
|
| 749 |
+
# returns a Tensor of shape [4, 3]. We have to remember to split the
|
| 750 |
+
# jacobian of shape [4, 3] into two:
|
| 751 |
+
# - one of shape [3, 3] for the first output
|
| 752 |
+
# - one of shape [ 3] for the second output
|
| 753 |
+
|
| 754 |
+
# Step 1: Construct grad_outputs by splitting the standard basis
|
| 755 |
+
output_numels = tuple(output.numel() for output in outputs)
|
| 756 |
+
grad_outputs = _construct_standard_basis_for(outputs, output_numels)
|
| 757 |
+
flat_outputs = tuple(output.reshape(-1) for output in outputs)
|
| 758 |
+
|
| 759 |
+
# Step 2: Call vmap + autograd.grad
|
| 760 |
+
def vjp(grad_output):
|
| 761 |
+
vj = list(
|
| 762 |
+
_autograd_grad(
|
| 763 |
+
flat_outputs,
|
| 764 |
+
inputs,
|
| 765 |
+
grad_output,
|
| 766 |
+
create_graph=create_graph,
|
| 767 |
+
is_grads_batched=True,
|
| 768 |
+
)
|
| 769 |
+
)
|
| 770 |
+
for el_idx, vj_el in enumerate(vj):
|
| 771 |
+
if vj_el is not None:
|
| 772 |
+
continue
|
| 773 |
+
vj[el_idx] = torch.zeros_like(inputs[el_idx]).expand(
|
| 774 |
+
(sum(output_numels),) + inputs[el_idx].shape
|
| 775 |
+
)
|
| 776 |
+
return tuple(vj)
|
| 777 |
+
|
| 778 |
+
jacobians_of_flat_output = vjp(grad_outputs)
|
| 779 |
+
|
| 780 |
+
# Step 3: The returned jacobian is one big tensor per input. In this step,
|
| 781 |
+
# we split each Tensor by output.
|
| 782 |
+
jacobian_input_output = []
|
| 783 |
+
for jac_input_i, input_i in zip(jacobians_of_flat_output, inputs):
|
| 784 |
+
jacobian_input_i_output = []
|
| 785 |
+
for jac, output_j in zip(
|
| 786 |
+
jac_input_i.split(output_numels, dim=0), outputs
|
| 787 |
+
):
|
| 788 |
+
jacobian_input_i_output_j = jac.view(output_j.shape + input_i.shape)
|
| 789 |
+
jacobian_input_i_output.append(jacobian_input_i_output_j)
|
| 790 |
+
jacobian_input_output.append(jacobian_input_i_output)
|
| 791 |
+
|
| 792 |
+
# Step 4: Right now, `jacobian` is a List[List[Tensor]].
|
| 793 |
+
# The outer List corresponds to the number of inputs,
|
| 794 |
+
# the inner List corresponds to the number of outputs.
|
| 795 |
+
# We need to exchange the order of these and convert to tuples
|
| 796 |
+
# before returning.
|
| 797 |
+
jacobian_output_input = tuple(zip(*jacobian_input_output))
|
| 798 |
+
|
| 799 |
+
jacobian_output_input = _grad_postprocess(
|
| 800 |
+
jacobian_output_input, create_graph
|
| 801 |
+
)
|
| 802 |
+
return _tuple_postprocess(
|
| 803 |
+
jacobian_output_input, (is_outputs_tuple, is_inputs_tuple)
|
| 804 |
+
)
|
| 805 |
+
|
| 806 |
+
jacobian: tuple[torch.Tensor, ...] = ()
|
| 807 |
+
|
| 808 |
+
for i, out in enumerate(outputs):
|
| 809 |
+
# mypy complains that expression and variable have different types due to the empty list
|
| 810 |
+
jac_i: tuple[list[torch.Tensor]] = tuple([] for _ in range(len(inputs))) # type: ignore[assignment]
|
| 811 |
+
for j in range(out.nelement()):
|
| 812 |
+
vj = _autograd_grad(
|
| 813 |
+
(out.reshape(-1)[j],),
|
| 814 |
+
inputs,
|
| 815 |
+
retain_graph=True,
|
| 816 |
+
create_graph=create_graph,
|
| 817 |
+
)
|
| 818 |
+
|
| 819 |
+
for el_idx, (jac_i_el, vj_el, inp_el) in enumerate(
|
| 820 |
+
zip(jac_i, vj, inputs)
|
| 821 |
+
):
|
| 822 |
+
if vj_el is not None:
|
| 823 |
+
if strict and create_graph and not vj_el.requires_grad:
|
| 824 |
+
msg = (
|
| 825 |
+
"The jacobian of the user-provided function is "
|
| 826 |
+
f"independent of input {i}. This is not allowed in "
|
| 827 |
+
"strict mode when create_graph=True."
|
| 828 |
+
)
|
| 829 |
+
raise RuntimeError(msg)
|
| 830 |
+
jac_i_el.append(vj_el)
|
| 831 |
+
else:
|
| 832 |
+
if strict:
|
| 833 |
+
msg = (
|
| 834 |
+
f"Output {i} of the user-provided function is "
|
| 835 |
+
f"independent of input {el_idx}. This is not allowed in "
|
| 836 |
+
"strict mode."
|
| 837 |
+
)
|
| 838 |
+
raise RuntimeError(msg)
|
| 839 |
+
jac_i_el.append(torch.zeros_like(inp_el))
|
| 840 |
+
|
| 841 |
+
# pyrefly: ignore [bad-assignment]
|
| 842 |
+
jacobian += (
|
| 843 |
+
tuple(
|
| 844 |
+
torch.stack(jac_i_el, dim=0).view(
|
| 845 |
+
out.size() + inputs[el_idx].size() # type: ignore[operator]
|
| 846 |
+
)
|
| 847 |
+
for (el_idx, jac_i_el) in enumerate(jac_i)
|
| 848 |
+
),
|
| 849 |
+
)
|
| 850 |
+
|
| 851 |
+
jacobian = _grad_postprocess(jacobian, create_graph)
|
| 852 |
+
|
| 853 |
+
return _tuple_postprocess(jacobian, (is_outputs_tuple, is_inputs_tuple))
|
| 854 |
+
|
| 855 |
+
|
| 856 |
+
def hessian(
|
| 857 |
+
func,
|
| 858 |
+
inputs,
|
| 859 |
+
create_graph=False,
|
| 860 |
+
strict=False,
|
| 861 |
+
vectorize=False,
|
| 862 |
+
outer_jacobian_strategy="reverse-mode",
|
| 863 |
+
):
|
| 864 |
+
r"""Compute the Hessian of a given scalar function.
|
| 865 |
+
|
| 866 |
+
Args:
|
| 867 |
+
func (function): a Python function that takes Tensor inputs and returns
|
| 868 |
+
a Tensor with a single element.
|
| 869 |
+
inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
|
| 870 |
+
create_graph (bool, optional): If ``True``, the Hessian will be computed in
|
| 871 |
+
a differentiable manner. Note that when ``strict`` is ``False``, the result can not
|
| 872 |
+
require gradients or be disconnected from the inputs.
|
| 873 |
+
Defaults to ``False``.
|
| 874 |
+
strict (bool, optional): If ``True``, an error will be raised when we detect that there exists an input
|
| 875 |
+
such that all the outputs are independent of it. If ``False``, we return a Tensor of zeros as the
|
| 876 |
+
hessian for said inputs, which is the expected mathematical value.
|
| 877 |
+
Defaults to ``False``.
|
| 878 |
+
vectorize (bool, optional): This feature is experimental.
|
| 879 |
+
Please consider using :func:`torch.func.hessian`
|
| 880 |
+
instead if you are looking for something less experimental and more performant.
|
| 881 |
+
When computing the hessian, usually we invoke
|
| 882 |
+
``autograd.grad`` once per row of the hessian. If this flag is
|
| 883 |
+
``True``, we use the vmap prototype feature as the backend to
|
| 884 |
+
vectorize calls to ``autograd.grad`` so we only invoke it once
|
| 885 |
+
instead of once per row. This should lead to performance
|
| 886 |
+
improvements in many use cases, however, due to this feature
|
| 887 |
+
being incomplete, there may be performance cliffs. Please
|
| 888 |
+
use `torch._C._debug_only_display_vmap_fallback_warnings(True)`
|
| 889 |
+
to show any performance warnings and file us issues if
|
| 890 |
+
warnings exist for your use case. Defaults to ``False``.
|
| 891 |
+
outer_jacobian_strategy (str, optional): The Hessian is computed by
|
| 892 |
+
computing the Jacobian of a Jacobian. The inner Jacobian is always
|
| 893 |
+
computed in reverse-mode AD. Setting strategy to ``"forward-mode"``
|
| 894 |
+
or ``"reverse-mode"`` determines whether the outer Jacobian will be
|
| 895 |
+
computed with forward or reverse mode AD. Currently, computing the outer
|
| 896 |
+
Jacobian in ``"forward-mode"`` requires ``vectorized=True``. Defaults
|
| 897 |
+
to ``"reverse-mode"``.
|
| 898 |
+
|
| 899 |
+
Returns:
|
| 900 |
+
Hessian (Tensor or a tuple of tuple of Tensors): if there is a single input,
|
| 901 |
+
this will be a single Tensor containing the Hessian for the input.
|
| 902 |
+
If it is a tuple, then the Hessian will be a tuple of tuples where
|
| 903 |
+
``Hessian[i][j]`` will contain the Hessian of the ``i``\th input
|
| 904 |
+
and ``j``\th input with size the sum of the size of the ``i``\th input plus
|
| 905 |
+
the size of the ``j``\th input. ``Hessian[i][j]`` will have the same
|
| 906 |
+
dtype and device as the corresponding ``i``\th input.
|
| 907 |
+
|
| 908 |
+
Example:
|
| 909 |
+
|
| 910 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 911 |
+
>>> def pow_reducer(x):
|
| 912 |
+
... return x.pow(3).sum()
|
| 913 |
+
>>> inputs = torch.rand(2, 2)
|
| 914 |
+
>>> # xdoctest: +IGNORE_WANT("non-deterministic")
|
| 915 |
+
>>> hessian(pow_reducer, inputs)
|
| 916 |
+
tensor([[[[5.2265, 0.0000],
|
| 917 |
+
[0.0000, 0.0000]],
|
| 918 |
+
[[0.0000, 4.8221],
|
| 919 |
+
[0.0000, 0.0000]]],
|
| 920 |
+
[[[0.0000, 0.0000],
|
| 921 |
+
[1.9456, 0.0000]],
|
| 922 |
+
[[0.0000, 0.0000],
|
| 923 |
+
[0.0000, 3.2550]]]])
|
| 924 |
+
|
| 925 |
+
>>> hessian(pow_reducer, inputs, create_graph=True)
|
| 926 |
+
tensor([[[[5.2265, 0.0000],
|
| 927 |
+
[0.0000, 0.0000]],
|
| 928 |
+
[[0.0000, 4.8221],
|
| 929 |
+
[0.0000, 0.0000]]],
|
| 930 |
+
[[[0.0000, 0.0000],
|
| 931 |
+
[1.9456, 0.0000]],
|
| 932 |
+
[[0.0000, 0.0000],
|
| 933 |
+
[0.0000, 3.2550]]]], grad_fn=<ViewBackward>)
|
| 934 |
+
|
| 935 |
+
|
| 936 |
+
>>> def pow_adder_reducer(x, y):
|
| 937 |
+
... return (2 * x.pow(2) + 3 * y.pow(2)).sum()
|
| 938 |
+
>>> inputs = (torch.rand(2), torch.rand(2))
|
| 939 |
+
>>> hessian(pow_adder_reducer, inputs)
|
| 940 |
+
((tensor([[4., 0.],
|
| 941 |
+
[0., 4.]]),
|
| 942 |
+
tensor([[0., 0.],
|
| 943 |
+
[0., 0.]])),
|
| 944 |
+
(tensor([[0., 0.],
|
| 945 |
+
[0., 0.]]),
|
| 946 |
+
tensor([[6., 0.],
|
| 947 |
+
[0., 6.]])))
|
| 948 |
+
"""
|
| 949 |
+
is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "hessian")
|
| 950 |
+
if outer_jacobian_strategy not in (
|
| 951 |
+
"forward-mode",
|
| 952 |
+
"reverse-mode",
|
| 953 |
+
):
|
| 954 |
+
raise AssertionError(
|
| 955 |
+
'Expected strategy to be either "forward-mode" or "reverse-mode".'
|
| 956 |
+
)
|
| 957 |
+
|
| 958 |
+
def ensure_single_output_function(*inp):
|
| 959 |
+
out = func(*inp)
|
| 960 |
+
is_out_tuple, t_out = _as_tuple(
|
| 961 |
+
out, "outputs of the user-provided function", "hessian"
|
| 962 |
+
)
|
| 963 |
+
_check_requires_grad(t_out, "outputs", strict=strict)
|
| 964 |
+
|
| 965 |
+
if is_out_tuple or not isinstance(out, torch.Tensor):
|
| 966 |
+
raise RuntimeError(
|
| 967 |
+
"The function given to hessian should return a single Tensor"
|
| 968 |
+
)
|
| 969 |
+
|
| 970 |
+
if out.nelement() != 1:
|
| 971 |
+
raise RuntimeError(
|
| 972 |
+
"The Tensor returned by the function given to hessian should contain a single element"
|
| 973 |
+
)
|
| 974 |
+
|
| 975 |
+
return out.squeeze()
|
| 976 |
+
|
| 977 |
+
def jac_func(*inp):
|
| 978 |
+
if outer_jacobian_strategy == "forward-mode":
|
| 979 |
+
# _grad_preprocess requires create_graph=True and input to require_grad
|
| 980 |
+
# or else the input will be detached
|
| 981 |
+
inp = tuple(t.requires_grad_(True) for t in inp)
|
| 982 |
+
jac = jacobian(ensure_single_output_function, inp, create_graph=True)
|
| 983 |
+
_check_requires_grad(jac, "jacobian", strict=strict)
|
| 984 |
+
return jac
|
| 985 |
+
|
| 986 |
+
res = jacobian(
|
| 987 |
+
jac_func,
|
| 988 |
+
inputs,
|
| 989 |
+
create_graph=create_graph,
|
| 990 |
+
strict=strict,
|
| 991 |
+
vectorize=vectorize,
|
| 992 |
+
strategy=outer_jacobian_strategy,
|
| 993 |
+
)
|
| 994 |
+
return _tuple_postprocess(res, (is_inputs_tuple, is_inputs_tuple))
|
| 995 |
+
|
| 996 |
+
|
| 997 |
+
def vhp(func, inputs, v=None, create_graph=False, strict=False):
|
| 998 |
+
r"""Compute the dot product between vector ``v`` and Hessian of a given scalar function at a specified point.
|
| 999 |
+
|
| 1000 |
+
Args:
|
| 1001 |
+
func (function): a Python function that takes Tensor inputs and returns
|
| 1002 |
+
a Tensor with a single element.
|
| 1003 |
+
inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
|
| 1004 |
+
v (tuple of Tensors or Tensor): The vector for which the vector Hessian
|
| 1005 |
+
product is computed. Must be the same size as the input of
|
| 1006 |
+
``func``. This argument is optional when ``func``'s input contains
|
| 1007 |
+
a single element and (if it is not provided) will be set as a
|
| 1008 |
+
Tensor containing a single ``1``.
|
| 1009 |
+
create_graph (bool, optional): If ``True``, both the output and result
|
| 1010 |
+
will be computed in a differentiable way. Note that when ``strict``
|
| 1011 |
+
is ``False``, the result can not require gradients or be
|
| 1012 |
+
disconnected from the inputs.
|
| 1013 |
+
Defaults to ``False``.
|
| 1014 |
+
strict (bool, optional): If ``True``, an error will be raised when we
|
| 1015 |
+
detect that there exists an input such that all the outputs are
|
| 1016 |
+
independent of it. If ``False``, we return a Tensor of zeros as the
|
| 1017 |
+
vhp for said inputs, which is the expected mathematical value.
|
| 1018 |
+
Defaults to ``False``.
|
| 1019 |
+
|
| 1020 |
+
Returns:
|
| 1021 |
+
output (tuple): tuple with:
|
| 1022 |
+
func_output (tuple of Tensors or Tensor): output of ``func(inputs)``
|
| 1023 |
+
|
| 1024 |
+
vhp (tuple of Tensors or Tensor): result of the dot product with the
|
| 1025 |
+
same shape as the inputs.
|
| 1026 |
+
|
| 1027 |
+
Example:
|
| 1028 |
+
|
| 1029 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 1030 |
+
>>> def pow_reducer(x):
|
| 1031 |
+
... return x.pow(3).sum()
|
| 1032 |
+
>>> inputs = torch.rand(2, 2)
|
| 1033 |
+
>>> v = torch.ones(2, 2)
|
| 1034 |
+
>>> # xdoctest: +IGNORE_WANT("non-deterministic")
|
| 1035 |
+
>>> vhp(pow_reducer, inputs, v)
|
| 1036 |
+
(tensor(0.5591),
|
| 1037 |
+
tensor([[1.0689, 1.2431],
|
| 1038 |
+
[3.0989, 4.4456]]))
|
| 1039 |
+
>>> vhp(pow_reducer, inputs, v, create_graph=True)
|
| 1040 |
+
(tensor(0.5591, grad_fn=<SumBackward0>),
|
| 1041 |
+
tensor([[1.0689, 1.2431],
|
| 1042 |
+
[3.0989, 4.4456]], grad_fn=<MulBackward0>))
|
| 1043 |
+
>>> def pow_adder_reducer(x, y):
|
| 1044 |
+
... return (2 * x.pow(2) + 3 * y.pow(2)).sum()
|
| 1045 |
+
>>> inputs = (torch.rand(2), torch.rand(2))
|
| 1046 |
+
>>> v = (torch.zeros(2), torch.ones(2))
|
| 1047 |
+
>>> vhp(pow_adder_reducer, inputs, v)
|
| 1048 |
+
(tensor(4.8053),
|
| 1049 |
+
(tensor([0., 0.]),
|
| 1050 |
+
tensor([6., 6.])))
|
| 1051 |
+
"""
|
| 1052 |
+
with torch.enable_grad():
|
| 1053 |
+
is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "vhp")
|
| 1054 |
+
inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True)
|
| 1055 |
+
|
| 1056 |
+
if v is not None:
|
| 1057 |
+
_, v = _as_tuple(v, "v", "vhp")
|
| 1058 |
+
v = _grad_preprocess(v, create_graph=create_graph, need_graph=False)
|
| 1059 |
+
_validate_v(v, inputs, is_inputs_tuple)
|
| 1060 |
+
else:
|
| 1061 |
+
if len(inputs) != 1 or inputs[0].nelement() != 1:
|
| 1062 |
+
raise RuntimeError(
|
| 1063 |
+
"The vector v can only be None if the input to the user-provided function "
|
| 1064 |
+
"is a single Tensor with a single element."
|
| 1065 |
+
)
|
| 1066 |
+
outputs = func(*inputs)
|
| 1067 |
+
is_outputs_tuple, outputs = _as_tuple(
|
| 1068 |
+
outputs, "outputs of the user-provided function", "vhp"
|
| 1069 |
+
)
|
| 1070 |
+
_check_requires_grad(outputs, "outputs", strict=strict)
|
| 1071 |
+
|
| 1072 |
+
if is_outputs_tuple or not isinstance(outputs[0], torch.Tensor):
|
| 1073 |
+
raise RuntimeError(
|
| 1074 |
+
"The function given to vhp should return a single Tensor"
|
| 1075 |
+
)
|
| 1076 |
+
|
| 1077 |
+
if outputs[0].nelement() != 1:
|
| 1078 |
+
raise RuntimeError(
|
| 1079 |
+
"The Tensor returned by the function given to vhp should contain a single element"
|
| 1080 |
+
)
|
| 1081 |
+
|
| 1082 |
+
jac = _autograd_grad(outputs, inputs, create_graph=True)
|
| 1083 |
+
_check_requires_grad(jac, "jacobian", strict=strict)
|
| 1084 |
+
|
| 1085 |
+
enable_grad = True if create_graph else torch.is_grad_enabled()
|
| 1086 |
+
with torch.set_grad_enabled(enable_grad):
|
| 1087 |
+
grad_res = _autograd_grad(jac, inputs, v, create_graph=create_graph)
|
| 1088 |
+
vhp = _fill_in_zeros(grad_res, inputs, strict, create_graph, "double_back")
|
| 1089 |
+
|
| 1090 |
+
outputs = _grad_postprocess(outputs, create_graph)
|
| 1091 |
+
vhp = _grad_postprocess(vhp, create_graph)
|
| 1092 |
+
|
| 1093 |
+
return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess(
|
| 1094 |
+
vhp, is_inputs_tuple
|
| 1095 |
+
)
|
| 1096 |
+
|
| 1097 |
+
|
| 1098 |
+
def hvp(func, inputs, v=None, create_graph=False, strict=False):
|
| 1099 |
+
r"""Compute the dot product between the scalar function's Hessian and a vector ``v`` at a specified point.
|
| 1100 |
+
|
| 1101 |
+
Args:
|
| 1102 |
+
func (function): a Python function that takes Tensor inputs and returns
|
| 1103 |
+
a Tensor with a single element.
|
| 1104 |
+
inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
|
| 1105 |
+
v (tuple of Tensors or Tensor): The vector for which the Hessian vector
|
| 1106 |
+
product is computed. Must be the same size as the input of
|
| 1107 |
+
``func``. This argument is optional when ``func``'s input contains
|
| 1108 |
+
a single element and (if it is not provided) will be set as a
|
| 1109 |
+
Tensor containing a single ``1``.
|
| 1110 |
+
create_graph (bool, optional): If ``True``, both the output and result will be
|
| 1111 |
+
computed in a differentiable way. Note that when ``strict`` is
|
| 1112 |
+
``False``, the result can not require gradients or be disconnected
|
| 1113 |
+
from the inputs. Defaults to ``False``.
|
| 1114 |
+
strict (bool, optional): If ``True``, an error will be raised when we
|
| 1115 |
+
detect that there exists an input such that all the outputs are
|
| 1116 |
+
independent of it. If ``False``, we return a Tensor of zeros as the
|
| 1117 |
+
hvp for said inputs, which is the expected mathematical value.
|
| 1118 |
+
Defaults to ``False``.
|
| 1119 |
+
Returns:
|
| 1120 |
+
output (tuple): tuple with:
|
| 1121 |
+
func_output (tuple of Tensors or Tensor): output of ``func(inputs)``
|
| 1122 |
+
|
| 1123 |
+
hvp (tuple of Tensors or Tensor): result of the dot product with
|
| 1124 |
+
the same shape as the inputs.
|
| 1125 |
+
|
| 1126 |
+
Example:
|
| 1127 |
+
|
| 1128 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 1129 |
+
>>> def pow_reducer(x):
|
| 1130 |
+
... return x.pow(3).sum()
|
| 1131 |
+
>>> inputs = torch.rand(2, 2)
|
| 1132 |
+
>>> v = torch.ones(2, 2)
|
| 1133 |
+
>>> # xdoctest: +IGNORE_WANT("non-deterministic")
|
| 1134 |
+
>>> hvp(pow_reducer, inputs, v)
|
| 1135 |
+
(tensor(0.1448),
|
| 1136 |
+
tensor([[2.0239, 1.6456],
|
| 1137 |
+
[2.4988, 1.4310]]))
|
| 1138 |
+
|
| 1139 |
+
>>> hvp(pow_reducer, inputs, v, create_graph=True)
|
| 1140 |
+
(tensor(0.1448, grad_fn=<SumBackward0>),
|
| 1141 |
+
tensor([[2.0239, 1.6456],
|
| 1142 |
+
[2.4988, 1.4310]], grad_fn=<MulBackward0>))
|
| 1143 |
+
|
| 1144 |
+
|
| 1145 |
+
>>> def pow_adder_reducer(x, y):
|
| 1146 |
+
... return (2 * x.pow(2) + 3 * y.pow(2)).sum()
|
| 1147 |
+
>>> inputs = (torch.rand(2), torch.rand(2))
|
| 1148 |
+
>>> v = (torch.zeros(2), torch.ones(2))
|
| 1149 |
+
>>> hvp(pow_adder_reducer, inputs, v)
|
| 1150 |
+
(tensor(2.3030),
|
| 1151 |
+
(tensor([0., 0.]),
|
| 1152 |
+
tensor([6., 6.])))
|
| 1153 |
+
|
| 1154 |
+
Note:
|
| 1155 |
+
|
| 1156 |
+
This function is significantly slower than `vhp` due to backward mode AD constraints.
|
| 1157 |
+
If your functions is twice continuously differentiable, then hvp = vhp.t(). So if you
|
| 1158 |
+
know that your function satisfies this condition, you should use vhp instead that is
|
| 1159 |
+
much faster with the current implementation.
|
| 1160 |
+
|
| 1161 |
+
"""
|
| 1162 |
+
with torch.enable_grad():
|
| 1163 |
+
is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "hvp")
|
| 1164 |
+
inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True)
|
| 1165 |
+
|
| 1166 |
+
if v is not None:
|
| 1167 |
+
_, v = _as_tuple(v, "v", "hvp")
|
| 1168 |
+
v = _grad_preprocess(v, create_graph=create_graph, need_graph=False)
|
| 1169 |
+
_validate_v(v, inputs, is_inputs_tuple)
|
| 1170 |
+
else:
|
| 1171 |
+
if len(inputs) != 1 or inputs[0].nelement() != 1:
|
| 1172 |
+
raise RuntimeError(
|
| 1173 |
+
"The vector v can only be None if the input to the user-provided function "
|
| 1174 |
+
"is a single Tensor with a single element."
|
| 1175 |
+
)
|
| 1176 |
+
outputs = func(*inputs)
|
| 1177 |
+
is_outputs_tuple, outputs = _as_tuple(
|
| 1178 |
+
outputs, "outputs of the user-provided function", "hvp"
|
| 1179 |
+
)
|
| 1180 |
+
_check_requires_grad(outputs, "outputs", strict=strict)
|
| 1181 |
+
|
| 1182 |
+
if is_outputs_tuple or not isinstance(outputs[0], torch.Tensor):
|
| 1183 |
+
raise RuntimeError(
|
| 1184 |
+
"The function given to hvp should return a single Tensor"
|
| 1185 |
+
)
|
| 1186 |
+
|
| 1187 |
+
if outputs[0].nelement() != 1:
|
| 1188 |
+
raise RuntimeError(
|
| 1189 |
+
"The Tensor returned by the function given to hvp should contain a single element"
|
| 1190 |
+
)
|
| 1191 |
+
|
| 1192 |
+
jac = _autograd_grad(outputs, inputs, create_graph=True)
|
| 1193 |
+
_check_requires_grad(jac, "jacobian", strict=strict)
|
| 1194 |
+
|
| 1195 |
+
grad_jac = tuple(torch.zeros_like(inp, requires_grad=True) for inp in inputs)
|
| 1196 |
+
|
| 1197 |
+
double_back = _autograd_grad(jac, inputs, grad_jac, create_graph=True)
|
| 1198 |
+
_check_requires_grad(jac, "hessian", strict=strict)
|
| 1199 |
+
|
| 1200 |
+
enable_grad = True if create_graph else torch.is_grad_enabled()
|
| 1201 |
+
with torch.set_grad_enabled(enable_grad):
|
| 1202 |
+
grad_res = _autograd_grad(double_back, grad_jac, v, create_graph=create_graph)
|
| 1203 |
+
hvp = _fill_in_zeros(
|
| 1204 |
+
grad_res, inputs, strict, create_graph, "double_back_trick"
|
| 1205 |
+
)
|
| 1206 |
+
|
| 1207 |
+
outputs = _grad_postprocess(outputs, create_graph)
|
| 1208 |
+
hvp = _grad_postprocess(hvp, create_graph)
|
| 1209 |
+
|
| 1210 |
+
return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess(
|
| 1211 |
+
hvp, is_inputs_tuple
|
| 1212 |
+
)
|
western_ai/usa/pytorch/torch/autograd/grad_mode.py
ADDED
|
@@ -0,0 +1,489 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
from typing import Any
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from torch.utils._contextlib import (
|
| 6 |
+
_DecoratorContextManager,
|
| 7 |
+
_NoParamDecoratorContextManager,
|
| 8 |
+
F,
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
__all__ = [
|
| 13 |
+
"no_grad",
|
| 14 |
+
"enable_grad",
|
| 15 |
+
"set_grad_enabled",
|
| 16 |
+
"inference_mode",
|
| 17 |
+
"set_multithreading_enabled",
|
| 18 |
+
"enforce_grad_layout_policy",
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class no_grad(_NoParamDecoratorContextManager):
|
| 23 |
+
r"""Context-manager that disables gradient calculation.
|
| 24 |
+
|
| 25 |
+
Disabling gradient calculation is useful for inference, when you are sure
|
| 26 |
+
that you will not call :meth:`Tensor.backward()`. It will reduce memory
|
| 27 |
+
consumption for computations that would otherwise have `requires_grad=True`.
|
| 28 |
+
|
| 29 |
+
In this mode, the result of every computation will have
|
| 30 |
+
`requires_grad=False`, even when the inputs have `requires_grad=True`.
|
| 31 |
+
There is an exception! All factory functions, or functions that create
|
| 32 |
+
a new Tensor and take a requires_grad kwarg, will NOT be affected by
|
| 33 |
+
this mode.
|
| 34 |
+
|
| 35 |
+
This context manager is thread local; it will not affect computation
|
| 36 |
+
in other threads.
|
| 37 |
+
|
| 38 |
+
Also functions as a decorator.
|
| 39 |
+
|
| 40 |
+
.. note::
|
| 41 |
+
No-grad is one of several mechanisms that can enable or
|
| 42 |
+
disable gradients locally see :ref:`locally-disable-grad-doc` for
|
| 43 |
+
more information on how they compare.
|
| 44 |
+
|
| 45 |
+
.. note::
|
| 46 |
+
This API does not apply to :ref:`forward-mode AD <forward-mode-ad>`.
|
| 47 |
+
If you want to disable forward AD for a computation, you can unpack
|
| 48 |
+
your dual tensors.
|
| 49 |
+
|
| 50 |
+
Example::
|
| 51 |
+
>>> # xdoctest: +SKIP
|
| 52 |
+
>>> x = torch.tensor([1.], requires_grad=True)
|
| 53 |
+
>>> with torch.no_grad():
|
| 54 |
+
... y = x * 2
|
| 55 |
+
>>> y.requires_grad
|
| 56 |
+
False
|
| 57 |
+
>>> @torch.no_grad()
|
| 58 |
+
... def doubler(x):
|
| 59 |
+
... return x * 2
|
| 60 |
+
>>> z = doubler(x)
|
| 61 |
+
>>> z.requires_grad
|
| 62 |
+
False
|
| 63 |
+
>>> @torch.no_grad()
|
| 64 |
+
... def tripler(x):
|
| 65 |
+
... return x * 3
|
| 66 |
+
>>> z = tripler(x)
|
| 67 |
+
>>> z.requires_grad
|
| 68 |
+
False
|
| 69 |
+
>>> # factory function exception
|
| 70 |
+
>>> with torch.no_grad():
|
| 71 |
+
... a = torch.nn.Parameter(torch.rand(10))
|
| 72 |
+
>>> a.requires_grad
|
| 73 |
+
True
|
| 74 |
+
"""
|
| 75 |
+
|
| 76 |
+
def __init__(self) -> None:
|
| 77 |
+
if not torch._jit_internal.is_scripting():
|
| 78 |
+
super().__init__()
|
| 79 |
+
self.prev = False
|
| 80 |
+
|
| 81 |
+
def __enter__(self) -> None:
|
| 82 |
+
self.prev = torch.is_grad_enabled()
|
| 83 |
+
torch.set_grad_enabled(False)
|
| 84 |
+
|
| 85 |
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
| 86 |
+
torch.set_grad_enabled(self.prev)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
class enable_grad(_NoParamDecoratorContextManager):
|
| 90 |
+
r"""Context-manager that enables gradient calculation.
|
| 91 |
+
|
| 92 |
+
Enables gradient calculation, if it has been disabled via :class:`~no_grad`
|
| 93 |
+
or :class:`~set_grad_enabled`.
|
| 94 |
+
|
| 95 |
+
This context manager is thread local; it will not affect computation
|
| 96 |
+
in other threads.
|
| 97 |
+
|
| 98 |
+
Also functions as a decorator.
|
| 99 |
+
|
| 100 |
+
.. note::
|
| 101 |
+
enable_grad is one of several mechanisms that can enable or
|
| 102 |
+
disable gradients locally see :ref:`locally-disable-grad-doc` for
|
| 103 |
+
more information on how they compare.
|
| 104 |
+
|
| 105 |
+
.. note::
|
| 106 |
+
This API does not apply to :ref:`forward-mode AD <forward-mode-ad>`.
|
| 107 |
+
|
| 108 |
+
Example::
|
| 109 |
+
>>> # xdoctest: +SKIP
|
| 110 |
+
>>> x = torch.tensor([1.], requires_grad=True)
|
| 111 |
+
>>> with torch.no_grad():
|
| 112 |
+
... with torch.enable_grad():
|
| 113 |
+
... y = x * 2
|
| 114 |
+
>>> y.requires_grad
|
| 115 |
+
True
|
| 116 |
+
>>> y.backward()
|
| 117 |
+
>>> x.grad
|
| 118 |
+
tensor([2.])
|
| 119 |
+
>>> @torch.enable_grad()
|
| 120 |
+
... def doubler(x):
|
| 121 |
+
... return x * 2
|
| 122 |
+
>>> with torch.no_grad():
|
| 123 |
+
... z = doubler(x)
|
| 124 |
+
>>> z.requires_grad
|
| 125 |
+
True
|
| 126 |
+
>>> @torch.enable_grad()
|
| 127 |
+
... def tripler(x):
|
| 128 |
+
... return x * 3
|
| 129 |
+
>>> with torch.no_grad():
|
| 130 |
+
... z = tripler(x)
|
| 131 |
+
>>> z.requires_grad
|
| 132 |
+
True
|
| 133 |
+
|
| 134 |
+
"""
|
| 135 |
+
|
| 136 |
+
def __enter__(self) -> None:
|
| 137 |
+
self.prev = torch.is_grad_enabled()
|
| 138 |
+
torch._C._set_grad_enabled(True)
|
| 139 |
+
|
| 140 |
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
| 141 |
+
torch._C._set_grad_enabled(self.prev)
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
class set_grad_enabled(_DecoratorContextManager):
|
| 145 |
+
r"""Context-manager that sets gradient calculation on or off.
|
| 146 |
+
|
| 147 |
+
``set_grad_enabled`` will enable or disable grads based on its argument :attr:`mode`.
|
| 148 |
+
It can be used as a context-manager or as a function.
|
| 149 |
+
|
| 150 |
+
This context manager is thread local; it will not affect computation
|
| 151 |
+
in other threads.
|
| 152 |
+
|
| 153 |
+
Args:
|
| 154 |
+
mode (bool): Flag whether to enable grad (``True``), or disable
|
| 155 |
+
(``False``). This can be used to conditionally enable
|
| 156 |
+
gradients.
|
| 157 |
+
|
| 158 |
+
.. note::
|
| 159 |
+
set_grad_enabled is one of several mechanisms that can enable or
|
| 160 |
+
disable gradients locally see :ref:`locally-disable-grad-doc` for
|
| 161 |
+
more information on how they compare.
|
| 162 |
+
|
| 163 |
+
.. note::
|
| 164 |
+
This API does not apply to :ref:`forward-mode AD <forward-mode-ad>`.
|
| 165 |
+
|
| 166 |
+
Example::
|
| 167 |
+
>>> # xdoctest: +SKIP
|
| 168 |
+
>>> x = torch.tensor([1.], requires_grad=True)
|
| 169 |
+
>>> is_train = False
|
| 170 |
+
>>> with torch.set_grad_enabled(is_train):
|
| 171 |
+
... y = x * 2
|
| 172 |
+
>>> y.requires_grad
|
| 173 |
+
False
|
| 174 |
+
>>> _ = torch.set_grad_enabled(True)
|
| 175 |
+
>>> y = x * 2
|
| 176 |
+
>>> y.requires_grad
|
| 177 |
+
True
|
| 178 |
+
>>> _ = torch.set_grad_enabled(False)
|
| 179 |
+
>>> y = x * 2
|
| 180 |
+
>>> y.requires_grad
|
| 181 |
+
False
|
| 182 |
+
|
| 183 |
+
"""
|
| 184 |
+
|
| 185 |
+
def __init__(self, mode: bool) -> None:
|
| 186 |
+
self.prev = torch.is_grad_enabled()
|
| 187 |
+
self.mode = mode
|
| 188 |
+
torch._C._set_grad_enabled(mode)
|
| 189 |
+
|
| 190 |
+
def __call__(self, orig_func: F) -> F:
|
| 191 |
+
torch._C._set_grad_enabled(self.prev)
|
| 192 |
+
return super().__call__(orig_func)
|
| 193 |
+
|
| 194 |
+
def __enter__(self) -> None:
|
| 195 |
+
torch._C._set_grad_enabled(self.mode)
|
| 196 |
+
|
| 197 |
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
| 198 |
+
torch._C._set_grad_enabled(self.prev)
|
| 199 |
+
|
| 200 |
+
def __str__(self) -> str:
|
| 201 |
+
return f"{torch.typename(self)}(mode={self.mode})"
|
| 202 |
+
|
| 203 |
+
def __repr__(self) -> str:
|
| 204 |
+
return str(self)
|
| 205 |
+
|
| 206 |
+
def clone(self) -> "set_grad_enabled":
|
| 207 |
+
r"""
|
| 208 |
+
Create a copy of this class
|
| 209 |
+
"""
|
| 210 |
+
return self.__class__(self.mode)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
class inference_mode(_DecoratorContextManager):
|
| 214 |
+
r"""Context manager that enables or disables inference mode.
|
| 215 |
+
|
| 216 |
+
InferenceMode is analogous to :class:`~no_grad` and should be used
|
| 217 |
+
when you are certain your operations will not interact with autograd
|
| 218 |
+
(e.g., during data loading or model evaluation). Compared to
|
| 219 |
+
:class:`~no_grad`, it removes additional overhead by disabling view
|
| 220 |
+
tracking and version counter bumps. It is also more restrictive, in
|
| 221 |
+
that tensors created in this mode cannot be used in computations
|
| 222 |
+
recorded by autograd.
|
| 223 |
+
|
| 224 |
+
This context manager is thread-local; it does not affect computation
|
| 225 |
+
in other threads.
|
| 226 |
+
|
| 227 |
+
Also functions as a decorator.
|
| 228 |
+
|
| 229 |
+
.. note::
|
| 230 |
+
Inference mode is one of several mechanisms that can locally enable
|
| 231 |
+
or disable gradients. See :ref:`locally-disable-grad-doc` for a
|
| 232 |
+
comparison. If avoiding the use of tensors created in inference mode
|
| 233 |
+
in autograd-tracked regions is difficult, consider benchmarking your
|
| 234 |
+
code with and without inference mode to weigh the performance benefits
|
| 235 |
+
against the trade-offs. You can always use :class:`~no_grad` instead.
|
| 236 |
+
|
| 237 |
+
.. note::
|
| 238 |
+
Unlike some other mechanisms that locally enable or disable grad,
|
| 239 |
+
entering inference_mode also disables :ref:`forward-mode AD <forward-mode-ad>`.
|
| 240 |
+
|
| 241 |
+
.. warning::
|
| 242 |
+
`inference_mode` does NOT automatically set the model to evaluation mode.
|
| 243 |
+
For proper inference behavior (e.g., disabling dropout, using running statistics
|
| 244 |
+
in batch normalization), you must explicitly set your model to evaluation mode using
|
| 245 |
+
`model.eval()` in addition to using this context manager.
|
| 246 |
+
|
| 247 |
+
Args:
|
| 248 |
+
mode (bool or function): Either a boolean flag to enable or disable
|
| 249 |
+
inference mode, or a Python function to decorate with inference
|
| 250 |
+
mode enabled.
|
| 251 |
+
|
| 252 |
+
Example::
|
| 253 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 254 |
+
>>> import torch
|
| 255 |
+
>>> x = torch.ones(1, 2, 3, requires_grad=True)
|
| 256 |
+
>>> with torch.inference_mode():
|
| 257 |
+
... y = x * x
|
| 258 |
+
>>> y.requires_grad
|
| 259 |
+
False
|
| 260 |
+
>>> # xdoctest: +SKIP("want string isn't quite right")
|
| 261 |
+
>>> y._version
|
| 262 |
+
Traceback (most recent call last):
|
| 263 |
+
File "<stdin>", line 1, in <module>
|
| 264 |
+
RuntimeError: Inference tensors do not track version counter.
|
| 265 |
+
>>> @torch.inference_mode()
|
| 266 |
+
... def func(x):
|
| 267 |
+
... return x * x
|
| 268 |
+
>>> out = func(x)
|
| 269 |
+
>>> out.requires_grad
|
| 270 |
+
False
|
| 271 |
+
>>> @torch.inference_mode()
|
| 272 |
+
... def doubler(x):
|
| 273 |
+
... return x * 2
|
| 274 |
+
>>> out = doubler(x)
|
| 275 |
+
>>> out.requires_grad
|
| 276 |
+
False
|
| 277 |
+
|
| 278 |
+
"""
|
| 279 |
+
|
| 280 |
+
def __init__(self, mode: bool = True) -> None:
|
| 281 |
+
if not torch._jit_internal.is_scripting():
|
| 282 |
+
super().__init__()
|
| 283 |
+
self.mode = mode
|
| 284 |
+
|
| 285 |
+
def __new__(cls, mode=True):
|
| 286 |
+
if isinstance(mode, bool):
|
| 287 |
+
return super().__new__(cls)
|
| 288 |
+
return cls()(mode)
|
| 289 |
+
|
| 290 |
+
def __enter__(self) -> None:
|
| 291 |
+
self._inference_mode_context = torch._C._InferenceMode(self.mode)
|
| 292 |
+
self._inference_mode_context.__enter__()
|
| 293 |
+
|
| 294 |
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
| 295 |
+
self._inference_mode_context.__exit__(exc_type, exc_value, traceback)
|
| 296 |
+
|
| 297 |
+
def clone(self) -> "inference_mode":
|
| 298 |
+
r"""
|
| 299 |
+
Create a copy of this class
|
| 300 |
+
"""
|
| 301 |
+
return self.__class__(self.mode)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def _enter_inference_mode(mode):
|
| 305 |
+
mode_context = torch._C._InferenceMode(mode)
|
| 306 |
+
mode_context.__enter__()
|
| 307 |
+
return mode_context
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def _exit_inference_mode(mode):
|
| 311 |
+
mode.__exit__(None, None, None)
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
class set_multithreading_enabled(_DecoratorContextManager):
|
| 315 |
+
r"""Context-manager that enables or disables multithreaded backward.
|
| 316 |
+
|
| 317 |
+
Ordinarily, when :ref:`accelerator<accelerators>` devices are in use,
|
| 318 |
+
the backward pass runs on device-specific worker threads. The engine
|
| 319 |
+
creates these threads based on the number of available devices and
|
| 320 |
+
reuses them across iterations.
|
| 321 |
+
|
| 322 |
+
When ``mode=False``, the backward pass runs on the calling thread
|
| 323 |
+
instead. ``mode=True`` restores the default behavior.
|
| 324 |
+
|
| 325 |
+
This can be used as a context-manager or as a function. It is
|
| 326 |
+
thread-local and will not affect computation in other threads.
|
| 327 |
+
|
| 328 |
+
Args:
|
| 329 |
+
mode (bool): Whether to enable multithreaded backward (``True``,
|
| 330 |
+
default) or disable (``False``).
|
| 331 |
+
|
| 332 |
+
.. note::
|
| 333 |
+
This API does not apply to :ref:`forward-mode AD <forward-mode-ad>`,
|
| 334 |
+
which never uses multithreading.
|
| 335 |
+
|
| 336 |
+
"""
|
| 337 |
+
|
| 338 |
+
def __init__(self, mode: bool) -> None:
|
| 339 |
+
self.prev = torch._C._is_multithreading_enabled()
|
| 340 |
+
torch._C._set_multithreading_enabled(mode)
|
| 341 |
+
self.mode = mode
|
| 342 |
+
|
| 343 |
+
def __enter__(self) -> None:
|
| 344 |
+
pass
|
| 345 |
+
|
| 346 |
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
| 347 |
+
torch._C._set_multithreading_enabled(self.prev)
|
| 348 |
+
|
| 349 |
+
def clone(self) -> "set_multithreading_enabled":
|
| 350 |
+
r"""
|
| 351 |
+
Create a copy of this class
|
| 352 |
+
"""
|
| 353 |
+
return self.__class__(self.mode)
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
class enforce_grad_layout_policy(_DecoratorContextManager):
|
| 357 |
+
r"""Context-manager that controls the gradient layout policy enforcement.
|
| 358 |
+
|
| 359 |
+
The gradient layout contract ensures that accumulated gradients have
|
| 360 |
+
strides matching their corresponding parameters (for non-overlapping
|
| 361 |
+
dense parameters) or are row-major contiguous (otherwise). This aids
|
| 362 |
+
performance in optimizers and distributed reducers.
|
| 363 |
+
|
| 364 |
+
When ``enable=False``, the autograd engine relaxes this enforcement:
|
| 365 |
+
|
| 366 |
+
- Stealable gradients whose layout does *not* match the parameter can
|
| 367 |
+
still be stolen directly (avoiding an extra copy).
|
| 368 |
+
- The "grad and param do not obey the gradient layout contract" warning
|
| 369 |
+
is suppressed.
|
| 370 |
+
|
| 371 |
+
The logic that *creates* a brand-new gradient (e.g., cloning into the
|
| 372 |
+
correct layout when the gradient is not stealable) is **not** affected
|
| 373 |
+
by this flag.
|
| 374 |
+
|
| 375 |
+
This can be used as a context-manager or as a function. It is
|
| 376 |
+
thread-local and will not affect computation in other threads.
|
| 377 |
+
|
| 378 |
+
Args:
|
| 379 |
+
enable (bool): Whether to enforce the gradient layout contract
|
| 380 |
+
(``True``, default) or relax enforcement (``False``).
|
| 381 |
+
|
| 382 |
+
Example::
|
| 383 |
+
>>> # xdoctest: +SKIP
|
| 384 |
+
>>> import torch
|
| 385 |
+
>>> p = torch.empty(2, 3, 4).permute(2, 0, 1).requires_grad_()
|
| 386 |
+
>>> with torch.autograd.enforce_grad_layout_policy(False):
|
| 387 |
+
... (p * 2).sum().backward()
|
| 388 |
+
... # p.grad may now have the same strides as the incoming
|
| 389 |
+
... # gradient rather than being forced to match p's strides.
|
| 390 |
+
"""
|
| 391 |
+
|
| 392 |
+
def __init__(self, enable: bool = True) -> None:
|
| 393 |
+
self.prev = torch._C._is_grad_layout_enforcement_enabled()
|
| 394 |
+
torch._C._set_grad_layout_enforcement_enabled(enable)
|
| 395 |
+
self.mode = enable
|
| 396 |
+
|
| 397 |
+
def __call__(self, orig_func: F) -> F:
|
| 398 |
+
torch._C._set_grad_layout_enforcement_enabled(self.prev)
|
| 399 |
+
return super().__call__(orig_func)
|
| 400 |
+
|
| 401 |
+
def __enter__(self) -> None:
|
| 402 |
+
pass
|
| 403 |
+
|
| 404 |
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
| 405 |
+
torch._C._set_grad_layout_enforcement_enabled(self.prev)
|
| 406 |
+
|
| 407 |
+
def clone(self) -> "enforce_grad_layout_policy":
|
| 408 |
+
r"""
|
| 409 |
+
Create a copy of this class
|
| 410 |
+
"""
|
| 411 |
+
return self.__class__(self.mode)
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
class _force_original_view_tracking(_DecoratorContextManager):
|
| 415 |
+
r"""Context-manager that sets whether or not to always enable view-replay in autograd.
|
| 416 |
+
|
| 417 |
+
``set_view_replay_enabled`` will enable or disable view-replay based on its argument :attr:`mode`.
|
| 418 |
+
It can be used as a context-manager or as a function.
|
| 419 |
+
|
| 420 |
+
This context manager is thread local; it will not affect computation
|
| 421 |
+
in other threads.
|
| 422 |
+
|
| 423 |
+
When a tensor view is mutated, the autograd engine needs to decide whether or not
|
| 424 |
+
to regenerate the "updated view" by either replaying the chain of views from the updated base,
|
| 425 |
+
or with a single call to as_strided.
|
| 426 |
+
|
| 427 |
+
If set_view_replay_enabled is set to True, then autograd will always use view replay.
|
| 428 |
+
Otherwise, it will fall back to its existing logic.
|
| 429 |
+
|
| 430 |
+
Args:
|
| 431 |
+
mode (bool): Flag whether to enable view-replay (``True``), or disable
|
| 432 |
+
(``False``).
|
| 433 |
+
|
| 434 |
+
"""
|
| 435 |
+
|
| 436 |
+
def __init__(self, mode: bool) -> None:
|
| 437 |
+
self.prev = torch._C._is_view_replay_enabled()
|
| 438 |
+
self.mode = mode
|
| 439 |
+
torch._C._set_view_replay_enabled(mode)
|
| 440 |
+
|
| 441 |
+
def __call__(self, orig_func: F) -> F:
|
| 442 |
+
torch._C._set_view_replay_enabled(self.prev)
|
| 443 |
+
return super().__call__(orig_func)
|
| 444 |
+
|
| 445 |
+
def __enter__(self) -> None:
|
| 446 |
+
torch._C._set_view_replay_enabled(self.mode)
|
| 447 |
+
|
| 448 |
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
| 449 |
+
torch._C._set_view_replay_enabled(self.prev)
|
| 450 |
+
|
| 451 |
+
def clone(self):
|
| 452 |
+
return self.__class__(self.mode)
|
| 453 |
+
|
| 454 |
+
|
| 455 |
+
class _unsafe_preserve_version_counter(_DecoratorContextManager):
|
| 456 |
+
r"""DO NOT USE THIS UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING.
|
| 457 |
+
|
| 458 |
+
This context manager can lead to arbitrary silent-correctness issues in any other part of your code
|
| 459 |
+
(even the ones not touched directly by the context manager)!
|
| 460 |
+
|
| 461 |
+
Ordinarily, autograd will track mutations to tensors by incrementing its `._version` attribute.
|
| 462 |
+
This is generally important for correctness, as for example, mutating a tensor that autograd has saved
|
| 463 |
+
for the backwards pass can result in incorrect gradients, and autograd uses the version counter to detect
|
| 464 |
+
and error out in this situation.
|
| 465 |
+
|
| 466 |
+
However, there are rare instances where it might be useful to hide mutations from autograd. For example:
|
| 467 |
+
if a tensor is very large, and you'd like to free its memory by storing it elsewhere, and re-populate
|
| 468 |
+
the tensor right before it is needed by autograd.
|
| 469 |
+
|
| 470 |
+
Args:
|
| 471 |
+
tensor (torch.Tensor): the tensor in question, that you would like to preserve the version counter of.
|
| 472 |
+
|
| 473 |
+
.. note::
|
| 474 |
+
This API does not apply to :ref:`forward-mode AD <forward-mode-ad>`.
|
| 475 |
+
|
| 476 |
+
"""
|
| 477 |
+
|
| 478 |
+
def __init__(self, tensors: torch.Tensor | tuple[torch.Tensor, ...]) -> None:
|
| 479 |
+
self.tensors = (tensors,) if isinstance(tensors, torch.Tensor) else tensors
|
| 480 |
+
if not isinstance(self.tensors, tuple):
|
| 481 |
+
raise AssertionError("Expected tensors to be a tuple")
|
| 482 |
+
self.prev_versions = tuple(t._version for t in self.tensors)
|
| 483 |
+
|
| 484 |
+
def __enter__(self) -> None:
|
| 485 |
+
pass
|
| 486 |
+
|
| 487 |
+
# pyrefly: ignore [bad-override]
|
| 488 |
+
def __exit__(self, *args) -> None:
|
| 489 |
+
torch._C._autograd._unsafe_set_version_counter(self.tensors, self.prev_versions)
|
western_ai/usa/pytorch/torch/autograd/gradcheck.py
ADDED
|
@@ -0,0 +1,2324 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import collections
|
| 3 |
+
import functools
|
| 4 |
+
import warnings
|
| 5 |
+
from collections.abc import Callable, Iterable
|
| 6 |
+
from itertools import product
|
| 7 |
+
from typing_extensions import deprecated
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
import torch.testing
|
| 11 |
+
|
| 12 |
+
# pyrefly: ignore [deprecated]
|
| 13 |
+
from torch._vmap_internals import _vmap, vmap
|
| 14 |
+
from torch.overrides import is_tensor_like
|
| 15 |
+
from torch.types import _TensorOrOptionalTensors, _TensorOrTensors
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
# Note: `get_*_jacobian` functions are added here even though we didn't intend to make them public
|
| 19 |
+
# since they have been exposed from before we added `__all__` and we already maintain BC for them
|
| 20 |
+
# We should eventually deprecate them and remove them from `__all__`
|
| 21 |
+
__all__ = [
|
| 22 |
+
"gradcheck",
|
| 23 |
+
"gradgradcheck",
|
| 24 |
+
"GradcheckError",
|
| 25 |
+
"get_numerical_jacobian",
|
| 26 |
+
"get_analytical_jacobian",
|
| 27 |
+
"get_numerical_jacobian_wrt_specific_input",
|
| 28 |
+
]
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class GradcheckError(RuntimeError):
|
| 32 |
+
r"""Error raised by :func:`gradcheck` and :func:`gradgradcheck`."""
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _is_sparse_compressed_tensor(obj: torch.Tensor):
|
| 36 |
+
return obj.layout in {
|
| 37 |
+
torch.sparse_csr,
|
| 38 |
+
torch.sparse_csc,
|
| 39 |
+
torch.sparse_bsr,
|
| 40 |
+
torch.sparse_bsc,
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _is_sparse_any_tensor(obj: torch.Tensor):
|
| 45 |
+
return _is_sparse_compressed_tensor(obj) or obj.layout is torch.sparse_coo
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _is_float_or_complex_tensor(obj):
|
| 49 |
+
return is_tensor_like(obj) and (obj.is_floating_point() or obj.is_complex())
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _allocate_jacobians_with_inputs(
|
| 53 |
+
input_tensors: tuple, numel_output
|
| 54 |
+
) -> tuple[torch.Tensor, ...]:
|
| 55 |
+
# Makes zero-filled tensors from inputs. If `numel_output` is not None, for
|
| 56 |
+
# each tensor in `input_tensors`, returns a new zero-filled tensor with height
|
| 57 |
+
# of `t.numel` and width of `numel_output`. Otherwise, for each tensor, returns
|
| 58 |
+
# a 1-d tensor with size `(t.numel,)`. Each new tensor will be strided and have
|
| 59 |
+
# the same dtype and device as those of the corresponding input.
|
| 60 |
+
out: list[torch.Tensor] = [
|
| 61 |
+
t.new_zeros((t.numel(), numel_output), layout=torch.strided)
|
| 62 |
+
for t in input_tensors
|
| 63 |
+
if _is_float_or_complex_tensor(t) and t.requires_grad
|
| 64 |
+
]
|
| 65 |
+
return tuple(out)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _allocate_jacobians_with_outputs(
|
| 69 |
+
output_tensors: tuple, numel_input, dtype=None, device=None
|
| 70 |
+
) -> tuple[torch.Tensor, ...]:
|
| 71 |
+
# Makes zero-filled tensors from outputs. If `dim` is not None, for each tensor
|
| 72 |
+
# in `output_tensors`, returns a new zero-filled tensor with height of `dim` and
|
| 73 |
+
# width of `t.numel`. Otherwise, for each tensor, returns a 1-d tensor with size
|
| 74 |
+
# (t.numel,).
|
| 75 |
+
options = {"dtype": dtype, "device": device, "layout": torch.strided}
|
| 76 |
+
out: list[torch.Tensor] = [
|
| 77 |
+
t.new_zeros((numel_input, t.numel()), **options)
|
| 78 |
+
for t in output_tensors
|
| 79 |
+
if _is_float_or_complex_tensor(t)
|
| 80 |
+
]
|
| 81 |
+
return tuple(out)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def _iter_tensors(
|
| 85 |
+
x: torch.Tensor | Iterable[torch.Tensor], only_requiring_grad: bool = False
|
| 86 |
+
) -> Iterable[torch.Tensor]:
|
| 87 |
+
if is_tensor_like(x):
|
| 88 |
+
# mypy doesn't narrow type of `x` to torch.Tensor
|
| 89 |
+
if x.requires_grad or not only_requiring_grad: # type: ignore[union-attr]
|
| 90 |
+
yield x # type: ignore[misc]
|
| 91 |
+
elif isinstance(x, collections.abc.Iterable) and not isinstance(x, str):
|
| 92 |
+
for elem in x:
|
| 93 |
+
yield from _iter_tensors(elem, only_requiring_grad)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _densify(x):
|
| 97 |
+
# return a copy of sparse x with all unspecified elements
|
| 98 |
+
# "replaced" with zero-valued elements
|
| 99 |
+
if isinstance(x, (list, tuple)):
|
| 100 |
+
return type(x)(map(_densify, x))
|
| 101 |
+
elif not is_tensor_like(x) or x.layout in {torch.strided, torch._mkldnn}: # type: ignore[attr-defined] # no attr _mkldnn
|
| 102 |
+
return x
|
| 103 |
+
elif x.layout is torch.sparse_coo:
|
| 104 |
+
device = x.device
|
| 105 |
+
indices_dtype = x._indices().dtype
|
| 106 |
+
tmp = torch.ones(x.shape[: x.sparse_dim()], dtype=torch.int8, device=device)
|
| 107 |
+
indices = tmp.nonzero().t().to(dtype=indices_dtype)
|
| 108 |
+
values = torch.zeros(
|
| 109 |
+
(tmp.numel(), *x.shape[x.sparse_dim() :]), dtype=x.dtype, device=device
|
| 110 |
+
)
|
| 111 |
+
x_coalesced = x.detach().coalesce()
|
| 112 |
+
if x_coalesced.numel() > 0:
|
| 113 |
+
stride = tmp.stride()
|
| 114 |
+
flat_indices = (
|
| 115 |
+
x_coalesced.indices()
|
| 116 |
+
.mul(
|
| 117 |
+
torch.tensor(stride, dtype=indices_dtype, device=device).unsqueeze(
|
| 118 |
+
1
|
| 119 |
+
)
|
| 120 |
+
)
|
| 121 |
+
.sum(0)
|
| 122 |
+
)
|
| 123 |
+
values[flat_indices] = x_coalesced.values()
|
| 124 |
+
return (
|
| 125 |
+
torch.sparse_coo_tensor(indices, values, x.shape)
|
| 126 |
+
._coalesced_(True)
|
| 127 |
+
.requires_grad_(x.requires_grad)
|
| 128 |
+
)
|
| 129 |
+
elif _is_sparse_compressed_tensor(x):
|
| 130 |
+
blocksize = (
|
| 131 |
+
x.values().shape[1:3]
|
| 132 |
+
if x.layout in {torch.sparse_bsr, torch.sparse_bsc}
|
| 133 |
+
else None
|
| 134 |
+
)
|
| 135 |
+
compressed_indices = (
|
| 136 |
+
x.crow_indices()
|
| 137 |
+
if x.layout in {torch.sparse_csr, torch.sparse_bsr}
|
| 138 |
+
else x.ccol_indices()
|
| 139 |
+
)
|
| 140 |
+
# We'll use intermediate sparse COO for simplicity
|
| 141 |
+
r = _densify(x.detach().to_sparse(layout=torch.sparse_coo)).to_sparse(
|
| 142 |
+
layout=x.layout, blocksize=blocksize
|
| 143 |
+
)
|
| 144 |
+
# Check that all elements are specified also after `to_sparse` op:
|
| 145 |
+
dense_numel = r.values().numel() // max(1, r.values().shape[0])
|
| 146 |
+
batch_numel = compressed_indices.numel() // compressed_indices.shape[-1]
|
| 147 |
+
sparse_numel = r.numel() // max(1, dense_numel * batch_numel)
|
| 148 |
+
if sparse_numel != r._nnz():
|
| 149 |
+
raise AssertionError(
|
| 150 |
+
f"{x.layout} densify failed: expected nnz={sparse_numel} but got {r._nnz()}"
|
| 151 |
+
)
|
| 152 |
+
return r.requires_grad_(x.requires_grad)
|
| 153 |
+
elif _is_sparse_any_tensor(x):
|
| 154 |
+
raise NotImplementedError(x.layout)
|
| 155 |
+
return x
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def _iter_tensor(x_tensor):
|
| 159 |
+
# (Only used for slow gradcheck) Returns a generator that yields the following
|
| 160 |
+
# elements at each iteration:
|
| 161 |
+
# 1) a tensor: the same tensor is returned across all iterations. The tensor
|
| 162 |
+
# is not the same as the original x_tensor as given as input - it is
|
| 163 |
+
# prepared so that it can be modified in-place. Depending on whether the
|
| 164 |
+
# input tensor is strided, sparse, or dense, the returned tensor may or may
|
| 165 |
+
# not share storage with x_tensor.
|
| 166 |
+
# 2) a tuple of indices that can be used with advanced indexing (yielded in
|
| 167 |
+
# dictionary order)
|
| 168 |
+
# 3) flattened index that will be used to index into the Jacobian tensor
|
| 169 |
+
#
|
| 170 |
+
# For a tensor t with size (2, 2), _iter_tensor yields:
|
| 171 |
+
# `x, (0, 0), 0`, `x, (0, 1), 1`, `x, (1, 0), 2`, `x, (1, 1), 3`
|
| 172 |
+
#
|
| 173 |
+
# where x is the t.data of the original tensor. Perturbing the entry of x
|
| 174 |
+
# at index (1, 1) yields the 3rd column of the overall Jacobian matrix.
|
| 175 |
+
if _is_sparse_any_tensor(x_tensor):
|
| 176 |
+
|
| 177 |
+
def get_stride(size):
|
| 178 |
+
dim = len(size)
|
| 179 |
+
tmp = 1
|
| 180 |
+
stride = [0] * dim
|
| 181 |
+
for i in reversed(range(dim)):
|
| 182 |
+
stride[i] = tmp
|
| 183 |
+
tmp *= size[i]
|
| 184 |
+
return stride
|
| 185 |
+
|
| 186 |
+
x_nnz = x_tensor._nnz()
|
| 187 |
+
x_size = list(x_tensor.size())
|
| 188 |
+
if x_tensor.layout is torch.sparse_coo:
|
| 189 |
+
x_indices = x_tensor._indices().t()
|
| 190 |
+
x_values = x_tensor._values()
|
| 191 |
+
elif x_tensor.layout is torch.sparse_csr:
|
| 192 |
+
x_indices = torch._convert_indices_from_csr_to_coo(
|
| 193 |
+
x_tensor.crow_indices(), x_tensor.col_indices()
|
| 194 |
+
).t()
|
| 195 |
+
x_values = x_tensor.values()
|
| 196 |
+
elif x_tensor.layout is torch.sparse_csc:
|
| 197 |
+
x_indices = torch._convert_indices_from_csr_to_coo(
|
| 198 |
+
x_tensor.ccol_indices(), x_tensor.row_indices(), transpose=True
|
| 199 |
+
).t()
|
| 200 |
+
x_values = x_tensor.values()
|
| 201 |
+
elif x_tensor.layout is torch.sparse_bsr:
|
| 202 |
+
x_block_values = x_tensor.values()
|
| 203 |
+
x_blocksize = x_block_values.size()[1:3]
|
| 204 |
+
x_indices = (
|
| 205 |
+
torch._convert_indices_from_csr_to_coo(
|
| 206 |
+
x_tensor.crow_indices(), x_tensor.col_indices()
|
| 207 |
+
)
|
| 208 |
+
.repeat_interleave(x_blocksize[0] * x_blocksize[1], 1)
|
| 209 |
+
.mul_(torch.tensor(x_blocksize, device=x_tensor.device).reshape(2, 1))
|
| 210 |
+
.add_(
|
| 211 |
+
torch.stack(
|
| 212 |
+
torch.where(torch.ones(x_blocksize, device=x_tensor.device))
|
| 213 |
+
).repeat(1, x_nnz)
|
| 214 |
+
)
|
| 215 |
+
.t()
|
| 216 |
+
)
|
| 217 |
+
x_values = x_block_values.flatten(0, 2)
|
| 218 |
+
x_nnz = x_values.size(0)
|
| 219 |
+
elif x_tensor.layout is torch.sparse_bsc:
|
| 220 |
+
x_block_values = x_tensor.values()
|
| 221 |
+
x_blocksize = x_block_values.size()[1:3]
|
| 222 |
+
x_indices = (
|
| 223 |
+
torch._convert_indices_from_csr_to_coo(
|
| 224 |
+
x_tensor.ccol_indices(), x_tensor.row_indices(), transpose=True
|
| 225 |
+
)
|
| 226 |
+
.repeat_interleave(x_blocksize[0] * x_blocksize[1], 1)
|
| 227 |
+
.mul_(torch.tensor(x_blocksize, device=x_tensor.device).reshape(2, 1))
|
| 228 |
+
.add_(
|
| 229 |
+
torch.stack(
|
| 230 |
+
torch.where(torch.ones(x_blocksize, device=x_tensor.device))
|
| 231 |
+
).repeat(1, x_nnz)
|
| 232 |
+
)
|
| 233 |
+
.t()
|
| 234 |
+
)
|
| 235 |
+
x_values = x_block_values.flatten(0, 2)
|
| 236 |
+
x_nnz = x_values.size(0)
|
| 237 |
+
else:
|
| 238 |
+
raise NotImplementedError(f"_iter_tensor for {x_tensor.layout} input")
|
| 239 |
+
x_stride = get_stride(x_size)
|
| 240 |
+
# Use .data here to get around the version check
|
| 241 |
+
x_values = x_values.data
|
| 242 |
+
for i in range(x_nnz):
|
| 243 |
+
x_value = x_values[i]
|
| 244 |
+
for x_idx in product(*[range(m) for m in x_values.size()[1:]]):
|
| 245 |
+
indices = x_indices[i].tolist() + list(x_idx)
|
| 246 |
+
d_idx = sum(indices[k] * x_stride[k] for k in range(len(x_size)))
|
| 247 |
+
yield x_value, x_idx, d_idx
|
| 248 |
+
elif x_tensor.layout == torch._mkldnn: # type: ignore[attr-defined]
|
| 249 |
+
for d_idx, x_idx in enumerate(product(*[range(m) for m in x_tensor.size()])):
|
| 250 |
+
# this is really inefficient, but without indexing implemented, there's
|
| 251 |
+
# not really a better way than converting back and forth
|
| 252 |
+
x_tensor_dense = x_tensor.to_dense()
|
| 253 |
+
yield x_tensor_dense, x_idx, d_idx
|
| 254 |
+
else:
|
| 255 |
+
# Use .data here to get around the version check
|
| 256 |
+
x_tensor = x_tensor.data
|
| 257 |
+
for d_idx, x_idx in enumerate(product(*[range(m) for m in x_tensor.size()])):
|
| 258 |
+
yield x_tensor, x_idx, d_idx
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def _get_numerical_jacobian(
|
| 262 |
+
fn, inputs, outputs=None, target=None, eps=1e-3, is_forward_ad=False
|
| 263 |
+
) -> list[tuple[torch.Tensor, ...]]:
|
| 264 |
+
"""Compute the numerical Jacobian of `fn(inputs)` with respect to `target`.
|
| 265 |
+
|
| 266 |
+
If not specified, targets are the input. Returns M * N Jacobians where N is the
|
| 267 |
+
number of tensors in target that require grad and M is the number of non-integral
|
| 268 |
+
outputs.
|
| 269 |
+
|
| 270 |
+
Args:
|
| 271 |
+
fn: the function to compute the jacobian for
|
| 272 |
+
inputs: inputs to `fn`
|
| 273 |
+
outputs: provide precomputed outputs to avoid one extra invocation of fn
|
| 274 |
+
target: the Tensors wrt whom Jacobians are calculated (default=`inputs`)
|
| 275 |
+
eps: the magnitude of the perturbation during finite differencing
|
| 276 |
+
(default=`1e-3`)
|
| 277 |
+
is_forward_ad: if this numerical jacobian is computed to be checked wrt
|
| 278 |
+
forward AD gradients (this is used for error checking only)
|
| 279 |
+
|
| 280 |
+
Returns:
|
| 281 |
+
A list of M N-tuples of tensors
|
| 282 |
+
|
| 283 |
+
Note that `target` may not even be part of `input` to `fn`, so please be
|
| 284 |
+
**very careful** in this to not clone `target`.
|
| 285 |
+
"""
|
| 286 |
+
jacobians: list[tuple[torch.Tensor, ...]] = []
|
| 287 |
+
if outputs is None:
|
| 288 |
+
outputs = _as_tuple(fn(*_as_tuple(inputs)))
|
| 289 |
+
if not is_forward_ad and any(o.is_complex() for o in outputs):
|
| 290 |
+
raise ValueError(
|
| 291 |
+
"Expected output to be non-complex. get_numerical_jacobian no "
|
| 292 |
+
"longer supports functions that return complex outputs."
|
| 293 |
+
)
|
| 294 |
+
if target is None:
|
| 295 |
+
target = inputs
|
| 296 |
+
inp_indices = [
|
| 297 |
+
i for i, a in enumerate(target) if is_tensor_like(a) and a.requires_grad
|
| 298 |
+
]
|
| 299 |
+
for inp, inp_idx in zip(_iter_tensors(target, True), inp_indices):
|
| 300 |
+
jacobians += [
|
| 301 |
+
get_numerical_jacobian_wrt_specific_input(
|
| 302 |
+
fn,
|
| 303 |
+
inp_idx,
|
| 304 |
+
inputs,
|
| 305 |
+
outputs,
|
| 306 |
+
eps,
|
| 307 |
+
input=inp,
|
| 308 |
+
is_forward_ad=is_forward_ad,
|
| 309 |
+
)
|
| 310 |
+
]
|
| 311 |
+
return jacobians
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
@deprecated(
|
| 315 |
+
"`get_numerical_jacobian` was part of PyTorch's private API and not "
|
| 316 |
+
"meant to be exposed. We are deprecating it and it will be removed "
|
| 317 |
+
"in a future version of PyTorch. If you have a specific use for "
|
| 318 |
+
"this or feature request for this to be a stable API, please file "
|
| 319 |
+
"us an issue at https://github.com/pytorch/pytorch/issues/new",
|
| 320 |
+
category=FutureWarning,
|
| 321 |
+
)
|
| 322 |
+
def get_numerical_jacobian(fn, inputs, target=None, eps=1e-3, grad_out=1.0):
|
| 323 |
+
"""Compute the numerical Jacobian for a given fn and its inputs.
|
| 324 |
+
|
| 325 |
+
This is a Deprecated API.
|
| 326 |
+
|
| 327 |
+
Args:
|
| 328 |
+
fn: the function to compute the Jacobian for (must take inputs as a tuple)
|
| 329 |
+
inputs: input to `fn`
|
| 330 |
+
target: the Tensors wrt whom Jacobians are calculated (default=`input`)
|
| 331 |
+
eps: the magnitude of the perturbation during finite differencing
|
| 332 |
+
(default=`1e-3`)
|
| 333 |
+
grad_out: defaults to 1.0.
|
| 334 |
+
|
| 335 |
+
Returns:
|
| 336 |
+
A list of Jacobians of `fn` (restricted to its first output) with respect to
|
| 337 |
+
each input or target, if provided.
|
| 338 |
+
|
| 339 |
+
Note that `target` may not even be part of `input` to `fn`, so please be
|
| 340 |
+
**very careful** in this to not clone `target`.
|
| 341 |
+
"""
|
| 342 |
+
if (
|
| 343 |
+
grad_out != 1.0
|
| 344 |
+
): # grad_out param is only kept for backward compatibility reasons
|
| 345 |
+
raise ValueError(
|
| 346 |
+
"Expected grad_out to be 1.0. get_numerical_jacobian no longer "
|
| 347 |
+
"supports values of grad_out != 1.0."
|
| 348 |
+
)
|
| 349 |
+
|
| 350 |
+
def fn_pack_inps(*inps):
|
| 351 |
+
return fn(inps)
|
| 352 |
+
|
| 353 |
+
jacobians = _get_numerical_jacobian(fn_pack_inps, inputs, None, target, eps)
|
| 354 |
+
|
| 355 |
+
return tuple(jacobian_for_each_output[0] for jacobian_for_each_output in jacobians)
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def _compute_numerical_gradient(fn, entry, v, norm_v, nbhd_checks_fn):
|
| 359 |
+
# Computes numerical directional derivative as finite difference
|
| 360 |
+
# of function `fn` at input `entry`, perturbed by vector `v`.
|
| 361 |
+
if _is_sparse_compressed_tensor(entry):
|
| 362 |
+
# sparse compressed tensors don't implement sub/add/copy_
|
| 363 |
+
# yet. However, in non-masked semantics context entry and v
|
| 364 |
+
# have the same sparse indices ...
|
| 365 |
+
if entry.layout != v.layout:
|
| 366 |
+
raise AssertionError(
|
| 367 |
+
f"Expected entry and v to have the same layout, but got {entry.layout} and {v.layout}"
|
| 368 |
+
)
|
| 369 |
+
if entry._nnz() != v._nnz():
|
| 370 |
+
raise AssertionError(
|
| 371 |
+
f"Expected entry and v to have the same nnz, but got {entry._nnz()} and {v._nnz()} "
|
| 372 |
+
f"with entry shape {entry.shape}"
|
| 373 |
+
)
|
| 374 |
+
# ... the finite differencing can be performed on values only:
|
| 375 |
+
entry = entry.values()
|
| 376 |
+
v = v.values()
|
| 377 |
+
# we'll detach to avoid backward computations that sparse
|
| 378 |
+
# tensors have limited support for.
|
| 379 |
+
entry = entry.detach()
|
| 380 |
+
|
| 381 |
+
orig = entry.clone()
|
| 382 |
+
entry.copy_(orig - v)
|
| 383 |
+
outa = fn()
|
| 384 |
+
entry.copy_(orig + v)
|
| 385 |
+
outb = fn()
|
| 386 |
+
entry.copy_(orig)
|
| 387 |
+
|
| 388 |
+
def compute(a, b):
|
| 389 |
+
nbhd_checks_fn(a, b)
|
| 390 |
+
ret = (b - a) / (2 * norm_v) # use central difference approx
|
| 391 |
+
return ret.detach().reshape(-1)
|
| 392 |
+
|
| 393 |
+
return tuple(compute(a, b) for (a, b) in zip(outa, outb))
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
def _compute_numerical_jvps_wrt_specific_input(
|
| 397 |
+
jvp_fn, delta, input_is_complex, is_forward_ad=False
|
| 398 |
+
) -> list[torch.Tensor]:
|
| 399 |
+
# Computing the jacobian only works for real delta
|
| 400 |
+
# For details on the algorithm used here, refer:
|
| 401 |
+
# Section 3.5.3 https://arxiv.org/pdf/1701.00392.pdf
|
| 402 |
+
# s = fn(z) where z = x for real valued input
|
| 403 |
+
# and z = x + yj for complex valued input
|
| 404 |
+
jvps: list[torch.Tensor] = []
|
| 405 |
+
ds_dx_tup = jvp_fn(delta[0] if isinstance(delta, tuple) else delta)
|
| 406 |
+
|
| 407 |
+
if input_is_complex: # C -> R
|
| 408 |
+
ds_dy_tup = (
|
| 409 |
+
jvp_fn(delta[1] * 1j) if isinstance(delta, tuple) else jvp_fn(delta * 1j)
|
| 410 |
+
)
|
| 411 |
+
for ds_dx, ds_dy in zip(ds_dx_tup, ds_dy_tup):
|
| 412 |
+
if ds_dx.is_complex():
|
| 413 |
+
raise AssertionError("Expected ds_dx to be real-valued, not complex")
|
| 414 |
+
# conjugate wirtinger derivative
|
| 415 |
+
conj_w_d = ds_dx + ds_dy * 1j
|
| 416 |
+
jvps.append(conj_w_d)
|
| 417 |
+
else:
|
| 418 |
+
for ds_dx in ds_dx_tup: # R -> R or (R -> C for the forward AD case)
|
| 419 |
+
if not is_forward_ad and ds_dx.is_complex():
|
| 420 |
+
raise AssertionError("Expected ds_dx to be real-valued, not complex.")
|
| 421 |
+
jvps.append(ds_dx)
|
| 422 |
+
return jvps
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
def _combine_jacobian_cols(
|
| 426 |
+
jacobians_cols: dict[int, list[torch.Tensor]], outputs, input, numel
|
| 427 |
+
) -> tuple[torch.Tensor, ...]:
|
| 428 |
+
# jacobian_cols maps column_idx -> output_idx -> single column of jacobian Tensor
|
| 429 |
+
# we return a list that maps output_idx -> full jacobian Tensor
|
| 430 |
+
jacobians = _allocate_jacobians_with_outputs(
|
| 431 |
+
outputs, numel, dtype=input.dtype if input.dtype.is_complex else None
|
| 432 |
+
)
|
| 433 |
+
for i, jacobian in enumerate(jacobians):
|
| 434 |
+
for k, v in jacobians_cols.items():
|
| 435 |
+
jacobian[k] = v[i]
|
| 436 |
+
return jacobians
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def _prepare_input(
|
| 440 |
+
input: torch.Tensor, maybe_perturbed_input: torch.Tensor | None, fast_mode=False
|
| 441 |
+
) -> torch.Tensor:
|
| 442 |
+
# Prepares the inputs to be passed into the function while including the new
|
| 443 |
+
# modified input.
|
| 444 |
+
if input.layout == torch._mkldnn: # type: ignore[attr-defined] # no attr _mkldnn
|
| 445 |
+
# Convert back to mkldnn
|
| 446 |
+
if maybe_perturbed_input is not None:
|
| 447 |
+
return maybe_perturbed_input.to_mkldnn()
|
| 448 |
+
else:
|
| 449 |
+
return input
|
| 450 |
+
elif _is_sparse_any_tensor(input):
|
| 451 |
+
if fast_mode and maybe_perturbed_input is not None:
|
| 452 |
+
# entry is already a "cloned" version of the original tensor
|
| 453 |
+
# thus changes to entry are not reflected in the input
|
| 454 |
+
return maybe_perturbed_input
|
| 455 |
+
else:
|
| 456 |
+
return input
|
| 457 |
+
else:
|
| 458 |
+
# We cannot use entry (input.data) if we want gradgrad to work because
|
| 459 |
+
# fn (in the gradgrad case) needs to compute grad wrt input
|
| 460 |
+
return input
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
def _check_outputs_same_dtype_and_shape(output1, output2, eps, idx=None) -> None:
|
| 464 |
+
# Check that the returned outputs don't have different dtype or shape when you
|
| 465 |
+
# perturb the input
|
| 466 |
+
on_index = f"on index {idx} " if idx is not None else ""
|
| 467 |
+
if output1.shape != output2.shape:
|
| 468 |
+
raise AssertionError(
|
| 469 |
+
f"Expected `func` to return outputs with the same shape"
|
| 470 |
+
f" when inputs are perturbed {on_index}by {eps}, but got:"
|
| 471 |
+
f" shapes {output1.shape} and {output2.shape}."
|
| 472 |
+
)
|
| 473 |
+
if output1.dtype != output2.dtype:
|
| 474 |
+
raise AssertionError(
|
| 475 |
+
f"Expected `func` to return outputs with the same dtype"
|
| 476 |
+
f" when inputs are perturbed {on_index}by {eps}, but got:"
|
| 477 |
+
f" dtypes {output1.dtype} and {output2.dtype}."
|
| 478 |
+
)
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
def get_numerical_jacobian_wrt_specific_input(
|
| 482 |
+
fn, input_idx, inputs, outputs, eps, input=None, is_forward_ad=False
|
| 483 |
+
) -> tuple[torch.Tensor, ...]:
|
| 484 |
+
# Computes the numerical jacobians wrt to a single input. Returns N jacobian
|
| 485 |
+
# tensors, where N is the number of outputs. We use a dictionary for
|
| 486 |
+
# jacobian_cols because indices aren't necessarily consecutive for sparse inputs
|
| 487 |
+
# When we perturb only a single element of the input tensor at a time, the jvp
|
| 488 |
+
# is equivalent to a single col of the Jacobian matrix of fn.
|
| 489 |
+
jacobian_cols: dict[int, list[torch.Tensor]] = {}
|
| 490 |
+
input = inputs[input_idx] if input is None else input
|
| 491 |
+
if not input.requires_grad:
|
| 492 |
+
raise AssertionError("Expected input to have requires_grad=True")
|
| 493 |
+
for x, idx, d_idx in _iter_tensor(input):
|
| 494 |
+
wrapped_fn = _with_prepare_inputs(fn, inputs, input_idx, x)
|
| 495 |
+
input_to_perturb = x[idx]
|
| 496 |
+
nbhd_checks_fn = functools.partial(
|
| 497 |
+
_check_outputs_same_dtype_and_shape, idx=idx, eps=eps
|
| 498 |
+
)
|
| 499 |
+
jvp_fn = _get_numerical_jvp_fn(
|
| 500 |
+
wrapped_fn, input_to_perturb, eps, nbhd_checks_fn
|
| 501 |
+
)
|
| 502 |
+
jacobian_cols[d_idx] = _compute_numerical_jvps_wrt_specific_input(
|
| 503 |
+
jvp_fn, eps, x.is_complex(), is_forward_ad
|
| 504 |
+
)
|
| 505 |
+
return _combine_jacobian_cols(jacobian_cols, outputs, input, input.numel())
|
| 506 |
+
|
| 507 |
+
|
| 508 |
+
def _get_analytical_jacobian_forward_ad(
|
| 509 |
+
fn, inputs, outputs, *, check_grad_dtypes=False, all_u=None
|
| 510 |
+
) -> tuple[tuple[torch.Tensor, ...], ...]:
|
| 511 |
+
"""Compute the analytical Jacobian using forward mode AD of `fn(inputs)` using forward mode AD with respect to `target`.
|
| 512 |
+
|
| 513 |
+
Return N * M Jacobians where N is the number of tensors in target that require grad and
|
| 514 |
+
M is the number of non-integral outputs.
|
| 515 |
+
Contrary to other functions here, this function requires "inputs" to actually be used by the function.
|
| 516 |
+
The computed value is expected to be wrong if the function captures the inputs by side effect instead of
|
| 517 |
+
using the passed ones (many torch.nn tests do this).
|
| 518 |
+
|
| 519 |
+
Args:
|
| 520 |
+
fn: the function to compute the jacobian for
|
| 521 |
+
inputs: inputs to `fn`
|
| 522 |
+
outputs: provide precomputed outputs to avoid one extra invocation of fn
|
| 523 |
+
check_grad_dtypes: if True, will check that the gradient dtype are valid
|
| 524 |
+
all_u (optional): if provided, the Jacobian will be right multiplied with this vector
|
| 525 |
+
|
| 526 |
+
Returns:
|
| 527 |
+
A tuple of M N-tuples of tensors
|
| 528 |
+
"""
|
| 529 |
+
# To avoid early import issues
|
| 530 |
+
fwAD = torch.autograd.forward_ad
|
| 531 |
+
|
| 532 |
+
tensor_inputs = tuple(i for i in inputs if is_tensor_like(i) and i.requires_grad)
|
| 533 |
+
|
| 534 |
+
if any(i.is_complex() for i in tensor_inputs):
|
| 535 |
+
raise ValueError(
|
| 536 |
+
"Expected inputs to be non-complex for _get_analytical_jacobian_forward_ad."
|
| 537 |
+
)
|
| 538 |
+
|
| 539 |
+
if all_u:
|
| 540 |
+
jacobians = tuple(
|
| 541 |
+
_allocate_jacobians_with_outputs(outputs, 1) for i in tensor_inputs
|
| 542 |
+
)
|
| 543 |
+
else:
|
| 544 |
+
jacobians = tuple(
|
| 545 |
+
_allocate_jacobians_with_outputs(outputs, i.numel()) for i in tensor_inputs
|
| 546 |
+
)
|
| 547 |
+
|
| 548 |
+
with fwAD.dual_level():
|
| 549 |
+
fw_grads = []
|
| 550 |
+
dual_inputs = []
|
| 551 |
+
for inp in inputs:
|
| 552 |
+
if is_tensor_like(inp) and inp.requires_grad:
|
| 553 |
+
if inp.layout == torch._mkldnn: # type: ignore[attr-defined]
|
| 554 |
+
raise ValueError(
|
| 555 |
+
"MKLDNN inputs are not support for forward AD gradcheck."
|
| 556 |
+
)
|
| 557 |
+
|
| 558 |
+
inp = fwAD.make_dual(inp.detach(), torch.zeros_like(inp))
|
| 559 |
+
# If inp is a differentiable view, the dual might not be the tangent given to
|
| 560 |
+
# make_dual, so read it explicitly from the dual tensor
|
| 561 |
+
fw_grads.append(fwAD.unpack_dual(inp)[1])
|
| 562 |
+
dual_inputs.append(inp)
|
| 563 |
+
|
| 564 |
+
if all_u:
|
| 565 |
+
# Do the full reduction in one pass
|
| 566 |
+
# To be consistent with numerical evaluation, we actually compute one reduction per input
|
| 567 |
+
for i, (fw_grad, u) in enumerate(zip(fw_grads, all_u)):
|
| 568 |
+
fw_grad.copy_(u.view_as(fw_grad))
|
| 569 |
+
raw_outputs = _as_tuple(fn(*dual_inputs))
|
| 570 |
+
dual_outputs = filter(_is_float_or_complex_tensor, raw_outputs)
|
| 571 |
+
for index_o, d_o in enumerate(dual_outputs):
|
| 572 |
+
val, res = fwAD.unpack_dual(d_o)
|
| 573 |
+
if (
|
| 574 |
+
check_grad_dtypes
|
| 575 |
+
and res is not None
|
| 576 |
+
and val.is_complex() != res.is_complex()
|
| 577 |
+
):
|
| 578 |
+
raise GradcheckError("Forward AD gradient has dtype mismatch.")
|
| 579 |
+
|
| 580 |
+
# Remove extra dimension of size 1 corresponding to the reduced input
|
| 581 |
+
jacobians[i][index_o].squeeze_(0)
|
| 582 |
+
if res is None:
|
| 583 |
+
jacobians[i][index_o].zero_()
|
| 584 |
+
else:
|
| 585 |
+
jacobians[i][index_o].copy_(res.reshape(-1))
|
| 586 |
+
fw_grad.zero_()
|
| 587 |
+
else:
|
| 588 |
+
# Reconstruct the full Jacobian column by column
|
| 589 |
+
for i, fw_grad in enumerate(fw_grads):
|
| 590 |
+
for lin_idx, grad_idx in enumerate(
|
| 591 |
+
product(*[range(m) for m in fw_grad.size()])
|
| 592 |
+
):
|
| 593 |
+
fw_grad[grad_idx] = 1.0
|
| 594 |
+
raw_outputs = _as_tuple(fn(*dual_inputs))
|
| 595 |
+
dual_outputs = filter(_is_float_or_complex_tensor, raw_outputs)
|
| 596 |
+
for index_o, d_o in enumerate(dual_outputs):
|
| 597 |
+
val, res = fwAD.unpack_dual(d_o)
|
| 598 |
+
if (
|
| 599 |
+
check_grad_dtypes
|
| 600 |
+
and res is not None
|
| 601 |
+
and val.is_complex() != res.is_complex()
|
| 602 |
+
):
|
| 603 |
+
raise GradcheckError(
|
| 604 |
+
"Forward AD gradient has dtype mismatch."
|
| 605 |
+
)
|
| 606 |
+
|
| 607 |
+
if res is None:
|
| 608 |
+
jacobians[i][index_o][lin_idx].zero_()
|
| 609 |
+
else:
|
| 610 |
+
jacobians[i][index_o][lin_idx].copy_(res.reshape(-1))
|
| 611 |
+
fw_grad[grad_idx] = 0.0
|
| 612 |
+
|
| 613 |
+
return jacobians
|
| 614 |
+
|
| 615 |
+
|
| 616 |
+
def _get_input_to_perturb(input):
|
| 617 |
+
# Prepare the input so that it can be modified in-place and do certain
|
| 618 |
+
# operations that require the tensor to have strides. If fast_mode=False,
|
| 619 |
+
# _iter_tensor would handle the below cases:
|
| 620 |
+
if input.layout == torch._mkldnn: # type: ignore[attr-defined] # no attr _mkldnn
|
| 621 |
+
# Convert to dense so we can perform operations that require strided tensors
|
| 622 |
+
input_to_perturb = input.to_dense()
|
| 623 |
+
elif _is_sparse_any_tensor(input):
|
| 624 |
+
# Clone because input may require grad, and copy_ calls resize_,
|
| 625 |
+
# which is not allowed for .data
|
| 626 |
+
input_to_perturb = input.clone()
|
| 627 |
+
else:
|
| 628 |
+
input_to_perturb = input.data
|
| 629 |
+
return input_to_perturb
|
| 630 |
+
|
| 631 |
+
|
| 632 |
+
def _with_prepare_inputs(fn, inputs, input_idx, input_to_perturb, fast_mode=False):
|
| 633 |
+
# Wraps `fn` so that its inputs are already supplied
|
| 634 |
+
def wrapped_fn():
|
| 635 |
+
inp = tuple(
|
| 636 |
+
_prepare_input(a, input_to_perturb if i == input_idx else None, fast_mode)
|
| 637 |
+
if is_tensor_like(a)
|
| 638 |
+
else a
|
| 639 |
+
for i, a in enumerate(_as_tuple(inputs))
|
| 640 |
+
)
|
| 641 |
+
return tuple(a.clone() for a in _as_tuple(fn(*inp)))
|
| 642 |
+
|
| 643 |
+
return wrapped_fn
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
def _get_numerical_jvp_fn(wrapped_fn, input_to_perturb, eps, nbhd_checks_fn):
|
| 647 |
+
# Wraps jvp_fn so that certain arguments are already supplied
|
| 648 |
+
def jvp_fn(delta):
|
| 649 |
+
return _compute_numerical_gradient(
|
| 650 |
+
wrapped_fn, input_to_perturb, delta, eps, nbhd_checks_fn
|
| 651 |
+
)
|
| 652 |
+
|
| 653 |
+
return jvp_fn
|
| 654 |
+
|
| 655 |
+
|
| 656 |
+
def _reshape_tensor_or_tuple(u, shape):
|
| 657 |
+
# We don't need to reshape when input corresponding to u is sparse
|
| 658 |
+
if isinstance(u, tuple):
|
| 659 |
+
if not _is_sparse_any_tensor(u[0]):
|
| 660 |
+
return (u[0].reshape(shape), u[1].reshape(shape))
|
| 661 |
+
else:
|
| 662 |
+
if not _is_sparse_any_tensor(u):
|
| 663 |
+
return u.reshape(shape)
|
| 664 |
+
return u
|
| 665 |
+
|
| 666 |
+
|
| 667 |
+
def _mul_tensor_or_tuple(u, k):
|
| 668 |
+
if isinstance(u, tuple):
|
| 669 |
+
return (k * u[0], k * u[1])
|
| 670 |
+
else:
|
| 671 |
+
return k * u
|
| 672 |
+
|
| 673 |
+
|
| 674 |
+
def _get_numerical_jvp_wrt_specific_input(
|
| 675 |
+
fn, input_idx, inputs, u, eps, is_forward_ad=False
|
| 676 |
+
) -> list[torch.Tensor]:
|
| 677 |
+
input = inputs[input_idx]
|
| 678 |
+
input_to_perturb = _get_input_to_perturb(input)
|
| 679 |
+
wrapped_fn = _with_prepare_inputs(fn, inputs, input_idx, input_to_perturb, True)
|
| 680 |
+
nbhd_checks_fn = functools.partial(_check_outputs_same_dtype_and_shape, eps=eps)
|
| 681 |
+
jvp_fn = _get_numerical_jvp_fn(wrapped_fn, input_to_perturb, eps, nbhd_checks_fn)
|
| 682 |
+
u = _reshape_tensor_or_tuple(u, input_to_perturb.shape)
|
| 683 |
+
u = _mul_tensor_or_tuple(u, eps)
|
| 684 |
+
return _compute_numerical_jvps_wrt_specific_input(
|
| 685 |
+
jvp_fn, u, input.is_complex(), is_forward_ad
|
| 686 |
+
)
|
| 687 |
+
|
| 688 |
+
|
| 689 |
+
def _get_numerical_vJu(
|
| 690 |
+
fn, inputs, inp_indices, func_out, all_u, all_v, eps, is_forward_ad
|
| 691 |
+
):
|
| 692 |
+
# Note that all_v can also be None, in that case, this function only computes Ju.
|
| 693 |
+
reduced_jacobians: list[list[torch.Tensor]] = []
|
| 694 |
+
for inp_idx, u in zip(inp_indices, all_u):
|
| 695 |
+
all_Ju = _get_numerical_jvp_wrt_specific_input(
|
| 696 |
+
fn, inp_idx, inputs, u, eps, is_forward_ad
|
| 697 |
+
)
|
| 698 |
+
# Filter out the Ju for non floating point outputs
|
| 699 |
+
filtered_Ju = []
|
| 700 |
+
func_out = _as_tuple(func_out)
|
| 701 |
+
if len(all_Ju) != len(func_out):
|
| 702 |
+
raise AssertionError(
|
| 703 |
+
f"Expected all_Ju and func_out to have the same length, "
|
| 704 |
+
f"but got {len(all_Ju)} and {len(func_out)}"
|
| 705 |
+
)
|
| 706 |
+
for Ju, output in zip(all_Ju, func_out):
|
| 707 |
+
if _is_float_or_complex_tensor(output):
|
| 708 |
+
filtered_Ju.append(Ju)
|
| 709 |
+
else:
|
| 710 |
+
# TODO: handle the other Ju
|
| 711 |
+
pass
|
| 712 |
+
if all_v is not None:
|
| 713 |
+
jacobian_scalars: list[torch.Tensor] = []
|
| 714 |
+
for v, Ju in zip(all_v, filtered_Ju):
|
| 715 |
+
jacobian_scalars.append(_dot_with_type_promotion(v, Ju))
|
| 716 |
+
reduced_jacobians.append(jacobian_scalars)
|
| 717 |
+
else:
|
| 718 |
+
reduced_jacobians.append(filtered_Ju)
|
| 719 |
+
return reduced_jacobians
|
| 720 |
+
|
| 721 |
+
|
| 722 |
+
def _check_jacobians_equal(j1, j2, atol):
|
| 723 |
+
# Check whether the max difference between two Jacobian tensors are within some
|
| 724 |
+
# tolerance `atol`.
|
| 725 |
+
for j1_x, j2_x in zip(j1, j2):
|
| 726 |
+
if j1_x.numel() != 0 and (j1_x - j2_x).abs().max() > atol:
|
| 727 |
+
return False
|
| 728 |
+
return True
|
| 729 |
+
|
| 730 |
+
|
| 731 |
+
def _stack_and_check_tensors(
|
| 732 |
+
list_of_list_of_tensors, inputs, numel_outputs
|
| 733 |
+
) -> tuple[tuple[torch.Tensor, ...], bool, bool]:
|
| 734 |
+
# For the ith tensor in the inner list checks whether it has the same size and
|
| 735 |
+
# dtype as the ith differentiable input.
|
| 736 |
+
out_jacobians = _allocate_jacobians_with_inputs(inputs, numel_outputs)
|
| 737 |
+
diff_input_list = list(_iter_tensors(inputs, True))
|
| 738 |
+
correct_grad_sizes = True
|
| 739 |
+
correct_grad_types = True
|
| 740 |
+
for i, tensor_list in enumerate(list_of_list_of_tensors):
|
| 741 |
+
inp = diff_input_list[i]
|
| 742 |
+
out_jacobian = out_jacobians[i]
|
| 743 |
+
for j, tensor in enumerate(tensor_list):
|
| 744 |
+
if tensor is not None and tensor.size() != inp.size():
|
| 745 |
+
correct_grad_sizes = False
|
| 746 |
+
elif tensor is not None and tensor.dtype != inp.dtype:
|
| 747 |
+
correct_grad_types = False
|
| 748 |
+
if tensor is None:
|
| 749 |
+
out_jacobian[:, j].zero_()
|
| 750 |
+
else:
|
| 751 |
+
dense = tensor.to_dense() if tensor.layout != torch.strided else tensor
|
| 752 |
+
if out_jacobian[:, j].numel() != dense.numel():
|
| 753 |
+
raise AssertionError(
|
| 754 |
+
f"Expected out_jacobian column to have {dense.numel()} elements, "
|
| 755 |
+
f"but got {out_jacobian[:, j].numel()}"
|
| 756 |
+
)
|
| 757 |
+
out_jacobian[:, j] = dense.reshape(-1)
|
| 758 |
+
return out_jacobians, correct_grad_sizes, correct_grad_types
|
| 759 |
+
|
| 760 |
+
|
| 761 |
+
FAILED_NONDET_MSG = """\n
|
| 762 |
+
NOTE: If your op relies on non-deterministic operations i.e., it is listed here:
|
| 763 |
+
https://pytorch.org/docs/stable/generated/torch.use_deterministic_algorithms.html
|
| 764 |
+
this failure might be expected.
|
| 765 |
+
|
| 766 |
+
If you are adding a new operator, please file an issue and then use one of the
|
| 767 |
+
workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck.
|
| 768 |
+
If the test
|
| 769 |
+
- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
|
| 770 |
+
with `nondet_tol=<tol>` as a keyword argument.
|
| 771 |
+
- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
|
| 772 |
+
to have `gradcheck_nondet_tol=<tol>`.
|
| 773 |
+
- is a Module test (e.g., in common_nn.py), then modify the corresponding
|
| 774 |
+
module_test entry to have `gradcheck_nondet_tol=<tol>`
|
| 775 |
+
"""
|
| 776 |
+
|
| 777 |
+
|
| 778 |
+
def _check_analytical_jacobian_attributes(
|
| 779 |
+
inputs, output, nondet_tol, check_grad_dtypes, fast_mode=False, v=None
|
| 780 |
+
) -> tuple[torch.Tensor, ...]:
|
| 781 |
+
# This is used by both fast and slow mode:
|
| 782 |
+
# - For slow mode, vjps[i][j] is the jth row of the Jacobian wrt the ith
|
| 783 |
+
# input.
|
| 784 |
+
# - For fast mode, vjps[i][0] is a linear combination of the rows
|
| 785 |
+
# of the Jacobian wrt the ith input
|
| 786 |
+
diff_input_list = list(_iter_tensors(inputs, True))
|
| 787 |
+
|
| 788 |
+
def vjp_fn(grad_output):
|
| 789 |
+
return torch.autograd.grad(
|
| 790 |
+
output, diff_input_list, grad_output, retain_graph=True, allow_unused=True
|
| 791 |
+
)
|
| 792 |
+
|
| 793 |
+
# Compute everything twice to check for nondeterminism (which we call reentrancy)
|
| 794 |
+
if fast_mode:
|
| 795 |
+
vjps1 = _get_analytical_vjps_wrt_specific_output(vjp_fn, output.clone(), v)
|
| 796 |
+
vjps2 = _get_analytical_vjps_wrt_specific_output(vjp_fn, output.clone(), v)
|
| 797 |
+
else:
|
| 798 |
+
vjps1 = _compute_analytical_jacobian_rows(vjp_fn, output.clone())
|
| 799 |
+
vjps2 = _compute_analytical_jacobian_rows(vjp_fn, output.clone())
|
| 800 |
+
|
| 801 |
+
output_numel = output.numel() if not fast_mode else 1
|
| 802 |
+
jacobians1, types_ok, sizes_ok = _stack_and_check_tensors(
|
| 803 |
+
vjps1, inputs, output_numel
|
| 804 |
+
)
|
| 805 |
+
jacobians2, _, _ = _stack_and_check_tensors(vjps2, inputs, output_numel)
|
| 806 |
+
reentrant = _check_jacobians_equal(jacobians1, jacobians2, nondet_tol)
|
| 807 |
+
|
| 808 |
+
if not types_ok and check_grad_dtypes:
|
| 809 |
+
raise GradcheckError("Gradient has dtype mismatch")
|
| 810 |
+
if not sizes_ok:
|
| 811 |
+
raise GradcheckError("Analytical gradient has incorrect size")
|
| 812 |
+
if not reentrant:
|
| 813 |
+
raise GradcheckError(
|
| 814 |
+
"Backward is not reentrant, i.e., running backward with "
|
| 815 |
+
"same input and grad_output multiple times gives different values, "
|
| 816 |
+
"although analytical gradient matches numerical gradient."
|
| 817 |
+
f"The tolerance for nondeterminism was {nondet_tol}." + FAILED_NONDET_MSG
|
| 818 |
+
)
|
| 819 |
+
return jacobians1
|
| 820 |
+
|
| 821 |
+
|
| 822 |
+
def _get_analytical_vJu_backward_mode(
|
| 823 |
+
inputs, outputs, nondet_tol, check_grad_dtypes, all_v, all_u
|
| 824 |
+
):
|
| 825 |
+
reduced_jacobians: list[list[torch.Tensor]] = []
|
| 826 |
+
for output, v in zip(outputs, all_v):
|
| 827 |
+
all_vJ = _check_analytical_jacobian_attributes(
|
| 828 |
+
inputs, output, nondet_tol, check_grad_dtypes, fast_mode=True, v=v
|
| 829 |
+
)
|
| 830 |
+
jacobian_scalars: list[torch.Tensor] = []
|
| 831 |
+
for vJ, u in zip(all_vJ, all_u):
|
| 832 |
+
# Why do we need squeeze here? vJ is a 2-d tensor so that we can reuse
|
| 833 |
+
# the error checking logic from slow mode
|
| 834 |
+
vJ = vJ.T.squeeze(0)
|
| 835 |
+
if vJ.is_complex(): # C -> R
|
| 836 |
+
tv = torch.view_as_real(vJ.resolve_conj())
|
| 837 |
+
tr = tv.select(-1, 0)
|
| 838 |
+
ti = tv.select(-1, 1)
|
| 839 |
+
jacobian_scalars.append(tr.dot(u[0]) + 1j * ti.dot(u[1]))
|
| 840 |
+
else: # R -> R
|
| 841 |
+
jacobian_scalars.append(vJ.dot(u))
|
| 842 |
+
reduced_jacobians.append(jacobian_scalars)
|
| 843 |
+
return reduced_jacobians
|
| 844 |
+
|
| 845 |
+
|
| 846 |
+
@deprecated(
|
| 847 |
+
"`get_analytical_jacobian` was part of PyTorch's private API and not "
|
| 848 |
+
"meant to be exposed. We are deprecating it and it will be removed "
|
| 849 |
+
"in a future version of PyTorch. If you have a specific use for "
|
| 850 |
+
"this or feature request for this to be a stable API, please file "
|
| 851 |
+
"us an issue at https://github.com/pytorch/pytorch/issues/new",
|
| 852 |
+
category=FutureWarning,
|
| 853 |
+
)
|
| 854 |
+
def get_analytical_jacobian(inputs, output, nondet_tol=0.0, grad_out=1.0):
|
| 855 |
+
# Replicates the behavior of the old get_analytical_jacobian before the refactor
|
| 856 |
+
# This shares much of its code with _check_analytical_jacobian_attributes
|
| 857 |
+
if (
|
| 858 |
+
grad_out != 1.0
|
| 859 |
+
): # grad_out param is only kept for backward compatibility reasons
|
| 860 |
+
raise ValueError(
|
| 861 |
+
"Expected grad_out to be 1.0. get_analytical_jacobian no longer "
|
| 862 |
+
"supports values of grad_out != 1.0."
|
| 863 |
+
)
|
| 864 |
+
if output.is_complex():
|
| 865 |
+
raise ValueError(
|
| 866 |
+
"Expected output to be non-complex. get_analytical_jacobian no "
|
| 867 |
+
"longer supports functions that return complex outputs."
|
| 868 |
+
)
|
| 869 |
+
diff_input_list = list(_iter_tensors(inputs, True))
|
| 870 |
+
|
| 871 |
+
def vjp_fn(grad_output):
|
| 872 |
+
return torch.autograd.grad(
|
| 873 |
+
output, diff_input_list, grad_output, retain_graph=True, allow_unused=True
|
| 874 |
+
)
|
| 875 |
+
|
| 876 |
+
# Compute everything twice to check for nondeterminism (which we call reentrancy)
|
| 877 |
+
vjps1 = _compute_analytical_jacobian_rows(vjp_fn, output.clone())
|
| 878 |
+
vjps2 = _compute_analytical_jacobian_rows(vjp_fn, output.clone())
|
| 879 |
+
|
| 880 |
+
output_numel = output.numel()
|
| 881 |
+
jacobians1, types_ok, sizes_ok = _stack_and_check_tensors(
|
| 882 |
+
vjps1, inputs, output_numel
|
| 883 |
+
)
|
| 884 |
+
jacobians2, _, _ = _stack_and_check_tensors(vjps2, inputs, output_numel)
|
| 885 |
+
reentrant = _check_jacobians_equal(jacobians1, jacobians2, nondet_tol)
|
| 886 |
+
|
| 887 |
+
return jacobians1, reentrant, sizes_ok, types_ok
|
| 888 |
+
|
| 889 |
+
|
| 890 |
+
def _get_analytical_jacobian(inputs, outputs, input_idx, output_idx):
|
| 891 |
+
# Computes the analytical Jacobian in slow mode for a single input-output pair.
|
| 892 |
+
# Forgoes performing checks on dtype, shape, and reentrancy.
|
| 893 |
+
jacobians = _check_analytical_jacobian_attributes(
|
| 894 |
+
inputs, outputs[output_idx], nondet_tol=float("inf"), check_grad_dtypes=False
|
| 895 |
+
)
|
| 896 |
+
return jacobians[input_idx]
|
| 897 |
+
|
| 898 |
+
|
| 899 |
+
def _compute_analytical_jacobian_rows(
|
| 900 |
+
vjp_fn, sample_output
|
| 901 |
+
) -> list[list[torch.Tensor | None]]:
|
| 902 |
+
# Computes Jacobian row-by-row by projecting `vjp_fn` = v^T J on standard basis
|
| 903 |
+
# vectors: vjp_fn(e) = e^T J is a corresponding row of the Jacobian.
|
| 904 |
+
# NB: this function does not assume vjp_fn(v) to return tensors with the same
|
| 905 |
+
# number of elements for different v. This is checked when we later combine the
|
| 906 |
+
# rows into a single tensor.
|
| 907 |
+
grad_out_base = torch.zeros_like(
|
| 908 |
+
sample_output, memory_format=torch.legacy_contiguous_format
|
| 909 |
+
)
|
| 910 |
+
flat_grad_out = grad_out_base.view(-1)
|
| 911 |
+
# jacobians_rows[i][j] is the Jacobian jth row for the ith input
|
| 912 |
+
jacobians_rows: list[list[torch.Tensor | None]] = []
|
| 913 |
+
for j in range(flat_grad_out.numel()):
|
| 914 |
+
flat_grad_out.zero_()
|
| 915 |
+
flat_grad_out[j] = 1.0 # projection for jth row of Jacobian
|
| 916 |
+
grad_inputs = vjp_fn(grad_out_base)
|
| 917 |
+
for i, d_x in enumerate(grad_inputs):
|
| 918 |
+
if j == 0:
|
| 919 |
+
jacobians_rows.append([])
|
| 920 |
+
jacobians_rows[i] += [
|
| 921 |
+
d_x.clone() if isinstance(d_x, torch.Tensor) else None
|
| 922 |
+
]
|
| 923 |
+
return jacobians_rows
|
| 924 |
+
|
| 925 |
+
|
| 926 |
+
def _get_analytical_vjps_wrt_specific_output(
|
| 927 |
+
vjp_fn, sample_output, v
|
| 928 |
+
) -> list[list[torch.Tensor | None]]:
|
| 929 |
+
grad_inputs = vjp_fn(v.reshape(sample_output.shape))
|
| 930 |
+
vjps: list[list[torch.Tensor | None]] = [
|
| 931 |
+
[vjp.clone() if isinstance(vjp, torch.Tensor) else None] for vjp in grad_inputs
|
| 932 |
+
]
|
| 933 |
+
return vjps
|
| 934 |
+
|
| 935 |
+
|
| 936 |
+
def _check_inputs(tupled_inputs) -> bool:
|
| 937 |
+
# Make sure that gradients are saved for at least one input
|
| 938 |
+
any_input_requiring_grad = False
|
| 939 |
+
for idx, inp in enumerate(tupled_inputs):
|
| 940 |
+
if is_tensor_like(inp) and inp.requires_grad:
|
| 941 |
+
if not (inp.dtype == torch.float64 or inp.dtype == torch.complex128):
|
| 942 |
+
warnings.warn(
|
| 943 |
+
f"Input #{idx} requires gradient and "
|
| 944 |
+
"is not a double precision floating point or complex. "
|
| 945 |
+
"This check will likely fail if all the inputs are "
|
| 946 |
+
"not of double precision floating point or complex. ",
|
| 947 |
+
stacklevel=2,
|
| 948 |
+
)
|
| 949 |
+
if inp.is_sparse:
|
| 950 |
+
content = inp._values()
|
| 951 |
+
elif _is_sparse_compressed_tensor(inp):
|
| 952 |
+
content = inp.values()
|
| 953 |
+
else:
|
| 954 |
+
content = inp
|
| 955 |
+
# TODO: To cover more problematic cases, replace stride = 0 check with
|
| 956 |
+
# "any overlap in memory" once we have a proper function to check it.
|
| 957 |
+
if content.layout is not torch._mkldnn: # type: ignore[attr-defined]
|
| 958 |
+
if not all(
|
| 959 |
+
st > 0 or sz <= 1
|
| 960 |
+
for st, sz in zip(content.stride(), content.size())
|
| 961 |
+
):
|
| 962 |
+
raise RuntimeError(
|
| 963 |
+
f"The {idx}th input has a dimension with stride 0. gradcheck only "
|
| 964 |
+
"supports inputs that are non-overlapping to be able to "
|
| 965 |
+
"compute the numerical gradients correctly. You should call "
|
| 966 |
+
".contiguous on the input before passing it to gradcheck."
|
| 967 |
+
)
|
| 968 |
+
any_input_requiring_grad = True
|
| 969 |
+
|
| 970 |
+
if not any_input_requiring_grad:
|
| 971 |
+
raise ValueError(
|
| 972 |
+
"gradcheck expects at least one input tensor to require gradient, "
|
| 973 |
+
"but none of them have requires_grad=True."
|
| 974 |
+
)
|
| 975 |
+
return True
|
| 976 |
+
|
| 977 |
+
|
| 978 |
+
def _check_outputs(outputs) -> None:
|
| 979 |
+
if any(_is_sparse_any_tensor(t) for t in outputs if isinstance(t, torch.Tensor)):
|
| 980 |
+
# it is easier to call to_dense() on the sparse output than
|
| 981 |
+
# to modify analytical jacobian
|
| 982 |
+
raise ValueError(
|
| 983 |
+
"Sparse output is not supported at gradcheck yet. "
|
| 984 |
+
"Please call to_dense(masked_grad=...) on the output of fn for gradcheck."
|
| 985 |
+
)
|
| 986 |
+
if any(t.layout == torch._mkldnn for t in outputs if isinstance(t, torch.Tensor)): # type: ignore[attr-defined]
|
| 987 |
+
raise ValueError(
|
| 988 |
+
"MKLDNN output is not supported at gradcheck yet. "
|
| 989 |
+
"Please call to_dense(masked_grad=...) on the output of fn for gradcheck."
|
| 990 |
+
)
|
| 991 |
+
|
| 992 |
+
|
| 993 |
+
def _check_no_differentiable_outputs(
|
| 994 |
+
func, inputs, func_out, eps, *, is_forward_ad
|
| 995 |
+
) -> bool:
|
| 996 |
+
# When there are no differentiable outputs, numerical gradient for a function is
|
| 997 |
+
# expected to be zero.
|
| 998 |
+
jacobians_all_inputs_outputs = _get_numerical_jacobian(
|
| 999 |
+
func, inputs, func_out, eps=eps, is_forward_ad=is_forward_ad
|
| 1000 |
+
)
|
| 1001 |
+
for jacobians_all_outputs_and_fixed_input in jacobians_all_inputs_outputs:
|
| 1002 |
+
for jacobian in jacobians_all_outputs_and_fixed_input:
|
| 1003 |
+
if torch.ne(jacobian, 0).sum() > 0:
|
| 1004 |
+
raise GradcheckError(
|
| 1005 |
+
"Numerical gradient for function expected to be zero"
|
| 1006 |
+
)
|
| 1007 |
+
return True
|
| 1008 |
+
|
| 1009 |
+
|
| 1010 |
+
def _check_no_differentiable_outputs_fast(
|
| 1011 |
+
func, func_out, all_inputs, inputs_indices, all_u, eps, nondet_tol
|
| 1012 |
+
):
|
| 1013 |
+
for inp_idx, u in zip(inputs_indices, all_u):
|
| 1014 |
+
jvps = _get_numerical_jvp_wrt_specific_input(func, inp_idx, all_inputs, u, eps)
|
| 1015 |
+
for jvp in jvps:
|
| 1016 |
+
if jvp.numel() == 0:
|
| 1017 |
+
continue
|
| 1018 |
+
if (jvp - torch.zeros_like(jvp)).abs().max() > nondet_tol:
|
| 1019 |
+
raise GradcheckError(
|
| 1020 |
+
"Numerical gradient for function expected to be zero"
|
| 1021 |
+
)
|
| 1022 |
+
return True
|
| 1023 |
+
|
| 1024 |
+
|
| 1025 |
+
FAILED_BATCHED_GRAD_MSG = """
|
| 1026 |
+
gradcheck or gradgradcheck failed while testing batched gradient computation.
|
| 1027 |
+
This could have been invoked in a number of ways (via a test that calls
|
| 1028 |
+
gradcheck/gradgradcheck directly or via an autogenerated test).
|
| 1029 |
+
|
| 1030 |
+
If you are adding a new operator, please file an issue and then use one of the
|
| 1031 |
+
workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck.
|
| 1032 |
+
If the test
|
| 1033 |
+
- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
|
| 1034 |
+
with `check_batched_grad=False` as a keyword argument.
|
| 1035 |
+
- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
|
| 1036 |
+
to have `check_batched_grad=False` and/or `check_batched_gradgrad=False`.
|
| 1037 |
+
|
| 1038 |
+
If you're modifying an existing operator that supports batched grad computation,
|
| 1039 |
+
or wish to make a new operator work with batched grad computation, please read
|
| 1040 |
+
the following.
|
| 1041 |
+
|
| 1042 |
+
To compute batched grads (e.g., jacobians, hessians), we vmap over the backward
|
| 1043 |
+
computation. The most common failure case is if there is a 'vmap-incompatible
|
| 1044 |
+
operation' in the backward pass. Please see
|
| 1045 |
+
NOTE: [How to write vmap-compatible backward formulas]
|
| 1046 |
+
in the codebase for an explanation of how to fix this.
|
| 1047 |
+
""".strip()
|
| 1048 |
+
|
| 1049 |
+
FAILED_BATCHED_GRAD_MSG_FWD_AD = """
|
| 1050 |
+
gradcheck failed while testing batched gradient computation with forward-mode AD.
|
| 1051 |
+
This test is enabled automatically when both `check_batched_grad=True`
|
| 1052 |
+
and `check_forward_ad=True`, but can be disabled in the following ways
|
| 1053 |
+
dependong on how the test was invoked (via a test that calls gradcheck
|
| 1054 |
+
directly or via an autogenerated test).
|
| 1055 |
+
|
| 1056 |
+
If you are adding a new operator, please file an issue and then use one of the
|
| 1057 |
+
workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck.
|
| 1058 |
+
If the test
|
| 1059 |
+
- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
|
| 1060 |
+
with `check_batched_forward_grad=False` as a keyword argument.
|
| 1061 |
+
- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
|
| 1062 |
+
to have `check_batched_forward_grad=False`
|
| 1063 |
+
"""
|
| 1064 |
+
|
| 1065 |
+
|
| 1066 |
+
def _get_failed_batched_grad_test_msg(
|
| 1067 |
+
output_idx, input_idx, res, exp, is_forward_ad=False
|
| 1068 |
+
):
|
| 1069 |
+
return f"""
|
| 1070 |
+
For output {output_idx} and input {input_idx}:
|
| 1071 |
+
|
| 1072 |
+
{FAILED_BATCHED_GRAD_MSG_FWD_AD if is_forward_ad else FAILED_BATCHED_GRAD_MSG}
|
| 1073 |
+
|
| 1074 |
+
Got:
|
| 1075 |
+
{res}
|
| 1076 |
+
|
| 1077 |
+
Expected:
|
| 1078 |
+
{exp}
|
| 1079 |
+
""".strip()
|
| 1080 |
+
|
| 1081 |
+
|
| 1082 |
+
def _test_batched_grad_forward_ad(func, inputs) -> bool:
|
| 1083 |
+
fwAD = torch.autograd.forward_ad # To avoid early import issues (do we need this?)
|
| 1084 |
+
if not isinstance(inputs, tuple):
|
| 1085 |
+
raise AssertionError("Expected inputs to be a tuple")
|
| 1086 |
+
|
| 1087 |
+
for input_idx, current_input in enumerate(inputs):
|
| 1088 |
+
if not (is_tensor_like(current_input) and current_input.requires_grad):
|
| 1089 |
+
continue
|
| 1090 |
+
|
| 1091 |
+
def jvp(tangent: torch.Tensor):
|
| 1092 |
+
with fwAD.dual_level():
|
| 1093 |
+
dual = fwAD.make_dual(current_input.detach(), tangent)
|
| 1094 |
+
inputs_with_dual = tuple(
|
| 1095 |
+
dual
|
| 1096 |
+
if idx == input_idx
|
| 1097 |
+
else (inp.detach() if is_tensor_like(inp) else inp)
|
| 1098 |
+
for idx, inp in enumerate(inputs)
|
| 1099 |
+
)
|
| 1100 |
+
dual_outputs = _as_tuple(func(*inputs_with_dual))
|
| 1101 |
+
ret = []
|
| 1102 |
+
for dual_output in dual_outputs:
|
| 1103 |
+
if dual_output is None:
|
| 1104 |
+
continue
|
| 1105 |
+
primal_out, tangent_out = fwAD.unpack_dual(dual_output)
|
| 1106 |
+
if tangent_out is not None:
|
| 1107 |
+
ret.append(tangent_out)
|
| 1108 |
+
else:
|
| 1109 |
+
ret.append(
|
| 1110 |
+
torch.zeros(
|
| 1111 |
+
[], dtype=primal_out.dtype, device=primal_out.device
|
| 1112 |
+
).expand(primal_out.shape)
|
| 1113 |
+
)
|
| 1114 |
+
return tuple(ret)
|
| 1115 |
+
|
| 1116 |
+
if not _is_float_or_complex_tensor(current_input):
|
| 1117 |
+
continue
|
| 1118 |
+
|
| 1119 |
+
tangents = [torch.randn_like(current_input) for _ in range(2)]
|
| 1120 |
+
expected = [jvp(t) for t in tangents]
|
| 1121 |
+
expected = [torch.stack(shards) for shards in zip(*expected)]
|
| 1122 |
+
|
| 1123 |
+
try:
|
| 1124 |
+
result = _vmap(jvp)(torch.stack(tangents))
|
| 1125 |
+
except RuntimeError as ex:
|
| 1126 |
+
# Rethrow to provide a better error message
|
| 1127 |
+
raise GradcheckError(
|
| 1128 |
+
f"While computing batched gradients, got: {ex}\n\n{FAILED_BATCHED_GRAD_MSG_FWD_AD}"
|
| 1129 |
+
) from ex
|
| 1130 |
+
|
| 1131 |
+
for input_idx, (res, exp) in enumerate(zip(result, expected)):
|
| 1132 |
+
if torch.allclose(res, exp):
|
| 1133 |
+
continue
|
| 1134 |
+
raise GradcheckError(
|
| 1135 |
+
_get_failed_batched_grad_test_msg(
|
| 1136 |
+
input_idx, input_idx, res, exp, is_forward_ad=True
|
| 1137 |
+
)
|
| 1138 |
+
)
|
| 1139 |
+
return True
|
| 1140 |
+
|
| 1141 |
+
|
| 1142 |
+
def _test_batched_grad(input, output, output_idx) -> bool:
|
| 1143 |
+
# NB: _test_batched_grad compares two autograd.grad invocations with a single
|
| 1144 |
+
# vmap(autograd.grad) invocation. It's not exactly a "gradcheck" in the
|
| 1145 |
+
# sense that we're not comparing an analytical jacobian with a numeric one,
|
| 1146 |
+
# but it is morally similar (we could have computed a full analytic jac
|
| 1147 |
+
# via vmap, but that is potentially slow)
|
| 1148 |
+
diff_input_list = list(_iter_tensors(input, True))
|
| 1149 |
+
grad = functools.partial(
|
| 1150 |
+
torch.autograd.grad,
|
| 1151 |
+
output,
|
| 1152 |
+
diff_input_list,
|
| 1153 |
+
retain_graph=True,
|
| 1154 |
+
allow_unused=True,
|
| 1155 |
+
)
|
| 1156 |
+
|
| 1157 |
+
def vjp(v):
|
| 1158 |
+
results = grad(v)
|
| 1159 |
+
results = tuple(
|
| 1160 |
+
grad
|
| 1161 |
+
if grad is not None
|
| 1162 |
+
else torch.zeros([], dtype=inp.dtype, device=inp.device).expand(inp.shape)
|
| 1163 |
+
for grad, inp in zip(results, diff_input_list)
|
| 1164 |
+
)
|
| 1165 |
+
return results
|
| 1166 |
+
|
| 1167 |
+
grad_outputs = [torch.randn_like(output) for _ in range(2)]
|
| 1168 |
+
|
| 1169 |
+
expected = [vjp(gO) for gO in grad_outputs]
|
| 1170 |
+
expected = [torch.stack(shards) for shards in zip(*expected)]
|
| 1171 |
+
|
| 1172 |
+
# Squash warnings since these are expected to happen in most cases
|
| 1173 |
+
# NB: this doesn't work for CUDA tests: https://github.com/pytorch/pytorch/issues/50209
|
| 1174 |
+
with warnings.catch_warnings():
|
| 1175 |
+
warnings.filterwarnings("ignore", message="There is a performance drop")
|
| 1176 |
+
warnings.filterwarnings("ignore", message="Please use `torch.vmap`")
|
| 1177 |
+
try:
|
| 1178 |
+
result = vmap(vjp)(torch.stack(grad_outputs))
|
| 1179 |
+
except RuntimeError as ex:
|
| 1180 |
+
# It's OK that we're not raising the error at the correct callsite.
|
| 1181 |
+
# That's because the callsite is always going to inside the Python
|
| 1182 |
+
# autograd.grad instead of the C++ traceback of what line in the
|
| 1183 |
+
# backward formula
|
| 1184 |
+
raise GradcheckError(
|
| 1185 |
+
f"While computing batched gradients, got: {ex}\n\n{FAILED_BATCHED_GRAD_MSG}"
|
| 1186 |
+
) from ex
|
| 1187 |
+
|
| 1188 |
+
for input_idx, (res, exp) in enumerate(zip(result, expected)):
|
| 1189 |
+
if torch.allclose(res, exp):
|
| 1190 |
+
continue
|
| 1191 |
+
raise GradcheckError(
|
| 1192 |
+
_get_failed_batched_grad_test_msg(output_idx, input_idx, res, exp)
|
| 1193 |
+
)
|
| 1194 |
+
return True
|
| 1195 |
+
|
| 1196 |
+
|
| 1197 |
+
def _test_backward_mul_by_grad_output(outputs, inputs, masked) -> bool:
|
| 1198 |
+
# Tests that backward is multiplied by grad_output
|
| 1199 |
+
diff_input_list: list[torch.Tensor] = list(_iter_tensors(inputs, True))
|
| 1200 |
+
if not diff_input_list:
|
| 1201 |
+
raise GradcheckError("no Tensors requiring grad found in input")
|
| 1202 |
+
grads_input = torch.autograd.grad(
|
| 1203 |
+
outputs,
|
| 1204 |
+
diff_input_list,
|
| 1205 |
+
[
|
| 1206 |
+
torch.zeros_like(o, memory_format=torch.legacy_contiguous_format)
|
| 1207 |
+
for o in outputs
|
| 1208 |
+
],
|
| 1209 |
+
allow_unused=True,
|
| 1210 |
+
)
|
| 1211 |
+
for gi, di in zip(grads_input, diff_input_list):
|
| 1212 |
+
if gi is None:
|
| 1213 |
+
continue
|
| 1214 |
+
if isinstance(gi, torch.Tensor) and gi.layout != torch.strided:
|
| 1215 |
+
if gi.layout != di.layout:
|
| 1216 |
+
raise GradcheckError(
|
| 1217 |
+
"grad is incorrect layout ("
|
| 1218 |
+
+ str(gi.layout)
|
| 1219 |
+
+ " is not "
|
| 1220 |
+
+ str(di.layout)
|
| 1221 |
+
+ ")"
|
| 1222 |
+
)
|
| 1223 |
+
if _is_sparse_any_tensor(gi):
|
| 1224 |
+
sparse_kind = str(gi.layout).replace("torch.", "").replace("_coo", "")
|
| 1225 |
+
if gi.sparse_dim() != di.sparse_dim():
|
| 1226 |
+
raise GradcheckError(
|
| 1227 |
+
f"grad is {sparse_kind} tensor, but has incorrect sparse_dim"
|
| 1228 |
+
f" {gi.sparse_dim()}, expected {di.sparse_dim()}"
|
| 1229 |
+
)
|
| 1230 |
+
if gi.dense_dim() != di.dense_dim():
|
| 1231 |
+
raise GradcheckError(
|
| 1232 |
+
f"grad is {sparse_kind} tensor, but has incorrect dense_dim"
|
| 1233 |
+
f" {gi.dense_dim()}, expected {di.dense_dim()}"
|
| 1234 |
+
)
|
| 1235 |
+
gi = gi.to_dense()
|
| 1236 |
+
di = di.to_dense()
|
| 1237 |
+
if masked:
|
| 1238 |
+
if not torch.allclose(gi, torch.zeros_like(gi)):
|
| 1239 |
+
raise GradcheckError("backward not multiplied by grad_output")
|
| 1240 |
+
elif not gi.eq(0).all():
|
| 1241 |
+
raise GradcheckError("backward not multiplied by grad_output")
|
| 1242 |
+
if gi.dtype != di.dtype:
|
| 1243 |
+
raise GradcheckError("grad is incorrect type")
|
| 1244 |
+
if gi.device != di.device:
|
| 1245 |
+
raise GradcheckError("grad is incorrect device")
|
| 1246 |
+
if gi.size() != di.size():
|
| 1247 |
+
raise GradcheckError("grad is incorrect size")
|
| 1248 |
+
return True
|
| 1249 |
+
|
| 1250 |
+
|
| 1251 |
+
def _test_undefined_forward_mode(func, outputs, inputs):
|
| 1252 |
+
fwAD = torch.autograd.forward_ad
|
| 1253 |
+
|
| 1254 |
+
_inp_tensors_idx, inp_tensors = _get_inp_tensors(inputs)
|
| 1255 |
+
_all_v, all_u, _all_u_dense = _make_vectors(
|
| 1256 |
+
inp_tensors, outputs, use_forward_ad=True
|
| 1257 |
+
)
|
| 1258 |
+
|
| 1259 |
+
with fwAD.dual_level():
|
| 1260 |
+
fw_grads = []
|
| 1261 |
+
dual_inputs = []
|
| 1262 |
+
tensor_indices = set()
|
| 1263 |
+
for i, inp in enumerate(inputs):
|
| 1264 |
+
if is_tensor_like(inp) and inp.requires_grad:
|
| 1265 |
+
if inp.layout == torch._mkldnn: # type: ignore[attr-defined]
|
| 1266 |
+
raise ValueError(
|
| 1267 |
+
"MKLDNN inputs are not support for forward AD gradcheck."
|
| 1268 |
+
)
|
| 1269 |
+
|
| 1270 |
+
inp = fwAD.make_dual(inp.detach(), torch.zeros_like(inp))
|
| 1271 |
+
# If inp is a differentiable view, the dual might not be the tangent given to
|
| 1272 |
+
# make_dual, so read it explicitly from the dual tensor
|
| 1273 |
+
fw_grads.append(fwAD.unpack_dual(inp)[1])
|
| 1274 |
+
tensor_indices.add(i)
|
| 1275 |
+
dual_inputs.append(inp)
|
| 1276 |
+
|
| 1277 |
+
for fw_grad, u in zip(fw_grads, all_u):
|
| 1278 |
+
fw_grad.copy_(u.view_as(fw_grad))
|
| 1279 |
+
|
| 1280 |
+
for idx, inp in enumerate(inputs):
|
| 1281 |
+
if idx not in tensor_indices:
|
| 1282 |
+
continue
|
| 1283 |
+
dual_inp_obj = dual_inputs[idx]
|
| 1284 |
+
|
| 1285 |
+
# case 1 (Materialized Zero Tensor Tangent)
|
| 1286 |
+
dual_inputs[idx] = fwAD.make_dual(inp.detach(), torch.zeros_like(inp))
|
| 1287 |
+
raw_outputs = _as_tuple(func(*dual_inputs))
|
| 1288 |
+
dual_outputs1 = filter(_is_float_or_complex_tensor, raw_outputs)
|
| 1289 |
+
|
| 1290 |
+
# case 2 (Efficient Zero Tensor Tangent since we don't make a dual object and pass a regular tensor)
|
| 1291 |
+
dual_inputs[idx] = inp.detach()
|
| 1292 |
+
raw_outputs = _as_tuple(func(*dual_inputs))
|
| 1293 |
+
dual_outputs2 = filter(_is_float_or_complex_tensor, raw_outputs)
|
| 1294 |
+
|
| 1295 |
+
# reset
|
| 1296 |
+
dual_inputs[idx] = dual_inp_obj
|
| 1297 |
+
|
| 1298 |
+
for index_o, (d_o1, d_o2) in enumerate(zip(dual_outputs1, dual_outputs2)):
|
| 1299 |
+
_val1, res1 = fwAD.unpack_dual(d_o1)
|
| 1300 |
+
_val2, res2 = fwAD.unpack_dual(d_o2)
|
| 1301 |
+
|
| 1302 |
+
if not (res1 is None or res2 is None):
|
| 1303 |
+
if not torch.allclose(res1, res2):
|
| 1304 |
+
raise GradcheckError(
|
| 1305 |
+
"Mismatch in tangent values for output with index: ",
|
| 1306 |
+
index_o,
|
| 1307 |
+
" when input: ",
|
| 1308 |
+
inp,
|
| 1309 |
+
" has an undefined tangent value. ",
|
| 1310 |
+
" Got: ",
|
| 1311 |
+
res1,
|
| 1312 |
+
" but expected: ",
|
| 1313 |
+
res2,
|
| 1314 |
+
)
|
| 1315 |
+
return True
|
| 1316 |
+
|
| 1317 |
+
|
| 1318 |
+
def _test_undefined_backward_mode(func, outputs, inputs) -> bool:
|
| 1319 |
+
diff_input_list: list[torch.Tensor] = list(_iter_tensors(inputs, True))
|
| 1320 |
+
if not diff_input_list:
|
| 1321 |
+
raise GradcheckError("no Tensors requiring grad found in input")
|
| 1322 |
+
|
| 1323 |
+
def warn_bc_breaking():
|
| 1324 |
+
warnings.warn(
|
| 1325 |
+
"Backwards compatibility: New undefined gradient support checking "
|
| 1326 |
+
"feature is enabled by default, but it may break existing callers "
|
| 1327 |
+
"of this function. If this is true for you, you can call this "
|
| 1328 |
+
'function with "check_undefined_grad=False" to disable the feature',
|
| 1329 |
+
stacklevel=2,
|
| 1330 |
+
)
|
| 1331 |
+
|
| 1332 |
+
def check_undefined_grad_support(output_to_check):
|
| 1333 |
+
grads_output = [
|
| 1334 |
+
torch.zeros_like(o, memory_format=torch.legacy_contiguous_format)
|
| 1335 |
+
for o in output_to_check
|
| 1336 |
+
]
|
| 1337 |
+
try:
|
| 1338 |
+
grads_input = torch.autograd.grad(
|
| 1339 |
+
output_to_check, diff_input_list, grads_output, allow_unused=True
|
| 1340 |
+
)
|
| 1341 |
+
except RuntimeError as e:
|
| 1342 |
+
warn_bc_breaking()
|
| 1343 |
+
raise GradcheckError(
|
| 1344 |
+
"Expected backward function to handle undefined output grads. "
|
| 1345 |
+
'Please look at "Notes about undefined output gradients" in '
|
| 1346 |
+
'"tools/autograd/derivatives.yaml"'
|
| 1347 |
+
) from e
|
| 1348 |
+
|
| 1349 |
+
for gi in grads_input:
|
| 1350 |
+
if (gi is not None) and (not gi.eq(0).all()):
|
| 1351 |
+
warn_bc_breaking()
|
| 1352 |
+
raise GradcheckError(
|
| 1353 |
+
"Expected all input grads to be undefined or zero when all output grads are undefined "
|
| 1354 |
+
'or zero. Please look at "Notes about undefined output gradients" in '
|
| 1355 |
+
'"tools/autograd/derivatives.yaml"'
|
| 1356 |
+
)
|
| 1357 |
+
return True
|
| 1358 |
+
|
| 1359 |
+
# All backward functions must work properly if all output grads are undefined
|
| 1360 |
+
outputs_to_check = [
|
| 1361 |
+
[
|
| 1362 |
+
torch._C._functions.UndefinedGrad()(o)
|
| 1363 |
+
for o in _differentiable_outputs(func(*inputs))
|
| 1364 |
+
# This check filters out Tensor-likes that aren't instances of Tensor.
|
| 1365 |
+
if isinstance(o, torch.Tensor)
|
| 1366 |
+
]
|
| 1367 |
+
]
|
| 1368 |
+
|
| 1369 |
+
# If there are multiple output grads, we should be able to undef one at a time without error
|
| 1370 |
+
if len(outputs_to_check[0]) > 1:
|
| 1371 |
+
for undef_grad_idx in range(len(outputs)):
|
| 1372 |
+
output_to_check = _differentiable_outputs(func(*inputs))
|
| 1373 |
+
outputs_to_check.append(
|
| 1374 |
+
[
|
| 1375 |
+
torch._C._functions.UndefinedGrad()(o)
|
| 1376 |
+
if idx == undef_grad_idx
|
| 1377 |
+
else o
|
| 1378 |
+
for idx, o in enumerate(output_to_check)
|
| 1379 |
+
]
|
| 1380 |
+
)
|
| 1381 |
+
|
| 1382 |
+
return all(check_undefined_grad_support(output) for output in outputs_to_check)
|
| 1383 |
+
|
| 1384 |
+
|
| 1385 |
+
def _as_tuple(x):
|
| 1386 |
+
if isinstance(x, tuple):
|
| 1387 |
+
return x
|
| 1388 |
+
elif isinstance(x, list):
|
| 1389 |
+
return tuple(x)
|
| 1390 |
+
else:
|
| 1391 |
+
return (x,)
|
| 1392 |
+
|
| 1393 |
+
|
| 1394 |
+
def _differentiable_outputs(x):
|
| 1395 |
+
return tuple(o for o in _as_tuple(x) if o.requires_grad)
|
| 1396 |
+
|
| 1397 |
+
|
| 1398 |
+
def _get_notallclose_msg(
|
| 1399 |
+
analytical,
|
| 1400 |
+
numerical,
|
| 1401 |
+
output_idx,
|
| 1402 |
+
input_idx,
|
| 1403 |
+
complex_indices,
|
| 1404 |
+
test_imag=False,
|
| 1405 |
+
is_forward_ad=False,
|
| 1406 |
+
) -> str:
|
| 1407 |
+
out_is_complex = (
|
| 1408 |
+
(not is_forward_ad) and complex_indices and output_idx in complex_indices
|
| 1409 |
+
)
|
| 1410 |
+
inp_is_complex = is_forward_ad and complex_indices and input_idx in complex_indices
|
| 1411 |
+
part = "imaginary" if test_imag else "real"
|
| 1412 |
+
element = "inputs" if is_forward_ad else "outputs"
|
| 1413 |
+
prefix = (
|
| 1414 |
+
""
|
| 1415 |
+
if not (out_is_complex or inp_is_complex)
|
| 1416 |
+
else f"While considering the {part} part of complex {element} only, "
|
| 1417 |
+
)
|
| 1418 |
+
mode = "computed with forward mode " if is_forward_ad else ""
|
| 1419 |
+
return (
|
| 1420 |
+
prefix
|
| 1421 |
+
+ f"Jacobian {mode}mismatch for output {output_idx:d} with respect to input {input_idx:d},\n"
|
| 1422 |
+
f"numerical:{numerical}\nanalytical:{analytical}\n"
|
| 1423 |
+
)
|
| 1424 |
+
|
| 1425 |
+
|
| 1426 |
+
def _transpose(matrix_of_tensors):
|
| 1427 |
+
# returns list of tuples
|
| 1428 |
+
return list(zip(*matrix_of_tensors))
|
| 1429 |
+
|
| 1430 |
+
|
| 1431 |
+
def _real_and_imag_output(fn):
|
| 1432 |
+
# returns new functions real(fn), and imag(fn) where real(fn) and imag(fn) behave the same as
|
| 1433 |
+
# the original fn, except torch.real or torch.imag are applied to the complex outputs
|
| 1434 |
+
def apply_to_c_outs(fn, fn_to_apply):
|
| 1435 |
+
def wrapped_fn(*inputs):
|
| 1436 |
+
outs = _as_tuple(fn(*inputs))
|
| 1437 |
+
return tuple(fn_to_apply(o) if o.is_complex() else o for o in outs)
|
| 1438 |
+
|
| 1439 |
+
return wrapped_fn
|
| 1440 |
+
|
| 1441 |
+
return apply_to_c_outs(fn, torch.real), apply_to_c_outs(fn, torch.imag)
|
| 1442 |
+
|
| 1443 |
+
|
| 1444 |
+
def _real_and_imag_input(fn, complex_inp_indices, tupled_inputs):
|
| 1445 |
+
# returns new functions that take real inputs instead of complex inputs as
|
| 1446 |
+
# (x, y) -> fn(x + y * 1j). And it computes: inp -> fn(inp + y * 1j) and inp -> fn(x + inp * 1j).
|
| 1447 |
+
# In each case, the other part is considered constant.
|
| 1448 |
+
# We do not use 0 for the constant here to make sure we always call the user function with a valid input.
|
| 1449 |
+
def apply_to_c_inps(fn, fn_to_apply):
|
| 1450 |
+
def wrapped_fn(*inputs):
|
| 1451 |
+
new_inputs = list(inputs)
|
| 1452 |
+
for should_be_complex in complex_inp_indices:
|
| 1453 |
+
new_inputs[should_be_complex] = fn_to_apply(
|
| 1454 |
+
new_inputs[should_be_complex], tupled_inputs[should_be_complex]
|
| 1455 |
+
)
|
| 1456 |
+
return _as_tuple(fn(*new_inputs))
|
| 1457 |
+
|
| 1458 |
+
return wrapped_fn
|
| 1459 |
+
|
| 1460 |
+
real_fn = apply_to_c_inps(fn, lambda inp, orig: inp + orig.imag * 1j)
|
| 1461 |
+
imag_fn = apply_to_c_inps(fn, lambda inp, orig: orig.real + inp * 1j)
|
| 1462 |
+
return real_fn, imag_fn
|
| 1463 |
+
|
| 1464 |
+
|
| 1465 |
+
def _gradcheck_real_imag(
|
| 1466 |
+
gradcheck_fn,
|
| 1467 |
+
func,
|
| 1468 |
+
func_out,
|
| 1469 |
+
tupled_inputs,
|
| 1470 |
+
outputs,
|
| 1471 |
+
eps,
|
| 1472 |
+
rtol,
|
| 1473 |
+
atol,
|
| 1474 |
+
check_grad_dtypes,
|
| 1475 |
+
check_forward_ad,
|
| 1476 |
+
check_backward_ad,
|
| 1477 |
+
nondet_tol,
|
| 1478 |
+
check_undefined_grad,
|
| 1479 |
+
):
|
| 1480 |
+
complex_out_indices = [i for i, o in enumerate(outputs) if o.is_complex()]
|
| 1481 |
+
has_any_complex_output = any(o.is_complex() for o in _as_tuple(func_out))
|
| 1482 |
+
if check_backward_ad:
|
| 1483 |
+
if has_any_complex_output:
|
| 1484 |
+
real_fn, imag_fn = _real_and_imag_output(func)
|
| 1485 |
+
|
| 1486 |
+
imag_func_out = imag_fn(*tupled_inputs)
|
| 1487 |
+
imag_outputs = _differentiable_outputs(imag_func_out)
|
| 1488 |
+
gradcheck_fn(
|
| 1489 |
+
imag_fn,
|
| 1490 |
+
imag_func_out,
|
| 1491 |
+
tupled_inputs,
|
| 1492 |
+
imag_outputs,
|
| 1493 |
+
eps,
|
| 1494 |
+
rtol,
|
| 1495 |
+
atol,
|
| 1496 |
+
check_grad_dtypes,
|
| 1497 |
+
nondet_tol,
|
| 1498 |
+
complex_indices=complex_out_indices,
|
| 1499 |
+
test_imag=True,
|
| 1500 |
+
)
|
| 1501 |
+
|
| 1502 |
+
real_func_out = real_fn(*tupled_inputs)
|
| 1503 |
+
real_outputs = _differentiable_outputs(real_func_out)
|
| 1504 |
+
gradcheck_fn(
|
| 1505 |
+
real_fn,
|
| 1506 |
+
real_func_out,
|
| 1507 |
+
tupled_inputs,
|
| 1508 |
+
real_outputs,
|
| 1509 |
+
eps,
|
| 1510 |
+
rtol,
|
| 1511 |
+
atol,
|
| 1512 |
+
check_grad_dtypes,
|
| 1513 |
+
nondet_tol,
|
| 1514 |
+
complex_indices=complex_out_indices,
|
| 1515 |
+
)
|
| 1516 |
+
else:
|
| 1517 |
+
gradcheck_fn(
|
| 1518 |
+
func,
|
| 1519 |
+
func_out,
|
| 1520 |
+
tupled_inputs,
|
| 1521 |
+
outputs,
|
| 1522 |
+
eps,
|
| 1523 |
+
rtol,
|
| 1524 |
+
atol,
|
| 1525 |
+
check_grad_dtypes,
|
| 1526 |
+
nondet_tol,
|
| 1527 |
+
)
|
| 1528 |
+
|
| 1529 |
+
if check_forward_ad:
|
| 1530 |
+
complex_inp_indices = [
|
| 1531 |
+
i
|
| 1532 |
+
for i, inp in enumerate(tupled_inputs)
|
| 1533 |
+
if is_tensor_like(inp) and inp.is_complex()
|
| 1534 |
+
]
|
| 1535 |
+
if complex_inp_indices:
|
| 1536 |
+
real_fn, imag_fn = _real_and_imag_input(
|
| 1537 |
+
func, complex_inp_indices, tupled_inputs
|
| 1538 |
+
)
|
| 1539 |
+
|
| 1540 |
+
imag_inputs = [
|
| 1541 |
+
inp.imag if is_tensor_like(inp) and inp.is_complex() else inp
|
| 1542 |
+
for inp in tupled_inputs
|
| 1543 |
+
]
|
| 1544 |
+
imag_func_out = imag_fn(*imag_inputs)
|
| 1545 |
+
diff_imag_func_out = _differentiable_outputs(imag_func_out)
|
| 1546 |
+
gradcheck_fn(
|
| 1547 |
+
imag_fn,
|
| 1548 |
+
imag_func_out,
|
| 1549 |
+
imag_inputs,
|
| 1550 |
+
diff_imag_func_out,
|
| 1551 |
+
eps,
|
| 1552 |
+
rtol,
|
| 1553 |
+
atol,
|
| 1554 |
+
check_grad_dtypes,
|
| 1555 |
+
nondet_tol,
|
| 1556 |
+
complex_indices=complex_inp_indices,
|
| 1557 |
+
test_imag=True,
|
| 1558 |
+
use_forward_ad=True,
|
| 1559 |
+
)
|
| 1560 |
+
|
| 1561 |
+
real_inputs = [
|
| 1562 |
+
inp.real if is_tensor_like(inp) and inp.is_complex() else inp
|
| 1563 |
+
for inp in tupled_inputs
|
| 1564 |
+
]
|
| 1565 |
+
real_func_out = real_fn(*real_inputs)
|
| 1566 |
+
diff_real_func_out = _differentiable_outputs(real_func_out)
|
| 1567 |
+
gradcheck_fn(
|
| 1568 |
+
real_fn,
|
| 1569 |
+
real_func_out,
|
| 1570 |
+
real_inputs,
|
| 1571 |
+
diff_real_func_out,
|
| 1572 |
+
eps,
|
| 1573 |
+
rtol,
|
| 1574 |
+
atol,
|
| 1575 |
+
check_grad_dtypes,
|
| 1576 |
+
nondet_tol,
|
| 1577 |
+
complex_indices=complex_inp_indices,
|
| 1578 |
+
use_forward_ad=True,
|
| 1579 |
+
)
|
| 1580 |
+
if check_undefined_grad:
|
| 1581 |
+
_test_undefined_forward_mode(imag_fn, imag_func_out, imag_inputs)
|
| 1582 |
+
_test_undefined_forward_mode(real_fn, real_func_out, real_inputs)
|
| 1583 |
+
else:
|
| 1584 |
+
gradcheck_fn(
|
| 1585 |
+
func,
|
| 1586 |
+
func_out,
|
| 1587 |
+
tupled_inputs,
|
| 1588 |
+
outputs,
|
| 1589 |
+
eps,
|
| 1590 |
+
rtol,
|
| 1591 |
+
atol,
|
| 1592 |
+
check_grad_dtypes,
|
| 1593 |
+
nondet_tol,
|
| 1594 |
+
use_forward_ad=True,
|
| 1595 |
+
)
|
| 1596 |
+
if check_undefined_grad:
|
| 1597 |
+
_test_undefined_forward_mode(func, outputs, tupled_inputs)
|
| 1598 |
+
|
| 1599 |
+
|
| 1600 |
+
def _slow_gradcheck(
|
| 1601 |
+
func,
|
| 1602 |
+
func_out,
|
| 1603 |
+
tupled_inputs,
|
| 1604 |
+
outputs,
|
| 1605 |
+
eps,
|
| 1606 |
+
rtol,
|
| 1607 |
+
atol,
|
| 1608 |
+
check_grad_dtypes,
|
| 1609 |
+
nondet_tol,
|
| 1610 |
+
*,
|
| 1611 |
+
use_forward_ad=False,
|
| 1612 |
+
complex_indices=None,
|
| 1613 |
+
test_imag=False,
|
| 1614 |
+
masked=False,
|
| 1615 |
+
):
|
| 1616 |
+
func_out = _as_tuple(func_out)
|
| 1617 |
+
if not outputs:
|
| 1618 |
+
return _check_no_differentiable_outputs(
|
| 1619 |
+
func, tupled_inputs, func_out, eps=eps, is_forward_ad=use_forward_ad
|
| 1620 |
+
)
|
| 1621 |
+
tupled_inputs_numerical = tupled_inputs if masked else _densify(tupled_inputs)
|
| 1622 |
+
|
| 1623 |
+
numerical = _transpose(
|
| 1624 |
+
_get_numerical_jacobian(
|
| 1625 |
+
func,
|
| 1626 |
+
tupled_inputs_numerical,
|
| 1627 |
+
func_out,
|
| 1628 |
+
eps=eps,
|
| 1629 |
+
is_forward_ad=use_forward_ad,
|
| 1630 |
+
)
|
| 1631 |
+
)
|
| 1632 |
+
# Note: [numerical vs analytical output length]
|
| 1633 |
+
# The numerical path returns jacobian quantity for all outputs, even if requires_grad of that
|
| 1634 |
+
# output is False. This behavior is necessary for _check_no_differentiable_outputs to work.
|
| 1635 |
+
numerical = [nj for o, nj in zip(func_out, numerical) if o.requires_grad]
|
| 1636 |
+
if use_forward_ad:
|
| 1637 |
+
analytical_forward = _get_analytical_jacobian_forward_ad(
|
| 1638 |
+
func, tupled_inputs, func_out, check_grad_dtypes=check_grad_dtypes
|
| 1639 |
+
)
|
| 1640 |
+
|
| 1641 |
+
for i, n_per_out in enumerate(numerical):
|
| 1642 |
+
for j, n in enumerate(n_per_out):
|
| 1643 |
+
a = analytical_forward[j][i]
|
| 1644 |
+
if not _allclose_with_type_promotion(a, n.to(a.device), rtol, atol):
|
| 1645 |
+
raise GradcheckError(
|
| 1646 |
+
_get_notallclose_msg(
|
| 1647 |
+
a, n, i, j, complex_indices, test_imag, is_forward_ad=True
|
| 1648 |
+
)
|
| 1649 |
+
)
|
| 1650 |
+
else:
|
| 1651 |
+
for i, o in enumerate(outputs):
|
| 1652 |
+
analytical = _check_analytical_jacobian_attributes(
|
| 1653 |
+
tupled_inputs, o, nondet_tol, check_grad_dtypes
|
| 1654 |
+
)
|
| 1655 |
+
|
| 1656 |
+
for j, (a, n) in enumerate(zip(analytical, numerical[i])):
|
| 1657 |
+
if not _allclose_with_type_promotion(a, n.to(a.device), rtol, atol):
|
| 1658 |
+
raise GradcheckError(
|
| 1659 |
+
_get_notallclose_msg(a, n, i, j, complex_indices, test_imag)
|
| 1660 |
+
)
|
| 1661 |
+
|
| 1662 |
+
return True
|
| 1663 |
+
|
| 1664 |
+
|
| 1665 |
+
def _dot_with_type_promotion(u, v):
|
| 1666 |
+
if u.dim() != 1 or v.dim() != 1:
|
| 1667 |
+
raise AssertionError(
|
| 1668 |
+
f"Expected u and v to be 1D tensors, but got dims {u.dim()} and {v.dim()}"
|
| 1669 |
+
)
|
| 1670 |
+
return (u * v).sum()
|
| 1671 |
+
|
| 1672 |
+
|
| 1673 |
+
def _allclose_with_type_promotion(a, b, rtol, atol):
|
| 1674 |
+
promoted_type = torch.promote_types(a.dtype, b.dtype)
|
| 1675 |
+
a = a.to(dtype=promoted_type)
|
| 1676 |
+
b = b.to(dtype=promoted_type)
|
| 1677 |
+
return torch.allclose(a, b, rtol, atol)
|
| 1678 |
+
|
| 1679 |
+
|
| 1680 |
+
def _to_real_dtype(dtype):
|
| 1681 |
+
if dtype == torch.complex128:
|
| 1682 |
+
return torch.float64
|
| 1683 |
+
elif dtype == torch.complex64:
|
| 1684 |
+
return torch.float32
|
| 1685 |
+
else:
|
| 1686 |
+
return dtype
|
| 1687 |
+
|
| 1688 |
+
|
| 1689 |
+
def _vec_from_tensor(x, generator, downcast_complex=False):
|
| 1690 |
+
# Create a random vector with the same number of elements as x and the same
|
| 1691 |
+
# dtype/device. If x is complex and downcast_complex is False, we create a
|
| 1692 |
+
# complex tensor with only real component.
|
| 1693 |
+
if x.layout == torch.sparse_coo:
|
| 1694 |
+
# For sparse, create a random sparse vec with random values in the same
|
| 1695 |
+
# indices. Make sure size is set so that it isn't inferred to be smaller.
|
| 1696 |
+
x_values = x._values()
|
| 1697 |
+
dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype
|
| 1698 |
+
values = (
|
| 1699 |
+
torch.rand(x_values.numel(), generator=generator)
|
| 1700 |
+
.to(dtype=dtype, device=x.device)
|
| 1701 |
+
.view(x_values.shape)
|
| 1702 |
+
)
|
| 1703 |
+
values /= values.norm()
|
| 1704 |
+
vec = torch.sparse_coo_tensor(x._indices(), values, x.size(), device=x.device)
|
| 1705 |
+
elif _is_sparse_compressed_tensor(x):
|
| 1706 |
+
if x.layout in {torch.sparse_csr, torch.sparse_bsr}:
|
| 1707 |
+
compressed_indices, plain_indices = x.crow_indices(), x.col_indices()
|
| 1708 |
+
else:
|
| 1709 |
+
compressed_indices, plain_indices = x.ccol_indices(), x.row_indices()
|
| 1710 |
+
x_values = x.values()
|
| 1711 |
+
dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype
|
| 1712 |
+
values = (
|
| 1713 |
+
torch.rand(x_values.numel(), generator=generator)
|
| 1714 |
+
.to(dtype=dtype, device=x.device)
|
| 1715 |
+
.view(x_values.shape)
|
| 1716 |
+
)
|
| 1717 |
+
values /= values.norm()
|
| 1718 |
+
vec = torch.sparse_compressed_tensor(
|
| 1719 |
+
compressed_indices,
|
| 1720 |
+
plain_indices,
|
| 1721 |
+
values,
|
| 1722 |
+
x.size(),
|
| 1723 |
+
layout=x.layout,
|
| 1724 |
+
device=x.device,
|
| 1725 |
+
)
|
| 1726 |
+
else:
|
| 1727 |
+
dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype
|
| 1728 |
+
vec = torch.rand(x.numel(), generator=generator).to(
|
| 1729 |
+
dtype=dtype, device=x.device
|
| 1730 |
+
)
|
| 1731 |
+
vec /= vec.norm()
|
| 1732 |
+
return vec
|
| 1733 |
+
|
| 1734 |
+
|
| 1735 |
+
def _get_inp_tensors(tupled_inputs):
|
| 1736 |
+
inp_idx_tup = [
|
| 1737 |
+
(i, t)
|
| 1738 |
+
for i, t in enumerate(tupled_inputs)
|
| 1739 |
+
if is_tensor_like(t) and t.requires_grad
|
| 1740 |
+
]
|
| 1741 |
+
return [tup[0] for tup in inp_idx_tup], [tup[1] for tup in inp_idx_tup]
|
| 1742 |
+
|
| 1743 |
+
|
| 1744 |
+
def _adjusted_atol(atol, u, v):
|
| 1745 |
+
# In slow gradcheck, we compare A and B element-wise, i.e., for some a, b we
|
| 1746 |
+
# allow: |a - b| < atol + rtol * b. But since we now compare q1 = v^T A u and
|
| 1747 |
+
# q2 = v^T B u, we must allow |q1 - q2| < v^T E u + rtol * v^T B u, where E is
|
| 1748 |
+
# the correctly sized matrix in which each entry is atol.
|
| 1749 |
+
#
|
| 1750 |
+
# We see that atol needs to be scaled by v^T M u (where M is an all-ones M x N
|
| 1751 |
+
# matrix): v^T M u = \sum_{i} \sum_{j} u[i] * v[j] = sum(u) * sum(v).
|
| 1752 |
+
|
| 1753 |
+
# For the case of complex inputs, u has re and im. components: u = (ur, ui).
|
| 1754 |
+
# Let q = a - b = (qr, qi) in the above notation, eg. q is the difference between analytic and numerical
|
| 1755 |
+
# Jacobians. Then the transformed tolerance checks being done in the torch.allclose ops
|
| 1756 |
+
# are of form abs(Re{q_r u_r - q_i u_i}) < atol * (abs(u_r) + abs(u_i)), and equivalently
|
| 1757 |
+
# for imaginary components abs(Im{q_r u_r - q_i ui}) < atol * (abs(u_r) + abs(u_i)).
|
| 1758 |
+
# Since u is drawn randomly non-negative, the end effect is a factor
|
| 1759 |
+
# (sum(u_r) + sum(u_i)) * sum(v) increase in atol for complex inputs, eg.
|
| 1760 |
+
# a statistical factor of 2 as compared to the real case.
|
| 1761 |
+
|
| 1762 |
+
sum_v = 1.0 if v is None else v.sum()
|
| 1763 |
+
|
| 1764 |
+
if isinstance(u, tuple):
|
| 1765 |
+
# case of complex input
|
| 1766 |
+
ur, ui = u[0], u[1]
|
| 1767 |
+
sum_ur = ur.sum()
|
| 1768 |
+
sum_ui = ui.sum()
|
| 1769 |
+
complex_modified_atol = atol * (float(sum_ur) + float(sum_ui)) * float(sum_v)
|
| 1770 |
+
return complex_modified_atol
|
| 1771 |
+
|
| 1772 |
+
# case of real input
|
| 1773 |
+
sum_u = u.sum()
|
| 1774 |
+
modified_atol = atol * float(sum_u) * float(sum_v)
|
| 1775 |
+
return modified_atol
|
| 1776 |
+
|
| 1777 |
+
|
| 1778 |
+
FAST_FAIL_SLOW_OK_MSG = """
|
| 1779 |
+
Fast gradcheck failed but element-wise differences are small. This means that the
|
| 1780 |
+
test might've passed in slow_mode!
|
| 1781 |
+
|
| 1782 |
+
If you are adding a new operator, please file an issue and then use one of the
|
| 1783 |
+
workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck:
|
| 1784 |
+
|
| 1785 |
+
If the test
|
| 1786 |
+
- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
|
| 1787 |
+
with `fast_mode=False` as a keyword argument.
|
| 1788 |
+
- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
|
| 1789 |
+
to have `gradcheck_fast_mode=False`
|
| 1790 |
+
- is a Module test (e.g., in common_nn.py), then modify the corresponding
|
| 1791 |
+
module_test entry to have `gradcheck_fast_mode=False`
|
| 1792 |
+
""".strip()
|
| 1793 |
+
|
| 1794 |
+
|
| 1795 |
+
def _run_slow_mode_and_get_error(
|
| 1796 |
+
func, tupled_inputs, outputs, input_idx, output_idx, rtol, atol, eps, is_forward_ad
|
| 1797 |
+
):
|
| 1798 |
+
# Compute jacobians in slow mode for better error message
|
| 1799 |
+
slow_numerical = _get_numerical_jacobian(
|
| 1800 |
+
func, tupled_inputs, outputs, eps=eps, is_forward_ad=is_forward_ad
|
| 1801 |
+
)[input_idx][output_idx]
|
| 1802 |
+
if is_forward_ad:
|
| 1803 |
+
|
| 1804 |
+
def new_fn(inp):
|
| 1805 |
+
new_inputs = list(tupled_inputs)
|
| 1806 |
+
new_inputs[input_idx] = inp
|
| 1807 |
+
return _as_tuple(func(*new_inputs))[output_idx]
|
| 1808 |
+
|
| 1809 |
+
slow_analytical = _get_analytical_jacobian_forward_ad(
|
| 1810 |
+
new_fn, (tupled_inputs[input_idx],), (outputs[output_idx],)
|
| 1811 |
+
)[0][0]
|
| 1812 |
+
else:
|
| 1813 |
+
slow_analytical = _get_analytical_jacobian(
|
| 1814 |
+
tupled_inputs, outputs, input_idx, output_idx
|
| 1815 |
+
)
|
| 1816 |
+
|
| 1817 |
+
# Assume jacobians are non-empty and have the same shape
|
| 1818 |
+
slow_max_diff = (slow_numerical - slow_analytical).abs().max()
|
| 1819 |
+
|
| 1820 |
+
slow_allclose = torch.allclose(slow_analytical, slow_numerical, rtol, atol)
|
| 1821 |
+
msg = (
|
| 1822 |
+
"\nThe above quantities relating the numerical and analytical jacobians are computed \n"
|
| 1823 |
+
"in fast mode. See: https://github.com/pytorch/pytorch/issues/53876 for more background \n"
|
| 1824 |
+
"about fast mode. Below, we recompute numerical and analytical jacobians in slow mode:\n\n"
|
| 1825 |
+
f"Numerical:\n {slow_numerical}\n"
|
| 1826 |
+
f"Analytical:\n{slow_analytical}\n\n"
|
| 1827 |
+
f"The max per-element difference (slow mode) is: {slow_max_diff}.\n"
|
| 1828 |
+
)
|
| 1829 |
+
if slow_allclose:
|
| 1830 |
+
# Slow gradcheck would've passed!
|
| 1831 |
+
msg += FAST_FAIL_SLOW_OK_MSG
|
| 1832 |
+
return msg
|
| 1833 |
+
|
| 1834 |
+
|
| 1835 |
+
def _to_flat_dense_if_sparse(tensor):
|
| 1836 |
+
if _is_sparse_any_tensor(tensor):
|
| 1837 |
+
return tensor.to_dense().reshape(-1)
|
| 1838 |
+
else:
|
| 1839 |
+
return tensor
|
| 1840 |
+
|
| 1841 |
+
|
| 1842 |
+
def _make_vectors(inp_tensors, outputs, *, use_forward_ad):
|
| 1843 |
+
# Use our own generator to avoid messing with the user's RNG state
|
| 1844 |
+
g_cpu = torch.Generator()
|
| 1845 |
+
|
| 1846 |
+
def _vec_from_tensor_cpu(*args):
|
| 1847 |
+
# Default allocate all tensors on CPU, so they are on the same device as the generator
|
| 1848 |
+
# even if the user specified a default device
|
| 1849 |
+
with torch.device("cpu"):
|
| 1850 |
+
return _vec_from_tensor(*args)
|
| 1851 |
+
|
| 1852 |
+
all_u = []
|
| 1853 |
+
all_u_dense = []
|
| 1854 |
+
for inp in inp_tensors:
|
| 1855 |
+
ur = _vec_from_tensor_cpu(inp, g_cpu, True)
|
| 1856 |
+
ur_dense = _to_flat_dense_if_sparse(ur)
|
| 1857 |
+
if inp.is_complex():
|
| 1858 |
+
ui = _vec_from_tensor_cpu(inp, g_cpu, True)
|
| 1859 |
+
all_u.append((ur, ui))
|
| 1860 |
+
ui_dense = _to_flat_dense_if_sparse(ui)
|
| 1861 |
+
all_u_dense.append((ur_dense, ui_dense))
|
| 1862 |
+
else:
|
| 1863 |
+
all_u.append(ur)
|
| 1864 |
+
all_u_dense.append(ur_dense)
|
| 1865 |
+
all_v = (
|
| 1866 |
+
None
|
| 1867 |
+
if use_forward_ad
|
| 1868 |
+
else [_vec_from_tensor_cpu(out, g_cpu) for out in outputs]
|
| 1869 |
+
)
|
| 1870 |
+
return all_v, all_u, all_u_dense
|
| 1871 |
+
|
| 1872 |
+
|
| 1873 |
+
def _check_analytical_numerical_equal(
|
| 1874 |
+
all_analytical,
|
| 1875 |
+
all_numerical,
|
| 1876 |
+
complex_indices,
|
| 1877 |
+
tupled_inputs,
|
| 1878 |
+
outputs,
|
| 1879 |
+
func,
|
| 1880 |
+
all_v,
|
| 1881 |
+
all_u,
|
| 1882 |
+
rtol,
|
| 1883 |
+
atol,
|
| 1884 |
+
eps,
|
| 1885 |
+
test_imag,
|
| 1886 |
+
*,
|
| 1887 |
+
is_forward_ad=False,
|
| 1888 |
+
):
|
| 1889 |
+
for i, all_numerical_for_input_i in enumerate(all_numerical):
|
| 1890 |
+
for j, n in enumerate(all_numerical_for_input_i):
|
| 1891 |
+
# Forward AD generates the transpose of what this function expects
|
| 1892 |
+
if is_forward_ad:
|
| 1893 |
+
a = all_analytical[i][j]
|
| 1894 |
+
else:
|
| 1895 |
+
a = all_analytical[j][i]
|
| 1896 |
+
n = n.to(device=a.device)
|
| 1897 |
+
updated_atol = _adjusted_atol(atol, all_u[i], all_v[j] if all_v else None)
|
| 1898 |
+
if not _allclose_with_type_promotion(a, n.to(a.device), rtol, updated_atol):
|
| 1899 |
+
jacobians_str = _run_slow_mode_and_get_error(
|
| 1900 |
+
func, tupled_inputs, outputs, i, j, rtol, atol, eps, is_forward_ad
|
| 1901 |
+
)
|
| 1902 |
+
raise GradcheckError(
|
| 1903 |
+
_get_notallclose_msg(
|
| 1904 |
+
a, n, j, i, complex_indices, test_imag, is_forward_ad
|
| 1905 |
+
)
|
| 1906 |
+
+ jacobians_str
|
| 1907 |
+
)
|
| 1908 |
+
|
| 1909 |
+
|
| 1910 |
+
def _fast_gradcheck(
|
| 1911 |
+
func,
|
| 1912 |
+
func_out,
|
| 1913 |
+
inputs,
|
| 1914 |
+
outputs,
|
| 1915 |
+
eps,
|
| 1916 |
+
rtol,
|
| 1917 |
+
atol,
|
| 1918 |
+
check_grad_dtypes,
|
| 1919 |
+
nondet_tol,
|
| 1920 |
+
*,
|
| 1921 |
+
use_forward_ad=False,
|
| 1922 |
+
complex_indices=None,
|
| 1923 |
+
test_imag=False,
|
| 1924 |
+
masked=False,
|
| 1925 |
+
):
|
| 1926 |
+
# See https://github.com/pytorch/pytorch/issues/53876 for details
|
| 1927 |
+
inp_tensors_idx, inp_tensors = _get_inp_tensors(inputs)
|
| 1928 |
+
# Backward mode computes v^T * J (VJP)
|
| 1929 |
+
# Since we computed J * u (JVP) through finite difference method, we perform an equality check
|
| 1930 |
+
# between VJP * u, v * JVP
|
| 1931 |
+
# ----
|
| 1932 |
+
# Forward mode computes J * u (JVP)
|
| 1933 |
+
# Since we already compute JVP through finite difference method,
|
| 1934 |
+
# we don't need v for correctness check here as asserted below
|
| 1935 |
+
all_v, all_u, all_u_dense = _make_vectors(
|
| 1936 |
+
inp_tensors, outputs, use_forward_ad=use_forward_ad
|
| 1937 |
+
)
|
| 1938 |
+
|
| 1939 |
+
inputs_numerical, all_u_numerical, all_v_numerical = (
|
| 1940 |
+
(inputs, all_u, all_v) if masked else _densify((inputs, all_u, all_v))
|
| 1941 |
+
)
|
| 1942 |
+
|
| 1943 |
+
numerical_vJu = _get_numerical_vJu(
|
| 1944 |
+
func,
|
| 1945 |
+
inputs_numerical,
|
| 1946 |
+
inp_tensors_idx,
|
| 1947 |
+
func_out,
|
| 1948 |
+
all_u_numerical,
|
| 1949 |
+
all_v_numerical,
|
| 1950 |
+
eps,
|
| 1951 |
+
is_forward_ad=use_forward_ad,
|
| 1952 |
+
)
|
| 1953 |
+
# TODO: replicate https://github.com/pytorch/pytorch/pull/77743 for fast gradcheck as well
|
| 1954 |
+
if use_forward_ad:
|
| 1955 |
+
if all_v is not None:
|
| 1956 |
+
raise AssertionError("Expected all_v to be None.")
|
| 1957 |
+
analytical_vJu = _get_analytical_jacobian_forward_ad(
|
| 1958 |
+
func,
|
| 1959 |
+
inputs,
|
| 1960 |
+
_as_tuple(func_out),
|
| 1961 |
+
all_u=all_u,
|
| 1962 |
+
check_grad_dtypes=check_grad_dtypes,
|
| 1963 |
+
)
|
| 1964 |
+
else:
|
| 1965 |
+
if not outputs:
|
| 1966 |
+
_check_no_differentiable_outputs_fast(
|
| 1967 |
+
func, func_out, inputs, inp_tensors_idx, all_u, eps, nondet_tol
|
| 1968 |
+
)
|
| 1969 |
+
|
| 1970 |
+
analytical_vJu = _get_analytical_vJu_backward_mode(
|
| 1971 |
+
inputs, outputs, nondet_tol, check_grad_dtypes, all_v, all_u_dense
|
| 1972 |
+
)
|
| 1973 |
+
|
| 1974 |
+
_check_analytical_numerical_equal(
|
| 1975 |
+
analytical_vJu,
|
| 1976 |
+
numerical_vJu,
|
| 1977 |
+
complex_indices,
|
| 1978 |
+
inputs,
|
| 1979 |
+
outputs,
|
| 1980 |
+
func,
|
| 1981 |
+
all_v,
|
| 1982 |
+
all_u,
|
| 1983 |
+
rtol,
|
| 1984 |
+
atol,
|
| 1985 |
+
eps,
|
| 1986 |
+
test_imag,
|
| 1987 |
+
is_forward_ad=use_forward_ad,
|
| 1988 |
+
)
|
| 1989 |
+
|
| 1990 |
+
return True
|
| 1991 |
+
|
| 1992 |
+
|
| 1993 |
+
# Note [VarArg of Tensors]
|
| 1994 |
+
# ~~~~~~~~~~~~~~~~~~~~~~~~
|
| 1995 |
+
# 'func' accepts a vararg of tensors, which isn't expressible in the type system at the moment.
|
| 1996 |
+
# If https://mypy.readthedocs.io/en/latest/additional_features.html?highlight=callable#extended-callable-types is accepted,
|
| 1997 |
+
# the '...' first argument of Callable can be replaced with VarArg(Tensor).
|
| 1998 |
+
# For now, we permit any input.
|
| 1999 |
+
def gradcheck(
|
| 2000 |
+
func: Callable[..., _TensorOrTensors], # See Note [VarArg of Tensors]
|
| 2001 |
+
inputs: _TensorOrTensors,
|
| 2002 |
+
*,
|
| 2003 |
+
eps: float = 1e-6,
|
| 2004 |
+
atol: float = 1e-5,
|
| 2005 |
+
rtol: float = 1e-3,
|
| 2006 |
+
raise_exception: bool = True,
|
| 2007 |
+
nondet_tol: float = 0.0,
|
| 2008 |
+
check_undefined_grad: bool = True,
|
| 2009 |
+
check_grad_dtypes: bool = False,
|
| 2010 |
+
check_batched_grad: bool = False,
|
| 2011 |
+
check_batched_forward_grad: bool = False,
|
| 2012 |
+
check_forward_ad: bool = False,
|
| 2013 |
+
check_backward_ad: bool = True,
|
| 2014 |
+
fast_mode: bool = False,
|
| 2015 |
+
masked: bool | None = None,
|
| 2016 |
+
) -> bool:
|
| 2017 |
+
r"""Check gradients computed via small finite differences against analytical
|
| 2018 |
+
gradients wrt tensors in :attr:`inputs` that are of floating point or complex type
|
| 2019 |
+
and with ``requires_grad=True``.
|
| 2020 |
+
|
| 2021 |
+
The check between numerical and analytical gradients uses :func:`~torch.allclose`.
|
| 2022 |
+
|
| 2023 |
+
For most of the complex functions we consider for optimization purposes, no notion of
|
| 2024 |
+
Jacobian exists. Instead, gradcheck verifies if the numerical and analytical values of
|
| 2025 |
+
the Wirtinger and Conjugate Wirtinger derivatives are consistent. Because the gradient
|
| 2026 |
+
computation is done under the assumption that the overall function has a real-valued
|
| 2027 |
+
output, we treat functions with complex output in a special way. For these functions,
|
| 2028 |
+
gradcheck is applied to two real-valued functions corresponding to taking the real
|
| 2029 |
+
components of the complex outputs for the first, and taking the imaginary components
|
| 2030 |
+
of the complex outputs for the second. For more details, check out
|
| 2031 |
+
:ref:`complex_autograd-doc`.
|
| 2032 |
+
|
| 2033 |
+
.. note::
|
| 2034 |
+
The default values are designed for :attr:`input` of double precision.
|
| 2035 |
+
This check will likely fail if :attr:`input` is of less precision, e.g.,
|
| 2036 |
+
``FloatTensor``.
|
| 2037 |
+
|
| 2038 |
+
.. note::
|
| 2039 |
+
Gradcheck may fail when evaluated on non-differentiable points
|
| 2040 |
+
because the numerically computed gradients via finite differencing may differ
|
| 2041 |
+
those computed analytically (not necessarily because either is incorrect).
|
| 2042 |
+
For more context, see :ref:`non-differentiable-func-grad`.
|
| 2043 |
+
|
| 2044 |
+
.. warning::
|
| 2045 |
+
If any checked tensor in :attr:`input` has overlapping memory, i.e.,
|
| 2046 |
+
different indices pointing to the same memory address (e.g., from
|
| 2047 |
+
:func:`torch.Tensor.expand`), this check will likely fail because the numerical
|
| 2048 |
+
gradients computed by point perturbation at such indices will change
|
| 2049 |
+
values at all other indices that share the same memory address.
|
| 2050 |
+
|
| 2051 |
+
Args:
|
| 2052 |
+
func (function): a Python function that takes Tensor inputs and returns
|
| 2053 |
+
a Tensor or a tuple of Tensors
|
| 2054 |
+
inputs (tuple of Tensor or Tensor): inputs to the function
|
| 2055 |
+
eps (float, optional): perturbation for finite differences
|
| 2056 |
+
atol (float, optional): absolute tolerance
|
| 2057 |
+
rtol (float, optional): relative tolerance
|
| 2058 |
+
raise_exception (bool, optional): indicating whether to raise an exception if
|
| 2059 |
+
the check fails. The exception gives more information about the
|
| 2060 |
+
exact nature of the failure. This is helpful when debugging gradchecks.
|
| 2061 |
+
nondet_tol (float, optional): tolerance for non-determinism. When running
|
| 2062 |
+
identical inputs through the differentiation, the results must either match
|
| 2063 |
+
exactly (default, 0.0) or be within this tolerance.
|
| 2064 |
+
check_undefined_grad (bool, optional): if ``True``, check if undefined output grads
|
| 2065 |
+
are supported and treated as zeros, for ``Tensor`` outputs.
|
| 2066 |
+
check_batched_grad (bool, optional): if ``True``, check if we can compute
|
| 2067 |
+
batched gradients using prototype vmap support. Defaults to False.
|
| 2068 |
+
check_batched_forward_grad (bool, optional): if ``True``, checks if we can compute
|
| 2069 |
+
batched forward gradients using forward ad and prototype vmap support. Defaults to ``False``.
|
| 2070 |
+
check_forward_ad (bool, optional): if ``True``, check that the gradients computed with forward
|
| 2071 |
+
mode AD match the numerical ones. Defaults to ``False``.
|
| 2072 |
+
check_backward_ad (bool, optional): if ``False``, do not perform any checks that rely on
|
| 2073 |
+
backward mode AD to be implemented. Defaults to ``True``.
|
| 2074 |
+
fast_mode (bool, optional): Fast mode for gradcheck and gradgradcheck is currently only
|
| 2075 |
+
implemented for R to R functions. If none of the inputs and outputs are complex
|
| 2076 |
+
a faster implementation of gradcheck that no longer computes the entire jacobian
|
| 2077 |
+
is run; otherwise, we fall back to the slow implementation.
|
| 2078 |
+
masked (bool, optional): if ``True``, the gradients of unspecified elements of
|
| 2079 |
+
sparse tensors are ignored. Defaults to ``False``.
|
| 2080 |
+
Returns:
|
| 2081 |
+
``True`` if all differences satisfy allclose condition
|
| 2082 |
+
|
| 2083 |
+
"""
|
| 2084 |
+
if not (check_forward_ad or check_backward_ad):
|
| 2085 |
+
raise AssertionError(
|
| 2086 |
+
"Expected at least one of check_forward_ad or check_backward_ad to be True"
|
| 2087 |
+
)
|
| 2088 |
+
if check_batched_grad and not check_backward_ad:
|
| 2089 |
+
raise AssertionError(
|
| 2090 |
+
"Setting check_batched_grad=True requires check_backward_ad to be True"
|
| 2091 |
+
)
|
| 2092 |
+
if check_batched_forward_grad and not check_forward_ad:
|
| 2093 |
+
raise AssertionError(
|
| 2094 |
+
"Setting check_batched_forward_grad=True requires check_forward_ad to be True"
|
| 2095 |
+
)
|
| 2096 |
+
args = locals().copy()
|
| 2097 |
+
args.pop("raise_exception")
|
| 2098 |
+
if not raise_exception:
|
| 2099 |
+
try:
|
| 2100 |
+
return _gradcheck_helper(**args)
|
| 2101 |
+
except GradcheckError:
|
| 2102 |
+
return False
|
| 2103 |
+
else:
|
| 2104 |
+
return _gradcheck_helper(**args)
|
| 2105 |
+
|
| 2106 |
+
|
| 2107 |
+
def _gradcheck_helper(
|
| 2108 |
+
func,
|
| 2109 |
+
inputs,
|
| 2110 |
+
eps,
|
| 2111 |
+
atol,
|
| 2112 |
+
rtol,
|
| 2113 |
+
nondet_tol,
|
| 2114 |
+
check_undefined_grad,
|
| 2115 |
+
check_grad_dtypes,
|
| 2116 |
+
check_batched_grad,
|
| 2117 |
+
check_batched_forward_grad,
|
| 2118 |
+
check_forward_ad,
|
| 2119 |
+
check_backward_ad,
|
| 2120 |
+
fast_mode,
|
| 2121 |
+
masked,
|
| 2122 |
+
):
|
| 2123 |
+
tupled_inputs = _as_tuple(inputs)
|
| 2124 |
+
_check_inputs(tupled_inputs)
|
| 2125 |
+
|
| 2126 |
+
func_out = func(*tupled_inputs)
|
| 2127 |
+
outputs = _differentiable_outputs(func_out)
|
| 2128 |
+
_check_outputs(outputs)
|
| 2129 |
+
|
| 2130 |
+
gradcheck_fn = functools.partial(
|
| 2131 |
+
_fast_gradcheck if fast_mode else _slow_gradcheck, masked=masked
|
| 2132 |
+
)
|
| 2133 |
+
_gradcheck_real_imag(
|
| 2134 |
+
gradcheck_fn,
|
| 2135 |
+
func,
|
| 2136 |
+
func_out,
|
| 2137 |
+
tupled_inputs,
|
| 2138 |
+
outputs,
|
| 2139 |
+
eps,
|
| 2140 |
+
rtol,
|
| 2141 |
+
atol,
|
| 2142 |
+
check_grad_dtypes,
|
| 2143 |
+
check_forward_ad=check_forward_ad,
|
| 2144 |
+
check_backward_ad=check_backward_ad,
|
| 2145 |
+
nondet_tol=nondet_tol,
|
| 2146 |
+
check_undefined_grad=check_undefined_grad,
|
| 2147 |
+
)
|
| 2148 |
+
|
| 2149 |
+
if check_batched_forward_grad:
|
| 2150 |
+
_test_batched_grad_forward_ad(func, tupled_inputs)
|
| 2151 |
+
|
| 2152 |
+
# Short circuit because remaining tests rely on backward AD to be implemented
|
| 2153 |
+
if not check_backward_ad:
|
| 2154 |
+
return True
|
| 2155 |
+
|
| 2156 |
+
for i, o in enumerate(outputs):
|
| 2157 |
+
if check_batched_grad:
|
| 2158 |
+
_test_batched_grad(tupled_inputs, o, i)
|
| 2159 |
+
|
| 2160 |
+
_test_backward_mul_by_grad_output(outputs, tupled_inputs, masked)
|
| 2161 |
+
|
| 2162 |
+
if check_undefined_grad and check_backward_ad:
|
| 2163 |
+
_test_undefined_backward_mode(func, outputs, tupled_inputs)
|
| 2164 |
+
return True
|
| 2165 |
+
|
| 2166 |
+
|
| 2167 |
+
def gradgradcheck(
|
| 2168 |
+
func: Callable[..., _TensorOrTensors], # See Note [VarArg of Tensors]
|
| 2169 |
+
inputs: _TensorOrTensors,
|
| 2170 |
+
grad_outputs: _TensorOrOptionalTensors | None = None,
|
| 2171 |
+
*,
|
| 2172 |
+
eps: float = 1e-6,
|
| 2173 |
+
atol: float = 1e-5,
|
| 2174 |
+
rtol: float = 1e-3,
|
| 2175 |
+
gen_non_contig_grad_outputs: bool = False,
|
| 2176 |
+
raise_exception: bool = True,
|
| 2177 |
+
nondet_tol: float = 0.0,
|
| 2178 |
+
check_undefined_grad: bool = True,
|
| 2179 |
+
check_grad_dtypes: bool = False,
|
| 2180 |
+
check_batched_grad: bool = False,
|
| 2181 |
+
check_fwd_over_rev: bool = False,
|
| 2182 |
+
check_rev_over_rev: bool = True,
|
| 2183 |
+
fast_mode: bool = False,
|
| 2184 |
+
masked: bool = False,
|
| 2185 |
+
) -> bool:
|
| 2186 |
+
r"""Check gradients of gradients computed via small finite differences
|
| 2187 |
+
against analytical gradients wrt tensors in :attr:`inputs` and
|
| 2188 |
+
:attr:`grad_outputs` that are of floating point or complex type and with
|
| 2189 |
+
``requires_grad=True``.
|
| 2190 |
+
|
| 2191 |
+
This function checks that backpropagating through the gradients computed
|
| 2192 |
+
to the given :attr:`grad_outputs` are correct.
|
| 2193 |
+
|
| 2194 |
+
The check between numerical and analytical gradients uses :func:`~torch.allclose`.
|
| 2195 |
+
|
| 2196 |
+
.. note::
|
| 2197 |
+
The default values are designed for :attr:`input` and
|
| 2198 |
+
:attr:`grad_outputs` of double precision. This check will likely fail if
|
| 2199 |
+
they are of less precision, e.g., ``FloatTensor``.
|
| 2200 |
+
|
| 2201 |
+
.. warning::
|
| 2202 |
+
If any checked tensor in :attr:`input` and :attr:`grad_outputs` has
|
| 2203 |
+
overlapping memory, i.e., different indices pointing to the same memory
|
| 2204 |
+
address (e.g., from :func:`torch.Tensor.expand`), this check will likely fail
|
| 2205 |
+
because the numerical gradients computed by point perturbation at such
|
| 2206 |
+
indices will change values at all other indices that share the same
|
| 2207 |
+
memory address.
|
| 2208 |
+
|
| 2209 |
+
Args:
|
| 2210 |
+
func (function): a Python function that takes Tensor inputs and returns
|
| 2211 |
+
a Tensor or a tuple of Tensors
|
| 2212 |
+
inputs (tuple of Tensor or Tensor): inputs to the function
|
| 2213 |
+
grad_outputs (tuple of [Tensor or None] or Tensor, optional): The gradients with
|
| 2214 |
+
respect to the function's outputs.
|
| 2215 |
+
eps (float, optional): perturbation for finite differences
|
| 2216 |
+
atol (float, optional): absolute tolerance
|
| 2217 |
+
rtol (float, optional): relative tolerance
|
| 2218 |
+
gen_non_contig_grad_outputs (bool, optional): if :attr:`grad_outputs` is
|
| 2219 |
+
``None`` and :attr:`gen_non_contig_grad_outputs` is ``True``, the
|
| 2220 |
+
randomly generated gradient outputs are made to be noncontiguous
|
| 2221 |
+
raise_exception (bool, optional): indicating whether to raise an exception if
|
| 2222 |
+
the check fails. The exception gives more information about the
|
| 2223 |
+
exact nature of the failure. This is helpful when debugging gradchecks.
|
| 2224 |
+
nondet_tol (float, optional): tolerance for non-determinism. When running
|
| 2225 |
+
identical inputs through the differentiation, the results must either match
|
| 2226 |
+
exactly (default, 0.0) or be within this tolerance. Note that a small amount
|
| 2227 |
+
of nondeterminism in the gradient will lead to larger inaccuracies in
|
| 2228 |
+
the second derivative.
|
| 2229 |
+
check_undefined_grad (bool, optional): if True, check if undefined output grads
|
| 2230 |
+
are supported and treated as zeros
|
| 2231 |
+
check_batched_grad (bool, optional): if True, check if we can compute
|
| 2232 |
+
batched gradients using prototype vmap support. Defaults to False.
|
| 2233 |
+
fast_mode (bool, optional): if True, run a faster implementation of gradgradcheck that
|
| 2234 |
+
no longer computes the entire jacobian.
|
| 2235 |
+
masked (bool, optional): if True, the gradients of unspecified elements of
|
| 2236 |
+
sparse tensors are ignored (default, False).
|
| 2237 |
+
Returns:
|
| 2238 |
+
True if all differences satisfy allclose condition
|
| 2239 |
+
"""
|
| 2240 |
+
if not (check_fwd_over_rev or check_rev_over_rev):
|
| 2241 |
+
raise AssertionError(
|
| 2242 |
+
"Expected at least one of check_fwd_over_rev or check_rev_over_rev to be True"
|
| 2243 |
+
)
|
| 2244 |
+
if check_undefined_grad and not check_rev_over_rev:
|
| 2245 |
+
raise AssertionError(
|
| 2246 |
+
"Setting check_undefined_grad=True requires check_rev_over_rev to be True"
|
| 2247 |
+
)
|
| 2248 |
+
if check_batched_grad and not check_rev_over_rev:
|
| 2249 |
+
raise AssertionError(
|
| 2250 |
+
"Setting check_batched_grad=True requires check_rev_over_rev to be True"
|
| 2251 |
+
)
|
| 2252 |
+
# TODO: do we want to test this too?
|
| 2253 |
+
# assert not (check_batched_forward_grad and not check_fwd_over_rev), (
|
| 2254 |
+
# "Setting check_batched_forward_grad=True requires check_fwd_over_rev to be True")
|
| 2255 |
+
tupled_inputs = _as_tuple(inputs)
|
| 2256 |
+
|
| 2257 |
+
if grad_outputs is None:
|
| 2258 |
+
# If grad_outputs is not specified, create random Tensors of the same shape, type, and device as the outputs
|
| 2259 |
+
|
| 2260 |
+
outputs = _differentiable_outputs(func(*tupled_inputs))
|
| 2261 |
+
tupled_grad_outputs = tuple(
|
| 2262 |
+
torch.testing.make_tensor(
|
| 2263 |
+
x.shape,
|
| 2264 |
+
dtype=x.dtype
|
| 2265 |
+
if x.is_floating_point() or x.is_complex()
|
| 2266 |
+
else torch.double,
|
| 2267 |
+
device=x.device,
|
| 2268 |
+
low=-1,
|
| 2269 |
+
high=1,
|
| 2270 |
+
requires_grad=True,
|
| 2271 |
+
noncontiguous=gen_non_contig_grad_outputs,
|
| 2272 |
+
)
|
| 2273 |
+
for x in outputs
|
| 2274 |
+
)
|
| 2275 |
+
else:
|
| 2276 |
+
tupled_grad_outputs = _as_tuple(grad_outputs)
|
| 2277 |
+
|
| 2278 |
+
num_outputs = len(tupled_grad_outputs)
|
| 2279 |
+
|
| 2280 |
+
# NB: We need to save the requires_grad information about the inputs here because gradcheck detaches inputs
|
| 2281 |
+
# before running forward mode AD
|
| 2282 |
+
diff_input_args_indices = {
|
| 2283 |
+
i for i, x in enumerate(tupled_inputs) if is_tensor_like(x) and x.requires_grad
|
| 2284 |
+
}
|
| 2285 |
+
diff_grad_output_indices = {
|
| 2286 |
+
i for i, x in enumerate(tupled_grad_outputs) if x.requires_grad
|
| 2287 |
+
}
|
| 2288 |
+
|
| 2289 |
+
def new_func(*args):
|
| 2290 |
+
# Restore the requires_grad information
|
| 2291 |
+
input_args = tuple(
|
| 2292 |
+
x.requires_grad_() if i in diff_input_args_indices else x
|
| 2293 |
+
for i, x in enumerate(args[:-num_outputs])
|
| 2294 |
+
)
|
| 2295 |
+
outputs = _differentiable_outputs(func(*input_args))
|
| 2296 |
+
grad_outputs = tuple(
|
| 2297 |
+
x.requires_grad_() if i in diff_grad_output_indices else x
|
| 2298 |
+
for i, x in enumerate(args[-num_outputs:])
|
| 2299 |
+
)
|
| 2300 |
+
diff_input_args = tuple(
|
| 2301 |
+
x for i, x in enumerate(input_args) if i in diff_input_args_indices
|
| 2302 |
+
)
|
| 2303 |
+
grad_inputs = torch.autograd.grad(
|
| 2304 |
+
outputs, diff_input_args, grad_outputs, create_graph=True, allow_unused=True
|
| 2305 |
+
)
|
| 2306 |
+
grad_inputs = tuple(g for g in grad_inputs if g is not None)
|
| 2307 |
+
return grad_inputs
|
| 2308 |
+
|
| 2309 |
+
return gradcheck(
|
| 2310 |
+
new_func,
|
| 2311 |
+
tupled_inputs + tupled_grad_outputs,
|
| 2312 |
+
eps=eps,
|
| 2313 |
+
atol=atol,
|
| 2314 |
+
rtol=rtol,
|
| 2315 |
+
raise_exception=raise_exception,
|
| 2316 |
+
nondet_tol=nondet_tol,
|
| 2317 |
+
check_undefined_grad=check_undefined_grad,
|
| 2318 |
+
check_grad_dtypes=check_grad_dtypes,
|
| 2319 |
+
check_batched_grad=check_batched_grad,
|
| 2320 |
+
fast_mode=fast_mode,
|
| 2321 |
+
check_forward_ad=check_fwd_over_rev,
|
| 2322 |
+
check_backward_ad=check_rev_over_rev,
|
| 2323 |
+
masked=masked,
|
| 2324 |
+
)
|
western_ai/usa/pytorch/torch/autograd/graph.py
ADDED
|
@@ -0,0 +1,989 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import abc
|
| 2 |
+
import contextlib
|
| 3 |
+
import contextvars
|
| 4 |
+
import functools
|
| 5 |
+
import logging
|
| 6 |
+
import threading
|
| 7 |
+
from collections import defaultdict, deque
|
| 8 |
+
from collections.abc import (
|
| 9 |
+
Callable,
|
| 10 |
+
Generator,
|
| 11 |
+
Iterable,
|
| 12 |
+
Iterator,
|
| 13 |
+
MutableMapping,
|
| 14 |
+
Sequence,
|
| 15 |
+
)
|
| 16 |
+
from typing import (
|
| 17 |
+
Any,
|
| 18 |
+
cast,
|
| 19 |
+
Literal,
|
| 20 |
+
NamedTuple,
|
| 21 |
+
Optional,
|
| 22 |
+
TYPE_CHECKING,
|
| 23 |
+
TypeAlias,
|
| 24 |
+
Union,
|
| 25 |
+
)
|
| 26 |
+
from weakref import WeakKeyDictionary, WeakValueDictionary
|
| 27 |
+
|
| 28 |
+
import torch
|
| 29 |
+
from torch.autograd.variable import Variable
|
| 30 |
+
from torch.utils._python_dispatch import TorchDispatchMode
|
| 31 |
+
from torch.utils.hooks import RemovableHandle
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
if TYPE_CHECKING:
|
| 35 |
+
from torch._ops import OpOverload
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
__all__ = [
|
| 39 |
+
"saved_tensors_hooks",
|
| 40 |
+
"save_on_cpu",
|
| 41 |
+
"disable_saved_tensors_hooks",
|
| 42 |
+
"register_multi_grad_hook",
|
| 43 |
+
"allow_mutation_on_saved_tensors",
|
| 44 |
+
"Node",
|
| 45 |
+
"GradientEdge",
|
| 46 |
+
"get_gradient_edge",
|
| 47 |
+
"increment_version",
|
| 48 |
+
"region_activation_memory_budget",
|
| 49 |
+
"set_warn_on_accumulate_grad_stream_mismatch",
|
| 50 |
+
"set_override_stale_capture_stream",
|
| 51 |
+
]
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
log = logging.getLogger(__name__)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class Node(abc.ABC):
|
| 58 |
+
@abc.abstractmethod
|
| 59 |
+
def name(self) -> str:
|
| 60 |
+
r"""Return the name.
|
| 61 |
+
|
| 62 |
+
Example::
|
| 63 |
+
|
| 64 |
+
>>> import torch
|
| 65 |
+
>>> a = torch.tensor([0., 0., 0.], requires_grad=True)
|
| 66 |
+
>>> b = a.clone()
|
| 67 |
+
>>> assert isinstance(b.grad_fn, torch.autograd.graph.Node)
|
| 68 |
+
>>> print(b.grad_fn.name())
|
| 69 |
+
CloneBackward0
|
| 70 |
+
"""
|
| 71 |
+
raise NotImplementedError
|
| 72 |
+
|
| 73 |
+
@property
|
| 74 |
+
@abc.abstractmethod
|
| 75 |
+
def next_functions(self) -> tuple[tuple[Optional["Node"], int], ...]:
|
| 76 |
+
raise NotImplementedError
|
| 77 |
+
|
| 78 |
+
@abc.abstractmethod
|
| 79 |
+
def metadata(self) -> dict:
|
| 80 |
+
r"""Return the metadata."""
|
| 81 |
+
raise NotImplementedError
|
| 82 |
+
|
| 83 |
+
@abc.abstractmethod
|
| 84 |
+
def _sequence_nr(self) -> int:
|
| 85 |
+
raise NotImplementedError
|
| 86 |
+
|
| 87 |
+
@property
|
| 88 |
+
@abc.abstractmethod
|
| 89 |
+
def _input_metadata(self) -> list[Any]:
|
| 90 |
+
raise NotImplementedError
|
| 91 |
+
|
| 92 |
+
@abc.abstractmethod
|
| 93 |
+
def _register_hook_dict(self, tensor: torch.Tensor) -> None:
|
| 94 |
+
raise NotImplementedError
|
| 95 |
+
|
| 96 |
+
@abc.abstractmethod
|
| 97 |
+
def register_hook(self, fn: Callable[..., Any]) -> RemovableHandle:
|
| 98 |
+
r"""Register a backward hook.
|
| 99 |
+
|
| 100 |
+
The hook will be called every time a gradient with respect to the
|
| 101 |
+
Node is computed. The hook should have the following signature::
|
| 102 |
+
|
| 103 |
+
hook(grad_inputs: Tuple[Tensor], grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
The hook should not modify its argument, but it can optionally return
|
| 107 |
+
a new gradient which will be used in place of :attr:`grad_inputs`.
|
| 108 |
+
|
| 109 |
+
This function returns a handle with a method ``handle.remove()``
|
| 110 |
+
that removes the hook from the module.
|
| 111 |
+
|
| 112 |
+
.. note::
|
| 113 |
+
See :ref:`backward-hooks-execution` for more information on how when this hook
|
| 114 |
+
is executed, and how its execution is ordered relative to other hooks.
|
| 115 |
+
|
| 116 |
+
.. note::
|
| 117 |
+
In the rare case where the hook is registered while the Node has already
|
| 118 |
+
begun execution, there is no longer any guarantee on :attr:`grad_outputs`
|
| 119 |
+
content (it might be as usual or empty depending on other factors). The
|
| 120 |
+
hook can still optionally return a new gradient to be used in place of
|
| 121 |
+
:attr:`grad_inputs` independent of :attr:`grad_outputs`.
|
| 122 |
+
|
| 123 |
+
Example::
|
| 124 |
+
|
| 125 |
+
>>> import torch
|
| 126 |
+
>>> a = torch.tensor([0., 0., 0.], requires_grad=True)
|
| 127 |
+
>>> b = a.clone()
|
| 128 |
+
>>> assert isinstance(b.grad_fn, torch.autograd.graph.Node)
|
| 129 |
+
>>> handle = b.grad_fn.register_hook(lambda gI, gO: (gO[0] * 2,))
|
| 130 |
+
>>> b.sum().backward(retain_graph=True)
|
| 131 |
+
>>> print(a.grad)
|
| 132 |
+
tensor([2., 2., 2.])
|
| 133 |
+
>>> handle.remove() # Removes the hook
|
| 134 |
+
>>> a.grad = None
|
| 135 |
+
>>> b.sum().backward(retain_graph=True)
|
| 136 |
+
>>> print(a.grad)
|
| 137 |
+
tensor([1., 1., 1.])
|
| 138 |
+
"""
|
| 139 |
+
raise NotImplementedError
|
| 140 |
+
|
| 141 |
+
@abc.abstractmethod
|
| 142 |
+
def register_prehook(self, fn: Callable[..., Any]) -> RemovableHandle:
|
| 143 |
+
r"""Register a backward pre-hook.
|
| 144 |
+
|
| 145 |
+
The hook will be called every time a gradient with respect to the
|
| 146 |
+
Node is computed. The hook should have the following signature::
|
| 147 |
+
|
| 148 |
+
hook(grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None
|
| 149 |
+
|
| 150 |
+
The hook should not modify its argument, but it can optionally return
|
| 151 |
+
a new gradient which will be used in place of :attr:`grad_outputs`.
|
| 152 |
+
|
| 153 |
+
This function returns a handle with a method ``handle.remove()``
|
| 154 |
+
that removes the hook from the module.
|
| 155 |
+
|
| 156 |
+
.. note::
|
| 157 |
+
See :ref:`backward-hooks-execution` for more information on how when this hook
|
| 158 |
+
is executed, and how its execution is ordered relative to other hooks.
|
| 159 |
+
|
| 160 |
+
Example::
|
| 161 |
+
|
| 162 |
+
>>> a = torch.tensor([0., 0., 0.], requires_grad=True)
|
| 163 |
+
>>> b = a.clone()
|
| 164 |
+
>>> assert isinstance(b.grad_fn, torch.autograd.graph.Node)
|
| 165 |
+
>>> handle = b.grad_fn.register_prehook(lambda gI: (gI[0] * 2,))
|
| 166 |
+
>>> b.sum().backward(retain_graph=True)
|
| 167 |
+
>>> print(a.grad)
|
| 168 |
+
tensor([2., 2., 2.])
|
| 169 |
+
>>> handle.remove()
|
| 170 |
+
>>> a.grad = None
|
| 171 |
+
>>> b.sum().backward(retain_graph=True)
|
| 172 |
+
>>> print(a.grad)
|
| 173 |
+
tensor([1., 1., 1.])
|
| 174 |
+
"""
|
| 175 |
+
raise NotImplementedError
|
| 176 |
+
|
| 177 |
+
@classmethod
|
| 178 |
+
def __subclasshook__(cls, subclass: type) -> bool:
|
| 179 |
+
if cls is Node and (
|
| 180 |
+
(
|
| 181 |
+
subclass is not None
|
| 182 |
+
and subclass is getattr(torch._C._functions, subclass.__name__, None)
|
| 183 |
+
)
|
| 184 |
+
or issubclass(subclass, torch.autograd.function.BackwardCFunction)
|
| 185 |
+
):
|
| 186 |
+
return True
|
| 187 |
+
return NotImplemented
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def _get_grad_fn_or_grad_acc(t: Union[torch.Tensor, "GradientEdge"]) -> Node:
|
| 191 |
+
if isinstance(t, GradientEdge):
|
| 192 |
+
return t.node
|
| 193 |
+
if t.requires_grad and t.grad_fn is None:
|
| 194 |
+
with torch.enable_grad():
|
| 195 |
+
node = t.view_as(t).grad_fn.next_functions[0][0] # type: ignore[union-attr]
|
| 196 |
+
else:
|
| 197 |
+
node = t.grad_fn
|
| 198 |
+
if node is None:
|
| 199 |
+
raise AssertionError("Expected gradient function to be set")
|
| 200 |
+
return node
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
class GradientEdge(NamedTuple):
|
| 204 |
+
"""Object representing a given gradient edge within the autograd graph.
|
| 205 |
+
|
| 206 |
+
To get the gradient edge where a given Tensor gradient will be computed,
|
| 207 |
+
you can do ``edge = autograd.graph.get_gradient_edge(tensor)``.
|
| 208 |
+
"""
|
| 209 |
+
|
| 210 |
+
node: Node
|
| 211 |
+
output_nr: int
|
| 212 |
+
# This token can be used to ensure the graph stays alive when it cannot be
|
| 213 |
+
# done via the node field
|
| 214 |
+
ownership_token: Node | None = None
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def get_gradient_edge(tensor: torch.Tensor) -> GradientEdge:
|
| 218 |
+
"""Get the gradient edge for computing the gradient of the given Tensor.
|
| 219 |
+
|
| 220 |
+
In particular, it is equivalent to call
|
| 221 |
+
``g = autograd.grad(loss, input)`` and ``g = autograd.grad(loss, get_gradient_edge(input))``.
|
| 222 |
+
"""
|
| 223 |
+
if not tensor.requires_grad:
|
| 224 |
+
raise RuntimeError(
|
| 225 |
+
"It is not possible to get the gradient edge for a Tensor "
|
| 226 |
+
"that does not require gradients",
|
| 227 |
+
)
|
| 228 |
+
grad_fn = _get_grad_fn_or_grad_acc(tensor)
|
| 229 |
+
|
| 230 |
+
# Python-based Node are owned by the C++ side meaning the python grad_fn
|
| 231 |
+
# object we hold here does NOT keep the C++ graph alive.
|
| 232 |
+
# Create an ownership token by creating a new C++ node that own the graph
|
| 233 |
+
# we care about here.
|
| 234 |
+
token = None
|
| 235 |
+
if isinstance(grad_fn, torch._C._FunctionBase):
|
| 236 |
+
with torch.enable_grad():
|
| 237 |
+
token = tensor.view_as(tensor).grad_fn
|
| 238 |
+
|
| 239 |
+
# Note that output_nr default to 0 which is the right value
|
| 240 |
+
# for the AccumulateGrad node.
|
| 241 |
+
# pyrefly: ignore [bad-argument-type]
|
| 242 |
+
return GradientEdge(grad_fn, tensor.output_nr, ownership_token=token)
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def increment_version(tensor: torch.Tensor | Iterable[torch.Tensor]) -> None:
|
| 246 |
+
"""Update autograd metadata tracking whether the given Tensor was modified in place.
|
| 247 |
+
|
| 248 |
+
This is to enable more accurate error checking within the autograd engine.
|
| 249 |
+
It is already done automatically by PyTorch functions and within custom Function
|
| 250 |
+
when mark_dirty() is called appropriately so you only need to call this explicitly
|
| 251 |
+
if you are doing inplace operation on the Tensor data in a way that Pytorch doesn't
|
| 252 |
+
know about. For example a custom kernel that reads the Tensor data_ptr and modifies
|
| 253 |
+
the memory inplace based on this pointer. Can accept either a tensor, or a list of tensors.
|
| 254 |
+
|
| 255 |
+
Note that incrementing the version counter multiple times for a single inplace operation
|
| 256 |
+
is not problematic.
|
| 257 |
+
|
| 258 |
+
Note that if you pass in tensor constructed under torch.inference_mode(),
|
| 259 |
+
we will not bump its version counter (because your tensor does not have one).
|
| 260 |
+
"""
|
| 261 |
+
if isinstance(tensor, torch.Tensor):
|
| 262 |
+
tensor = (tensor,)
|
| 263 |
+
torch._C._increment_version(tensor)
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
class saved_tensors_hooks:
|
| 267 |
+
"""Context-manager that sets a pair of pack / unpack hooks for saved tensors.
|
| 268 |
+
|
| 269 |
+
Use this context-manager to define how intermediary results of an operation
|
| 270 |
+
should be packed before saving, and unpacked on retrieval.
|
| 271 |
+
|
| 272 |
+
In that context, the ``pack_hook`` function will be called every time an
|
| 273 |
+
operation saves a tensor for backward (this includes intermediary results
|
| 274 |
+
saved using
|
| 275 |
+
:func:`~torch.autograd.function._ContextMethodMixin.save_for_backward` but
|
| 276 |
+
also those recorded by a PyTorch-defined operation). The output of
|
| 277 |
+
``pack_hook`` is then stored in the computation graph instead of the
|
| 278 |
+
original tensor.
|
| 279 |
+
|
| 280 |
+
The ``unpack_hook`` is called when the saved tensor needs to be accessed,
|
| 281 |
+
namely when executing :func:`torch.Tensor.backward()` or
|
| 282 |
+
:func:`torch.autograd.grad()`. It takes as argument the *packed* object
|
| 283 |
+
returned by ``pack_hook`` and should return a tensor which has the same
|
| 284 |
+
content as the original tensor (passed as input to the corresponding
|
| 285 |
+
``pack_hook``).
|
| 286 |
+
|
| 287 |
+
The hooks should have the following signatures:
|
| 288 |
+
|
| 289 |
+
pack_hook(tensor: Tensor) -> Any
|
| 290 |
+
|
| 291 |
+
unpack_hook(Any) -> Tensor
|
| 292 |
+
|
| 293 |
+
where the return value of ``pack_hook`` is a valid input to ``unpack_hook``.
|
| 294 |
+
|
| 295 |
+
In general, you want ``unpack_hook(pack_hook(t))`` to be equal to ``t`` in terms
|
| 296 |
+
of value, size, dtype and device.
|
| 297 |
+
|
| 298 |
+
Example::
|
| 299 |
+
|
| 300 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 301 |
+
>>> def pack_hook(x):
|
| 302 |
+
... print("Packing", x)
|
| 303 |
+
... return x.detach()
|
| 304 |
+
>>>
|
| 305 |
+
>>> def unpack_hook(x):
|
| 306 |
+
... print("Unpacking", x)
|
| 307 |
+
... return x
|
| 308 |
+
>>>
|
| 309 |
+
>>> a = torch.ones(5, requires_grad=True)
|
| 310 |
+
>>> b = torch.ones(5, requires_grad=True) * 2
|
| 311 |
+
>>> with torch.autograd.graph.saved_tensors_hooks(pack_hook, unpack_hook):
|
| 312 |
+
... y = a * b
|
| 313 |
+
Packing tensor([1., 1., 1., 1., 1.], requires_grad=True)
|
| 314 |
+
Packing tensor([2., 2., 2., 2., 2.], grad_fn=<MulBackward0>)
|
| 315 |
+
>>> y.sum().backward()
|
| 316 |
+
Unpacking tensor([1., 1., 1., 1., 1.], requires_grad=True)
|
| 317 |
+
Unpacking tensor([2., 2., 2., 2., 2.], grad_fn=<MulBackward0>)
|
| 318 |
+
|
| 319 |
+
.. warning ::
|
| 320 |
+
Performing an inplace operation on the input to either hooks may lead
|
| 321 |
+
to undefined behavior.
|
| 322 |
+
|
| 323 |
+
.. warning ::
|
| 324 |
+
Only one pair of hooks is allowed at a time. When recursively nesting this
|
| 325 |
+
context-manager, only the inner-most pair of hooks will be applied.
|
| 326 |
+
|
| 327 |
+
.. warning ::
|
| 328 |
+
To avoid reference cycle, the return value of ``pack_hook`` cannot hold a
|
| 329 |
+
reference to the input tensor. For example, use `lambda x: x.detach()`
|
| 330 |
+
instead of `lambda x: x` as the pack hook.
|
| 331 |
+
"""
|
| 332 |
+
|
| 333 |
+
def __init__(
|
| 334 |
+
self,
|
| 335 |
+
pack_hook: Callable[[torch.Tensor], Any],
|
| 336 |
+
unpack_hook: Callable[[Any], torch.Tensor],
|
| 337 |
+
) -> None:
|
| 338 |
+
self.pack_hook = pack_hook
|
| 339 |
+
self.unpack_hook = unpack_hook
|
| 340 |
+
|
| 341 |
+
def __enter__(self) -> None:
|
| 342 |
+
torch._C._autograd._push_saved_tensors_default_hooks(
|
| 343 |
+
self.pack_hook, self.unpack_hook
|
| 344 |
+
)
|
| 345 |
+
|
| 346 |
+
def __exit__(self, *args: object) -> None:
|
| 347 |
+
torch._C._autograd._pop_saved_tensors_default_hooks()
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
class save_on_cpu(saved_tensors_hooks):
|
| 351 |
+
"""Context manager under which tensors saved by the forward pass will be stored on cpu, then retrieved for backward.
|
| 352 |
+
|
| 353 |
+
When performing operations within this context manager, intermediary
|
| 354 |
+
results saved in the graph during the forward pass will be moved to CPU,
|
| 355 |
+
then copied back to the original device when needed for the backward pass.
|
| 356 |
+
If the graph was already on CPU, no tensor copy is performed.
|
| 357 |
+
|
| 358 |
+
Use this context-manager to trade compute for GPU memory usage (e.g.
|
| 359 |
+
when your model doesn't fit in GPU memory during training).
|
| 360 |
+
|
| 361 |
+
.. warning::
|
| 362 |
+
|
| 363 |
+
When ``pin_memory=True``, the GPU to CPU copy during packing is
|
| 364 |
+
asynchronous. Accessing saved tensors on CPU (e.g. via
|
| 365 |
+
``grad_fn._saved_self``) before the CUDA stream has finished may
|
| 366 |
+
yield incorrect data. Call :func:`torch.cuda.synchronize` first
|
| 367 |
+
if you need to read them.
|
| 368 |
+
|
| 369 |
+
Args:
|
| 370 |
+
pin_memory (bool): If ``True`` tensors will be saved to CPU pinned memory
|
| 371 |
+
during packing and copied to GPU asynchronously during both
|
| 372 |
+
packing and unpacking.
|
| 373 |
+
Defaults to ``False``.
|
| 374 |
+
Also see :ref:`cuda-memory-pinning`.
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
Example::
|
| 378 |
+
|
| 379 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA)
|
| 380 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
|
| 381 |
+
>>> a = torch.randn(5, requires_grad=True, device="cuda")
|
| 382 |
+
>>> b = torch.randn(5, requires_grad=True, device="cuda")
|
| 383 |
+
>>> c = torch.randn(5, requires_grad=True, device="cuda")
|
| 384 |
+
>>>
|
| 385 |
+
>>> def f(a, b, c):
|
| 386 |
+
... prod_1 = a * b # a and b are saved on GPU
|
| 387 |
+
... with torch.autograd.graph.save_on_cpu():
|
| 388 |
+
... prod_2 = prod_1 * c # prod_1 and c are saved on CPU
|
| 389 |
+
... y = prod_2 * a # prod_2 and a are saved on GPU
|
| 390 |
+
... return y
|
| 391 |
+
>>>
|
| 392 |
+
>>> y = f(a, b, c)
|
| 393 |
+
>>> del a, b, c # for illustration only
|
| 394 |
+
>>> # the content of a, b, and prod_2 are still alive on GPU
|
| 395 |
+
>>> # the content of prod_1 and c only live on CPU
|
| 396 |
+
>>> y.sum().backward() # all CPU tensors are moved back to GPU, for backward
|
| 397 |
+
>>> # all intermediary tensors are released (deleted) after the call to backward
|
| 398 |
+
"""
|
| 399 |
+
|
| 400 |
+
def __init__(self, pin_memory: bool = False, device_type: str = "cuda") -> None:
|
| 401 |
+
device_module = getattr(torch, device_type, torch.cuda)
|
| 402 |
+
|
| 403 |
+
def pack_to_cpu(tensor: torch.Tensor) -> tuple[torch.device, torch.Tensor]:
|
| 404 |
+
if not pin_memory:
|
| 405 |
+
return (tensor.device, tensor.cpu())
|
| 406 |
+
is_pinnable = device_module.is_available() and not tensor.is_sparse
|
| 407 |
+
packed = torch.empty(
|
| 408 |
+
tensor.size(),
|
| 409 |
+
dtype=tensor.dtype,
|
| 410 |
+
layout=tensor.layout,
|
| 411 |
+
pin_memory=is_pinnable,
|
| 412 |
+
)
|
| 413 |
+
packed.copy_(tensor, non_blocking=is_pinnable)
|
| 414 |
+
return (tensor.device, packed)
|
| 415 |
+
|
| 416 |
+
def unpack_from_cpu(packed: tuple[torch.device, torch.Tensor]) -> torch.Tensor:
|
| 417 |
+
device, tensor = packed
|
| 418 |
+
return tensor.to(device, non_blocking=pin_memory)
|
| 419 |
+
|
| 420 |
+
super().__init__(pack_to_cpu, unpack_from_cpu)
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
@contextlib.contextmanager
|
| 424 |
+
def disable_saved_tensors_hooks(error_message: str) -> Generator[None, None, None]:
|
| 425 |
+
"""Context-manager that disables the saved tensors default hooks feature.
|
| 426 |
+
|
| 427 |
+
Useful for if you are creating a feature that does not work with saved
|
| 428 |
+
tensors default hooks.
|
| 429 |
+
|
| 430 |
+
Args:
|
| 431 |
+
error_message (str): When saved tensors default hooks are used when they
|
| 432 |
+
have been disabled, a RuntimeError with this
|
| 433 |
+
error message gets raised.
|
| 434 |
+
|
| 435 |
+
Example::
|
| 436 |
+
|
| 437 |
+
>>> # xdoctest: +SKIP(failing)
|
| 438 |
+
>>> message = "saved tensors default hooks are disabled"
|
| 439 |
+
>>> with torch.autograd.graph.disable_saved_tensors_hooks(message):
|
| 440 |
+
... # Raises RuntimeError: saved tensors default hooks are disabled
|
| 441 |
+
... with torch.autograd.graph.save_on_cpu():
|
| 442 |
+
... pass
|
| 443 |
+
"""
|
| 444 |
+
maybe_prev_message = None
|
| 445 |
+
try:
|
| 446 |
+
maybe_prev_message = (
|
| 447 |
+
torch._C._autograd._saved_tensors_hooks_get_disabled_error_message()
|
| 448 |
+
)
|
| 449 |
+
torch._C._autograd._saved_tensors_hooks_disable(error_message)
|
| 450 |
+
yield
|
| 451 |
+
finally:
|
| 452 |
+
# See NOTE: [disabled_error_message invariant]
|
| 453 |
+
if maybe_prev_message is None:
|
| 454 |
+
torch._C._autograd._saved_tensors_hooks_enable()
|
| 455 |
+
else:
|
| 456 |
+
torch._C._autograd._saved_tensors_hooks_disable(maybe_prev_message)
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
def region_activation_memory_budget(
|
| 460 |
+
budget: float,
|
| 461 |
+
) -> contextlib.AbstractContextManager[None]:
|
| 462 |
+
r"""Context-manager that sets the activation memory budget for the region of
|
| 463 |
+
a compiled forward traced under it.
|
| 464 |
+
|
| 465 |
+
.. warning::
|
| 466 |
+
This is a prototype feature and is subject to change.
|
| 467 |
+
|
| 468 |
+
Under :func:`torch.compile`, the min-cut partitioner chooses which
|
| 469 |
+
activations to save versus recompute in the backward pass to stay under a
|
| 470 |
+
memory budget. ``budget`` is a ratio in ``[0, 1]``: ``0.0`` corresponds to
|
| 471 |
+
the activation memory from applying activation checkpointing to the full
|
| 472 |
+
region, and ``1.0`` corresponds to the activation memory from the default
|
| 473 |
+
runtime-optimized strategy. So ``0.4`` would result in a strategy that saves
|
| 474 |
+
40% of the activations compared to the default strategy. It solves a 0-1
|
| 475 |
+
knapsack to find the minimum recompute necessary to stay below the budget.
|
| 476 |
+
This overrides the global ``torch._functorch.config.activation_memory_budget``
|
| 477 |
+
for the annotated region.
|
| 478 |
+
|
| 479 |
+
.. note::
|
| 480 |
+
Today the partitioner only supports a single budget per compiled graph,
|
| 481 |
+
so the annotation must cover every forward op in the graph (a partial
|
| 482 |
+
annotation is rejected rather than silently applied graph-wide), and all
|
| 483 |
+
annotated nodes must agree on the budget. To use different budgets for
|
| 484 |
+
different parts of a model, separate them with a graph break (e.g.
|
| 485 |
+
``torch._dynamo.graph_break()``) so each part becomes its own graph.
|
| 486 |
+
|
| 487 |
+
This only has an effect under :func:`torch.compile`; using it outside of a
|
| 488 |
+
compiled region raises a ``RuntimeError``.
|
| 489 |
+
|
| 490 |
+
Args:
|
| 491 |
+
budget (float): Activation memory budget ratio in ``[0, 1]``.
|
| 492 |
+
|
| 493 |
+
Example::
|
| 494 |
+
|
| 495 |
+
>>> # xdoctest: +SKIP
|
| 496 |
+
>>> with torch.autograd.graph.region_activation_memory_budget(0.0):
|
| 497 |
+
... x = layer(x) # recompute this region's activations in backward
|
| 498 |
+
"""
|
| 499 |
+
import torch.fx.traceback as fx_traceback
|
| 500 |
+
|
| 501 |
+
if isinstance(budget, bool) or not isinstance(budget, (int, float)):
|
| 502 |
+
raise TypeError(
|
| 503 |
+
f"torch.autograd.graph.region_activation_memory_budget: expects a "
|
| 504 |
+
f"float, got {type(budget).__name__}"
|
| 505 |
+
)
|
| 506 |
+
if not 0.0 <= budget <= 1.0:
|
| 507 |
+
raise ValueError(
|
| 508 |
+
f"torch.autograd.graph.region_activation_memory_budget: must be in "
|
| 509 |
+
f"[0, 1], got {budget}"
|
| 510 |
+
)
|
| 511 |
+
# The budget only takes effect when read back by the partitioner during
|
| 512 |
+
# compilation. Dynamo folds torch.compiler.is_compiling() to True while
|
| 513 |
+
# tracing this (inlined) call, so this guard only fires in eager mode.
|
| 514 |
+
if not torch.compiler.is_compiling():
|
| 515 |
+
raise RuntimeError(
|
| 516 |
+
"torch.autograd.graph.region_activation_memory_budget can only be "
|
| 517 |
+
"used inside a torch.compile region; it has no effect in eager mode."
|
| 518 |
+
)
|
| 519 |
+
return fx_traceback.annotate(
|
| 520 |
+
{fx_traceback.MEMORY_BUDGET_ANNOTATION_KEY: float(budget)}
|
| 521 |
+
)
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def set_warn_on_accumulate_grad_stream_mismatch(enabled: bool) -> None:
|
| 525 |
+
"""Whether to warn when the AccumulateGrad node's stream does not match the stream
|
| 526 |
+
of the node that produced the incoming gradient.
|
| 527 |
+
"""
|
| 528 |
+
return torch._C._set_warn_on_accumulate_grad_stream_mismatch(enabled)
|
| 529 |
+
|
| 530 |
+
|
| 531 |
+
def set_override_stale_capture_stream(enabled: bool) -> None:
|
| 532 |
+
"""Control behavior when autograd detects a stale non-capturing stream during
|
| 533 |
+
CUDA graph capture.
|
| 534 |
+
|
| 535 |
+
During CUDA graph capture, autograd nodes may reference a stale stream
|
| 536 |
+
that is not part of the capture. With the flag disabled (the
|
| 537 |
+
process-initial state), autograd raises a ``RuntimeError`` when the stale
|
| 538 |
+
stream is the default stream (stream 0), because this case always
|
| 539 |
+
invalidates the capture: ``cudaStreamWaitEvent`` on the default stream
|
| 540 |
+
pulls a non-capturing stream into the graph. For non-default stale streams
|
| 541 |
+
the stream reference is left unchanged; the capture will succeed if the
|
| 542 |
+
user has joined the stream into the capture (e.g. via
|
| 543 |
+
``capture_stream.wait_stream(stale_stream)``) and will otherwise fail with
|
| 544 |
+
a CUDA runtime error.
|
| 545 |
+
|
| 546 |
+
When ``enabled=True``, any stale non-capturing stream (default or
|
| 547 |
+
non-default) is automatically overridden with the producer's capturing
|
| 548 |
+
stream, allowing the capture to proceed. This is a process-global setting
|
| 549 |
+
and is not thread-local.
|
| 550 |
+
|
| 551 |
+
Args:
|
| 552 |
+
enabled (bool): If ``True``, override stale non-capturing streams with
|
| 553 |
+
the producer's capturing stream during CUDA graph capture. If
|
| 554 |
+
``False`` (the process-initial state), raise an error only when the
|
| 555 |
+
stale stream is the default stream (stream 0); other stale streams
|
| 556 |
+
are left unchanged.
|
| 557 |
+
"""
|
| 558 |
+
return torch._C._set_override_stale_capture_stream(enabled)
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
class _MultiHandle(RemovableHandle):
|
| 562 |
+
handles: tuple[RemovableHandle, ...]
|
| 563 |
+
|
| 564 |
+
def __init__(self, handles: tuple[RemovableHandle, ...]) -> None:
|
| 565 |
+
self.handles = handles
|
| 566 |
+
|
| 567 |
+
def remove(self) -> None:
|
| 568 |
+
for handle in self.handles:
|
| 569 |
+
handle.remove()
|
| 570 |
+
|
| 571 |
+
def __getstate__(self) -> tuple[RemovableHandle, ...]:
|
| 572 |
+
return self.handles
|
| 573 |
+
|
| 574 |
+
def __setstate__(self, state: tuple[RemovableHandle, ...]) -> None:
|
| 575 |
+
self.handles = state
|
| 576 |
+
|
| 577 |
+
|
| 578 |
+
def register_multi_grad_hook(
|
| 579 |
+
tensors: Sequence[torch.Tensor],
|
| 580 |
+
fn: Callable[[Sequence[torch.Tensor | None]], None]
|
| 581 |
+
| Callable[[torch.Tensor], None],
|
| 582 |
+
*,
|
| 583 |
+
mode: Literal["all", "any"] = "all",
|
| 584 |
+
) -> RemovableHandle:
|
| 585 |
+
r"""Register a multi-grad backward hook.
|
| 586 |
+
|
| 587 |
+
There are two supported modes: ``"all"`` and ``"any"``.
|
| 588 |
+
|
| 589 |
+
Under the ``"all"`` mode, the hook will be called after gradients with respect to every tensor in
|
| 590 |
+
:attr:`tensors` have been computed. If a tensor is in :attr:`tensors` but
|
| 591 |
+
is not part of the graph, or if a tensor is not needed to compute the gradients
|
| 592 |
+
for any ``inputs`` specified for the current ``.backward()`` or ``.grad()`` call,
|
| 593 |
+
this tensor will be ignored and the hook will not wait for its gradient to be
|
| 594 |
+
computed.
|
| 595 |
+
|
| 596 |
+
After every non-ignored tensor's gradient has been computed, :attr:`fn` will be
|
| 597 |
+
called with those gradients. ``None`` will be passed for tensors that did not
|
| 598 |
+
have their gradients computed.
|
| 599 |
+
|
| 600 |
+
Under the ``"any"`` mode, the hook will be called after the first gradient
|
| 601 |
+
with respect to a tensor in :attr:`tensors` has been computed. The hook
|
| 602 |
+
will be called with that gradient as its argument.
|
| 603 |
+
|
| 604 |
+
The hook should not modify its arguments.
|
| 605 |
+
|
| 606 |
+
This function returns a handle with a method ``handle.remove()`` that removes the hook.
|
| 607 |
+
|
| 608 |
+
.. note::
|
| 609 |
+
See :ref:`backward-hooks-execution` for more information on how when this hook
|
| 610 |
+
is executed, and how its execution is ordered relative to other hooks.
|
| 611 |
+
|
| 612 |
+
Example::
|
| 613 |
+
|
| 614 |
+
>>> import torch
|
| 615 |
+
>>>
|
| 616 |
+
>>> a = torch.rand(2, 3, requires_grad=True)
|
| 617 |
+
>>> b = torch.rand(2, 3, requires_grad=True)
|
| 618 |
+
>>> c = a * b
|
| 619 |
+
>>> d = a * b
|
| 620 |
+
>>>
|
| 621 |
+
>>> def fn(grads):
|
| 622 |
+
... print([g is not None for g in grads])
|
| 623 |
+
...
|
| 624 |
+
>>> torch.autograd.graph.register_multi_grad_hook((a, b, c, d), fn)
|
| 625 |
+
>>>
|
| 626 |
+
>>> c.sum().backward(retain_graph=True)
|
| 627 |
+
[True, True, True, False]
|
| 628 |
+
>>> c.sum().backward(inputs=(a,), retain_graph=True)
|
| 629 |
+
[True, False, True, False]
|
| 630 |
+
>>>
|
| 631 |
+
"""
|
| 632 |
+
supported_modes = ("all", "any")
|
| 633 |
+
lock = threading.Lock()
|
| 634 |
+
|
| 635 |
+
if mode not in supported_modes:
|
| 636 |
+
raise ValueError(f"Expects mode to be one of {supported_modes} but got {mode}")
|
| 637 |
+
|
| 638 |
+
if mode == "all":
|
| 639 |
+
count: dict[int, int] = {}
|
| 640 |
+
nb_calls = None
|
| 641 |
+
buffer: dict[int, list[torch.Tensor | None]] = {}
|
| 642 |
+
|
| 643 |
+
grad_fns = list(map(_get_grad_fn_or_grad_acc, tensors))
|
| 644 |
+
len_tensors = len(tensors)
|
| 645 |
+
|
| 646 |
+
def get_inner_hook(idx: int) -> Callable[[torch.Tensor], None]:
|
| 647 |
+
def inner_hook(grad: torch.Tensor) -> None:
|
| 648 |
+
nonlocal count, nb_calls, buffer, fn
|
| 649 |
+
id = torch._C._current_graph_task_id()
|
| 650 |
+
if id == -1:
|
| 651 |
+
raise AssertionError(
|
| 652 |
+
"expected this hook to be called inside a backward call"
|
| 653 |
+
)
|
| 654 |
+
count[id] = count.get(id, 0)
|
| 655 |
+
# pyrefly: ignore [unsupported-operation]
|
| 656 |
+
buffer[id] = buffer.get(id, [None] * len_tensors)
|
| 657 |
+
|
| 658 |
+
with lock:
|
| 659 |
+
curr_count, count[id] = count[id], count[id] + 1
|
| 660 |
+
|
| 661 |
+
if curr_count == 0:
|
| 662 |
+
# On the first call, compute the actual nb_calls and buffer
|
| 663 |
+
nb_calls = sum(
|
| 664 |
+
map(torch._C._will_engine_execute_node, grad_fns)
|
| 665 |
+
)
|
| 666 |
+
|
| 667 |
+
buffer[id][idx] = grad
|
| 668 |
+
|
| 669 |
+
if nb_calls is None:
|
| 670 |
+
raise AssertionError("Expected nb_calls to be set")
|
| 671 |
+
if curr_count == nb_calls - 1:
|
| 672 |
+
fn = cast(Callable[[Sequence[torch.Tensor | None]], None], fn)
|
| 673 |
+
fn(buffer[id])
|
| 674 |
+
del count[id]
|
| 675 |
+
del buffer[id]
|
| 676 |
+
|
| 677 |
+
return inner_hook
|
| 678 |
+
|
| 679 |
+
handles = tuple(
|
| 680 |
+
t.register_hook(get_inner_hook(i)) for i, t in enumerate(tensors)
|
| 681 |
+
)
|
| 682 |
+
elif mode == "any":
|
| 683 |
+
fn = cast(Callable[[torch.Tensor], None], fn)
|
| 684 |
+
ran_hook: dict[int, bool] = defaultdict(bool)
|
| 685 |
+
|
| 686 |
+
@functools.wraps(fn)
|
| 687 |
+
def wrapped_fn(grad: torch.Tensor) -> None:
|
| 688 |
+
nonlocal ran_hook
|
| 689 |
+
id = torch._C._current_graph_task_id()
|
| 690 |
+
if id == -1:
|
| 691 |
+
raise AssertionError(
|
| 692 |
+
"expected this hook to be called inside a backward call"
|
| 693 |
+
)
|
| 694 |
+
with lock:
|
| 695 |
+
prev, ran_hook[id] = ran_hook[id], True
|
| 696 |
+
if prev:
|
| 697 |
+
return
|
| 698 |
+
fn(grad)
|
| 699 |
+
|
| 700 |
+
handles = tuple(
|
| 701 |
+
tensor.register_hook(wrapped_fn)
|
| 702 |
+
for tensor in tensors
|
| 703 |
+
if tensor.requires_grad
|
| 704 |
+
)
|
| 705 |
+
|
| 706 |
+
return _MultiHandle(handles) # type: ignore[possibly-undefined]
|
| 707 |
+
|
| 708 |
+
|
| 709 |
+
# NOTE [Allow mutation on tensors saved for backward]
|
| 710 |
+
#
|
| 711 |
+
# 1. Tensor gets saved for backward
|
| 712 |
+
# - remember the python object id and the version of the tensor
|
| 713 |
+
# - remember aliasing information (data_ptr of base + version)
|
| 714 |
+
# - save the original so we control its lifetime
|
| 715 |
+
# 2. Any time a tensor gets in-placed
|
| 716 |
+
# - for each tensor aliased to it:
|
| 717 |
+
# - check using its object id and version to see if it has been saved
|
| 718 |
+
# - if it has been saved, clone it
|
| 719 |
+
# - delete the reference to the original
|
| 720 |
+
# 3. during backward
|
| 721 |
+
# - if the clone exists, the tensor must've been modified in-place
|
| 722 |
+
_allow_mutation_on_saved_tensors_enabled: bool = False
|
| 723 |
+
|
| 724 |
+
|
| 725 |
+
_TID: TypeAlias = tuple[int, int, int]
|
| 726 |
+
_SID: TypeAlias = tuple[int, int]
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
def _get_tid(tensor: torch.Tensor) -> _TID:
|
| 730 |
+
# FIXME: This is almost definitely a bug.
|
| 731 |
+
if isinstance(
|
| 732 |
+
tensor,
|
| 733 |
+
(
|
| 734 |
+
torch._subclasses.fake_tensor.FakeTensor,
|
| 735 |
+
torch._subclasses.functional_tensor.FunctionalTensor,
|
| 736 |
+
),
|
| 737 |
+
):
|
| 738 |
+
data_ptr = 0
|
| 739 |
+
else:
|
| 740 |
+
data_ptr = tensor.data_ptr()
|
| 741 |
+
return (id(tensor), data_ptr, tensor._version)
|
| 742 |
+
|
| 743 |
+
|
| 744 |
+
def _get_sid(tensor: torch.Tensor) -> _SID:
|
| 745 |
+
# FIXME: This is almost definitely a bug.
|
| 746 |
+
if isinstance(
|
| 747 |
+
tensor,
|
| 748 |
+
(
|
| 749 |
+
torch._subclasses.fake_tensor.FakeTensor,
|
| 750 |
+
torch._subclasses.functional_tensor.FunctionalTensor,
|
| 751 |
+
),
|
| 752 |
+
):
|
| 753 |
+
data_ptr = 0
|
| 754 |
+
else:
|
| 755 |
+
data_ptr = tensor.data_ptr()
|
| 756 |
+
return (data_ptr, tensor._version)
|
| 757 |
+
|
| 758 |
+
|
| 759 |
+
class _Handle:
|
| 760 |
+
pass
|
| 761 |
+
|
| 762 |
+
|
| 763 |
+
class _swap_with_cloned(saved_tensors_hooks):
|
| 764 |
+
def __init__(self, ctx: "_AllowMutationOnSavedContext") -> None:
|
| 765 |
+
def pack_hook(tensor: torch.Tensor) -> _Handle:
|
| 766 |
+
tid = _get_tid(tensor)
|
| 767 |
+
sid = _get_sid(tensor)
|
| 768 |
+
# Tensors saved for backward have an entry in _tid_to_weakhandle
|
| 769 |
+
handle: _Handle | None = None
|
| 770 |
+
|
| 771 |
+
# Save aliasing information
|
| 772 |
+
ctx.sid_to_tid[sid].add(tid)
|
| 773 |
+
|
| 774 |
+
# NB: The same tensor (of the same version) can be saved multiple times
|
| 775 |
+
if tid not in ctx.tid_to_weakhandle:
|
| 776 |
+
handle = _Handle()
|
| 777 |
+
ctx.tid_to_weakhandle[tid] = handle
|
| 778 |
+
ctx.original[handle] = tensor
|
| 779 |
+
else:
|
| 780 |
+
# Store an additional strong reference to the handle
|
| 781 |
+
handle = ctx.tid_to_weakhandle[tid]
|
| 782 |
+
return handle
|
| 783 |
+
|
| 784 |
+
def unpack_hook(handle: _Handle) -> torch.Tensor:
|
| 785 |
+
error_msg = (
|
| 786 |
+
"Trying to backward outside of the 'allow_mutation_on_saved_tensors' context"
|
| 787 |
+
"in which the graph was originally recorded."
|
| 788 |
+
)
|
| 789 |
+
if not _allow_mutation_on_saved_tensors_enabled:
|
| 790 |
+
raise AssertionError(error_msg)
|
| 791 |
+
if handle in ctx.cloned:
|
| 792 |
+
res = ctx.cloned[handle]
|
| 793 |
+
else:
|
| 794 |
+
if handle not in ctx.original:
|
| 795 |
+
raise AssertionError(error_msg)
|
| 796 |
+
res = ctx.original[handle]
|
| 797 |
+
return res
|
| 798 |
+
|
| 799 |
+
super().__init__(pack_hook, unpack_hook)
|
| 800 |
+
|
| 801 |
+
|
| 802 |
+
class _CloneArgBeforeMutateMode(TorchDispatchMode):
|
| 803 |
+
def __init__(self, ctx: "_AllowMutationOnSavedContext") -> None:
|
| 804 |
+
self.ctx = ctx
|
| 805 |
+
|
| 806 |
+
def __torch_dispatch__(
|
| 807 |
+
self,
|
| 808 |
+
func: "OpOverload",
|
| 809 |
+
types: Iterable[type],
|
| 810 |
+
args: tuple[Any, ...] = (),
|
| 811 |
+
kwargs: dict[Any, Any] | None = None,
|
| 812 |
+
) -> Any:
|
| 813 |
+
kwargs = kwargs or {}
|
| 814 |
+
|
| 815 |
+
def maybe_clone(t: torch.Tensor) -> None:
|
| 816 |
+
tid = _get_tid(t)
|
| 817 |
+
sid = _get_sid(t)
|
| 818 |
+
ctx = self.ctx
|
| 819 |
+
if sid in ctx.sid_to_tid:
|
| 820 |
+
for tid in ctx.sid_to_tid[sid]:
|
| 821 |
+
if tid not in ctx.tid_to_weakhandle:
|
| 822 |
+
# We know that if tid is in sid_to_tid, then it must also be in
|
| 823 |
+
# tid_to_weakhandle. However, it is possible for the tensor to be
|
| 824 |
+
# saved at one point, but cleared by backward before it is modified
|
| 825 |
+
# in-place. Consider the following example:
|
| 826 |
+
#
|
| 827 |
+
# >>> a = torch.randn(2, 3, requires_grad=True).clone()
|
| 828 |
+
# >>> out = (a**2).sum()
|
| 829 |
+
# >>> out.backward()
|
| 830 |
+
# >>> a.sin_()
|
| 831 |
+
continue
|
| 832 |
+
handle = ctx.tid_to_weakhandle[tid]
|
| 833 |
+
if handle in ctx.cloned:
|
| 834 |
+
# The same exact tensor has been cloned already
|
| 835 |
+
continue
|
| 836 |
+
ctx.cloned[handle] = ctx.original[handle].clone()
|
| 837 |
+
del ctx.original[handle]
|
| 838 |
+
|
| 839 |
+
for idx, arg in enumerate(func._schema.arguments):
|
| 840 |
+
if arg.alias_info is not None and arg.alias_info.is_write:
|
| 841 |
+
if arg.is_out:
|
| 842 |
+
maybe_clone(kwargs["out"])
|
| 843 |
+
elif isinstance(args[idx], list):
|
| 844 |
+
# Foreach case. (Possible optimization: if most of the
|
| 845 |
+
# tensors need to be cloned, use a for each clone?)
|
| 846 |
+
for t in args[idx]:
|
| 847 |
+
maybe_clone(t)
|
| 848 |
+
else:
|
| 849 |
+
maybe_clone(args[idx])
|
| 850 |
+
|
| 851 |
+
return func(*args, **kwargs)
|
| 852 |
+
|
| 853 |
+
|
| 854 |
+
class _AllowMutationOnSavedContext:
|
| 855 |
+
def __init__(self) -> None:
|
| 856 |
+
self.cloned: MutableMapping[_Handle, torch.Tensor] = WeakKeyDictionary()
|
| 857 |
+
self.original: MutableMapping[_Handle, torch.Tensor] = WeakKeyDictionary()
|
| 858 |
+
self.tid_to_weakhandle: MutableMapping[_TID, _Handle] = WeakValueDictionary()
|
| 859 |
+
self.sid_to_tid: dict[_SID, set[_TID]] = defaultdict(set)
|
| 860 |
+
|
| 861 |
+
def clear(self) -> None:
|
| 862 |
+
self.cloned.clear()
|
| 863 |
+
self.original.clear()
|
| 864 |
+
self.tid_to_weakhandle.clear()
|
| 865 |
+
self.sid_to_tid.clear()
|
| 866 |
+
|
| 867 |
+
|
| 868 |
+
@contextlib.contextmanager
|
| 869 |
+
def allow_mutation_on_saved_tensors() -> Generator[
|
| 870 |
+
_AllowMutationOnSavedContext, None, None
|
| 871 |
+
]:
|
| 872 |
+
"""Context manager under which mutating tensors saved for backward is allowed.
|
| 873 |
+
|
| 874 |
+
Under this context manager, tensors saved for backward are cloned on mutation,
|
| 875 |
+
so the original version can still be used during backward. Normally, mutating a tensor
|
| 876 |
+
saved for backward will result in an error raised when it's used during backward.
|
| 877 |
+
|
| 878 |
+
To ensure the correct behavior, both the forward and backward should be run under
|
| 879 |
+
the same context manager.
|
| 880 |
+
|
| 881 |
+
Returns:
|
| 882 |
+
An _AllowMutationOnSavedContext object storing the state managed by this
|
| 883 |
+
context manager. This object can be useful for debugging purposes. The state
|
| 884 |
+
managed by the context manager is automatically cleared upon exiting.
|
| 885 |
+
|
| 886 |
+
Example::
|
| 887 |
+
|
| 888 |
+
>>> import torch
|
| 889 |
+
>>> with torch.autograd.graph.allow_mutation_on_saved_tensors():
|
| 890 |
+
... # forward
|
| 891 |
+
... a = torch.ones(2, 3, requires_grad=True)
|
| 892 |
+
... b = a.clone()
|
| 893 |
+
... out = (b**2).sum()
|
| 894 |
+
... b.sin_()
|
| 895 |
+
... # backward
|
| 896 |
+
... out.sum().backward()
|
| 897 |
+
...
|
| 898 |
+
tensor([[0.8415, 0.8415, 0.8415],
|
| 899 |
+
[0.8415, 0.8415, 0.8415]], grad_fn=<SinBackward0>)
|
| 900 |
+
"""
|
| 901 |
+
global _allow_mutation_on_saved_tensors_enabled
|
| 902 |
+
|
| 903 |
+
ctx = _AllowMutationOnSavedContext()
|
| 904 |
+
|
| 905 |
+
with _swap_with_cloned(ctx), _CloneArgBeforeMutateMode(ctx):
|
| 906 |
+
try:
|
| 907 |
+
if _allow_mutation_on_saved_tensors_enabled:
|
| 908 |
+
raise RuntimeError(
|
| 909 |
+
"allow_mutation_on_saved_tensors contexts cannot be nested"
|
| 910 |
+
)
|
| 911 |
+
_allow_mutation_on_saved_tensors_enabled = True
|
| 912 |
+
yield ctx
|
| 913 |
+
finally:
|
| 914 |
+
ctx.clear()
|
| 915 |
+
_allow_mutation_on_saved_tensors_enabled = False
|
| 916 |
+
|
| 917 |
+
|
| 918 |
+
def _register_logging_hooks_on_whole_graph(
|
| 919 |
+
t_outputs: Sequence[torch.Tensor | GradientEdge],
|
| 920 |
+
) -> Callable[[], None]:
|
| 921 |
+
grad_fns = list(map(_get_grad_fn_or_grad_acc, t_outputs))
|
| 922 |
+
|
| 923 |
+
def iter_graph(roots: list[Node]) -> Iterator[Node]:
|
| 924 |
+
if not roots:
|
| 925 |
+
return
|
| 926 |
+
seen: set[Node] = set()
|
| 927 |
+
q: deque[Node] = deque()
|
| 928 |
+
for node in roots:
|
| 929 |
+
if node is not None:
|
| 930 |
+
seen.add(node)
|
| 931 |
+
q.append(node)
|
| 932 |
+
|
| 933 |
+
while q:
|
| 934 |
+
node = q.popleft()
|
| 935 |
+
for fn, _ in node.next_functions:
|
| 936 |
+
if fn in seen or fn is None:
|
| 937 |
+
continue
|
| 938 |
+
seen.add(fn)
|
| 939 |
+
q.append(fn)
|
| 940 |
+
|
| 941 |
+
yield node
|
| 942 |
+
|
| 943 |
+
def fmt(t: torch.Tensor | None) -> str:
|
| 944 |
+
# Avoid circular import
|
| 945 |
+
from torch.utils._dtype_abbrs import dtype_abbrs
|
| 946 |
+
|
| 947 |
+
if t is None:
|
| 948 |
+
return "None"
|
| 949 |
+
return f"{dtype_abbrs[t.dtype]}[{', '.join(map(str, t.shape))}]"
|
| 950 |
+
|
| 951 |
+
def prehook(grad_outputs: Sequence[torch.Tensor | None]) -> None:
|
| 952 |
+
node = torch._C._current_autograd_node()
|
| 953 |
+
grad_outputs_str = f"[{','.join(fmt(t) for t in grad_outputs)}]"
|
| 954 |
+
log_str = f"Executing: {node} with grad_outputs: {grad_outputs_str}"
|
| 955 |
+
log.debug(log_str)
|
| 956 |
+
|
| 957 |
+
handles = [node.register_prehook(prehook) for node in iter_graph(grad_fns)]
|
| 958 |
+
|
| 959 |
+
def unregister_hooks() -> None:
|
| 960 |
+
for handle in handles:
|
| 961 |
+
handle.remove()
|
| 962 |
+
|
| 963 |
+
return unregister_hooks
|
| 964 |
+
|
| 965 |
+
|
| 966 |
+
def _engine_run_backward(
|
| 967 |
+
t_outputs: Sequence[torch.Tensor | GradientEdge],
|
| 968 |
+
*args: Any,
|
| 969 |
+
**kwargs: Any,
|
| 970 |
+
) -> tuple[torch.Tensor, ...]:
|
| 971 |
+
attach_logging_hooks = log.getEffectiveLevel() <= logging.DEBUG
|
| 972 |
+
if attach_logging_hooks:
|
| 973 |
+
unregister_hooks = _register_logging_hooks_on_whole_graph(t_outputs)
|
| 974 |
+
|
| 975 |
+
# Need to save the context so compiler config will be visible in device threads
|
| 976 |
+
torch._C._stash_obj_in_tls("context", contextvars.copy_context())
|
| 977 |
+
|
| 978 |
+
try:
|
| 979 |
+
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 980 |
+
t_outputs, *args, **kwargs
|
| 981 |
+
) # Calls into the C++ engine to run the backward pass
|
| 982 |
+
finally:
|
| 983 |
+
if attach_logging_hooks:
|
| 984 |
+
unregister_hooks() # type: ignore[possibly-undefined]
|
| 985 |
+
# Erase rather than overwrite-with-None so the thread_local map is
|
| 986 |
+
# truly empty. SafePyObject's destructor needs the GIL; if a thread
|
| 987 |
+
# exits while a SafePyObject is still in its thread_local,
|
| 988 |
+
# __call_tls_dtors fires the destructor → take_gil → deadlock.
|
| 989 |
+
torch._C._remove_obj_from_tls("context")
|
western_ai/usa/pytorch/torch/autograd/profiler.py
ADDED
|
@@ -0,0 +1,1799 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import ast
|
| 3 |
+
import copy
|
| 4 |
+
import json
|
| 5 |
+
import logging
|
| 6 |
+
import re
|
| 7 |
+
import uuid
|
| 8 |
+
from bisect import bisect_left
|
| 9 |
+
from collections import defaultdict
|
| 10 |
+
from collections.abc import Iterable
|
| 11 |
+
from dataclasses import dataclass
|
| 12 |
+
from time import perf_counter_ns
|
| 13 |
+
from typing import Any, Optional
|
| 14 |
+
from warnings import warn
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
log = logging.getLogger(__name__)
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
import torch.cuda
|
| 21 |
+
from torch._C import _get_privateuse1_backend_name
|
| 22 |
+
from torch._C._profiler import _ExperimentalConfig
|
| 23 |
+
from torch.autograd import (
|
| 24 |
+
_disable_profiler,
|
| 25 |
+
_enable_profiler,
|
| 26 |
+
_kineto_step,
|
| 27 |
+
_prepare_profiler,
|
| 28 |
+
_ProfilerResult,
|
| 29 |
+
_supported_activities,
|
| 30 |
+
_toggle_collection_dynamic,
|
| 31 |
+
DeviceType,
|
| 32 |
+
kineto_available,
|
| 33 |
+
ProfilerActivity,
|
| 34 |
+
ProfilerConfig,
|
| 35 |
+
ProfilerState,
|
| 36 |
+
)
|
| 37 |
+
from torch.autograd.profiler_util import (
|
| 38 |
+
_filter_name,
|
| 39 |
+
_filter_stack_entry,
|
| 40 |
+
_rewrite_name,
|
| 41 |
+
EventList,
|
| 42 |
+
FunctionEvent,
|
| 43 |
+
MEMORY_EVENT_NAME,
|
| 44 |
+
MemRecordsAcc,
|
| 45 |
+
OUT_OF_MEMORY_EVENT_NAME,
|
| 46 |
+
)
|
| 47 |
+
from torch.futures import Future
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
__all__ = [
|
| 51 |
+
"profile",
|
| 52 |
+
"record_function",
|
| 53 |
+
"emit_itt",
|
| 54 |
+
"emit_nvtx",
|
| 55 |
+
"load_nvprof",
|
| 56 |
+
"EnforceUnique",
|
| 57 |
+
"parse_nvprof_trace",
|
| 58 |
+
"KinetoStepTracker",
|
| 59 |
+
"EventList",
|
| 60 |
+
"FunctionEvent",
|
| 61 |
+
"MemRecordsAcc",
|
| 62 |
+
]
|
| 63 |
+
|
| 64 |
+
try:
|
| 65 |
+
# Available in Python >= 3.2
|
| 66 |
+
from contextlib import ContextDecorator as _ContextDecorator
|
| 67 |
+
except ImportError:
|
| 68 |
+
import functools
|
| 69 |
+
|
| 70 |
+
class _ContextDecorator: # type: ignore[no-redef]
|
| 71 |
+
def __enter__(self):
|
| 72 |
+
raise NotImplementedError
|
| 73 |
+
|
| 74 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 75 |
+
raise NotImplementedError
|
| 76 |
+
|
| 77 |
+
def __call__(self, func):
|
| 78 |
+
@functools.wraps(func)
|
| 79 |
+
def wrapped(*args, **kwargs):
|
| 80 |
+
with self:
|
| 81 |
+
return func(*args, **kwargs)
|
| 82 |
+
|
| 83 |
+
return wrapped
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
# global python state - whether profiler is currently enabled
|
| 87 |
+
# useful for fast python checks to reduce latency
|
| 88 |
+
_is_profiler_enabled: bool = False
|
| 89 |
+
|
| 90 |
+
# https://github.com/pytorch/kineto/blob/a054a4be0db117c579a21747debf19c863631f26/libkineto/src/output_json.cpp#L559
|
| 91 |
+
# Kernel and CUDA runtime API events are connected by ac2g; forward and backward
|
| 92 |
+
# ATen ops are connected by fwdbwd.
|
| 93 |
+
_profile_flow_types = {"fwdbwd", "ac2g"}
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _set_is_profiler_enabled(enable: bool):
|
| 97 |
+
global _is_profiler_enabled
|
| 98 |
+
_is_profiler_enabled = enable
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _run_on_profiler_start():
|
| 102 |
+
_set_is_profiler_enabled(True)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _run_on_profiler_stop():
|
| 106 |
+
_set_is_profiler_enabled(False)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
@dataclass
|
| 110 |
+
class _ProfilerStats:
|
| 111 |
+
"Profiler timing and stats used by developers to catch issues/regressions"
|
| 112 |
+
|
| 113 |
+
profiling_window_duration_sec: float = 0
|
| 114 |
+
number_of_events: int = 0
|
| 115 |
+
profiler_prepare_call_duration_us: int = 0
|
| 116 |
+
profiler_enable_call_duration_us: int = 0
|
| 117 |
+
profiler_disable_call_duration_us: int = 0
|
| 118 |
+
parse_kineto_call_duration_us: int = 0
|
| 119 |
+
function_events_build_tree_call_duration_us: int = 0
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
class _EventItem:
|
| 123 |
+
def __init__(self, event):
|
| 124 |
+
self.e = event
|
| 125 |
+
self.start, self.end = self._interval_of()
|
| 126 |
+
|
| 127 |
+
def _interval_of(self):
|
| 128 |
+
start = self.e["ts"]
|
| 129 |
+
duration = self.e.get("dur", 0)
|
| 130 |
+
return start, start + duration
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _find_events_covered_in(
|
| 134 |
+
events: list[dict[str, Any]],
|
| 135 |
+
top_level_events: list[dict[str, Any]],
|
| 136 |
+
):
|
| 137 |
+
"""Find events covered in top level events.
|
| 138 |
+
|
| 139 |
+
Returns a dict mapping top-level event uid to covered event uids.
|
| 140 |
+
"""
|
| 141 |
+
top_level_by_tid = defaultdict(list)
|
| 142 |
+
for event in top_level_events:
|
| 143 |
+
top_level_by_tid[event["tid"]].append(_EventItem(event))
|
| 144 |
+
for tid in top_level_by_tid:
|
| 145 |
+
top_level_by_tid[tid] = sorted(top_level_by_tid[tid], key=lambda x: x.start)
|
| 146 |
+
starts_by_tid = {
|
| 147 |
+
tid: [event.start for event in event_items]
|
| 148 |
+
for tid, event_items in top_level_by_tid.items()
|
| 149 |
+
}
|
| 150 |
+
top_level_id_to_events = defaultdict(set)
|
| 151 |
+
|
| 152 |
+
for event in events:
|
| 153 |
+
if event.get("cat") not in {
|
| 154 |
+
"cuda_runtime",
|
| 155 |
+
"cuLaunchKernel",
|
| 156 |
+
"cpu_op",
|
| 157 |
+
"cuda_driver",
|
| 158 |
+
"gpu_memset",
|
| 159 |
+
"python_function",
|
| 160 |
+
}:
|
| 161 |
+
continue
|
| 162 |
+
if "CallFrom" in event.get("args", {}):
|
| 163 |
+
continue
|
| 164 |
+
tid = event.get("tid")
|
| 165 |
+
if tid not in top_level_by_tid:
|
| 166 |
+
continue
|
| 167 |
+
item = _EventItem(event)
|
| 168 |
+
event_items = top_level_by_tid[tid]
|
| 169 |
+
idx = bisect_left(starts_by_tid[tid], item.start) - 1
|
| 170 |
+
if idx < 0 or idx >= len(event_items):
|
| 171 |
+
continue
|
| 172 |
+
top_level_item = event_items[idx]
|
| 173 |
+
if top_level_item.start <= item.start and item.end <= top_level_item.end:
|
| 174 |
+
top_level_id_to_events[top_level_item.e["uid"]].add(event["uid"])
|
| 175 |
+
return top_level_id_to_events
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
class profile:
|
| 179 |
+
"""Context manager that manages autograd profiler state and holds a summary of results.
|
| 180 |
+
|
| 181 |
+
.. note::
|
| 182 |
+
This is the backend, most people should use :mod:`torch.profiler` instead.
|
| 183 |
+
|
| 184 |
+
Under the hood it just records events of functions being executed in C++ and
|
| 185 |
+
exposes those events to Python. You can wrap any code into it and it will
|
| 186 |
+
only report runtime of PyTorch functions.
|
| 187 |
+
Note: profiler is thread local and is automatically propagated into the async tasks
|
| 188 |
+
|
| 189 |
+
Args:
|
| 190 |
+
enabled (bool, optional): Setting this to False makes this context manager a no-op.
|
| 191 |
+
|
| 192 |
+
use_cuda (bool, optional): Enables timing of CUDA events as well
|
| 193 |
+
using the cudaEvent API. (will be deprecated)
|
| 194 |
+
|
| 195 |
+
use_device (str, optional): Enables timing of device events.
|
| 196 |
+
Adds approximately 4us of overhead to each tensor operation when use cuda.
|
| 197 |
+
The valid devices options are 'cuda', 'xpu', 'mtia' and 'privateuseone'.
|
| 198 |
+
|
| 199 |
+
record_shapes (bool, optional): If shapes recording is set, information
|
| 200 |
+
about input dimensions will be collected. This allows one to see which
|
| 201 |
+
dimensions have been used under the hood and further group by them
|
| 202 |
+
using prof.key_averages(group_by_input_shape=True). Please note that
|
| 203 |
+
shape recording might skew your profiling data. It is recommended to
|
| 204 |
+
use separate runs with and without shape recording to validate the timing.
|
| 205 |
+
Most likely the skew will be negligible for bottom most events (in a case
|
| 206 |
+
of nested function calls). But for higher level functions the total
|
| 207 |
+
self cpu time might be artificially increased because of the shape
|
| 208 |
+
collection.
|
| 209 |
+
|
| 210 |
+
with_flops (bool, optional): If with_flops is set, the profiler will estimate
|
| 211 |
+
the FLOPs (floating point operations) value using the operator's input shape.
|
| 212 |
+
This allows one to estimate the hardware performance. Currently,
|
| 213 |
+
this option only works for the matrix multiplication and 2D convolution operators.
|
| 214 |
+
|
| 215 |
+
profile_memory (bool, optional): track tensor memory allocation/deallocation.
|
| 216 |
+
|
| 217 |
+
with_stack (bool, optional): record source information (file and line number) for the ops.
|
| 218 |
+
|
| 219 |
+
with_modules (bool): record module hierarchy (including function names)
|
| 220 |
+
corresponding to the callstack of the op. e.g. If module A's forward call's
|
| 221 |
+
module B's forward which contains an aten::add op,
|
| 222 |
+
then aten::add's module hierarchy is A.B
|
| 223 |
+
Note that this support exist, at the moment, only for TorchScript models
|
| 224 |
+
and not eager mode models.
|
| 225 |
+
|
| 226 |
+
use_kineto (bool, optional): experimental, enable profiling with Kineto profiler.
|
| 227 |
+
|
| 228 |
+
use_cpu (bool, optional): profile CPU events; setting to ``False`` requires
|
| 229 |
+
``use_kineto=True`` and can be used to lower the overhead for GPU-only profiling.
|
| 230 |
+
|
| 231 |
+
experimental_config (_ExperimentalConfig) : A set of experimental options
|
| 232 |
+
used by profiler libraries like Kineto. Note, backward compatibility is not guaranteed.
|
| 233 |
+
|
| 234 |
+
acc_events (bool): Enable the accumulation of FunctionEvents across multiple profiling cycles
|
| 235 |
+
|
| 236 |
+
post_processing_timeout_s (float): Optional timeout in seconds for post-processing profiler
|
| 237 |
+
results. In this context, post-processing happens after the profiling itself has finished.
|
| 238 |
+
If specified, event parsing will stop after this duration and return partial results. Useful
|
| 239 |
+
for handling large traces that may take too long to process.
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
.. warning::
|
| 243 |
+
Enabling memory profiling or source attribution incurs additional profiler
|
| 244 |
+
overhead
|
| 245 |
+
|
| 246 |
+
.. warning::
|
| 247 |
+
This context managers should not be called recursively, i.e. no nested
|
| 248 |
+
instances are allowed
|
| 249 |
+
|
| 250 |
+
.. warning::
|
| 251 |
+
Due to some CUDA multiprocessing limitations (see :ref:`multiprocessing-cuda-note`),
|
| 252 |
+
one cannot use the profiler with ``use_device = 'cuda'`` to benchmark
|
| 253 |
+
DataLoaders with ``num_workers > 0``. If you wish to benchmark data loading,
|
| 254 |
+
please use ``use_device = None`` or ``num_workers = 0``.
|
| 255 |
+
|
| 256 |
+
Example:
|
| 257 |
+
>>> # xdoctest: +SKIP
|
| 258 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER)
|
| 259 |
+
>>> x = torch.randn((1, 1), requires_grad=True)
|
| 260 |
+
>>> with torch.autograd.profiler.profile() as prof:
|
| 261 |
+
>>> for _ in range(100): # any normal python code, really!
|
| 262 |
+
>>> y = x ** 2
|
| 263 |
+
>>> y.backward()
|
| 264 |
+
>>> # NOTE: some columns were removed for brevity
|
| 265 |
+
>>> print(prof.key_averages().table(sort_by="self_cpu_time_total"))
|
| 266 |
+
----------------------------------- --------------- --------------- ---------------
|
| 267 |
+
Name Self CPU total CPU time avg Number of Calls
|
| 268 |
+
----------------------------------- --------------- --------------- ---------------
|
| 269 |
+
mul 32.048ms 32.048ms 200
|
| 270 |
+
pow 27.041ms 27.041ms 200
|
| 271 |
+
PowBackward0 9.727ms 55.483ms 100
|
| 272 |
+
torch::autograd::AccumulateGrad 9.148ms 9.148ms 100
|
| 273 |
+
torch::autograd::GraphRoot 691.816us 691.816us 100
|
| 274 |
+
----------------------------------- --------------- --------------- ---------------
|
| 275 |
+
|
| 276 |
+
"""
|
| 277 |
+
|
| 278 |
+
def __init__(
|
| 279 |
+
self,
|
| 280 |
+
enabled=True,
|
| 281 |
+
*,
|
| 282 |
+
use_cuda=False, # Deprecated
|
| 283 |
+
use_device=None,
|
| 284 |
+
record_shapes=False,
|
| 285 |
+
with_flops=False,
|
| 286 |
+
profile_memory=False,
|
| 287 |
+
with_stack=False,
|
| 288 |
+
with_modules=False,
|
| 289 |
+
use_kineto=False,
|
| 290 |
+
use_cpu=True,
|
| 291 |
+
experimental_config=None,
|
| 292 |
+
acc_events=False,
|
| 293 |
+
custom_trace_id_callback=None,
|
| 294 |
+
post_processing_timeout_s: float | None = None,
|
| 295 |
+
activity_filters: dict[ProfilerActivity, set[str]] | None = None,
|
| 296 |
+
):
|
| 297 |
+
self.enabled: bool = enabled
|
| 298 |
+
if not self.enabled:
|
| 299 |
+
return
|
| 300 |
+
self.use_cuda = use_cuda
|
| 301 |
+
if self.use_cuda:
|
| 302 |
+
warn(
|
| 303 |
+
"The attribute `use_cuda` will be deprecated soon, "
|
| 304 |
+
"please use ``use_device = 'cuda'`` instead.",
|
| 305 |
+
FutureWarning,
|
| 306 |
+
stacklevel=2,
|
| 307 |
+
)
|
| 308 |
+
self.use_device: str | None = "cuda"
|
| 309 |
+
else:
|
| 310 |
+
self.use_device = use_device
|
| 311 |
+
# TODO Consider changing _function_events into data structure with size cap
|
| 312 |
+
self._function_events: EventList | None = None
|
| 313 |
+
self._old_function_events: EventList | None = None
|
| 314 |
+
# Function event processing is done lazily
|
| 315 |
+
self._needs_processing = False
|
| 316 |
+
self.entered = False
|
| 317 |
+
self.record_shapes = record_shapes
|
| 318 |
+
self.with_flops = with_flops
|
| 319 |
+
self.record_shapes |= self.with_flops
|
| 320 |
+
self.profile_memory = profile_memory
|
| 321 |
+
self.with_stack = with_stack
|
| 322 |
+
self.with_modules = with_modules
|
| 323 |
+
self.use_cpu = use_cpu
|
| 324 |
+
self.acc_events = acc_events
|
| 325 |
+
if experimental_config is None:
|
| 326 |
+
experimental_config = _ExperimentalConfig()
|
| 327 |
+
if experimental_config.trace_only and with_stack:
|
| 328 |
+
warn(
|
| 329 |
+
"trace_only=True is incompatible with with_stack=True "
|
| 330 |
+
"(stack traces require event post-processing). "
|
| 331 |
+
"Disabling trace_only."
|
| 332 |
+
)
|
| 333 |
+
experimental_config = copy.copy(experimental_config)
|
| 334 |
+
experimental_config.trace_only = False
|
| 335 |
+
if (
|
| 336 |
+
experimental_config.profiler_metrics
|
| 337 |
+
or experimental_config.profiler_measure_per_kernel
|
| 338 |
+
):
|
| 339 |
+
warn(
|
| 340 |
+
"profiler_metrics and profiler_measure_per_kernel are deprecated "
|
| 341 |
+
"and ignored. These options will be removed in a future release.",
|
| 342 |
+
FutureWarning,
|
| 343 |
+
stacklevel=2,
|
| 344 |
+
)
|
| 345 |
+
self.experimental_config = experimental_config
|
| 346 |
+
self.kineto_results: _ProfilerResult | None = None
|
| 347 |
+
self.profiling_start_time_ns = 0
|
| 348 |
+
self.profiling_end_time_ns = 0
|
| 349 |
+
self._stats = _ProfilerStats()
|
| 350 |
+
self.custom_trace_id_callback = custom_trace_id_callback
|
| 351 |
+
self.post_processing_timeout_s = post_processing_timeout_s
|
| 352 |
+
self.activity_filters = activity_filters or {}
|
| 353 |
+
self.trace_id = ""
|
| 354 |
+
if not self.use_cpu:
|
| 355 |
+
if not use_kineto:
|
| 356 |
+
raise AssertionError(
|
| 357 |
+
"Device-only events supported only with Kineto (use_kineto=True)"
|
| 358 |
+
)
|
| 359 |
+
|
| 360 |
+
if self.use_device is not None:
|
| 361 |
+
VALID_DEVICE_OPTIONS = ["cuda", "xpu", "mtia", "hpu"]
|
| 362 |
+
if _get_privateuse1_backend_name() != "privateuseone":
|
| 363 |
+
VALID_DEVICE_OPTIONS.append(_get_privateuse1_backend_name())
|
| 364 |
+
if self.use_device not in VALID_DEVICE_OPTIONS:
|
| 365 |
+
warn(
|
| 366 |
+
f"The {self.use_device} is not a valid device option.", stacklevel=2
|
| 367 |
+
)
|
| 368 |
+
self.use_device = None
|
| 369 |
+
|
| 370 |
+
if self.use_device == "cuda" and not torch.cuda.is_available():
|
| 371 |
+
warn("CUDA is not available, disabling CUDA profiling", stacklevel=2)
|
| 372 |
+
self.use_cuda = False
|
| 373 |
+
self.use_device = None
|
| 374 |
+
|
| 375 |
+
if self.use_device == "xpu" and not torch.xpu.is_available():
|
| 376 |
+
warn("XPU is not available, disabling XPU profiling", stacklevel=2)
|
| 377 |
+
self.use_device = None
|
| 378 |
+
|
| 379 |
+
if self.use_device == "hpu" and not (
|
| 380 |
+
hasattr(torch, "hpu") and torch.hpu.is_available()
|
| 381 |
+
):
|
| 382 |
+
warn("HPU is not available, disabling HPU profiling", stacklevel=2)
|
| 383 |
+
self.use_device = None
|
| 384 |
+
|
| 385 |
+
self.kineto_activities = set()
|
| 386 |
+
if self.use_cpu:
|
| 387 |
+
self.kineto_activities.add(ProfilerActivity.CPU)
|
| 388 |
+
|
| 389 |
+
self.profiler_kind = ProfilerState.KINETO
|
| 390 |
+
if self.use_device == "cuda":
|
| 391 |
+
if not use_kineto or ProfilerActivity.CUDA not in _supported_activities():
|
| 392 |
+
if not self.use_cpu:
|
| 393 |
+
raise AssertionError("Legacy CUDA profiling requires use_cpu=True")
|
| 394 |
+
self.profiler_kind = ProfilerState.KINETO_GPU_FALLBACK
|
| 395 |
+
else:
|
| 396 |
+
self.kineto_activities.add(ProfilerActivity.CUDA)
|
| 397 |
+
elif self.use_device == "xpu":
|
| 398 |
+
if not (use_kineto and ProfilerActivity.XPU in _supported_activities()):
|
| 399 |
+
raise AssertionError(
|
| 400 |
+
"Legacy XPU profiling is not supported. Requires use_kineto=True on XPU devices."
|
| 401 |
+
)
|
| 402 |
+
self.kineto_activities.add(ProfilerActivity.XPU)
|
| 403 |
+
elif self.use_device == "mtia":
|
| 404 |
+
if not (use_kineto and ProfilerActivity.MTIA in _supported_activities()):
|
| 405 |
+
raise AssertionError(
|
| 406 |
+
"Legacy MTIA profiling is not supported. Requires use_kineto=True on MTIA devices."
|
| 407 |
+
)
|
| 408 |
+
self.kineto_activities.add(ProfilerActivity.MTIA)
|
| 409 |
+
elif self.use_device == "hpu":
|
| 410 |
+
if not (use_kineto and ProfilerActivity.HPU in _supported_activities()):
|
| 411 |
+
raise AssertionError(
|
| 412 |
+
"Legacy HPU profiling is not supported. Requires use_kineto=True on HPU devices."
|
| 413 |
+
)
|
| 414 |
+
self.kineto_activities.add(ProfilerActivity.HPU)
|
| 415 |
+
elif self.use_device is not None and self.use_device != "privateuseone":
|
| 416 |
+
if use_kineto:
|
| 417 |
+
# Native tracing mode: use KINETO_PRIVATEUSE1 with registered IActivityProfiler
|
| 418 |
+
self.profiler_kind = ProfilerState.KINETO_PRIVATEUSE1
|
| 419 |
+
if ProfilerActivity.PrivateUse1 in _supported_activities():
|
| 420 |
+
self.kineto_activities.add(ProfilerActivity.PrivateUse1)
|
| 421 |
+
else:
|
| 422 |
+
# Marker-only mode: use fallback state
|
| 423 |
+
if not self.use_cpu:
|
| 424 |
+
raise AssertionError(
|
| 425 |
+
"Legacy privateuse1 profiling requires use_cpu=True"
|
| 426 |
+
)
|
| 427 |
+
self.profiler_kind = ProfilerState.KINETO_PRIVATEUSE1_FALLBACK
|
| 428 |
+
|
| 429 |
+
if len(self.kineto_activities) == 0:
|
| 430 |
+
raise AssertionError("No activities specified for the profiler")
|
| 431 |
+
|
| 432 |
+
if (
|
| 433 |
+
self.post_processing_timeout_s is not None
|
| 434 |
+
and self.post_processing_timeout_s < 0
|
| 435 |
+
):
|
| 436 |
+
raise ValueError("post_processing_timeout_s must be non-negative")
|
| 437 |
+
|
| 438 |
+
def default_trace_id(self):
|
| 439 |
+
# Generate a UUID
|
| 440 |
+
uuid_raw = uuid.uuid4()
|
| 441 |
+
|
| 442 |
+
return f"{uuid_raw.int:032X}"
|
| 443 |
+
|
| 444 |
+
def create_trace_id(self):
|
| 445 |
+
if self.custom_trace_id_callback:
|
| 446 |
+
return self.custom_trace_id_callback()
|
| 447 |
+
return self.default_trace_id()
|
| 448 |
+
|
| 449 |
+
def config(self, create_trace_id=False):
|
| 450 |
+
# only need to generate new trace id upon prepare trace not start trace
|
| 451 |
+
if create_trace_id:
|
| 452 |
+
trace_id = self.create_trace_id()
|
| 453 |
+
self.trace_id = trace_id
|
| 454 |
+
return ProfilerConfig(
|
| 455 |
+
self.profiler_kind,
|
| 456 |
+
self.record_shapes,
|
| 457 |
+
self.profile_memory,
|
| 458 |
+
self.with_stack,
|
| 459 |
+
self.with_flops,
|
| 460 |
+
self.with_modules,
|
| 461 |
+
self.experimental_config,
|
| 462 |
+
self.trace_id,
|
| 463 |
+
)
|
| 464 |
+
|
| 465 |
+
def __enter__(self):
|
| 466 |
+
if not self.enabled:
|
| 467 |
+
return
|
| 468 |
+
if self.entered:
|
| 469 |
+
raise RuntimeError("Profiler context manager is not reentrant")
|
| 470 |
+
self._prepare_trace()
|
| 471 |
+
self._start_trace()
|
| 472 |
+
return self
|
| 473 |
+
|
| 474 |
+
def _prepare_trace(self):
|
| 475 |
+
self.entered = True
|
| 476 |
+
t0 = perf_counter_ns()
|
| 477 |
+
_prepare_profiler(
|
| 478 |
+
self.config(create_trace_id=True),
|
| 479 |
+
self.kineto_activities,
|
| 480 |
+
activity_filter=self.activity_filters,
|
| 481 |
+
)
|
| 482 |
+
t1 = perf_counter_ns()
|
| 483 |
+
self._stats.profiler_prepare_call_duration_us = int((t1 - t0) / 1000)
|
| 484 |
+
|
| 485 |
+
def _start_trace(self):
|
| 486 |
+
self.entered = True
|
| 487 |
+
_run_on_profiler_start()
|
| 488 |
+
t0 = perf_counter_ns()
|
| 489 |
+
_enable_profiler(self.config(create_trace_id=False), self.kineto_activities)
|
| 490 |
+
t1 = perf_counter_ns()
|
| 491 |
+
self._stats.profiler_enable_call_duration_us = int((t1 - t0) / 1000)
|
| 492 |
+
self.profiling_start_time_ns = t1
|
| 493 |
+
|
| 494 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 495 |
+
if not self.enabled:
|
| 496 |
+
return
|
| 497 |
+
if self.use_device and hasattr(torch, self.use_device):
|
| 498 |
+
device_module = getattr(torch, self.use_device)
|
| 499 |
+
if hasattr(device_module, "synchronize"):
|
| 500 |
+
device_module.synchronize()
|
| 501 |
+
|
| 502 |
+
if self._function_events and self.acc_events:
|
| 503 |
+
self._old_function_events = self._function_events
|
| 504 |
+
self._function_events = None
|
| 505 |
+
self._needs_processing = True
|
| 506 |
+
|
| 507 |
+
t0 = perf_counter_ns()
|
| 508 |
+
|
| 509 |
+
self.kineto_results = _disable_profiler()
|
| 510 |
+
t1 = perf_counter_ns()
|
| 511 |
+
self._stats.profiler_disable_call_duration_us = int((t1 - t0) / 1000)
|
| 512 |
+
self.profiling_end_time_ns = t0
|
| 513 |
+
|
| 514 |
+
_run_on_profiler_stop()
|
| 515 |
+
|
| 516 |
+
self._stats.profiling_window_duration_sec = (
|
| 517 |
+
(self.profiling_end_time_ns - self.profiling_start_time_ns) * 1.0 / 1e9
|
| 518 |
+
)
|
| 519 |
+
|
| 520 |
+
# If we plan to accumulate events we should post process the function events
|
| 521 |
+
# right away to retain the state across multiple start/stop calls
|
| 522 |
+
if self.acc_events and not self.experimental_config.trace_only:
|
| 523 |
+
self._ensure_function_events()
|
| 524 |
+
return False
|
| 525 |
+
|
| 526 |
+
def __repr__(self):
|
| 527 |
+
if self._needs_processing:
|
| 528 |
+
self._ensure_function_events()
|
| 529 |
+
if self._function_events is None:
|
| 530 |
+
return "<unfinished torch.autograd.profile>"
|
| 531 |
+
return repr(self._function_events)
|
| 532 |
+
|
| 533 |
+
def __str__(self):
|
| 534 |
+
if self._needs_processing:
|
| 535 |
+
self._ensure_function_events()
|
| 536 |
+
if self._function_events is None:
|
| 537 |
+
return "<unfinished torch.autograd.profile>"
|
| 538 |
+
return str(self._function_events)
|
| 539 |
+
|
| 540 |
+
def _ensure_function_events(self):
|
| 541 |
+
"""Process function events lazily if required"""
|
| 542 |
+
if self.experimental_config.trace_only:
|
| 543 |
+
raise RuntimeError(
|
| 544 |
+
"events() is not available when trace_only=True in "
|
| 545 |
+
"ExperimentalConfig. Use export_chrome_trace() instead."
|
| 546 |
+
)
|
| 547 |
+
if self._function_events is not None:
|
| 548 |
+
return
|
| 549 |
+
self._needs_processing = False
|
| 550 |
+
|
| 551 |
+
t0 = perf_counter_ns()
|
| 552 |
+
parsed_results = []
|
| 553 |
+
if self.kineto_results:
|
| 554 |
+
parsed_results = self._parse_kineto_results(
|
| 555 |
+
self.kineto_results, timeout_s=self.post_processing_timeout_s
|
| 556 |
+
)
|
| 557 |
+
t1 = perf_counter_ns()
|
| 558 |
+
self._stats.parse_kineto_call_duration_us = int((t1 - t0) / 1000)
|
| 559 |
+
|
| 560 |
+
self._function_events = EventList(
|
| 561 |
+
parsed_results,
|
| 562 |
+
use_device=self.use_device,
|
| 563 |
+
profile_memory=self.profile_memory,
|
| 564 |
+
with_flops=self.with_flops,
|
| 565 |
+
)
|
| 566 |
+
t0 = perf_counter_ns()
|
| 567 |
+
self._function_events._build_tree()
|
| 568 |
+
t1 = perf_counter_ns()
|
| 569 |
+
self._stats.function_events_build_tree_call_duration_us = int((t1 - t0) / 1000)
|
| 570 |
+
self._stats.number_of_events = len(self._function_events)
|
| 571 |
+
|
| 572 |
+
if self._old_function_events and self.acc_events:
|
| 573 |
+
for evt in self._old_function_events:
|
| 574 |
+
self._function_events.append(evt)
|
| 575 |
+
self._old_function_events = None
|
| 576 |
+
|
| 577 |
+
@property
|
| 578 |
+
def function_events(self):
|
| 579 |
+
if self._function_events is None or self._needs_processing:
|
| 580 |
+
self._ensure_function_events()
|
| 581 |
+
return self._function_events
|
| 582 |
+
|
| 583 |
+
def table(
|
| 584 |
+
self,
|
| 585 |
+
sort_by=None,
|
| 586 |
+
row_limit=100,
|
| 587 |
+
max_src_column_width=75,
|
| 588 |
+
max_name_column_width=55,
|
| 589 |
+
max_shapes_column_width=80,
|
| 590 |
+
header=None,
|
| 591 |
+
top_level_events_only=False,
|
| 592 |
+
):
|
| 593 |
+
self._ensure_function_events()
|
| 594 |
+
if self._function_events is None:
|
| 595 |
+
raise AssertionError("Expected profiling results")
|
| 596 |
+
return self._function_events.table(
|
| 597 |
+
sort_by=sort_by,
|
| 598 |
+
row_limit=row_limit,
|
| 599 |
+
max_src_column_width=max_src_column_width,
|
| 600 |
+
max_name_column_width=max_name_column_width,
|
| 601 |
+
max_shapes_column_width=max_shapes_column_width,
|
| 602 |
+
header=header,
|
| 603 |
+
top_level_events_only=top_level_events_only,
|
| 604 |
+
)
|
| 605 |
+
|
| 606 |
+
table.__doc__ = EventList.table.__doc__
|
| 607 |
+
|
| 608 |
+
def _assign_uniq_id_to_event(self, trace):
|
| 609 |
+
if trace.get("uid_assigned"):
|
| 610 |
+
return {event["uid"]: event for event in trace["traceEvents"]}
|
| 611 |
+
uid_2_events = {}
|
| 612 |
+
for uid, event in enumerate(trace["traceEvents"]):
|
| 613 |
+
event["uid"] = uid
|
| 614 |
+
uid_2_events[uid] = event
|
| 615 |
+
trace["uid_assigned"] = True
|
| 616 |
+
return uid_2_events
|
| 617 |
+
|
| 618 |
+
def _build_flow_mapping(self, trace, flow_events):
|
| 619 |
+
"""Build src/dst event mappings from profiler flow events."""
|
| 620 |
+
if not flow_events:
|
| 621 |
+
return {}, {}
|
| 622 |
+
|
| 623 |
+
flow_pair: dict[int, list[int | None]] = {}
|
| 624 |
+
prev_event = None
|
| 625 |
+
for event in trace["traceEvents"]:
|
| 626 |
+
if (
|
| 627 |
+
event.get("name") not in _profile_flow_types
|
| 628 |
+
and event.get("cat") not in _profile_flow_types
|
| 629 |
+
):
|
| 630 |
+
prev_event = event
|
| 631 |
+
continue
|
| 632 |
+
if event.get("ph") == "s":
|
| 633 |
+
if prev_event is None:
|
| 634 |
+
continue
|
| 635 |
+
pair = flow_pair.setdefault(int(event["id"]), [None, None])
|
| 636 |
+
pair[0] = prev_event["uid"]
|
| 637 |
+
elif event.get("ph") == "f":
|
| 638 |
+
if prev_event is None:
|
| 639 |
+
continue
|
| 640 |
+
pair = flow_pair.setdefault(int(event["id"]), [None, None])
|
| 641 |
+
pair[1] = prev_event["uid"]
|
| 642 |
+
prev_event = event
|
| 643 |
+
|
| 644 |
+
src2dst = {}
|
| 645 |
+
dst2src = {}
|
| 646 |
+
for src, dst in flow_pair.values():
|
| 647 |
+
if src is None or dst is None:
|
| 648 |
+
continue
|
| 649 |
+
src2dst[src] = dst
|
| 650 |
+
dst2src[dst] = src
|
| 651 |
+
return src2dst, dst2src
|
| 652 |
+
|
| 653 |
+
def _maybe_triton_call(self, kernel_name):
|
| 654 |
+
return kernel_name.startswith("triton_")
|
| 655 |
+
|
| 656 |
+
def _has_kernel_info(self, compile_info, graph_key, kernel_name):
|
| 657 |
+
graph_info = compile_info.get(graph_key, {})
|
| 658 |
+
if kernel_name in graph_info:
|
| 659 |
+
return True
|
| 660 |
+
kernel_prefix = kernel_name + ":"
|
| 661 |
+
return any(name.startswith(kernel_prefix) for name in graph_info)
|
| 662 |
+
|
| 663 |
+
def _maybe_extern_call(self, event, compile_info, fwd_key, bw_keys, need_bw):
|
| 664 |
+
kernel_name = event["name"]
|
| 665 |
+
return not self._has_kernel_info(compile_info, fwd_key, kernel_name) and (
|
| 666 |
+
not need_bw
|
| 667 |
+
or all(
|
| 668 |
+
not self._has_kernel_info(compile_info, bw_key, kernel_name)
|
| 669 |
+
for bw_key in bw_keys
|
| 670 |
+
)
|
| 671 |
+
)
|
| 672 |
+
|
| 673 |
+
def add_to_chrome_trace(self, origin_trace):
|
| 674 |
+
"""Add Inductor kernel stack information to a Chrome trace."""
|
| 675 |
+
from torch._inductor.debug import get_kernel_information_jsons
|
| 676 |
+
|
| 677 |
+
compile_linenos = get_kernel_information_jsons()
|
| 678 |
+
if not compile_linenos:
|
| 679 |
+
return origin_trace
|
| 680 |
+
|
| 681 |
+
trace = origin_trace
|
| 682 |
+
uid_2_events = self._assign_uniq_id_to_event(trace)
|
| 683 |
+
|
| 684 |
+
real_events = []
|
| 685 |
+
flow_events = []
|
| 686 |
+
compiled_events = []
|
| 687 |
+
extern_events = []
|
| 688 |
+
for event in trace["traceEvents"]:
|
| 689 |
+
name = event.get("name", "")
|
| 690 |
+
cat = event.get("cat", "")
|
| 691 |
+
if name in _profile_flow_types or cat in _profile_flow_types:
|
| 692 |
+
flow_events.append(event)
|
| 693 |
+
continue
|
| 694 |
+
|
| 695 |
+
real_events.append(event)
|
| 696 |
+
if cat == "cpu_op" and (
|
| 697 |
+
name == "CompiledFunctionBackward" or "Torch-Compiled Region" in name
|
| 698 |
+
):
|
| 699 |
+
compiled_events.append(event)
|
| 700 |
+
if (
|
| 701 |
+
name
|
| 702 |
+
and "extern_kernels" in name
|
| 703 |
+
and cat
|
| 704 |
+
in (
|
| 705 |
+
"cpu_op",
|
| 706 |
+
"python_function",
|
| 707 |
+
)
|
| 708 |
+
):
|
| 709 |
+
extern_events.append(event)
|
| 710 |
+
|
| 711 |
+
compiled_events = sorted(compiled_events, key=lambda x: _EventItem(x).start)
|
| 712 |
+
compiled_event_items = [_EventItem(event) for event in compiled_events]
|
| 713 |
+
compiled_event_starts = [item.start for item in compiled_event_items]
|
| 714 |
+
ops_in_compile_region = _find_events_covered_in(real_events, compiled_events)
|
| 715 |
+
ops_in_extern_region = _find_events_covered_in(real_events, extern_events)
|
| 716 |
+
src2dst, dst2src = self._build_flow_mapping(trace, flow_events)
|
| 717 |
+
kernel_uids_by_external_id = defaultdict(list)
|
| 718 |
+
for event in real_events:
|
| 719 |
+
if event.get("cat") != "kernel":
|
| 720 |
+
continue
|
| 721 |
+
external_id = event.get("args", {}).get("External id")
|
| 722 |
+
if external_id is not None:
|
| 723 |
+
kernel_uids_by_external_id[external_id].append(event["uid"])
|
| 724 |
+
|
| 725 |
+
def _related_compile_region(compile_event):
|
| 726 |
+
src_event = uid_2_events[dst2src[compile_event["uid"]]]
|
| 727 |
+
src_item = _EventItem(src_event)
|
| 728 |
+
idx = bisect_left(compiled_event_starts, src_item.start) - 1
|
| 729 |
+
# Compiled regions are expected to be nearly nested around their
|
| 730 |
+
# source events, so the nearest preceding region usually matches.
|
| 731 |
+
while idx >= 0:
|
| 732 |
+
region_item = compiled_event_items[idx]
|
| 733 |
+
if (
|
| 734 |
+
src_item.start > region_item.start
|
| 735 |
+
and src_item.end < region_item.end
|
| 736 |
+
):
|
| 737 |
+
return region_item.e
|
| 738 |
+
idx -= 1
|
| 739 |
+
raise ValueError(f"Cannot find compile region for {compile_event}")
|
| 740 |
+
|
| 741 |
+
def _parse_compile_region_key(key):
|
| 742 |
+
try:
|
| 743 |
+
parsed_key = ast.literal_eval(key)
|
| 744 |
+
except (SyntaxError, ValueError):
|
| 745 |
+
return None
|
| 746 |
+
if (
|
| 747 |
+
not isinstance(parsed_key, tuple)
|
| 748 |
+
or len(parsed_key) != 2
|
| 749 |
+
or not isinstance(parsed_key[0], str)
|
| 750 |
+
or not isinstance(parsed_key[1], bool)
|
| 751 |
+
):
|
| 752 |
+
return None
|
| 753 |
+
return parsed_key
|
| 754 |
+
|
| 755 |
+
def _backward_graph_keys(compile_info, compile_name):
|
| 756 |
+
bw_graph_key = str((compile_name, True))
|
| 757 |
+
keys = [bw_graph_key]
|
| 758 |
+
if bw_graph_key in compile_info:
|
| 759 |
+
return keys
|
| 760 |
+
|
| 761 |
+
prefix = compile_name + "_"
|
| 762 |
+
for key, info in compile_info.items():
|
| 763 |
+
if not info:
|
| 764 |
+
continue
|
| 765 |
+
parsed_key = _parse_compile_region_key(key)
|
| 766 |
+
if parsed_key is None:
|
| 767 |
+
continue
|
| 768 |
+
graph_name, is_backward = parsed_key
|
| 769 |
+
if is_backward and graph_name.startswith(prefix):
|
| 770 |
+
keys.append(key)
|
| 771 |
+
return keys
|
| 772 |
+
|
| 773 |
+
def _stack_from_kernel_info(kernel_info):
|
| 774 |
+
if isinstance(kernel_info, dict):
|
| 775 |
+
return kernel_info.get("stack_traces")
|
| 776 |
+
return kernel_info
|
| 777 |
+
|
| 778 |
+
def _stack_for_kernel(compile_info, graph_key, kernel_name):
|
| 779 |
+
graph_info = compile_info.get(graph_key, {})
|
| 780 |
+
kernel_info = graph_info.get(kernel_name)
|
| 781 |
+
if kernel_info is None:
|
| 782 |
+
kernel_prefix = kernel_name + ":"
|
| 783 |
+
for name, info in graph_info.items():
|
| 784 |
+
if name.startswith(kernel_prefix):
|
| 785 |
+
kernel_info = info
|
| 786 |
+
break
|
| 787 |
+
return _stack_from_kernel_info(kernel_info)
|
| 788 |
+
|
| 789 |
+
def _single_stack_for_graph(compile_info, graph_key):
|
| 790 |
+
stacks = [
|
| 791 |
+
stack
|
| 792 |
+
for stack in (
|
| 793 |
+
_stack_from_kernel_info(kernel_info)
|
| 794 |
+
for kernel_info in compile_info.get(graph_key, {}).values()
|
| 795 |
+
)
|
| 796 |
+
if stack is not None
|
| 797 |
+
]
|
| 798 |
+
if len(stacks) == 1:
|
| 799 |
+
return stacks[0]
|
| 800 |
+
return None
|
| 801 |
+
|
| 802 |
+
def _assign_stack(event, compile_info, fwd_key, bw_keys, kernel_name):
|
| 803 |
+
for bw_key in bw_keys:
|
| 804 |
+
stack = _stack_for_kernel(compile_info, bw_key, kernel_name)
|
| 805 |
+
if stack is not None:
|
| 806 |
+
event.setdefault("args", {})["stack"] = stack
|
| 807 |
+
return True
|
| 808 |
+
stack = _stack_for_kernel(compile_info, fwd_key, kernel_name)
|
| 809 |
+
if stack is not None:
|
| 810 |
+
event.setdefault("args", {})["stack"] = stack
|
| 811 |
+
return True
|
| 812 |
+
return False
|
| 813 |
+
|
| 814 |
+
def _assign_single_stack(event, compile_info, fwd_key, bw_keys):
|
| 815 |
+
for bw_key in bw_keys:
|
| 816 |
+
stack = _single_stack_for_graph(compile_info, bw_key)
|
| 817 |
+
if stack is not None:
|
| 818 |
+
event.setdefault("args", {})["stack"] = stack
|
| 819 |
+
return True
|
| 820 |
+
stack = _single_stack_for_graph(compile_info, fwd_key)
|
| 821 |
+
if stack is not None:
|
| 822 |
+
event.setdefault("args", {})["stack"] = stack
|
| 823 |
+
return True
|
| 824 |
+
return False
|
| 825 |
+
|
| 826 |
+
def _kernel_events_for_op(op_id):
|
| 827 |
+
if op_id in src2dst:
|
| 828 |
+
return [uid_2_events[src2dst[op_id]]]
|
| 829 |
+
external_id = uid_2_events[op_id].get("args", {}).get("External id")
|
| 830 |
+
if external_id is None:
|
| 831 |
+
return []
|
| 832 |
+
return [
|
| 833 |
+
uid_2_events[kernel_uid]
|
| 834 |
+
for kernel_uid in kernel_uids_by_external_id.get(external_id, [])
|
| 835 |
+
]
|
| 836 |
+
|
| 837 |
+
def _extern_kernel_name(name):
|
| 838 |
+
match = re.search(r"\b(extern\w+)\s*,", name)
|
| 839 |
+
return match.group(1) if match else name
|
| 840 |
+
|
| 841 |
+
for compile_func_id, scoped_ops in ops_in_compile_region.items():
|
| 842 |
+
compile_event = uid_2_events[compile_func_id]
|
| 843 |
+
compile_name = compile_event["name"]
|
| 844 |
+
need_bw = "Backward" in compile_name
|
| 845 |
+
|
| 846 |
+
if need_bw:
|
| 847 |
+
try:
|
| 848 |
+
compile_name = _related_compile_region(compile_event)["name"]
|
| 849 |
+
except ValueError:
|
| 850 |
+
continue
|
| 851 |
+
|
| 852 |
+
fw_graph_key = str((compile_name, False))
|
| 853 |
+
bw_graph_keys = _backward_graph_keys(compile_linenos, compile_name)
|
| 854 |
+
if fw_graph_key not in compile_linenos and all(
|
| 855 |
+
bw_graph_key not in compile_linenos for bw_graph_key in bw_graph_keys
|
| 856 |
+
):
|
| 857 |
+
continue
|
| 858 |
+
|
| 859 |
+
for op_id in scoped_ops:
|
| 860 |
+
kernel_events = _kernel_events_for_op(op_id)
|
| 861 |
+
if not kernel_events:
|
| 862 |
+
continue
|
| 863 |
+
for kernel_event in kernel_events:
|
| 864 |
+
assigned_stack = False
|
| 865 |
+
warn_if_unrecognized = False
|
| 866 |
+
if self._maybe_extern_call(
|
| 867 |
+
kernel_event,
|
| 868 |
+
compile_linenos,
|
| 869 |
+
fw_graph_key,
|
| 870 |
+
bw_graph_keys,
|
| 871 |
+
need_bw,
|
| 872 |
+
):
|
| 873 |
+
for extern_call, related_ops in ops_in_extern_region.items():
|
| 874 |
+
if op_id in related_ops:
|
| 875 |
+
extern_event = uid_2_events[extern_call]
|
| 876 |
+
assigned_stack = _assign_stack(
|
| 877 |
+
kernel_event,
|
| 878 |
+
compile_linenos,
|
| 879 |
+
fw_graph_key,
|
| 880 |
+
bw_graph_keys,
|
| 881 |
+
_extern_kernel_name(extern_event.get("name", "")),
|
| 882 |
+
)
|
| 883 |
+
if assigned_stack:
|
| 884 |
+
break
|
| 885 |
+
if not assigned_stack:
|
| 886 |
+
assigned_stack = _assign_single_stack(
|
| 887 |
+
kernel_event,
|
| 888 |
+
compile_linenos,
|
| 889 |
+
fw_graph_key,
|
| 890 |
+
bw_graph_keys,
|
| 891 |
+
)
|
| 892 |
+
elif not self._maybe_triton_call(kernel_event.get("name", "")):
|
| 893 |
+
warn_if_unrecognized = True
|
| 894 |
+
assigned_stack = _assign_single_stack(
|
| 895 |
+
kernel_event,
|
| 896 |
+
compile_linenos,
|
| 897 |
+
fw_graph_key,
|
| 898 |
+
bw_graph_keys,
|
| 899 |
+
)
|
| 900 |
+
else:
|
| 901 |
+
assigned_stack = _assign_stack(
|
| 902 |
+
kernel_event,
|
| 903 |
+
compile_linenos,
|
| 904 |
+
fw_graph_key,
|
| 905 |
+
bw_graph_keys,
|
| 906 |
+
kernel_event.get("name", ""),
|
| 907 |
+
)
|
| 908 |
+
if not assigned_stack and warn_if_unrecognized:
|
| 909 |
+
log.warning(
|
| 910 |
+
"Kernel %s cannot be recognized as a custom kernel or triton kernel. "
|
| 911 |
+
"Please try profile with stack=True.",
|
| 912 |
+
kernel_event.get("name", ""),
|
| 913 |
+
)
|
| 914 |
+
return trace
|
| 915 |
+
|
| 916 |
+
def export_chrome_trace(self, path, metadata=None, use_python_export=False):
|
| 917 |
+
"""
|
| 918 |
+
Exports the collected trace in Chrome JSON format. If kineto is enabled, only
|
| 919 |
+
last cycle in schedule is exported.
|
| 920 |
+
"""
|
| 921 |
+
if use_python_export and kineto_available():
|
| 922 |
+
from torch.profiler._chrome_trace_export import (
|
| 923 |
+
export_chrome_trace as _export,
|
| 924 |
+
)
|
| 925 |
+
|
| 926 |
+
_export(self.kineto_results, path, metadata) # type: ignore[union-attr]
|
| 927 |
+
elif kineto_available():
|
| 928 |
+
self.kineto_results.save(path) # type: ignore[union-attr]
|
| 929 |
+
else:
|
| 930 |
+
self._ensure_function_events()
|
| 931 |
+
return self._function_events.export_chrome_trace(path) # type: ignore[union-attr]
|
| 932 |
+
|
| 933 |
+
import torch._inductor.config as inductor_config
|
| 934 |
+
|
| 935 |
+
if inductor_config.trace.provenance_tracking_to_timeline:
|
| 936 |
+
from torch._inductor.debug import get_kernel_information_jsons
|
| 937 |
+
|
| 938 |
+
try:
|
| 939 |
+
# Kineto owns the initial serialization path. Re-read the exported
|
| 940 |
+
# trace here so timeline provenance stays decoupled from Kineto's
|
| 941 |
+
# JSON writer and remains a Python-only post-processing step.
|
| 942 |
+
with open(path) as f:
|
| 943 |
+
trace = json.load(f)
|
| 944 |
+
|
| 945 |
+
num_events = len(trace.get("traceEvents", []))
|
| 946 |
+
max_events = inductor_config.trace.provenance_tracking_max_events
|
| 947 |
+
if max_events > 0 and num_events > max_events:
|
| 948 |
+
log.warning(
|
| 949 |
+
"Skipping provenance tracking: trace has %d events "
|
| 950 |
+
"(exceeds limit of %d). Set TORCH_COMPILE_DEBUG_MAX_EVENTS=0 "
|
| 951 |
+
"to disable this protection or increase the limit.",
|
| 952 |
+
num_events,
|
| 953 |
+
max_events,
|
| 954 |
+
)
|
| 955 |
+
return
|
| 956 |
+
|
| 957 |
+
if not inductor_config.triton.unique_kernel_names:
|
| 958 |
+
log.warning(
|
| 959 |
+
"Profiling trace does not contain Triton kernel stack traces "
|
| 960 |
+
"because TORCHINDUCTOR_UNIQUE_KERNEL_NAMES=0."
|
| 961 |
+
)
|
| 962 |
+
if inductor_config.cpp_wrapper:
|
| 963 |
+
log.warning(
|
| 964 |
+
"Profiling trace does not contain compiled kernel stack traces "
|
| 965 |
+
"because cpp_wrapper is enabled."
|
| 966 |
+
)
|
| 967 |
+
return
|
| 968 |
+
try:
|
| 969 |
+
log.info("Add stack trace to compiled kernel.")
|
| 970 |
+
trace = self.add_to_chrome_trace(trace)
|
| 971 |
+
with open(path, "w") as f:
|
| 972 |
+
json.dump(trace, f, indent=1)
|
| 973 |
+
except MemoryError:
|
| 974 |
+
log.error(
|
| 975 |
+
"MemoryError during add_to_chrome_trace. "
|
| 976 |
+
"Try increasing TORCH_COMPILE_DEBUG_MAX_EVENTS or disable provenance tracking."
|
| 977 |
+
)
|
| 978 |
+
raise
|
| 979 |
+
except Exception:
|
| 980 |
+
log.exception("Failed to add stack trace to compiled kernel")
|
| 981 |
+
finally:
|
| 982 |
+
get_kernel_information_jsons().clear()
|
| 983 |
+
|
| 984 |
+
export_chrome_trace.__doc__ = EventList.export_chrome_trace.__doc__
|
| 985 |
+
|
| 986 |
+
def export_stacks(self, path: str, metric: str = "self_cpu_time_total"):
|
| 987 |
+
self._ensure_function_events()
|
| 988 |
+
if self._function_events is None:
|
| 989 |
+
raise AssertionError("Expected profiling results")
|
| 990 |
+
if not self.with_stack:
|
| 991 |
+
raise AssertionError("export_stacks() requires with_stack=True")
|
| 992 |
+
return self._function_events.export_stacks(path, metric)
|
| 993 |
+
|
| 994 |
+
def toggle_collection_dynamic(
|
| 995 |
+
self, enabled: bool, activities: Iterable[ProfilerActivity]
|
| 996 |
+
):
|
| 997 |
+
"""
|
| 998 |
+
Toggles the collection of activities for the current profiler instance.
|
| 999 |
+
"""
|
| 1000 |
+
return _toggle_collection_dynamic(enabled, set(activities))
|
| 1001 |
+
|
| 1002 |
+
def key_averages(
|
| 1003 |
+
self,
|
| 1004 |
+
group_by_input_shape=False,
|
| 1005 |
+
group_by_stack_n=0,
|
| 1006 |
+
group_by_overload_name=False,
|
| 1007 |
+
include_python_functions=False,
|
| 1008 |
+
):
|
| 1009 |
+
self._ensure_function_events()
|
| 1010 |
+
if self._function_events is None:
|
| 1011 |
+
raise AssertionError("Expected profiling results")
|
| 1012 |
+
return self._function_events.key_averages(
|
| 1013 |
+
group_by_input_shape,
|
| 1014 |
+
group_by_stack_n,
|
| 1015 |
+
group_by_overload_name,
|
| 1016 |
+
include_python_functions,
|
| 1017 |
+
)
|
| 1018 |
+
|
| 1019 |
+
key_averages.__doc__ = EventList.key_averages.__doc__
|
| 1020 |
+
|
| 1021 |
+
def total_average(self):
|
| 1022 |
+
self._ensure_function_events()
|
| 1023 |
+
if self._function_events is None:
|
| 1024 |
+
raise AssertionError("Expected profiling results")
|
| 1025 |
+
return self._function_events.total_average()
|
| 1026 |
+
|
| 1027 |
+
total_average.__doc__ = EventList.total_average.__doc__
|
| 1028 |
+
|
| 1029 |
+
@property
|
| 1030 |
+
def self_cpu_time_total(self):
|
| 1031 |
+
"""Returns total time spent on CPU.
|
| 1032 |
+
|
| 1033 |
+
The total time is a sum of all self times across all the events.
|
| 1034 |
+
"""
|
| 1035 |
+
self._ensure_function_events()
|
| 1036 |
+
if self._function_events is None:
|
| 1037 |
+
raise AssertionError("Expected profiling results")
|
| 1038 |
+
return self._function_events.self_cpu_time_total
|
| 1039 |
+
|
| 1040 |
+
def _parse_kineto_results(
|
| 1041 |
+
self, result: _ProfilerResult, timeout_s: float | None = None
|
| 1042 |
+
):
|
| 1043 |
+
# result.events() has most of the events - PyTorch op-level and device-level events
|
| 1044 |
+
|
| 1045 |
+
timeout_ns = int(timeout_s * 1e9) if timeout_s is not None else None
|
| 1046 |
+
result_events = result.events()
|
| 1047 |
+
if timeout_ns is not None and timeout_ns < 0:
|
| 1048 |
+
raise ValueError("timeout_s must be non-negative")
|
| 1049 |
+
start_time_ns = perf_counter_ns()
|
| 1050 |
+
timed_out = False
|
| 1051 |
+
|
| 1052 |
+
def _check_timeout() -> bool:
|
| 1053 |
+
"""Check if timeout has been exceeded. Returns True if timed out."""
|
| 1054 |
+
nonlocal timed_out
|
| 1055 |
+
if timeout_ns is not None and not timed_out:
|
| 1056 |
+
elapsed_ns = perf_counter_ns() - start_time_ns
|
| 1057 |
+
if elapsed_ns >= timeout_ns:
|
| 1058 |
+
timed_out = True
|
| 1059 |
+
return timed_out
|
| 1060 |
+
|
| 1061 |
+
trace_start_ns = result.trace_start_ns()
|
| 1062 |
+
mem_records = [
|
| 1063 |
+
[evt, False] for evt in result_events if evt.name() == MEMORY_EVENT_NAME
|
| 1064 |
+
]
|
| 1065 |
+
oom_records = [
|
| 1066 |
+
evt for evt in result_events if evt.name() == OUT_OF_MEMORY_EVENT_NAME
|
| 1067 |
+
]
|
| 1068 |
+
mem_records_acc = MemRecordsAcc(mem_records)
|
| 1069 |
+
|
| 1070 |
+
def _cpu_memory_usage(mem_record):
|
| 1071 |
+
return (
|
| 1072 |
+
mem_record.nbytes()
|
| 1073 |
+
if mem_record.device_type()
|
| 1074 |
+
in [DeviceType.CPU, DeviceType.MKLDNN, DeviceType.IDEEP]
|
| 1075 |
+
else 0
|
| 1076 |
+
)
|
| 1077 |
+
|
| 1078 |
+
def _device_memory_usage(mem_record):
|
| 1079 |
+
return (
|
| 1080 |
+
mem_record.nbytes()
|
| 1081 |
+
if mem_record.device_type()
|
| 1082 |
+
in [
|
| 1083 |
+
DeviceType.CUDA,
|
| 1084 |
+
DeviceType.PrivateUse1,
|
| 1085 |
+
DeviceType.HIP,
|
| 1086 |
+
DeviceType.XPU,
|
| 1087 |
+
]
|
| 1088 |
+
else 0
|
| 1089 |
+
)
|
| 1090 |
+
|
| 1091 |
+
# Create and return FunctionEvent list, which contains all function events
|
| 1092 |
+
# Here 2 function events are created:
|
| 1093 |
+
# all_function_events contains all events associated with each kineto event from result
|
| 1094 |
+
all_function_events = []
|
| 1095 |
+
# frontend_function_events contains the events in aten or torch frontend level,
|
| 1096 |
+
# whose correlation id is 0
|
| 1097 |
+
frontend_function_events = []
|
| 1098 |
+
device_corr_map: dict[int, list[FunctionEvent]] = {}
|
| 1099 |
+
max_evt_id = 0
|
| 1100 |
+
for kineto_event in result_events:
|
| 1101 |
+
if _check_timeout():
|
| 1102 |
+
break
|
| 1103 |
+
|
| 1104 |
+
if (
|
| 1105 |
+
_filter_name(kineto_event.name())
|
| 1106 |
+
or getattr(kineto_event, "is_hidden_event", lambda: False)()
|
| 1107 |
+
):
|
| 1108 |
+
continue
|
| 1109 |
+
rel_start_ns = kineto_event.start_ns() - trace_start_ns
|
| 1110 |
+
rel_end_ns = kineto_event.end_ns() - trace_start_ns
|
| 1111 |
+
|
| 1112 |
+
cpu_memory_usage = 0
|
| 1113 |
+
device_memory_usage = 0
|
| 1114 |
+
if kineto_event.device_type() == DeviceType.CPU:
|
| 1115 |
+
# find the corresponding memory allocation events
|
| 1116 |
+
for mem_record in mem_records_acc.in_interval(
|
| 1117 |
+
kineto_event.start_ns(), kineto_event.end_ns()
|
| 1118 |
+
):
|
| 1119 |
+
cpu_memory_usage += _cpu_memory_usage(mem_record[0])
|
| 1120 |
+
device_memory_usage += _device_memory_usage(mem_record[0])
|
| 1121 |
+
mem_record[1] = True
|
| 1122 |
+
|
| 1123 |
+
is_async = kineto_event.is_async() or (
|
| 1124 |
+
kineto_event.start_thread_id() != kineto_event.end_thread_id()
|
| 1125 |
+
)
|
| 1126 |
+
|
| 1127 |
+
fe = FunctionEvent(
|
| 1128 |
+
id=kineto_event.correlation_id(),
|
| 1129 |
+
name=_rewrite_name(name=kineto_event.name(), with_wildcard=True),
|
| 1130 |
+
overload_name=kineto_event.overload_name(),
|
| 1131 |
+
trace_name=_rewrite_name(name=kineto_event.name(), with_wildcard=False),
|
| 1132 |
+
thread=kineto_event.start_thread_id(),
|
| 1133 |
+
start_us=rel_start_ns / 1000,
|
| 1134 |
+
end_us=rel_end_ns / 1000,
|
| 1135 |
+
fwd_thread=kineto_event.fwd_thread_id(),
|
| 1136 |
+
input_shapes=kineto_event.shapes(),
|
| 1137 |
+
concrete_inputs=kineto_event.concrete_inputs(),
|
| 1138 |
+
kwinputs=kineto_event.kwinputs(),
|
| 1139 |
+
stack=[
|
| 1140 |
+
entry
|
| 1141 |
+
for entry in kineto_event.stack()
|
| 1142 |
+
if _filter_stack_entry(entry)
|
| 1143 |
+
],
|
| 1144 |
+
scope=kineto_event.scope(),
|
| 1145 |
+
use_device=self.use_device,
|
| 1146 |
+
cpu_memory_usage=cpu_memory_usage,
|
| 1147 |
+
device_memory_usage=device_memory_usage,
|
| 1148 |
+
is_async=is_async,
|
| 1149 |
+
sequence_nr=kineto_event.sequence_nr(),
|
| 1150 |
+
device_type=kineto_event.device_type(),
|
| 1151 |
+
device_index=kineto_event.device_index(),
|
| 1152 |
+
device_resource_id=kineto_event.device_resource_id(),
|
| 1153 |
+
flops=kineto_event.flops(),
|
| 1154 |
+
is_user_annotation=kineto_event.is_user_annotation(),
|
| 1155 |
+
is_python_function=kineto_event.is_python_function(),
|
| 1156 |
+
activity_type=kineto_event.activity_type(),
|
| 1157 |
+
metadata_json=kineto_event.metadata_json(),
|
| 1158 |
+
extra_meta=kineto_event.extra_meta() or None,
|
| 1159 |
+
flow_id=kineto_event.flow_id(),
|
| 1160 |
+
flow_type=kineto_event.flow_type(),
|
| 1161 |
+
flow_start=kineto_event.flow_start(),
|
| 1162 |
+
external_id=kineto_event.external_id(),
|
| 1163 |
+
linked_correlation_id=kineto_event.linked_correlation_id(),
|
| 1164 |
+
structured_input_shapes=kineto_event.structured_input_shapes(),
|
| 1165 |
+
structured_input_strides=kineto_event.structured_input_strides(),
|
| 1166 |
+
input_dtypes=kineto_event.dtypes(),
|
| 1167 |
+
python_id=kineto_event.python_id(),
|
| 1168 |
+
python_parent_id=kineto_event.python_parent_id(),
|
| 1169 |
+
python_module_id=kineto_event.python_module_id(),
|
| 1170 |
+
)
|
| 1171 |
+
max_evt_id = max(max_evt_id, fe.id)
|
| 1172 |
+
if fe.device_type == DeviceType.CPU and not fe.is_async:
|
| 1173 |
+
if self.use_device == _get_privateuse1_backend_name():
|
| 1174 |
+
privateuse1_time = kineto_event.privateuse1_elapsed_us()
|
| 1175 |
+
if privateuse1_time > 0:
|
| 1176 |
+
fe.append_kernel(fe.name, fe.device_index, privateuse1_time)
|
| 1177 |
+
fe.is_legacy = True
|
| 1178 |
+
elif self.use_device == "cuda":
|
| 1179 |
+
# Check if we have CUDA time as a fallback
|
| 1180 |
+
cuda_time = kineto_event.cuda_elapsed_us()
|
| 1181 |
+
if cuda_time > 0:
|
| 1182 |
+
fe.append_kernel(fe.name, fe.device_index, cuda_time)
|
| 1183 |
+
fe.is_legacy = True
|
| 1184 |
+
all_function_events.append(fe)
|
| 1185 |
+
corr_id = kineto_event.linked_correlation_id()
|
| 1186 |
+
if corr_id > 0:
|
| 1187 |
+
if corr_id not in device_corr_map:
|
| 1188 |
+
device_corr_map[corr_id] = []
|
| 1189 |
+
device_corr_map[corr_id].append(fe)
|
| 1190 |
+
elif corr_id == 0:
|
| 1191 |
+
frontend_function_events.append(fe)
|
| 1192 |
+
else:
|
| 1193 |
+
raise RuntimeError(
|
| 1194 |
+
f"Got negative correlation id {corr_id} in profiler post processing"
|
| 1195 |
+
)
|
| 1196 |
+
|
| 1197 |
+
# associate device kernels and device runtime (CPU) with CPU events
|
| 1198 |
+
for fe in frontend_function_events:
|
| 1199 |
+
if (
|
| 1200 |
+
fe.device_type == DeviceType.CPU
|
| 1201 |
+
and not fe.is_async
|
| 1202 |
+
and fe.id in device_corr_map
|
| 1203 |
+
):
|
| 1204 |
+
for f_evt in device_corr_map[fe.id]:
|
| 1205 |
+
if f_evt.device_type in [
|
| 1206 |
+
DeviceType.CUDA,
|
| 1207 |
+
DeviceType.PrivateUse1,
|
| 1208 |
+
DeviceType.XPU,
|
| 1209 |
+
]:
|
| 1210 |
+
fe.append_kernel(
|
| 1211 |
+
f_evt.name,
|
| 1212 |
+
f_evt.device_index,
|
| 1213 |
+
f_evt.time_range.end - f_evt.time_range.start,
|
| 1214 |
+
)
|
| 1215 |
+
elif f_evt.device_type == DeviceType.CPU:
|
| 1216 |
+
# make sure that 'thread' of a CPU Kineto (e.g. Device Runtime) event is associated
|
| 1217 |
+
# with the 'thread' of the corresponding linked PyTorch event to properly track
|
| 1218 |
+
# parents and children
|
| 1219 |
+
f_evt.thread = fe.thread
|
| 1220 |
+
|
| 1221 |
+
def _create_function_event_for_memory_events(evt):
|
| 1222 |
+
rel_start_ns = evt.start_ns() - trace_start_ns
|
| 1223 |
+
fe = FunctionEvent(
|
| 1224 |
+
id=max_evt_id,
|
| 1225 |
+
name=evt.name(),
|
| 1226 |
+
overload_name="",
|
| 1227 |
+
trace_name=None, # not outputting in the trace
|
| 1228 |
+
thread=evt.start_thread_id(),
|
| 1229 |
+
start_us=rel_start_ns / 1000,
|
| 1230 |
+
end_us=rel_start_ns / 1000, # no duration
|
| 1231 |
+
fwd_thread=evt.start_thread_id(),
|
| 1232 |
+
input_shapes=[],
|
| 1233 |
+
stack=[],
|
| 1234 |
+
scope=0, # RecordScope::FUNCTION
|
| 1235 |
+
use_device=self.use_device,
|
| 1236 |
+
cpu_memory_usage=_cpu_memory_usage(evt),
|
| 1237 |
+
device_memory_usage=_device_memory_usage(evt),
|
| 1238 |
+
is_async=False,
|
| 1239 |
+
sequence_nr=-1,
|
| 1240 |
+
device_type=DeviceType.CPU,
|
| 1241 |
+
device_index=0,
|
| 1242 |
+
)
|
| 1243 |
+
return fe
|
| 1244 |
+
|
| 1245 |
+
# output top-level memory events
|
| 1246 |
+
for mem_record in mem_records:
|
| 1247 |
+
if _check_timeout():
|
| 1248 |
+
break
|
| 1249 |
+
|
| 1250 |
+
if not mem_record[1]:
|
| 1251 |
+
max_evt_id += 1
|
| 1252 |
+
fe = _create_function_event_for_memory_events(mem_record[0])
|
| 1253 |
+
all_function_events.append(fe)
|
| 1254 |
+
|
| 1255 |
+
for oom_record in oom_records:
|
| 1256 |
+
if _check_timeout():
|
| 1257 |
+
break
|
| 1258 |
+
|
| 1259 |
+
max_evt_id += 1
|
| 1260 |
+
fe = _create_function_event_for_memory_events(oom_record)
|
| 1261 |
+
all_function_events.append(fe)
|
| 1262 |
+
|
| 1263 |
+
if timed_out:
|
| 1264 |
+
log.warning(
|
| 1265 |
+
"Profiler _parse_kineto_results timed out after %.3f seconds, "
|
| 1266 |
+
"returning partial results with %d events",
|
| 1267 |
+
timeout_s,
|
| 1268 |
+
len(all_function_events),
|
| 1269 |
+
)
|
| 1270 |
+
|
| 1271 |
+
all_function_events.sort(
|
| 1272 |
+
key=lambda evt: [evt.time_range.start, -evt.time_range.end]
|
| 1273 |
+
)
|
| 1274 |
+
return all_function_events
|
| 1275 |
+
|
| 1276 |
+
|
| 1277 |
+
# Set by torch.profiler to the active cupti_monitor ProfilerObserver while a session is
|
| 1278 |
+
# running (None otherwise). record_function routes regions to it via push/pop_annotation.
|
| 1279 |
+
# Held as an opaque object -- NOT imported from the cupti package -- so record_function never
|
| 1280 |
+
# pulls in the cupti chain on a non-cupti run, and there is a single "is a session active"
|
| 1281 |
+
# signal (this reference) rather than a separate flag.
|
| 1282 |
+
_active_cupti_profiler_observer: Any = None
|
| 1283 |
+
|
| 1284 |
+
|
| 1285 |
+
def _set_active_cupti_profiler_observer(observer: Any) -> None:
|
| 1286 |
+
global _active_cupti_profiler_observer
|
| 1287 |
+
_active_cupti_profiler_observer = observer
|
| 1288 |
+
|
| 1289 |
+
|
| 1290 |
+
class record_function(_ContextDecorator): # pyrefly: ignore [invalid-inheritance]
|
| 1291 |
+
"""Context manager/function decorator that adds a label to a code block/function when running autograd profiler.
|
| 1292 |
+
Label will only appear if CPU activity tracing is enabled.
|
| 1293 |
+
|
| 1294 |
+
It is useful when tracing the code profile.
|
| 1295 |
+
|
| 1296 |
+
Args:
|
| 1297 |
+
name (str): Label assigned to the block of code.
|
| 1298 |
+
node_id (int): ID of node, for distributed profiling. Unset in
|
| 1299 |
+
non-distributed cases.
|
| 1300 |
+
|
| 1301 |
+
Example:
|
| 1302 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER)
|
| 1303 |
+
>>> x = torch.randn((1, 1), requires_grad=True)
|
| 1304 |
+
>>> with torch.autograd.profiler.profile() as prof:
|
| 1305 |
+
... y = x**2
|
| 1306 |
+
... with torch.autograd.profiler.record_function(
|
| 1307 |
+
... "label-z"
|
| 1308 |
+
... ): # label the block
|
| 1309 |
+
... z = y**3
|
| 1310 |
+
... y.backward()
|
| 1311 |
+
>>> # xdoctest: +IGNORE_WANT
|
| 1312 |
+
>>> # NOTE: some columns were removed for brevity
|
| 1313 |
+
>>> print(prof.key_averages().table(sort_by="self_cpu_time_total"))
|
| 1314 |
+
----------------------------------- --------------- --------------- ---------------
|
| 1315 |
+
Name Self CPU total % CPU time avg Number of Calls
|
| 1316 |
+
----------------------------------- --------------- --------------- ---------------
|
| 1317 |
+
pow 60.77% 47.470us 3
|
| 1318 |
+
mul 21.73% 25.465us 2
|
| 1319 |
+
PowBackward0 12.03% 121.891us 1
|
| 1320 |
+
torch::autograd::AccumulateGrad 2.70% 6.324us 1
|
| 1321 |
+
label-z 2.13% 12.421us 1
|
| 1322 |
+
torch::autograd::GraphRoot 0.64% 1.503us 1
|
| 1323 |
+
----------------------------------- --------------- --------------- ---------------
|
| 1324 |
+
Self CPU time total: 234.344us
|
| 1325 |
+
CUDA time total: 0.000us
|
| 1326 |
+
|
| 1327 |
+
"""
|
| 1328 |
+
|
| 1329 |
+
def __init__(self, name: str, args: str | None = None):
|
| 1330 |
+
self.name: str = name
|
| 1331 |
+
self.args: str | None = args
|
| 1332 |
+
# Whether or not we should run record function's end callbacks when exiting.
|
| 1333 |
+
self.run_callbacks_on_exit: bool = True
|
| 1334 |
+
# TODO: TorchScript ignores standard type annotation here
|
| 1335 |
+
# self.record: Optional["torch.classes.profiler._RecordFunction"] = None
|
| 1336 |
+
self.record = torch.jit.annotate(
|
| 1337 |
+
# pyrefly: ignore [not-a-type]
|
| 1338 |
+
Optional["torch.classes.profiler._RecordFunction"],
|
| 1339 |
+
None,
|
| 1340 |
+
)
|
| 1341 |
+
self._cupti_monitor_external_id: int | None = None
|
| 1342 |
+
|
| 1343 |
+
def __enter__(self):
|
| 1344 |
+
self.record = torch.ops.profiler._record_function_enter_new(
|
| 1345 |
+
self.name, self.args
|
| 1346 |
+
)
|
| 1347 |
+
# Route the region to the active cupti_monitor observer, if any. The reference is
|
| 1348 |
+
# None unless a cupti_monitor profile is running, so a non-cupti run never touches
|
| 1349 |
+
# the cupti chain. Guarded by is_scripting() (the global access doesn't compile under
|
| 1350 |
+
# TorchScript), and the global is read inside the guard so it is dead-code-eliminated.
|
| 1351 |
+
if not torch.jit.is_scripting():
|
| 1352 |
+
observer = _active_cupti_profiler_observer
|
| 1353 |
+
if observer is not None:
|
| 1354 |
+
self._cupti_monitor_external_id = observer.push_annotation(self.name)
|
| 1355 |
+
return self
|
| 1356 |
+
|
| 1357 |
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any):
|
| 1358 |
+
if not torch.jit.is_scripting():
|
| 1359 |
+
if self._cupti_monitor_external_id is not None:
|
| 1360 |
+
observer = _active_cupti_profiler_observer
|
| 1361 |
+
if observer is not None:
|
| 1362 |
+
observer.pop_annotation()
|
| 1363 |
+
self._cupti_monitor_external_id = None
|
| 1364 |
+
if not self.run_callbacks_on_exit:
|
| 1365 |
+
return
|
| 1366 |
+
|
| 1367 |
+
# Local variable is needed by TorchScript to refine Optional[T] to T
|
| 1368 |
+
record = self.record
|
| 1369 |
+
if record is None:
|
| 1370 |
+
raise AssertionError("Expected record to be set")
|
| 1371 |
+
|
| 1372 |
+
# TODO: Too slow with __torch_function__ handling enabled
|
| 1373 |
+
# See https://github.com/pytorch/pytorch/issues/76410
|
| 1374 |
+
if not torch.jit.is_scripting():
|
| 1375 |
+
with torch._C.DisableTorchFunctionSubclass():
|
| 1376 |
+
torch.ops.profiler._record_function_exit._RecordFunction(record)
|
| 1377 |
+
else:
|
| 1378 |
+
torch.ops.profiler._record_function_exit(record)
|
| 1379 |
+
|
| 1380 |
+
def _call_end_callbacks_on_future(self, fut: Future[Any]) -> Future[Any]:
|
| 1381 |
+
"""Use for profiling async calls that return a future.
|
| 1382 |
+
|
| 1383 |
+
Calling this function will extend recording beyond this scope, until the future is
|
| 1384 |
+
satisfied. It is useful for profiling the end to end time of asynchronous calls.
|
| 1385 |
+
This function should only be called once to attach the callback onto the future, and
|
| 1386 |
+
will throw if called multiple times.
|
| 1387 |
+
|
| 1388 |
+
Args:
|
| 1389 |
+
fut: (torch._C.Future): future for which to schedule
|
| 1390 |
+
callback for.
|
| 1391 |
+
|
| 1392 |
+
Returns:
|
| 1393 |
+
A future that completes with the value of the passed in future when
|
| 1394 |
+
the profiling callbacks have ran.
|
| 1395 |
+
|
| 1396 |
+
"""
|
| 1397 |
+
# Throw if we have already attached a callback onto the future.
|
| 1398 |
+
if not self.run_callbacks_on_exit:
|
| 1399 |
+
raise RuntimeError("_call_end_callbacks_on_future can only be called once.")
|
| 1400 |
+
|
| 1401 |
+
# We are scheduling to run this RecordFunction's end callbacks when the
|
| 1402 |
+
# passed in future completes, so don't run end callbacks on exit.
|
| 1403 |
+
self.run_callbacks_on_exit = False
|
| 1404 |
+
|
| 1405 |
+
# Local variable is needed by TorchScript to refine Optional[T] to T
|
| 1406 |
+
record = self.record
|
| 1407 |
+
if record is None:
|
| 1408 |
+
raise AssertionError("Expected record to be set")
|
| 1409 |
+
|
| 1410 |
+
# TODO: Too slow with __torch_function__ handling enabled
|
| 1411 |
+
# See https://github.com/pytorch/pytorch/issues/76410
|
| 1412 |
+
if not torch.jit.is_scripting():
|
| 1413 |
+
with torch._C.DisableTorchFunctionSubclass():
|
| 1414 |
+
profiled_future = (
|
| 1415 |
+
torch.ops.profiler._call_end_callbacks_on_jit_fut._RecordFunction(
|
| 1416 |
+
record, fut
|
| 1417 |
+
)
|
| 1418 |
+
)
|
| 1419 |
+
else:
|
| 1420 |
+
profiled_future = torch.ops.profiler._call_end_callbacks_on_jit_fut(
|
| 1421 |
+
record, fut
|
| 1422 |
+
)
|
| 1423 |
+
return profiled_future
|
| 1424 |
+
|
| 1425 |
+
|
| 1426 |
+
class emit_itt:
|
| 1427 |
+
"""Context manager that makes every autograd operation emit an ITT range.
|
| 1428 |
+
|
| 1429 |
+
It is useful when running the program under Intel(R) VTune Profiler::
|
| 1430 |
+
|
| 1431 |
+
vtune <--vtune-flags> <regular command here>
|
| 1432 |
+
|
| 1433 |
+
The Instrumentation and Tracing Technology (ITT) API enables your application to generate and
|
| 1434 |
+
control the collection of trace data during its execution across different Intel tools.
|
| 1435 |
+
This context manager is to annotate Intel(R) VTune Profiling trace. With help of this context manager,
|
| 1436 |
+
you will be able to see labeled ranges in Intel(R) VTune Profiler GUI.
|
| 1437 |
+
|
| 1438 |
+
.. warning:
|
| 1439 |
+
This context manager should not be called recursively, i.e. at most one
|
| 1440 |
+
instance should be enabled at any given time.
|
| 1441 |
+
|
| 1442 |
+
Args:
|
| 1443 |
+
enabled (bool, optional): Setting ``enabled=False`` makes this context manager a no-op.
|
| 1444 |
+
Default: ``True``.
|
| 1445 |
+
record_shapes (bool, optional): If ``record_shapes=True``, the itt range wrapping
|
| 1446 |
+
each autograd op will append information about the sizes of Tensor arguments received
|
| 1447 |
+
by that op, in the following format:
|
| 1448 |
+
``[[arg0.size(0), arg0.size(1), ...], [arg1.size(0), arg1.size(1), ...], ...]``
|
| 1449 |
+
Non-tensor arguments will be represented by ``[]``.
|
| 1450 |
+
Arguments will be listed in the order they are received by the backend op.
|
| 1451 |
+
Please note that this order may not match the order in which those arguments were passed
|
| 1452 |
+
on the Python side. Also note that shape recording may increase the overhead of itt range creation.
|
| 1453 |
+
Default: ``False``
|
| 1454 |
+
|
| 1455 |
+
Example:
|
| 1456 |
+
>>> # xdoctest: +SKIP("Undefined variables")
|
| 1457 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER)
|
| 1458 |
+
>>> with torch.autograd.profiler.emit_itt():
|
| 1459 |
+
... model(x)
|
| 1460 |
+
|
| 1461 |
+
"""
|
| 1462 |
+
|
| 1463 |
+
def __init__(self, enabled=True, record_shapes=False):
|
| 1464 |
+
self.enabled = enabled
|
| 1465 |
+
self.entered = False
|
| 1466 |
+
self.record_shapes = record_shapes
|
| 1467 |
+
|
| 1468 |
+
def __enter__(self):
|
| 1469 |
+
if not self.enabled:
|
| 1470 |
+
return
|
| 1471 |
+
if self.entered:
|
| 1472 |
+
raise RuntimeError("ITT annotation context manager is not reentrant")
|
| 1473 |
+
self.entered = True
|
| 1474 |
+
_run_on_profiler_start()
|
| 1475 |
+
_enable_profiler(
|
| 1476 |
+
ProfilerConfig(
|
| 1477 |
+
ProfilerState.ITT,
|
| 1478 |
+
self.record_shapes,
|
| 1479 |
+
False,
|
| 1480 |
+
False,
|
| 1481 |
+
False,
|
| 1482 |
+
False,
|
| 1483 |
+
_ExperimentalConfig(),
|
| 1484 |
+
),
|
| 1485 |
+
set(),
|
| 1486 |
+
)
|
| 1487 |
+
return self
|
| 1488 |
+
|
| 1489 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 1490 |
+
if not self.enabled:
|
| 1491 |
+
return
|
| 1492 |
+
_disable_profiler()
|
| 1493 |
+
_run_on_profiler_stop()
|
| 1494 |
+
return False
|
| 1495 |
+
|
| 1496 |
+
|
| 1497 |
+
class emit_nvtx:
|
| 1498 |
+
"""Context manager that makes every autograd operation emit an NVTX range.
|
| 1499 |
+
|
| 1500 |
+
It is useful when running the program under nvprof::
|
| 1501 |
+
|
| 1502 |
+
nvprof --profile-from-start off -o trace_name.prof -- <regular command here>
|
| 1503 |
+
|
| 1504 |
+
Unfortunately, there's no way to force nvprof to flush the data it collected
|
| 1505 |
+
to disk, so for CUDA profiling one has to use this context manager to annotate
|
| 1506 |
+
nvprof traces and wait for the process to exit before inspecting them.
|
| 1507 |
+
Then, either NVIDIA Visual Profiler (nvvp) can be used to visualize the timeline, or
|
| 1508 |
+
:func:`torch.autograd.profiler.load_nvprof` can load the results for inspection
|
| 1509 |
+
e.g. in Python REPL.
|
| 1510 |
+
|
| 1511 |
+
.. warning:
|
| 1512 |
+
This context manager should not be called recursively, i.e. at most one
|
| 1513 |
+
instance should be enabled at any given time.
|
| 1514 |
+
|
| 1515 |
+
Args:
|
| 1516 |
+
enabled (bool, optional): Setting ``enabled=False`` makes this context manager a no-op.
|
| 1517 |
+
Default: ``True``.
|
| 1518 |
+
record_shapes (bool, optional): If ``record_shapes=True``, the nvtx range wrapping
|
| 1519 |
+
each autograd op will append information about the sizes of Tensor arguments received
|
| 1520 |
+
by that op, in the following format:
|
| 1521 |
+
``[[arg0.size(0), arg0.size(1), ...], [arg1.size(0), arg1.size(1), ...], ...]``
|
| 1522 |
+
Non-tensor arguments will be represented by ``[]``.
|
| 1523 |
+
Arguments will be listed in the order they are received by the backend op.
|
| 1524 |
+
Please note that this order may not match the order in which those arguments were passed
|
| 1525 |
+
on the Python side. Also note that shape recording may increase the overhead of nvtx range creation.
|
| 1526 |
+
Default: ``False``
|
| 1527 |
+
|
| 1528 |
+
Example:
|
| 1529 |
+
>>> # xdoctest: +SKIP("undefined variables")
|
| 1530 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER)
|
| 1531 |
+
>>> with torch.cuda.profiler.profile():
|
| 1532 |
+
... model(x) # Warmup CUDA memory allocator and profiler
|
| 1533 |
+
... with torch.autograd.profiler.emit_nvtx():
|
| 1534 |
+
... model(x)
|
| 1535 |
+
|
| 1536 |
+
**Forward-backward correlation**
|
| 1537 |
+
|
| 1538 |
+
When viewing a profile created using :class:`emit_nvtx` in the Nvidia Visual Profiler,
|
| 1539 |
+
correlating each backward-pass op with the corresponding forward-pass op can be difficult.
|
| 1540 |
+
To ease this task, :class:`emit_nvtx` appends sequence number information to the ranges it
|
| 1541 |
+
generates.
|
| 1542 |
+
|
| 1543 |
+
During the forward pass, each function range is decorated with ``seq=<N>``. ``seq`` is a running
|
| 1544 |
+
counter, incremented each time a new backward Function object is created and stashed for backward.
|
| 1545 |
+
Thus, the ``seq=<N>`` annotation associated with each forward function range tells you that
|
| 1546 |
+
if a backward Function object is created by this forward function,
|
| 1547 |
+
the backward object will receive sequence number N.
|
| 1548 |
+
During the backward pass, the top-level range wrapping each C++ backward Function's
|
| 1549 |
+
``apply()`` call is decorated with ``stashed seq=<M>``. ``M`` is the sequence number that
|
| 1550 |
+
the backward object was created with. By comparing ``stashed seq`` numbers in backward with ``seq``
|
| 1551 |
+
numbers in forward, you can track down which forward op created each backward Function.
|
| 1552 |
+
|
| 1553 |
+
Any functions executed during the backward pass are also decorated with ``seq=<N>``. During
|
| 1554 |
+
default backward (with ``create_graph=False``) this information is irrelevant, and in fact,
|
| 1555 |
+
``N`` may simply be 0 for all such functions. Only the top-level ranges associated with
|
| 1556 |
+
backward Function objects' ``apply()`` methods are useful, as a way to correlate these Function
|
| 1557 |
+
objects with the earlier forward pass.
|
| 1558 |
+
|
| 1559 |
+
**Double-backward**
|
| 1560 |
+
|
| 1561 |
+
If, on the other hand, a backward pass with ``create_graph=True`` is underway (in other words,
|
| 1562 |
+
if you are setting up for a double-backward), each function's execution during backward
|
| 1563 |
+
is given a nonzero, useful ``seq=<N>``. Those functions may themselves create Function objects
|
| 1564 |
+
to be executed later during double-backward, just as the original functions in the forward pass did.
|
| 1565 |
+
The relationship between backward and double-backward is conceptually the same as the relationship
|
| 1566 |
+
between forward and backward: The functions still emit current-sequence-number-tagged ranges,
|
| 1567 |
+
the Function objects they create still stash those sequence numbers, and during the eventual
|
| 1568 |
+
double-backward, the Function objects' ``apply()`` ranges are still tagged with ``stashed seq``
|
| 1569 |
+
numbers, which can be compared to `seq` numbers from the backward pass.
|
| 1570 |
+
|
| 1571 |
+
.. warning:
|
| 1572 |
+
The sequence number is thread-local, and some forward functions don't create an associated
|
| 1573 |
+
backward Function object (instead delegating that to sub-functions further down the call chain).
|
| 1574 |
+
For these reasons, the correspondence of stashed sequence numbers in
|
| 1575 |
+
backward Function ``apply()`` ranges with `seq` numbers in forward-pass ranges is
|
| 1576 |
+
not guaranteed to be 1 to 1. The sequence numbers alone may not be enough to fully
|
| 1577 |
+
disambiguate which forward function created which
|
| 1578 |
+
backward Function object. You may need to make a judgment based on analytic knowledge of what
|
| 1579 |
+
the expected correspondence should be.
|
| 1580 |
+
"""
|
| 1581 |
+
|
| 1582 |
+
def __init__(self, enabled=True, record_shapes=False):
|
| 1583 |
+
self.enabled = enabled
|
| 1584 |
+
self.entered = False
|
| 1585 |
+
self.record_shapes = record_shapes
|
| 1586 |
+
|
| 1587 |
+
def __enter__(self):
|
| 1588 |
+
if not self.enabled:
|
| 1589 |
+
return
|
| 1590 |
+
if self.entered:
|
| 1591 |
+
raise RuntimeError("NVTX annotation context manager is not reentrant")
|
| 1592 |
+
self.entered = True
|
| 1593 |
+
torch.cuda.synchronize()
|
| 1594 |
+
_run_on_profiler_start()
|
| 1595 |
+
_enable_profiler(
|
| 1596 |
+
ProfilerConfig(
|
| 1597 |
+
ProfilerState.NVTX,
|
| 1598 |
+
self.record_shapes,
|
| 1599 |
+
False,
|
| 1600 |
+
False,
|
| 1601 |
+
False,
|
| 1602 |
+
False,
|
| 1603 |
+
_ExperimentalConfig(),
|
| 1604 |
+
),
|
| 1605 |
+
set(),
|
| 1606 |
+
)
|
| 1607 |
+
return self
|
| 1608 |
+
|
| 1609 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 1610 |
+
if not self.enabled:
|
| 1611 |
+
return
|
| 1612 |
+
torch.cuda.synchronize()
|
| 1613 |
+
_disable_profiler()
|
| 1614 |
+
_run_on_profiler_stop()
|
| 1615 |
+
return False
|
| 1616 |
+
|
| 1617 |
+
|
| 1618 |
+
def load_nvprof(path):
|
| 1619 |
+
"""Open an nvprof trace file and parses autograd annotations.
|
| 1620 |
+
|
| 1621 |
+
Args:
|
| 1622 |
+
path (str): path to nvprof trace
|
| 1623 |
+
"""
|
| 1624 |
+
return EventList(parse_nvprof_trace(path))
|
| 1625 |
+
|
| 1626 |
+
|
| 1627 |
+
class EnforceUnique:
|
| 1628 |
+
"""Raises an error if a key is seen more than once."""
|
| 1629 |
+
|
| 1630 |
+
def __init__(self):
|
| 1631 |
+
self.seen = set()
|
| 1632 |
+
|
| 1633 |
+
def see(self, *key):
|
| 1634 |
+
r"""
|
| 1635 |
+
Observe a key and raise an error if it is seen multiple times.
|
| 1636 |
+
"""
|
| 1637 |
+
if key in self.seen:
|
| 1638 |
+
raise RuntimeError("duplicate key: " + str(key))
|
| 1639 |
+
self.seen.add(key)
|
| 1640 |
+
|
| 1641 |
+
|
| 1642 |
+
def parse_nvprof_trace(path):
|
| 1643 |
+
import sqlite3
|
| 1644 |
+
|
| 1645 |
+
conn = sqlite3.connect(path)
|
| 1646 |
+
conn.row_factory = sqlite3.Row
|
| 1647 |
+
|
| 1648 |
+
# Parse strings table
|
| 1649 |
+
strings = {}
|
| 1650 |
+
for r in conn.execute("SELECT _id_ as id, value FROM StringTable"):
|
| 1651 |
+
strings[r["id"]] = torch._C._demangle(r["value"])
|
| 1652 |
+
|
| 1653 |
+
# First, find all functions and create FunctionEvents for them
|
| 1654 |
+
marker_query = """
|
| 1655 |
+
SELECT
|
| 1656 |
+
start.id AS marker_id, start.name, start.timestamp AS start_time, end.timestamp AS end_time
|
| 1657 |
+
FROM
|
| 1658 |
+
CUPTI_ACTIVITY_KIND_MARKER AS start INNER JOIN CUPTI_ACTIVITY_KIND_MARKER AS end
|
| 1659 |
+
ON start.id = end.id
|
| 1660 |
+
WHERE
|
| 1661 |
+
start.name != 0 AND end.name = 0
|
| 1662 |
+
"""
|
| 1663 |
+
functions = []
|
| 1664 |
+
functions_map = {}
|
| 1665 |
+
unique = EnforceUnique()
|
| 1666 |
+
for row in conn.execute(marker_query):
|
| 1667 |
+
unique.see(row["marker_id"])
|
| 1668 |
+
evt = FunctionEvent(
|
| 1669 |
+
id=row["marker_id"],
|
| 1670 |
+
node_id=0, # missing a node_id when calling FunctionEvent. This is just to ensure
|
| 1671 |
+
# that pytorch doesn't crash when creating a FunctionEvent() object
|
| 1672 |
+
name=strings[row["name"]],
|
| 1673 |
+
start_us=row["start_time"],
|
| 1674 |
+
end_us=row["end_time"],
|
| 1675 |
+
thread=0,
|
| 1676 |
+
) # TODO: find in sqlite database
|
| 1677 |
+
functions.append(evt)
|
| 1678 |
+
functions_map[evt.id] = evt
|
| 1679 |
+
|
| 1680 |
+
# Now, correlate all kernels with FunctionEvents
|
| 1681 |
+
kernel_query = """
|
| 1682 |
+
SELECT
|
| 1683 |
+
start.id AS marker_id, start.name, start.timestamp, end.timestamp,
|
| 1684 |
+
runtime._id_ AS runtime_id, runtime.cbid, runtime.start AS runtime_start, runtime.end AS runtime_end,
|
| 1685 |
+
kernel.start AS kernel_start, kernel.end AS kernel_end, kernel.name AS kernel_name
|
| 1686 |
+
FROM
|
| 1687 |
+
CUPTI_ACTIVITY_KIND_MARKER AS start
|
| 1688 |
+
INNER JOIN CUPTI_ACTIVITY_KIND_MARKER AS end
|
| 1689 |
+
ON start.id = end.id
|
| 1690 |
+
INNER JOIN CUPTI_ACTIVITY_KIND_RUNTIME as runtime
|
| 1691 |
+
ON (start.timestamp < runtime.start AND runtime.end < end.timestamp)
|
| 1692 |
+
INNER JOIN CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL AS kernel
|
| 1693 |
+
ON kernel.correlationId = runtime.correlationId
|
| 1694 |
+
"""
|
| 1695 |
+
unique = EnforceUnique()
|
| 1696 |
+
for row in conn.execute(kernel_query):
|
| 1697 |
+
unique.see(row["marker_id"], row["runtime_id"])
|
| 1698 |
+
# 211 is cudaKernelLaunch for cuda >= 9.2
|
| 1699 |
+
if row["cbid"] != 211:
|
| 1700 |
+
raise AssertionError(f"Expected cbid to be 211, but got {row['cbid']}")
|
| 1701 |
+
evt = functions_map[row["marker_id"]]
|
| 1702 |
+
evt.append_kernel(
|
| 1703 |
+
row["kernel_name"], 0, row["kernel_end"] - row["kernel_start"]
|
| 1704 |
+
)
|
| 1705 |
+
|
| 1706 |
+
functions.sort(key=lambda evt: evt.time_range.start)
|
| 1707 |
+
return functions
|
| 1708 |
+
|
| 1709 |
+
|
| 1710 |
+
class KinetoStepTracker:
|
| 1711 |
+
"""Provides an abstraction for incrementing the step count globally.
|
| 1712 |
+
|
| 1713 |
+
Previously, we only had one place to mark that a step() has occurred
|
| 1714 |
+
in the program via pytorch profiler step(). We will now add step hooks
|
| 1715 |
+
in the Optimizer class https://github.com/pytorch/pytorch/issues/88446
|
| 1716 |
+
|
| 1717 |
+
- This could mean programs that already call profiler.step() every
|
| 1718 |
+
iteration can end up double incrementing step count.
|
| 1719 |
+
- If a model uses multiple optimizers we can also have double or more
|
| 1720 |
+
counting of the step.
|
| 1721 |
+
|
| 1722 |
+
We fix this by adding a layer of abstraction before calling step()
|
| 1723 |
+
to the kineto library. The idea is to maintain steps per requester in a dict:
|
| 1724 |
+
|
| 1725 |
+
.. code-block::
|
| 1726 |
+
|
| 1727 |
+
{
|
| 1728 |
+
"ProfilerStep": 100, # triggered by profiler step() call
|
| 1729 |
+
"Optimizer1Step": 100, # Optimizer 1 or 2 are just examples, could be SGD, Adam etc
|
| 1730 |
+
"Optimizer2Step": 100,
|
| 1731 |
+
}
|
| 1732 |
+
|
| 1733 |
+
To figure out the global step count just take the max of dict values (100).
|
| 1734 |
+
|
| 1735 |
+
If one of the count increments the max will go up.
|
| 1736 |
+
|
| 1737 |
+
.. code-block::
|
| 1738 |
+
|
| 1739 |
+
{
|
| 1740 |
+
"ProfilerStep": 100,
|
| 1741 |
+
"Optimizer1Step": 101, # Optimizer1 got incremented first say
|
| 1742 |
+
"Optimizer2Step": 100,
|
| 1743 |
+
}
|
| 1744 |
+
|
| 1745 |
+
Then global step count is 101
|
| 1746 |
+
We only call the kineto step() function when global count increments.
|
| 1747 |
+
|
| 1748 |
+
NOTE: Please do not use the KinetoStepTracker in modules beside the Optimizer
|
| 1749 |
+
for now. The result could be incorrect increments of the step count.
|
| 1750 |
+
"""
|
| 1751 |
+
|
| 1752 |
+
_current_step = 0
|
| 1753 |
+
_step_dict: dict[str, int] = defaultdict(int)
|
| 1754 |
+
|
| 1755 |
+
@classmethod
|
| 1756 |
+
def init_step_count(cls, requester: str):
|
| 1757 |
+
r"""
|
| 1758 |
+
Initialize for a given requester.
|
| 1759 |
+
"""
|
| 1760 |
+
cls._step_dict[requester] = cls._current_step
|
| 1761 |
+
|
| 1762 |
+
@classmethod
|
| 1763 |
+
def erase_step_count(cls, requester: str) -> bool:
|
| 1764 |
+
r"""
|
| 1765 |
+
Remove a given requester.
|
| 1766 |
+
"""
|
| 1767 |
+
return cls._step_dict.pop(requester, None) is not None
|
| 1768 |
+
|
| 1769 |
+
@classmethod
|
| 1770 |
+
def increment_step(cls, requester: str) -> int:
|
| 1771 |
+
"""Increments the step count for the requester.
|
| 1772 |
+
|
| 1773 |
+
Additionally if the max over all step counts has incremented then
|
| 1774 |
+
trigger the _kineto_step() returns global step count
|
| 1775 |
+
"""
|
| 1776 |
+
if requester not in cls._step_dict:
|
| 1777 |
+
cls.init_step_count(requester)
|
| 1778 |
+
cls._step_dict[requester] += 1
|
| 1779 |
+
|
| 1780 |
+
new_step = max(cls._step_dict.values())
|
| 1781 |
+
if new_step > cls._current_step:
|
| 1782 |
+
delta = new_step - cls._current_step
|
| 1783 |
+
if delta > 1:
|
| 1784 |
+
warn(
|
| 1785 |
+
"Profiler step count has increased more than 1 - "
|
| 1786 |
+
f"current_step = {cls._current_step} step dict = {cls._step_dict}",
|
| 1787 |
+
stacklevel=2,
|
| 1788 |
+
)
|
| 1789 |
+
for _ in range(delta):
|
| 1790 |
+
_kineto_step()
|
| 1791 |
+
cls._current_step = new_step
|
| 1792 |
+
return cls._current_step
|
| 1793 |
+
|
| 1794 |
+
@classmethod
|
| 1795 |
+
def current_step(cls) -> int:
|
| 1796 |
+
r"""
|
| 1797 |
+
Get the latest step for any requester
|
| 1798 |
+
"""
|
| 1799 |
+
return cls._current_step
|
western_ai/usa/pytorch/torch/autograd/profiler_legacy.py
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import itertools
|
| 3 |
+
import warnings
|
| 4 |
+
from typing_extensions import deprecated
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
import torch.cuda
|
| 8 |
+
from torch.autograd import (
|
| 9 |
+
_disable_profiler_legacy,
|
| 10 |
+
_enable_profiler_legacy,
|
| 11 |
+
DeviceType,
|
| 12 |
+
ProfilerConfig,
|
| 13 |
+
ProfilerState,
|
| 14 |
+
)
|
| 15 |
+
from torch.autograd.profiler_util import (
|
| 16 |
+
_filter_name,
|
| 17 |
+
_filter_stack_entry,
|
| 18 |
+
_rewrite_name,
|
| 19 |
+
EventList,
|
| 20 |
+
FunctionEvent,
|
| 21 |
+
MEMORY_EVENT_NAME,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
__all__ = ["profile"]
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@deprecated(
|
| 29 |
+
"`torch.autograd.profiler_legacy.profile` is deprecated and will be removed in a future release. "
|
| 30 |
+
"Please use `torch.profiler` instead.",
|
| 31 |
+
category=None, # TODO: change to `FutureWarning`
|
| 32 |
+
)
|
| 33 |
+
class profile:
|
| 34 |
+
"""DEPRECATED: use torch.profiler instead."""
|
| 35 |
+
|
| 36 |
+
def __init__(
|
| 37 |
+
self,
|
| 38 |
+
enabled=True,
|
| 39 |
+
*,
|
| 40 |
+
use_cuda=False,
|
| 41 |
+
record_shapes=False,
|
| 42 |
+
with_flops=False,
|
| 43 |
+
profile_memory=False,
|
| 44 |
+
with_stack=False,
|
| 45 |
+
with_modules=False,
|
| 46 |
+
):
|
| 47 |
+
self.enabled: bool = enabled
|
| 48 |
+
if not self.enabled:
|
| 49 |
+
return
|
| 50 |
+
self.use_cuda = use_cuda
|
| 51 |
+
self.function_events = None
|
| 52 |
+
self.entered = False
|
| 53 |
+
self.record_shapes = record_shapes
|
| 54 |
+
self.with_flops = with_flops
|
| 55 |
+
self.record_shapes |= self.with_flops
|
| 56 |
+
self.profile_memory = profile_memory
|
| 57 |
+
self.with_stack = with_stack
|
| 58 |
+
self.with_modules = with_modules
|
| 59 |
+
|
| 60 |
+
if self.use_cuda and not torch.cuda.is_available():
|
| 61 |
+
warnings.warn(
|
| 62 |
+
"CUDA is not available, disabling CUDA profiling",
|
| 63 |
+
stacklevel=2,
|
| 64 |
+
)
|
| 65 |
+
self.use_cuda = False
|
| 66 |
+
|
| 67 |
+
if self.use_cuda:
|
| 68 |
+
self.profiler_kind = ProfilerState.CUDA
|
| 69 |
+
else:
|
| 70 |
+
self.profiler_kind = ProfilerState.CPU
|
| 71 |
+
|
| 72 |
+
def config(self):
|
| 73 |
+
return ProfilerConfig(
|
| 74 |
+
self.profiler_kind,
|
| 75 |
+
self.record_shapes,
|
| 76 |
+
self.profile_memory,
|
| 77 |
+
self.with_stack,
|
| 78 |
+
self.with_flops,
|
| 79 |
+
self.with_modules,
|
| 80 |
+
# avoid exposing _ExperimentalConfig this in legacy public API
|
| 81 |
+
torch._C._profiler._ExperimentalConfig(),
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
def __enter__(self):
|
| 85 |
+
if not self.enabled:
|
| 86 |
+
return
|
| 87 |
+
if self.entered:
|
| 88 |
+
raise RuntimeError("Profiler context manager is not reentrant")
|
| 89 |
+
self.entered = True
|
| 90 |
+
self._start_trace()
|
| 91 |
+
return self
|
| 92 |
+
|
| 93 |
+
def _start_trace(self):
|
| 94 |
+
_enable_profiler_legacy(self.config())
|
| 95 |
+
|
| 96 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 97 |
+
if not self.enabled:
|
| 98 |
+
return
|
| 99 |
+
if self.use_cuda:
|
| 100 |
+
torch.cuda.synchronize()
|
| 101 |
+
|
| 102 |
+
records = _disable_profiler_legacy()
|
| 103 |
+
parsed_results = _parse_legacy_records(records)
|
| 104 |
+
# pyrefly: ignore [bad-assignment]
|
| 105 |
+
self.function_events = EventList(
|
| 106 |
+
parsed_results,
|
| 107 |
+
use_device="cuda" if self.use_cuda else None,
|
| 108 |
+
profile_memory=self.profile_memory,
|
| 109 |
+
with_flops=self.with_flops,
|
| 110 |
+
)
|
| 111 |
+
# pyrefly: ignore [missing-attribute]
|
| 112 |
+
self.function_events._build_tree()
|
| 113 |
+
return False
|
| 114 |
+
|
| 115 |
+
def __repr__(self):
|
| 116 |
+
if self.function_events is None:
|
| 117 |
+
return "<unfinished profiler_legacy.profile>"
|
| 118 |
+
return repr(self.function_events)
|
| 119 |
+
|
| 120 |
+
def __str__(self):
|
| 121 |
+
if self.function_events is None:
|
| 122 |
+
return "<unfinished profile.profiler_legacy.profile>"
|
| 123 |
+
return str(self.function_events)
|
| 124 |
+
|
| 125 |
+
def _check_finish(self):
|
| 126 |
+
if self.function_events is None:
|
| 127 |
+
raise RuntimeError("Profiler didn't finish running")
|
| 128 |
+
|
| 129 |
+
def table(
|
| 130 |
+
self,
|
| 131 |
+
sort_by=None,
|
| 132 |
+
row_limit=100,
|
| 133 |
+
max_src_column_width=75,
|
| 134 |
+
max_name_column_width=55,
|
| 135 |
+
max_shapes_column_width=80,
|
| 136 |
+
header=None,
|
| 137 |
+
top_level_events_only=False,
|
| 138 |
+
):
|
| 139 |
+
self._check_finish()
|
| 140 |
+
if self.function_events is None:
|
| 141 |
+
raise AssertionError("Expected profiling results")
|
| 142 |
+
return self.function_events.table(
|
| 143 |
+
sort_by=sort_by,
|
| 144 |
+
row_limit=row_limit,
|
| 145 |
+
max_src_column_width=max_src_column_width,
|
| 146 |
+
max_name_column_width=max_name_column_width,
|
| 147 |
+
max_shapes_column_width=max_shapes_column_width,
|
| 148 |
+
header=header,
|
| 149 |
+
top_level_events_only=top_level_events_only,
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
table.__doc__ = EventList.table.__doc__
|
| 153 |
+
|
| 154 |
+
def export_chrome_trace(self, path):
|
| 155 |
+
self._check_finish()
|
| 156 |
+
if self.function_events is None:
|
| 157 |
+
raise AssertionError("Expected profiling results")
|
| 158 |
+
return self.function_events.export_chrome_trace(path)
|
| 159 |
+
|
| 160 |
+
export_chrome_trace.__doc__ = EventList.export_chrome_trace.__doc__
|
| 161 |
+
|
| 162 |
+
def export_stacks(self, path: str, metric: str = "self_cpu_time_total"):
|
| 163 |
+
self._check_finish()
|
| 164 |
+
if self.function_events is None:
|
| 165 |
+
raise AssertionError("Expected profiling results")
|
| 166 |
+
if not self.with_stack:
|
| 167 |
+
raise AssertionError("export_stacks() requires with_stack=True")
|
| 168 |
+
return self.function_events.export_stacks(path, metric)
|
| 169 |
+
|
| 170 |
+
def key_averages(self, group_by_input_shape=False, group_by_stack_n=0):
|
| 171 |
+
self._check_finish()
|
| 172 |
+
if self.function_events is None:
|
| 173 |
+
raise AssertionError("Expected profiling results")
|
| 174 |
+
return self.function_events.key_averages(group_by_input_shape, group_by_stack_n)
|
| 175 |
+
|
| 176 |
+
key_averages.__doc__ = EventList.key_averages.__doc__
|
| 177 |
+
|
| 178 |
+
def total_average(self):
|
| 179 |
+
self._check_finish()
|
| 180 |
+
if self.function_events is None:
|
| 181 |
+
raise AssertionError("Expected profiling results")
|
| 182 |
+
return self.function_events.total_average()
|
| 183 |
+
|
| 184 |
+
total_average.__doc__ = EventList.total_average.__doc__
|
| 185 |
+
|
| 186 |
+
@property
|
| 187 |
+
def self_cpu_time_total(self):
|
| 188 |
+
"""Return CPU time as the sum of self times across all events."""
|
| 189 |
+
self._check_finish()
|
| 190 |
+
if self.function_events is None:
|
| 191 |
+
raise AssertionError("Expected profiling results")
|
| 192 |
+
return self.function_events.self_cpu_time_total
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def _parse_legacy_records(thread_records):
|
| 196 |
+
def _get_record_key(record):
|
| 197 |
+
"""Return a tuple for correlating start and end records in `_parse_legacy_records`."""
|
| 198 |
+
return (record.handle(), record.node_id())
|
| 199 |
+
|
| 200 |
+
start_record = None
|
| 201 |
+
functions = []
|
| 202 |
+
|
| 203 |
+
# '__start_profile' is not guaranteed to be first, so we must find it here
|
| 204 |
+
for record in itertools.chain.from_iterable(thread_records):
|
| 205 |
+
name = record.name()
|
| 206 |
+
if start_record is None and name == "__start_profile":
|
| 207 |
+
start_record = record
|
| 208 |
+
|
| 209 |
+
if start_record is None or start_record.is_remote():
|
| 210 |
+
raise AssertionError("Expected a valid local start_record")
|
| 211 |
+
|
| 212 |
+
for thread_record_list in thread_records:
|
| 213 |
+
# accumulated memory allocations per handle
|
| 214 |
+
cpu_memory_allocs = {}
|
| 215 |
+
cuda_memory_allocs = {}
|
| 216 |
+
# ranges per handle
|
| 217 |
+
range_starts = {}
|
| 218 |
+
|
| 219 |
+
filtered_handles = set()
|
| 220 |
+
prev_record = None
|
| 221 |
+
for record in thread_record_list:
|
| 222 |
+
record_key = _get_record_key(record)
|
| 223 |
+
if _filter_name(record.name()) or record_key in filtered_handles:
|
| 224 |
+
filtered_handles.add(record_key)
|
| 225 |
+
continue
|
| 226 |
+
|
| 227 |
+
if record.kind() == "push":
|
| 228 |
+
# workaround to reduce double logging from operator
|
| 229 |
+
# wrappers and redispatch
|
| 230 |
+
if prev_record is not None:
|
| 231 |
+
duplicate = (
|
| 232 |
+
prev_record.name() == record.name()
|
| 233 |
+
and prev_record.kind() == record.kind()
|
| 234 |
+
and prev_record.node_id() == record.node_id()
|
| 235 |
+
)
|
| 236 |
+
if duplicate:
|
| 237 |
+
filtered_handles.add(record_key)
|
| 238 |
+
continue
|
| 239 |
+
|
| 240 |
+
range_starts[record_key] = record
|
| 241 |
+
cpu_memory_allocs[record_key] = 0
|
| 242 |
+
cuda_memory_allocs[record_key] = 0
|
| 243 |
+
elif record.kind() == "pop":
|
| 244 |
+
if record_key not in range_starts:
|
| 245 |
+
raise AssertionError(
|
| 246 |
+
f"Expected record with key {record_key} to exist in range_starts. "
|
| 247 |
+
"This means that the pop event did not have a corresponding push."
|
| 248 |
+
)
|
| 249 |
+
|
| 250 |
+
start = range_starts[record_key]
|
| 251 |
+
|
| 252 |
+
cpu_memory_usage = cpu_memory_allocs[record_key]
|
| 253 |
+
cuda_memory_usage = cuda_memory_allocs[record_key]
|
| 254 |
+
is_async = start.is_async() or (start.thread_id() != record.thread_id())
|
| 255 |
+
is_remote_event = record.is_remote()
|
| 256 |
+
start_flops = start.flops()
|
| 257 |
+
|
| 258 |
+
fe = FunctionEvent(
|
| 259 |
+
id=record.handle(),
|
| 260 |
+
node_id=record.node_id(),
|
| 261 |
+
name=_rewrite_name(name=start.name(), with_wildcard=True),
|
| 262 |
+
trace_name=_rewrite_name(name=start.name(), with_wildcard=False),
|
| 263 |
+
thread=start.thread_id(),
|
| 264 |
+
start_us=start_record.cpu_elapsed_us(start),
|
| 265 |
+
end_us=start_record.cpu_elapsed_us(record),
|
| 266 |
+
fwd_thread=start.fwd_thread_id(),
|
| 267 |
+
input_shapes=start.shapes(),
|
| 268 |
+
stack=[
|
| 269 |
+
entry for entry in start.stack() if _filter_stack_entry(entry)
|
| 270 |
+
],
|
| 271 |
+
scope=start.scope(),
|
| 272 |
+
use_device="cuda" if start.has_cuda() else None,
|
| 273 |
+
cpu_memory_usage=cpu_memory_usage,
|
| 274 |
+
device_memory_usage=cuda_memory_usage,
|
| 275 |
+
is_async=is_async,
|
| 276 |
+
is_remote=is_remote_event,
|
| 277 |
+
sequence_nr=start.sequence_nr(),
|
| 278 |
+
device_type=DeviceType.CPU,
|
| 279 |
+
is_legacy=True,
|
| 280 |
+
flops=start_flops,
|
| 281 |
+
)
|
| 282 |
+
# note: async events have only cpu total time
|
| 283 |
+
if not is_async and start.has_cuda():
|
| 284 |
+
duration = start.cuda_elapsed_us(record)
|
| 285 |
+
if duration > 0:
|
| 286 |
+
fe.append_kernel(start.name(), start.device(), duration)
|
| 287 |
+
functions.append(fe)
|
| 288 |
+
del range_starts[record_key]
|
| 289 |
+
del cpu_memory_allocs[record_key]
|
| 290 |
+
del cuda_memory_allocs[record_key]
|
| 291 |
+
elif record.kind() == "memory_alloc":
|
| 292 |
+
num_open_handles_cpu = len(cpu_memory_allocs)
|
| 293 |
+
num_open_handles_cuda = len(cuda_memory_allocs)
|
| 294 |
+
if num_open_handles_cpu != num_open_handles_cuda:
|
| 295 |
+
raise AssertionError(
|
| 296 |
+
f"Expected CPU and CUDA memory allocation handles to match, "
|
| 297 |
+
f"but got {num_open_handles_cpu} CPU and {num_open_handles_cuda} CUDA"
|
| 298 |
+
)
|
| 299 |
+
for handle in cpu_memory_allocs:
|
| 300 |
+
cpu_memory_allocs[handle] += record.cpu_memory_usage()
|
| 301 |
+
for handle in cuda_memory_allocs:
|
| 302 |
+
cuda_memory_allocs[handle] += record.cuda_memory_usage()
|
| 303 |
+
if num_open_handles_cpu == 0:
|
| 304 |
+
# output event as a top-level memory event
|
| 305 |
+
fe = FunctionEvent(
|
| 306 |
+
id=0,
|
| 307 |
+
name=MEMORY_EVENT_NAME,
|
| 308 |
+
trace_name=None,
|
| 309 |
+
thread=0,
|
| 310 |
+
start_us=0,
|
| 311 |
+
end_us=0,
|
| 312 |
+
stack=[],
|
| 313 |
+
cpu_memory_usage=record.cpu_memory_usage(),
|
| 314 |
+
device_memory_usage=record.cuda_memory_usage(),
|
| 315 |
+
is_legacy=True,
|
| 316 |
+
)
|
| 317 |
+
functions.append(fe)
|
| 318 |
+
prev_record = record
|
| 319 |
+
|
| 320 |
+
# Sort functions by start time then by end time ascending.
|
| 321 |
+
# This ensures that--in the case of nested events which
|
| 322 |
+
# have the same start time (which may happen due to the
|
| 323 |
+
# granularity of the given clock tick)--we always show
|
| 324 |
+
# the outermost nested call first. This adds stability
|
| 325 |
+
# in how FunctionEvents appear
|
| 326 |
+
functions.sort(key=lambda evt: [evt.time_range.start, -evt.time_range.end])
|
| 327 |
+
return functions
|
western_ai/usa/pytorch/torch/autograd/profiler_util.py
ADDED
|
@@ -0,0 +1,1617 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import bisect
|
| 3 |
+
import itertools
|
| 4 |
+
import json
|
| 5 |
+
import math
|
| 6 |
+
from collections import defaultdict, namedtuple
|
| 7 |
+
from collections.abc import Callable
|
| 8 |
+
from operator import attrgetter
|
| 9 |
+
from typing import Any, NamedTuple
|
| 10 |
+
from typing_extensions import deprecated
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
from torch.autograd import DeviceType
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
__all__ = [
|
| 17 |
+
"EventList",
|
| 18 |
+
"EventMetadata",
|
| 19 |
+
"FormattedTimesMixin",
|
| 20 |
+
"Interval",
|
| 21 |
+
"Kernel",
|
| 22 |
+
"FunctionEvent",
|
| 23 |
+
"FunctionEventAvg",
|
| 24 |
+
"StringTable",
|
| 25 |
+
"MemRecordsAcc",
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class EventList(list):
|
| 30 |
+
"""A list of profiling events with helper methods for analysis and visualization.
|
| 31 |
+
|
| 32 |
+
EventList extends the standard Python list to provide specialized methods for
|
| 33 |
+
working with profiling events (FunctionEvent or FunctionEventAvg objects).
|
| 34 |
+
It includes utilities for aggregating statistics, formatting output tables,
|
| 35 |
+
and exporting profiling data.
|
| 36 |
+
|
| 37 |
+
This class is typically returned by profiler methods and should not be
|
| 38 |
+
instantiated directly by users.
|
| 39 |
+
|
| 40 |
+
Args:
|
| 41 |
+
*args: Standard list arguments.
|
| 42 |
+
use_device (str, optional): Device type for profiling ("cuda", "xpu", etc.).
|
| 43 |
+
profile_memory (bool, optional): Whether memory profiling was enabled. Default: False.
|
| 44 |
+
with_flops (bool, optional): Whether to include FLOP counts. Default: False.
|
| 45 |
+
|
| 46 |
+
Attributes:
|
| 47 |
+
_use_device (str): Device type being profiled.
|
| 48 |
+
_profile_memory (bool): Whether memory profiling is enabled.
|
| 49 |
+
_with_flops (bool): Whether FLOP counting is enabled.
|
| 50 |
+
_tree_built (bool): Whether the event tree structure has been built.
|
| 51 |
+
|
| 52 |
+
Key Methods:
|
| 53 |
+
table(...): Format events as a table string for display.
|
| 54 |
+
export_chrome_trace(path): Export to Chrome tracing format.
|
| 55 |
+
export_stacks(path, metric): Export stack traces with metrics.
|
| 56 |
+
key_averages(...): Compute averaged statistics grouped by operation name.
|
| 57 |
+
total_average(): Compute aggregate totals across all events (sums, not averages).
|
| 58 |
+
|
| 59 |
+
Properties:
|
| 60 |
+
self_cpu_time_total: Sum of self CPU time across all events.
|
| 61 |
+
|
| 62 |
+
Example::
|
| 63 |
+
|
| 64 |
+
import torch
|
| 65 |
+
from torch.profiler import profile, ProfilerActivity
|
| 66 |
+
|
| 67 |
+
with profile(activities=[ProfilerActivity.CPU]) as prof:
|
| 68 |
+
x = torch.randn(100, 100)
|
| 69 |
+
y = torch.matmul(x, x)
|
| 70 |
+
|
| 71 |
+
# EventList is returned by prof.events()
|
| 72 |
+
events = prof.events()
|
| 73 |
+
|
| 74 |
+
# Display as formatted table
|
| 75 |
+
print(
|
| 76 |
+
events.table(
|
| 77 |
+
sort_by="cpu_time_total", row_limit=20, top_level_events_only=False
|
| 78 |
+
)
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
# Export to Chrome tracing format
|
| 82 |
+
events.export_chrome_trace("trace.json")
|
| 83 |
+
|
| 84 |
+
# Get averaged statistics
|
| 85 |
+
avg_events = events.key_averages()
|
| 86 |
+
print(avg_events.table())
|
| 87 |
+
|
| 88 |
+
# Export stack traces
|
| 89 |
+
events.export_stacks("stacks.txt", "self_cpu_time_total")
|
| 90 |
+
|
| 91 |
+
See Also:
|
| 92 |
+
- :class:`FunctionEvent`: Individual profiling event
|
| 93 |
+
- :class:`FunctionEventAvg`: Averaged profiling statistics
|
| 94 |
+
- :meth:`table`: Format events as a readable table
|
| 95 |
+
- :meth:`key_averages`: Aggregate events by operation name
|
| 96 |
+
"""
|
| 97 |
+
|
| 98 |
+
def __init__(self, *args, **kwargs):
|
| 99 |
+
use_device = kwargs.pop("use_device", None)
|
| 100 |
+
profile_memory = kwargs.pop("profile_memory", False)
|
| 101 |
+
with_flops = kwargs.pop("with_flops", False)
|
| 102 |
+
# pyrefly: ignore [not-iterable]
|
| 103 |
+
super().__init__(*args, **kwargs)
|
| 104 |
+
self._use_device = use_device
|
| 105 |
+
self._profile_memory = profile_memory
|
| 106 |
+
self._tree_built = False
|
| 107 |
+
self._with_flops = with_flops
|
| 108 |
+
|
| 109 |
+
def _build_tree(self):
|
| 110 |
+
self._populate_cpu_children()
|
| 111 |
+
self._remove_dup_nodes()
|
| 112 |
+
self._set_backward_stacktraces()
|
| 113 |
+
self._tree_built = True
|
| 114 |
+
|
| 115 |
+
def __str__(self):
|
| 116 |
+
return self.table()
|
| 117 |
+
|
| 118 |
+
def _remove_dup_nodes(self):
|
| 119 |
+
while True:
|
| 120 |
+
to_delete = set()
|
| 121 |
+
|
| 122 |
+
for idx in range(len(self)):
|
| 123 |
+
if (
|
| 124 |
+
self[idx].cpu_parent is not None
|
| 125 |
+
and self[idx].cpu_parent.name == self[idx].name
|
| 126 |
+
and len(self[idx].cpu_parent.cpu_children) == 1
|
| 127 |
+
):
|
| 128 |
+
self[idx].cpu_parent.cpu_children = self[idx].cpu_children
|
| 129 |
+
self[idx].cpu_parent.kernels = self[idx].kernels # lift kernels up
|
| 130 |
+
for ch in self[idx].cpu_children:
|
| 131 |
+
ch.cpu_parent = self[idx].cpu_parent
|
| 132 |
+
to_delete.add(idx)
|
| 133 |
+
if len(to_delete) == 0:
|
| 134 |
+
break
|
| 135 |
+
|
| 136 |
+
new_evts = [ev for ind, ev in enumerate(self) if ind not in to_delete]
|
| 137 |
+
|
| 138 |
+
self.clear()
|
| 139 |
+
|
| 140 |
+
self.extend(new_evts)
|
| 141 |
+
|
| 142 |
+
def _populate_cpu_children(self):
|
| 143 |
+
"""Populate child events into each underlying FunctionEvent object.
|
| 144 |
+
|
| 145 |
+
One event is a child of another if [s1, e1) is inside [s2, e2). Where
|
| 146 |
+
s1 and e1 would be start and end of the child event's interval. And
|
| 147 |
+
s2 and e2 start and end of the parent event's interval
|
| 148 |
+
|
| 149 |
+
Example: In event list [[0, 10], [1, 3], [3, 4]] would have make [0, 10]
|
| 150 |
+
be a parent of two other intervals.
|
| 151 |
+
|
| 152 |
+
If for any reason two intervals intersect only partially, this function
|
| 153 |
+
will not record a parent child relationship between then.
|
| 154 |
+
"""
|
| 155 |
+
# Some events can be async (i.e. start and end on different threads),
|
| 156 |
+
# since it's generally undefined how to attribute children ranges to
|
| 157 |
+
# async ranges, we do not use them when calculating nested ranges and stats
|
| 158 |
+
sync_events = [
|
| 159 |
+
evt
|
| 160 |
+
for evt in self
|
| 161 |
+
if not evt.is_async and evt.device_type == DeviceType.CPU
|
| 162 |
+
]
|
| 163 |
+
events = sorted(
|
| 164 |
+
sync_events,
|
| 165 |
+
key=attrgetter("thread"),
|
| 166 |
+
)
|
| 167 |
+
# Group by both thread and node_id, so that events that happen to have
|
| 168 |
+
# the same thread_id but are from different nodes aren't incorrectly
|
| 169 |
+
# grouped together.
|
| 170 |
+
threads = itertools.groupby(
|
| 171 |
+
events, key=lambda event: (event.thread, event.node_id)
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
# For each thread we keep a stack of current nested parents.
|
| 175 |
+
# We maintain the invariant that each interval is a subset of all other
|
| 176 |
+
# intervals lower in the stack.
|
| 177 |
+
#
|
| 178 |
+
# First we sort the intervals by their start time. Then we iterate over them.
|
| 179 |
+
# Every time we see a new interval we remove several parents from
|
| 180 |
+
# the top until we restore the invariant. Then parent child relationship
|
| 181 |
+
# if recorded if the stack is not empty.
|
| 182 |
+
# Finally we add new interval to the list
|
| 183 |
+
#
|
| 184 |
+
# Algorithm has O(N * log(N)) complexity where N is number of
|
| 185 |
+
# intervals
|
| 186 |
+
for _thread_id, thread_events in threads:
|
| 187 |
+
thread_events_ = sorted(
|
| 188 |
+
thread_events,
|
| 189 |
+
key=lambda event: [event.time_range.start, -event.time_range.end],
|
| 190 |
+
)
|
| 191 |
+
current_events: list[FunctionEvent] = []
|
| 192 |
+
for event in thread_events_:
|
| 193 |
+
while len(current_events) > 0:
|
| 194 |
+
parent = current_events[-1]
|
| 195 |
+
if (
|
| 196 |
+
event.time_range.start >= parent.time_range.end
|
| 197 |
+
or event.time_range.end > parent.time_range.end
|
| 198 |
+
):
|
| 199 |
+
# this can't be a parent
|
| 200 |
+
current_events.pop()
|
| 201 |
+
else:
|
| 202 |
+
parent.append_cpu_child(event)
|
| 203 |
+
if event.cpu_parent is not None:
|
| 204 |
+
raise AssertionError(
|
| 205 |
+
f"There is already a CPU parent event for {event.key}"
|
| 206 |
+
)
|
| 207 |
+
event.set_cpu_parent(parent)
|
| 208 |
+
break
|
| 209 |
+
|
| 210 |
+
current_events.append(event)
|
| 211 |
+
|
| 212 |
+
def _set_backward_stacktraces(self):
|
| 213 |
+
def bw_parent(evt):
|
| 214 |
+
if evt is None:
|
| 215 |
+
return None
|
| 216 |
+
elif evt.scope == 1: # BACKWARD_FUNCTION
|
| 217 |
+
return evt
|
| 218 |
+
else:
|
| 219 |
+
return bw_parent(evt.cpu_parent)
|
| 220 |
+
|
| 221 |
+
fwd_stacks = {}
|
| 222 |
+
for evt in self:
|
| 223 |
+
if bw_parent(evt) is None and evt.stack is not None:
|
| 224 |
+
t = (evt.sequence_nr, evt.thread)
|
| 225 |
+
if t not in fwd_stacks:
|
| 226 |
+
fwd_stacks[t] = evt.stack
|
| 227 |
+
|
| 228 |
+
for evt in self:
|
| 229 |
+
p = bw_parent(evt)
|
| 230 |
+
if p is not None:
|
| 231 |
+
if p.fwd_thread is None:
|
| 232 |
+
raise AssertionError(
|
| 233 |
+
"Expected fwd_thread to be set for backward parent"
|
| 234 |
+
)
|
| 235 |
+
t = (p.sequence_nr, p.fwd_thread)
|
| 236 |
+
evt.stack = fwd_stacks.get(t, [])
|
| 237 |
+
|
| 238 |
+
@property
|
| 239 |
+
def self_cpu_time_total(self):
|
| 240 |
+
return sum(event.self_cpu_time_total for event in self)
|
| 241 |
+
|
| 242 |
+
def table(
|
| 243 |
+
self,
|
| 244 |
+
sort_by=None,
|
| 245 |
+
row_limit=100,
|
| 246 |
+
max_src_column_width=75,
|
| 247 |
+
max_name_column_width=55,
|
| 248 |
+
max_shapes_column_width=80,
|
| 249 |
+
header=None,
|
| 250 |
+
top_level_events_only=False,
|
| 251 |
+
time_unit=None,
|
| 252 |
+
):
|
| 253 |
+
"""Print an EventList as a nicely formatted table.
|
| 254 |
+
|
| 255 |
+
Args:
|
| 256 |
+
sort_by (str, optional): Attribute used to sort entries. By default
|
| 257 |
+
they are printed in the same order as they were registered.
|
| 258 |
+
Valid keys include: ``cpu_time``, ``cuda_time``, ``xpu_time``,
|
| 259 |
+
``cpu_time_total``, ``cuda_time_total``, ``xpu_time_total``,
|
| 260 |
+
``cpu_memory_usage``, ``cuda_memory_usage``, ``xpu_memory_usage``,
|
| 261 |
+
``self_cpu_memory_usage``, ``self_cuda_memory_usage``,
|
| 262 |
+
``self_xpu_memory_usage``, ``count``.
|
| 263 |
+
top_level_events_only(bool, optional): Boolean flag to determine the
|
| 264 |
+
selection of events to display. If true, the profiler will only
|
| 265 |
+
display events at top level like top-level invocation of python
|
| 266 |
+
`lstm`, python `add` or other functions, nested events like low-level
|
| 267 |
+
cpu/cuda/xpu ops events are omitted for profiler result readability.
|
| 268 |
+
time_unit(str, optional): A time unit to be used for all values in the
|
| 269 |
+
table. Valid options are: ``s``, ``ms`` and ``us``.
|
| 270 |
+
|
| 271 |
+
Returns:
|
| 272 |
+
A string containing the table.
|
| 273 |
+
"""
|
| 274 |
+
return _build_table(
|
| 275 |
+
self,
|
| 276 |
+
sort_by=sort_by,
|
| 277 |
+
row_limit=row_limit,
|
| 278 |
+
max_src_column_width=max_src_column_width,
|
| 279 |
+
max_name_column_width=max_name_column_width,
|
| 280 |
+
max_shapes_column_width=max_shapes_column_width,
|
| 281 |
+
header=header,
|
| 282 |
+
profile_memory=self._profile_memory,
|
| 283 |
+
with_flops=self._with_flops,
|
| 284 |
+
top_level_events_only=top_level_events_only,
|
| 285 |
+
time_unit=time_unit,
|
| 286 |
+
)
|
| 287 |
+
|
| 288 |
+
def export_chrome_trace(self, path):
|
| 289 |
+
"""Export an EventList as a Chrome tracing tools file.
|
| 290 |
+
|
| 291 |
+
The checkpoint can be later loaded and inspected under ``chrome://tracing`` URL.
|
| 292 |
+
|
| 293 |
+
Args:
|
| 294 |
+
path (str): Path where the trace will be written.
|
| 295 |
+
"""
|
| 296 |
+
import os
|
| 297 |
+
|
| 298 |
+
device_name = "cuda" if not self._use_device else self._use_device
|
| 299 |
+
with open(path, "w") as f:
|
| 300 |
+
next_id = 0
|
| 301 |
+
# Use file IO over using json.dump since JSON dumping is very slow and
|
| 302 |
+
# this technique is proven to give a 4x speedup.
|
| 303 |
+
f.write("[")
|
| 304 |
+
for evt in self:
|
| 305 |
+
if evt.trace_name is None:
|
| 306 |
+
continue
|
| 307 |
+
f.write(
|
| 308 |
+
'{{"name": "{}", '
|
| 309 |
+
'"ph": "X", '
|
| 310 |
+
'"ts": {}, '
|
| 311 |
+
'"dur": {}, '
|
| 312 |
+
'"tid": {}, '
|
| 313 |
+
'"pid": "CPU functions", '
|
| 314 |
+
'"args": {{}}}}, '.format(
|
| 315 |
+
evt.trace_name,
|
| 316 |
+
evt.time_range.start,
|
| 317 |
+
evt.time_range.elapsed_us(),
|
| 318 |
+
evt.thread
|
| 319 |
+
if not evt.is_remote
|
| 320 |
+
else f'" node_id:{evt.node_id}, thread_id:{evt.thread} "',
|
| 321 |
+
)
|
| 322 |
+
)
|
| 323 |
+
for _ in evt.kernels:
|
| 324 |
+
# 's' and 'f' draw Flow arrows from
|
| 325 |
+
# the CPU launch to the GPU kernel
|
| 326 |
+
f.write(
|
| 327 |
+
f'{{"name": "{evt.trace_name}", '
|
| 328 |
+
'"ph": "s", '
|
| 329 |
+
f'"ts": {evt.time_range.start}, '
|
| 330 |
+
f'"tid": {evt.thread}, '
|
| 331 |
+
'"pid": "CPU functions", '
|
| 332 |
+
f'"id": {next_id}, '
|
| 333 |
+
f'"cat": "cpu_to_{device_name}", '
|
| 334 |
+
'"args": {}}, '
|
| 335 |
+
)
|
| 336 |
+
# Note: use torch.profiler to get device kernel trace
|
| 337 |
+
next_id += 1
|
| 338 |
+
if len(self) > 0:
|
| 339 |
+
# remove trailing whitespace and comma
|
| 340 |
+
f.seek(f.tell() - 2, os.SEEK_SET)
|
| 341 |
+
f.truncate()
|
| 342 |
+
f.write("]")
|
| 343 |
+
|
| 344 |
+
def supported_export_stacks_metrics(self):
|
| 345 |
+
return [
|
| 346 |
+
"self_cpu_time_total",
|
| 347 |
+
"self_cuda_time_total",
|
| 348 |
+
"self_xpu_time_total",
|
| 349 |
+
"self_privateuse1_time_total",
|
| 350 |
+
]
|
| 351 |
+
|
| 352 |
+
def export_stacks(self, path: str, metric: str):
|
| 353 |
+
if metric not in self.supported_export_stacks_metrics():
|
| 354 |
+
raise ValueError(
|
| 355 |
+
"metric should be one of: "
|
| 356 |
+
+ str(self.supported_export_stacks_metrics())
|
| 357 |
+
)
|
| 358 |
+
translate_table = str.maketrans(" ;\t\n", "____")
|
| 359 |
+
with open(path, "w") as f:
|
| 360 |
+
for evt in self:
|
| 361 |
+
if evt.stack and len(evt.stack) > 0:
|
| 362 |
+
metric_value = getattr(
|
| 363 |
+
evt,
|
| 364 |
+
metric.replace("cuda", "device")
|
| 365 |
+
.replace("xpu", "device")
|
| 366 |
+
.replace("privateuse1", "device"),
|
| 367 |
+
)
|
| 368 |
+
if int(metric_value) > 0:
|
| 369 |
+
stack_str = ""
|
| 370 |
+
for entry in reversed(evt.stack):
|
| 371 |
+
stack_str += entry.translate(translate_table)
|
| 372 |
+
stack_str += ";"
|
| 373 |
+
stack_str = stack_str[:-1] + " " + str(int(metric_value))
|
| 374 |
+
f.write(stack_str + "\n")
|
| 375 |
+
|
| 376 |
+
def key_averages(
|
| 377 |
+
self,
|
| 378 |
+
group_by_input_shapes=False,
|
| 379 |
+
group_by_stack_n=0,
|
| 380 |
+
group_by_overload_name=False,
|
| 381 |
+
include_python_functions=False,
|
| 382 |
+
):
|
| 383 |
+
"""Averages all function events over their keys.
|
| 384 |
+
|
| 385 |
+
Args:
|
| 386 |
+
group_by_input_shapes: group entries by
|
| 387 |
+
(event name, input shapes) rather than just event name.
|
| 388 |
+
This is useful to see which input shapes contribute to the runtime
|
| 389 |
+
the most and may help with size-specific optimizations or
|
| 390 |
+
choosing the best candidates for quantization (aka fitting a roof line)
|
| 391 |
+
|
| 392 |
+
group_by_stack_n: group by top n stack trace entries
|
| 393 |
+
|
| 394 |
+
group_by_overload_name: Differentiate operators by their overload name e.g. aten::add.Tensor
|
| 395 |
+
and aten::add.out will be aggregated separately
|
| 396 |
+
|
| 397 |
+
include_python_functions: include Python function events (e.g. individual
|
| 398 |
+
Python callsite entries captured with ``with_stack=True``) in the
|
| 399 |
+
averages. By default these are excluded because they tend to appear as
|
| 400 |
+
misleading hotspots (e.g. ``threading.py: wait``) that obscure the
|
| 401 |
+
real operator-level breakdown. Set to ``True`` to restore the raw
|
| 402 |
+
per-callsite view.
|
| 403 |
+
|
| 404 |
+
Returns:
|
| 405 |
+
An EventList containing FunctionEventAvg objects.
|
| 406 |
+
"""
|
| 407 |
+
if not self._tree_built:
|
| 408 |
+
raise AssertionError(
|
| 409 |
+
"Expected tree to be built before calling key_averages"
|
| 410 |
+
)
|
| 411 |
+
stats: dict[tuple[str, ...], FunctionEventAvg] = defaultdict(FunctionEventAvg)
|
| 412 |
+
|
| 413 |
+
def get_key(
|
| 414 |
+
event, group_by_input_shapes, group_by_stack_n, group_by_overload_name
|
| 415 |
+
) -> tuple[str, ...]:
|
| 416 |
+
key = [
|
| 417 |
+
str(event.key),
|
| 418 |
+
str(event.node_id),
|
| 419 |
+
str(event.device_type),
|
| 420 |
+
str(event.is_legacy),
|
| 421 |
+
str(event.is_user_annotation),
|
| 422 |
+
]
|
| 423 |
+
if group_by_overload_name:
|
| 424 |
+
key.append(evt.overload_name)
|
| 425 |
+
if group_by_input_shapes:
|
| 426 |
+
key.append(str(event.input_shapes))
|
| 427 |
+
if group_by_stack_n > 0:
|
| 428 |
+
key += event.stack[:group_by_stack_n]
|
| 429 |
+
return tuple(key)
|
| 430 |
+
|
| 431 |
+
for evt in self:
|
| 432 |
+
if evt.is_python_function and not include_python_functions:
|
| 433 |
+
continue
|
| 434 |
+
stats[
|
| 435 |
+
get_key(
|
| 436 |
+
evt, group_by_input_shapes, group_by_stack_n, group_by_overload_name
|
| 437 |
+
)
|
| 438 |
+
].add(evt)
|
| 439 |
+
|
| 440 |
+
avg_list = EventList(
|
| 441 |
+
stats.values(),
|
| 442 |
+
use_device=self._use_device,
|
| 443 |
+
profile_memory=self._profile_memory,
|
| 444 |
+
with_flops=self._with_flops,
|
| 445 |
+
)
|
| 446 |
+
for evt in avg_list:
|
| 447 |
+
evt.stack = evt.stack[:group_by_stack_n]
|
| 448 |
+
if not group_by_input_shapes:
|
| 449 |
+
evt.input_shapes = ""
|
| 450 |
+
if not group_by_overload_name:
|
| 451 |
+
evt.overload_name = ""
|
| 452 |
+
return avg_list
|
| 453 |
+
|
| 454 |
+
def total_average(self):
|
| 455 |
+
"""Compute aggregate statistics across all events.
|
| 456 |
+
|
| 457 |
+
Accumulates statistics from all events into a single FunctionEventAvg object.
|
| 458 |
+
This is primarily useful for computing total metrics (total CPU time, total
|
| 459 |
+
memory usage, etc.) across the entire profiling session, regardless of
|
| 460 |
+
operation type.
|
| 461 |
+
|
| 462 |
+
Note:
|
| 463 |
+
This sums up times and counts across ALL different operations, so the
|
| 464 |
+
"average" metrics (like cpu_time) represent the average time per operation
|
| 465 |
+
call across the entire session, mixing all operation types together.
|
| 466 |
+
For per-operation averages, use :meth:`key_averages` instead.
|
| 467 |
+
|
| 468 |
+
Returns:
|
| 469 |
+
FunctionEventAvg: A single aggregate object with key="Total" containing
|
| 470 |
+
accumulated statistics.
|
| 471 |
+
|
| 472 |
+
"""
|
| 473 |
+
total_stat = FunctionEventAvg()
|
| 474 |
+
for evt in self:
|
| 475 |
+
total_stat += evt
|
| 476 |
+
total_stat.key = None
|
| 477 |
+
total_stat.key = "Total"
|
| 478 |
+
return total_stat
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
def _format_time(time_us):
|
| 482 |
+
"""Define how to format time in FunctionEvent."""
|
| 483 |
+
US_IN_SECOND = 1000.0 * 1000.0
|
| 484 |
+
US_IN_MS = 1000.0
|
| 485 |
+
if time_us >= US_IN_SECOND:
|
| 486 |
+
return f"{time_us / US_IN_SECOND:.3f}s"
|
| 487 |
+
if time_us >= US_IN_MS:
|
| 488 |
+
return f"{time_us / US_IN_MS:.3f}ms"
|
| 489 |
+
return f"{time_us:.3f}us"
|
| 490 |
+
|
| 491 |
+
|
| 492 |
+
def _format_time_share(time_us, total_time_us):
|
| 493 |
+
"""Define how to format time in FunctionEvent."""
|
| 494 |
+
if total_time_us == 0:
|
| 495 |
+
if time_us != 0:
|
| 496 |
+
raise AssertionError(f"Expected time_us == 0 but got {time_us}")
|
| 497 |
+
return "NaN"
|
| 498 |
+
return f"{time_us * 100.0 / total_time_us:.2f}%"
|
| 499 |
+
|
| 500 |
+
|
| 501 |
+
def _format_memory(nbytes):
|
| 502 |
+
"""Return a formatted memory size string."""
|
| 503 |
+
KB = 1024
|
| 504 |
+
MB = 1024 * KB
|
| 505 |
+
GB = 1024 * MB
|
| 506 |
+
if abs(nbytes) >= GB:
|
| 507 |
+
return f"{nbytes * 1.0 / GB:.2f} GB"
|
| 508 |
+
elif abs(nbytes) >= MB:
|
| 509 |
+
return f"{nbytes * 1.0 / MB:.2f} MB"
|
| 510 |
+
elif abs(nbytes) >= KB:
|
| 511 |
+
return f"{nbytes * 1.0 / KB:.2f} KB"
|
| 512 |
+
else:
|
| 513 |
+
return str(nbytes) + " B"
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
def _attr_formatter(name):
|
| 517 |
+
return property(lambda self: _format_time(getattr(self, name)))
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
class FormattedTimesMixin:
|
| 521 |
+
"""Helpers for FunctionEvent and FunctionEventAvg.
|
| 522 |
+
|
| 523 |
+
The subclass should define `*_time_total` and `count` attributes.
|
| 524 |
+
"""
|
| 525 |
+
|
| 526 |
+
cpu_time_str = _attr_formatter("cpu_time")
|
| 527 |
+
device_time_str = _attr_formatter("device_time")
|
| 528 |
+
cpu_time_total_str = _attr_formatter("cpu_time_total")
|
| 529 |
+
device_time_total_str = _attr_formatter("device_time_total")
|
| 530 |
+
self_cpu_time_total_str = _attr_formatter("self_cpu_time_total")
|
| 531 |
+
self_device_time_total_str = _attr_formatter("self_device_time_total")
|
| 532 |
+
|
| 533 |
+
@property
|
| 534 |
+
def cpu_time(self):
|
| 535 |
+
return 0.0 if self.count == 0 else 1.0 * self.cpu_time_total / self.count # type: ignore[attr-defined]
|
| 536 |
+
|
| 537 |
+
@property
|
| 538 |
+
def device_time(self):
|
| 539 |
+
return 0.0 if self.count == 0 else 1.0 * self.device_time_total / self.count # type: ignore[attr-defined]
|
| 540 |
+
|
| 541 |
+
@property
|
| 542 |
+
@deprecated(
|
| 543 |
+
"`cuda_time` is deprecated, please use `device_time` instead.",
|
| 544 |
+
category=FutureWarning,
|
| 545 |
+
)
|
| 546 |
+
def cuda_time(self): # To be deprecated
|
| 547 |
+
return self.device_time
|
| 548 |
+
|
| 549 |
+
|
| 550 |
+
class Interval:
|
| 551 |
+
def __init__(self, start, end):
|
| 552 |
+
self.start = start
|
| 553 |
+
self.end = end
|
| 554 |
+
|
| 555 |
+
def elapsed_us(self):
|
| 556 |
+
r"""
|
| 557 |
+
Returns the length of the interval
|
| 558 |
+
"""
|
| 559 |
+
return self.end - self.start
|
| 560 |
+
|
| 561 |
+
|
| 562 |
+
Kernel = namedtuple("Kernel", ["name", "device", "duration"])
|
| 563 |
+
|
| 564 |
+
|
| 565 |
+
class EventMetadata(NamedTuple):
|
| 566 |
+
# Kernel fields
|
| 567 |
+
registers_per_thread: int | None
|
| 568 |
+
shared_memory: int | None
|
| 569 |
+
grid: list[int] | None
|
| 570 |
+
block: list[int] | None
|
| 571 |
+
priority: int | None
|
| 572 |
+
blocks_per_sm: float | None
|
| 573 |
+
warps_per_sm: float | None
|
| 574 |
+
occupancy: dict[str, Any] | None
|
| 575 |
+
est_occupancy_pct: float | None
|
| 576 |
+
queued: int | None
|
| 577 |
+
graph_id: int | None
|
| 578 |
+
graph_node_id: int | None
|
| 579 |
+
stream: int | None
|
| 580 |
+
context: int | None
|
| 581 |
+
channel: int | None
|
| 582 |
+
channel_type: int | None
|
| 583 |
+
# Memory fields
|
| 584 |
+
bytes: int | None
|
| 585 |
+
bandwidth_gb_s: float | None
|
| 586 |
+
# NCCL fields
|
| 587 |
+
collective_name: str | None
|
| 588 |
+
dtype: str | None
|
| 589 |
+
in_msg_nelems: int | None
|
| 590 |
+
out_msg_nelems: int | None
|
| 591 |
+
in_split_size: str | None
|
| 592 |
+
out_split_size: str | None
|
| 593 |
+
global_rank_start: int | None
|
| 594 |
+
global_rank_stride: int | None
|
| 595 |
+
group_size: int | None
|
| 596 |
+
process_group_name: str | None
|
| 597 |
+
process_group_desc: str | None
|
| 598 |
+
group_ranks: str | None
|
| 599 |
+
rank: int | None
|
| 600 |
+
src_rank: int | None
|
| 601 |
+
dst_rank: int | None
|
| 602 |
+
seq: int | None
|
| 603 |
+
is_async: bool | None
|
| 604 |
+
|
| 605 |
+
|
| 606 |
+
def _to_str(v: str) -> str:
|
| 607 |
+
return v.strip('"')
|
| 608 |
+
|
| 609 |
+
|
| 610 |
+
def _to_bool(v: str) -> bool:
|
| 611 |
+
return v in ("1", "true")
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
# Kineto key → (EventMetadata field name, converter from string)
|
| 615 |
+
_EVENT_METADATA_KEYS: dict[str, tuple[str, Callable[[str], Any]]] = {
|
| 616 |
+
"registers per thread": ("registers_per_thread", int),
|
| 617 |
+
"shared memory": ("shared_memory", int),
|
| 618 |
+
"grid": ("grid", json.loads), # list[int]
|
| 619 |
+
"block": ("block", json.loads), # list[int]
|
| 620 |
+
"priority": ("priority", int),
|
| 621 |
+
"blocks per SM": ("blocks_per_sm", float),
|
| 622 |
+
"warps per SM": ("warps_per_sm", float),
|
| 623 |
+
"est. achieved occupancy %": ("est_occupancy_pct", float),
|
| 624 |
+
"occupancy": ("occupancy", json.loads), # dict[str, Any]
|
| 625 |
+
"queued": ("queued", int),
|
| 626 |
+
"graph id": ("graph_id", int),
|
| 627 |
+
"graph node id": ("graph_node_id", int),
|
| 628 |
+
"stream": ("stream", int),
|
| 629 |
+
"context": ("context", int),
|
| 630 |
+
"channel": ("channel", int),
|
| 631 |
+
"channel_type": ("channel_type", int),
|
| 632 |
+
"bytes": ("bytes", int),
|
| 633 |
+
"memory bandwidth (GB/s)": ("bandwidth_gb_s", float),
|
| 634 |
+
"Collective name": ("collective_name", _to_str),
|
| 635 |
+
"dtype": ("dtype", _to_str),
|
| 636 |
+
"In msg nelems": ("in_msg_nelems", int),
|
| 637 |
+
"Out msg nelems": ("out_msg_nelems", int),
|
| 638 |
+
"In split size": ("in_split_size", _to_str),
|
| 639 |
+
"Out split size": ("out_split_size", _to_str),
|
| 640 |
+
"Global rank start": ("global_rank_start", int),
|
| 641 |
+
"Global rank stride": ("global_rank_stride", int),
|
| 642 |
+
"Group size": ("group_size", int),
|
| 643 |
+
"Process Group Name": ("process_group_name", _to_str),
|
| 644 |
+
"Process Group Description": ("process_group_desc", _to_str),
|
| 645 |
+
"Process Group Ranks": ("group_ranks", _to_str),
|
| 646 |
+
"Rank": ("rank", int),
|
| 647 |
+
"Src Rank": ("src_rank", int),
|
| 648 |
+
"Dst Rank": ("dst_rank", int),
|
| 649 |
+
"Seq": ("seq", int),
|
| 650 |
+
"Is asynchronized op": ("is_async", _to_bool),
|
| 651 |
+
}
|
| 652 |
+
|
| 653 |
+
|
| 654 |
+
def _build_metadata(extra_meta):
|
| 655 |
+
fields: dict[str, Any] = {}
|
| 656 |
+
any_populated = False
|
| 657 |
+
for kineto_key, (field_name, convert) in _EVENT_METADATA_KEYS.items():
|
| 658 |
+
v = extra_meta.get(kineto_key)
|
| 659 |
+
if v is not None:
|
| 660 |
+
fields[field_name] = convert(v)
|
| 661 |
+
any_populated = True
|
| 662 |
+
else:
|
| 663 |
+
fields[field_name] = None
|
| 664 |
+
return EventMetadata(**fields) if any_populated else None
|
| 665 |
+
|
| 666 |
+
|
| 667 |
+
class FunctionEvent(FormattedTimesMixin):
|
| 668 |
+
"""Profiling information about a single function.
|
| 669 |
+
|
| 670 |
+
FunctionEvent records the execution of a single operation during profiling.
|
| 671 |
+
These events are obtained from the profiler/kineto and contain detailed
|
| 672 |
+
timing and memory usage information.
|
| 673 |
+
|
| 674 |
+
.. note::
|
| 675 |
+
FunctionEvent objects are typically created by the profiler/kineto and should not
|
| 676 |
+
be instantiated directly by users. Access them through the profiler's output.
|
| 677 |
+
|
| 678 |
+
Attributes:
|
| 679 |
+
id (int): Unique identifier for this event.
|
| 680 |
+
node_id (int): Node identifier for distributed profiling (-1 if not applicable).
|
| 681 |
+
name (str): Name of the profiled function/operator.
|
| 682 |
+
overload_name (str): Overload name for the operator (requires _ExperimentalConfig(capture_overload_names=True) set).
|
| 683 |
+
trace_name (str): Same as name, just changes ProfilerStep* to ProfilerStep#
|
| 684 |
+
time_range (Interval): Time interval containing start and end timestamps in microseconds.
|
| 685 |
+
thread (int): Thread ID where the operation started.
|
| 686 |
+
fwd_thread (int): Thread ID of the corresponding forward operation.
|
| 687 |
+
kernels (List[Kernel]): List of device kernels launched by this operation.
|
| 688 |
+
count (int): Number of times this event was called (usually 1).
|
| 689 |
+
cpu_children (List[FunctionEvent]): Direct CPU child operations.
|
| 690 |
+
cpu_parent (FunctionEvent): Direct CPU parent operation.
|
| 691 |
+
input_shapes (List[List[int]]): Shapes of input tensors (requires record_shapes=True).
|
| 692 |
+
For plain tensor inputs, each entry is a list of dimensions (e.g. ``[16, 16]``).
|
| 693 |
+
TensorList inputs are represented as an empty list ``[]``; use
|
| 694 |
+
``structured_input_shapes`` to get per-element shapes for TensorList inputs.
|
| 695 |
+
concrete_inputs (List[Any]): Concrete input values (requires record_shapes=true).
|
| 696 |
+
kwinputs (Dict[str, Any]): Keyword arguments (requires record_shapes=true).
|
| 697 |
+
stack (List[str]): Python stack trace where the operation was called (requires with_stack=true).
|
| 698 |
+
scope (int): at::RecordScope identifier (0=forward, 1=backward, etc.).
|
| 699 |
+
use_device (str): Device type being profiled ("cuda", "xpu", etc.).
|
| 700 |
+
cpu_memory_usage (int): CPU memory allocated in bytes.
|
| 701 |
+
device_memory_usage (int): Device memory allocated in bytes.
|
| 702 |
+
is_async (bool): Whether this is an asynchronous operation.
|
| 703 |
+
is_remote (bool): Whether this operation occurred on a remote node.
|
| 704 |
+
sequence_nr (int): Sequence number for autograd operations.
|
| 705 |
+
device_type (DeviceType): Type of device (CPU, CUDA, XPU, PrivateUse1, etc.).
|
| 706 |
+
device_index (int): Index of the device (e.g., GPU 0, 1, 2).
|
| 707 |
+
device_resource_id (int): Resource ID on the device (ie. stream ID).
|
| 708 |
+
is_legacy (bool): Whether this is from the legacy profiler.
|
| 709 |
+
flops (int): Estimated floating point operations.
|
| 710 |
+
is_user_annotation (bool): Whether this is a user-annotated region.
|
| 711 |
+
metadata_json (str): Deprecated. Use event_metadata instead.
|
| 712 |
+
event_metadata (EventMetadata): Additional metadata in structured format.
|
| 713 |
+
structured_input_shapes (List[List[int] | List[List[int]]]): Like ``input_shapes``
|
| 714 |
+
but distinguishes TensorList inputs. Plain tensor inputs are ``List[int]``;
|
| 715 |
+
TensorList inputs are ``List[List[int]]`` containing one shape per tensor in the list.
|
| 716 |
+
Matches the ``"Input Dims"`` field in the Chrome trace JSON.
|
| 717 |
+
structured_input_strides (List[List[int] | List[List[int]]]): Strides of input
|
| 718 |
+
tensors in the same format as ``structured_input_shapes`` (requires
|
| 719 |
+
record_shapes=True).
|
| 720 |
+
|
| 721 |
+
Properties:
|
| 722 |
+
cpu_time_total (float): Total CPU time in microseconds.
|
| 723 |
+
device_time_total (float): Total device (CUDA/XPU/etc) time in microseconds.
|
| 724 |
+
self_cpu_time_total (float): CPU time excluding child operations.
|
| 725 |
+
self_device_time_total (float): Device time excluding child operations.
|
| 726 |
+
self_cpu_memory_usage (int): CPU memory usage excluding child operations.
|
| 727 |
+
self_device_memory_usage (int): Device memory usage excluding child operations.
|
| 728 |
+
cpu_time (float): Average CPU time per call.
|
| 729 |
+
device_time (float): Average device time per call.
|
| 730 |
+
key (str): Key used for grouping events (usually same as name).
|
| 731 |
+
|
| 732 |
+
See Also:
|
| 733 |
+
- :class:`torch.profiler.profile`: Context manager for profiling
|
| 734 |
+
- :class:`EventList`: List container for FunctionEvent objects with helper methods
|
| 735 |
+
- :class:`FunctionEventAvg`: Averaged statistics over multiple FunctionEvent objects
|
| 736 |
+
"""
|
| 737 |
+
|
| 738 |
+
def __init__(
|
| 739 |
+
self,
|
| 740 |
+
id,
|
| 741 |
+
name,
|
| 742 |
+
thread,
|
| 743 |
+
start_us,
|
| 744 |
+
end_us,
|
| 745 |
+
overload_name=None,
|
| 746 |
+
fwd_thread=None,
|
| 747 |
+
input_shapes=None,
|
| 748 |
+
stack=None,
|
| 749 |
+
scope=0,
|
| 750 |
+
use_device=None,
|
| 751 |
+
cpu_memory_usage=0,
|
| 752 |
+
device_memory_usage=0,
|
| 753 |
+
is_async=False,
|
| 754 |
+
is_remote=False,
|
| 755 |
+
sequence_nr=-1,
|
| 756 |
+
node_id=-1,
|
| 757 |
+
device_type=DeviceType.CPU,
|
| 758 |
+
device_index=0,
|
| 759 |
+
device_resource_id=None,
|
| 760 |
+
is_legacy=False,
|
| 761 |
+
flops=None,
|
| 762 |
+
trace_name=None,
|
| 763 |
+
concrete_inputs=None,
|
| 764 |
+
kwinputs=None,
|
| 765 |
+
is_user_annotation=False,
|
| 766 |
+
is_python_function=False,
|
| 767 |
+
activity_type=None,
|
| 768 |
+
metadata_json=None,
|
| 769 |
+
flow_id=None,
|
| 770 |
+
flow_type=None,
|
| 771 |
+
flow_start=None,
|
| 772 |
+
external_id=0,
|
| 773 |
+
linked_correlation_id=0,
|
| 774 |
+
extra_meta=None,
|
| 775 |
+
structured_input_shapes=None,
|
| 776 |
+
structured_input_strides=None,
|
| 777 |
+
input_dtypes=None,
|
| 778 |
+
python_id=-1,
|
| 779 |
+
python_parent_id=-1,
|
| 780 |
+
python_module_id=-1,
|
| 781 |
+
):
|
| 782 |
+
self.id: int = id
|
| 783 |
+
self.node_id: int = node_id
|
| 784 |
+
self.name: str = name
|
| 785 |
+
# pyrefly: ignore [bad-assignment]
|
| 786 |
+
self.overload_name: str = overload_name
|
| 787 |
+
# pyrefly: ignore [bad-assignment]
|
| 788 |
+
self.trace_name: str = trace_name
|
| 789 |
+
self.time_range: Interval = Interval(start_us, end_us)
|
| 790 |
+
self.thread: int = thread
|
| 791 |
+
self.fwd_thread: int | None = fwd_thread
|
| 792 |
+
self.kernels: list[Kernel] = []
|
| 793 |
+
self.count: int = 1
|
| 794 |
+
self.cpu_children: list[FunctionEvent] = []
|
| 795 |
+
self.cpu_parent: FunctionEvent | None = None
|
| 796 |
+
# pyrefly: ignore [bad-assignment]
|
| 797 |
+
self.input_shapes: tuple[int, ...] = input_shapes
|
| 798 |
+
# pyrefly: ignore [bad-assignment]
|
| 799 |
+
self.concrete_inputs: list[Any] = concrete_inputs
|
| 800 |
+
# pyrefly: ignore [bad-assignment]
|
| 801 |
+
self.kwinputs: dict[str, Any] = kwinputs
|
| 802 |
+
# pyrefly: ignore [bad-assignment]
|
| 803 |
+
self.stack: list = stack
|
| 804 |
+
self.scope: int = scope
|
| 805 |
+
self.use_device: str | None = use_device
|
| 806 |
+
self.cpu_memory_usage: int = cpu_memory_usage
|
| 807 |
+
self.device_memory_usage: int = device_memory_usage
|
| 808 |
+
self.is_async: bool = is_async
|
| 809 |
+
self.is_remote: bool = is_remote
|
| 810 |
+
self.sequence_nr: int = sequence_nr
|
| 811 |
+
self.device_type: DeviceType = device_type
|
| 812 |
+
self.device_index: int = device_index
|
| 813 |
+
self.device_resource_id: int = (
|
| 814 |
+
thread if device_resource_id is None else device_resource_id
|
| 815 |
+
)
|
| 816 |
+
self.is_legacy: bool = is_legacy
|
| 817 |
+
self.flops: int | None = flops
|
| 818 |
+
self.is_user_annotation: bool | None = is_user_annotation
|
| 819 |
+
self.is_python_function: bool = is_python_function
|
| 820 |
+
self.activity_type: str | None = activity_type
|
| 821 |
+
self.self_cpu_percent = -1
|
| 822 |
+
self.total_cpu_percent = -1
|
| 823 |
+
self.total_device_percent = -1
|
| 824 |
+
self._metadata_json = metadata_json
|
| 825 |
+
self.flow_id: int | None = flow_id
|
| 826 |
+
self.flow_type: int | None = flow_type
|
| 827 |
+
self.flow_start: bool | None = flow_start
|
| 828 |
+
self.external_id: int = external_id
|
| 829 |
+
self.linked_correlation_id: int = linked_correlation_id
|
| 830 |
+
self.event_metadata: EventMetadata | None = (
|
| 831 |
+
_build_metadata(extra_meta) if extra_meta else None
|
| 832 |
+
)
|
| 833 |
+
# pyrefly: ignore [bad-assignment]
|
| 834 |
+
self.structured_input_shapes: list = structured_input_shapes
|
| 835 |
+
# pyrefly: ignore [bad-assignment]
|
| 836 |
+
self.structured_input_strides: list = structured_input_strides
|
| 837 |
+
# pyrefly: ignore [bad-assignment]
|
| 838 |
+
self.input_dtypes: list[str] = input_dtypes
|
| 839 |
+
self.python_id: int = python_id
|
| 840 |
+
self.python_parent_id: int = python_parent_id
|
| 841 |
+
self.python_module_id: int = python_module_id
|
| 842 |
+
|
| 843 |
+
def append_kernel(self, name, device, duration):
|
| 844 |
+
if self.device_type != DeviceType.CPU:
|
| 845 |
+
raise AssertionError("Expected device_type to be CPU")
|
| 846 |
+
self.kernels.append(Kernel(name, device, duration))
|
| 847 |
+
|
| 848 |
+
def append_cpu_child(self, child):
|
| 849 |
+
"""Append a CPU child of type FunctionEvent.
|
| 850 |
+
|
| 851 |
+
One is supposed to append only direct children to the event to have
|
| 852 |
+
correct self cpu time being reported.
|
| 853 |
+
"""
|
| 854 |
+
if self.device_type != DeviceType.CPU:
|
| 855 |
+
raise AssertionError("Expected device_type to be CPU")
|
| 856 |
+
if not isinstance(child, FunctionEvent):
|
| 857 |
+
raise AssertionError("Expected child to be a FunctionEvent")
|
| 858 |
+
if child.device_type != DeviceType.CPU:
|
| 859 |
+
raise AssertionError("Expected child device_type to be CPU")
|
| 860 |
+
self.cpu_children.append(child)
|
| 861 |
+
|
| 862 |
+
def set_cpu_parent(self, parent):
|
| 863 |
+
"""Set the immediate CPU parent of type FunctionEvent.
|
| 864 |
+
|
| 865 |
+
One profiling FunctionEvent should have only one CPU parent such that
|
| 866 |
+
the child's range interval is completely inside the parent's. We use
|
| 867 |
+
this connection to determine the event is from top-level op or not.
|
| 868 |
+
"""
|
| 869 |
+
if self.device_type != DeviceType.CPU:
|
| 870 |
+
raise AssertionError("Expected device_type to be CPU")
|
| 871 |
+
if not isinstance(parent, FunctionEvent):
|
| 872 |
+
raise AssertionError("Expected parent to be a FunctionEvent")
|
| 873 |
+
if parent.device_type != DeviceType.CPU:
|
| 874 |
+
raise AssertionError("Expected parent device_type to be CPU")
|
| 875 |
+
self.cpu_parent = parent
|
| 876 |
+
|
| 877 |
+
# Note: async events don't have children, are not used when computing 'self'
|
| 878 |
+
# metrics of other events, have only total cpu time
|
| 879 |
+
@property
|
| 880 |
+
def self_cpu_memory_usage(self):
|
| 881 |
+
if self.is_async or self.device_type != DeviceType.CPU:
|
| 882 |
+
return 0
|
| 883 |
+
return self.cpu_memory_usage - sum(
|
| 884 |
+
child.cpu_memory_usage for child in self.cpu_children
|
| 885 |
+
)
|
| 886 |
+
|
| 887 |
+
@property
|
| 888 |
+
def self_device_memory_usage(self):
|
| 889 |
+
if self.is_async or self.device_type != DeviceType.CPU:
|
| 890 |
+
return 0
|
| 891 |
+
return self.device_memory_usage - sum(
|
| 892 |
+
child.device_memory_usage for child in self.cpu_children
|
| 893 |
+
)
|
| 894 |
+
|
| 895 |
+
@property
|
| 896 |
+
@deprecated(
|
| 897 |
+
"`metadata_json` is deprecated. Use `event_metadata` instead.",
|
| 898 |
+
category=FutureWarning,
|
| 899 |
+
)
|
| 900 |
+
def metadata_json(self):
|
| 901 |
+
return self._metadata_json
|
| 902 |
+
|
| 903 |
+
@property
|
| 904 |
+
@deprecated(
|
| 905 |
+
"`self_cuda_memory_usage` is deprecated. Use `self_device_memory_usage` instead.",
|
| 906 |
+
category=FutureWarning,
|
| 907 |
+
)
|
| 908 |
+
def self_cuda_memory_usage(self): # To be deprecated
|
| 909 |
+
return self.self_device_memory_usage
|
| 910 |
+
|
| 911 |
+
@property
|
| 912 |
+
def cpu_time_total(self):
|
| 913 |
+
if self.device_type == DeviceType.CPU:
|
| 914 |
+
return self.time_range.elapsed_us()
|
| 915 |
+
else:
|
| 916 |
+
return 0
|
| 917 |
+
|
| 918 |
+
@property
|
| 919 |
+
def self_cpu_time_total(self):
|
| 920 |
+
if self.is_async or self.device_type != DeviceType.CPU:
|
| 921 |
+
return 0
|
| 922 |
+
return self.cpu_time_total - sum(
|
| 923 |
+
child.cpu_time_total for child in self.cpu_children
|
| 924 |
+
)
|
| 925 |
+
|
| 926 |
+
@property
|
| 927 |
+
def device_time_total(self):
|
| 928 |
+
if self.is_async or not self.use_device:
|
| 929 |
+
return 0
|
| 930 |
+
if self.device_type == DeviceType.CPU:
|
| 931 |
+
if not self.is_legacy:
|
| 932 |
+
# account for the kernels in the children ops
|
| 933 |
+
return sum(kinfo.duration for kinfo in self.kernels) + sum(
|
| 934 |
+
ch.device_time_total for ch in self.cpu_children
|
| 935 |
+
)
|
| 936 |
+
else:
|
| 937 |
+
# each legacy cpu events has a single (fake) kernel
|
| 938 |
+
return sum(kinfo.duration for kinfo in self.kernels)
|
| 939 |
+
else:
|
| 940 |
+
if self.device_type not in [
|
| 941 |
+
DeviceType.CUDA,
|
| 942 |
+
DeviceType.PrivateUse1,
|
| 943 |
+
DeviceType.MTIA,
|
| 944 |
+
DeviceType.HPU,
|
| 945 |
+
DeviceType.XPU,
|
| 946 |
+
]:
|
| 947 |
+
raise AssertionError(
|
| 948 |
+
f"Expected device_type to be CUDA, PrivateUse1, MTIA, HPU or XPU, but got {self.device_type}"
|
| 949 |
+
)
|
| 950 |
+
return self.time_range.elapsed_us()
|
| 951 |
+
|
| 952 |
+
@property
|
| 953 |
+
@deprecated(
|
| 954 |
+
"`cuda_time_total` is deprecated. Use `device_time_total` instead.",
|
| 955 |
+
category=FutureWarning,
|
| 956 |
+
)
|
| 957 |
+
def cuda_time_total(self): # To be deprecated
|
| 958 |
+
return self.device_time_total
|
| 959 |
+
|
| 960 |
+
@property
|
| 961 |
+
def self_device_time_total(self):
|
| 962 |
+
if self.is_async or not self.use_device:
|
| 963 |
+
return 0
|
| 964 |
+
if self.device_type == DeviceType.CPU:
|
| 965 |
+
return self.device_time_total - sum(
|
| 966 |
+
child.device_time_total for child in self.cpu_children
|
| 967 |
+
)
|
| 968 |
+
else:
|
| 969 |
+
if self.device_type not in [
|
| 970 |
+
DeviceType.CUDA,
|
| 971 |
+
DeviceType.PrivateUse1,
|
| 972 |
+
DeviceType.MTIA,
|
| 973 |
+
DeviceType.HPU,
|
| 974 |
+
DeviceType.XPU,
|
| 975 |
+
]:
|
| 976 |
+
raise AssertionError(
|
| 977 |
+
f"Expected device_type to be CUDA, PrivateUse1, MTIA, HPU or XPU, but got {self.device_type}"
|
| 978 |
+
)
|
| 979 |
+
return self.device_time_total
|
| 980 |
+
|
| 981 |
+
@property
|
| 982 |
+
@deprecated(
|
| 983 |
+
"`self_cuda_time_total` is deprecated. Use `self_device_time_total` instead.",
|
| 984 |
+
category=FutureWarning,
|
| 985 |
+
)
|
| 986 |
+
def self_cuda_time_total(self): # To be deprecated
|
| 987 |
+
return self.self_device_time_total
|
| 988 |
+
|
| 989 |
+
@property
|
| 990 |
+
def key(self):
|
| 991 |
+
return self.name
|
| 992 |
+
|
| 993 |
+
def __repr__(self):
|
| 994 |
+
device_name = self.use_device
|
| 995 |
+
device_time = self.device_time_str
|
| 996 |
+
device_memory_usage = self.device_memory_usage
|
| 997 |
+
return (
|
| 998 |
+
f"<FunctionEvent id={self.id} name={self.name} overload_name={self.overload_name} "
|
| 999 |
+
f"device_type={self.device_type} node_id={self.node_id} cpu_time={self.cpu_time_str} "
|
| 1000 |
+
f"start_us={self.time_range.start} end_us={self.time_range.end} "
|
| 1001 |
+
f"cpu_children={str([child.id for child in self.cpu_children])} {device_name}_time={device_time} "
|
| 1002 |
+
f"name={self.name} thread={self.thread} input_shapes={str(self.input_shapes)} "
|
| 1003 |
+
f"cpu_memory_usage={self.cpu_memory_usage} {device_name}_memory_usage={device_memory_usage} "
|
| 1004 |
+
f"is_async={self.is_async} is_remote={self.is_remote} seq_nr={self.sequence_nr} is_legacy={self.is_legacy}>"
|
| 1005 |
+
)
|
| 1006 |
+
|
| 1007 |
+
|
| 1008 |
+
class FunctionEventAvg(FormattedTimesMixin):
|
| 1009 |
+
"""Averaged profiling statistics over multiple FunctionEvent objects.
|
| 1010 |
+
|
| 1011 |
+
FunctionEventAvg aggregates statistics from multiple FunctionEvent objects
|
| 1012 |
+
with the same key (typically same operation name). This is useful for getting
|
| 1013 |
+
average performance metrics across multiple invocations of the same operation.
|
| 1014 |
+
|
| 1015 |
+
This class is typically created by calling :meth:`EventList.key_averages()` on
|
| 1016 |
+
a profiler's event list.
|
| 1017 |
+
|
| 1018 |
+
Attributes:
|
| 1019 |
+
key (str): Grouping key for the events (typically operation name).
|
| 1020 |
+
count (int): Total number of events aggregated.
|
| 1021 |
+
node_id (int): Node identifier for distributed profiling (-1 if not applicable).
|
| 1022 |
+
is_async (bool): Whether the operations are asynchronous.
|
| 1023 |
+
is_remote (bool): Whether the operations occurred on a remote node.
|
| 1024 |
+
use_device (str): Device type being profiled ("cuda", "xpu", etc.).
|
| 1025 |
+
cpu_time_total (int): Accumulated total CPU time in microseconds.
|
| 1026 |
+
device_time_total (int): Accumulated total device time in microseconds.
|
| 1027 |
+
self_cpu_time_total (int): Accumulated self CPU time (excluding children) in microseconds.
|
| 1028 |
+
self_device_time_total (int): Accumulated self device time (excluding children) in microseconds.
|
| 1029 |
+
input_shapes (List[List[int]]): Input tensor shapes (requires record_shapes=true).
|
| 1030 |
+
overload_name (str): Operator overload name (requires _ExperimentalConfig(capture_overload_names=True) set).
|
| 1031 |
+
stack (List[str]): Python stack trace where the operation was called (requires with_stack=true).
|
| 1032 |
+
scope (int): at::RecordScope identifier (0=forward, 1=backward, etc.).
|
| 1033 |
+
cpu_memory_usage (int): Accumulated CPU memory usage in bytes.
|
| 1034 |
+
device_memory_usage (int): Accumulated device memory usage in bytes.
|
| 1035 |
+
self_cpu_memory_usage (int): Accumulated self CPU memory usage in bytes.
|
| 1036 |
+
self_device_memory_usage (int): Accumulated self device memory usage in bytes.
|
| 1037 |
+
cpu_children (List[FunctionEvent]): CPU child events.
|
| 1038 |
+
cpu_parent (FunctionEvent): CPU parent event.
|
| 1039 |
+
device_type (DeviceType): Type of device (CPU, CUDA, XPU, PrivateUse1, etc.).
|
| 1040 |
+
is_legacy (bool): Whether from legacy profiler.
|
| 1041 |
+
flops (int): Total floating point operations.
|
| 1042 |
+
is_user_annotation (bool): Whether this is a user-annotated region.
|
| 1043 |
+
|
| 1044 |
+
Properties:
|
| 1045 |
+
cpu_time (float): Average CPU time per invocation.
|
| 1046 |
+
device_time (float): Average device time per invocation.
|
| 1047 |
+
|
| 1048 |
+
See Also:
|
| 1049 |
+
- :class:`EventList.key_averages`: Method that creates FunctionEventAvg objects
|
| 1050 |
+
- :class:`FunctionEvent`: Individual profiling event
|
| 1051 |
+
- :class:`EventList`: Container for profiling events
|
| 1052 |
+
"""
|
| 1053 |
+
|
| 1054 |
+
def __init__(self) -> None:
|
| 1055 |
+
self.key: str | None = None
|
| 1056 |
+
self.count: int = 0
|
| 1057 |
+
self.node_id: int = 0
|
| 1058 |
+
self.is_async: bool = False
|
| 1059 |
+
self.is_remote: bool = False
|
| 1060 |
+
self.use_device: str | None = None
|
| 1061 |
+
self.cpu_time_total: int = 0
|
| 1062 |
+
self.device_time_total: int = 0
|
| 1063 |
+
self.self_cpu_time_total: int = 0
|
| 1064 |
+
self.self_device_time_total: int = 0
|
| 1065 |
+
self.input_shapes: list[list[int]] | None = None
|
| 1066 |
+
self.overload_name: str | None = None
|
| 1067 |
+
self.stack: list | None = None
|
| 1068 |
+
self.scope: int | None = None
|
| 1069 |
+
self.cpu_memory_usage: int = 0
|
| 1070 |
+
self.device_memory_usage: int = 0
|
| 1071 |
+
self.self_cpu_memory_usage: int = 0
|
| 1072 |
+
self.self_device_memory_usage: int = 0
|
| 1073 |
+
self.cpu_children: list[FunctionEvent] | None = None
|
| 1074 |
+
self.cpu_parent: FunctionEvent | None = None
|
| 1075 |
+
self.device_type: DeviceType = DeviceType.CPU
|
| 1076 |
+
self.is_legacy: bool = False
|
| 1077 |
+
self.flops: int = 0
|
| 1078 |
+
|
| 1079 |
+
def add(self, other):
|
| 1080 |
+
if self.key is None:
|
| 1081 |
+
# First function being recorded as part of FunctionEventAvg, propagate
|
| 1082 |
+
# fields.
|
| 1083 |
+
self.key = other.key
|
| 1084 |
+
self.node_id = other.node_id
|
| 1085 |
+
self.is_async = other.is_async
|
| 1086 |
+
self.is_remote = other.is_remote
|
| 1087 |
+
self.cpu_parent = other.cpu_parent
|
| 1088 |
+
self.cpu_children = other.cpu_children
|
| 1089 |
+
|
| 1090 |
+
self.overload_name = other.overload_name
|
| 1091 |
+
self.input_shapes = other.input_shapes
|
| 1092 |
+
self.stack = other.stack
|
| 1093 |
+
self.scope = other.scope
|
| 1094 |
+
self.device_type = other.device_type
|
| 1095 |
+
self.is_legacy = other.is_legacy
|
| 1096 |
+
self.use_device = other.use_device
|
| 1097 |
+
self.is_user_annotation = other.is_user_annotation
|
| 1098 |
+
|
| 1099 |
+
if not isinstance(other, (FunctionEvent, FunctionEventAvg)):
|
| 1100 |
+
raise AssertionError(
|
| 1101 |
+
"Expected other to be a FunctionEvent or FunctionEventAvg"
|
| 1102 |
+
)
|
| 1103 |
+
if other.key != self.key:
|
| 1104 |
+
raise AssertionError(
|
| 1105 |
+
f"Expected keys to match, but got {other.key} vs {self.key}"
|
| 1106 |
+
)
|
| 1107 |
+
|
| 1108 |
+
self.cpu_time_total += other.cpu_time_total
|
| 1109 |
+
self.device_time_total += other.device_time_total
|
| 1110 |
+
self.self_cpu_time_total += other.self_cpu_time_total
|
| 1111 |
+
self.self_device_time_total += other.self_device_time_total
|
| 1112 |
+
self.cpu_memory_usage += other.cpu_memory_usage
|
| 1113 |
+
self.device_memory_usage += other.device_memory_usage
|
| 1114 |
+
self.self_cpu_memory_usage += other.self_cpu_memory_usage
|
| 1115 |
+
self.self_device_memory_usage += other.self_device_memory_usage
|
| 1116 |
+
self.count += other.count
|
| 1117 |
+
if self.flops is None:
|
| 1118 |
+
# pyrefly: ignore [bad-assignment]
|
| 1119 |
+
self.flops = other.flops
|
| 1120 |
+
elif other.flops is not None:
|
| 1121 |
+
self.flops += other.flops
|
| 1122 |
+
return self
|
| 1123 |
+
|
| 1124 |
+
def __iadd__(self, other):
|
| 1125 |
+
return self.add(other)
|
| 1126 |
+
|
| 1127 |
+
def __repr__(self):
|
| 1128 |
+
device_name = "cuda" if not self.use_device else self.use_device
|
| 1129 |
+
self_device_time = self.self_device_time_total_str
|
| 1130 |
+
device_time = self.device_time_str
|
| 1131 |
+
device_memory = self.device_memory_usage
|
| 1132 |
+
return (
|
| 1133 |
+
f"<FunctionEventAvg key={self.key} self_cpu_time={self.self_cpu_time_total_str} cpu_time={self.cpu_time_str} "
|
| 1134 |
+
f" self_{device_name}_time={self_device_time} {device_name}_time={device_time} input_shapes={str(self.input_shapes)} "
|
| 1135 |
+
f"cpu_memory_usage={self.cpu_memory_usage} {device_name}_memory_usage={device_memory}>"
|
| 1136 |
+
)
|
| 1137 |
+
|
| 1138 |
+
|
| 1139 |
+
class StringTable(defaultdict):
|
| 1140 |
+
def __missing__(self, key):
|
| 1141 |
+
# manage cases like 't' (demangled to 'unsigned short') separately,
|
| 1142 |
+
# for now simply check the length to avoid unexpected results for
|
| 1143 |
+
# the short sequences
|
| 1144 |
+
self[key] = torch._C._demangle(key) if len(key) > 1 else key
|
| 1145 |
+
return self[key]
|
| 1146 |
+
|
| 1147 |
+
|
| 1148 |
+
class MemRecordsAcc:
|
| 1149 |
+
"""Acceleration structure for accessing mem_records in interval."""
|
| 1150 |
+
|
| 1151 |
+
def __init__(self, mem_records):
|
| 1152 |
+
self._mem_records = mem_records
|
| 1153 |
+
self._start_nses: list[int] = []
|
| 1154 |
+
self._indices: list[int] = []
|
| 1155 |
+
if len(mem_records) > 0:
|
| 1156 |
+
tmp = sorted([(r[0].start_ns(), i) for i, r in enumerate(mem_records)])
|
| 1157 |
+
self._start_nses, self._indices = zip(*tmp) # type: ignore[assignment]
|
| 1158 |
+
|
| 1159 |
+
def in_interval(self, start_ns, end_ns):
|
| 1160 |
+
r"""
|
| 1161 |
+
Return all records in the given interval
|
| 1162 |
+
"""
|
| 1163 |
+
start_idx = bisect.bisect_left(self._start_nses, start_ns)
|
| 1164 |
+
end_idx = bisect.bisect_right(self._start_nses, end_ns)
|
| 1165 |
+
for i in range(start_idx, end_idx):
|
| 1166 |
+
yield self._mem_records[self._indices[i]]
|
| 1167 |
+
|
| 1168 |
+
|
| 1169 |
+
def _filter_stack_entry(entry):
|
| 1170 |
+
filtered_entries = [
|
| 1171 |
+
("autograd/__init__", "_make_grads"),
|
| 1172 |
+
("autograd/__init__", "backward"),
|
| 1173 |
+
("torch/tensor", "backward"),
|
| 1174 |
+
("_internal/common_utils", "prof_callable"),
|
| 1175 |
+
("_internal/common_utils", "prof_func_call"),
|
| 1176 |
+
("_internal/common_utils", "prof_meth_call"),
|
| 1177 |
+
]
|
| 1178 |
+
return all(not (f[0] in entry and f[1] in entry) for f in filtered_entries)
|
| 1179 |
+
|
| 1180 |
+
|
| 1181 |
+
MEMORY_EVENT_NAME = "[memory]"
|
| 1182 |
+
OUT_OF_MEMORY_EVENT_NAME = "[OutOfMemory]"
|
| 1183 |
+
|
| 1184 |
+
|
| 1185 |
+
def _filter_name(name):
|
| 1186 |
+
# ignoring the following utility ops
|
| 1187 |
+
filtered_out_names = [
|
| 1188 |
+
MEMORY_EVENT_NAME, # used only for the top-level memory events
|
| 1189 |
+
OUT_OF_MEMORY_EVENT_NAME,
|
| 1190 |
+
"profiler::_record_function_enter",
|
| 1191 |
+
"profiler::_record_function_enter_new",
|
| 1192 |
+
"profiler::_record_function_exit",
|
| 1193 |
+
"aten::is_leaf",
|
| 1194 |
+
"aten::output_nr",
|
| 1195 |
+
"aten::_version",
|
| 1196 |
+
]
|
| 1197 |
+
return name in filtered_out_names
|
| 1198 |
+
|
| 1199 |
+
|
| 1200 |
+
# Demangles and optionally rewrites the provided event name,
|
| 1201 |
+
# with_wildcard - whether to replace certain numbered event names
|
| 1202 |
+
# with a wildcard name to aggregate them together in the profiler table
|
| 1203 |
+
# output
|
| 1204 |
+
def _rewrite_name(name, with_wildcard=False):
|
| 1205 |
+
string_table = StringTable()
|
| 1206 |
+
name = string_table[name]
|
| 1207 |
+
if with_wildcard:
|
| 1208 |
+
if name.startswith("ProfilerStep#"):
|
| 1209 |
+
name = "ProfilerStep*"
|
| 1210 |
+
return name
|
| 1211 |
+
|
| 1212 |
+
|
| 1213 |
+
def _build_table(
|
| 1214 |
+
events,
|
| 1215 |
+
sort_by=None,
|
| 1216 |
+
header=None,
|
| 1217 |
+
row_limit=100,
|
| 1218 |
+
max_src_column_width=75,
|
| 1219 |
+
max_name_column_width=55,
|
| 1220 |
+
max_shapes_column_width=80,
|
| 1221 |
+
with_flops=False,
|
| 1222 |
+
profile_memory=False,
|
| 1223 |
+
top_level_events_only=False,
|
| 1224 |
+
time_unit=None,
|
| 1225 |
+
):
|
| 1226 |
+
"""Print a summary of events (which can be a list of FunctionEvent or FunctionEventAvg)."""
|
| 1227 |
+
if len(events) == 0:
|
| 1228 |
+
return ""
|
| 1229 |
+
|
| 1230 |
+
has_device_time = any(event.self_device_time_total > 0 for event in events)
|
| 1231 |
+
has_device_mem = any(event.self_device_memory_usage > 0 for event in events)
|
| 1232 |
+
use_device = events[0].use_device
|
| 1233 |
+
# Running on PrivateUse1 device with profiler but not enable
|
| 1234 |
+
# ProfilerActivity.PrivateUse1 can also catch privateuse1 memory usage.
|
| 1235 |
+
# Here only need to check has_privateuse1_time if not use_device.
|
| 1236 |
+
if not use_device and has_device_time:
|
| 1237 |
+
raise RuntimeError("use_device is None, but there is device performance data.")
|
| 1238 |
+
|
| 1239 |
+
has_input_shapes = any(
|
| 1240 |
+
(event.input_shapes is not None and len(event.input_shapes) > 0)
|
| 1241 |
+
for event in events
|
| 1242 |
+
)
|
| 1243 |
+
|
| 1244 |
+
has_overload_names = any(
|
| 1245 |
+
(event.overload_name is not None and len(event.overload_name) > 0)
|
| 1246 |
+
for event in events
|
| 1247 |
+
)
|
| 1248 |
+
|
| 1249 |
+
if sort_by is not None:
|
| 1250 |
+
events = EventList(
|
| 1251 |
+
sorted(
|
| 1252 |
+
events,
|
| 1253 |
+
key=lambda evt: getattr(
|
| 1254 |
+
evt,
|
| 1255 |
+
sort_by.replace("cuda", "device")
|
| 1256 |
+
.replace("xpu", "device")
|
| 1257 |
+
.replace("privateuse1", "device"),
|
| 1258 |
+
),
|
| 1259 |
+
reverse=True,
|
| 1260 |
+
),
|
| 1261 |
+
use_device=use_device,
|
| 1262 |
+
profile_memory=profile_memory,
|
| 1263 |
+
with_flops=with_flops,
|
| 1264 |
+
)
|
| 1265 |
+
|
| 1266 |
+
name_column_width = max(len(evt.key) for evt in events) + 4
|
| 1267 |
+
if max_name_column_width is not None:
|
| 1268 |
+
name_column_width = min(name_column_width, max_name_column_width)
|
| 1269 |
+
|
| 1270 |
+
shapes_column_width = max(len(str(evt.input_shapes)) for evt in events) + 4
|
| 1271 |
+
if max_shapes_column_width is not None:
|
| 1272 |
+
shapes_column_width = min(shapes_column_width, max_shapes_column_width)
|
| 1273 |
+
|
| 1274 |
+
DEFAULT_COLUMN_WIDTH = 12
|
| 1275 |
+
flops_column_width = DEFAULT_COLUMN_WIDTH
|
| 1276 |
+
|
| 1277 |
+
src_column_width = None
|
| 1278 |
+
stacks = [
|
| 1279 |
+
evt.stack for evt in events if evt.stack is not None and len(evt.stack) > 0
|
| 1280 |
+
]
|
| 1281 |
+
has_stack = len(stacks) > 0
|
| 1282 |
+
if has_stack:
|
| 1283 |
+
src_column_width = (
|
| 1284 |
+
max(max(len(entry) for entry in stack) for stack in stacks) + 4
|
| 1285 |
+
)
|
| 1286 |
+
if max_src_column_width is not None:
|
| 1287 |
+
src_column_width = min(src_column_width, max_src_column_width)
|
| 1288 |
+
|
| 1289 |
+
headers = ["Name"]
|
| 1290 |
+
if has_overload_names:
|
| 1291 |
+
headers.append("Overload Name")
|
| 1292 |
+
headers += [
|
| 1293 |
+
"Self CPU %",
|
| 1294 |
+
"Self CPU",
|
| 1295 |
+
"CPU total %",
|
| 1296 |
+
"CPU total",
|
| 1297 |
+
"CPU time avg",
|
| 1298 |
+
]
|
| 1299 |
+
|
| 1300 |
+
device_name = use_device.upper() if use_device is not None else "None"
|
| 1301 |
+
if has_device_time:
|
| 1302 |
+
headers.extend(
|
| 1303 |
+
[
|
| 1304 |
+
f"Self {device_name}",
|
| 1305 |
+
f"Self {device_name} %",
|
| 1306 |
+
f"{device_name} total",
|
| 1307 |
+
f"{device_name} time avg",
|
| 1308 |
+
]
|
| 1309 |
+
)
|
| 1310 |
+
if profile_memory:
|
| 1311 |
+
headers.extend(
|
| 1312 |
+
[
|
| 1313 |
+
"CPU Mem",
|
| 1314 |
+
"Self CPU Mem",
|
| 1315 |
+
]
|
| 1316 |
+
)
|
| 1317 |
+
if use_device and has_device_mem:
|
| 1318 |
+
headers.extend(
|
| 1319 |
+
[
|
| 1320 |
+
f"{device_name} Mem",
|
| 1321 |
+
f"Self {device_name} Mem",
|
| 1322 |
+
]
|
| 1323 |
+
)
|
| 1324 |
+
headers.append("# of Calls")
|
| 1325 |
+
# Only append Node ID if any event has a valid (>= 0) Node ID
|
| 1326 |
+
append_node_id = any(evt.node_id != -1 for evt in events)
|
| 1327 |
+
if append_node_id:
|
| 1328 |
+
headers.append("Node ID")
|
| 1329 |
+
|
| 1330 |
+
SPACING_SIZE = 2
|
| 1331 |
+
row_format_lst = [""]
|
| 1332 |
+
header_sep_lst = [""]
|
| 1333 |
+
line_length_lst = [-SPACING_SIZE]
|
| 1334 |
+
|
| 1335 |
+
def add_column(padding, text_dir=">"):
|
| 1336 |
+
row_format_lst[0] += (
|
| 1337 |
+
"{: " + text_dir + str(padding) + "}" + (" " * SPACING_SIZE)
|
| 1338 |
+
)
|
| 1339 |
+
header_sep_lst[0] += "-" * padding + (" " * SPACING_SIZE)
|
| 1340 |
+
line_length_lst[0] += padding + SPACING_SIZE
|
| 1341 |
+
|
| 1342 |
+
def auto_scale_flops(flops):
|
| 1343 |
+
flop_headers = [
|
| 1344 |
+
"FLOPs",
|
| 1345 |
+
"KFLOPs",
|
| 1346 |
+
"MFLOPs",
|
| 1347 |
+
"GFLOPs",
|
| 1348 |
+
"TFLOPs",
|
| 1349 |
+
"PFLOPs",
|
| 1350 |
+
]
|
| 1351 |
+
if flops <= 0:
|
| 1352 |
+
raise AssertionError(f"Expected flops to be positive, but got {flops}")
|
| 1353 |
+
# pyrefly: ignore [no-matching-overload]
|
| 1354 |
+
log_flops = max(0, min(math.log10(flops) / 3, float(len(flop_headers) - 1)))
|
| 1355 |
+
if not (log_flops >= 0 and log_flops < len(flop_headers)):
|
| 1356 |
+
raise AssertionError(
|
| 1357 |
+
f"Expected log_flops to be in range [0, {len(flop_headers)}), but got {log_flops}"
|
| 1358 |
+
)
|
| 1359 |
+
return (pow(10, (math.floor(log_flops) * -3.0)), flop_headers[int(log_flops)])
|
| 1360 |
+
|
| 1361 |
+
add_column(name_column_width)
|
| 1362 |
+
if has_overload_names:
|
| 1363 |
+
add_column(name_column_width)
|
| 1364 |
+
for _ in headers[1 + has_overload_names :]:
|
| 1365 |
+
add_column(DEFAULT_COLUMN_WIDTH)
|
| 1366 |
+
|
| 1367 |
+
if has_input_shapes:
|
| 1368 |
+
headers.append("Input Shapes")
|
| 1369 |
+
add_column(shapes_column_width)
|
| 1370 |
+
|
| 1371 |
+
if has_stack:
|
| 1372 |
+
headers.append("Source Location")
|
| 1373 |
+
add_column(src_column_width, text_dir="<")
|
| 1374 |
+
|
| 1375 |
+
if with_flops:
|
| 1376 |
+
# Auto-scaling of flops header
|
| 1377 |
+
raw_flops = [evt.flops for evt in events if evt.flops > 0]
|
| 1378 |
+
if len(raw_flops) != 0:
|
| 1379 |
+
(flops_scale, flops_header) = auto_scale_flops(min(raw_flops))
|
| 1380 |
+
headers.append(f"Total {flops_header}")
|
| 1381 |
+
add_column(flops_column_width)
|
| 1382 |
+
else:
|
| 1383 |
+
with_flops = False # can't find any valid flops
|
| 1384 |
+
|
| 1385 |
+
row_format = row_format_lst[0]
|
| 1386 |
+
header_sep = header_sep_lst[0]
|
| 1387 |
+
line_length = line_length_lst[0]
|
| 1388 |
+
add_column = None # type: ignore[assignment]
|
| 1389 |
+
|
| 1390 |
+
result = []
|
| 1391 |
+
|
| 1392 |
+
def append(s):
|
| 1393 |
+
result.append(s)
|
| 1394 |
+
result.append("\n") # Yes, newline after the end as well
|
| 1395 |
+
|
| 1396 |
+
sum_self_cpu_time_total = 0
|
| 1397 |
+
sum_self_device_time_total = 0
|
| 1398 |
+
for evt in events:
|
| 1399 |
+
sum_self_cpu_time_total += evt.self_cpu_time_total
|
| 1400 |
+
if evt.device_type == DeviceType.CPU and evt.is_legacy:
|
| 1401 |
+
# in legacy profiler, kernel info is stored in cpu events
|
| 1402 |
+
sum_self_device_time_total += evt.self_device_time_total
|
| 1403 |
+
elif (
|
| 1404 |
+
evt.device_type
|
| 1405 |
+
in [
|
| 1406 |
+
DeviceType.CUDA,
|
| 1407 |
+
DeviceType.PrivateUse1,
|
| 1408 |
+
DeviceType.MTIA,
|
| 1409 |
+
DeviceType.XPU,
|
| 1410 |
+
]
|
| 1411 |
+
and not evt.is_user_annotation
|
| 1412 |
+
):
|
| 1413 |
+
# in kineto profiler, there're events with the correct device type (e.g. CUDA)
|
| 1414 |
+
sum_self_device_time_total += evt.self_device_time_total
|
| 1415 |
+
|
| 1416 |
+
# Actual printing
|
| 1417 |
+
if header is not None:
|
| 1418 |
+
append("=" * line_length)
|
| 1419 |
+
append(header)
|
| 1420 |
+
if top_level_events_only:
|
| 1421 |
+
append("=" * line_length)
|
| 1422 |
+
append("This report only display top-level ops statistics")
|
| 1423 |
+
append(header_sep)
|
| 1424 |
+
append(row_format.format(*headers))
|
| 1425 |
+
|
| 1426 |
+
append(header_sep)
|
| 1427 |
+
|
| 1428 |
+
def trim_path(path, src_column_width):
|
| 1429 |
+
if len(path) > src_column_width:
|
| 1430 |
+
offset = len(path) - src_column_width
|
| 1431 |
+
path = path[offset:]
|
| 1432 |
+
if len(path) > 3:
|
| 1433 |
+
path = "..." + path[3:]
|
| 1434 |
+
return path
|
| 1435 |
+
|
| 1436 |
+
def override_time_unit(time_us, default_str, time_unit):
|
| 1437 |
+
US_IN_SECOND = 1000.0 * 1000.0
|
| 1438 |
+
US_IN_MS = 1000.0
|
| 1439 |
+
if time_unit == "s":
|
| 1440 |
+
return f"{time_us / US_IN_SECOND:.3f}s"
|
| 1441 |
+
elif time_unit == "ms":
|
| 1442 |
+
return f"{time_us / US_IN_MS:.3f}ms"
|
| 1443 |
+
elif time_unit == "us":
|
| 1444 |
+
return f"{time_us:.3f}us"
|
| 1445 |
+
else:
|
| 1446 |
+
return default_str
|
| 1447 |
+
|
| 1448 |
+
event_limit = 0
|
| 1449 |
+
for evt in events:
|
| 1450 |
+
if event_limit == row_limit:
|
| 1451 |
+
break
|
| 1452 |
+
if top_level_events_only and evt.cpu_parent is not None:
|
| 1453 |
+
continue
|
| 1454 |
+
else:
|
| 1455 |
+
event_limit += 1
|
| 1456 |
+
name = evt.key
|
| 1457 |
+
if max_name_column_width is not None and len(name) >= max_name_column_width - 3:
|
| 1458 |
+
name = name[: (max_name_column_width - 3)] + "..."
|
| 1459 |
+
|
| 1460 |
+
evt.self_cpu_percent = _format_time_share(
|
| 1461 |
+
evt.self_cpu_time_total, sum_self_cpu_time_total
|
| 1462 |
+
)
|
| 1463 |
+
evt.total_cpu_percent = (
|
| 1464 |
+
_format_time_share(evt.cpu_time_total, sum_self_cpu_time_total)
|
| 1465 |
+
if not evt.is_async
|
| 1466 |
+
else 0
|
| 1467 |
+
)
|
| 1468 |
+
|
| 1469 |
+
row_values = [name]
|
| 1470 |
+
if has_overload_names:
|
| 1471 |
+
overload_name = evt.overload_name
|
| 1472 |
+
if (
|
| 1473 |
+
max_name_column_width is not None
|
| 1474 |
+
and len(overload_name) >= max_name_column_width - 3
|
| 1475 |
+
):
|
| 1476 |
+
overload_name = overload_name[: (max_name_column_width - 3)] + "..."
|
| 1477 |
+
row_values += [overload_name]
|
| 1478 |
+
row_values += [
|
| 1479 |
+
# Self CPU total %, 0 for async events.
|
| 1480 |
+
evt.self_cpu_percent,
|
| 1481 |
+
override_time_unit(
|
| 1482 |
+
evt.self_cpu_time_total, evt.self_cpu_time_total_str, time_unit
|
| 1483 |
+
), # Self CPU total
|
| 1484 |
+
# CPU total %, 0 for async events.
|
| 1485 |
+
evt.total_cpu_percent,
|
| 1486 |
+
override_time_unit(
|
| 1487 |
+
evt.cpu_time_total, evt.cpu_time_total_str, time_unit
|
| 1488 |
+
), # CPU total
|
| 1489 |
+
override_time_unit(
|
| 1490 |
+
evt.cpu_time, evt.cpu_time_str, time_unit
|
| 1491 |
+
), # CPU time avg
|
| 1492 |
+
]
|
| 1493 |
+
if has_device_time:
|
| 1494 |
+
evt.total_device_percent = _format_time_share(
|
| 1495 |
+
evt.self_device_time_total, sum_self_device_time_total
|
| 1496 |
+
)
|
| 1497 |
+
row_values.extend(
|
| 1498 |
+
[
|
| 1499 |
+
override_time_unit(
|
| 1500 |
+
evt.self_device_time_total,
|
| 1501 |
+
evt.self_device_time_total_str,
|
| 1502 |
+
time_unit,
|
| 1503 |
+
),
|
| 1504 |
+
# device time total %
|
| 1505 |
+
evt.total_device_percent,
|
| 1506 |
+
override_time_unit(
|
| 1507 |
+
evt.device_time_total, evt.device_time_total_str, time_unit
|
| 1508 |
+
),
|
| 1509 |
+
override_time_unit(
|
| 1510 |
+
evt.device_time, evt.device_time_str, time_unit
|
| 1511 |
+
), # device time avg
|
| 1512 |
+
]
|
| 1513 |
+
)
|
| 1514 |
+
if profile_memory:
|
| 1515 |
+
row_values.extend(
|
| 1516 |
+
[
|
| 1517 |
+
# CPU Mem Total
|
| 1518 |
+
_format_memory(evt.cpu_memory_usage),
|
| 1519 |
+
# Self CPU Mem Total
|
| 1520 |
+
_format_memory(evt.self_cpu_memory_usage),
|
| 1521 |
+
]
|
| 1522 |
+
)
|
| 1523 |
+
if use_device and has_device_mem:
|
| 1524 |
+
row_values.extend(
|
| 1525 |
+
[
|
| 1526 |
+
# Device Mem Total
|
| 1527 |
+
_format_memory(evt.device_memory_usage),
|
| 1528 |
+
# Self Device Mem Total
|
| 1529 |
+
_format_memory(evt.self_device_memory_usage),
|
| 1530 |
+
]
|
| 1531 |
+
)
|
| 1532 |
+
row_values.append(
|
| 1533 |
+
evt.count, # Number of calls
|
| 1534 |
+
)
|
| 1535 |
+
|
| 1536 |
+
if append_node_id:
|
| 1537 |
+
row_values.append(evt.node_id)
|
| 1538 |
+
if has_input_shapes:
|
| 1539 |
+
row_values.append(str(evt.input_shapes)[:shapes_column_width])
|
| 1540 |
+
if with_flops:
|
| 1541 |
+
if evt.flops <= 0:
|
| 1542 |
+
row_values.append("--")
|
| 1543 |
+
else:
|
| 1544 |
+
row_values.append(f"{evt.flops * flops_scale:8.3f}") # type: ignore[possibly-undefined]
|
| 1545 |
+
if has_stack:
|
| 1546 |
+
src_field = ""
|
| 1547 |
+
if len(evt.stack) > 0:
|
| 1548 |
+
src_field = trim_path(evt.stack[0], src_column_width)
|
| 1549 |
+
row_values.append(src_field)
|
| 1550 |
+
append(row_format.format(*row_values))
|
| 1551 |
+
|
| 1552 |
+
if has_stack:
|
| 1553 |
+
empty_headers = [""] * (len(headers) - 1)
|
| 1554 |
+
for entry in evt.stack[1:]:
|
| 1555 |
+
append(
|
| 1556 |
+
row_format.format(
|
| 1557 |
+
*(empty_headers + [trim_path(entry, src_column_width)])
|
| 1558 |
+
)
|
| 1559 |
+
)
|
| 1560 |
+
empty_headers.append("")
|
| 1561 |
+
append(row_format.format(*empty_headers))
|
| 1562 |
+
|
| 1563 |
+
append(header_sep)
|
| 1564 |
+
append(
|
| 1565 |
+
f"Self CPU time total: {override_time_unit(sum_self_cpu_time_total, _format_time(sum_self_cpu_time_total), time_unit)}"
|
| 1566 |
+
)
|
| 1567 |
+
if has_device_time:
|
| 1568 |
+
append(
|
| 1569 |
+
f"Self {use_device.upper() if use_device is not None else 'None'} "
|
| 1570 |
+
f"time total: {override_time_unit(sum_self_device_time_total, _format_time(sum_self_device_time_total), time_unit)}"
|
| 1571 |
+
)
|
| 1572 |
+
return "".join(result)
|
| 1573 |
+
|
| 1574 |
+
|
| 1575 |
+
# Collect all events with stack traces and format them canonically
|
| 1576 |
+
def _canonicalize_profiler_events(events):
|
| 1577 |
+
"""
|
| 1578 |
+
Extract and format all events with stack traces in a canonical way
|
| 1579 |
+
for deterministic testing.
|
| 1580 |
+
"""
|
| 1581 |
+
events_with_traces = []
|
| 1582 |
+
|
| 1583 |
+
for event in events:
|
| 1584 |
+
# Extract relevant fields
|
| 1585 |
+
event_name = event.get("name", "")
|
| 1586 |
+
node_name = event["args"].get("node_name", "")
|
| 1587 |
+
stack_trace = event["args"].get("stack_trace", "")
|
| 1588 |
+
|
| 1589 |
+
# Get the last non-empty line of the stack trace that is actual source,
|
| 1590 |
+
# not a caret-marker line. Python 3.11+ appends "^^^^"/"~~~~" indicator
|
| 1591 |
+
# lines below the source when a FrameSummary has colno/end_colno set
|
| 1592 |
+
# (e.g. dynamo-generated stack traces); those lines must be skipped so
|
| 1593 |
+
# we still surface the source code line in canonicalized output.
|
| 1594 |
+
lines = [s.strip() for s in stack_trace.split("\n") if s.strip()]
|
| 1595 |
+
source_lines = [s for s in lines if not set(s).issubset({"^", "~", " "})]
|
| 1596 |
+
stack_trace = source_lines[-1] if source_lines else ""
|
| 1597 |
+
|
| 1598 |
+
events_with_traces.append(
|
| 1599 |
+
{
|
| 1600 |
+
"event_name": event_name[:30],
|
| 1601 |
+
"node_name": node_name,
|
| 1602 |
+
"stack_trace": stack_trace,
|
| 1603 |
+
"start_time": event.get("ts", 0),
|
| 1604 |
+
}
|
| 1605 |
+
)
|
| 1606 |
+
|
| 1607 |
+
# Sort by node_name for deterministic ordering
|
| 1608 |
+
events_with_traces.sort(key=lambda x: x["start_time"])
|
| 1609 |
+
|
| 1610 |
+
# Format as a string
|
| 1611 |
+
lines: list[str] = []
|
| 1612 |
+
for evt in events_with_traces:
|
| 1613 |
+
lines.append(
|
| 1614 |
+
f"event={evt['event_name']} node={evt['node_name']} stack_trace={evt['stack_trace']}"
|
| 1615 |
+
)
|
| 1616 |
+
|
| 1617 |
+
return "\n".join(lines)
|
western_ai/usa/pytorch/torch/autograd/variable.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import torch
|
| 3 |
+
from torch._C import _ImperativeEngine as ImperativeEngine
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
__all__ = ["VariableMeta", "Variable"]
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class VariableMeta(type):
|
| 10 |
+
def __instancecheck__(cls, other):
|
| 11 |
+
return isinstance(other, torch.Tensor)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class Variable(torch._C._LegacyVariableBase, metaclass=VariableMeta): # type: ignore[misc]
|
| 15 |
+
_execution_engine = ImperativeEngine()
|
western_ai/usa/pytorch/torch/backends/__init__.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import sys
|
| 3 |
+
import types
|
| 4 |
+
from contextlib import contextmanager
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
# The idea for this parameter is that we forbid bare assignment
|
| 10 |
+
# to torch.backends.<cudnn|mkldnn>.enabled and friends when running our
|
| 11 |
+
# test suite, where it's very easy to forget to undo the change
|
| 12 |
+
# later.
|
| 13 |
+
__allow_nonbracketed_mutation_flag = True
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def disable_global_flags():
|
| 17 |
+
global __allow_nonbracketed_mutation_flag
|
| 18 |
+
__allow_nonbracketed_mutation_flag = False
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def flags_frozen():
|
| 22 |
+
return not __allow_nonbracketed_mutation_flag
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@contextmanager
|
| 26 |
+
def __allow_nonbracketed_mutation():
|
| 27 |
+
global __allow_nonbracketed_mutation_flag
|
| 28 |
+
old = __allow_nonbracketed_mutation_flag
|
| 29 |
+
__allow_nonbracketed_mutation_flag = True
|
| 30 |
+
try:
|
| 31 |
+
yield
|
| 32 |
+
finally:
|
| 33 |
+
__allow_nonbracketed_mutation_flag = old
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class ContextProp:
|
| 37 |
+
def __init__(self, getter, setter):
|
| 38 |
+
self.getter = getter
|
| 39 |
+
self.setter = setter
|
| 40 |
+
|
| 41 |
+
def __get__(self, obj, objtype):
|
| 42 |
+
return self.getter()
|
| 43 |
+
|
| 44 |
+
def __set__(self, obj, val):
|
| 45 |
+
if not flags_frozen():
|
| 46 |
+
self.setter(val)
|
| 47 |
+
else:
|
| 48 |
+
raise RuntimeError(
|
| 49 |
+
f"not allowed to set {obj.__name__} flags "
|
| 50 |
+
"after disable_global_flags; please use flags() context manager instead"
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class PropModule(types.ModuleType):
|
| 55 |
+
def __init__(self, m, name):
|
| 56 |
+
super().__init__(name)
|
| 57 |
+
self.m = m
|
| 58 |
+
|
| 59 |
+
def __getattr__(self, attr):
|
| 60 |
+
return self.m.__getattribute__(attr)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class _FP32Precision:
|
| 64 |
+
def __init__(self, backend, op):
|
| 65 |
+
self.backend = backend
|
| 66 |
+
self.op = op
|
| 67 |
+
|
| 68 |
+
def __setattr__(self, name, value):
|
| 69 |
+
if name == "fp32_precision":
|
| 70 |
+
torch._C._set_fp32_precision_setter(self.backend, self.op, value)
|
| 71 |
+
elif name in ("backend", "op"):
|
| 72 |
+
super().__setattr__(name, value)
|
| 73 |
+
else:
|
| 74 |
+
raise AttributeError("Unknown attribute " + name)
|
| 75 |
+
|
| 76 |
+
def __getattr__(self, name):
|
| 77 |
+
if name == "fp32_precision":
|
| 78 |
+
return torch._C._get_fp32_precision_getter(self.backend, self.op)
|
| 79 |
+
else:
|
| 80 |
+
raise AttributeError("Unknown attribute " + name)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def set_flags(_fp32_precision="none"):
|
| 84 |
+
orig_flags = (torch._C._get_fp32_precision_getter("generic", "all"),)
|
| 85 |
+
if _fp32_precision is not None:
|
| 86 |
+
torch._C._set_fp32_precision_setter("generic", "all", _fp32_precision)
|
| 87 |
+
return orig_flags
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
@contextmanager
|
| 91 |
+
def flags(fp32_precision="none"):
|
| 92 |
+
with __allow_nonbracketed_mutation():
|
| 93 |
+
orig_flags = set_flags(fp32_precision)
|
| 94 |
+
try:
|
| 95 |
+
yield
|
| 96 |
+
finally:
|
| 97 |
+
with __allow_nonbracketed_mutation():
|
| 98 |
+
set_flags(*orig_flags)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _get_fp32_precision_getter(backend, op):
|
| 102 |
+
def inner():
|
| 103 |
+
return torch._C._get_fp32_precision_getter(backend, op)
|
| 104 |
+
|
| 105 |
+
return inner
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _set_fp32_precision_setter(backend, op):
|
| 109 |
+
def inner(precision):
|
| 110 |
+
return torch._C._set_fp32_precision_setter(backend, op, precision)
|
| 111 |
+
|
| 112 |
+
return inner
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
class GenericModule(PropModule):
|
| 116 |
+
fp32_precision = ContextProp(
|
| 117 |
+
_get_fp32_precision_getter("generic", "all"),
|
| 118 |
+
_set_fp32_precision_setter("generic", "all"),
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
sys.modules[__name__] = GenericModule(sys.modules[__name__], __name__)
|
| 123 |
+
|
| 124 |
+
from torch.backends import (
|
| 125 |
+
cpu as cpu,
|
| 126 |
+
cuda as cuda,
|
| 127 |
+
cudnn as cudnn,
|
| 128 |
+
cusparselt as cusparselt,
|
| 129 |
+
kleidiai as kleidiai,
|
| 130 |
+
mha as mha,
|
| 131 |
+
miopen as miopen,
|
| 132 |
+
mkl as mkl,
|
| 133 |
+
mkldnn as mkldnn,
|
| 134 |
+
mps as mps,
|
| 135 |
+
nnpack as nnpack,
|
| 136 |
+
openmp as openmp,
|
| 137 |
+
opt_einsum as opt_einsum,
|
| 138 |
+
python_native as python_native,
|
| 139 |
+
quantized as quantized,
|
| 140 |
+
)
|
western_ai/usa/pytorch/torch/backends/_coreml/__init__.py
ADDED
|
File without changes
|
western_ai/usa/pytorch/torch/backends/_coreml/preprocess.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import hashlib
|
| 3 |
+
import json
|
| 4 |
+
|
| 5 |
+
import coremltools as ct # type: ignore[import]
|
| 6 |
+
from coremltools.converters.mil.input_types import TensorType # type: ignore[import]
|
| 7 |
+
from coremltools.converters.mil.mil import types # type: ignore[import]
|
| 8 |
+
from coremltools.models.neural_network import quantization_utils # type: ignore[import]
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
CT_METADATA_VERSION = "com.github.apple.coremltools.version"
|
| 14 |
+
CT_METADATA_SOURCE = "com.github.apple.coremltools.source"
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class ScalarType:
|
| 18 |
+
Float = 0
|
| 19 |
+
Double = 1
|
| 20 |
+
Int = 2
|
| 21 |
+
Long = 3
|
| 22 |
+
Undefined = 4
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
# Supported Tensor types in coremltools:
|
| 26 |
+
# https://github.com/apple/coremltools/blob/main/coremltools/converters/mil/frontend/torch/converter.py#L28
|
| 27 |
+
torch_to_mil_types = {
|
| 28 |
+
ScalarType.Float: types.fp32,
|
| 29 |
+
ScalarType.Double: types.fp64,
|
| 30 |
+
ScalarType.Int: types.int32,
|
| 31 |
+
ScalarType.Long: types.int64,
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class CoreMLComputeUnit:
|
| 36 |
+
CPU = "cpuOnly"
|
| 37 |
+
CPUAndGPU = "cpuAndGPU"
|
| 38 |
+
ALL = "all"
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class CoreMLQuantizationMode:
|
| 42 |
+
LINEAR = "linear"
|
| 43 |
+
LINEAR_SYMMETRIC = "linear_symmetric"
|
| 44 |
+
NONE = "none"
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def TensorSpec(shape, dtype=ScalarType.Float):
|
| 48 |
+
return (shape, dtype)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def CompileSpec(
|
| 52 |
+
inputs,
|
| 53 |
+
outputs,
|
| 54 |
+
backend=CoreMLComputeUnit.CPU,
|
| 55 |
+
allow_low_precision=True,
|
| 56 |
+
quantization_mode=CoreMLQuantizationMode.NONE,
|
| 57 |
+
mlmodel_export_path=None,
|
| 58 |
+
convert_to=None,
|
| 59 |
+
):
|
| 60 |
+
return (
|
| 61 |
+
inputs,
|
| 62 |
+
outputs,
|
| 63 |
+
backend,
|
| 64 |
+
allow_low_precision,
|
| 65 |
+
quantization_mode,
|
| 66 |
+
mlmodel_export_path,
|
| 67 |
+
convert_to,
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _check_enumerated_shape(shape):
|
| 72 |
+
for s in shape:
|
| 73 |
+
if not isinstance(s, (list, tuple)):
|
| 74 |
+
return False
|
| 75 |
+
return True
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def _convert_to_mil_type(shape, dtype, name: str):
|
| 79 |
+
mil_shape = shape
|
| 80 |
+
if _check_enumerated_shape(shape):
|
| 81 |
+
mil_shape = ct.EnumeratedShapes(shape)
|
| 82 |
+
ml_type = TensorType(shape=mil_shape, dtype=torch_to_mil_types[dtype])
|
| 83 |
+
ml_type.name = name
|
| 84 |
+
return ml_type
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def preprocess(script_module: torch._C.ScriptObject, compile_spec: dict[str, tuple]):
|
| 88 |
+
spec = compile_spec["forward"]
|
| 89 |
+
(
|
| 90 |
+
input_specs,
|
| 91 |
+
output_specs,
|
| 92 |
+
backend,
|
| 93 |
+
allow_low_precision,
|
| 94 |
+
quantization_mode,
|
| 95 |
+
mlmodel_export_path,
|
| 96 |
+
convert_to,
|
| 97 |
+
) = spec
|
| 98 |
+
mil_inputs = []
|
| 99 |
+
inputs = []
|
| 100 |
+
for index, input in enumerate(input_specs):
|
| 101 |
+
shape, dtype = input
|
| 102 |
+
name = "input_" + str(index)
|
| 103 |
+
inputs.append([name, str(dtype), str(shape)])
|
| 104 |
+
ml_type = _convert_to_mil_type(shape, dtype, name)
|
| 105 |
+
mil_inputs.append(ml_type)
|
| 106 |
+
model = torch.jit.RecursiveScriptModule._construct(script_module, lambda x: None)
|
| 107 |
+
mlmodel = ct.convert(model, inputs=mil_inputs, convert_to=convert_to)
|
| 108 |
+
|
| 109 |
+
if quantization_mode != CoreMLQuantizationMode.NONE:
|
| 110 |
+
quant_model_spec = quantization_utils.quantize_weights(
|
| 111 |
+
mlmodel, nbits=8, quantization_mode=quantization_mode
|
| 112 |
+
)
|
| 113 |
+
mlmodel = ct.models.MLModel(quant_model_spec)
|
| 114 |
+
|
| 115 |
+
spec = mlmodel.get_spec()
|
| 116 |
+
if len(spec.description.output) != len(output_specs): # type: ignore[attr-defined]
|
| 117 |
+
raise AssertionError(
|
| 118 |
+
f"Number of outputs in spec ({len(spec.description.output)}) " # type: ignore[attr-defined]
|
| 119 |
+
f"does not match output_specs ({len(output_specs)})"
|
| 120 |
+
)
|
| 121 |
+
outputs = []
|
| 122 |
+
for index, output in enumerate(output_specs):
|
| 123 |
+
shape, dtype = output
|
| 124 |
+
name = spec.description.output[index].name # type: ignore[attr-defined]
|
| 125 |
+
outputs.append([name, str(dtype), str(shape)])
|
| 126 |
+
mlmodel = ct.models.model.MLModel(spec)
|
| 127 |
+
print(mlmodel)
|
| 128 |
+
|
| 129 |
+
if mlmodel_export_path is not None:
|
| 130 |
+
print(f"Saving CoreML .mlmodel file to {mlmodel_export_path}")
|
| 131 |
+
mlmodel.save(mlmodel_export_path)
|
| 132 |
+
|
| 133 |
+
config = {
|
| 134 |
+
"spec_ver": str(spec.specificationVersion), # type: ignore[attr-defined]
|
| 135 |
+
"backend": backend,
|
| 136 |
+
"allow_low_precision": str(allow_low_precision),
|
| 137 |
+
}
|
| 138 |
+
metadata = {
|
| 139 |
+
"coremltool_ver": mlmodel.user_defined_metadata[CT_METADATA_VERSION],
|
| 140 |
+
"torch_ver": mlmodel.user_defined_metadata[CT_METADATA_SOURCE],
|
| 141 |
+
}
|
| 142 |
+
coreml_compile_spec = {
|
| 143 |
+
"inputs": inputs,
|
| 144 |
+
"outputs": outputs,
|
| 145 |
+
"config": config,
|
| 146 |
+
"metadata": metadata,
|
| 147 |
+
}
|
| 148 |
+
mlmodel = spec.SerializeToString() # type: ignore[attr-defined]
|
| 149 |
+
|
| 150 |
+
return {
|
| 151 |
+
"model": mlmodel,
|
| 152 |
+
"hash": str(hashlib.sha256(mlmodel).hexdigest()),
|
| 153 |
+
"extra": json.dumps(coreml_compile_spec),
|
| 154 |
+
}
|
western_ai/usa/pytorch/torch/backends/_nnapi/__init__.py
ADDED
|
File without changes
|
western_ai/usa/pytorch/torch/backends/_nnapi/prepare.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-decorators
|
| 2 |
+
# mypy: allow-untyped-defs
|
| 3 |
+
|
| 4 |
+
from typing import Optional
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from torch.backends._nnapi.serializer import _NnapiSerializer
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
ANEURALNETWORKS_PREFER_LOW_POWER = 0
|
| 11 |
+
ANEURALNETWORKS_PREFER_FAST_SINGLE_ANSWER = 1
|
| 12 |
+
ANEURALNETWORKS_PREFER_SUSTAINED_SPEED = 2
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class NnapiModule(torch.nn.Module):
|
| 16 |
+
"""Torch Module that wraps an NNAPI Compilation.
|
| 17 |
+
|
| 18 |
+
This module handles preparing the weights, initializing the
|
| 19 |
+
NNAPI TorchBind object, and adjusting the memory formats
|
| 20 |
+
of all inputs and outputs.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
# _nnapi.Compilation is defined
|
| 24 |
+
comp: Optional[torch.classes._nnapi.Compilation] # type: ignore[name-defined] # noqa: UP045
|
| 25 |
+
weights: list[torch.Tensor]
|
| 26 |
+
out_templates: list[torch.Tensor]
|
| 27 |
+
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
shape_compute_module: torch.nn.Module,
|
| 31 |
+
ser_model: torch.Tensor,
|
| 32 |
+
weights: list[torch.Tensor],
|
| 33 |
+
inp_mem_fmts: list[int],
|
| 34 |
+
out_mem_fmts: list[int],
|
| 35 |
+
compilation_preference: int,
|
| 36 |
+
relax_f32_to_f16: bool,
|
| 37 |
+
):
|
| 38 |
+
super().__init__()
|
| 39 |
+
self.shape_compute_module = shape_compute_module
|
| 40 |
+
self.ser_model = ser_model
|
| 41 |
+
self.weights = weights
|
| 42 |
+
self.inp_mem_fmts = inp_mem_fmts
|
| 43 |
+
self.out_mem_fmts = out_mem_fmts
|
| 44 |
+
self.out_templates = []
|
| 45 |
+
self.comp = None
|
| 46 |
+
self.compilation_preference = compilation_preference
|
| 47 |
+
self.relax_f32_to_f16 = relax_f32_to_f16
|
| 48 |
+
|
| 49 |
+
@torch.jit.export
|
| 50 |
+
def init(self, args: list[torch.Tensor]):
|
| 51 |
+
if self.comp is not None:
|
| 52 |
+
raise AssertionError("comp must be None before initialization")
|
| 53 |
+
self.out_templates = self.shape_compute_module.prepare(self.ser_model, args) # type: ignore[operator]
|
| 54 |
+
self.weights = [w.contiguous() for w in self.weights]
|
| 55 |
+
comp = torch.classes._nnapi.Compilation()
|
| 56 |
+
comp.init2(
|
| 57 |
+
self.ser_model,
|
| 58 |
+
self.weights,
|
| 59 |
+
self.compilation_preference,
|
| 60 |
+
self.relax_f32_to_f16,
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
self.comp = comp
|
| 64 |
+
|
| 65 |
+
def forward(self, args: list[torch.Tensor]) -> list[torch.Tensor]:
|
| 66 |
+
if self.comp is None:
|
| 67 |
+
self.init(args)
|
| 68 |
+
comp = self.comp
|
| 69 |
+
if comp is None:
|
| 70 |
+
raise AssertionError("comp must not be None")
|
| 71 |
+
outs = [torch.empty_like(out) for out in self.out_templates]
|
| 72 |
+
|
| 73 |
+
if len(args) != len(self.inp_mem_fmts):
|
| 74 |
+
raise AssertionError(
|
| 75 |
+
f"args length {len(args)} != inp_mem_fmts length {len(self.inp_mem_fmts)}"
|
| 76 |
+
)
|
| 77 |
+
fixed_args = []
|
| 78 |
+
for idx in range(len(args)):
|
| 79 |
+
fmt = self.inp_mem_fmts[idx]
|
| 80 |
+
# These constants match the values in DimOrder in serializer.py
|
| 81 |
+
# TODO: See if it's possible to use those directly.
|
| 82 |
+
if fmt == 0:
|
| 83 |
+
fixed_args.append(args[idx].contiguous())
|
| 84 |
+
elif fmt == 1:
|
| 85 |
+
fixed_args.append(args[idx].permute(0, 2, 3, 1).contiguous())
|
| 86 |
+
else:
|
| 87 |
+
raise ValueError("Invalid mem_fmt")
|
| 88 |
+
comp.run(fixed_args, outs)
|
| 89 |
+
if len(outs) != len(self.out_mem_fmts):
|
| 90 |
+
raise AssertionError(
|
| 91 |
+
f"outs length {len(outs)} != out_mem_fmts length {len(self.out_mem_fmts)}"
|
| 92 |
+
)
|
| 93 |
+
for idx in range(len(self.out_templates)):
|
| 94 |
+
fmt = self.out_mem_fmts[idx]
|
| 95 |
+
# These constants match the values in DimOrder in serializer.py
|
| 96 |
+
# TODO: See if it's possible to use those directly.
|
| 97 |
+
if fmt in (0, 2):
|
| 98 |
+
pass
|
| 99 |
+
elif fmt == 1:
|
| 100 |
+
outs[idx] = outs[idx].permute(0, 3, 1, 2)
|
| 101 |
+
else:
|
| 102 |
+
raise ValueError("Invalid mem_fmt")
|
| 103 |
+
return outs
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def convert_model_to_nnapi(
|
| 107 |
+
model,
|
| 108 |
+
inputs,
|
| 109 |
+
serializer=None,
|
| 110 |
+
return_shapes=None,
|
| 111 |
+
use_int16_for_qint16=False,
|
| 112 |
+
compilation_preference=ANEURALNETWORKS_PREFER_SUSTAINED_SPEED,
|
| 113 |
+
relax_f32_to_f16=False,
|
| 114 |
+
):
|
| 115 |
+
(
|
| 116 |
+
shape_compute_module,
|
| 117 |
+
ser_model_tensor,
|
| 118 |
+
used_weights,
|
| 119 |
+
inp_mem_fmts,
|
| 120 |
+
out_mem_fmts,
|
| 121 |
+
retval_count,
|
| 122 |
+
) = process_for_nnapi(
|
| 123 |
+
model, inputs, serializer, return_shapes, use_int16_for_qint16
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
nnapi_model = NnapiModule(
|
| 127 |
+
shape_compute_module,
|
| 128 |
+
ser_model_tensor,
|
| 129 |
+
used_weights,
|
| 130 |
+
inp_mem_fmts,
|
| 131 |
+
out_mem_fmts,
|
| 132 |
+
compilation_preference,
|
| 133 |
+
relax_f32_to_f16,
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
class NnapiInterfaceWrapper(torch.nn.Module):
|
| 137 |
+
"""NNAPI list-ifying and de-list-ifying wrapper.
|
| 138 |
+
|
| 139 |
+
NNAPI always expects a list of inputs and provides a list of outputs.
|
| 140 |
+
This module allows us to accept inputs as separate arguments.
|
| 141 |
+
It returns results as either a single tensor or tuple,
|
| 142 |
+
matching the original module.
|
| 143 |
+
"""
|
| 144 |
+
|
| 145 |
+
def __init__(self, mod):
|
| 146 |
+
super().__init__()
|
| 147 |
+
self.mod = mod
|
| 148 |
+
|
| 149 |
+
wrapper_model_py = NnapiInterfaceWrapper(nnapi_model)
|
| 150 |
+
wrapper_model = torch.jit.script(wrapper_model_py)
|
| 151 |
+
# TODO: Maybe make these names match the original.
|
| 152 |
+
arg_list = ", ".join(f"arg_{idx}" for idx in range(len(inputs)))
|
| 153 |
+
if retval_count < 0:
|
| 154 |
+
ret_expr = "retvals[0]"
|
| 155 |
+
else:
|
| 156 |
+
ret_expr = "".join(f"retvals[{idx}], " for idx in range(retval_count))
|
| 157 |
+
wrapper_model.define(
|
| 158 |
+
f"def forward(self, {arg_list}):\n"
|
| 159 |
+
f" retvals = self.mod([{arg_list}])\n"
|
| 160 |
+
f" return {ret_expr}\n"
|
| 161 |
+
)
|
| 162 |
+
return wrapper_model
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def process_for_nnapi(
|
| 166 |
+
model, inputs, serializer=None, return_shapes=None, use_int16_for_qint16=False
|
| 167 |
+
):
|
| 168 |
+
model = torch.jit.freeze(model)
|
| 169 |
+
|
| 170 |
+
if isinstance(inputs, torch.Tensor):
|
| 171 |
+
inputs = [inputs]
|
| 172 |
+
|
| 173 |
+
serializer = serializer or _NnapiSerializer(
|
| 174 |
+
config=None, use_int16_for_qint16=use_int16_for_qint16
|
| 175 |
+
)
|
| 176 |
+
(
|
| 177 |
+
ser_model,
|
| 178 |
+
used_weights,
|
| 179 |
+
inp_mem_fmts,
|
| 180 |
+
out_mem_fmts,
|
| 181 |
+
shape_compute_lines,
|
| 182 |
+
retval_count,
|
| 183 |
+
) = serializer.serialize_model(model, inputs, return_shapes)
|
| 184 |
+
ser_model_tensor = torch.tensor(ser_model, dtype=torch.int32)
|
| 185 |
+
|
| 186 |
+
# We have to create a new class here every time this function is called
|
| 187 |
+
# because module.define adds a method to the *class*, not the instance.
|
| 188 |
+
class ShapeComputeModule(torch.nn.Module):
|
| 189 |
+
"""Code-gen-ed module for tensor shape computation.
|
| 190 |
+
|
| 191 |
+
module.prepare will mutate ser_model according to the computed operand
|
| 192 |
+
shapes, based on the shapes of args. Returns a list of output templates.
|
| 193 |
+
"""
|
| 194 |
+
|
| 195 |
+
shape_compute_module = torch.jit.script(ShapeComputeModule())
|
| 196 |
+
real_shape_compute_lines = [
|
| 197 |
+
"def prepare(self, ser_model: torch.Tensor, args: List[torch.Tensor]) -> List[torch.Tensor]:\n",
|
| 198 |
+
] + [f" {line}\n" for line in shape_compute_lines]
|
| 199 |
+
shape_compute_module.define("".join(real_shape_compute_lines))
|
| 200 |
+
|
| 201 |
+
return (
|
| 202 |
+
shape_compute_module,
|
| 203 |
+
ser_model_tensor,
|
| 204 |
+
used_weights,
|
| 205 |
+
inp_mem_fmts,
|
| 206 |
+
out_mem_fmts,
|
| 207 |
+
retval_count,
|
| 208 |
+
)
|
western_ai/usa/pytorch/torch/backends/_nnapi/serializer.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
western_ai/usa/pytorch/torch/backends/cpu/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
__all__ = [
|
| 5 |
+
"get_cpu_capability",
|
| 6 |
+
]
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def get_cpu_capability() -> str:
|
| 10 |
+
r"""Return cpu capability as a string value.
|
| 11 |
+
|
| 12 |
+
Possible values:
|
| 13 |
+
- "DEFAULT"
|
| 14 |
+
- "VSX"
|
| 15 |
+
- "Z VECTOR"
|
| 16 |
+
- "NO AVX"
|
| 17 |
+
- "AVX2"
|
| 18 |
+
- "AVX512"
|
| 19 |
+
- "SVE256"
|
| 20 |
+
"""
|
| 21 |
+
return torch._C._get_cpu_capability()
|
western_ai/usa/pytorch/torch/backends/cuda/__init__.py
ADDED
|
@@ -0,0 +1,711 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import contextlib
|
| 3 |
+
from typing import Any
|
| 4 |
+
from typing_extensions import deprecated
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
__all__ = [
|
| 10 |
+
"is_built",
|
| 11 |
+
"cuFFTPlanCacheAttrContextProp",
|
| 12 |
+
"cuFFTPlanCache",
|
| 13 |
+
"cuFFTPlanCacheManager",
|
| 14 |
+
"cuBLASModule",
|
| 15 |
+
"preferred_linalg_library",
|
| 16 |
+
"preferred_blas_library",
|
| 17 |
+
"cublas_workspace_size",
|
| 18 |
+
"cublaslt_workspace_size",
|
| 19 |
+
"blas_workspace_size",
|
| 20 |
+
"preferred_rocm_fa_library",
|
| 21 |
+
"is_ck_sdpa_available",
|
| 22 |
+
"cufft_plan_cache",
|
| 23 |
+
"matmul",
|
| 24 |
+
"SDPAParams",
|
| 25 |
+
"enable_cudnn_sdp",
|
| 26 |
+
"cudnn_sdp_enabled",
|
| 27 |
+
"enable_flash_sdp",
|
| 28 |
+
"flash_sdp_enabled",
|
| 29 |
+
"enable_mem_efficient_sdp",
|
| 30 |
+
"mem_efficient_sdp_enabled",
|
| 31 |
+
"math_sdp_enabled",
|
| 32 |
+
"enable_math_sdp",
|
| 33 |
+
"allow_fp16_bf16_reduction_math_sdp",
|
| 34 |
+
"fp16_bf16_reduction_math_sdp_allowed",
|
| 35 |
+
"is_flash_attention_available",
|
| 36 |
+
"can_use_flash_attention",
|
| 37 |
+
"can_use_efficient_attention",
|
| 38 |
+
"can_use_cudnn_attention",
|
| 39 |
+
"sdp_kernel",
|
| 40 |
+
]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def is_built():
|
| 44 |
+
r"""
|
| 45 |
+
Return whether PyTorch is built with CUDA support.
|
| 46 |
+
|
| 47 |
+
Note that this doesn't necessarily mean CUDA is available; just that if this PyTorch
|
| 48 |
+
binary were run on a machine with working CUDA drivers and devices, we would be able to use it.
|
| 49 |
+
"""
|
| 50 |
+
return torch._C._has_cuda
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class cuFFTPlanCacheAttrContextProp:
|
| 54 |
+
# Like regular ContextProp, but uses the `.device_index` attribute from the
|
| 55 |
+
# calling object as the first argument to the getter and setter.
|
| 56 |
+
def __init__(self, getter, setter):
|
| 57 |
+
self.getter = getter
|
| 58 |
+
self.setter = setter
|
| 59 |
+
|
| 60 |
+
def __get__(self, obj, objtype):
|
| 61 |
+
return self.getter(obj.device_index)
|
| 62 |
+
|
| 63 |
+
def __set__(self, obj, val):
|
| 64 |
+
if isinstance(self.setter, str):
|
| 65 |
+
raise RuntimeError(self.setter)
|
| 66 |
+
self.setter(obj.device_index, val)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class cuFFTPlanCache:
|
| 70 |
+
r"""
|
| 71 |
+
Represent a specific plan cache for a specific `device_index`.
|
| 72 |
+
|
| 73 |
+
The attributes `size` and `max_size`, and method `clear`, can fetch and/ or
|
| 74 |
+
change properties of the C++ cuFFT plan cache.
|
| 75 |
+
"""
|
| 76 |
+
|
| 77 |
+
def __init__(self, device_index):
|
| 78 |
+
self.device_index = device_index
|
| 79 |
+
|
| 80 |
+
size = cuFFTPlanCacheAttrContextProp(
|
| 81 |
+
torch._cufft_get_plan_cache_size,
|
| 82 |
+
".size is a read-only property showing the number of plans currently in the "
|
| 83 |
+
"cache. To change the cache capacity, set cufft_plan_cache.max_size.",
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
max_size = cuFFTPlanCacheAttrContextProp(
|
| 87 |
+
torch._cufft_get_plan_cache_max_size, torch._cufft_set_plan_cache_max_size
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
def clear(self):
|
| 91 |
+
return torch._cufft_clear_plan_cache(self.device_index)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class cuFFTPlanCacheManager:
|
| 95 |
+
r"""
|
| 96 |
+
Represent all cuFFT plan caches, return the cuFFTPlanCache for a given device when indexed.
|
| 97 |
+
|
| 98 |
+
Finally, this object, when used directly as a `cuFFTPlanCache` object (e.g.,
|
| 99 |
+
setting the `.max_size`) attribute, the current device's cuFFT plan cache is
|
| 100 |
+
used.
|
| 101 |
+
"""
|
| 102 |
+
|
| 103 |
+
__initialized = False
|
| 104 |
+
|
| 105 |
+
def __init__(self):
|
| 106 |
+
self.caches = []
|
| 107 |
+
self.__initialized = True
|
| 108 |
+
|
| 109 |
+
def __getitem__(self, device):
|
| 110 |
+
index = torch.cuda._utils._get_device_index(device)
|
| 111 |
+
if index < 0 or index >= torch.cuda.device_count():
|
| 112 |
+
raise RuntimeError(
|
| 113 |
+
f"cufft_plan_cache: expected 0 <= device index < {torch.cuda.device_count()}, but got "
|
| 114 |
+
f"device with index {index}"
|
| 115 |
+
)
|
| 116 |
+
if len(self.caches) == 0:
|
| 117 |
+
self.caches.extend(
|
| 118 |
+
cuFFTPlanCache(index) for index in range(torch.cuda.device_count())
|
| 119 |
+
)
|
| 120 |
+
return self.caches[index]
|
| 121 |
+
|
| 122 |
+
def __getattr__(self, name):
|
| 123 |
+
return getattr(self[torch.cuda.current_device()], name)
|
| 124 |
+
|
| 125 |
+
def __setattr__(self, name, value):
|
| 126 |
+
if self.__initialized:
|
| 127 |
+
return setattr(self[torch.cuda.current_device()], name, value)
|
| 128 |
+
else:
|
| 129 |
+
return super().__setattr__(name, value)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class cuBLASModule:
|
| 133 |
+
@staticmethod
|
| 134 |
+
def _parse_reduction_setting(value: Any, attr_name: str) -> tuple[bool, bool]:
|
| 135 |
+
def _ensure_bool(obj: Any, which: str) -> bool:
|
| 136 |
+
if isinstance(obj, bool):
|
| 137 |
+
return obj
|
| 138 |
+
raise TypeError(
|
| 139 |
+
f"{attr_name} expects a bool for {which}, but got {type(obj)!r}"
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
if isinstance(value, bool):
|
| 143 |
+
return value, True
|
| 144 |
+
if isinstance(value, (list, tuple)):
|
| 145 |
+
if not value:
|
| 146 |
+
raise TypeError(f"{attr_name} expects at least one boolean argument")
|
| 147 |
+
if len(value) > 2:
|
| 148 |
+
raise TypeError(f"{attr_name} expects at most two boolean arguments")
|
| 149 |
+
allow_reduced_precision = _ensure_bool(value[0], "allow_reduced_precision")
|
| 150 |
+
if len(value) == 1:
|
| 151 |
+
return allow_reduced_precision, True
|
| 152 |
+
allow_splitk = _ensure_bool(value[1], "allow_splitk")
|
| 153 |
+
return allow_reduced_precision, allow_splitk
|
| 154 |
+
raise TypeError(
|
| 155 |
+
f"{attr_name} expects a bool or a tuple/list of bools, but got {type(value)!r}"
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
def __getattr__(self, name):
|
| 159 |
+
if name == "allow_tf32":
|
| 160 |
+
return torch._C._get_cublas_allow_tf32()
|
| 161 |
+
elif name == "allow_fp16_reduced_precision_reduction":
|
| 162 |
+
allow_reduced_precision, _ = (
|
| 163 |
+
torch._C._get_cublas_allow_fp16_reduced_precision_reduction()
|
| 164 |
+
)
|
| 165 |
+
return allow_reduced_precision
|
| 166 |
+
elif name == "allow_fp16_reduced_precision_reduction_split_k":
|
| 167 |
+
_, allow_splitk = (
|
| 168 |
+
torch._C._get_cublas_allow_fp16_reduced_precision_reduction()
|
| 169 |
+
)
|
| 170 |
+
return allow_splitk
|
| 171 |
+
elif name == "allow_bf16_reduced_precision_reduction":
|
| 172 |
+
allow_reduced_precision, _ = (
|
| 173 |
+
torch._C._get_cublas_allow_bf16_reduced_precision_reduction()
|
| 174 |
+
)
|
| 175 |
+
return allow_reduced_precision
|
| 176 |
+
elif name == "allow_bf16_reduced_precision_reduction_split_k":
|
| 177 |
+
_, allow_splitk = (
|
| 178 |
+
torch._C._get_cublas_allow_bf16_reduced_precision_reduction()
|
| 179 |
+
)
|
| 180 |
+
return allow_splitk
|
| 181 |
+
elif name == "allow_fp16_accumulation":
|
| 182 |
+
return torch._C._get_cublas_allow_fp16_accumulation()
|
| 183 |
+
elif name == "fp32_precision":
|
| 184 |
+
return torch._C._get_fp32_precision_getter("cuda", "matmul")
|
| 185 |
+
raise AttributeError("Unknown attribute " + name)
|
| 186 |
+
|
| 187 |
+
def __setattr__(self, name, value):
|
| 188 |
+
if name == "allow_tf32":
|
| 189 |
+
return torch._C._set_cublas_allow_tf32(value)
|
| 190 |
+
elif name == "allow_fp16_reduced_precision_reduction":
|
| 191 |
+
allow_reduced_precision, allow_splitk = self._parse_reduction_setting(
|
| 192 |
+
value, "allow_fp16_reduced_precision_reduction"
|
| 193 |
+
)
|
| 194 |
+
return torch._C._set_cublas_allow_fp16_reduced_precision_reduction(
|
| 195 |
+
allow_reduced_precision,
|
| 196 |
+
allow_splitk,
|
| 197 |
+
)
|
| 198 |
+
elif name == "allow_bf16_reduced_precision_reduction":
|
| 199 |
+
allow_reduced_precision, allow_splitk = self._parse_reduction_setting(
|
| 200 |
+
value, "allow_bf16_reduced_precision_reduction"
|
| 201 |
+
)
|
| 202 |
+
return torch._C._set_cublas_allow_bf16_reduced_precision_reduction(
|
| 203 |
+
allow_reduced_precision,
|
| 204 |
+
allow_splitk,
|
| 205 |
+
)
|
| 206 |
+
elif name == "allow_fp16_accumulation":
|
| 207 |
+
return torch._C._set_cublas_allow_fp16_accumulation(value)
|
| 208 |
+
elif name == "fp32_precision":
|
| 209 |
+
return torch._C._set_fp32_precision_setter("cuda", "matmul", value)
|
| 210 |
+
raise AttributeError("Unknown attribute " + name)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
_LinalgBackends = {
|
| 214 |
+
"default": torch._C._LinalgBackend.Default,
|
| 215 |
+
"cusolver": torch._C._LinalgBackend.Cusolver,
|
| 216 |
+
"magma": torch._C._LinalgBackend.Magma,
|
| 217 |
+
}
|
| 218 |
+
_LinalgBackends_str = ", ".join(_LinalgBackends.keys())
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def preferred_linalg_library(
|
| 222 |
+
backend: None | str | torch._C._LinalgBackend = None,
|
| 223 |
+
) -> torch._C._LinalgBackend:
|
| 224 |
+
r"""
|
| 225 |
+
Override the heuristic PyTorch uses to choose between cuSOLVER and MAGMA for CUDA linear algebra operations.
|
| 226 |
+
|
| 227 |
+
.. warning:: This flag is experimental and subject to change.
|
| 228 |
+
|
| 229 |
+
When PyTorch runs a CUDA linear algebra operation it often uses the cuSOLVER or MAGMA libraries,
|
| 230 |
+
and if both are available it decides which to use with a heuristic.
|
| 231 |
+
This flag (a :class:`str`) allows overriding those heuristics.
|
| 232 |
+
|
| 233 |
+
* If `"cusolver"` is set then cuSOLVER will be used wherever possible.
|
| 234 |
+
* If `"magma"` is set then MAGMA will be used wherever possible.
|
| 235 |
+
* If `"default"` (the default) is set then heuristics will be used to pick between
|
| 236 |
+
cuSOLVER and MAGMA if both are available.
|
| 237 |
+
* When no input is given, this function returns the currently preferred library.
|
| 238 |
+
* User may use the environment variable TORCH_LINALG_PREFER_CUSOLVER=1 to set the preferred library to cuSOLVER
|
| 239 |
+
globally.
|
| 240 |
+
This flag only sets the initial value of the preferred library and the preferred library
|
| 241 |
+
may still be overridden by this function call later in your script.
|
| 242 |
+
|
| 243 |
+
Note: When a library is preferred other libraries may still be used if the preferred library
|
| 244 |
+
doesn't implement the operation(s) called.
|
| 245 |
+
This flag may achieve better performance if PyTorch's heuristic library selection is incorrect
|
| 246 |
+
for your application's inputs.
|
| 247 |
+
|
| 248 |
+
Currently supported linalg operators:
|
| 249 |
+
|
| 250 |
+
* :func:`torch.linalg.inv`
|
| 251 |
+
* :func:`torch.linalg.inv_ex`
|
| 252 |
+
* :func:`torch.linalg.cholesky`
|
| 253 |
+
* :func:`torch.linalg.cholesky_ex`
|
| 254 |
+
* :func:`torch.cholesky_solve`
|
| 255 |
+
* :func:`torch.cholesky_inverse`
|
| 256 |
+
* :func:`torch.linalg.lu_factor`
|
| 257 |
+
* :func:`torch.linalg.lu`
|
| 258 |
+
* :func:`torch.linalg.lu_solve`
|
| 259 |
+
* :func:`torch.linalg.qr`
|
| 260 |
+
* :func:`torch.linalg.eigh`
|
| 261 |
+
* :func:`torch.linalg.eighvals`
|
| 262 |
+
* :func:`torch.linalg.svd`
|
| 263 |
+
* :func:`torch.linalg.svdvals`
|
| 264 |
+
"""
|
| 265 |
+
if backend is None:
|
| 266 |
+
pass
|
| 267 |
+
elif isinstance(backend, str):
|
| 268 |
+
if backend not in _LinalgBackends:
|
| 269 |
+
raise RuntimeError(
|
| 270 |
+
f"Unknown input value. Choose from: {_LinalgBackends_str}."
|
| 271 |
+
)
|
| 272 |
+
torch._C._set_linalg_preferred_backend(_LinalgBackends[backend])
|
| 273 |
+
elif isinstance(backend, torch._C._LinalgBackend):
|
| 274 |
+
torch._C._set_linalg_preferred_backend(backend)
|
| 275 |
+
else:
|
| 276 |
+
raise RuntimeError("Unknown input value type.")
|
| 277 |
+
|
| 278 |
+
return torch._C._get_linalg_preferred_backend()
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
_BlasBackends = {
|
| 282 |
+
"default": torch._C._BlasBackend.Default,
|
| 283 |
+
"cublas": torch._C._BlasBackend.Cublas,
|
| 284 |
+
"hipblas": torch._C._BlasBackend.Cublas, # alias
|
| 285 |
+
"cublaslt": torch._C._BlasBackend.Cublaslt,
|
| 286 |
+
"hipblaslt": torch._C._BlasBackend.Cublaslt, # alias
|
| 287 |
+
"ck": torch._C._BlasBackend.Ck,
|
| 288 |
+
}
|
| 289 |
+
_BlasBackends_str = ", ".join(_BlasBackends.keys())
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
def preferred_blas_library(
|
| 293 |
+
backend: None | str | torch._C._BlasBackend = None,
|
| 294 |
+
) -> torch._C._BlasBackend:
|
| 295 |
+
r"""
|
| 296 |
+
Override the library PyTorch uses for BLAS operations. Choose between cuBLAS, cuBLASLt, and CK [ROCm-only].
|
| 297 |
+
|
| 298 |
+
.. warning:: This flag is experimental and subject to change.
|
| 299 |
+
|
| 300 |
+
When PyTorch runs a CUDA BLAS operation it defaults to cuBLAS even if both cuBLAS and cuBLASLt are available.
|
| 301 |
+
For PyTorch built for ROCm, hipBLAS, hipBLASLt, and CK may offer different performance.
|
| 302 |
+
This flag (a :class:`str`) allows overriding which BLAS library to use.
|
| 303 |
+
|
| 304 |
+
* If `"cublas"` is set then cuBLAS will be used wherever possible.
|
| 305 |
+
* If `"cublaslt"` is set then cuBLASLt will be used wherever possible.
|
| 306 |
+
* If `"ck"` is set then CK will be used wherever possible.
|
| 307 |
+
* If `"default"` (the default) is set then heuristics will be used to pick between the other options.
|
| 308 |
+
* When no input is given, this function returns the currently preferred library.
|
| 309 |
+
* User may use the environment variable TORCH_BLAS_PREFER_CUBLASLT=1 to set the preferred library to cuBLASLt
|
| 310 |
+
globally.
|
| 311 |
+
This flag only sets the initial value of the preferred library and the preferred library
|
| 312 |
+
may still be overridden by this function call later in your script.
|
| 313 |
+
|
| 314 |
+
Note: When a library is preferred other libraries may still be used if the preferred library
|
| 315 |
+
doesn't implement the operation(s) called.
|
| 316 |
+
This flag may achieve better performance if PyTorch's library selection is incorrect
|
| 317 |
+
for your application's inputs.
|
| 318 |
+
|
| 319 |
+
"""
|
| 320 |
+
if backend is None:
|
| 321 |
+
pass
|
| 322 |
+
elif isinstance(backend, str):
|
| 323 |
+
if backend not in _BlasBackends:
|
| 324 |
+
raise RuntimeError(
|
| 325 |
+
f"Unknown input value. Choose from: {_BlasBackends_str}."
|
| 326 |
+
)
|
| 327 |
+
torch._C._set_blas_preferred_backend(_BlasBackends[backend])
|
| 328 |
+
elif isinstance(backend, torch._C._BlasBackend):
|
| 329 |
+
torch._C._set_blas_preferred_backend(backend)
|
| 330 |
+
else:
|
| 331 |
+
raise RuntimeError("Unknown input value type.")
|
| 332 |
+
|
| 333 |
+
return torch._C._get_blas_preferred_backend()
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def cublas_workspace_size(size: None | int = None) -> int:
|
| 337 |
+
r"""Query or set the cuBLAS workspace size in bytes.
|
| 338 |
+
|
| 339 |
+
When called with no arguments, returns the current workspace size.
|
| 340 |
+
When called with a size argument, sets the workspace size and returns the new value.
|
| 341 |
+
Setting the workspace size will take precedence over the CUBLAS_WORKSPACE_CONFIG environment variable.
|
| 342 |
+
Changes take effect lazily: only handles used after the change get new workspaces.
|
| 343 |
+
|
| 344 |
+
Args:
|
| 345 |
+
size (int, optional): workspace size in bytes. Must be non-negative.
|
| 346 |
+
|
| 347 |
+
Returns:
|
| 348 |
+
int: the current (or newly set) workspace size in bytes.
|
| 349 |
+
"""
|
| 350 |
+
if size is not None:
|
| 351 |
+
torch._C._cuda_setCublasWorkspaceSize(size)
|
| 352 |
+
return torch._C._cuda_getCublasWorkspaceSize()
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
def cublaslt_workspace_size(size: None | int = None) -> int:
|
| 356 |
+
r"""Query or set the cuBLASLt workspace size in bytes.
|
| 357 |
+
|
| 358 |
+
When called with no arguments, returns the current workspace size.
|
| 359 |
+
When called with a size argument, sets the workspace size and returns the new value.
|
| 360 |
+
Setting the workspace size will take precedence over the CUBLASLT_WORKSPACE_SIZE environment variable.
|
| 361 |
+
Changes take effect lazily: only handles used after the change get new workspaces.
|
| 362 |
+
|
| 363 |
+
Args:
|
| 364 |
+
size (int, optional): workspace size in bytes. Must be non-negative.
|
| 365 |
+
|
| 366 |
+
Returns:
|
| 367 |
+
int: the current (or newly set) workspace size in bytes.
|
| 368 |
+
"""
|
| 369 |
+
if size is not None:
|
| 370 |
+
torch._C._cuda_setCublasLtWorkspaceSize(size)
|
| 371 |
+
return torch._C._cuda_getCublasLtWorkspaceSize()
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def blas_workspace_size(
|
| 375 |
+
size: None | int = None,
|
| 376 |
+
backend: None | str | torch._C._BlasBackend = None,
|
| 377 |
+
) -> int:
|
| 378 |
+
r"""Query or set the BLAS workspace size for a given backend.
|
| 379 |
+
|
| 380 |
+
Convenience wrapper that dispatches to :func:`cublas_workspace_size` or
|
| 381 |
+
:func:`cublaslt_workspace_size` depending on the backend.
|
| 382 |
+
|
| 383 |
+
When *backend* is ``None`` the current :func:`preferred_blas_library` is
|
| 384 |
+
used. ``Default`` is resolved to the platform's default backend (cuBLAS
|
| 385 |
+
on NVIDIA, potentially hipBLASLt on supported ROCm architectures).
|
| 386 |
+
|
| 387 |
+
.. note::
|
| 388 |
+
|
| 389 |
+
When ``TORCH_CUBLASLT_UNIFIED_WORKSPACE`` is enabled (the default on
|
| 390 |
+
open-source CUDA builds), the cuBLASLt workspace is capped at the
|
| 391 |
+
cuBLAS workspace size and physically reuses the same allocation.
|
| 392 |
+
Setting a large cuBLASLt workspace via this function will therefore
|
| 393 |
+
*not* increase memory beyond the cuBLAS workspace size.
|
| 394 |
+
|
| 395 |
+
.. note::
|
| 396 |
+
|
| 397 |
+
Setting the workspace size for the cublas backend will take precedence
|
| 398 |
+
over the CUBLAS_WORKSPACE_CONFIG environment variable, and setting the
|
| 399 |
+
workspace size for the cublaslt backend will take precedence over the
|
| 400 |
+
CUBLASLT_WORKSPACE_SIZE environment variable.
|
| 401 |
+
|
| 402 |
+
Args:
|
| 403 |
+
size (int, optional): workspace size in bytes. Must be non-negative.
|
| 404 |
+
When omitted the current size is returned without modification.
|
| 405 |
+
backend (str | torch._C._BlasBackend, optional): which backend's
|
| 406 |
+
workspace to query/set. Accepts the same strings as
|
| 407 |
+
:func:`preferred_blas_library` (e.g. ``"cublas"``, ``"cublaslt"``).
|
| 408 |
+
|
| 409 |
+
Returns:
|
| 410 |
+
int: the current (or newly set) workspace size in bytes.
|
| 411 |
+
|
| 412 |
+
Raises:
|
| 413 |
+
RuntimeError: if the resolved backend is CK (no workspace concept).
|
| 414 |
+
"""
|
| 415 |
+
if backend is None:
|
| 416 |
+
resolved = preferred_blas_library()
|
| 417 |
+
elif isinstance(backend, str):
|
| 418 |
+
if backend not in _BlasBackends:
|
| 419 |
+
raise RuntimeError(
|
| 420 |
+
f"Unknown backend string. Choose from: {_BlasBackends_str}."
|
| 421 |
+
)
|
| 422 |
+
resolved = _BlasBackends[backend]
|
| 423 |
+
elif isinstance(backend, torch._C._BlasBackend):
|
| 424 |
+
resolved = backend
|
| 425 |
+
else:
|
| 426 |
+
raise RuntimeError("Unknown backend type.")
|
| 427 |
+
|
| 428 |
+
if resolved == torch._C._BlasBackend.Default:
|
| 429 |
+
resolved = torch._C._get_blas_default_backend()
|
| 430 |
+
|
| 431 |
+
if resolved == torch._C._BlasBackend.Ck:
|
| 432 |
+
raise RuntimeError("CK backend does not use a workspace.")
|
| 433 |
+
|
| 434 |
+
if resolved == torch._C._BlasBackend.Cublaslt:
|
| 435 |
+
return cublaslt_workspace_size(size)
|
| 436 |
+
return cublas_workspace_size(size)
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
_ROCmFABackends = {
|
| 440 |
+
"default": torch._C._ROCmFABackend.Default,
|
| 441 |
+
"aotriton": torch._C._ROCmFABackend.AOTriton,
|
| 442 |
+
"ck": torch._C._ROCmFABackend.Ck,
|
| 443 |
+
}
|
| 444 |
+
_ROCmFABackends_str = ", ".join(_ROCmFABackends.keys())
|
| 445 |
+
|
| 446 |
+
|
| 447 |
+
from torch._C import _SDPAParams as SDPAParams, _SDPBackend as SDPBackend
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def preferred_rocm_fa_library(
|
| 451 |
+
backend: None | str | torch._C._ROCmFABackend = None,
|
| 452 |
+
) -> torch._C._ROCmFABackend:
|
| 453 |
+
r"""
|
| 454 |
+
[ROCm-only]
|
| 455 |
+
Override the backend PyTorch uses in ROCm environments for Flash Attention. Choose between AOTriton and CK
|
| 456 |
+
|
| 457 |
+
.. warning:: This flag is experimental and subject to change.
|
| 458 |
+
|
| 459 |
+
When Flash Attention is enabled and desired, PyTorch defaults to using AOTriton as the backend.
|
| 460 |
+
This flag (a :class:`str`) allows users to override this backend to use composable_kernel
|
| 461 |
+
|
| 462 |
+
* If `"default"` is set then the default backend will be used wherever possible. Currently AOTriton.
|
| 463 |
+
* If `"aotriton"` is set then AOTriton will be used wherever possible.
|
| 464 |
+
* If `"ck"` is set then CK will be used wherever possible.
|
| 465 |
+
* When no input is given, this function returns the currently preferred library.
|
| 466 |
+
* User may use the environment variable TORCH_ROCM_FA_PREFER_CK=1 to set the preferred library to CK
|
| 467 |
+
globally.
|
| 468 |
+
|
| 469 |
+
Note: When a library is preferred other libraries may still be used if the preferred library
|
| 470 |
+
doesn't implement the operation(s) called.
|
| 471 |
+
This flag may achieve better performance if PyTorch's library selection is incorrect
|
| 472 |
+
for your application's inputs.
|
| 473 |
+
"""
|
| 474 |
+
if backend is None:
|
| 475 |
+
pass
|
| 476 |
+
elif isinstance(backend, str):
|
| 477 |
+
if backend not in _ROCmFABackends:
|
| 478 |
+
raise RuntimeError(
|
| 479 |
+
f"Unknown input value. Choose from: {_ROCmFABackends_str}."
|
| 480 |
+
)
|
| 481 |
+
torch._C._set_rocm_fa_preferred_backend(_ROCmFABackends[backend])
|
| 482 |
+
elif isinstance(backend, torch._C._ROCmFABackend):
|
| 483 |
+
torch._C._set_rocm_fa_preferred_backend(backend)
|
| 484 |
+
else:
|
| 485 |
+
raise ValueError(f"Unknown input value. Choose from: {_ROCmFABackends_str}.")
|
| 486 |
+
|
| 487 |
+
return torch._C._get_rocm_fa_preferred_backend()
|
| 488 |
+
|
| 489 |
+
|
| 490 |
+
# Set the __module__ attribute
|
| 491 |
+
SDPAParams.__module__ = "torch.backends.cuda"
|
| 492 |
+
SDPAParams.__name__ = "SDPAParams"
|
| 493 |
+
|
| 494 |
+
|
| 495 |
+
def is_ck_sdpa_available() -> bool:
|
| 496 |
+
r"""
|
| 497 |
+
.. warning:: This flag is beta and subject to change.
|
| 498 |
+
|
| 499 |
+
Returns whether composable_kernel may be used as the backend for
|
| 500 |
+
scaled-dot-product-attention.
|
| 501 |
+
"""
|
| 502 |
+
# pyrefly: ignore [missing-attribute]
|
| 503 |
+
return torch._C._is_ck_sdpa_available()
|
| 504 |
+
|
| 505 |
+
|
| 506 |
+
def flash_sdp_enabled():
|
| 507 |
+
r"""
|
| 508 |
+
.. warning:: This flag is beta and subject to change.
|
| 509 |
+
|
| 510 |
+
Returns whether flash scaled dot product attention is enabled or not.
|
| 511 |
+
"""
|
| 512 |
+
return torch._C._get_flash_sdp_enabled()
|
| 513 |
+
|
| 514 |
+
|
| 515 |
+
def enable_flash_sdp(enabled: bool):
|
| 516 |
+
r"""
|
| 517 |
+
.. warning:: This flag is beta and subject to change.
|
| 518 |
+
|
| 519 |
+
Enables or disables flash scaled dot product attention.
|
| 520 |
+
"""
|
| 521 |
+
torch._C._set_sdp_use_flash(enabled)
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def mem_efficient_sdp_enabled():
|
| 525 |
+
r"""
|
| 526 |
+
.. warning:: This flag is beta and subject to change.
|
| 527 |
+
|
| 528 |
+
Returns whether memory efficient scaled dot product attention is enabled or not.
|
| 529 |
+
"""
|
| 530 |
+
return torch._C._get_mem_efficient_sdp_enabled()
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def enable_mem_efficient_sdp(enabled: bool):
|
| 534 |
+
r"""
|
| 535 |
+
.. warning:: This flag is beta and subject to change.
|
| 536 |
+
|
| 537 |
+
Enables or disables memory efficient scaled dot product attention.
|
| 538 |
+
"""
|
| 539 |
+
torch._C._set_sdp_use_mem_efficient(enabled)
|
| 540 |
+
|
| 541 |
+
|
| 542 |
+
def math_sdp_enabled():
|
| 543 |
+
r"""
|
| 544 |
+
.. warning:: This flag is beta and subject to change.
|
| 545 |
+
|
| 546 |
+
Returns whether math scaled dot product attention is enabled or not.
|
| 547 |
+
"""
|
| 548 |
+
return torch._C._get_math_sdp_enabled()
|
| 549 |
+
|
| 550 |
+
|
| 551 |
+
def enable_math_sdp(enabled: bool):
|
| 552 |
+
r"""
|
| 553 |
+
.. warning:: This flag is beta and subject to change.
|
| 554 |
+
|
| 555 |
+
Enables or disables math scaled dot product attention.
|
| 556 |
+
"""
|
| 557 |
+
torch._C._set_sdp_use_math(enabled)
|
| 558 |
+
|
| 559 |
+
|
| 560 |
+
def allow_fp16_bf16_reduction_math_sdp(enabled: bool):
|
| 561 |
+
r"""
|
| 562 |
+
.. warning:: This flag is beta and subject to change.
|
| 563 |
+
|
| 564 |
+
Enables or disables fp16/bf16 reduction in math scaled dot product attention.
|
| 565 |
+
"""
|
| 566 |
+
torch._C._set_math_sdp_allow_fp16_bf16_reduction(enabled)
|
| 567 |
+
|
| 568 |
+
|
| 569 |
+
def fp16_bf16_reduction_math_sdp_allowed():
|
| 570 |
+
r"""
|
| 571 |
+
.. warning:: This flag is beta and subject to change.
|
| 572 |
+
|
| 573 |
+
Returns whether fp16/bf16 reduction in math scaled dot product attention is enabled or not.
|
| 574 |
+
"""
|
| 575 |
+
return torch._C._get_math_sdp_allow_fp16_bf16_reduction()
|
| 576 |
+
|
| 577 |
+
|
| 578 |
+
def is_flash_attention_available() -> bool:
|
| 579 |
+
r"""Check if PyTorch was built with FlashAttention for scaled_dot_product_attention.
|
| 580 |
+
|
| 581 |
+
Returns:
|
| 582 |
+
True if FlashAttention is built and available; otherwise, False.
|
| 583 |
+
|
| 584 |
+
Note:
|
| 585 |
+
This function is dependent on a CUDA-enabled build of PyTorch. It will return False
|
| 586 |
+
in non-CUDA environments.
|
| 587 |
+
"""
|
| 588 |
+
return torch._C._is_flash_attention_available()
|
| 589 |
+
|
| 590 |
+
|
| 591 |
+
def can_use_flash_attention(params: SDPAParams, debug: bool = False) -> bool:
|
| 592 |
+
r"""Check if FlashAttention can be utilized in scaled_dot_product_attention.
|
| 593 |
+
|
| 594 |
+
Args:
|
| 595 |
+
params: An instance of SDPAParams containing the tensors for query,
|
| 596 |
+
key, value, an optional attention mask, dropout rate, and
|
| 597 |
+
a flag indicating if the attention is causal.
|
| 598 |
+
debug: Whether to logging.warn debug information as to why FlashAttention could not be run.
|
| 599 |
+
Defaults to False.
|
| 600 |
+
|
| 601 |
+
Returns:
|
| 602 |
+
True if FlashAttention can be used with the given parameters; otherwise, False.
|
| 603 |
+
|
| 604 |
+
Note:
|
| 605 |
+
This function is dependent on a CUDA-enabled build of PyTorch. It will return False
|
| 606 |
+
in non-CUDA environments.
|
| 607 |
+
"""
|
| 608 |
+
return torch._C._can_use_flash_attention(params, debug)
|
| 609 |
+
|
| 610 |
+
|
| 611 |
+
def can_use_efficient_attention(params: SDPAParams, debug: bool = False) -> bool:
|
| 612 |
+
r"""Check if efficient_attention can be utilized in scaled_dot_product_attention.
|
| 613 |
+
|
| 614 |
+
Args:
|
| 615 |
+
params: An instance of SDPAParams containing the tensors for query,
|
| 616 |
+
key, value, an optional attention mask, dropout rate, and
|
| 617 |
+
a flag indicating if the attention is causal.
|
| 618 |
+
debug: Whether to logging.warn with information as to why efficient_attention could not be run.
|
| 619 |
+
Defaults to False.
|
| 620 |
+
|
| 621 |
+
Returns:
|
| 622 |
+
True if efficient_attention can be used with the given parameters; otherwise, False.
|
| 623 |
+
|
| 624 |
+
Note:
|
| 625 |
+
This function is dependent on a CUDA-enabled build of PyTorch. It will return False
|
| 626 |
+
in non-CUDA environments.
|
| 627 |
+
"""
|
| 628 |
+
return torch._C._can_use_mem_efficient_attention(params, debug)
|
| 629 |
+
|
| 630 |
+
|
| 631 |
+
def can_use_cudnn_attention(params: SDPAParams, debug: bool = False) -> bool:
|
| 632 |
+
r"""Check if cudnn_attention can be utilized in scaled_dot_product_attention.
|
| 633 |
+
|
| 634 |
+
Args:
|
| 635 |
+
params: An instance of SDPAParams containing the tensors for query,
|
| 636 |
+
key, value, an optional attention mask, dropout rate, and
|
| 637 |
+
a flag indicating if the attention is causal.
|
| 638 |
+
debug: Whether to logging.warn with information as to why cuDNN attention could not be run.
|
| 639 |
+
Defaults to False.
|
| 640 |
+
|
| 641 |
+
Returns:
|
| 642 |
+
True if cuDNN can be used with the given parameters; otherwise, False.
|
| 643 |
+
|
| 644 |
+
Note:
|
| 645 |
+
This function is dependent on a CUDA-enabled build of PyTorch. It will return False
|
| 646 |
+
in non-CUDA environments.
|
| 647 |
+
"""
|
| 648 |
+
return torch._C._can_use_cudnn_attention(params, debug)
|
| 649 |
+
|
| 650 |
+
|
| 651 |
+
def cudnn_sdp_enabled():
|
| 652 |
+
r"""
|
| 653 |
+
.. warning:: This flag is beta and subject to change.
|
| 654 |
+
|
| 655 |
+
Returns whether cuDNN scaled dot product attention is enabled or not.
|
| 656 |
+
"""
|
| 657 |
+
return torch._C._get_cudnn_sdp_enabled()
|
| 658 |
+
|
| 659 |
+
|
| 660 |
+
def enable_cudnn_sdp(enabled: bool):
|
| 661 |
+
r"""
|
| 662 |
+
.. warning:: This flag is beta and subject to change.
|
| 663 |
+
|
| 664 |
+
Enables or disables cuDNN scaled dot product attention.
|
| 665 |
+
"""
|
| 666 |
+
torch._C._set_sdp_use_cudnn(enabled)
|
| 667 |
+
|
| 668 |
+
|
| 669 |
+
@contextlib.contextmanager
|
| 670 |
+
@deprecated(
|
| 671 |
+
(
|
| 672 |
+
"`torch.backends.cuda.sdp_kernel()` is deprecated. "
|
| 673 |
+
"In the future, this context manager will be removed. "
|
| 674 |
+
"Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, "
|
| 675 |
+
"with updated signature."
|
| 676 |
+
),
|
| 677 |
+
category=FutureWarning,
|
| 678 |
+
)
|
| 679 |
+
def sdp_kernel(
|
| 680 |
+
enable_flash: bool = True,
|
| 681 |
+
enable_math: bool = True,
|
| 682 |
+
enable_mem_efficient: bool = True,
|
| 683 |
+
enable_cudnn: bool = True,
|
| 684 |
+
):
|
| 685 |
+
r"""
|
| 686 |
+
.. warning:: This flag is beta and subject to change.
|
| 687 |
+
|
| 688 |
+
This context manager can be used to temporarily enable or disable any of the three backends for scaled dot product attention.
|
| 689 |
+
Upon exiting the context manager, the previous state of the flags will be restored.
|
| 690 |
+
"""
|
| 691 |
+
from torch.nn.attention import sdpa_kernel
|
| 692 |
+
|
| 693 |
+
backend_list = []
|
| 694 |
+
if enable_flash:
|
| 695 |
+
backend_list.append(SDPBackend.FLASH_ATTENTION)
|
| 696 |
+
if enable_mem_efficient:
|
| 697 |
+
backend_list.append(SDPBackend.EFFICIENT_ATTENTION)
|
| 698 |
+
if enable_math:
|
| 699 |
+
backend_list.append(SDPBackend.MATH)
|
| 700 |
+
if enable_cudnn:
|
| 701 |
+
backend_list.append(SDPBackend.CUDNN_ATTENTION)
|
| 702 |
+
|
| 703 |
+
with sdpa_kernel(backend_list) as context:
|
| 704 |
+
try:
|
| 705 |
+
yield context
|
| 706 |
+
finally:
|
| 707 |
+
pass
|
| 708 |
+
|
| 709 |
+
|
| 710 |
+
cufft_plan_cache = cuFFTPlanCacheManager()
|
| 711 |
+
matmul = cuBLASModule()
|
western_ai/usa/pytorch/torch/backends/cudnn/__init__.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import os
|
| 3 |
+
import sys
|
| 4 |
+
import warnings
|
| 5 |
+
from contextlib import contextmanager
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from torch.backends import (
|
| 10 |
+
__allow_nonbracketed_mutation,
|
| 11 |
+
_FP32Precision,
|
| 12 |
+
_get_fp32_precision_getter,
|
| 13 |
+
_set_fp32_precision_setter,
|
| 14 |
+
ContextProp,
|
| 15 |
+
PropModule,
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
from . import rnn
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
try:
|
| 22 |
+
from torch._C import _cudnn
|
| 23 |
+
except ImportError:
|
| 24 |
+
_cudnn = None # type: ignore[assignment]
|
| 25 |
+
|
| 26 |
+
# Write:
|
| 27 |
+
#
|
| 28 |
+
# torch.backends.cudnn.enabled = False
|
| 29 |
+
#
|
| 30 |
+
# to globally disable CuDNN/MIOpen
|
| 31 |
+
|
| 32 |
+
__cudnn_version: int | None = None
|
| 33 |
+
|
| 34 |
+
if _cudnn is not None:
|
| 35 |
+
|
| 36 |
+
def _init():
|
| 37 |
+
global __cudnn_version
|
| 38 |
+
if __cudnn_version is None:
|
| 39 |
+
# pyrefly: ignore [missing-attribute]
|
| 40 |
+
__cudnn_version = _cudnn.getVersionInt()
|
| 41 |
+
# pyrefly: ignore [missing-attribute]
|
| 42 |
+
runtime_version = _cudnn.getRuntimeVersion()
|
| 43 |
+
# pyrefly: ignore [missing-attribute]
|
| 44 |
+
compile_version = _cudnn.getCompileVersion()
|
| 45 |
+
runtime_major, runtime_minor, _ = runtime_version
|
| 46 |
+
compile_major, compile_minor, _ = compile_version
|
| 47 |
+
# Different major versions are always incompatible
|
| 48 |
+
# Starting with cuDNN 7, minor versions are backwards-compatible
|
| 49 |
+
# Not sure about MIOpen (ROCm), so always do a strict check
|
| 50 |
+
if runtime_major != compile_major:
|
| 51 |
+
cudnn_compatible = False
|
| 52 |
+
# pyrefly: ignore [missing-attribute]
|
| 53 |
+
elif runtime_major < 7 or not _cudnn.is_cuda:
|
| 54 |
+
cudnn_compatible = runtime_minor == compile_minor
|
| 55 |
+
else:
|
| 56 |
+
cudnn_compatible = runtime_minor >= compile_minor
|
| 57 |
+
if not cudnn_compatible:
|
| 58 |
+
if os.environ.get("PYTORCH_SKIP_CUDNN_COMPATIBILITY_CHECK", "0") == "1":
|
| 59 |
+
return True
|
| 60 |
+
base_error_msg = (
|
| 61 |
+
f"cuDNN version incompatibility: "
|
| 62 |
+
f"PyTorch was compiled against {compile_version} "
|
| 63 |
+
f"but found runtime version {runtime_version}. "
|
| 64 |
+
f"PyTorch already comes bundled with cuDNN. "
|
| 65 |
+
f"One option to resolving this error is to ensure PyTorch "
|
| 66 |
+
f"can find the bundled cuDNN. "
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
if "LD_LIBRARY_PATH" in os.environ:
|
| 70 |
+
ld_library_path = os.environ.get("LD_LIBRARY_PATH", "")
|
| 71 |
+
if any(
|
| 72 |
+
substring in ld_library_path for substring in ["cuda", "cudnn"]
|
| 73 |
+
):
|
| 74 |
+
raise RuntimeError(
|
| 75 |
+
f"{base_error_msg}"
|
| 76 |
+
f"Looks like your LD_LIBRARY_PATH contains incompatible version of cudnn. "
|
| 77 |
+
f"Please either remove it from the path or install cudnn {compile_version}"
|
| 78 |
+
)
|
| 79 |
+
else:
|
| 80 |
+
raise RuntimeError(
|
| 81 |
+
f"{base_error_msg}"
|
| 82 |
+
f"one possibility is that there is a "
|
| 83 |
+
f"conflicting cuDNN in LD_LIBRARY_PATH."
|
| 84 |
+
)
|
| 85 |
+
else:
|
| 86 |
+
raise RuntimeError(base_error_msg)
|
| 87 |
+
# Check if cuDNN version is compatible with available CUDA devices
|
| 88 |
+
if torch.cuda.is_available() and not torch.version.hip:
|
| 89 |
+
min_cc = min(
|
| 90 |
+
[
|
| 91 |
+
torch.cuda.get_device_capability(i)
|
| 92 |
+
for i in range(torch.cuda.device_count())
|
| 93 |
+
]
|
| 94 |
+
)
|
| 95 |
+
if __cudnn_version >= 91100 and min_cc < (7, 5):
|
| 96 |
+
raise RuntimeError(
|
| 97 |
+
f"cuDNN version {__cudnn_version} is not compatible with devices with SM < 7.5. "
|
| 98 |
+
f"Please install a version of PyTorch with a compatible cuDNN version. "
|
| 99 |
+
f"https://github.com/pytorch/pytorch/blob/main/RELEASE.md#release-compatibility-matrix"
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
return True
|
| 103 |
+
|
| 104 |
+
else:
|
| 105 |
+
|
| 106 |
+
def _init():
|
| 107 |
+
return False
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def version():
|
| 111 |
+
"""Return the version of cuDNN."""
|
| 112 |
+
if not _init():
|
| 113 |
+
return None
|
| 114 |
+
return __cudnn_version
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
CUDNN_TENSOR_DTYPES = {
|
| 118 |
+
torch.half,
|
| 119 |
+
torch.float,
|
| 120 |
+
torch.double,
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def is_available():
|
| 125 |
+
r"""Return a bool indicating if CUDNN is currently available."""
|
| 126 |
+
return torch._C._has_cudnn
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def is_acceptable(tensor):
|
| 130 |
+
if not torch._C._get_cudnn_enabled():
|
| 131 |
+
return False
|
| 132 |
+
if tensor.device.type != "cuda" or tensor.dtype not in CUDNN_TENSOR_DTYPES:
|
| 133 |
+
return False
|
| 134 |
+
if not is_available():
|
| 135 |
+
warnings.warn(
|
| 136 |
+
"PyTorch was compiled without cuDNN/MIOpen support. To use cuDNN/MIOpen, rebuild "
|
| 137 |
+
"PyTorch making sure the library is visible to the build system.",
|
| 138 |
+
stacklevel=2,
|
| 139 |
+
)
|
| 140 |
+
return False
|
| 141 |
+
if not _init():
|
| 142 |
+
warnings.warn(
|
| 143 |
+
"cuDNN/MIOpen library not found. Check your {libpath}".format(
|
| 144 |
+
libpath={"darwin": "DYLD_LIBRARY_PATH", "win32": "PATH"}.get(
|
| 145 |
+
sys.platform, "LD_LIBRARY_PATH"
|
| 146 |
+
)
|
| 147 |
+
),
|
| 148 |
+
stacklevel=2,
|
| 149 |
+
)
|
| 150 |
+
return False
|
| 151 |
+
return True
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def set_flags(
|
| 155 |
+
_enabled=None,
|
| 156 |
+
_benchmark=None,
|
| 157 |
+
_benchmark_limit=None,
|
| 158 |
+
_deterministic=None,
|
| 159 |
+
_allow_tf32=None,
|
| 160 |
+
_fp32_precision="none",
|
| 161 |
+
_depthwise_kernel=None,
|
| 162 |
+
):
|
| 163 |
+
orig_flags = (
|
| 164 |
+
torch._C._get_cudnn_enabled(),
|
| 165 |
+
torch._C._get_cudnn_benchmark(),
|
| 166 |
+
None if not is_available() else torch._C._cuda_get_cudnn_benchmark_limit(),
|
| 167 |
+
torch._C._get_cudnn_deterministic(),
|
| 168 |
+
torch._C._get_cudnn_allow_tf32(),
|
| 169 |
+
torch._C._get_fp32_precision_getter("cuda", "all"),
|
| 170 |
+
torch._C._get_cudnn_depthwise_kernel(),
|
| 171 |
+
)
|
| 172 |
+
if _enabled is not None:
|
| 173 |
+
torch._C._set_cudnn_enabled(_enabled)
|
| 174 |
+
if _benchmark is not None:
|
| 175 |
+
torch._C._set_cudnn_benchmark(_benchmark)
|
| 176 |
+
if _benchmark_limit is not None and is_available():
|
| 177 |
+
torch._C._cuda_set_cudnn_benchmark_limit(_benchmark_limit)
|
| 178 |
+
if _deterministic is not None:
|
| 179 |
+
torch._C._set_cudnn_deterministic(_deterministic)
|
| 180 |
+
if _allow_tf32 is not None:
|
| 181 |
+
torch._C._set_cudnn_allow_tf32(_allow_tf32)
|
| 182 |
+
if _fp32_precision is not None:
|
| 183 |
+
torch._C._set_fp32_precision_setter("cuda", "all", _fp32_precision)
|
| 184 |
+
if _depthwise_kernel is not None:
|
| 185 |
+
torch._C._set_cudnn_depthwise_kernel(_depthwise_kernel)
|
| 186 |
+
return orig_flags
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
@contextmanager
|
| 190 |
+
def flags(
|
| 191 |
+
enabled=False,
|
| 192 |
+
benchmark=False,
|
| 193 |
+
benchmark_limit=10,
|
| 194 |
+
deterministic=False,
|
| 195 |
+
allow_tf32=True,
|
| 196 |
+
fp32_precision="none",
|
| 197 |
+
depthwise_kernel="auto",
|
| 198 |
+
):
|
| 199 |
+
with __allow_nonbracketed_mutation():
|
| 200 |
+
orig_flags = set_flags(
|
| 201 |
+
enabled,
|
| 202 |
+
benchmark,
|
| 203 |
+
benchmark_limit,
|
| 204 |
+
deterministic,
|
| 205 |
+
allow_tf32,
|
| 206 |
+
fp32_precision,
|
| 207 |
+
depthwise_kernel,
|
| 208 |
+
)
|
| 209 |
+
try:
|
| 210 |
+
yield
|
| 211 |
+
finally:
|
| 212 |
+
# recover the previous values
|
| 213 |
+
with __allow_nonbracketed_mutation():
|
| 214 |
+
set_flags(*orig_flags)
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
# The magic here is to allow us to intercept code like this:
|
| 218 |
+
#
|
| 219 |
+
# torch.backends.<cudnn|mkldnn>.enabled = True
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
class CudnnModule(PropModule):
|
| 223 |
+
enabled = ContextProp(torch._C._get_cudnn_enabled, torch._C._set_cudnn_enabled)
|
| 224 |
+
deterministic = ContextProp(
|
| 225 |
+
torch._C._get_cudnn_deterministic, torch._C._set_cudnn_deterministic
|
| 226 |
+
)
|
| 227 |
+
benchmark = ContextProp(
|
| 228 |
+
torch._C._get_cudnn_benchmark, torch._C._set_cudnn_benchmark
|
| 229 |
+
)
|
| 230 |
+
benchmark_limit = None
|
| 231 |
+
if is_available():
|
| 232 |
+
benchmark_limit = ContextProp(
|
| 233 |
+
torch._C._cuda_get_cudnn_benchmark_limit,
|
| 234 |
+
torch._C._cuda_set_cudnn_benchmark_limit,
|
| 235 |
+
)
|
| 236 |
+
allow_tf32 = ContextProp(
|
| 237 |
+
torch._C._get_cudnn_allow_tf32, torch._C._set_cudnn_allow_tf32
|
| 238 |
+
)
|
| 239 |
+
conv = _FP32Precision("cuda", "conv")
|
| 240 |
+
fp32_precision = ContextProp(
|
| 241 |
+
_get_fp32_precision_getter("cuda", "all"),
|
| 242 |
+
_set_fp32_precision_setter("cuda", "all"),
|
| 243 |
+
)
|
| 244 |
+
depthwise_kernel = ContextProp(
|
| 245 |
+
torch._C._get_cudnn_depthwise_kernel,
|
| 246 |
+
torch._C._set_cudnn_depthwise_kernel,
|
| 247 |
+
)
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
# This is the sys.modules replacement trick, see
|
| 251 |
+
# https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273
|
| 252 |
+
sys.modules[__name__] = CudnnModule(sys.modules[__name__], __name__)
|
| 253 |
+
|
| 254 |
+
# Add type annotation for the replaced module
|
| 255 |
+
enabled: bool
|
| 256 |
+
deterministic: bool
|
| 257 |
+
benchmark: bool
|
| 258 |
+
allow_tf32: bool
|
| 259 |
+
fp32_precision: str
|
| 260 |
+
benchmark_limit: int
|
| 261 |
+
depthwise_kernel: str
|
western_ai/usa/pytorch/torch/backends/cudnn/rnn.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import sys
|
| 3 |
+
|
| 4 |
+
import torch._C
|
| 5 |
+
import torch.cuda
|
| 6 |
+
from torch.backends import (
|
| 7 |
+
_get_fp32_precision_getter,
|
| 8 |
+
_set_fp32_precision_setter,
|
| 9 |
+
PropModule,
|
| 10 |
+
)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
try:
|
| 14 |
+
from torch._C import _cudnn
|
| 15 |
+
except ImportError:
|
| 16 |
+
# Uses of all the functions below should be guarded by torch.backends.cudnn.is_available(),
|
| 17 |
+
# so it's safe to not emit any checks here.
|
| 18 |
+
_cudnn = None # type: ignore[assignment]
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def get_cudnn_mode(mode):
|
| 22 |
+
if mode == "RNN_RELU":
|
| 23 |
+
# pyrefly: ignore [missing-attribute]
|
| 24 |
+
return int(_cudnn.RNNMode.rnn_relu)
|
| 25 |
+
elif mode == "RNN_TANH":
|
| 26 |
+
# pyrefly: ignore [missing-attribute]
|
| 27 |
+
return int(_cudnn.RNNMode.rnn_tanh)
|
| 28 |
+
elif mode == "LSTM":
|
| 29 |
+
# pyrefly: ignore [missing-attribute]
|
| 30 |
+
return int(_cudnn.RNNMode.lstm)
|
| 31 |
+
elif mode == "GRU":
|
| 32 |
+
# pyrefly: ignore [missing-attribute]
|
| 33 |
+
return int(_cudnn.RNNMode.gru)
|
| 34 |
+
else:
|
| 35 |
+
raise ValueError(f"Unknown mode: {mode}")
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
# NB: We don't actually need this class anymore (in fact, we could serialize the
|
| 39 |
+
# dropout state for even better reproducibility), but it is kept for backwards
|
| 40 |
+
# compatibility for old models.
|
| 41 |
+
class Unserializable:
|
| 42 |
+
def __init__(self, inner):
|
| 43 |
+
self.inner = inner
|
| 44 |
+
|
| 45 |
+
def get(self):
|
| 46 |
+
return self.inner
|
| 47 |
+
|
| 48 |
+
def __getstate__(self):
|
| 49 |
+
return "<unserializable>"
|
| 50 |
+
|
| 51 |
+
def __setstate__(self, state):
|
| 52 |
+
self.inner = None
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
# we would like to use ContextProp from backends here but the
|
| 56 |
+
# frozen flags appears to be overzealous
|
| 57 |
+
class ContextProp:
|
| 58 |
+
def __init__(self, getter, setter):
|
| 59 |
+
self.getter = getter
|
| 60 |
+
self.setter = setter
|
| 61 |
+
|
| 62 |
+
def __get__(self, obj, objtype):
|
| 63 |
+
return self.getter()
|
| 64 |
+
|
| 65 |
+
def __set__(self, obj, val):
|
| 66 |
+
self.setter(val)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def init_dropout_state(dropout, train, dropout_seed, dropout_state):
|
| 70 |
+
dropout_desc_name = "desc_" + str(torch.cuda.current_device())
|
| 71 |
+
dropout_p = dropout if train else 0
|
| 72 |
+
if (dropout_desc_name not in dropout_state) or (
|
| 73 |
+
dropout_state[dropout_desc_name].get() is None
|
| 74 |
+
):
|
| 75 |
+
if dropout_p == 0:
|
| 76 |
+
dropout_state[dropout_desc_name] = Unserializable(None)
|
| 77 |
+
else:
|
| 78 |
+
dropout_state[dropout_desc_name] = Unserializable(
|
| 79 |
+
torch._cudnn_init_dropout_state( # type: ignore[call-arg]
|
| 80 |
+
dropout_p,
|
| 81 |
+
train,
|
| 82 |
+
dropout_seed,
|
| 83 |
+
# pyrefly: ignore [unexpected-keyword]
|
| 84 |
+
self_ty=torch.uint8,
|
| 85 |
+
device=torch.device("cuda"),
|
| 86 |
+
)
|
| 87 |
+
)
|
| 88 |
+
dropout_ts = dropout_state[dropout_desc_name].get()
|
| 89 |
+
return dropout_ts
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
class CudnnRNNModule(PropModule):
|
| 93 |
+
def __init__(self, m, name):
|
| 94 |
+
super().__init__(m, name)
|
| 95 |
+
self.m.Unserializable = Unserializable
|
| 96 |
+
self.m.get_cudnn_mode = get_cudnn_mode
|
| 97 |
+
self.m.init_dropout_state = init_dropout_state
|
| 98 |
+
|
| 99 |
+
@staticmethod
|
| 100 |
+
def init_dropout_state(dropout, train, dropout_seed, dropout_state):
|
| 101 |
+
dropout_desc_name = "desc_" + str(torch.cuda.current_device())
|
| 102 |
+
dropout_p = dropout if train else 0
|
| 103 |
+
if (dropout_desc_name not in dropout_state) or (
|
| 104 |
+
dropout_state[dropout_desc_name].get() is None
|
| 105 |
+
):
|
| 106 |
+
if dropout_p == 0:
|
| 107 |
+
dropout_state[dropout_desc_name] = Unserializable(None)
|
| 108 |
+
else:
|
| 109 |
+
dropout_state[dropout_desc_name] = Unserializable(
|
| 110 |
+
torch._cudnn_init_dropout_state( # type: ignore[call-arg]
|
| 111 |
+
dropout_p,
|
| 112 |
+
train,
|
| 113 |
+
dropout_seed,
|
| 114 |
+
# pyrefly: ignore [unexpected-keyword]
|
| 115 |
+
self_ty=torch.uint8,
|
| 116 |
+
device=torch.device("cuda"),
|
| 117 |
+
)
|
| 118 |
+
)
|
| 119 |
+
dropout_ts = dropout_state[dropout_desc_name].get()
|
| 120 |
+
return dropout_ts
|
| 121 |
+
|
| 122 |
+
fp32_precision = ContextProp(
|
| 123 |
+
_get_fp32_precision_getter("cuda", "rnn"),
|
| 124 |
+
_set_fp32_precision_setter("cuda", "rnn"),
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
sys.modules[__name__] = CudnnRNNModule(sys.modules[__name__], __name__)
|
western_ai/usa/pytorch/torch/backends/cusparselt/__init__.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
__all__ = [
|
| 7 |
+
"version",
|
| 8 |
+
"is_available",
|
| 9 |
+
"get_max_alg_id",
|
| 10 |
+
]
|
| 11 |
+
|
| 12 |
+
try:
|
| 13 |
+
from torch._C import _cusparselt
|
| 14 |
+
except ImportError:
|
| 15 |
+
_cusparselt = None # type: ignore[assignment]
|
| 16 |
+
|
| 17 |
+
__cusparselt_version: int | None = None
|
| 18 |
+
__MAX_ALG_ID: int | None = None
|
| 19 |
+
|
| 20 |
+
if _cusparselt is not None:
|
| 21 |
+
|
| 22 |
+
def _init() -> bool:
|
| 23 |
+
global __cusparselt_version
|
| 24 |
+
global __MAX_ALG_ID
|
| 25 |
+
if __cusparselt_version is None:
|
| 26 |
+
# pyrefly: ignore [missing-attribute]
|
| 27 |
+
__cusparselt_version = _cusparselt.getVersionInt()
|
| 28 |
+
if __cusparselt_version == 400:
|
| 29 |
+
__MAX_ALG_ID = 4
|
| 30 |
+
elif __cusparselt_version == 502:
|
| 31 |
+
__MAX_ALG_ID = 5
|
| 32 |
+
elif __cusparselt_version == 602:
|
| 33 |
+
__MAX_ALG_ID = 37
|
| 34 |
+
return True
|
| 35 |
+
|
| 36 |
+
else:
|
| 37 |
+
|
| 38 |
+
def _init() -> bool:
|
| 39 |
+
return False
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def version() -> int | None:
|
| 43 |
+
"""Return the version of cuSPARSELt"""
|
| 44 |
+
if not _init():
|
| 45 |
+
return None
|
| 46 |
+
return __cusparselt_version
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def is_available() -> bool:
|
| 50 |
+
r"""Return a bool indicating if cuSPARSELt is currently available."""
|
| 51 |
+
return torch._C._has_cusparselt
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def get_max_alg_id() -> int | None:
|
| 55 |
+
if not _init():
|
| 56 |
+
return None
|
| 57 |
+
return __MAX_ALG_ID
|
western_ai/usa/pytorch/torch/backends/kleidiai/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import torch
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def is_available():
|
| 6 |
+
r"""Return whether PyTorch is built with KleidiAI support."""
|
| 7 |
+
return torch._C._has_kleidiai
|
western_ai/usa/pytorch/torch/backends/mha/__init__.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Config options to enable/disable C++ kernel for nn.functional.MHA
|
| 2 |
+
# and nn.TransformerEncoder
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
_is_fastpath_enabled: bool = True
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def get_fastpath_enabled() -> bool:
|
| 10 |
+
"""Returns whether fast path for TransformerEncoder and MultiHeadAttention
|
| 11 |
+
is enabled, or ``True`` if jit is scripting.
|
| 12 |
+
|
| 13 |
+
.. note::
|
| 14 |
+
The fastpath might not be run even if ``get_fastpath_enabled`` returns
|
| 15 |
+
``True`` unless all conditions on inputs are met.
|
| 16 |
+
"""
|
| 17 |
+
if not torch.jit.is_scripting():
|
| 18 |
+
return _is_fastpath_enabled
|
| 19 |
+
return True
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def set_fastpath_enabled(value: bool) -> None:
|
| 23 |
+
"""Sets whether fast path is enabled"""
|
| 24 |
+
global _is_fastpath_enabled
|
| 25 |
+
_is_fastpath_enabled = value
|
western_ai/usa/pytorch/torch/backends/miopen/__init__.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import sys
|
| 3 |
+
from contextlib import contextmanager
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from torch.backends import __allow_nonbracketed_mutation, ContextProp, PropModule
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def set_flags(
|
| 10 |
+
_immediate=None,
|
| 11 |
+
):
|
| 12 |
+
orig_flags = (torch._C._get_miopen_immediate(),)
|
| 13 |
+
if _immediate is not None:
|
| 14 |
+
torch._C._set_miopen_immediate(_immediate)
|
| 15 |
+
return orig_flags
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@contextmanager
|
| 19 |
+
def flags(
|
| 20 |
+
immediate=False,
|
| 21 |
+
):
|
| 22 |
+
with __allow_nonbracketed_mutation():
|
| 23 |
+
orig_flags = set_flags(
|
| 24 |
+
immediate,
|
| 25 |
+
)
|
| 26 |
+
try:
|
| 27 |
+
yield
|
| 28 |
+
finally:
|
| 29 |
+
# recover the previous values
|
| 30 |
+
with __allow_nonbracketed_mutation():
|
| 31 |
+
set_flags(*orig_flags)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# The magic here is to allow us to intercept code like this:
|
| 35 |
+
#
|
| 36 |
+
# torch.backends.<miopen|mkldnn>.immediate = True
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class MiopenModule(PropModule):
|
| 40 |
+
immediate = ContextProp(
|
| 41 |
+
torch._C._get_miopen_immediate, torch._C._set_miopen_immediate
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
# This is the sys.modules replacement trick, see
|
| 46 |
+
# https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273
|
| 47 |
+
sys.modules[__name__] = MiopenModule(sys.modules[__name__], __name__)
|
| 48 |
+
|
| 49 |
+
# Add type annotation for the replaced module
|
| 50 |
+
immediate: bool
|
western_ai/usa/pytorch/torch/backends/mkl/__init__.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import torch
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def is_available():
|
| 6 |
+
r"""Return whether PyTorch is built with MKL support."""
|
| 7 |
+
return torch._C.has_mkl
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
VERBOSE_OFF = 0
|
| 11 |
+
VERBOSE_ON = 1
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class verbose:
|
| 15 |
+
"""
|
| 16 |
+
On-demand oneMKL verbosing functionality.
|
| 17 |
+
|
| 18 |
+
To make it easier to debug performance issues, oneMKL can dump verbose
|
| 19 |
+
messages containing execution information like duration while executing
|
| 20 |
+
the kernel. The verbosing functionality can be invoked via an environment
|
| 21 |
+
variable named `MKL_VERBOSE`. However, this methodology dumps messages in
|
| 22 |
+
all steps. Those are a large amount of verbose messages. Moreover, for
|
| 23 |
+
investigating the performance issues, generally taking verbose messages
|
| 24 |
+
for one single iteration is enough. This on-demand verbosing functionality
|
| 25 |
+
makes it possible to control scope for verbose message dumping. In the
|
| 26 |
+
following example, verbose messages will be dumped out for the second
|
| 27 |
+
inference only.
|
| 28 |
+
|
| 29 |
+
.. highlight:: python
|
| 30 |
+
.. code-block:: python
|
| 31 |
+
|
| 32 |
+
import torch
|
| 33 |
+
|
| 34 |
+
model(data)
|
| 35 |
+
with torch.backends.mkl.verbose(torch.backends.mkl.VERBOSE_ON):
|
| 36 |
+
model(data)
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
level: Verbose level
|
| 40 |
+
- ``VERBOSE_OFF``: Disable verbosing
|
| 41 |
+
- ``VERBOSE_ON``: Enable verbosing
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
def __init__(self, enable):
|
| 45 |
+
self.enable = enable
|
| 46 |
+
|
| 47 |
+
def __enter__(self):
|
| 48 |
+
if self.enable == VERBOSE_OFF:
|
| 49 |
+
return
|
| 50 |
+
st = torch._C._verbose.mkl_set_verbose(self.enable)
|
| 51 |
+
if not st:
|
| 52 |
+
raise AssertionError(
|
| 53 |
+
"Failed to set MKL into verbose mode. Please consider to disable this verbose scope."
|
| 54 |
+
)
|
| 55 |
+
return self
|
| 56 |
+
|
| 57 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 58 |
+
torch._C._verbose.mkl_set_verbose(VERBOSE_OFF)
|
| 59 |
+
return False
|
western_ai/usa/pytorch/torch/backends/mkldnn/__init__.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import sys
|
| 3 |
+
from contextlib import contextmanager
|
| 4 |
+
from typing import TYPE_CHECKING
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from torch.backends import (
|
| 8 |
+
__allow_nonbracketed_mutation,
|
| 9 |
+
_FP32Precision,
|
| 10 |
+
_get_fp32_precision_getter,
|
| 11 |
+
_set_fp32_precision_setter,
|
| 12 |
+
ContextProp,
|
| 13 |
+
PropModule,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def is_available():
|
| 18 |
+
r"""Return whether PyTorch is built with MKL-DNN support."""
|
| 19 |
+
return torch._C._has_mkldnn
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def is_acl_available():
|
| 23 |
+
r"""Return whether PyTorch is built with MKL-DNN + ACL support."""
|
| 24 |
+
# pyrefly: ignore [missing-attribute]
|
| 25 |
+
return torch._C._has_mkldnn_acl
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
VERBOSE_OFF = 0
|
| 29 |
+
VERBOSE_ON = 1
|
| 30 |
+
VERBOSE_ON_CREATION = 2
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class verbose:
|
| 34 |
+
"""
|
| 35 |
+
On-demand oneDNN (former MKL-DNN) verbosing functionality.
|
| 36 |
+
|
| 37 |
+
To make it easier to debug performance issues, oneDNN can dump verbose
|
| 38 |
+
messages containing information like kernel size, input data size and
|
| 39 |
+
execution duration while executing the kernel. The verbosing functionality
|
| 40 |
+
can be invoked via an environment variable named `DNNL_VERBOSE`. However,
|
| 41 |
+
this methodology dumps messages in all steps. Those are a large amount of
|
| 42 |
+
verbose messages. Moreover, for investigating the performance issues,
|
| 43 |
+
generally taking verbose messages for one single iteration is enough.
|
| 44 |
+
This on-demand verbosing functionality makes it possible to control scope
|
| 45 |
+
for verbose message dumping. In the following example, verbose messages
|
| 46 |
+
will be dumped out for the second inference only.
|
| 47 |
+
|
| 48 |
+
.. highlight:: python
|
| 49 |
+
.. code-block:: python
|
| 50 |
+
|
| 51 |
+
import torch
|
| 52 |
+
|
| 53 |
+
model(data)
|
| 54 |
+
with torch.backends.mkldnn.verbose(torch.backends.mkldnn.VERBOSE_ON):
|
| 55 |
+
model(data)
|
| 56 |
+
|
| 57 |
+
Args:
|
| 58 |
+
level: Verbose level
|
| 59 |
+
- ``VERBOSE_OFF``: Disable verbosing
|
| 60 |
+
- ``VERBOSE_ON``: Enable verbosing
|
| 61 |
+
- ``VERBOSE_ON_CREATION``: Enable verbosing, including oneDNN kernel creation
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
def __init__(self, level):
|
| 65 |
+
self.level = level
|
| 66 |
+
|
| 67 |
+
def __enter__(self):
|
| 68 |
+
if self.level == VERBOSE_OFF:
|
| 69 |
+
return
|
| 70 |
+
st = torch._C._verbose.mkldnn_set_verbose(self.level)
|
| 71 |
+
if not st:
|
| 72 |
+
raise AssertionError(
|
| 73 |
+
"Failed to set MKLDNN into verbose mode. Please consider to disable this verbose scope."
|
| 74 |
+
)
|
| 75 |
+
return self
|
| 76 |
+
|
| 77 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 78 |
+
torch._C._verbose.mkldnn_set_verbose(VERBOSE_OFF)
|
| 79 |
+
return False
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def set_flags(
|
| 83 |
+
_enabled=None, _deterministic=None, _allow_tf32=None, _fp32_precision="none"
|
| 84 |
+
):
|
| 85 |
+
orig_flags = (
|
| 86 |
+
torch._C._get_mkldnn_enabled(),
|
| 87 |
+
torch._C._get_mkldnn_deterministic(),
|
| 88 |
+
torch._C._get_onednn_allow_tf32(),
|
| 89 |
+
torch._C._get_fp32_precision_getter("mkldnn", "all"),
|
| 90 |
+
)
|
| 91 |
+
if _enabled is not None:
|
| 92 |
+
torch._C._set_mkldnn_enabled(_enabled)
|
| 93 |
+
if _deterministic is not None:
|
| 94 |
+
torch._C._set_mkldnn_deterministic(_deterministic)
|
| 95 |
+
if _allow_tf32 is not None:
|
| 96 |
+
torch._C._set_onednn_allow_tf32(_allow_tf32)
|
| 97 |
+
if _fp32_precision is not None:
|
| 98 |
+
torch._C._set_fp32_precision_setter("mkldnn", "all", _fp32_precision)
|
| 99 |
+
return orig_flags
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
@contextmanager
|
| 103 |
+
def flags(enabled=False, deterministic=False, allow_tf32=True, fp32_precision="none"):
|
| 104 |
+
with __allow_nonbracketed_mutation():
|
| 105 |
+
orig_flags = set_flags(enabled, deterministic, allow_tf32, fp32_precision)
|
| 106 |
+
try:
|
| 107 |
+
yield
|
| 108 |
+
finally:
|
| 109 |
+
with __allow_nonbracketed_mutation():
|
| 110 |
+
set_flags(*orig_flags)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
class MkldnnModule(PropModule):
|
| 114 |
+
def is_available(self):
|
| 115 |
+
return is_available()
|
| 116 |
+
|
| 117 |
+
enabled = ContextProp(torch._C._get_mkldnn_enabled, torch._C._set_mkldnn_enabled)
|
| 118 |
+
deterministic = ContextProp(
|
| 119 |
+
torch._C._get_mkldnn_deterministic, torch._C._set_mkldnn_deterministic
|
| 120 |
+
)
|
| 121 |
+
allow_tf32 = ContextProp(
|
| 122 |
+
torch._C._get_onednn_allow_tf32, torch._C._set_onednn_allow_tf32
|
| 123 |
+
)
|
| 124 |
+
matmul = _FP32Precision("mkldnn", "matmul")
|
| 125 |
+
conv = _FP32Precision("mkldnn", "conv")
|
| 126 |
+
rnn = _FP32Precision("mkldnn", "rnn")
|
| 127 |
+
fp32_precision = ContextProp(
|
| 128 |
+
_get_fp32_precision_getter("mkldnn", "all"),
|
| 129 |
+
_set_fp32_precision_setter("generic", "all"),
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
if TYPE_CHECKING:
|
| 134 |
+
enabled: ContextProp
|
| 135 |
+
deterministic: ContextProp
|
| 136 |
+
allow_tf32: ContextProp
|
| 137 |
+
fp32_precision: str
|
| 138 |
+
|
| 139 |
+
sys.modules[__name__] = MkldnnModule(sys.modules[__name__], __name__)
|
western_ai/usa/pytorch/torch/backends/mps/__init__.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from functools import lru_cache as _lru_cache
|
| 2 |
+
from typing import Optional, TYPE_CHECKING
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from torch.library import Library as _Library
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
__all__ = [
|
| 9 |
+
"get_core_count",
|
| 10 |
+
"get_name",
|
| 11 |
+
"is_built",
|
| 12 |
+
"is_available",
|
| 13 |
+
"is_macos13_or_newer",
|
| 14 |
+
"is_macos_or_newer",
|
| 15 |
+
]
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def is_built() -> bool:
|
| 19 |
+
r"""Return whether PyTorch is built with MPS support.
|
| 20 |
+
|
| 21 |
+
Note that this doesn't necessarily mean MPS is available; just that
|
| 22 |
+
if this PyTorch binary were run a machine with working MPS drivers
|
| 23 |
+
and devices, we would be able to use it.
|
| 24 |
+
"""
|
| 25 |
+
return torch._C._has_mps
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@_lru_cache
|
| 29 |
+
def is_available() -> bool:
|
| 30 |
+
r"""Return a bool indicating if MPS is currently available."""
|
| 31 |
+
return torch._C._mps_is_available()
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@_lru_cache
|
| 35 |
+
def is_macos_or_newer(major: int, minor: int) -> bool:
|
| 36 |
+
r"""Return a bool indicating whether MPS is running on given MacOS or newer."""
|
| 37 |
+
return torch._C._mps_is_on_macos_or_newer(major, minor)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@_lru_cache
|
| 41 |
+
def is_macos13_or_newer(minor: int = 0) -> bool:
|
| 42 |
+
r"""Return a bool indicating whether MPS is running on MacOS 13 or newer."""
|
| 43 |
+
return torch._C._mps_is_on_macos_or_newer(13, minor)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@_lru_cache
|
| 47 |
+
def get_name() -> str:
|
| 48 |
+
r"""Return Metal device name"""
|
| 49 |
+
return torch._C._mps_get_name()
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
@_lru_cache
|
| 53 |
+
def get_core_count() -> int:
|
| 54 |
+
r"""Return GPU core count.
|
| 55 |
+
|
| 56 |
+
According to the documentation, one core is comprised of 16 Execution Units.
|
| 57 |
+
One execution Unit has 8 ALUs.
|
| 58 |
+
And one ALU can run 24 threads, i.e. one core is capable of executing 3072 threads concurrently.
|
| 59 |
+
"""
|
| 60 |
+
return torch._C._mps_get_core_count()
|
western_ai/usa/pytorch/torch/backends/nnpack/__init__.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
from contextlib import contextmanager
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from torch.backends import __allow_nonbracketed_mutation, ContextProp, PropModule
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
__all__ = ["is_available", "flags", "set_flags"]
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def is_available():
|
| 12 |
+
r"""Return whether PyTorch is built with NNPACK support."""
|
| 13 |
+
return torch._nnpack_available()
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def set_flags(_enabled):
|
| 17 |
+
r"""Set if nnpack is enabled globally"""
|
| 18 |
+
orig_flags = (torch._C._get_nnpack_enabled(),)
|
| 19 |
+
torch._C._set_nnpack_enabled(_enabled)
|
| 20 |
+
return orig_flags
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@contextmanager
|
| 24 |
+
def flags(enabled=False):
|
| 25 |
+
r"""Context manager for setting if nnpack is enabled globally"""
|
| 26 |
+
with __allow_nonbracketed_mutation():
|
| 27 |
+
orig_flags = set_flags(enabled)
|
| 28 |
+
try:
|
| 29 |
+
yield
|
| 30 |
+
finally:
|
| 31 |
+
with __allow_nonbracketed_mutation():
|
| 32 |
+
set_flags(orig_flags[0])
|
western_ai/usa/pytorch/torch/backends/openmp/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import torch
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def is_available():
|
| 6 |
+
r"""Return whether PyTorch is built with OpenMP support."""
|
| 7 |
+
return torch._C.has_openmp
|
western_ai/usa/pytorch/torch/backends/opt_einsum/__init__.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import sys
|
| 3 |
+
import warnings
|
| 4 |
+
from contextlib import contextmanager
|
| 5 |
+
from functools import lru_cache as _lru_cache
|
| 6 |
+
from typing import Any
|
| 7 |
+
|
| 8 |
+
from torch.backends import __allow_nonbracketed_mutation, ContextProp, PropModule
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
try:
|
| 12 |
+
import opt_einsum as _opt_einsum # type: ignore[import]
|
| 13 |
+
except ImportError:
|
| 14 |
+
_opt_einsum = None
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@_lru_cache
|
| 18 |
+
def is_available() -> bool:
|
| 19 |
+
r"""Return a bool indicating if opt_einsum is currently available.
|
| 20 |
+
|
| 21 |
+
You must install opt-einsum in order for torch to automatically optimize einsum. To
|
| 22 |
+
make opt-einsum available, you can install it along with torch: ``pip install torch[opt-einsum]``
|
| 23 |
+
or by itself: ``pip install opt-einsum``. If the package is installed, torch will import
|
| 24 |
+
it automatically and use it accordingly. Use this function to check whether opt-einsum
|
| 25 |
+
was installed and properly imported by torch.
|
| 26 |
+
"""
|
| 27 |
+
return _opt_einsum is not None
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def get_opt_einsum() -> Any:
|
| 31 |
+
r"""Return the opt_einsum package if opt_einsum is currently available, else None."""
|
| 32 |
+
return _opt_einsum
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _set_enabled(_enabled: bool) -> None:
|
| 36 |
+
if not is_available() and _enabled:
|
| 37 |
+
raise ValueError(
|
| 38 |
+
f"opt_einsum is not available, so setting `enabled` to {_enabled} will not reap "
|
| 39 |
+
"the benefits of calculating an optimal path for einsum. torch.einsum will "
|
| 40 |
+
"fall back to contracting from left to right. To enable this optimal path "
|
| 41 |
+
"calculation, please install opt-einsum."
|
| 42 |
+
)
|
| 43 |
+
global enabled
|
| 44 |
+
enabled = _enabled
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _get_enabled() -> bool:
|
| 48 |
+
return enabled
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _set_strategy(_strategy: str) -> None:
|
| 52 |
+
if not is_available():
|
| 53 |
+
raise ValueError(
|
| 54 |
+
f"opt_einsum is not available, so setting `strategy` to {_strategy} will not be meaningful. "
|
| 55 |
+
"torch.einsum will bypass path calculation and simply contract from left to right. "
|
| 56 |
+
"Please install opt_einsum or unset `strategy`."
|
| 57 |
+
)
|
| 58 |
+
if not enabled:
|
| 59 |
+
raise ValueError(
|
| 60 |
+
f"opt_einsum is not enabled, so setting a `strategy` to {_strategy} will not be meaningful. "
|
| 61 |
+
"torch.einsum will bypass path calculation and simply contract from left to right. "
|
| 62 |
+
"Please set `enabled` to `True` as well or unset `strategy`."
|
| 63 |
+
)
|
| 64 |
+
if _strategy not in ["auto", "greedy", "optimal"]:
|
| 65 |
+
raise ValueError(
|
| 66 |
+
f"`strategy` must be one of the following: [auto, greedy, optimal] but is {_strategy}"
|
| 67 |
+
)
|
| 68 |
+
global strategy
|
| 69 |
+
strategy = _strategy
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _get_strategy() -> str:
|
| 73 |
+
# pyrefly: ignore [bad-return]
|
| 74 |
+
return strategy
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def set_flags(_enabled=None, _strategy=None):
|
| 78 |
+
orig_flags = (enabled, None if not is_available() else strategy)
|
| 79 |
+
if _enabled is not None:
|
| 80 |
+
_set_enabled(_enabled)
|
| 81 |
+
if _strategy is not None:
|
| 82 |
+
_set_strategy(_strategy)
|
| 83 |
+
return orig_flags
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
@contextmanager
|
| 87 |
+
def flags(enabled=None, strategy=None):
|
| 88 |
+
with __allow_nonbracketed_mutation():
|
| 89 |
+
orig_flags = set_flags(enabled, strategy)
|
| 90 |
+
try:
|
| 91 |
+
yield
|
| 92 |
+
finally:
|
| 93 |
+
# recover the previous values
|
| 94 |
+
with __allow_nonbracketed_mutation():
|
| 95 |
+
set_flags(*orig_flags)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
# The magic here is to allow us to intercept code like this:
|
| 99 |
+
#
|
| 100 |
+
# torch.backends.opt_einsum.enabled = True
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
class OptEinsumModule(PropModule):
|
| 104 |
+
global enabled
|
| 105 |
+
enabled = ContextProp(_get_enabled, _set_enabled)
|
| 106 |
+
global strategy
|
| 107 |
+
strategy = None
|
| 108 |
+
if is_available():
|
| 109 |
+
strategy = ContextProp(_get_strategy, _set_strategy)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
# This is the sys.modules replacement trick, see
|
| 113 |
+
# https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273
|
| 114 |
+
sys.modules[__name__] = OptEinsumModule(sys.modules[__name__], __name__)
|
| 115 |
+
|
| 116 |
+
enabled = bool(is_available())
|
| 117 |
+
strategy = "auto" if is_available() else None
|
western_ai/usa/pytorch/torch/backends/python_native/__init__.py
ADDED
|
@@ -0,0 +1,387 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
User-facing API for controlling DSL operation overrides.
|
| 3 |
+
|
| 4 |
+
The torch.backends.python_native module provides control over DSL (Domain Specific Language)
|
| 5 |
+
operation overrides defined in torch._native. This allows users to selectively enable or disable
|
| 6 |
+
high-performance implementations from various DSLs like Triton and CuteDSL.
|
| 7 |
+
|
| 8 |
+
The module supports both coarse-grained control (entire DSLs) and fine-grained control
|
| 9 |
+
(individual operations or dispatch keys). All control operations support context managers
|
| 10 |
+
for temporary state changes.
|
| 11 |
+
|
| 12 |
+
Example usage::
|
| 13 |
+
|
| 14 |
+
import torch.backends.python_native as pn
|
| 15 |
+
|
| 16 |
+
# DSL-level control
|
| 17 |
+
pn.triton.enabled = False # Disable all triton ops
|
| 18 |
+
pn.cutedsl.enabled = True # Enable all cutedsl ops
|
| 19 |
+
|
| 20 |
+
# Individual operation control
|
| 21 |
+
pn.disable_operations("scaled_mm") # Disable specific op across all DSLs
|
| 22 |
+
pn.enable_operations("scaled_mm") # Re-enable specific op
|
| 23 |
+
|
| 24 |
+
# Context manager support
|
| 25 |
+
with pn.triton.disabled():
|
| 26 |
+
result = some_computation() # Triton ops disabled here
|
| 27 |
+
|
| 28 |
+
# Query capabilities
|
| 29 |
+
print(pn.available_dsls) # ['triton', 'cutedsl']
|
| 30 |
+
print(pn.get_dsl_operations("triton")) # Operations for triton
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
import functools
|
| 34 |
+
import sys
|
| 35 |
+
import types
|
| 36 |
+
from contextlib import contextmanager
|
| 37 |
+
|
| 38 |
+
from torch.backends import ContextProp, flags_frozen, PropModule
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
@contextmanager
|
| 42 |
+
def _preserve_filter_state():
|
| 43 |
+
"""Context manager to save and restore registry filter state."""
|
| 44 |
+
filter_state = _get_filter_state()
|
| 45 |
+
|
| 46 |
+
# Save original state
|
| 47 |
+
original_state = (
|
| 48 |
+
set(filter_state._dsl_names),
|
| 49 |
+
set(filter_state._op_symbols),
|
| 50 |
+
set(filter_state._dispatch_keys),
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
try:
|
| 54 |
+
yield filter_state
|
| 55 |
+
finally:
|
| 56 |
+
# Restore original state
|
| 57 |
+
filter_state._dsl_names.clear()
|
| 58 |
+
filter_state._op_symbols.clear()
|
| 59 |
+
filter_state._dispatch_keys.clear()
|
| 60 |
+
|
| 61 |
+
filter_state._dsl_names.update(original_state[0])
|
| 62 |
+
filter_state._op_symbols.update(original_state[1])
|
| 63 |
+
filter_state._dispatch_keys.update(original_state[2])
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _get_dsl_registry():
|
| 67 |
+
"""Lazy import to avoid circular imports."""
|
| 68 |
+
from torch._native.dsl_registry import dsl_registry
|
| 69 |
+
|
| 70 |
+
return dsl_registry
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _get_registry_functions():
|
| 74 |
+
"""Lazy import of registry functions."""
|
| 75 |
+
from torch._native.registry import (
|
| 76 |
+
_filter_state,
|
| 77 |
+
_graphs,
|
| 78 |
+
deregister_op_overrides,
|
| 79 |
+
reenable_op_overrides,
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
return deregister_op_overrides, reenable_op_overrides, _graphs, _filter_state
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _get_filter_state():
|
| 86 |
+
"""Direct access to filter state."""
|
| 87 |
+
return _get_registry_functions()[3]
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _get_dsl_module(dsl_name: str):
|
| 91 |
+
"""Get the registered DSL module for direct control.
|
| 92 |
+
|
| 93 |
+
Uses the DSL registry to dynamically look up DSL modules instead of
|
| 94 |
+
hard-coding the mapping. This makes the function automatically extensible
|
| 95 |
+
for new DSLs without code changes.
|
| 96 |
+
|
| 97 |
+
Args:
|
| 98 |
+
dsl_name (str): Name of the DSL to retrieve.
|
| 99 |
+
|
| 100 |
+
Returns:
|
| 101 |
+
DSLModuleProtocol: The registered DSL module.
|
| 102 |
+
|
| 103 |
+
Raises:
|
| 104 |
+
ValueError: If the DSL is not registered.
|
| 105 |
+
"""
|
| 106 |
+
registry = _get_dsl_registry()
|
| 107 |
+
|
| 108 |
+
# Use the public API to get the DSL module
|
| 109 |
+
dsl_module = registry.get_dsl_module(dsl_name)
|
| 110 |
+
if dsl_module is not None:
|
| 111 |
+
return dsl_module
|
| 112 |
+
else:
|
| 113 |
+
raise ValueError(
|
| 114 |
+
f"Unknown DSL: {dsl_name}. Available DSLs: {registry.list_all_dsls()}"
|
| 115 |
+
)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
class DSLController:
|
| 119 |
+
"""Controller for a specific DSL."""
|
| 120 |
+
|
| 121 |
+
def __init__(self, dsl_name: str):
|
| 122 |
+
self._dsl_name = dsl_name
|
| 123 |
+
|
| 124 |
+
@property
|
| 125 |
+
def name(self) -> str:
|
| 126 |
+
return self._dsl_name
|
| 127 |
+
|
| 128 |
+
@property
|
| 129 |
+
def available(self) -> bool:
|
| 130 |
+
"""Check if DSL runtime is available."""
|
| 131 |
+
registry = _get_dsl_registry()
|
| 132 |
+
return registry.is_dsl_available(self._dsl_name)
|
| 133 |
+
|
| 134 |
+
@property
|
| 135 |
+
def version(self):
|
| 136 |
+
"""Get DSL version."""
|
| 137 |
+
registry = _get_dsl_registry()
|
| 138 |
+
return registry.get_dsl_version(self._dsl_name)
|
| 139 |
+
|
| 140 |
+
@property
|
| 141 |
+
def enabled(self) -> bool:
|
| 142 |
+
"""Check if DSL is currently enabled."""
|
| 143 |
+
filter_state = _get_filter_state()
|
| 144 |
+
return self._dsl_name not in filter_state._dsl_names
|
| 145 |
+
|
| 146 |
+
@enabled.setter
|
| 147 |
+
def enabled(self, value: bool):
|
| 148 |
+
"""Enable or disable the DSL."""
|
| 149 |
+
if flags_frozen():
|
| 150 |
+
raise RuntimeError(
|
| 151 |
+
f"not allowed to set {self._dsl_name} DSL flags "
|
| 152 |
+
"after disable_global_flags; please use flags() context manager instead"
|
| 153 |
+
)
|
| 154 |
+
if value:
|
| 155 |
+
self.enable()
|
| 156 |
+
else:
|
| 157 |
+
self.disable()
|
| 158 |
+
|
| 159 |
+
def disable(self):
|
| 160 |
+
"""Disable all operations for this DSL."""
|
| 161 |
+
dsl_module = _get_dsl_module(self._dsl_name)
|
| 162 |
+
dsl_module.deregister_op_overrides()
|
| 163 |
+
|
| 164 |
+
def enable(self):
|
| 165 |
+
"""Re-enable all operations for this DSL."""
|
| 166 |
+
reenable_op_overrides = _get_registry_functions()[1]
|
| 167 |
+
reenable_op_overrides(enable_dsl_names=self._dsl_name)
|
| 168 |
+
|
| 169 |
+
@contextmanager
|
| 170 |
+
def disabled(self):
|
| 171 |
+
"""Context manager to temporarily disable DSL."""
|
| 172 |
+
original_state = self.enabled
|
| 173 |
+
try:
|
| 174 |
+
self.disable()
|
| 175 |
+
yield
|
| 176 |
+
finally:
|
| 177 |
+
if original_state:
|
| 178 |
+
self.enable()
|
| 179 |
+
|
| 180 |
+
def __repr__(self):
|
| 181 |
+
status = "available" if self.available else "unavailable"
|
| 182 |
+
enabled_status = "enabled" if self.enabled else "disabled"
|
| 183 |
+
return f"DSLController({self._dsl_name}, {status}, {enabled_status})"
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
class PythonNativeModule(PropModule):
|
| 187 |
+
"""Main module for python_native DSL control."""
|
| 188 |
+
|
| 189 |
+
def __init__(self, original_module):
|
| 190 |
+
super().__init__(original_module, original_module.__name__)
|
| 191 |
+
|
| 192 |
+
@property
|
| 193 |
+
def available_dsls(self) -> list[str]:
|
| 194 |
+
"""Get list of available DSLs."""
|
| 195 |
+
registry = _get_dsl_registry()
|
| 196 |
+
result = registry.list_available_dsls()
|
| 197 |
+
return list(result) if not isinstance(result, list) else result
|
| 198 |
+
|
| 199 |
+
@property
|
| 200 |
+
def all_dsls(self) -> list[str]:
|
| 201 |
+
"""Get list of all registered DSLs."""
|
| 202 |
+
registry = _get_dsl_registry()
|
| 203 |
+
result = registry.list_all_dsls()
|
| 204 |
+
return list(result) if not isinstance(result, list) else result
|
| 205 |
+
|
| 206 |
+
def get_dsl_operations(self, dsl_name: str) -> list[str]:
|
| 207 |
+
"""Get list of operations registered by a specific DSL.
|
| 208 |
+
|
| 209 |
+
Args:
|
| 210 |
+
dsl_name (str): Name of the DSL to query (e.g., 'triton', 'cutedsl').
|
| 211 |
+
|
| 212 |
+
Returns:
|
| 213 |
+
list[str]: Sorted list of operation names registered by the DSL.
|
| 214 |
+
|
| 215 |
+
Example::
|
| 216 |
+
|
| 217 |
+
ops = torch.backends.python_native.get_dsl_operations("triton")
|
| 218 |
+
print(ops) # ['triton_to_mxfp8_dim0', ...]
|
| 219 |
+
"""
|
| 220 |
+
from torch._native.registry import get_dsl_operations
|
| 221 |
+
|
| 222 |
+
return get_dsl_operations(dsl_name)
|
| 223 |
+
|
| 224 |
+
def disable_operations(self, *op_symbols: str):
|
| 225 |
+
"""Disable specific operations across all DSLs.
|
| 226 |
+
|
| 227 |
+
Args:
|
| 228 |
+
*op_symbols (str): Names of operations to disable.
|
| 229 |
+
|
| 230 |
+
Example::
|
| 231 |
+
|
| 232 |
+
# Disable scaled matrix multiply across all DSLs
|
| 233 |
+
torch.backends.python_native.disable_operations("scaled_mm")
|
| 234 |
+
|
| 235 |
+
# Disable multiple operations
|
| 236 |
+
torch.backends.python_native.disable_operations(
|
| 237 |
+
"scaled_mm", "flash_attention"
|
| 238 |
+
)
|
| 239 |
+
"""
|
| 240 |
+
deregister_op_overrides = _get_registry_functions()[0]
|
| 241 |
+
deregister_op_overrides(disable_op_symbols=list(op_symbols))
|
| 242 |
+
|
| 243 |
+
def enable_operations(self, *op_symbols: str):
|
| 244 |
+
"""Re-enable specific operations across all DSLs.
|
| 245 |
+
|
| 246 |
+
Args:
|
| 247 |
+
*op_symbols (str): Names of operations to re-enable.
|
| 248 |
+
|
| 249 |
+
Example::
|
| 250 |
+
|
| 251 |
+
# Re-enable previously disabled operations
|
| 252 |
+
torch.backends.python_native.enable_operations(
|
| 253 |
+
"scaled_mm", "flash_attention"
|
| 254 |
+
)
|
| 255 |
+
"""
|
| 256 |
+
reenable_op_overrides = _get_registry_functions()[1]
|
| 257 |
+
reenable_op_overrides(enable_op_symbols=list(op_symbols))
|
| 258 |
+
|
| 259 |
+
def disable_dispatch_keys(self, *dispatch_keys: str):
|
| 260 |
+
"""Disable operations at specific dispatch keys.
|
| 261 |
+
|
| 262 |
+
Args:
|
| 263 |
+
*dispatch_keys (str): Dispatch keys to disable (e.g., 'CUDA', 'CPU').
|
| 264 |
+
|
| 265 |
+
Example::
|
| 266 |
+
|
| 267 |
+
# Disable all native operations on CUDA
|
| 268 |
+
torch.backends.python_native.disable_dispatch_keys("CUDA")
|
| 269 |
+
"""
|
| 270 |
+
deregister_op_overrides = _get_registry_functions()[0]
|
| 271 |
+
deregister_op_overrides(disable_dispatch_keys=list(dispatch_keys))
|
| 272 |
+
|
| 273 |
+
def enable_dispatch_keys(self, *dispatch_keys: str):
|
| 274 |
+
"""Re-enable operations at specific dispatch keys.
|
| 275 |
+
|
| 276 |
+
Args:
|
| 277 |
+
*dispatch_keys (str): Dispatch keys to re-enable (e.g., 'CUDA', 'CPU').
|
| 278 |
+
|
| 279 |
+
Example::
|
| 280 |
+
|
| 281 |
+
# Re-enable native operations on CUDA
|
| 282 |
+
torch.backends.python_native.enable_dispatch_keys("CUDA")
|
| 283 |
+
"""
|
| 284 |
+
reenable_op_overrides = _get_registry_functions()[1]
|
| 285 |
+
reenable_op_overrides(enable_dispatch_keys=list(dispatch_keys))
|
| 286 |
+
|
| 287 |
+
@contextmanager
|
| 288 |
+
def operations_disabled(self, *op_symbols: str):
|
| 289 |
+
"""Context manager to temporarily disable operations.
|
| 290 |
+
|
| 291 |
+
Args:
|
| 292 |
+
*op_symbols (str): Names of operations to temporarily disable.
|
| 293 |
+
|
| 294 |
+
Example::
|
| 295 |
+
|
| 296 |
+
with torch.backends.python_native.operations_disabled("scaled_mm"):
|
| 297 |
+
# scaled_mm is disabled across all DSLs
|
| 298 |
+
result = model(input)
|
| 299 |
+
# scaled_mm is automatically re-enabled here
|
| 300 |
+
"""
|
| 301 |
+
filter_state = _get_filter_state()
|
| 302 |
+
previously_disabled_ops = {
|
| 303 |
+
op for op in op_symbols if op in filter_state._op_symbols
|
| 304 |
+
}
|
| 305 |
+
|
| 306 |
+
self.disable_operations(*op_symbols)
|
| 307 |
+
try:
|
| 308 |
+
yield
|
| 309 |
+
finally:
|
| 310 |
+
# Only re-enable operations that weren't already disabled
|
| 311 |
+
ops_to_reenable = [
|
| 312 |
+
op for op in op_symbols if op not in previously_disabled_ops
|
| 313 |
+
]
|
| 314 |
+
if ops_to_reenable:
|
| 315 |
+
self.enable_operations(*ops_to_reenable)
|
| 316 |
+
|
| 317 |
+
@functools.lru_cache(maxsize=16) # noqa: B019
|
| 318 |
+
def _get_dsl_controller(self, name: str) -> "DSLController":
|
| 319 |
+
"""Get or create a DSL controller (cached)."""
|
| 320 |
+
return DSLController(name)
|
| 321 |
+
|
| 322 |
+
def _get_registry_functions(self):
|
| 323 |
+
"""Expose registry functions for testing."""
|
| 324 |
+
return _get_registry_functions()
|
| 325 |
+
|
| 326 |
+
def is_operation_disabled(self, op_symbol: str) -> bool:
|
| 327 |
+
"""Check if an operation is currently disabled."""
|
| 328 |
+
filter_state = _get_filter_state()
|
| 329 |
+
return op_symbol in filter_state._op_symbols
|
| 330 |
+
|
| 331 |
+
def is_dsl_disabled(self, dsl_name: str) -> bool:
|
| 332 |
+
"""Check if a DSL is currently disabled."""
|
| 333 |
+
filter_state = _get_filter_state()
|
| 334 |
+
return dsl_name in filter_state._dsl_names
|
| 335 |
+
|
| 336 |
+
def __getattr__(self, name: str):
|
| 337 |
+
"""Dynamic attribute access for DSL controllers."""
|
| 338 |
+
# Skip dunder attributes to avoid triggering DSL registry lookups
|
| 339 |
+
# during torch initialization. inspect.getmodule() calls
|
| 340 |
+
# hasattr(module, '__file__') which would otherwise cause a circular
|
| 341 |
+
# import through _get_dsl_registry() while torch is still loading.
|
| 342 |
+
if name.startswith("__") and name.endswith("__"):
|
| 343 |
+
raise AttributeError(f"module '{self.__name__}' has no attribute '{name}'")
|
| 344 |
+
|
| 345 |
+
if name in self.all_dsls:
|
| 346 |
+
return self._get_dsl_controller(name)
|
| 347 |
+
|
| 348 |
+
# Expose private functions for testing
|
| 349 |
+
if name == "_get_dsl_module":
|
| 350 |
+
return _get_dsl_module
|
| 351 |
+
if name == "_get_registry_functions":
|
| 352 |
+
return self._get_registry_functions
|
| 353 |
+
if name == "_get_filter_state":
|
| 354 |
+
return _get_filter_state
|
| 355 |
+
|
| 356 |
+
raise AttributeError(f"module '{self.__name__}' has no attribute '{name}'")
|
| 357 |
+
|
| 358 |
+
def __dir__(self):
|
| 359 |
+
"""Return available attributes including DSL names."""
|
| 360 |
+
attrs = set(super().__dir__())
|
| 361 |
+
attrs.update(
|
| 362 |
+
{
|
| 363 |
+
"available_dsls",
|
| 364 |
+
"all_dsls",
|
| 365 |
+
"get_dsl_operations",
|
| 366 |
+
"disable_operations",
|
| 367 |
+
"enable_operations",
|
| 368 |
+
"disable_dispatch_keys",
|
| 369 |
+
"enable_dispatch_keys",
|
| 370 |
+
"operations_disabled",
|
| 371 |
+
"is_operation_disabled",
|
| 372 |
+
"is_dsl_disabled",
|
| 373 |
+
}
|
| 374 |
+
)
|
| 375 |
+
|
| 376 |
+
# Add DSL names
|
| 377 |
+
try:
|
| 378 |
+
attrs.update(self.all_dsls)
|
| 379 |
+
except Exception:
|
| 380 |
+
# If registry not available yet, skip DSL names
|
| 381 |
+
pass
|
| 382 |
+
|
| 383 |
+
return sorted(attrs)
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
# Replace the current module with our enhanced version
|
| 387 |
+
sys.modules[__name__] = PythonNativeModule(sys.modules[__name__])
|
western_ai/usa/pytorch/torch/backends/quantized/__init__.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import sys
|
| 3 |
+
import types
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
# This function should correspond to the enums present in c10/core/QEngine.h
|
| 9 |
+
def _get_qengine_id(qengine: str) -> int:
|
| 10 |
+
if qengine == "none" or qengine == "" or qengine is None:
|
| 11 |
+
ret = 0
|
| 12 |
+
elif qengine == "fbgemm":
|
| 13 |
+
ret = 1
|
| 14 |
+
elif qengine == "qnnpack":
|
| 15 |
+
ret = 2
|
| 16 |
+
elif qengine == "onednn":
|
| 17 |
+
ret = 3
|
| 18 |
+
elif qengine == "x86":
|
| 19 |
+
ret = 4
|
| 20 |
+
else:
|
| 21 |
+
ret = -1
|
| 22 |
+
raise RuntimeError(f"{qengine} is not a valid value for quantized engine")
|
| 23 |
+
return ret
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
# This function should correspond to the enums present in c10/core/QEngine.h
|
| 27 |
+
def _get_qengine_str(qengine: int) -> str:
|
| 28 |
+
all_engines = {0: "none", 1: "fbgemm", 2: "qnnpack", 3: "onednn", 4: "x86"}
|
| 29 |
+
return all_engines.get(qengine, "*undefined")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class _QEngineProp:
|
| 33 |
+
def __get__(self, obj, objtype) -> str:
|
| 34 |
+
return _get_qengine_str(torch._C._get_qengine())
|
| 35 |
+
|
| 36 |
+
def __set__(self, obj, val: str) -> None:
|
| 37 |
+
torch._C._set_qengine(_get_qengine_id(val))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class _SupportedQEnginesProp:
|
| 41 |
+
def __get__(self, obj, objtype) -> list[str]:
|
| 42 |
+
qengines = torch._C._supported_qengines()
|
| 43 |
+
return [_get_qengine_str(qe) for qe in qengines]
|
| 44 |
+
|
| 45 |
+
def __set__(self, obj, val) -> None:
|
| 46 |
+
raise RuntimeError("Assignment not supported")
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class QuantizedEngine(types.ModuleType):
|
| 50 |
+
def __init__(self, m, name):
|
| 51 |
+
super().__init__(name)
|
| 52 |
+
self.m = m
|
| 53 |
+
|
| 54 |
+
def __getattr__(self, attr):
|
| 55 |
+
return self.m.__getattribute__(attr)
|
| 56 |
+
|
| 57 |
+
engine = _QEngineProp()
|
| 58 |
+
supported_engines = _SupportedQEnginesProp()
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
# This is the sys.modules replacement trick, see
|
| 62 |
+
# https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273
|
| 63 |
+
sys.modules[__name__] = QuantizedEngine(sys.modules[__name__], __name__)
|
| 64 |
+
engine: str
|
| 65 |
+
supported_engines: list[str]
|
western_ai/usa/pytorch/torch/backends/xeon/__init__.py
ADDED
|
File without changes
|
western_ai/usa/pytorch/torch/backends/xeon/run_cpu.py
ADDED
|
@@ -0,0 +1,957 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
"""
|
| 3 |
+
This is a script for launching PyTorch inference on Intel(R) Xeon(R) Scalable Processors with optimal configurations.
|
| 4 |
+
|
| 5 |
+
Single instance inference, multi-instance inference are enabled.
|
| 6 |
+
|
| 7 |
+
Note: term "instance" here doesn't refer to a cloud instance. This script is executed as a single process. It invokes
|
| 8 |
+
multiple "instances" which are formed from multiple threads for each. "instance" is kind of group of threads in this
|
| 9 |
+
context.
|
| 10 |
+
|
| 11 |
+
Illustrated as below:
|
| 12 |
+
|
| 13 |
+
::
|
| 14 |
+
|
| 15 |
+
+-----------------------------+----------------------+-------+
|
| 16 |
+
| process | thread | core |
|
| 17 |
+
+=============================+======================+=======+
|
| 18 |
+
| torch.backends.xeon.run_cpu | instance 0: thread 0 | 0 |
|
| 19 |
+
| | thread 1 | 1 |
|
| 20 |
+
| +----------------------+-------+
|
| 21 |
+
| | instance 1: thread 0 | 2 |
|
| 22 |
+
| | thread 1 | 3 |
|
| 23 |
+
| +----------------------+-------+
|
| 24 |
+
| | ... | ... |
|
| 25 |
+
| +----------------------+-------+
|
| 26 |
+
| | instance N: thread 0 | M |
|
| 27 |
+
| | thread 1 | M+1 |
|
| 28 |
+
+-----------------------------+----------------------+-------+
|
| 29 |
+
|
| 30 |
+
To get the peak performance on Intel(R) Xeon(R) Scalable Processors, the script optimizes the configuration of thread and memory
|
| 31 |
+
management. For thread management, the script configures thread affinity and the preload of Intel OMP library.
|
| 32 |
+
For memory management, it configures NUMA binding and preload optimized memory allocation library (e.g. tcmalloc, jemalloc).
|
| 33 |
+
|
| 34 |
+
Environment variables that will be set by this script:
|
| 35 |
+
|
| 36 |
+
+------------------+-------------------------------------------------------------------------------------------------+
|
| 37 |
+
| Environ Variable | Value |
|
| 38 |
+
+==================+=================================================================================================+
|
| 39 |
+
| LD_PRELOAD | Depending on knobs you set, <lib>/libiomp5.so, <lib>/libjemalloc.so, <lib>/libtcmalloc.so might |
|
| 40 |
+
| | be appended to LD_PRELOAD. |
|
| 41 |
+
+------------------+-------------------------------------------------------------------------------------------------+
|
| 42 |
+
| KMP_AFFINITY | If libiomp5.so is preloaded, KMP_AFFINITY could be set to "granularity=fine,compact,1,0". |
|
| 43 |
+
+------------------+-------------------------------------------------------------------------------------------------+
|
| 44 |
+
| KMP_BLOCKTIME | If libiomp5.so is preloaded, KMP_BLOCKTIME is set to "1". |
|
| 45 |
+
+------------------+-------------------------------------------------------------------------------------------------+
|
| 46 |
+
| OMP_NUM_THREADS | value of ncores_per_instance |
|
| 47 |
+
+------------------+-------------------------------------------------------------------------------------------------+
|
| 48 |
+
| MALLOC_CONF | If libjemalloc.so is preloaded, MALLOC_CONF will be set to |
|
| 49 |
+
| | "oversize_threshold:1,background_thread:true,metadata_thp:auto". |
|
| 50 |
+
+------------------+-------------------------------------------------------------------------------------------------+
|
| 51 |
+
|
| 52 |
+
*Note*: This script respects environment variables set preliminarily. I.e. If you set the environment variables
|
| 53 |
+
mentioned above before running the script, the script will not overwrite the values in the script.
|
| 54 |
+
|
| 55 |
+
How to use this module:
|
| 56 |
+
~~~~~~~~~~~~~~~~~~~~~~~
|
| 57 |
+
|
| 58 |
+
Single instance inference
|
| 59 |
+
-------------------------
|
| 60 |
+
|
| 61 |
+
1. Run single-instance inference on a single node with all CPU nodes.
|
| 62 |
+
|
| 63 |
+
::
|
| 64 |
+
|
| 65 |
+
python -m torch.backends.xeon.run_cpu --throughput-mode script.py args
|
| 66 |
+
|
| 67 |
+
2. Run single-instance inference on a single CPU node.
|
| 68 |
+
|
| 69 |
+
::
|
| 70 |
+
|
| 71 |
+
python -m torch.backends.xeon.run_cpu --node-id 1 script.py args
|
| 72 |
+
|
| 73 |
+
Multi-instance inference
|
| 74 |
+
------------------------
|
| 75 |
+
|
| 76 |
+
1. Multi-instance
|
| 77 |
+
By default this tool runs one process per node. If you want to set the instance numbers and core per instance,
|
| 78 |
+
--ninstances and --ncores-per-instance should be set.
|
| 79 |
+
|
| 80 |
+
::
|
| 81 |
+
|
| 82 |
+
python -m torch.backends.xeon.run_cpu -- python_script args
|
| 83 |
+
|
| 84 |
+
eg: on an Intel(R) Xeon(R) Scalable Processor with 14 instance, 4 cores per instance
|
| 85 |
+
|
| 86 |
+
::
|
| 87 |
+
|
| 88 |
+
python -m torch.backends.xeon.run_cpu --ninstances 14 --ncores-per-instance 4 python_script args
|
| 89 |
+
|
| 90 |
+
2. Run single-instance inference among multiple instances.
|
| 91 |
+
By default, runs all ninstances. If you want to independently run a single instance among ninstances, specify rank.
|
| 92 |
+
|
| 93 |
+
eg: run 0th instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance (i.e., numactl -C 0-27)
|
| 94 |
+
|
| 95 |
+
::
|
| 96 |
+
|
| 97 |
+
python -m torch.backends.xeon.run_cpu --ninstances 2 --rank 0 python_script args
|
| 98 |
+
|
| 99 |
+
eg: run 1st instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance (i.e., numactl -C 28-55)
|
| 100 |
+
|
| 101 |
+
::
|
| 102 |
+
|
| 103 |
+
python -m torch.backends.xeon.run_cpu --ninstances 2 --rank 1 python_script args
|
| 104 |
+
|
| 105 |
+
eg: run 0th instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance, 2 cores per instance,
|
| 106 |
+
first four cores (i.e., numactl -C 0-1)
|
| 107 |
+
|
| 108 |
+
::
|
| 109 |
+
|
| 110 |
+
python -m torch.backends.xeon.run_cpu --core-list "0, 1, 2, 3" --ninstances 2 --ncores-per-instance 2
|
| 111 |
+
--rank 0 python_script args
|
| 112 |
+
|
| 113 |
+
3. To look up what optional arguments this module offers:
|
| 114 |
+
|
| 115 |
+
::
|
| 116 |
+
|
| 117 |
+
python -m torch.backends.xeon.run_cpu --help
|
| 118 |
+
|
| 119 |
+
Memory allocator
|
| 120 |
+
----------------
|
| 121 |
+
|
| 122 |
+
"--enable-tcmalloc" and "--enable-jemalloc" can be used to enable different memory allocator.
|
| 123 |
+
|
| 124 |
+
"""
|
| 125 |
+
|
| 126 |
+
import glob
|
| 127 |
+
import logging
|
| 128 |
+
import os
|
| 129 |
+
import platform
|
| 130 |
+
import re
|
| 131 |
+
import subprocess
|
| 132 |
+
import sys
|
| 133 |
+
from argparse import ArgumentParser, RawTextHelpFormatter, REMAINDER
|
| 134 |
+
from os.path import expanduser
|
| 135 |
+
|
| 136 |
+
from torch.distributed.elastic.multiprocessing import (
|
| 137 |
+
DefaultLogsSpecs as _DefaultLogsSpecs,
|
| 138 |
+
start_processes,
|
| 139 |
+
Std,
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
format_str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
| 144 |
+
logging.basicConfig(level=logging.INFO, format=format_str)
|
| 145 |
+
logger = logging.getLogger(__name__)
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
class _CPUinfo:
|
| 149 |
+
"""Get CPU information, such as cores list and NUMA information."""
|
| 150 |
+
|
| 151 |
+
def __init__(self, test_input=""):
|
| 152 |
+
self.cpuinfo = []
|
| 153 |
+
if platform.system() in ["Windows", "Darwin"]:
|
| 154 |
+
raise RuntimeError(f"{platform.system()} is not supported!!!")
|
| 155 |
+
elif platform.system() == "Linux":
|
| 156 |
+
# Sample output of: `lscpu --parse=CPU,Core,Socket,Node`
|
| 157 |
+
#
|
| 158 |
+
# # The following is the parsable format, which can be fed to other
|
| 159 |
+
# # programs. Each different item in every column has a unique ID
|
| 160 |
+
# # starting from zero.
|
| 161 |
+
# # CPU,Core,Socket,Node
|
| 162 |
+
# 0,0,0,0
|
| 163 |
+
# 1,1,0,0
|
| 164 |
+
# ...
|
| 165 |
+
if test_input == "":
|
| 166 |
+
lscpu_cmd = ["lscpu", "--parse=CPU,Core,Socket,Node"]
|
| 167 |
+
lscpu_info = subprocess.check_output(
|
| 168 |
+
lscpu_cmd, universal_newlines=True
|
| 169 |
+
).split("\n")
|
| 170 |
+
else:
|
| 171 |
+
lscpu_info = test_input.split("\n")
|
| 172 |
+
|
| 173 |
+
# Get information about cpu, core, socket and node
|
| 174 |
+
for line in lscpu_info:
|
| 175 |
+
pattern = r"^([\d]+,[\d]+,[\d]+,[\d]?)"
|
| 176 |
+
regex_out = re.search(pattern, line)
|
| 177 |
+
if regex_out:
|
| 178 |
+
self.cpuinfo.append(regex_out.group(1).strip().split(","))
|
| 179 |
+
|
| 180 |
+
# physical cores := core column in lscpu output
|
| 181 |
+
# logical cores := cPU column in lscpu output
|
| 182 |
+
self.node_nums = int(max(line[3] for line in self.cpuinfo)) + 1
|
| 183 |
+
self.node_physical_cores: list[list[int]] = [] # node_id is index
|
| 184 |
+
self.node_logical_cores: list[list[int]] = [] # node_id is index
|
| 185 |
+
self.physical_core_node_map = {} # physical core to numa node id
|
| 186 |
+
self.logical_core_node_map = {} # logical core to numa node id
|
| 187 |
+
|
| 188 |
+
for node_id in range(self.node_nums):
|
| 189 |
+
cur_node_physical_core = []
|
| 190 |
+
cur_node_logical_core = []
|
| 191 |
+
for cpuinfo in self.cpuinfo:
|
| 192 |
+
nid = cpuinfo[3] if cpuinfo[3] != "" else "0"
|
| 193 |
+
if node_id == int(nid):
|
| 194 |
+
if int(cpuinfo[1]) not in cur_node_physical_core:
|
| 195 |
+
cur_node_physical_core.append(int(cpuinfo[1]))
|
| 196 |
+
self.physical_core_node_map[int(cpuinfo[1])] = int(node_id)
|
| 197 |
+
cur_node_logical_core.append(int(cpuinfo[0]))
|
| 198 |
+
self.logical_core_node_map[int(cpuinfo[0])] = int(node_id)
|
| 199 |
+
self.node_physical_cores.append(cur_node_physical_core)
|
| 200 |
+
self.node_logical_cores.append(cur_node_logical_core)
|
| 201 |
+
|
| 202 |
+
def _physical_core_nums(self):
|
| 203 |
+
return len(self.node_physical_cores) * len(self.node_physical_cores[0])
|
| 204 |
+
|
| 205 |
+
def _logical_core_nums(self):
|
| 206 |
+
return len(self.node_logical_cores) * len(self.node_logical_cores[0])
|
| 207 |
+
|
| 208 |
+
def get_node_physical_cores(self, node_id):
|
| 209 |
+
if node_id < 0 or node_id > self.node_nums - 1:
|
| 210 |
+
raise ValueError(
|
| 211 |
+
f"Invalid node id: {node_id}. Valid node ids: {list(range(len(self.node_physical_cores)))}"
|
| 212 |
+
)
|
| 213 |
+
return self.node_physical_cores[node_id]
|
| 214 |
+
|
| 215 |
+
def get_node_logical_cores(self, node_id):
|
| 216 |
+
if node_id < 0 or node_id > self.node_nums - 1:
|
| 217 |
+
raise ValueError(
|
| 218 |
+
f"Invalid node id: {node_id}. Valid node ids: {list(range(len(self.node_physical_cores)))}"
|
| 219 |
+
)
|
| 220 |
+
return self.node_logical_cores[node_id]
|
| 221 |
+
|
| 222 |
+
def get_all_physical_cores(self):
|
| 223 |
+
all_cores = []
|
| 224 |
+
for cores in self.node_physical_cores:
|
| 225 |
+
all_cores.extend(cores)
|
| 226 |
+
return all_cores
|
| 227 |
+
|
| 228 |
+
def get_all_logical_cores(self):
|
| 229 |
+
all_cores = []
|
| 230 |
+
for cores in self.node_logical_cores:
|
| 231 |
+
all_cores.extend(cores)
|
| 232 |
+
return all_cores
|
| 233 |
+
|
| 234 |
+
def numa_aware_check(self, core_list):
|
| 235 |
+
"""
|
| 236 |
+
Check whether all cores in core_list are in the same NUMA node.
|
| 237 |
+
|
| 238 |
+
Cross NUMA will reduce performance.
|
| 239 |
+
We strongly advice to not use cores on different nodes.
|
| 240 |
+
"""
|
| 241 |
+
cores_numa_map = self.logical_core_node_map
|
| 242 |
+
numa_ids = []
|
| 243 |
+
for core in core_list:
|
| 244 |
+
numa_id = cores_numa_map[core]
|
| 245 |
+
if numa_id not in numa_ids:
|
| 246 |
+
numa_ids.append(numa_id)
|
| 247 |
+
if len(numa_ids) > 1:
|
| 248 |
+
logger.warning(
|
| 249 |
+
"Numa Aware: cores:%s on different NUMA nodes:%s. To avoid \
|
| 250 |
+
this behavior, please use --ncores-per-instance knob to make sure number of cores is divisible by --ncores-per-\
|
| 251 |
+
instance. Alternatively, please use --skip-cross-node-cores knob.",
|
| 252 |
+
core_list,
|
| 253 |
+
numa_ids,
|
| 254 |
+
)
|
| 255 |
+
if len(numa_ids) == 0:
|
| 256 |
+
raise RuntimeError(
|
| 257 |
+
"invalid number of NUMA nodes; please make sure numa_ids >= 1"
|
| 258 |
+
)
|
| 259 |
+
return numa_ids
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
class _Launcher:
|
| 263 |
+
r"""Class for launcher."""
|
| 264 |
+
|
| 265 |
+
msg_lib_notfound = (
|
| 266 |
+
f"Unable to find the {{0}} library file lib{{1}}.so in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib \
|
| 267 |
+
or /.local/lib/ or /usr/local/lib/ or /usr/local/lib64/ or /usr/lib or /usr/lib64 or \
|
| 268 |
+
{expanduser('~')}/.local/lib/ so the LD_PRELOAD environment variable will not be set."
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
def __init__(self) -> None:
|
| 272 |
+
self.cpuinfo = _CPUinfo()
|
| 273 |
+
|
| 274 |
+
def add_lib_preload(self, lib_type):
|
| 275 |
+
"""Enable TCMalloc/JeMalloc/intel OpenMP."""
|
| 276 |
+
library_paths = []
|
| 277 |
+
if "CONDA_PREFIX" in os.environ:
|
| 278 |
+
library_paths.append(f"{os.environ['CONDA_PREFIX']}/lib")
|
| 279 |
+
if "VIRTUAL_ENV" in os.environ:
|
| 280 |
+
library_paths.append(f"{os.environ['VIRTUAL_ENV']}/lib")
|
| 281 |
+
|
| 282 |
+
library_paths += [
|
| 283 |
+
f"{expanduser('~')}/.local/lib",
|
| 284 |
+
"/usr/local/lib",
|
| 285 |
+
"/usr/local/lib64",
|
| 286 |
+
"/usr/lib",
|
| 287 |
+
"/usr/lib64",
|
| 288 |
+
]
|
| 289 |
+
|
| 290 |
+
lib_find = False
|
| 291 |
+
lib_set = False
|
| 292 |
+
for item in os.getenv("LD_PRELOAD", "").split(":"):
|
| 293 |
+
if item.endswith(f"lib{lib_type}.so"):
|
| 294 |
+
lib_set = True
|
| 295 |
+
break
|
| 296 |
+
if not lib_set:
|
| 297 |
+
for lib_path in library_paths:
|
| 298 |
+
# pyrefly: ignore [unbound-name]
|
| 299 |
+
library_file = os.path.join(lib_path, f"lib{lib_type}.so")
|
| 300 |
+
matches = glob.glob(library_file)
|
| 301 |
+
if len(matches) > 0:
|
| 302 |
+
# pyrefly: ignore [unbound-name]
|
| 303 |
+
ld_preloads = [f"{matches[0]}", os.getenv("LD_PRELOAD", "")]
|
| 304 |
+
# pyrefly: ignore [unbound-name]
|
| 305 |
+
os.environ["LD_PRELOAD"] = os.pathsep.join(
|
| 306 |
+
# pyrefly: ignore [unbound-name]
|
| 307 |
+
[p.strip(os.pathsep) for p in ld_preloads if p]
|
| 308 |
+
)
|
| 309 |
+
lib_find = True
|
| 310 |
+
break
|
| 311 |
+
return lib_set or lib_find
|
| 312 |
+
|
| 313 |
+
def is_numactl_available(self):
|
| 314 |
+
numactl_available = False
|
| 315 |
+
try:
|
| 316 |
+
cmd = ["numactl", "-C", "0", "-m", "0", "hostname"]
|
| 317 |
+
r = subprocess.run(
|
| 318 |
+
cmd,
|
| 319 |
+
env=os.environ,
|
| 320 |
+
stdout=subprocess.DEVNULL,
|
| 321 |
+
stderr=subprocess.DEVNULL,
|
| 322 |
+
check=False,
|
| 323 |
+
)
|
| 324 |
+
if r.returncode == 0:
|
| 325 |
+
numactl_available = True
|
| 326 |
+
except Exception:
|
| 327 |
+
pass
|
| 328 |
+
return numactl_available
|
| 329 |
+
|
| 330 |
+
def set_memory_allocator(
|
| 331 |
+
self, enable_tcmalloc=True, enable_jemalloc=False, use_default_allocator=False
|
| 332 |
+
):
|
| 333 |
+
"""
|
| 334 |
+
Enable TCMalloc/JeMalloc with LD_PRELOAD and set configuration for JeMalloc.
|
| 335 |
+
|
| 336 |
+
By default, PTMalloc will be used for PyTorch, but TCMalloc and JeMalloc can get better
|
| 337 |
+
memory reuse and reduce page fault to improve performance.
|
| 338 |
+
"""
|
| 339 |
+
if enable_tcmalloc and enable_jemalloc:
|
| 340 |
+
raise RuntimeError(
|
| 341 |
+
"Unable to enable TCMalloc and JEMalloc at the same time."
|
| 342 |
+
)
|
| 343 |
+
|
| 344 |
+
if enable_tcmalloc:
|
| 345 |
+
find_tc = self.add_lib_preload(lib_type="tcmalloc")
|
| 346 |
+
if not find_tc:
|
| 347 |
+
msg = f'{self.msg_lib_notfound} you can use "conda install -c conda-forge gperftools" to install {{0}}'
|
| 348 |
+
logger.warning(msg.format("TCmalloc", "tcmalloc"))
|
| 349 |
+
else:
|
| 350 |
+
logger.info("Use TCMalloc memory allocator")
|
| 351 |
+
|
| 352 |
+
elif enable_jemalloc:
|
| 353 |
+
find_je = self.add_lib_preload(lib_type="jemalloc")
|
| 354 |
+
if not find_je:
|
| 355 |
+
msg = f'{self.msg_lib_notfound} you can use "conda install -c conda-forge jemalloc" to install {{0}}'
|
| 356 |
+
logger.warning(msg.format("Jemalloc", "jemalloc"))
|
| 357 |
+
else:
|
| 358 |
+
logger.info("Use JeMalloc memory allocator")
|
| 359 |
+
self.set_env(
|
| 360 |
+
"MALLOC_CONF",
|
| 361 |
+
"oversize_threshold:1,background_thread:true,metadata_thp:auto",
|
| 362 |
+
)
|
| 363 |
+
|
| 364 |
+
elif use_default_allocator:
|
| 365 |
+
pass
|
| 366 |
+
|
| 367 |
+
else:
|
| 368 |
+
find_tc = self.add_lib_preload(lib_type="tcmalloc")
|
| 369 |
+
if find_tc:
|
| 370 |
+
logger.info("Use TCMalloc memory allocator")
|
| 371 |
+
return
|
| 372 |
+
find_je = self.add_lib_preload(lib_type="jemalloc")
|
| 373 |
+
if find_je:
|
| 374 |
+
logger.info("Use JeMalloc memory allocator")
|
| 375 |
+
return
|
| 376 |
+
logger.warning(
|
| 377 |
+
"""Neither TCMalloc nor JeMalloc is found in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib
|
| 378 |
+
or /.local/lib/ or /usr/local/lib/ or /usr/local/lib64/ or /usr/lib or /usr/lib64 or
|
| 379 |
+
%s/.local/lib/ so the LD_PRELOAD environment variable will not be set.
|
| 380 |
+
This may drop the performance""",
|
| 381 |
+
expanduser("~"),
|
| 382 |
+
)
|
| 383 |
+
|
| 384 |
+
def log_env_var(self, env_var_name=""):
|
| 385 |
+
if env_var_name in os.environ:
|
| 386 |
+
logger.info("%s=%s", env_var_name, os.environ[env_var_name])
|
| 387 |
+
|
| 388 |
+
def set_env(self, env_name, env_value):
|
| 389 |
+
if not env_value:
|
| 390 |
+
logger.warning("%s is None", env_name)
|
| 391 |
+
if env_name not in os.environ:
|
| 392 |
+
os.environ[env_name] = env_value
|
| 393 |
+
elif os.environ[env_name] != env_value:
|
| 394 |
+
logger.warning(
|
| 395 |
+
"Overriding value with the one set in environment variable: %s. \
|
| 396 |
+
Value applied: %s. Value ignored: %s",
|
| 397 |
+
env_name,
|
| 398 |
+
os.environ[env_name],
|
| 399 |
+
env_value,
|
| 400 |
+
)
|
| 401 |
+
self.log_env_var(env_name)
|
| 402 |
+
|
| 403 |
+
# set_kmp_affinity is used to control whether to set KMP_AFFINITY or not.
|
| 404 |
+
# In scenario that use all cores on all nodes, including logical cores, setting KMP_AFFINITY disables logical cores.
|
| 405 |
+
# In this case, KMP_AFFINITY should not be set.
|
| 406 |
+
def set_multi_thread_and_allocator(
|
| 407 |
+
self,
|
| 408 |
+
ncores_per_instance,
|
| 409 |
+
disable_iomp=False,
|
| 410 |
+
set_kmp_affinity=True,
|
| 411 |
+
enable_tcmalloc=True,
|
| 412 |
+
enable_jemalloc=False,
|
| 413 |
+
use_default_allocator=False,
|
| 414 |
+
):
|
| 415 |
+
"""
|
| 416 |
+
Set multi-thread configuration and enable Intel openMP and TCMalloc/JeMalloc.
|
| 417 |
+
|
| 418 |
+
By default, GNU openMP and PTMalloc are used in PyTorch. but Intel openMP and TCMalloc/JeMalloc are better alternatives
|
| 419 |
+
to get performance benefit.
|
| 420 |
+
"""
|
| 421 |
+
self.set_memory_allocator(
|
| 422 |
+
enable_tcmalloc, enable_jemalloc, use_default_allocator
|
| 423 |
+
)
|
| 424 |
+
self.set_env("OMP_NUM_THREADS", str(ncores_per_instance))
|
| 425 |
+
if not disable_iomp:
|
| 426 |
+
find_iomp = self.add_lib_preload(lib_type="iomp5")
|
| 427 |
+
if not find_iomp:
|
| 428 |
+
msg = f'{self.msg_lib_notfound} you can use "conda install mkl" to install {{0}}'
|
| 429 |
+
logger.warning(msg.format("iomp", "iomp5"))
|
| 430 |
+
else:
|
| 431 |
+
logger.info("Using Intel OpenMP")
|
| 432 |
+
if set_kmp_affinity:
|
| 433 |
+
self.set_env("KMP_AFFINITY", "granularity=fine,compact,1,0")
|
| 434 |
+
self.set_env("KMP_BLOCKTIME", "1")
|
| 435 |
+
self.log_env_var("LD_PRELOAD")
|
| 436 |
+
|
| 437 |
+
r"""
|
| 438 |
+
Launcher for single instance and multi-instance
|
| 439 |
+
"""
|
| 440 |
+
|
| 441 |
+
def launch(self, args):
|
| 442 |
+
cores = []
|
| 443 |
+
set_kmp_affinity = True
|
| 444 |
+
enable_taskset = False
|
| 445 |
+
if args.core_list: # user specify what cores will be used by params
|
| 446 |
+
cores = [int(x) for x in args.core_list.split(",")]
|
| 447 |
+
if args.ncores_per_instance == -1:
|
| 448 |
+
raise RuntimeError(
|
| 449 |
+
'please specify the "--ncores-per-instance" if you have pass the --core-list params'
|
| 450 |
+
)
|
| 451 |
+
elif (
|
| 452 |
+
args.ninstances > 1
|
| 453 |
+
and args.ncores_per_instance * args.ninstances < len(cores)
|
| 454 |
+
):
|
| 455 |
+
logger.warning(
|
| 456 |
+
"only first %s cores will be used, \
|
| 457 |
+
but you specify %s cores in core_list",
|
| 458 |
+
args.ncores_per_instance * args.ninstances,
|
| 459 |
+
len(cores),
|
| 460 |
+
)
|
| 461 |
+
else:
|
| 462 |
+
args.ninstances = len(cores) // args.ncores_per_instance
|
| 463 |
+
|
| 464 |
+
else:
|
| 465 |
+
if args.use_logical_core:
|
| 466 |
+
if args.node_id != -1:
|
| 467 |
+
cores = self.cpuinfo.get_node_logical_cores(args.node_id)
|
| 468 |
+
else:
|
| 469 |
+
cores = self.cpuinfo.get_all_logical_cores()
|
| 470 |
+
# When using all cores on all nodes, including logical cores,
|
| 471 |
+
# setting KMP_AFFINITY disables logical cores. Thus, KMP_AFFINITY should not be set.
|
| 472 |
+
set_kmp_affinity = False
|
| 473 |
+
else:
|
| 474 |
+
if args.node_id != -1:
|
| 475 |
+
cores = self.cpuinfo.get_node_physical_cores(args.node_id)
|
| 476 |
+
else:
|
| 477 |
+
cores = self.cpuinfo.get_all_physical_cores()
|
| 478 |
+
if (
|
| 479 |
+
not args.multi_instance
|
| 480 |
+
and args.ninstances == -1
|
| 481 |
+
and args.ncores_per_instance == -1
|
| 482 |
+
):
|
| 483 |
+
args.ninstances = 1
|
| 484 |
+
args.ncores_per_instance = len(cores)
|
| 485 |
+
elif (
|
| 486 |
+
args.multi_instance
|
| 487 |
+
and args.ninstances == -1
|
| 488 |
+
and args.ncores_per_instance == -1
|
| 489 |
+
):
|
| 490 |
+
args.throughput_mode = True
|
| 491 |
+
elif args.ncores_per_instance == -1 and args.ninstances != -1:
|
| 492 |
+
if args.ninstances > len(cores):
|
| 493 |
+
raise RuntimeError(
|
| 494 |
+
f"there are {len(cores)} total cores but you specify {args.ninstances} ninstances; \
|
| 495 |
+
please make sure ninstances <= total_cores)"
|
| 496 |
+
)
|
| 497 |
+
else:
|
| 498 |
+
args.ncores_per_instance = len(cores) // args.ninstances
|
| 499 |
+
elif args.ncores_per_instance != -1 and args.ninstances == -1:
|
| 500 |
+
if not args.skip_cross_node_cores:
|
| 501 |
+
args.ninstances = len(cores) // args.ncores_per_instance
|
| 502 |
+
else:
|
| 503 |
+
ncore_per_node = len(self.cpuinfo.node_physical_cores[0])
|
| 504 |
+
num_leftover_cores = ncore_per_node % args.ncores_per_instance
|
| 505 |
+
if args.ncores_per_instance > ncore_per_node:
|
| 506 |
+
# too many ncores_per_instance to skip cross-node cores
|
| 507 |
+
logger.warning(
|
| 508 |
+
"there are %s core(s) per socket, but you specify %s ncores_per_instance and \
|
| 509 |
+
skip_cross_node_cores. Please make sure --ncores-per-instance < core(s) per \
|
| 510 |
+
socket",
|
| 511 |
+
ncore_per_node,
|
| 512 |
+
args.ncores_per_instance,
|
| 513 |
+
)
|
| 514 |
+
sys.exit(-1)
|
| 515 |
+
elif num_leftover_cores == 0:
|
| 516 |
+
# aren't any cross-node cores
|
| 517 |
+
logger.info(
|
| 518 |
+
"--skip-cross-node-cores is set, but there are no cross-node cores."
|
| 519 |
+
)
|
| 520 |
+
args.ninstances = len(cores) // args.ncores_per_instance
|
| 521 |
+
else:
|
| 522 |
+
# skip cross-node cores
|
| 523 |
+
if args.ninstances != -1:
|
| 524 |
+
logger.warning(
|
| 525 |
+
"--skip-cross-node-cores is exclusive to --ninstances. --ninstances \
|
| 526 |
+
won't take effect even if it is set explicitly."
|
| 527 |
+
)
|
| 528 |
+
|
| 529 |
+
i = 1
|
| 530 |
+
leftover_cores = set()
|
| 531 |
+
while ncore_per_node * i <= len(cores):
|
| 532 |
+
leftover_cores.update(
|
| 533 |
+
cores[
|
| 534 |
+
ncore_per_node * i
|
| 535 |
+
- num_leftover_cores : ncore_per_node * i
|
| 536 |
+
]
|
| 537 |
+
)
|
| 538 |
+
i += 1
|
| 539 |
+
cores = list(set(cores) - leftover_cores)
|
| 540 |
+
if len(cores) % args.ncores_per_instance != 0:
|
| 541 |
+
raise AssertionError(
|
| 542 |
+
f"Number of cores ({len(cores)}) must be divisible by "
|
| 543 |
+
f"ncores_per_instance ({args.ncores_per_instance})"
|
| 544 |
+
)
|
| 545 |
+
args.ninstances = len(cores) // args.ncores_per_instance
|
| 546 |
+
else:
|
| 547 |
+
if args.ninstances * args.ncores_per_instance > len(cores):
|
| 548 |
+
raise RuntimeError(
|
| 549 |
+
"Please make sure ninstances * ncores_per_instance <= total_cores"
|
| 550 |
+
)
|
| 551 |
+
if args.latency_mode:
|
| 552 |
+
logger.warning(
|
| 553 |
+
"--latency-mode is exclusive to --ninstances, --ncores-per-instance, --node-id and \
|
| 554 |
+
--use-logical-core. They won't take effect even they are set explicitly."
|
| 555 |
+
)
|
| 556 |
+
args.ncores_per_instance = 4
|
| 557 |
+
cores = self.cpuinfo.get_all_physical_cores()
|
| 558 |
+
args.ninstances = len(cores) // args.ncores_per_instance
|
| 559 |
+
|
| 560 |
+
if args.throughput_mode:
|
| 561 |
+
logger.warning(
|
| 562 |
+
"--throughput-mode is exclusive to --ninstances, --ncores-per-instance, --node-id and \
|
| 563 |
+
--use-logical-core. They won't take effect even they are set explicitly."
|
| 564 |
+
)
|
| 565 |
+
args.ninstances = self.cpuinfo.node_nums
|
| 566 |
+
cores = self.cpuinfo.get_all_physical_cores()
|
| 567 |
+
args.ncores_per_instance = len(cores) // args.ninstances
|
| 568 |
+
|
| 569 |
+
if args.ninstances > 1 and args.rank != -1:
|
| 570 |
+
logger.info(
|
| 571 |
+
"assigning %s cores for instance %s",
|
| 572 |
+
args.ncores_per_instance,
|
| 573 |
+
args.rank,
|
| 574 |
+
)
|
| 575 |
+
|
| 576 |
+
if not args.disable_numactl:
|
| 577 |
+
numactl_available = self.is_numactl_available()
|
| 578 |
+
if not numactl_available:
|
| 579 |
+
if not args.disable_taskset:
|
| 580 |
+
logger.warning(
|
| 581 |
+
"Core binding with numactl is not available. Disabling numactl and using taskset instead. \
|
| 582 |
+
This may affect performance in multi-socket system; please use numactl if memory binding is needed."
|
| 583 |
+
)
|
| 584 |
+
args.disable_numactl = True
|
| 585 |
+
enable_taskset = True
|
| 586 |
+
else:
|
| 587 |
+
logger.warning(
|
| 588 |
+
"Core binding with numactl is not available, and --disable_taskset is set. \
|
| 589 |
+
Please unset --disable_taskset to use taskset instead of numactl."
|
| 590 |
+
)
|
| 591 |
+
sys.exit(-1)
|
| 592 |
+
|
| 593 |
+
if not args.disable_taskset:
|
| 594 |
+
enable_taskset = True
|
| 595 |
+
|
| 596 |
+
self.set_multi_thread_and_allocator(
|
| 597 |
+
args.ncores_per_instance,
|
| 598 |
+
args.disable_iomp,
|
| 599 |
+
set_kmp_affinity,
|
| 600 |
+
args.enable_tcmalloc,
|
| 601 |
+
args.enable_jemalloc,
|
| 602 |
+
args.use_default_allocator,
|
| 603 |
+
)
|
| 604 |
+
entrypoint = ""
|
| 605 |
+
launch_args = {}
|
| 606 |
+
launch_envs: dict[int, dict] = {}
|
| 607 |
+
launch_tee = {}
|
| 608 |
+
# check whether is launched from torchrun with --nproc-per-node <num workers>
|
| 609 |
+
local_size = int(os.environ.get("LOCAL_WORLD_SIZE", 1))
|
| 610 |
+
local_rank = int(os.environ.get("LOCAL_RANK", 0))
|
| 611 |
+
for i in range(args.ninstances):
|
| 612 |
+
cmd = []
|
| 613 |
+
cur_process_cores = ""
|
| 614 |
+
if not args.disable_numactl or enable_taskset:
|
| 615 |
+
if not args.disable_numactl:
|
| 616 |
+
cmd = ["numactl"]
|
| 617 |
+
elif enable_taskset:
|
| 618 |
+
cmd = ["taskset"]
|
| 619 |
+
cores = sorted(cores)
|
| 620 |
+
if (
|
| 621 |
+
args.rank == -1
|
| 622 |
+
): # sequentially assign ncores_per_instance to ninstances
|
| 623 |
+
core_list = cores[
|
| 624 |
+
i * args.ncores_per_instance : (i + 1)
|
| 625 |
+
* args.ncores_per_instance
|
| 626 |
+
]
|
| 627 |
+
else: # assign ncores_per_instance from rank
|
| 628 |
+
core_list = cores[
|
| 629 |
+
args.rank * args.ncores_per_instance : (args.rank + 1)
|
| 630 |
+
* args.ncores_per_instance
|
| 631 |
+
]
|
| 632 |
+
|
| 633 |
+
core_ranges: list[dict] = []
|
| 634 |
+
if local_size > 1:
|
| 635 |
+
total_num_cores = len(core_list)
|
| 636 |
+
cores_per_rank = total_num_cores // local_size
|
| 637 |
+
if cores_per_rank < 1:
|
| 638 |
+
raise AssertionError(
|
| 639 |
+
f"At least one core needs to be assigned to each rank, "
|
| 640 |
+
f"got {total_num_cores} cores for {local_size} ranks"
|
| 641 |
+
)
|
| 642 |
+
core_list = core_list[
|
| 643 |
+
cores_per_rank * local_rank : cores_per_rank * (local_rank + 1)
|
| 644 |
+
]
|
| 645 |
+
for core in core_list:
|
| 646 |
+
if len(core_ranges) == 0:
|
| 647 |
+
range_elem = {"start": core, "end": core}
|
| 648 |
+
core_ranges.append(range_elem)
|
| 649 |
+
else:
|
| 650 |
+
if core - core_ranges[-1]["end"] == 1:
|
| 651 |
+
core_ranges[-1]["end"] = core
|
| 652 |
+
else:
|
| 653 |
+
range_elem = {"start": core, "end": core}
|
| 654 |
+
core_ranges.append(range_elem)
|
| 655 |
+
for r in core_ranges:
|
| 656 |
+
cur_process_cores = f"{cur_process_cores}{r['start']}-{r['end']},"
|
| 657 |
+
cur_process_cores = cur_process_cores[:-1]
|
| 658 |
+
if not args.disable_numactl:
|
| 659 |
+
numa_params = f"-C {cur_process_cores} "
|
| 660 |
+
numa_ids = ",".join(
|
| 661 |
+
[
|
| 662 |
+
str(numa_id)
|
| 663 |
+
for numa_id in self.cpuinfo.numa_aware_check(core_list)
|
| 664 |
+
]
|
| 665 |
+
)
|
| 666 |
+
numa_params += f"-m {numa_ids}"
|
| 667 |
+
cmd.extend(numa_params.split())
|
| 668 |
+
elif enable_taskset:
|
| 669 |
+
taskset_params = f"-c {cur_process_cores} "
|
| 670 |
+
cmd.extend(taskset_params.split())
|
| 671 |
+
with_python = not args.no_python
|
| 672 |
+
if with_python:
|
| 673 |
+
cmd.append(sys.executable)
|
| 674 |
+
cmd.append("-u")
|
| 675 |
+
if args.module:
|
| 676 |
+
cmd.append("-m")
|
| 677 |
+
cmd.append(args.program)
|
| 678 |
+
cmd.extend(args.program_args)
|
| 679 |
+
cmd_s = " ".join(cmd)
|
| 680 |
+
logger.info(cmd_s)
|
| 681 |
+
if entrypoint == "":
|
| 682 |
+
entrypoint = cmd[0]
|
| 683 |
+
del cmd[0]
|
| 684 |
+
launch_args[i] = tuple(cmd)
|
| 685 |
+
launch_envs[i] = {}
|
| 686 |
+
launch_tee[i] = Std.ALL
|
| 687 |
+
|
| 688 |
+
if args.rank != -1: # launches single instance, rank, only
|
| 689 |
+
break
|
| 690 |
+
|
| 691 |
+
ctx = start_processes(
|
| 692 |
+
name=args.log_file_prefix,
|
| 693 |
+
entrypoint=entrypoint,
|
| 694 |
+
args=launch_args,
|
| 695 |
+
envs=launch_envs,
|
| 696 |
+
logs_specs=_DefaultLogsSpecs(log_dir=args.log_path, tee=launch_tee),
|
| 697 |
+
)
|
| 698 |
+
ctx.wait()
|
| 699 |
+
|
| 700 |
+
|
| 701 |
+
def _add_memory_allocator_params(parser):
|
| 702 |
+
group = parser.add_argument_group("Memory Allocator Parameters")
|
| 703 |
+
# allocator control
|
| 704 |
+
group.add_argument(
|
| 705 |
+
"--enable-tcmalloc",
|
| 706 |
+
"--enable_tcmalloc",
|
| 707 |
+
action="store_true",
|
| 708 |
+
default=False,
|
| 709 |
+
help="Enable tcmalloc allocator",
|
| 710 |
+
)
|
| 711 |
+
group.add_argument(
|
| 712 |
+
"--enable-jemalloc",
|
| 713 |
+
"--enable_jemalloc",
|
| 714 |
+
action="store_true",
|
| 715 |
+
default=False,
|
| 716 |
+
help="Enable jemalloc allocator",
|
| 717 |
+
)
|
| 718 |
+
group.add_argument(
|
| 719 |
+
"--use-default-allocator",
|
| 720 |
+
"--use_default_allocator",
|
| 721 |
+
action="store_true",
|
| 722 |
+
default=False,
|
| 723 |
+
help="Use default memory allocator",
|
| 724 |
+
)
|
| 725 |
+
|
| 726 |
+
|
| 727 |
+
def _add_multi_instance_params(parser):
|
| 728 |
+
group = parser.add_argument_group("Multi-instance Parameters")
|
| 729 |
+
# multi-instance control
|
| 730 |
+
group.add_argument(
|
| 731 |
+
"--ncores-per-instance",
|
| 732 |
+
"--ncores_per_instance",
|
| 733 |
+
metavar="\b",
|
| 734 |
+
default=-1,
|
| 735 |
+
type=int,
|
| 736 |
+
help="Cores per instance",
|
| 737 |
+
)
|
| 738 |
+
group.add_argument(
|
| 739 |
+
"--ninstances",
|
| 740 |
+
metavar="\b",
|
| 741 |
+
default=-1,
|
| 742 |
+
type=int,
|
| 743 |
+
help="For multi-instance, you should give the cores number you used for per instance.",
|
| 744 |
+
)
|
| 745 |
+
group.add_argument(
|
| 746 |
+
"--skip-cross-node-cores",
|
| 747 |
+
"--skip_cross_node_cores",
|
| 748 |
+
action="store_true",
|
| 749 |
+
default=False,
|
| 750 |
+
help="If specified --ncores-per-instance, skips cross-node cores.",
|
| 751 |
+
)
|
| 752 |
+
group.add_argument(
|
| 753 |
+
"--rank",
|
| 754 |
+
metavar="\b",
|
| 755 |
+
default="-1",
|
| 756 |
+
type=int,
|
| 757 |
+
help="Specify instance index to assign ncores_per_instance for rank; \
|
| 758 |
+
otherwise ncores_per_instance will be assigned sequentially to ninstances. Please refer to \
|
| 759 |
+
https://github.com/intel/intel-extension-for-pytorch/blob/master/docs/tutorials/performance_tuning/launch_script.md",
|
| 760 |
+
)
|
| 761 |
+
group.add_argument(
|
| 762 |
+
"--latency-mode",
|
| 763 |
+
"--latency_mode",
|
| 764 |
+
action="store_true",
|
| 765 |
+
default=False,
|
| 766 |
+
help="By default 4 core per instance and use all physical cores",
|
| 767 |
+
)
|
| 768 |
+
group.add_argument(
|
| 769 |
+
"--throughput-mode",
|
| 770 |
+
"--throughput_mode",
|
| 771 |
+
action="store_true",
|
| 772 |
+
default=False,
|
| 773 |
+
help="By default one instance per node and use all physical cores",
|
| 774 |
+
)
|
| 775 |
+
group.add_argument(
|
| 776 |
+
"--node-id",
|
| 777 |
+
"--node_id",
|
| 778 |
+
metavar="\b",
|
| 779 |
+
default=-1,
|
| 780 |
+
type=int,
|
| 781 |
+
help="node id for multi-instance, by default all nodes will be used",
|
| 782 |
+
)
|
| 783 |
+
group.add_argument(
|
| 784 |
+
"--use-logical-core",
|
| 785 |
+
"--use_logical_core",
|
| 786 |
+
action="store_true",
|
| 787 |
+
default=False,
|
| 788 |
+
help="Whether only use physical cores",
|
| 789 |
+
)
|
| 790 |
+
group.add_argument(
|
| 791 |
+
"--disable-numactl",
|
| 792 |
+
"--disable_numactl",
|
| 793 |
+
action="store_true",
|
| 794 |
+
default=False,
|
| 795 |
+
help="Disable numactl",
|
| 796 |
+
)
|
| 797 |
+
group.add_argument(
|
| 798 |
+
"--disable-taskset",
|
| 799 |
+
"--disable_taskset",
|
| 800 |
+
action="store_true",
|
| 801 |
+
default=False,
|
| 802 |
+
help="Disable taskset",
|
| 803 |
+
)
|
| 804 |
+
group.add_argument(
|
| 805 |
+
"--core-list",
|
| 806 |
+
"--core_list",
|
| 807 |
+
metavar="\b",
|
| 808 |
+
default=None,
|
| 809 |
+
type=str,
|
| 810 |
+
help='Specify the core list as "core_id, core_id, ....", otherwise, all the cores will be used.',
|
| 811 |
+
)
|
| 812 |
+
group.add_argument(
|
| 813 |
+
"--log-path",
|
| 814 |
+
"--log_path",
|
| 815 |
+
metavar="\b",
|
| 816 |
+
default="",
|
| 817 |
+
type=str,
|
| 818 |
+
help="The log file directory. Default path is "
|
| 819 |
+
", which means disable logging to files.",
|
| 820 |
+
)
|
| 821 |
+
group.add_argument(
|
| 822 |
+
"--log-file-prefix",
|
| 823 |
+
"--log_file_prefix",
|
| 824 |
+
metavar="\b",
|
| 825 |
+
default="run",
|
| 826 |
+
type=str,
|
| 827 |
+
help="log file prefix",
|
| 828 |
+
)
|
| 829 |
+
|
| 830 |
+
|
| 831 |
+
def _add_kmp_iomp_params(parser):
|
| 832 |
+
group = parser.add_argument_group("IOMP Parameters")
|
| 833 |
+
group.add_argument(
|
| 834 |
+
"--disable-iomp",
|
| 835 |
+
"--disable_iomp",
|
| 836 |
+
action="store_true",
|
| 837 |
+
default=False,
|
| 838 |
+
help="By default, we use Intel OpenMP and libiomp5.so will be add to LD_PRELOAD",
|
| 839 |
+
)
|
| 840 |
+
|
| 841 |
+
|
| 842 |
+
def create_args(parser=None):
|
| 843 |
+
"""
|
| 844 |
+
Parse the command line options.
|
| 845 |
+
|
| 846 |
+
@retval ArgumentParser
|
| 847 |
+
"""
|
| 848 |
+
# pyrefly: ignore [missing-attribute]
|
| 849 |
+
parser.add_argument(
|
| 850 |
+
"--multi-instance",
|
| 851 |
+
"--multi_instance",
|
| 852 |
+
action="store_true",
|
| 853 |
+
default=False,
|
| 854 |
+
help="Enable multi-instance, by default one instance per node",
|
| 855 |
+
)
|
| 856 |
+
|
| 857 |
+
# pyrefly: ignore [missing-attribute]
|
| 858 |
+
parser.add_argument(
|
| 859 |
+
"-m",
|
| 860 |
+
"--module",
|
| 861 |
+
default=False,
|
| 862 |
+
action="store_true",
|
| 863 |
+
help="Changes each process to interpret the launch script "
|
| 864 |
+
"as a python module, executing with the same behavior as"
|
| 865 |
+
'"python -m".',
|
| 866 |
+
)
|
| 867 |
+
|
| 868 |
+
# pyrefly: ignore [missing-attribute]
|
| 869 |
+
parser.add_argument(
|
| 870 |
+
"--no-python",
|
| 871 |
+
"--no_python",
|
| 872 |
+
default=False,
|
| 873 |
+
action="store_true",
|
| 874 |
+
help='Do not prepend the --program script with "python" - just exec '
|
| 875 |
+
"it directly. Useful when the script is not a Python script.",
|
| 876 |
+
)
|
| 877 |
+
|
| 878 |
+
_add_memory_allocator_params(parser)
|
| 879 |
+
_add_kmp_iomp_params(parser)
|
| 880 |
+
|
| 881 |
+
_add_multi_instance_params(parser)
|
| 882 |
+
# positional
|
| 883 |
+
# pyrefly: ignore [missing-attribute]
|
| 884 |
+
parser.add_argument(
|
| 885 |
+
"program",
|
| 886 |
+
type=str,
|
| 887 |
+
help="The full path to the program/script to be launched. "
|
| 888 |
+
"followed by all the arguments for the script",
|
| 889 |
+
)
|
| 890 |
+
|
| 891 |
+
# rest from the training program
|
| 892 |
+
# pyrefly: ignore [missing-attribute]
|
| 893 |
+
parser.add_argument("program_args", nargs=REMAINDER)
|
| 894 |
+
|
| 895 |
+
|
| 896 |
+
def main(args):
|
| 897 |
+
env_before = set(os.environ.keys())
|
| 898 |
+
if platform.system() in ["Windows", "Darwin"]:
|
| 899 |
+
raise RuntimeError(f"{platform.system()} is not supported!!!")
|
| 900 |
+
|
| 901 |
+
if args.log_path:
|
| 902 |
+
os.makedirs(args.log_path, exist_ok=True)
|
| 903 |
+
else:
|
| 904 |
+
args.log_path = os.devnull
|
| 905 |
+
|
| 906 |
+
if args.latency_mode and args.throughput_mode:
|
| 907 |
+
raise RuntimeError(
|
| 908 |
+
"Either args.latency_mode or args.throughput_mode should be set"
|
| 909 |
+
)
|
| 910 |
+
|
| 911 |
+
if not args.no_python and not args.program.endswith(".py"):
|
| 912 |
+
raise RuntimeError(
|
| 913 |
+
'For non Python script, you should use "--no-python" parameter.'
|
| 914 |
+
)
|
| 915 |
+
|
| 916 |
+
# Verify LD_PRELOAD
|
| 917 |
+
if "LD_PRELOAD" in os.environ:
|
| 918 |
+
lst_valid = []
|
| 919 |
+
tmp_ldpreload = os.environ["LD_PRELOAD"]
|
| 920 |
+
for item in tmp_ldpreload.split(":"):
|
| 921 |
+
matches = glob.glob(item)
|
| 922 |
+
if len(matches) > 0:
|
| 923 |
+
lst_valid.append(item)
|
| 924 |
+
else:
|
| 925 |
+
logger.warning("%s doesn't exist. Removing it from LD_PRELOAD.", item)
|
| 926 |
+
if len(lst_valid) > 0:
|
| 927 |
+
os.environ["LD_PRELOAD"] = ":".join(lst_valid)
|
| 928 |
+
else:
|
| 929 |
+
os.environ["LD_PRELOAD"] = ""
|
| 930 |
+
|
| 931 |
+
launcher = _Launcher()
|
| 932 |
+
launcher.launch(args)
|
| 933 |
+
for x in sorted(set(os.environ.keys()) - env_before):
|
| 934 |
+
logger.debug("%s=%s", x, os.environ[x])
|
| 935 |
+
|
| 936 |
+
|
| 937 |
+
if __name__ == "__main__":
|
| 938 |
+
parser = ArgumentParser(
|
| 939 |
+
description="This is a script for launching PyTorch inference on Intel(R) Xeon(R) Scalable "
|
| 940 |
+
"Processors with optimal configurations. Single instance inference, "
|
| 941 |
+
"multi-instance inference are enable. To get the peak performance on Intel(R) "
|
| 942 |
+
"Xeon(R) Scalable Processors, the script optimizes the configuration "
|
| 943 |
+
"of thread and memory management. For thread management, the script configures thread "
|
| 944 |
+
"affinity and the preload of Intel OMP library. For memory management, it configures "
|
| 945 |
+
"NUMA binding and preload optimized memory allocation library (e.g. tcmalloc, jemalloc) "
|
| 946 |
+
"\n################################# Basic usage ############################# \n"
|
| 947 |
+
"\n 1. single instance\n"
|
| 948 |
+
"\n >>> python -m torch.backends.xeon.run_cpu python_script args \n"
|
| 949 |
+
"\n2. multi-instance \n"
|
| 950 |
+
"\n >>> python -m torch.backends.xeon.run_cpu --ninstances xxx "
|
| 951 |
+
"--ncores-per-instance xx python_script args\n"
|
| 952 |
+
"\n############################################################################# \n",
|
| 953 |
+
formatter_class=RawTextHelpFormatter,
|
| 954 |
+
)
|
| 955 |
+
create_args(parser)
|
| 956 |
+
args = parser.parse_args()
|
| 957 |
+
main(args)
|
western_ai/usa/pytorch/torch/backends/xnnpack/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import sys
|
| 3 |
+
import types
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class _XNNPACKEnabled:
|
| 9 |
+
def __get__(self, obj, objtype):
|
| 10 |
+
return torch._C._is_xnnpack_enabled()
|
| 11 |
+
|
| 12 |
+
def __set__(self, obj, val):
|
| 13 |
+
raise RuntimeError("Assignment not supported")
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class XNNPACKEngine(types.ModuleType):
|
| 17 |
+
def __init__(self, m, name):
|
| 18 |
+
super().__init__(name)
|
| 19 |
+
self.m = m
|
| 20 |
+
|
| 21 |
+
def __getattr__(self, attr):
|
| 22 |
+
return self.m.__getattribute__(attr)
|
| 23 |
+
|
| 24 |
+
enabled = _XNNPACKEnabled()
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
# This is the sys.modules replacement trick, see
|
| 28 |
+
# https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273
|
| 29 |
+
sys.modules[__name__] = XNNPACKEngine(sys.modules[__name__], __name__)
|
western_ai/usa/pytorch/torch/compiler/__init__.py
ADDED
|
@@ -0,0 +1,984 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import contextlib
|
| 3 |
+
import io
|
| 4 |
+
from collections.abc import Callable
|
| 5 |
+
from typing import Any, TYPE_CHECKING, TypeVar
|
| 6 |
+
from typing_extensions import ParamSpec
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from torch._higher_order_ops.invoke_subgraph import NestedCompileRegionOptions
|
| 10 |
+
|
| 11 |
+
from . import config
|
| 12 |
+
from ._cache import CacheInfo
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
if TYPE_CHECKING:
|
| 16 |
+
from torch._dynamo.eval_frame import StanceStr
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
__all__ = [
|
| 20 |
+
"compile",
|
| 21 |
+
"config",
|
| 22 |
+
"assume_constant_result",
|
| 23 |
+
"reset",
|
| 24 |
+
"nonstrict_trace",
|
| 25 |
+
"allow_in_graph",
|
| 26 |
+
"substitute_in_graph",
|
| 27 |
+
"list_backends",
|
| 28 |
+
"disable",
|
| 29 |
+
"set_default_backend",
|
| 30 |
+
"get_default_backend",
|
| 31 |
+
"set_stance",
|
| 32 |
+
"set_enable_guard_collectives",
|
| 33 |
+
"cudagraph_mark_step_begin",
|
| 34 |
+
"load_compiled_function",
|
| 35 |
+
"wrap_numpy",
|
| 36 |
+
"is_compiling",
|
| 37 |
+
"is_dynamo_compiling",
|
| 38 |
+
"is_exporting",
|
| 39 |
+
"save_cache_artifacts",
|
| 40 |
+
"load_cache_artifacts",
|
| 41 |
+
"keep_portable_guards_unsafe",
|
| 42 |
+
"skip_guard_on_inbuilt_nn_modules_unsafe",
|
| 43 |
+
"skip_guard_on_all_nn_modules_unsafe",
|
| 44 |
+
"keep_tensor_guards_unsafe",
|
| 45 |
+
"skip_guard_on_globals_unsafe",
|
| 46 |
+
"skip_all_guards_unsafe",
|
| 47 |
+
"nested_compile_region",
|
| 48 |
+
]
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
_P = ParamSpec("_P")
|
| 52 |
+
_R = TypeVar("_R")
|
| 53 |
+
FuncType = Callable[..., Any]
|
| 54 |
+
F = TypeVar("F", bound=FuncType)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def compile(*args, **kwargs):
|
| 58 |
+
"""
|
| 59 |
+
See :func:`torch.compile` for details on the arguments for this function.
|
| 60 |
+
"""
|
| 61 |
+
# pyrefly: ignore [not-iterable]
|
| 62 |
+
return torch.compile(*args, **kwargs)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def reset() -> None:
|
| 66 |
+
"""
|
| 67 |
+
Reset the in-process compiler state.
|
| 68 |
+
|
| 69 |
+
This function clears Dynamo's in-memory compilation caches and related
|
| 70 |
+
process-local state used by :func:`torch.compile`. It does not delete
|
| 71 |
+
filesystem caches, such as Inductor's disk cache.
|
| 72 |
+
"""
|
| 73 |
+
import torch._dynamo
|
| 74 |
+
|
| 75 |
+
torch._dynamo.reset()
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def allow_in_graph(fn):
|
| 79 |
+
"""
|
| 80 |
+
Tells the compiler frontend (Dynamo) to skip symbolic introspection of the function
|
| 81 |
+
and instead directly write it to the graph when encountered.
|
| 82 |
+
|
| 83 |
+
If you are using :func:`torch.compile` (with backend="inductor" (the default)), or
|
| 84 |
+
:func:`torch.export.export`, and trying to black-box a Python function throughout
|
| 85 |
+
all tracing, do not use this API.
|
| 86 |
+
Instead, please create a custom operator (see `PyTorch Custom Operators Landing Page
|
| 87 |
+
<https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html>`_)
|
| 88 |
+
|
| 89 |
+
.. warning::
|
| 90 |
+
|
| 91 |
+
If you're a typical torch.compile user (e.g. you're applying torch.compile to
|
| 92 |
+
a model to make it run faster), you probably don't want to use this function.
|
| 93 |
+
:func:`allow_in_graph` is a footgun because it skips the compiler frontend
|
| 94 |
+
(Dynamo) that is responsible for doing safety checks (graph breaks, handling
|
| 95 |
+
closures, etc). Incorrect usage will lead to difficult-to-debug silent
|
| 96 |
+
incorrectness issues.
|
| 97 |
+
|
| 98 |
+
Given a Python function with no allow_in_graph decorator, regular execution
|
| 99 |
+
of torch.compile traces through the function. :func:`allow_in_graph` changes
|
| 100 |
+
it so that the frontend does not trace inside the function, but the compiler
|
| 101 |
+
backend still traces through it. Compare this to custom operators, which
|
| 102 |
+
treats a function as a black box throughout the torch.compile stack. The following
|
| 103 |
+
table compares these mechanisms.
|
| 104 |
+
|
| 105 |
+
+------------------------+-----------------------+--------------------------------+
|
| 106 |
+
| Mechanism | Frontend (Dynamo) | Backend (AOTAutograd+Inductor) |
|
| 107 |
+
+========================+=======================+================================+
|
| 108 |
+
| no decorator | trace inside | trace inside |
|
| 109 |
+
+------------------------+-----------------------+--------------------------------+
|
| 110 |
+
| allow_in_graph | opaque callable | trace inside |
|
| 111 |
+
+------------------------+-----------------------+--------------------------------+
|
| 112 |
+
| custom op | opaque callable | opaque callable |
|
| 113 |
+
+------------------------+-----------------------+--------------------------------+
|
| 114 |
+
|
| 115 |
+
One common use case for :func:`allow_in_graph()` is as an escape hatch for the compiler
|
| 116 |
+
frontend: if you know the function works w.r.t. to the downstream components of the
|
| 117 |
+
compilation stack (AOTAutograd and Inductor) but there is a Dynamo bug that prevents it from
|
| 118 |
+
symbolically introspecting the function properly (or if your code is in C/C++ and
|
| 119 |
+
therefore cannot be introspected with Dynamo), then one can decorate said function
|
| 120 |
+
with :func:`allow_in_graph` to bypass Dynamo.
|
| 121 |
+
|
| 122 |
+
We require that ``fn`` adhere to the following restrictions. Failure to adhere
|
| 123 |
+
results in undefined behavior:
|
| 124 |
+
|
| 125 |
+
- The inputs to ``fn`` must be Proxy-able types in the FX graph. Valid types include:
|
| 126 |
+
Tensor/int/bool/float/None/List[Tensor?]/List[int?]/List[float?]
|
| 127 |
+
Tuple[Tensor?, ...]/Tuple[int?, ...]/Tuple[float?, ...]/torch.dtype/torch.device
|
| 128 |
+
- The outputs to ``fn`` must be Proxy-able types in the FX graph (see previous bullet)
|
| 129 |
+
- all Tensors used inside of ``fn`` must be passed directly as inputs to ``fn``
|
| 130 |
+
(as opposed to being captured variables).
|
| 131 |
+
|
| 132 |
+
See also :func:`nonstrict_trace()`, which has slightly fewer restrictions on the inputs.
|
| 133 |
+
|
| 134 |
+
Args:
|
| 135 |
+
fn: A callable representing the function to be included in the graph.
|
| 136 |
+
If ``fn`` is a list or tuple of callables it recursively applies
|
| 137 |
+
:func:`allow_in_graph()` to each function and returns a new list or
|
| 138 |
+
tuple containing the modified functions.
|
| 139 |
+
|
| 140 |
+
Example::
|
| 141 |
+
|
| 142 |
+
torch.compiler.allow_in_graph(my_custom_function)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
@torch.compile(...)
|
| 146 |
+
def fn(x):
|
| 147 |
+
x = torch.add(x, 1)
|
| 148 |
+
x = my_custom_function(x)
|
| 149 |
+
x = torch.add(x, 1)
|
| 150 |
+
return x
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
fn(...)
|
| 154 |
+
|
| 155 |
+
Will capture a single graph containing ``my_custom_function()``.
|
| 156 |
+
|
| 157 |
+
"""
|
| 158 |
+
import torch._dynamo
|
| 159 |
+
|
| 160 |
+
return torch._dynamo.allow_in_graph(fn)
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def nonstrict_trace(traceable_fn: Callable[_P, _R]) -> Callable[_P, _R]:
|
| 164 |
+
"""
|
| 165 |
+
Decorator to mark a function as nonstrict-traceable for dynamo.
|
| 166 |
+
|
| 167 |
+
A nonstrict-traced function appears as an opaque call in the dynamo graph.
|
| 168 |
+
Dynamo does not trace into the function body (hence the "nonstrict"), but
|
| 169 |
+
aot_autograd will trace into it.
|
| 170 |
+
|
| 171 |
+
This is similar to ``allow_in_graph`` but with enhanced support for:
|
| 172 |
+
- User-defined classes as inputs (must be registered with pytree)
|
| 173 |
+
- ``nn.Module`` as input arguments (parameters and buffers are tracked for autograd)
|
| 174 |
+
- Global/captured tensors treated as constants (assumed not updated during execution)
|
| 175 |
+
|
| 176 |
+
Note:
|
| 177 |
+
- With ``backend="eager"``, the original Python function runs directly.
|
| 178 |
+
With ``backend="aot_eager"``, the graph traced by aot_autograd runs.
|
| 179 |
+
With ``backend="inductor"``, the traced graph is compiled with inductor.
|
| 180 |
+
|
| 181 |
+
- Training is supported: you can call ``.backward()`` on outputs and gradients
|
| 182 |
+
will flow through the nonstrict-traced function.
|
| 183 |
+
|
| 184 |
+
Dangerous patterns (may cause silent incorrectness):
|
| 185 |
+
- Side effects between nonstric_trace'd fn and compiled region: The function should
|
| 186 |
+
not depend on variables mutated by other code inside the compiled function, and code
|
| 187 |
+
after the call should not depend on mutations made by it.
|
| 188 |
+
|
| 189 |
+
- Implicit inputs (closures/globals): Tensors captured from enclosing scopes
|
| 190 |
+
are treated as constants. Gradients will NOT flow back to them. Pass tensors
|
| 191 |
+
as explicit arguments if gradients are needed.
|
| 192 |
+
|
| 193 |
+
Restrictions:
|
| 194 |
+
- Both inputs and outputs must use pytree-compatible types. User-defined classes
|
| 195 |
+
must be registered via :func:`torch.utils._pytree.register_pytree_node`,
|
| 196 |
+
:func:`torch.utils._pytree.register_dataclass`, or
|
| 197 |
+
:func:`torch.utils._pytree.register_constant`. Tensors, Python primitives (int, float, bool, str),
|
| 198 |
+
symbolic types (SymInt, SymFloat, SymBool), and built-in containers (list,
|
| 199 |
+
tuple, dict) are already handled by default.
|
| 200 |
+
- Primitive values and container structure are specialized per call site:
|
| 201 |
+
each call site expects the same primitives and structure on every execution.
|
| 202 |
+
|
| 203 |
+
Example::
|
| 204 |
+
|
| 205 |
+
>>> import torch
|
| 206 |
+
>>> @torch.compiler.nonstrict_trace
|
| 207 |
+
... def traced_forward(model, x):
|
| 208 |
+
... # It's OK to have dynamo graph break within nonstrict_trace region
|
| 209 |
+
... torch._dynamo.graph_break()
|
| 210 |
+
... return model(x) + x
|
| 211 |
+
...
|
| 212 |
+
>>> class MyModule(torch.nn.Module):
|
| 213 |
+
... def __init__(self):
|
| 214 |
+
... super().__init__()
|
| 215 |
+
... self.inner = torch.nn.Linear(10, 10)
|
| 216 |
+
...
|
| 217 |
+
... def forward(self, x):
|
| 218 |
+
... return traced_forward(self.inner, x)
|
| 219 |
+
...
|
| 220 |
+
>>> # Compile and run
|
| 221 |
+
>>> model = MyModule()
|
| 222 |
+
>>> opt_model = torch.compile(model, backend="aot_eager", fullgraph=True)
|
| 223 |
+
>>> out = opt_model(torch.randn(10, 10))
|
| 224 |
+
>>> out.sum().backward() # Gradients flow through traced_forward
|
| 225 |
+
|
| 226 |
+
"""
|
| 227 |
+
import torch._dynamo
|
| 228 |
+
|
| 229 |
+
return torch._dynamo.nonstrict_trace(traceable_fn)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def substitute_in_graph(
|
| 233 |
+
original_fn: Callable[_P, _R],
|
| 234 |
+
*,
|
| 235 |
+
can_constant_fold_through: bool = False,
|
| 236 |
+
skip_signature_check: bool = False,
|
| 237 |
+
) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]:
|
| 238 |
+
"""
|
| 239 |
+
Register a polyfill handler for a function, usually a C function from the C extension, to be
|
| 240 |
+
used in place of the original function when inlining the original function in the graph.
|
| 241 |
+
|
| 242 |
+
.. note::
|
| 243 |
+
|
| 244 |
+
The polyfill handler is only used when inlining the original function. It is not used when
|
| 245 |
+
the original function is called directly. In the eager mode, the decorated function calls
|
| 246 |
+
the performant C function rather than the polyfill handler.
|
| 247 |
+
|
| 248 |
+
The polyfill handler is a function that will be called in place of the original function when
|
| 249 |
+
inlining the original function. The polyfill handler should have the same signature and the same
|
| 250 |
+
behavior as the original function.
|
| 251 |
+
|
| 252 |
+
Args:
|
| 253 |
+
original_fn (callable): The original function, usually a C function, to register a polyfill
|
| 254 |
+
handler for.
|
| 255 |
+
can_constant_fold_through (bool, optional): Whether the polyfill handler can be constant
|
| 256 |
+
folded through. That is, if the polyfill handler is a pure function and its arguments
|
| 257 |
+
are constant, the result of the polyfill handler can be constant folded during the
|
| 258 |
+
compilation. Defaults to ``False``.
|
| 259 |
+
skip_signature_check (bool, optional): Whether to skip the signature check between the
|
| 260 |
+
original function and the polyfill handler. Defaults to ``False``.
|
| 261 |
+
|
| 262 |
+
Returns:
|
| 263 |
+
A decorator that registers the polyfill handler for the original function.
|
| 264 |
+
|
| 265 |
+
Example::
|
| 266 |
+
|
| 267 |
+
>>> import operator
|
| 268 |
+
>>> operator.indexOf([1, 2, 3, 4, 5], 3)
|
| 269 |
+
2
|
| 270 |
+
>>> torch.compile(operator.indexOf, fullgraph=True)([1, 2, 3, 4, 5], 3)
|
| 271 |
+
... # xdoctest: +SKIP("Long tracebacks")
|
| 272 |
+
Traceback (most recent call last):
|
| 273 |
+
...
|
| 274 |
+
torch._dynamo.exc.Unsupported: ...
|
| 275 |
+
|
| 276 |
+
>>> @torch.compiler.substitute_in_graph(operator.indexOf)
|
| 277 |
+
... def indexOf(a, b, /):
|
| 278 |
+
... for i, item in enumerate(a):
|
| 279 |
+
... if item is b or item == b:
|
| 280 |
+
... return i
|
| 281 |
+
... raise ValueError("sequence.index(x): x not in sequence")
|
| 282 |
+
>>>
|
| 283 |
+
>>> torch.compile(operator.indexOf, fullgraph=True)([1, 2, 3, 4, 5], 3)
|
| 284 |
+
2
|
| 285 |
+
"""
|
| 286 |
+
import torch._dynamo
|
| 287 |
+
|
| 288 |
+
return torch._dynamo.substitute_in_graph(
|
| 289 |
+
original_fn,
|
| 290 |
+
can_constant_fold_through=can_constant_fold_through,
|
| 291 |
+
skip_signature_check=skip_signature_check,
|
| 292 |
+
)
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def list_backends(exclude_tags=("debug", "experimental")) -> list[str]:
|
| 296 |
+
"""
|
| 297 |
+
Return valid strings that can be passed to `torch.compile(..., backend="name")`.
|
| 298 |
+
|
| 299 |
+
Args:
|
| 300 |
+
exclude_tags(optional): A tuple of strings representing tags to exclude.
|
| 301 |
+
"""
|
| 302 |
+
import torch._dynamo
|
| 303 |
+
|
| 304 |
+
return torch._dynamo.list_backends(exclude_tags)
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def assume_constant_result(fn):
|
| 308 |
+
"""
|
| 309 |
+
This function is used to mark a function `fn` as having a constant result.
|
| 310 |
+
This allows the compiler to optimize away your function.
|
| 311 |
+
Returns The same function `fn`
|
| 312 |
+
|
| 313 |
+
Args:
|
| 314 |
+
fn: The function to be marked as having a constant result.
|
| 315 |
+
|
| 316 |
+
.. warning::
|
| 317 |
+
`assume_constant_result` can, if invalid, cause safety and soundness issues, :func:`torch.compile`
|
| 318 |
+
will not attempt to validate whether the constant assumption is true or not
|
| 319 |
+
|
| 320 |
+
"""
|
| 321 |
+
import torch._dynamo
|
| 322 |
+
|
| 323 |
+
return torch._dynamo.assume_constant_result(fn)
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def disable(fn=None, recursive=True, *, reason=None):
|
| 327 |
+
"""
|
| 328 |
+
This function provides a decorator to disable compilation on a function.
|
| 329 |
+
It also provides the option of recursively disabling called functions.
|
| 330 |
+
|
| 331 |
+
Args:
|
| 332 |
+
fn (optional): The function to disable
|
| 333 |
+
recursive (optional): A boolean value indicating whether the disabling should be recursive.
|
| 334 |
+
reason (optional): A string value indicating the reason for disabling the function.
|
| 335 |
+
"""
|
| 336 |
+
import torch._dynamo
|
| 337 |
+
|
| 338 |
+
return torch._dynamo.disable(fn, recursive, reason=reason)
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
def set_default_backend(backend: str | Callable[..., Any] | None) -> None:
|
| 342 |
+
"""Set the default backend for ``torch.compile`` when no ``backend`` argument is specified.
|
| 343 |
+
|
| 344 |
+
Passing ``None`` resets the default back to ``"inductor"``.
|
| 345 |
+
|
| 346 |
+
Args:
|
| 347 |
+
backend: A backend name (string), a callable backend, or ``None``.
|
| 348 |
+
|
| 349 |
+
Example::
|
| 350 |
+
|
| 351 |
+
>>> torch.compiler.set_default_backend("eager")
|
| 352 |
+
>>> torch.compiler.get_default_backend()
|
| 353 |
+
'eager'
|
| 354 |
+
>>> torch.compiler.set_default_backend(None) # reset
|
| 355 |
+
>>> torch.compiler.get_default_backend()
|
| 356 |
+
'inductor'
|
| 357 |
+
"""
|
| 358 |
+
from torch._dynamo.backends.registry import set_default_backend
|
| 359 |
+
|
| 360 |
+
set_default_backend(backend)
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
def get_default_backend() -> str | Callable[..., Any]:
|
| 364 |
+
"""Return the current default backend for ``torch.compile``.
|
| 365 |
+
|
| 366 |
+
Returns:
|
| 367 |
+
The current default backend (string or callable). Initially ``"inductor"``.
|
| 368 |
+
"""
|
| 369 |
+
from torch._dynamo.backends.registry import get_default_backend
|
| 370 |
+
|
| 371 |
+
return get_default_backend()
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def set_stance(
|
| 375 |
+
stance: "StanceStr" = "default",
|
| 376 |
+
*,
|
| 377 |
+
skip_guard_eval_unsafe: bool = False,
|
| 378 |
+
force_backend: str | Callable[..., Any] | None = None,
|
| 379 |
+
):
|
| 380 |
+
"""
|
| 381 |
+
Set the current stance of the compiler.
|
| 382 |
+
Can be used as a function, context manager, or decorator.
|
| 383 |
+
Do not use this function inside a `torch.compile` region - an error will be raised otherwise.
|
| 384 |
+
|
| 385 |
+
.. code-block:: python
|
| 386 |
+
|
| 387 |
+
@torch.compile
|
| 388 |
+
def foo(x): ...
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
@torch.compiler.set_stance("force_eager")
|
| 392 |
+
def bar():
|
| 393 |
+
# will not be compiled
|
| 394 |
+
foo(...)
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
bar()
|
| 398 |
+
|
| 399 |
+
with torch.compiler.set_stance("force_eager"):
|
| 400 |
+
# will also not be compiled
|
| 401 |
+
foo(...)
|
| 402 |
+
|
| 403 |
+
torch.compiler.set_stance("force_eager")
|
| 404 |
+
# will also not be compiled
|
| 405 |
+
foo(...)
|
| 406 |
+
torch.compiler.set_stance("default")
|
| 407 |
+
|
| 408 |
+
# will be compiled
|
| 409 |
+
foo(...)
|
| 410 |
+
|
| 411 |
+
Args:
|
| 412 |
+
stance: The stance to set the compiler to. Valid values are:
|
| 413 |
+
|
| 414 |
+
- "default": The default stance, used for normal compilation.
|
| 415 |
+
- "force_eager": Ignore all `torch.compile` directives.
|
| 416 |
+
- "eager_on_recompile": Run code eagerly when a recompile is necessary.
|
| 417 |
+
If there is cached compiled code valid for the input, it will still be used.
|
| 418 |
+
- "fail_on_recompile": Raise an error when recompiling a function.
|
| 419 |
+
- "eager_then_compile": Run the first invocation in eager mode, then compile on
|
| 420 |
+
subsequent calls. This is beneficial for dynamic shapes as it allows inferring
|
| 421 |
+
dynamism from the first two invocations instead of wasting a static compile on
|
| 422 |
+
the first invocation.
|
| 423 |
+
- "aot_eager_then_compile": Run the first invocation with AOT eager to get memory
|
| 424 |
+
benefits from activation checkpointing, then compile on subsequent calls. Like
|
| 425 |
+
eager_then_compile, this improves handling of dynamic shapes by avoiding an
|
| 426 |
+
initial static compile.
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
skip_guard_eval_unsafe: A flag to run only differentiating guards.
|
| 430 |
+
CAUTION - This flag is unsafe and should only be used if your setup
|
| 431 |
+
meets the following conditions.
|
| 432 |
+
|
| 433 |
+
torch.compile uses a guard system to support recompilations and
|
| 434 |
+
choose which compiled artifact to run at runtime. These guards,
|
| 435 |
+
though efficient, add some overhead, which may impact performance in
|
| 436 |
+
scenarios where you need to optimize for minimal guard processing
|
| 437 |
+
time. This API enables you to disable guard evaluation, assuming
|
| 438 |
+
that you have warmed up the compiled model with a sufficient variety
|
| 439 |
+
of inputs. This assumption means that, after the warmup phase, no
|
| 440 |
+
further recompilations will be necessary. If this assumption fails,
|
| 441 |
+
there is a risk of silently producing incorrect results (hence the
|
| 442 |
+
term "unsafe" in the API name).
|
| 443 |
+
|
| 444 |
+
force_backend: If `stance` is "default", this argument can be used to force `torch.compile`
|
| 445 |
+
to use a specific backend. Otherwise, an error is raised.
|
| 446 |
+
"""
|
| 447 |
+
import torch._dynamo
|
| 448 |
+
|
| 449 |
+
return torch._dynamo.set_stance(
|
| 450 |
+
stance,
|
| 451 |
+
skip_guard_eval_unsafe=skip_guard_eval_unsafe,
|
| 452 |
+
force_backend=force_backend,
|
| 453 |
+
)
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
# forbid in graph
|
| 457 |
+
set_stance._dynamo_forbidden = True # type: ignore[attr-defined]
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
def set_enable_guard_collectives(enabled: bool):
|
| 461 |
+
"""
|
| 462 |
+
Enables use of collectives *during* guard evaluation to synchronize behavior
|
| 463 |
+
across ranks. This is expensive: we have to issue a collective every time
|
| 464 |
+
we enter a compiled code region, even if no rank actually would need to
|
| 465 |
+
compile. This can help prevent NCCL hangs by ensuring that we never have a
|
| 466 |
+
situation where one rank starts recompiling while other ranks don't compile;
|
| 467 |
+
it is especially useful in conjunction with enable_compiler_collectives
|
| 468 |
+
where such a situation would immediately cause a hang (as it is necessary
|
| 469 |
+
for all ranks to compile at the same time to run compiler collectives). Like
|
| 470 |
+
compiler collectives, you can only run this on SPMD programs; you will hang
|
| 471 |
+
otherwise. Note that a guard collective is only issued if there is any
|
| 472 |
+
compiled code to guard on; if this is the first time we encounter a frame or
|
| 473 |
+
the frame is skipped, we don't issue collectives.
|
| 474 |
+
|
| 475 |
+
Returns the previous setting of enabled.
|
| 476 |
+
"""
|
| 477 |
+
from torch._C._dynamo.eval_frame import set_guard_complete_hook
|
| 478 |
+
from torch._dynamo.eval_frame import guard_collectives_hook
|
| 479 |
+
|
| 480 |
+
if enabled:
|
| 481 |
+
return set_guard_complete_hook(guard_collectives_hook) is not None # type: ignore[arg-type]
|
| 482 |
+
else:
|
| 483 |
+
return set_guard_complete_hook(None) is not None
|
| 484 |
+
|
| 485 |
+
|
| 486 |
+
set_enable_guard_collectives._dynamo_forbidden = True # type: ignore[attr-defined]
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
def cudagraph_mark_step_begin():
|
| 490 |
+
"""
|
| 491 |
+
Indicates that a new iteration of inference or training is about to begin.
|
| 492 |
+
|
| 493 |
+
CUDA Graphs will free tensors of a prior iteration. A new iteration is started on each invocation of
|
| 494 |
+
torch.compile, so long as there is not a pending backward that has not been called.
|
| 495 |
+
|
| 496 |
+
If that heuristic is wrong, such as in the following example, manually mark it with this api.
|
| 497 |
+
|
| 498 |
+
.. code-block:: python
|
| 499 |
+
|
| 500 |
+
@torch.compile(mode="reduce-overhead")
|
| 501 |
+
def rand_foo():
|
| 502 |
+
return torch.rand([4], device="cuda")
|
| 503 |
+
|
| 504 |
+
|
| 505 |
+
for _ in range(5):
|
| 506 |
+
torch.compiler.cudagraph_mark_step_begin()
|
| 507 |
+
rand_foo() + rand_foo()
|
| 508 |
+
|
| 509 |
+
For more details, see `torch.compiler_cudagraph_trees <https://docs.pytorch.org/docs/main/user_guide/torch_compiler/torch.compiler_cudagraph_trees.html>`__ # noqa: B950
|
| 510 |
+
"""
|
| 511 |
+
from torch._inductor import cudagraph_trees
|
| 512 |
+
|
| 513 |
+
cudagraph_trees.mark_step_begin()
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
def wrap_numpy(fn):
|
| 517 |
+
r"""Decorator that turns a function from ``np.ndarray``\ s to ``np.ndarray``\ s into a function
|
| 518 |
+
from ``torch.Tensor``\ s to ``torch.Tensor``\ s.
|
| 519 |
+
|
| 520 |
+
It is designed to be used with :func:`torch.compile` with ``fullgraph=True``. It allows you to
|
| 521 |
+
compile a NumPy function as if it were a PyTorch function. This allows you to run NumPy code
|
| 522 |
+
on CUDA or compute its gradients.
|
| 523 |
+
|
| 524 |
+
.. note::
|
| 525 |
+
|
| 526 |
+
This decorator does not work without :func:`torch.compile`.
|
| 527 |
+
|
| 528 |
+
Example::
|
| 529 |
+
|
| 530 |
+
>>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA)
|
| 531 |
+
>>> # Compile a NumPy function as a Tensor -> Tensor function
|
| 532 |
+
>>> @torch.compile(fullgraph=True)
|
| 533 |
+
>>> @torch.compiler.wrap_numpy
|
| 534 |
+
>>> def fn(a: np.ndarray):
|
| 535 |
+
>>> return np.sum(a * a)
|
| 536 |
+
>>> # Execute the NumPy function using Tensors on CUDA and compute the gradients
|
| 537 |
+
>>> x = torch.arange(6, dtype=torch.float32, device="cuda", requires_grad=True)
|
| 538 |
+
>>> out = fn(x)
|
| 539 |
+
>>> out.backward()
|
| 540 |
+
>>> print(x.grad)
|
| 541 |
+
tensor([ 0., 2., 4., 6., 8., 10.], device='cuda:0')
|
| 542 |
+
"""
|
| 543 |
+
from torch._dynamo.external_utils import wrap_numpy as wrap
|
| 544 |
+
|
| 545 |
+
return wrap(fn)
|
| 546 |
+
|
| 547 |
+
|
| 548 |
+
_is_compiling_flag: bool = False
|
| 549 |
+
_is_exporting_flag: bool = False
|
| 550 |
+
_is_non_strict_tracing_flag: bool = False
|
| 551 |
+
|
| 552 |
+
|
| 553 |
+
def is_compiling() -> bool:
|
| 554 |
+
"""
|
| 555 |
+
Indicates whether a graph is executed/traced as part of torch.compile() or torch.export().
|
| 556 |
+
|
| 557 |
+
Note that there are 2 other related flags that should deprecated eventually:
|
| 558 |
+
* torch._dynamo.external_utils.is_compiling()
|
| 559 |
+
* torch._utils.is_compiling()
|
| 560 |
+
|
| 561 |
+
Example::
|
| 562 |
+
|
| 563 |
+
>>> def forward(self, x):
|
| 564 |
+
>>> if not torch.compiler.is_compiling():
|
| 565 |
+
>>> pass # ...logic that is not needed in a compiled/traced graph...
|
| 566 |
+
>>>
|
| 567 |
+
>>> # ...rest of the function...
|
| 568 |
+
"""
|
| 569 |
+
if torch.jit.is_scripting():
|
| 570 |
+
return False
|
| 571 |
+
else:
|
| 572 |
+
return _is_compiling_flag
|
| 573 |
+
|
| 574 |
+
|
| 575 |
+
def _is_non_strict_tracing() -> bool:
|
| 576 |
+
"""
|
| 577 |
+
Indicates whether we are inside a non-strict make_fx-based tracing session.
|
| 578 |
+
"""
|
| 579 |
+
return _is_non_strict_tracing_flag
|
| 580 |
+
|
| 581 |
+
|
| 582 |
+
@contextlib.contextmanager
|
| 583 |
+
def _non_strict_tracing_context():
|
| 584 |
+
"""Context manager that sets the non-strict tracing flag."""
|
| 585 |
+
global _is_non_strict_tracing_flag
|
| 586 |
+
old = _is_non_strict_tracing_flag
|
| 587 |
+
try:
|
| 588 |
+
_is_non_strict_tracing_flag = True
|
| 589 |
+
yield
|
| 590 |
+
finally:
|
| 591 |
+
_is_non_strict_tracing_flag = old
|
| 592 |
+
|
| 593 |
+
|
| 594 |
+
@contextlib.contextmanager
|
| 595 |
+
def _compile_session_context():
|
| 596 |
+
"""Context manager that sets _is_compiling_flag for the duration of a
|
| 597 |
+
torch.compile session.
|
| 598 |
+
"""
|
| 599 |
+
global _is_compiling_flag
|
| 600 |
+
old = _is_compiling_flag
|
| 601 |
+
try:
|
| 602 |
+
_is_compiling_flag = True
|
| 603 |
+
yield
|
| 604 |
+
finally:
|
| 605 |
+
_is_compiling_flag = old
|
| 606 |
+
|
| 607 |
+
|
| 608 |
+
@contextlib.contextmanager
|
| 609 |
+
def _patch_autograd_grad():
|
| 610 |
+
"""Patch autograd.grad for non-strict make_fx tracing.
|
| 611 |
+
|
| 612 |
+
This patch installs autograd hooks so traced backward nodes preserve
|
| 613 |
+
stack trace, seq_nr, and autograd_backward metadata before delegating to
|
| 614 |
+
the real torch.autograd.grad.
|
| 615 |
+
"""
|
| 616 |
+
import functools
|
| 617 |
+
|
| 618 |
+
import torch.autograd
|
| 619 |
+
from torch._dynamo.utils import warn_once
|
| 620 |
+
from torch._functorch._aot_autograd.logging_utils import (
|
| 621 |
+
setup_stacktrace_preservation_hooks_from_tensors,
|
| 622 |
+
)
|
| 623 |
+
|
| 624 |
+
warn_once(
|
| 625 |
+
"torch.compiler._patch_autograd_grad() is deprecated; "
|
| 626 |
+
"use torch.compiler._patch_engine_backward() instead."
|
| 627 |
+
)
|
| 628 |
+
# TODO: Remove this helper once Titan no longer depends on it.
|
| 629 |
+
|
| 630 |
+
_orig_grad = torch.autograd.grad
|
| 631 |
+
|
| 632 |
+
@functools.wraps(_orig_grad)
|
| 633 |
+
def _patched_grad(outputs, inputs, *args, **kwargs):
|
| 634 |
+
if not _is_non_strict_tracing():
|
| 635 |
+
raise AssertionError(
|
| 636 |
+
"_patch_autograd_grad() must be used under "
|
| 637 |
+
"_non_strict_tracing_context()"
|
| 638 |
+
)
|
| 639 |
+
|
| 640 |
+
setup_stacktrace_preservation_hooks_from_tensors(outputs)
|
| 641 |
+
return _orig_grad(outputs, inputs, *args, **kwargs)
|
| 642 |
+
|
| 643 |
+
torch.autograd.grad = _patched_grad
|
| 644 |
+
try:
|
| 645 |
+
yield
|
| 646 |
+
finally:
|
| 647 |
+
torch.autograd.grad = _orig_grad
|
| 648 |
+
|
| 649 |
+
|
| 650 |
+
@contextlib.contextmanager
|
| 651 |
+
def _patch_engine_backward():
|
| 652 |
+
"""Patch _engine_run_backward for non-strict make_fx tracing.
|
| 653 |
+
|
| 654 |
+
This patch installs autograd hooks so traced backward nodes preserve
|
| 655 |
+
stack trace, seq_nr, and autograd_backward metadata before delegating to
|
| 656 |
+
the real autograd engine entrypoint used by backward().
|
| 657 |
+
"""
|
| 658 |
+
import functools
|
| 659 |
+
|
| 660 |
+
import torch.autograd
|
| 661 |
+
import torch.autograd.graph
|
| 662 |
+
from torch._functorch._aot_autograd.logging_utils import (
|
| 663 |
+
setup_stacktrace_preservation_hooks_from_tensors,
|
| 664 |
+
)
|
| 665 |
+
|
| 666 |
+
_orig_engine_run_backward = torch.autograd.graph._engine_run_backward
|
| 667 |
+
|
| 668 |
+
@functools.wraps(_orig_engine_run_backward)
|
| 669 |
+
def _patched_engine_backward(outputs, *args, **kwargs):
|
| 670 |
+
if not _is_non_strict_tracing():
|
| 671 |
+
raise AssertionError(
|
| 672 |
+
"_patch_engine_backward() must be used under "
|
| 673 |
+
"_non_strict_tracing_context()"
|
| 674 |
+
)
|
| 675 |
+
|
| 676 |
+
setup_stacktrace_preservation_hooks_from_tensors(outputs)
|
| 677 |
+
return _orig_engine_run_backward(outputs, *args, **kwargs)
|
| 678 |
+
|
| 679 |
+
torch.autograd.graph._engine_run_backward = _patched_engine_backward
|
| 680 |
+
torch.autograd._engine_run_backward = _patched_engine_backward
|
| 681 |
+
try:
|
| 682 |
+
yield
|
| 683 |
+
finally:
|
| 684 |
+
torch.autograd.graph._engine_run_backward = _orig_engine_run_backward
|
| 685 |
+
torch.autograd._engine_run_backward = _orig_engine_run_backward
|
| 686 |
+
|
| 687 |
+
|
| 688 |
+
def is_dynamo_compiling() -> bool:
|
| 689 |
+
"""
|
| 690 |
+
Indicates whether a graph is traced via TorchDynamo.
|
| 691 |
+
|
| 692 |
+
It's stricter than is_compiling() flag, as it would only be set to True when
|
| 693 |
+
TorchDynamo is used.
|
| 694 |
+
|
| 695 |
+
Example::
|
| 696 |
+
|
| 697 |
+
>>> def forward(self, x):
|
| 698 |
+
>>> if not torch.compiler.is_dynamo_compiling():
|
| 699 |
+
>>> pass # ...logic that is not needed in a TorchDynamo-traced graph...
|
| 700 |
+
>>>
|
| 701 |
+
>>> # ...rest of the function...
|
| 702 |
+
"""
|
| 703 |
+
return False
|
| 704 |
+
|
| 705 |
+
|
| 706 |
+
def is_exporting() -> bool:
|
| 707 |
+
"""
|
| 708 |
+
Indicates whether we're under exporting.
|
| 709 |
+
|
| 710 |
+
It's stricter than is_compiling() flag, as it would only be set to True when
|
| 711 |
+
torch.export is used.
|
| 712 |
+
|
| 713 |
+
Example::
|
| 714 |
+
|
| 715 |
+
>>> def forward(self, x):
|
| 716 |
+
>>> if not torch.compiler.is_exporting():
|
| 717 |
+
>>> pass # ...logic that is not needed in export...
|
| 718 |
+
>>>
|
| 719 |
+
>>> # ...rest of the function...
|
| 720 |
+
"""
|
| 721 |
+
return _is_exporting_flag
|
| 722 |
+
|
| 723 |
+
|
| 724 |
+
def save_cache_artifacts() -> tuple[bytes, CacheInfo] | None:
|
| 725 |
+
"""
|
| 726 |
+
Serializes all the cache artifacts that were created during the compilation
|
| 727 |
+
|
| 728 |
+
Example:
|
| 729 |
+
|
| 730 |
+
- Execute torch.compile
|
| 731 |
+
- Call torch.compiler.save_cache_artifacts()
|
| 732 |
+
"""
|
| 733 |
+
from ._cache import CacheArtifactManager
|
| 734 |
+
|
| 735 |
+
if torch._dynamo.config.caching_precompile:
|
| 736 |
+
from torch._dynamo.precompile_context import PrecompileContext
|
| 737 |
+
|
| 738 |
+
PrecompileContext.save_to_dynamo_cache()
|
| 739 |
+
|
| 740 |
+
return CacheArtifactManager.serialize()
|
| 741 |
+
|
| 742 |
+
|
| 743 |
+
def load_cache_artifacts(serialized_artifacts: bytes) -> CacheInfo | None:
|
| 744 |
+
"""
|
| 745 |
+
Hot loads cache artifacts that were previously serialized via
|
| 746 |
+
save_cache_artifacts
|
| 747 |
+
|
| 748 |
+
Example:
|
| 749 |
+
|
| 750 |
+
# From a previous invocation
|
| 751 |
+
artifacts = torch.compiler.save_cache_artifacts()
|
| 752 |
+
|
| 753 |
+
torch.compiler.load_cache_artifacts(artifacts[0])
|
| 754 |
+
"""
|
| 755 |
+
from ._cache import CacheArtifactManager, CacheInfo
|
| 756 |
+
|
| 757 |
+
artifacts = CacheArtifactManager.deserialize(serialized_artifacts)
|
| 758 |
+
if artifacts is not None:
|
| 759 |
+
return CacheArtifactManager.populate_caches(artifacts)
|
| 760 |
+
return None
|
| 761 |
+
|
| 762 |
+
|
| 763 |
+
def keep_portable_guards_unsafe(guard_entries):
|
| 764 |
+
"""
|
| 765 |
+
A common function to only keep guards that can be used in both Python and non-Python environments.
|
| 766 |
+
This includes:
|
| 767 |
+
- Tensor metadata and dynamic shape information.
|
| 768 |
+
- Global contexts state (e.g. autocast, no_grad, etc.)
|
| 769 |
+
|
| 770 |
+
This is unsafe to use by default.
|
| 771 |
+
To use this API, use guard_filter_fn argument while calling torch.compile
|
| 772 |
+
|
| 773 |
+
>> opt_mod = torch.compile(
|
| 774 |
+
>> mod,
|
| 775 |
+
>> options={"guard_filter_fn": torch.compiler.keep_global_context_and_tensor_guards_unsafe},
|
| 776 |
+
>> )
|
| 777 |
+
"""
|
| 778 |
+
return [
|
| 779 |
+
(
|
| 780 |
+
g.guard_type in ("GLOBAL_STATE", "SHAPE_ENV")
|
| 781 |
+
or (g.guard_type == "TENSOR_MATCH" and not g.is_global)
|
| 782 |
+
)
|
| 783 |
+
for g in guard_entries
|
| 784 |
+
]
|
| 785 |
+
|
| 786 |
+
|
| 787 |
+
def skip_guard_on_inbuilt_nn_modules_unsafe(guard_entries):
|
| 788 |
+
"""
|
| 789 |
+
A common function to skip guards on the inbuilt nn modules like
|
| 790 |
+
torch.nn.Linear. This is unsafe to use by default. But for majority of
|
| 791 |
+
torch.compile users, the model code does not modify the inbuilt nn module
|
| 792 |
+
attributes. They can benefit from reduction in guard latency overhead using
|
| 793 |
+
this API.
|
| 794 |
+
|
| 795 |
+
To use this API, use guard_filter_fn argument while calling torch.compile
|
| 796 |
+
|
| 797 |
+
>> opt_mod = torch.compile(
|
| 798 |
+
>> mod,
|
| 799 |
+
>> options={"guard_filter_fn": torch.compiler.skip_guard_on_all_nn_modules_unsafe},
|
| 800 |
+
>> )
|
| 801 |
+
"""
|
| 802 |
+
return [
|
| 803 |
+
not entry.orig_guard.source.is_unspecialized_builtin_nn_module()
|
| 804 |
+
for entry in guard_entries
|
| 805 |
+
]
|
| 806 |
+
|
| 807 |
+
|
| 808 |
+
def skip_guard_on_all_nn_modules_unsafe(guard_entries):
|
| 809 |
+
"""
|
| 810 |
+
A common function to skip guards on all nn modules, both user defined as
|
| 811 |
+
well inbuilt nn modules (like torch.nn.Linear). This is unsafe to use by
|
| 812 |
+
default. But for majority of torch.compile users, the model code does not
|
| 813 |
+
modify the nn module attributes. They can benefit from reduction in guard
|
| 814 |
+
latency overhead using this API.
|
| 815 |
+
|
| 816 |
+
To use this API, use guard_filter_fn argument while calling torch.compile
|
| 817 |
+
|
| 818 |
+
>> opt_mod = torch.compile(
|
| 819 |
+
>> mod,
|
| 820 |
+
>> options={"guard_filter_fn": torch.compiler.skip_guard_on_all_nn_modules_unsafe},
|
| 821 |
+
>> )
|
| 822 |
+
"""
|
| 823 |
+
|
| 824 |
+
return [
|
| 825 |
+
not entry.orig_guard.source.is_unspecialized_nn_module()
|
| 826 |
+
for entry in guard_entries
|
| 827 |
+
]
|
| 828 |
+
|
| 829 |
+
|
| 830 |
+
def keep_tensor_guards_unsafe(guard_entries, keep_parameters=False):
|
| 831 |
+
"""
|
| 832 |
+
A common function to keep tensor guards on all tensors. This is unsafe to
|
| 833 |
+
use by default. But if you don't expect any changes in the model code, you
|
| 834 |
+
can just keep the tensor guards.
|
| 835 |
+
|
| 836 |
+
|
| 837 |
+
>> opt_mod = torch.compile(
|
| 838 |
+
>> mod,
|
| 839 |
+
>> options={"guard_filter_fn": torch.compiler.keep_tensor_guards},
|
| 840 |
+
>> )
|
| 841 |
+
"""
|
| 842 |
+
|
| 843 |
+
keep_flags = []
|
| 844 |
+
for entry in guard_entries:
|
| 845 |
+
if entry.guard_type == "TENSOR_MATCH":
|
| 846 |
+
if not isinstance(entry.value, torch.nn.Parameter):
|
| 847 |
+
keep_flags.append(True)
|
| 848 |
+
elif keep_parameters:
|
| 849 |
+
keep_flags.append(True)
|
| 850 |
+
else:
|
| 851 |
+
keep_flags.append(False)
|
| 852 |
+
else:
|
| 853 |
+
keep_flags.append(False)
|
| 854 |
+
return keep_flags
|
| 855 |
+
|
| 856 |
+
|
| 857 |
+
def skip_guard_on_globals_unsafe(guard_entries):
|
| 858 |
+
"""
|
| 859 |
+
A common function to skip guards on all globals. This is unsafe to use by
|
| 860 |
+
default. But if you don't expect any changes in the globals, you can just
|
| 861 |
+
keep the tensor guards.
|
| 862 |
+
|
| 863 |
+
>> opt_mod = torch.compile(
|
| 864 |
+
>> mod,
|
| 865 |
+
>> options={"guard_filter_fn": torch.compiler.skip_guard_on_globals},
|
| 866 |
+
>> )
|
| 867 |
+
"""
|
| 868 |
+
|
| 869 |
+
return [not entry.is_global for entry in guard_entries]
|
| 870 |
+
|
| 871 |
+
|
| 872 |
+
def skip_all_guards_unsafe(guard_entries):
|
| 873 |
+
"""
|
| 874 |
+
A function for skipping all guards on a compiled function.
|
| 875 |
+
|
| 876 |
+
WARNING: This function will drop all the safety guarantees from Dynamo
|
| 877 |
+
compiled function. Use this with caution.
|
| 878 |
+
|
| 879 |
+
To use this API, use guard_filter_fn argument while calling torch.compile
|
| 880 |
+
|
| 881 |
+
>> opt_mod = torch.compile(
|
| 882 |
+
>> mod,
|
| 883 |
+
>> options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe},
|
| 884 |
+
>> )
|
| 885 |
+
"""
|
| 886 |
+
return [False for entry in guard_entries]
|
| 887 |
+
|
| 888 |
+
|
| 889 |
+
def nested_compile_region(
|
| 890 |
+
fn=None,
|
| 891 |
+
*,
|
| 892 |
+
options: NestedCompileRegionOptions | None = None,
|
| 893 |
+
max_reuse_entries: int = 8,
|
| 894 |
+
reuse_hash_fn=None,
|
| 895 |
+
):
|
| 896 |
+
"""
|
| 897 |
+
Tells **``torch.compile``** that the marked set of operations forms a nested
|
| 898 |
+
compile region (which is often repeated in the full model) whose code can be
|
| 899 |
+
compiled once and safely reused. ``nested_compile_region`` can also be used
|
| 900 |
+
as a decorator.
|
| 901 |
+
|
| 902 |
+
During **``torch.compile``** tracing, the compiler applies *hierarchical
|
| 903 |
+
compilation* with ``nested_compile_region``: it emits optimized code for the
|
| 904 |
+
marked region the first time it is encountered and re-emits (or "stamps
|
| 905 |
+
out") the previously compiled code on every subsequent invocation. This can
|
| 906 |
+
substantially reduce overall compile time for deeply-stacked,
|
| 907 |
+
structurally-identical components such as the transformer layers of a
|
| 908 |
+
large-language-model (LLM).
|
| 909 |
+
|
| 910 |
+
Outside a ``torch.compile`` context—i.e., in standard eager execution—the
|
| 911 |
+
call is a no-op, so existing workflows remain unaffected.
|
| 912 |
+
|
| 913 |
+
Note that ``nested_compile_region`` **does not** promise that a region will
|
| 914 |
+
be compiled exactly once. If the compiler detects that new input conditions
|
| 915 |
+
(shape, dtype, device, stride, globals etc.) make the cached version invalid
|
| 916 |
+
to reuse, it will transparently re-compile the region. Using it is
|
| 917 |
+
therefore *safe*: correctness is always preserved, and you pay the extra
|
| 918 |
+
compilation cost only when required.
|
| 919 |
+
|
| 920 |
+
Args:
|
| 921 |
+
fn: The function to wrap
|
| 922 |
+
options: Optional backend to use for compiling the subgraph.
|
| 923 |
+
Warning: this is an experimental feature under development and
|
| 924 |
+
not ready for use yet.
|
| 925 |
+
max_reuse_entries: Maximum number of reuse cache entries per function
|
| 926 |
+
before raising an error. If this limit is hit, guards keep failing
|
| 927 |
+
across invocations and hierarchical compilation is not effective.
|
| 928 |
+
reuse_hash_fn: Optional callable that takes the same ``*args, **kwargs``
|
| 929 |
+
as the wrapped function and returns an integer hash key. When
|
| 930 |
+
provided, Dynamo traces this function to obtain a constant integer
|
| 931 |
+
and uses it as the cache key for subgraph reuse, bypassing the
|
| 932 |
+
automatic fingerprint/guard machinery. Two calls that produce the
|
| 933 |
+
same hash key reuse the same cached subgraph. The hash function
|
| 934 |
+
must be fully traceable (no graph breaks) and must return a
|
| 935 |
+
constant integer.
|
| 936 |
+
"""
|
| 937 |
+
|
| 938 |
+
if options is not None:
|
| 939 |
+
from torch._dynamo import config as dynamo_config
|
| 940 |
+
|
| 941 |
+
if not dynamo_config.enable_invoke_subgraph_regional_compile:
|
| 942 |
+
raise RuntimeError(
|
| 943 |
+
"nested_compile_region config is an experimental feature for testing only."
|
| 944 |
+
)
|
| 945 |
+
|
| 946 |
+
from torch._higher_order_ops.invoke_subgraph import (
|
| 947 |
+
mark_compile_region as _mark_compile_region,
|
| 948 |
+
)
|
| 949 |
+
|
| 950 |
+
return _mark_compile_region(
|
| 951 |
+
fn,
|
| 952 |
+
options=options,
|
| 953 |
+
max_reuse_entries=max_reuse_entries,
|
| 954 |
+
reuse_hash_fn=reuse_hash_fn,
|
| 955 |
+
)
|
| 956 |
+
|
| 957 |
+
|
| 958 |
+
def load_compiled_function(
|
| 959 |
+
file: io.IOBase,
|
| 960 |
+
*,
|
| 961 |
+
f_globals: dict[str, object] | None = None,
|
| 962 |
+
external_data: dict[str, Any] | None = None,
|
| 963 |
+
) -> Callable[..., Any]:
|
| 964 |
+
"""
|
| 965 |
+
Load an aot-compiled function from a file.
|
| 966 |
+
|
| 967 |
+
.. warning::
|
| 968 |
+
|
| 969 |
+
This API is currently experimental and subject to change.
|
| 970 |
+
|
| 971 |
+
Args:
|
| 972 |
+
file: A file-like object containing the serialized compiled function.
|
| 973 |
+
f_globals: Optional global scope enclosing the compiled function.
|
| 974 |
+
external_data: Optional data to be loaded into the runtime environment
|
| 975 |
+
of the compiled function. This should contain the same
|
| 976 |
+
data as AOTCompileResult.external_data returned from save_compiled_function() call.
|
| 977 |
+
|
| 978 |
+
Returns:
|
| 979 |
+
A torch-compiled function with compilation preloaded from disk.
|
| 980 |
+
"""
|
| 981 |
+
from torch._dynamo.aot_compile import AOTCompiledFunction
|
| 982 |
+
|
| 983 |
+
data = file.read()
|
| 984 |
+
return AOTCompiledFunction.deserialize(data, f_globals, external_data)
|
western_ai/usa/pytorch/torch/compiler/_cache.py
ADDED
|
@@ -0,0 +1,342 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import dataclasses
|
| 3 |
+
import logging
|
| 4 |
+
from abc import ABC, abstractmethod
|
| 5 |
+
from collections import defaultdict
|
| 6 |
+
from collections.abc import Generator
|
| 7 |
+
from contextlib import contextmanager
|
| 8 |
+
from itertools import chain
|
| 9 |
+
from typing import Any
|
| 10 |
+
|
| 11 |
+
from torch.utils._appending_byte_serializer import (
|
| 12 |
+
AppendingByteSerializer,
|
| 13 |
+
BytesReader,
|
| 14 |
+
BytesWriter,
|
| 15 |
+
)
|
| 16 |
+
from torch.utils._ordered_set import OrderedSet
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
log = logging.getLogger(__name__)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclasses.dataclass(frozen=True)
|
| 23 |
+
class CacheArtifact(ABC):
|
| 24 |
+
"""
|
| 25 |
+
Data for each cache artifact that will be serialized and deserialized
|
| 26 |
+
"""
|
| 27 |
+
|
| 28 |
+
key: str
|
| 29 |
+
content: bytes = dataclasses.field(repr=False) # Do not display potential binary
|
| 30 |
+
|
| 31 |
+
@staticmethod
|
| 32 |
+
def serialize(writer: BytesWriter, cls: "CacheArtifact") -> None:
|
| 33 |
+
writer.write_str(cls.key)
|
| 34 |
+
writer.write_bytes(cls.content)
|
| 35 |
+
|
| 36 |
+
@staticmethod
|
| 37 |
+
def deserialize(artifact_type: str, reader: BytesReader) -> "CacheArtifact":
|
| 38 |
+
key = reader.read_str()
|
| 39 |
+
content = reader.read_bytes()
|
| 40 |
+
return CacheArtifactFactory.create(artifact_type, key, content)
|
| 41 |
+
|
| 42 |
+
@staticmethod
|
| 43 |
+
def encode(content: Any) -> bytes:
|
| 44 |
+
if not isinstance(content, bytes):
|
| 45 |
+
raise AssertionError(f"Expected bytes, got {type(content)}")
|
| 46 |
+
return content
|
| 47 |
+
|
| 48 |
+
@abstractmethod
|
| 49 |
+
def populate_cache(self) -> None:
|
| 50 |
+
pass
|
| 51 |
+
|
| 52 |
+
@staticmethod
|
| 53 |
+
def type() -> str:
|
| 54 |
+
"""
|
| 55 |
+
Returns the type of the artifact. Must be unique across all CacheArtifact classes.
|
| 56 |
+
|
| 57 |
+
CacheArtifactFactory.register will add property method to CacheInfo based on this (def {type}_artifacts)
|
| 58 |
+
that returns all artifacts for specific cache.
|
| 59 |
+
"""
|
| 60 |
+
raise RuntimeError("CacheArtifact is an abstract class, please use a subclass")
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class CacheArtifactFactory:
|
| 64 |
+
"""
|
| 65 |
+
Factory for creating CacheArtifact objects based on their type
|
| 66 |
+
"""
|
| 67 |
+
|
| 68 |
+
_artifact_types: dict[str, type[CacheArtifact]] = {}
|
| 69 |
+
|
| 70 |
+
@classmethod
|
| 71 |
+
def register(cls, artifact_cls: type[CacheArtifact]) -> type[CacheArtifact]:
|
| 72 |
+
artifact_type_key = artifact_cls.type()
|
| 73 |
+
if artifact_cls.type() in cls._artifact_types:
|
| 74 |
+
raise AssertionError(
|
| 75 |
+
f"Artifact of type={artifact_type_key} already registered in mega-cache artifact factory"
|
| 76 |
+
)
|
| 77 |
+
cls._artifact_types[artifact_type_key] = artifact_cls
|
| 78 |
+
setattr(
|
| 79 |
+
CacheInfo,
|
| 80 |
+
f"{artifact_type_key}_artifacts",
|
| 81 |
+
property(lambda self: self.artifacts[artifact_type_key]),
|
| 82 |
+
)
|
| 83 |
+
return artifact_cls
|
| 84 |
+
|
| 85 |
+
@classmethod
|
| 86 |
+
def _get_artifact_type(cls, artifact_type_key: str) -> type[CacheArtifact]:
|
| 87 |
+
if artifact_type_key not in cls._artifact_types:
|
| 88 |
+
raise AssertionError(
|
| 89 |
+
f"Artifact of type={artifact_type_key} not registered in mega-cache artifact factory"
|
| 90 |
+
)
|
| 91 |
+
return cls._artifact_types[artifact_type_key]
|
| 92 |
+
|
| 93 |
+
@classmethod
|
| 94 |
+
def create(cls, artifact_type_key: str, key: str, content: bytes) -> CacheArtifact:
|
| 95 |
+
artifact_cls = cls._get_artifact_type(artifact_type_key)
|
| 96 |
+
# pyrefly: ignore [bad-instantiation]
|
| 97 |
+
return artifact_cls(key, content)
|
| 98 |
+
|
| 99 |
+
@classmethod
|
| 100 |
+
def encode_create(
|
| 101 |
+
cls, artifact_type_key: str, key: str, content: Any
|
| 102 |
+
) -> CacheArtifact:
|
| 103 |
+
artifact_cls = cls._get_artifact_type(artifact_type_key)
|
| 104 |
+
# pyrefly: ignore [bad-instantiation]
|
| 105 |
+
return artifact_cls(key, artifact_cls.encode(content))
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@dataclasses.dataclass
|
| 109 |
+
class CacheInfo:
|
| 110 |
+
"""
|
| 111 |
+
Return value of serialization and deserialization for the purpose of
|
| 112 |
+
instrumentation
|
| 113 |
+
"""
|
| 114 |
+
|
| 115 |
+
artifacts: defaultdict[str, list[str]] = dataclasses.field(
|
| 116 |
+
default_factory=lambda: defaultdict(list)
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
# Methods set by CacheArtifactFactory.register based on CacheArtifact.type()
|
| 120 |
+
@property
|
| 121 |
+
def inductor_artifacts(self) -> list[str]: # type: ignore[empty-body]
|
| 122 |
+
...
|
| 123 |
+
|
| 124 |
+
@property
|
| 125 |
+
def autotune_artifacts(self) -> list[str]: # type: ignore[empty-body]
|
| 126 |
+
...
|
| 127 |
+
|
| 128 |
+
@property
|
| 129 |
+
def aot_autograd_artifacts(self) -> list[str]: # type: ignore[empty-body]
|
| 130 |
+
...
|
| 131 |
+
|
| 132 |
+
@property
|
| 133 |
+
def pgo_artifacts(self) -> list[str]: # type: ignore[empty-body]
|
| 134 |
+
...
|
| 135 |
+
|
| 136 |
+
@property
|
| 137 |
+
def precompile_artifacts(self) -> list[str]: # type: ignore[empty-body]
|
| 138 |
+
...
|
| 139 |
+
|
| 140 |
+
def add(self, artifact: CacheArtifact) -> None:
|
| 141 |
+
self.artifacts[artifact.type()].append(artifact.key)
|
| 142 |
+
|
| 143 |
+
def clear(self) -> None:
|
| 144 |
+
self.artifacts.clear()
|
| 145 |
+
|
| 146 |
+
def empty(self) -> bool:
|
| 147 |
+
return not self.artifacts
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def _serialize_single_cache(
|
| 151 |
+
writer: BytesWriter, cls: "tuple[str, list[CacheArtifact]]"
|
| 152 |
+
) -> None:
|
| 153 |
+
writer.write_str(cls[0])
|
| 154 |
+
writer.write_uint64(len(cls[1]))
|
| 155 |
+
for artifact in cls[1]:
|
| 156 |
+
CacheArtifact.serialize(writer, artifact)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def _deserialize_single_cache(
|
| 160 |
+
reader: BytesReader,
|
| 161 |
+
) -> "tuple[str, list[CacheArtifact]]":
|
| 162 |
+
artifacts = []
|
| 163 |
+
artifact_type_key = reader.read_str()
|
| 164 |
+
num_artifacts = reader.read_uint64()
|
| 165 |
+
for _ in range(num_artifacts):
|
| 166 |
+
artifacts.append(CacheArtifact.deserialize(artifact_type_key, reader))
|
| 167 |
+
|
| 168 |
+
return artifact_type_key, artifacts
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
CacheArtifactsResult = dict[str, list[CacheArtifact]]
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
@dataclasses.dataclass(frozen=True)
|
| 175 |
+
class CacheArtifactRecorder:
|
| 176 |
+
"""
|
| 177 |
+
Shared entry point for cache implementations to record artifacts.
|
| 178 |
+
"""
|
| 179 |
+
|
| 180 |
+
artifact_type: str
|
| 181 |
+
key: str
|
| 182 |
+
|
| 183 |
+
def record(self, content: Any) -> None:
|
| 184 |
+
CacheArtifactManager.record_artifact(self.artifact_type, self.key, content)
|
| 185 |
+
|
| 186 |
+
def record_if_present(self, content: Any | None) -> None:
|
| 187 |
+
if content is not None:
|
| 188 |
+
self.record(content)
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
class CacheArtifactManager:
|
| 192 |
+
"""
|
| 193 |
+
Lightweight manager class for collecting and processing cache artifacts for
|
| 194 |
+
hot loading
|
| 195 |
+
|
| 196 |
+
Intended Lifecycle:
|
| 197 |
+
- Execute code via torch.compile, this will call
|
| 198 |
+
CacheArtifactRecorder.record on each cache artifact
|
| 199 |
+
- Call CacheArtifactManager.serialize to convert all the cache artifacts
|
| 200 |
+
to portable format
|
| 201 |
+
- Call CacheArtifactManager.deserialize to hot load the cache artifacts on
|
| 202 |
+
a potentially different process
|
| 203 |
+
|
| 204 |
+
NOTE: There's no FB/FC guarantees, results of cache artifacts will not be
|
| 205 |
+
used unless code version matches.
|
| 206 |
+
"""
|
| 207 |
+
|
| 208 |
+
# Protected by the compile_lock
|
| 209 |
+
_new_cache_artifacts: CacheArtifactsResult = defaultdict(list)
|
| 210 |
+
# Keep a separate seen artifacts list to make avoid unnecessary duplicates
|
| 211 |
+
# This list will not be cleared between serialize() calls
|
| 212 |
+
_seen_artifacts: OrderedSet[CacheArtifact] = OrderedSet()
|
| 213 |
+
# When serialize() is called, artifacts are transferred from _cache_artifacts to
|
| 214 |
+
# internal data structure of the _serializer
|
| 215 |
+
# This allows us to only pay the cost of serialization if serialize() is called
|
| 216 |
+
_serializer: AppendingByteSerializer[tuple[str, list[CacheArtifact]]] = (
|
| 217 |
+
AppendingByteSerializer(serialize_fn=_serialize_single_cache)
|
| 218 |
+
)
|
| 219 |
+
_cache_info: CacheInfo = CacheInfo()
|
| 220 |
+
|
| 221 |
+
@classmethod
|
| 222 |
+
def clear(cls) -> None:
|
| 223 |
+
cls._new_cache_artifacts.clear()
|
| 224 |
+
cls._seen_artifacts.clear()
|
| 225 |
+
cls._serializer.clear()
|
| 226 |
+
cls._cache_info.clear()
|
| 227 |
+
|
| 228 |
+
@classmethod
|
| 229 |
+
@contextmanager
|
| 230 |
+
def with_fresh_cache(cls) -> Generator[None, None, None]:
|
| 231 |
+
original_new_cache_artifacts = cls._new_cache_artifacts
|
| 232 |
+
original_seen_artifacts = cls._seen_artifacts
|
| 233 |
+
original_serializer = cls._serializer
|
| 234 |
+
original_cache_info = cls._cache_info
|
| 235 |
+
|
| 236 |
+
cls._new_cache_artifacts = defaultdict(list)
|
| 237 |
+
cls._seen_artifacts = OrderedSet()
|
| 238 |
+
cls._serializer = AppendingByteSerializer(serialize_fn=_serialize_single_cache)
|
| 239 |
+
cls._cache_info = cls._cache_info.__class__()
|
| 240 |
+
try:
|
| 241 |
+
yield
|
| 242 |
+
finally:
|
| 243 |
+
cls._new_cache_artifacts = original_new_cache_artifacts
|
| 244 |
+
cls._seen_artifacts = original_seen_artifacts
|
| 245 |
+
cls._serializer = original_serializer
|
| 246 |
+
cls._cache_info = original_cache_info
|
| 247 |
+
|
| 248 |
+
@classmethod
|
| 249 |
+
def record_artifact(
|
| 250 |
+
cls,
|
| 251 |
+
artifact_type: str,
|
| 252 |
+
key: str,
|
| 253 |
+
content: Any,
|
| 254 |
+
) -> None:
|
| 255 |
+
"""
|
| 256 |
+
Called from each caching operation to record the artifact in this
|
| 257 |
+
"mega" list
|
| 258 |
+
"""
|
| 259 |
+
artifact = CacheArtifactFactory.encode_create(artifact_type, key, content)
|
| 260 |
+
if artifact in cls._seen_artifacts:
|
| 261 |
+
return
|
| 262 |
+
log.debug("Recording %s", artifact)
|
| 263 |
+
cls._new_cache_artifacts[artifact_type].append(artifact)
|
| 264 |
+
cls._seen_artifacts.add(artifact)
|
| 265 |
+
|
| 266 |
+
@classmethod
|
| 267 |
+
def need_serialize(cls) -> bool:
|
| 268 |
+
"""
|
| 269 |
+
Have we seen new artifacts since last serialize call?
|
| 270 |
+
"""
|
| 271 |
+
return len(cls._new_cache_artifacts) != 0
|
| 272 |
+
|
| 273 |
+
@classmethod
|
| 274 |
+
def serialize(cls) -> tuple[bytes, CacheInfo] | None:
|
| 275 |
+
"""
|
| 276 |
+
Converts the "mega" list into portable format
|
| 277 |
+
"""
|
| 278 |
+
for artifact in chain(*cls._new_cache_artifacts.values()):
|
| 279 |
+
log.debug("saving: %s", artifact)
|
| 280 |
+
cls._cache_info.add(artifact)
|
| 281 |
+
|
| 282 |
+
if cls._cache_info.empty():
|
| 283 |
+
# If there are no artifacts, don't just return bytes with
|
| 284 |
+
# version.
|
| 285 |
+
return None
|
| 286 |
+
|
| 287 |
+
try:
|
| 288 |
+
# We deep copy cls._cache_info since later compilations
|
| 289 |
+
# can keep adding to cache_info
|
| 290 |
+
info = copy.deepcopy(cls._cache_info)
|
| 291 |
+
cls._serializer.extend(cls._new_cache_artifacts.items())
|
| 292 |
+
artifact_bytes = cls._serializer.to_bytes()
|
| 293 |
+
cls._new_cache_artifacts.clear()
|
| 294 |
+
return artifact_bytes, info
|
| 295 |
+
except Exception:
|
| 296 |
+
log.warning("Failed to pickle cache artifacts", exc_info=True)
|
| 297 |
+
return None
|
| 298 |
+
|
| 299 |
+
@staticmethod
|
| 300 |
+
def deserialize(serialized_artifacts: bytes) -> CacheArtifactsResult | None:
|
| 301 |
+
"""
|
| 302 |
+
Converts the portable format back into CacheArtifacts
|
| 303 |
+
"""
|
| 304 |
+
try:
|
| 305 |
+
CacheArtifactManager._ensure_cache_artifacts_registered()
|
| 306 |
+
artifacts = dict(
|
| 307 |
+
AppendingByteSerializer.to_list(
|
| 308 |
+
serialized_artifacts,
|
| 309 |
+
deserialize_fn=_deserialize_single_cache,
|
| 310 |
+
)
|
| 311 |
+
)
|
| 312 |
+
except Exception:
|
| 313 |
+
log.warning("Failed to un-pickle cache artifacts", exc_info=True)
|
| 314 |
+
return None
|
| 315 |
+
|
| 316 |
+
return artifacts
|
| 317 |
+
|
| 318 |
+
@staticmethod
|
| 319 |
+
def populate_caches(artifacts: CacheArtifactsResult) -> CacheInfo:
|
| 320 |
+
info = CacheInfo()
|
| 321 |
+
for artifact in chain(*artifacts.values()):
|
| 322 |
+
log.debug("writing: %s", artifact)
|
| 323 |
+
info.add(artifact)
|
| 324 |
+
artifact.populate_cache()
|
| 325 |
+
|
| 326 |
+
return info
|
| 327 |
+
|
| 328 |
+
@classmethod
|
| 329 |
+
def _ensure_cache_artifacts_registered(cls) -> None:
|
| 330 |
+
"""When deserializing caches in fresh process, we need to ensure that all
|
| 331 |
+
cache artifacts are registered in the cache registry. This is done by
|
| 332 |
+
simply importing all the cache artifacts already wrapped with register call.
|
| 333 |
+
"""
|
| 334 |
+
from torch._dynamo.package import PrecompileCacheArtifact # noqa: F401
|
| 335 |
+
from torch._dynamo.pgo import PGOCacheArtifact # noqa: F401
|
| 336 |
+
from torch._functorch._aot_autograd.autograd_cache import ( # noqa: F401
|
| 337 |
+
AOTAutogradCacheArtifact,
|
| 338 |
+
)
|
| 339 |
+
from torch._inductor.codecache import InductorCacheArtifact # noqa: F401
|
| 340 |
+
from torch._inductor.runtime.autotune_cache import ( # noqa: F401
|
| 341 |
+
AutotuneCacheArtifact,
|
| 342 |
+
)
|
western_ai/usa/pytorch/torch/compiler/config.py
ADDED
|
@@ -0,0 +1,344 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
This is the top-level configuration module for the compiler, containing
|
| 3 |
+
cross-cutting configuration options that affect all parts of the compiler
|
| 4 |
+
stack.
|
| 5 |
+
|
| 6 |
+
You may also be interested in the per-component configuration modules, which
|
| 7 |
+
contain configuration options that affect only a specific part of the compiler:
|
| 8 |
+
|
| 9 |
+
* :mod:`torch._dynamo.config`
|
| 10 |
+
* :mod:`torch._inductor.config`
|
| 11 |
+
* :mod:`torch._functorch.config`
|
| 12 |
+
* :mod:`torch.fx.experimental.config`
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import sys
|
| 16 |
+
|
| 17 |
+
from torch.utils._config_module import Config, install_config_module
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
__all__ = [
|
| 21 |
+
"job_id",
|
| 22 |
+
"dynamic_shapes",
|
| 23 |
+
"assume_static_by_default",
|
| 24 |
+
"automatic_dynamic_shapes",
|
| 25 |
+
"recompile_limit",
|
| 26 |
+
"accumulated_recompile_limit",
|
| 27 |
+
"verbose",
|
| 28 |
+
"capture_scalar_outputs",
|
| 29 |
+
"capture_dynamic_output_shape_ops",
|
| 30 |
+
"log_file_name",
|
| 31 |
+
"fail_on_recompile_limit_hit",
|
| 32 |
+
"allow_unspec_int_on_nn_module",
|
| 33 |
+
"skip_tensor_guards_with_matching_dict_tags",
|
| 34 |
+
"enable_cpp_symbolic_shape_guards",
|
| 35 |
+
"wrap_top_frame",
|
| 36 |
+
"reorderable_logging_functions",
|
| 37 |
+
"force_disable_caches",
|
| 38 |
+
"compile_on_one_rank",
|
| 39 |
+
]
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
# NB: Docblocks go UNDER variable definitions! Use spacing to make the
|
| 43 |
+
# grouping clear.
|
| 44 |
+
|
| 45 |
+
# FB-internal note: you do NOT have to specify this explicitly specify this if
|
| 46 |
+
# you run on MAST, we will automatically default this to
|
| 47 |
+
# mast:MAST_JOB_NAME:MAST_JOB_VERSION.
|
| 48 |
+
job_id: str | None = Config(
|
| 49 |
+
env_name_default=["TORCH_COMPILE_JOB_ID", "TORCH_COMPILE_STICKY_PGO_KEY"],
|
| 50 |
+
default=None,
|
| 51 |
+
)
|
| 52 |
+
"""
|
| 53 |
+
Semantically, this should be an identifier that uniquely identifies, e.g., a
|
| 54 |
+
training job. You might have multiple attempts of the same job, e.g., if it was
|
| 55 |
+
preempted or needed to be restarted, but each attempt should be running
|
| 56 |
+
substantially the same workload with the same distributed topology. You can
|
| 57 |
+
set this by environment variable with :envvar:`TORCH_COMPILE_JOB_ID`.
|
| 58 |
+
|
| 59 |
+
Operationally, this controls the effect of profile-guided optimization related
|
| 60 |
+
persistent state. PGO state can affect how we perform compilation across
|
| 61 |
+
multiple invocations of PyTorch, e.g., the first time you run your program we
|
| 62 |
+
may compile twice as we discover what inputs are dynamic, and then PGO will
|
| 63 |
+
save this state so subsequent invocations only need to compile once, because
|
| 64 |
+
they remember it is dynamic. This profile information, however, is sensitive
|
| 65 |
+
to what workload you are running, so we require you to tell us that two jobs
|
| 66 |
+
are *related* (i.e., are the same workload) before we are willing to reuse
|
| 67 |
+
this information. Notably, PGO does nothing (even if explicitly enabled)
|
| 68 |
+
unless a valid ``job_id`` is available. In some situations, PyTorch can
|
| 69 |
+
configured to automatically compute a ``job_id`` based on the environment it
|
| 70 |
+
is running in.
|
| 71 |
+
|
| 72 |
+
Profiles are always collected on a per rank basis, so different ranks may have
|
| 73 |
+
different profiles. If you know your workload is truly SPMD, you can run with
|
| 74 |
+
:data:`torch._dynamo.config.enable_compiler_collectives` to ensure nodes get
|
| 75 |
+
consistent profiles across all ranks.
|
| 76 |
+
"""
|
| 77 |
+
|
| 78 |
+
pgo_extra_read_key: str | None = Config(
|
| 79 |
+
env_name_default="TORCH_COMPILE_STICKY_PGO_READ", default=None
|
| 80 |
+
)
|
| 81 |
+
pgo_extra_write_key: str | None = Config(
|
| 82 |
+
env_name_default="TORCH_COMPILE_STICKY_PGO_WRITE", default=None
|
| 83 |
+
)
|
| 84 |
+
"""
|
| 85 |
+
Additional read/write keys for PGO.
|
| 86 |
+
Write key: Besides writing to the default local/remote PGO state, this also writes to the specified key.
|
| 87 |
+
Read key: Besides reading from the default state, this also reads from the specified key (if written to before)
|
| 88 |
+
and merges it with the default state.
|
| 89 |
+
"""
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
cache_key_tag: str = Config(env_name_default="TORCH_COMPILE_CACHE_KEY_TAG", default="")
|
| 93 |
+
"""
|
| 94 |
+
Tag to be included in the cache key generation for all torch compile caching.
|
| 95 |
+
A common use case for such a tag is to break caches.
|
| 96 |
+
"""
|
| 97 |
+
|
| 98 |
+
force_disable_caches: bool = Config(
|
| 99 |
+
justknob="pytorch/remote_cache:force_disable_caches",
|
| 100 |
+
env_name_force=[
|
| 101 |
+
"TORCHINDUCTOR_FORCE_DISABLE_CACHES",
|
| 102 |
+
"TORCH_COMPILE_FORCE_DISABLE_CACHES",
|
| 103 |
+
],
|
| 104 |
+
default=False,
|
| 105 |
+
)
|
| 106 |
+
"""
|
| 107 |
+
Force disables all caching -- This will take precedence over and override any other caching flag
|
| 108 |
+
"""
|
| 109 |
+
|
| 110 |
+
compile_on_one_rank: bool = Config(
|
| 111 |
+
default=False,
|
| 112 |
+
env_name_default=[
|
| 113 |
+
"TORCH_COMPILE_ON_ONE_RANK",
|
| 114 |
+
"TORCH_DISTRIBUTED_COMPILE_ON_ONE_RANK",
|
| 115 |
+
],
|
| 116 |
+
)
|
| 117 |
+
"""
|
| 118 |
+
When enabled, device- and rank-specific values (devices, process groups) are computed at
|
| 119 |
+
runtime via custom ops rather than baked in at compile time, so a graph can be compiled on
|
| 120 |
+
one rank and run on all ranks. Read across the stack: make_fx, inductor, and distributed.
|
| 121 |
+
"""
|
| 122 |
+
|
| 123 |
+
dynamic_sources: str = Config(
|
| 124 |
+
env_name_default="TORCH_COMPILE_DYNAMIC_SOURCES", default=""
|
| 125 |
+
)
|
| 126 |
+
r"""
|
| 127 |
+
Comma delimited list of sources that should be marked as dynamic. Primarily useful for large
|
| 128 |
+
models with graph breaks where you need intermediate tensors and ints to be marked dynamic.
|
| 129 |
+
|
| 130 |
+
Each entry is either an exact source name or a Python regex (regex applies only to the
|
| 131 |
+
source name, not to the dim) matched with ``re.match``. An optional ``:N`` suffix restricts
|
| 132 |
+
the entry to dim ``N`` of the matched tensor; without the suffix all dims are marked dynamic.
|
| 133 |
+
``N`` must be a literal non-negative integer. To target multiple dims of the same source,
|
| 134 |
+
list it multiple times. Examples::
|
| 135 |
+
|
| 136 |
+
L['x'] # all dims of x dynamic (current behavior)
|
| 137 |
+
L['x']:0 # only dim 0 of x dynamic
|
| 138 |
+
L['x']:0, L['x']:2 # dims 0 and 2 of x dynamic
|
| 139 |
+
|
| 140 |
+
Regex examples (``re.match`` anchors at the start, not the end)::
|
| 141 |
+
|
| 142 |
+
L\['x.*'\] # all sources whose name starts with L['x (e.g. L['x'], L['x1'])
|
| 143 |
+
L\['x.*'\]:0 # dim 0 dynamic for every source matching L['x.*
|
| 144 |
+
.* # mark every source dynamic (all dims)
|
| 145 |
+
.*:0 # mark dim 0 dynamic for every tensor source
|
| 146 |
+
|
| 147 |
+
The ``:N`` suffix is ignored for non-tensor (int/scalar) sources, which have no dim.
|
| 148 |
+
|
| 149 |
+
Entries in this list are dominant over all other flags dynamic=False, force_nn_module_property_static_shapes
|
| 150 |
+
and force_parameter_static_shapes.
|
| 151 |
+
"""
|
| 152 |
+
|
| 153 |
+
dynamic_values: str = Config(
|
| 154 |
+
env_name_default="TORCH_COMPILE_DYNAMIC_VALUES", default=""
|
| 155 |
+
)
|
| 156 |
+
"""
|
| 157 |
+
Comma delimited list of integer values that should cause any matching int source or
|
| 158 |
+
tensor dim to be marked dynamic.
|
| 159 |
+
|
| 160 |
+
This is useful for JIT workflows with graph breaks where you don't know up-front which
|
| 161 |
+
sources need to be dynamic, but you can warm up the model with a distinctive "sentinel"
|
| 162 |
+
value (e.g. an odd batch size like 1111) and have any int or tensor dim that equals one
|
| 163 |
+
of those values be marked dynamic automatically. This is needed because with graph
|
| 164 |
+
breaks we do not propagate dynamic properties through eager code, so an int or tensor
|
| 165 |
+
derived in eager between two compiled regions cannot inherit the dynamic-ness of its
|
| 166 |
+
source — matching on the sentinel value lets us recover it on the next compiled region.
|
| 167 |
+
|
| 168 |
+
Example::
|
| 169 |
+
|
| 170 |
+
TORCH_COMPILE_DYNAMIC_VALUES="1111,2222"
|
| 171 |
+
|
| 172 |
+
Pick sentinel values unlikely to occur as legitimate shapes/ints in your
|
| 173 |
+
workload (e.g. 1111, 2222) — matching is purely by value, so any int source
|
| 174 |
+
or tensor dim equal to a listed value anywhere in the program will be
|
| 175 |
+
marked dynamic.
|
| 176 |
+
|
| 177 |
+
unlike ``dynamic_sources``, ``dynamic_values`` does NOT override
|
| 178 |
+
``force_parameter_static_shapes`` or ``force_nn_module_property_static_shapes``.
|
| 179 |
+
"""
|
| 180 |
+
|
| 181 |
+
unbacked_sources: str = Config(
|
| 182 |
+
env_name_default="TORCH_COMPILE_UNBACKED_SOURCES", default=""
|
| 183 |
+
)
|
| 184 |
+
"""
|
| 185 |
+
Comma delimited list of sources that should be marked as unbacked. Primarily useful for large
|
| 186 |
+
models with graph breaks where you need intermediate tensors marked unbacked.
|
| 187 |
+
|
| 188 |
+
Supports the same ``:N`` per-dim suffix syntax as ``dynamic_sources``; see that config's
|
| 189 |
+
docstring for details.
|
| 190 |
+
|
| 191 |
+
Entries in this list are dominant over all other flags dynamic=False, force_nn_module_property_static_shapes
|
| 192 |
+
and force_parameter_static_shapes.
|
| 193 |
+
"""
|
| 194 |
+
|
| 195 |
+
# force a python GC before recording cudagraphs
|
| 196 |
+
force_cudagraph_gc: bool = Config(env_name_default="TORCH_CUDAGRAPH_GC", default=False)
|
| 197 |
+
"""
|
| 198 |
+
If True (the backward-compatible behavior) then gc.collect() before recording
|
| 199 |
+
any cudagraph.
|
| 200 |
+
"""
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
# Cross-cutting configuration options that affect the entire compilation pipeline
|
| 204 |
+
|
| 205 |
+
dynamic_shapes: bool = Config(alias="torch._dynamo.config.dynamic_shapes")
|
| 206 |
+
"""
|
| 207 |
+
Controls whether the compilation pipeline supports dynamic tensor shapes.
|
| 208 |
+
When enabled, the compiler can handle tensors with varying dimensions across
|
| 209 |
+
different invocations. This is a cross-cutting setting that affects shape
|
| 210 |
+
inference, guard generation, and code generation across the entire compilation
|
| 211 |
+
stack.
|
| 212 |
+
"""
|
| 213 |
+
|
| 214 |
+
assume_static_by_default: bool = Config(
|
| 215 |
+
alias="torch._dynamo.config.assume_static_by_default"
|
| 216 |
+
)
|
| 217 |
+
"""
|
| 218 |
+
When enabled, all tensor dimensions are assumed to be static unless explicitly
|
| 219 |
+
marked as dynamic or detected as changing. This compilation-wide behavior affects
|
| 220 |
+
how the entire stack handles shape specialization and can improve performance
|
| 221 |
+
for static workloads.
|
| 222 |
+
"""
|
| 223 |
+
|
| 224 |
+
automatic_dynamic_shapes: bool = Config(
|
| 225 |
+
alias="torch._dynamo.config.automatic_dynamic_shapes"
|
| 226 |
+
)
|
| 227 |
+
"""
|
| 228 |
+
Enables automatic detection and handling of dynamic shapes. When a tensor's
|
| 229 |
+
shape changes between compilations, the system automatically marks those
|
| 230 |
+
dimensions as dynamic rather than requiring manual specification. This
|
| 231 |
+
cross-cutting optimization improves the user experience by reducing recompilations.
|
| 232 |
+
"""
|
| 233 |
+
|
| 234 |
+
recompile_limit: int = Config(alias="torch._dynamo.config.recompile_limit")
|
| 235 |
+
"""
|
| 236 |
+
Maximum number of recompilations allowed for a single function before falling
|
| 237 |
+
back to eager execution. This compilation performance control prevents excessive
|
| 238 |
+
recompilation overhead that can degrade overall performance.
|
| 239 |
+
"""
|
| 240 |
+
|
| 241 |
+
accumulated_recompile_limit: int = Config(
|
| 242 |
+
alias="torch._dynamo.config.accumulated_recompile_limit"
|
| 243 |
+
)
|
| 244 |
+
"""
|
| 245 |
+
Global limit on total recompilations across all compiled functions to prevent
|
| 246 |
+
runaway recompilation scenarios. This safeguard protects against compilation
|
| 247 |
+
performance issues that could affect the entire program.
|
| 248 |
+
"""
|
| 249 |
+
|
| 250 |
+
verbose: bool = Config(alias="torch._dynamo.config.verbose")
|
| 251 |
+
"""
|
| 252 |
+
Enables verbose debugging output for Dynamo. When enabled, provides detailed
|
| 253 |
+
information about Dynamo's compilation decisions, optimizations, and potential
|
| 254 |
+
issues.
|
| 255 |
+
"""
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
# TorchDynamo-specific configuration options
|
| 259 |
+
|
| 260 |
+
capture_scalar_outputs: bool = Config(
|
| 261 |
+
alias="torch._dynamo.config.capture_scalar_outputs"
|
| 262 |
+
)
|
| 263 |
+
"""
|
| 264 |
+
Controls whether TorchDynamo captures operations that return scalar values (like .item())
|
| 265 |
+
into the FX graph. When disabled, these operations cause graph breaks. This is a
|
| 266 |
+
TorchDynamo-specific tracing behavior that affects how the tracer handles
|
| 267 |
+
scalar-returning operations.
|
| 268 |
+
"""
|
| 269 |
+
|
| 270 |
+
capture_dynamic_output_shape_ops: bool = Config(
|
| 271 |
+
alias="torch._dynamo.config.capture_dynamic_output_shape_ops"
|
| 272 |
+
)
|
| 273 |
+
"""
|
| 274 |
+
Controls whether TorchDynamo captures operations with dynamic output shapes (like
|
| 275 |
+
nonzero, unique) into the FX graph. When disabled, these operations cause graph breaks.
|
| 276 |
+
This is a TorchDynamo-specific setting for handling operations with unpredictable
|
| 277 |
+
output shapes during tracing.
|
| 278 |
+
"""
|
| 279 |
+
|
| 280 |
+
log_file_name: str | None = Config(alias="torch._dynamo.config.log_file_name")
|
| 281 |
+
"""
|
| 282 |
+
Specifies a file path for TorchDynamo-specific logging output. When set, internal
|
| 283 |
+
TorchDynamo debug information is written to this file rather than stdout. This is
|
| 284 |
+
useful for debugging TorchDynamo's internal tracing behavior.
|
| 285 |
+
"""
|
| 286 |
+
|
| 287 |
+
fail_on_recompile_limit_hit: bool = Config(
|
| 288 |
+
alias="torch._dynamo.config.fail_on_recompile_limit_hit"
|
| 289 |
+
)
|
| 290 |
+
"""
|
| 291 |
+
Raises a hard error when recompile limits are exceeded instead of falling back
|
| 292 |
+
to eager execution. This is useful for detecting excessive recompilation in
|
| 293 |
+
performance-critical deployments where you want to ensure compilation overhead
|
| 294 |
+
is kept under control.
|
| 295 |
+
"""
|
| 296 |
+
|
| 297 |
+
allow_unspec_int_on_nn_module: bool = Config(
|
| 298 |
+
alias="torch._dynamo.config.allow_unspec_int_on_nn_module"
|
| 299 |
+
)
|
| 300 |
+
"""
|
| 301 |
+
Allows integer attributes of nn.Module instances to be unspecialized through
|
| 302 |
+
the dynamic shape mechanism. By default, TorchDynamo specializes on all integer
|
| 303 |
+
module attributes, but this can cause excessive recompilation when integers
|
| 304 |
+
like step counters change frequently.
|
| 305 |
+
"""
|
| 306 |
+
|
| 307 |
+
skip_tensor_guards_with_matching_dict_tags: bool = Config(
|
| 308 |
+
alias="torch._dynamo.config.skip_tensor_guards_with_matching_dict_tags"
|
| 309 |
+
)
|
| 310 |
+
"""
|
| 311 |
+
Optimizes guard generation by treating tensors as immutable when they are
|
| 312 |
+
dictionary values with consistent dictionary tags across invocations. This
|
| 313 |
+
reduces guard overhead for tensors stored in persistent data structures.
|
| 314 |
+
"""
|
| 315 |
+
|
| 316 |
+
enable_cpp_symbolic_shape_guards: bool = Config(
|
| 317 |
+
alias="torch._dynamo.config.enable_cpp_symbolic_shape_guards"
|
| 318 |
+
)
|
| 319 |
+
"""
|
| 320 |
+
Uses C++ implementation for symbolic shape guard evaluation to improve performance.
|
| 321 |
+
The C++ guard manager can significantly speed up guard checking for symbolic shapes
|
| 322 |
+
in shape-polymorphic compilations.
|
| 323 |
+
"""
|
| 324 |
+
|
| 325 |
+
wrap_top_frame: bool = Config(alias="torch._dynamo.config.wrap_top_frame")
|
| 326 |
+
"""
|
| 327 |
+
Wraps the top-level decorated function/module in a frame wrapper to ensure
|
| 328 |
+
nn.Module hooks are compiled within the same frame as the main function. This
|
| 329 |
+
improves compilation coverage for models that rely on hooks.
|
| 330 |
+
"""
|
| 331 |
+
|
| 332 |
+
reorderable_logging_functions: set = Config(
|
| 333 |
+
alias="torch._dynamo.config.reorderable_logging_functions"
|
| 334 |
+
)
|
| 335 |
+
"""
|
| 336 |
+
A set of logging functions that can be reordered to execute after the compiled
|
| 337 |
+
portion of the graph, allowing larger graphs to be captured. Functions in this
|
| 338 |
+
set will have their execution deferred to avoid graph breaks, though this may
|
| 339 |
+
affect the timing of log output. In particular, mutated values will not be logged
|
| 340 |
+
at the right time, leading to incorrect logging.
|
| 341 |
+
"""
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
install_config_module(sys.modules[__name__])
|
western_ai/usa/pytorch/torch/contrib/__init__.py
ADDED
|
File without changes
|
western_ai/usa/pytorch/torch/contrib/_tensorboard_vis.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import time
|
| 3 |
+
from collections import defaultdict
|
| 4 |
+
from functools import partial
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
# Unfortunately it doesn't seem as if there was any way to get TensorBoard to do
|
| 10 |
+
# anything without having TF installed, and so this file has a hard dependency on it
|
| 11 |
+
# as well. It really is a debugging tool, so it doesn't matter.
|
| 12 |
+
try:
|
| 13 |
+
from tensorflow.core.framework import graph_pb2
|
| 14 |
+
from tensorflow.core.util import event_pb2
|
| 15 |
+
from tensorflow.python.summary.writer.writer import FileWriter
|
| 16 |
+
except ImportError:
|
| 17 |
+
raise ImportError(
|
| 18 |
+
"TensorBoard visualization of GraphExecutors requires having "
|
| 19 |
+
"TensorFlow installed"
|
| 20 |
+
) from None
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def dump_tensorboard_summary(graph_executor, logdir):
|
| 24 |
+
with FileWriter(logdir) as w:
|
| 25 |
+
pb_graph = visualize(graph_executor)
|
| 26 |
+
evt = event_pb2.Event(
|
| 27 |
+
wall_time=time.time(), graph_def=pb_graph.SerializeToString()
|
| 28 |
+
)
|
| 29 |
+
w.add_event(evt)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def visualize(graph, name_prefix="", pb_graph=None, executors_it=None):
|
| 33 |
+
"""Visualizes an independent graph, or a graph executor."""
|
| 34 |
+
value_map = {}
|
| 35 |
+
pb_graph = pb_graph or graph_pb2.GraphDef()
|
| 36 |
+
|
| 37 |
+
if isinstance(graph, torch._C.GraphExecutorState):
|
| 38 |
+
visualize_graph_executor(
|
| 39 |
+
graph, name_prefix, pb_graph, partial(visualize, pb_graph=pb_graph)
|
| 40 |
+
)
|
| 41 |
+
return pb_graph
|
| 42 |
+
|
| 43 |
+
# Set up an input node
|
| 44 |
+
pb_graph.node.add(op="input", name=name_prefix + "input")
|
| 45 |
+
for i, value in enumerate(graph.param_node().outputs()):
|
| 46 |
+
value_map[value.unique()] = name_prefix + "input:" + str(i)
|
| 47 |
+
|
| 48 |
+
visualize_rec(graph, value_map, name_prefix, pb_graph, executors_it)
|
| 49 |
+
|
| 50 |
+
# Gather all outputs
|
| 51 |
+
return_node = pb_graph.node.add(op="output", name=name_prefix + "output")
|
| 52 |
+
for value in graph.return_node().inputs():
|
| 53 |
+
return_node.input.append(value_map[value.unique()])
|
| 54 |
+
|
| 55 |
+
return pb_graph
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def visualize_graph_executor(state, name_prefix, pb_graph, inline_graph):
|
| 59 |
+
"""Append the state of a given GraphExecutor to the graph protobuf.
|
| 60 |
+
|
| 61 |
+
Args:
|
| 62 |
+
state (GraphExecutor or GraphExecutorState): GraphExecutor to display.
|
| 63 |
+
name_prefix (str): Name prefix of the containing subgraph.
|
| 64 |
+
pb_graph (GraphDef): graph to append to.
|
| 65 |
+
inline_graph (Callable): a function that handles setting up a value_map,
|
| 66 |
+
so that some graphs in here can be inlined. This is necessary, because
|
| 67 |
+
this will simply be `visualize` for the top-level GraphExecutor,
|
| 68 |
+
or `inline_graph` for all nested ones.
|
| 69 |
+
|
| 70 |
+
The signature should look like (Graph, name_prefix) -> ().
|
| 71 |
+
It will be called exactly once.
|
| 72 |
+
|
| 73 |
+
The strategy is to embed all different configurations as independent subgraphs,
|
| 74 |
+
while inlining the original graph as the one that actually produces the values.
|
| 75 |
+
"""
|
| 76 |
+
if state.autograd_fallback_graph is not None:
|
| 77 |
+
visualize(
|
| 78 |
+
graph=state.autograd_fallback_graph,
|
| 79 |
+
name_prefix=name_prefix + "autograd_fallback/",
|
| 80 |
+
pb_graph=pb_graph,
|
| 81 |
+
executors_it=iter(state.autograd_fallback.executors()),
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
for i, (arg_spec, plan) in enumerate(state.execution_plans.items()):
|
| 85 |
+
subgraph_name = name_prefix + f"plan{i}/"
|
| 86 |
+
|
| 87 |
+
# Create a disconnected node that will keep information regarding the input
|
| 88 |
+
# types of this trace. This is unfortunately a bit too verbose to be included
|
| 89 |
+
# in the subgraph name.
|
| 90 |
+
input_kinds = pb_graph.node.add(op="INPUT_KIND", name=subgraph_name)
|
| 91 |
+
input_kinds.attr["inputs"].s = repr(arg_spec).encode("ascii")
|
| 92 |
+
|
| 93 |
+
visualize(plan.graph, subgraph_name, pb_graph, iter(plan.code.executors()))
|
| 94 |
+
|
| 95 |
+
# Show gradient as an independent subgraph of this plan
|
| 96 |
+
if plan.grad_executor is not None:
|
| 97 |
+
grad_subgraph_name = subgraph_name + "grad/"
|
| 98 |
+
visualize(plan.grad_executor, grad_subgraph_name, pb_graph)
|
| 99 |
+
|
| 100 |
+
return inline_graph(state.graph, name_prefix + "original/")
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def visualize_rec(graph, value_map, name_prefix, pb_graph, executors_it=None):
|
| 104 |
+
"""Recursive part of visualize (basically skips setting up the input and output nodes)."""
|
| 105 |
+
|
| 106 |
+
def inline_graph(subgraph, name, node):
|
| 107 |
+
rec_value_map = {
|
| 108 |
+
inp.unique(): value_map[val.unique()]
|
| 109 |
+
for inp, val in zip(subgraph.inputs(), node.inputs())
|
| 110 |
+
}
|
| 111 |
+
visualize_rec(
|
| 112 |
+
graph=subgraph, value_map=rec_value_map, name_prefix=name, pb_graph=pb_graph
|
| 113 |
+
)
|
| 114 |
+
for out, val in zip(subgraph.outputs(), node.outputs()):
|
| 115 |
+
value_map[val.unique()] = rec_value_map[out.unique()]
|
| 116 |
+
|
| 117 |
+
op_id_counter: defaultdict[str, int] = defaultdict(int)
|
| 118 |
+
|
| 119 |
+
def name_for(node):
|
| 120 |
+
kind = node.kind()[node.kind().index("::") + 2 :]
|
| 121 |
+
op_id_counter[kind] += 1
|
| 122 |
+
return kind, name_prefix + kind + "_" + str(op_id_counter[kind])
|
| 123 |
+
|
| 124 |
+
def add_fusion_group(node):
|
| 125 |
+
op, name = name_for(node)
|
| 126 |
+
inline_graph(node.g("Subgraph"), name + "/", node)
|
| 127 |
+
|
| 128 |
+
def add_graph_executor(node):
|
| 129 |
+
op, name = name_for(node)
|
| 130 |
+
if executors_it is None:
|
| 131 |
+
add_node(node)
|
| 132 |
+
else:
|
| 133 |
+
ge = next(executors_it)
|
| 134 |
+
visualize_graph_executor(
|
| 135 |
+
ge, name + "/", pb_graph, partial(inline_graph, node=node)
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
def add_node(node):
|
| 139 |
+
if node.kind() == "prim::FusionGroup":
|
| 140 |
+
return add_fusion_group(node)
|
| 141 |
+
elif node.kind() == "prim::GraphExecutor":
|
| 142 |
+
return add_graph_executor(node)
|
| 143 |
+
op, name = name_for(node)
|
| 144 |
+
pb_node = pb_graph.node.add(op=op, name=name)
|
| 145 |
+
for value in node.inputs():
|
| 146 |
+
pb_node.input.append(value_map[value.unique()])
|
| 147 |
+
# TODO: handle attrs
|
| 148 |
+
for i, value in enumerate(node.outputs()):
|
| 149 |
+
value_map[value.unique()] = name + ":" + str(i)
|
| 150 |
+
|
| 151 |
+
for node in graph.nodes():
|
| 152 |
+
add_node(node)
|
western_ai/usa/pytorch/torch/cpu/__init__.py
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
r"""
|
| 3 |
+
This package implements abstractions found in ``torch.cuda``
|
| 4 |
+
to facilitate writing device-agnostic code.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from collections.abc import Mapping
|
| 8 |
+
from contextlib import AbstractContextManager
|
| 9 |
+
from functools import lru_cache
|
| 10 |
+
from types import MappingProxyType
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
from .. import device as _device
|
| 16 |
+
from . import amp
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
__all__ = [
|
| 20 |
+
"is_available",
|
| 21 |
+
"is_initialized",
|
| 22 |
+
"synchronize",
|
| 23 |
+
"current_device",
|
| 24 |
+
"current_stream",
|
| 25 |
+
"stream",
|
| 26 |
+
"set_device",
|
| 27 |
+
"device_count",
|
| 28 |
+
"Stream",
|
| 29 |
+
"StreamContext",
|
| 30 |
+
"Event",
|
| 31 |
+
"get_capabilities",
|
| 32 |
+
]
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@lru_cache(None)
|
| 36 |
+
def get_capabilities() -> Mapping[str, Any]:
|
| 37 |
+
"""
|
| 38 |
+
Returns an immutable mapping of CPU capabilities detected at runtime.
|
| 39 |
+
|
| 40 |
+
This function queries the CPU for supported instruction sets and features
|
| 41 |
+
using cpuinfo. The result is cached after the first call for efficiency.
|
| 42 |
+
|
| 43 |
+
The returned mapping contains architecture-specific capabilities:
|
| 44 |
+
|
| 45 |
+
For x86/x86_64:
|
| 46 |
+
- SSE family: sse, sse2, sse3, ssse3, sse4_1, sse4_2, sse4a
|
| 47 |
+
- AVX family: avx, avx2, avx_vnni
|
| 48 |
+
- AVX-512 family: avx512_f, avx512_cd, avx512_dq, avx512_bw, avx512_vl,
|
| 49 |
+
avx512_ifma, avx512_vbmi, avx512_vbmi2, avx512_bitalg, avx512_vpopcntdq,
|
| 50 |
+
avx512_vnni, avx512_bf16, avx512_fp16, avx512_vp2intersect,
|
| 51 |
+
avx512_4vnniw, avx512_4fmaps
|
| 52 |
+
- AVX10 family: avx10_1, avx10_2
|
| 53 |
+
- AVX-VNNI-INT: avx_vnni_int8, avx_vnni_int16, avx_ne_convert
|
| 54 |
+
- AMX: amx_bf16, amx_tile, amx_int8, amx_fp16
|
| 55 |
+
- FMA: fma3, fma4
|
| 56 |
+
- Other: f16c, bmi, bmi2, popcnt, lzcnt, aes, sha, clflush, clflushopt, clwb
|
| 57 |
+
|
| 58 |
+
For ARM64:
|
| 59 |
+
- SIMD: neon, fp16_arith, bf16, i8mm, dot
|
| 60 |
+
- SVE: sve, sve2, sve_bf16, sve_max_length (when supported)
|
| 61 |
+
- SME: sme, sme2, sme_max_length (when supported)
|
| 62 |
+
- Other: atomics, fhm, rdm, crc32, aes, sha1, sha2, pmull
|
| 63 |
+
|
| 64 |
+
Common to all architectures:
|
| 65 |
+
- architecture: string identifying the CPU architecture
|
| 66 |
+
|
| 67 |
+
Returns:
|
| 68 |
+
MappingProxyType: An immutable mapping where keys are capability names
|
| 69 |
+
(e.g., 'avx2', 'sve') and values are booleans indicating
|
| 70 |
+
support, or integers for properties like vector lengths.
|
| 71 |
+
|
| 72 |
+
Example:
|
| 73 |
+
>>> caps = torch.cpu.get_capabilities()
|
| 74 |
+
>>> if caps.get("avx2", False):
|
| 75 |
+
... print("AVX2 is supported")
|
| 76 |
+
>>> print(f"Architecture: {caps['architecture']}")
|
| 77 |
+
"""
|
| 78 |
+
return MappingProxyType(torch._C._cpu._get_cpu_capability())
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _is_avx2_supported() -> bool:
|
| 82 |
+
r"""Returns a bool indicating if CPU supports AVX2."""
|
| 83 |
+
return get_capabilities().get("avx2", False)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _is_avx512_supported() -> bool:
|
| 87 |
+
r"""Returns a bool indicating if CPU supports AVX512."""
|
| 88 |
+
return get_capabilities().get("avx512_f", False)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _is_avx512_bf16_supported() -> bool:
|
| 92 |
+
r"""Returns a bool indicating if CPU supports AVX512_BF16."""
|
| 93 |
+
return get_capabilities().get("avx512_bf16", False)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _is_vnni_supported() -> bool:
|
| 97 |
+
r"""Returns a bool indicating if CPU supports VNNI."""
|
| 98 |
+
# Note: Currently, it only checks avx512_vnni, will add the support of avx2_vnni later.
|
| 99 |
+
return get_capabilities().get("avx512_vnni", False)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _is_amx_tile_supported() -> bool:
|
| 103 |
+
r"""Returns a bool indicating if CPU supports AMX_TILE."""
|
| 104 |
+
return get_capabilities().get("amx_tile", False)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _is_amx_fp16_supported() -> bool:
|
| 108 |
+
r"""Returns a bool indicating if CPU supports AMX FP16."""
|
| 109 |
+
return get_capabilities().get("amx_fp16", False)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def _init_amx() -> bool:
|
| 113 |
+
r"""Initializes AMX instructions."""
|
| 114 |
+
return torch._C._cpu._init_amx()
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def is_available() -> bool:
|
| 118 |
+
r"""Returns a bool indicating if CPU is currently available.
|
| 119 |
+
|
| 120 |
+
N.B. This function only exists to facilitate device-agnostic code
|
| 121 |
+
|
| 122 |
+
"""
|
| 123 |
+
return True
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def synchronize(device: torch.types.Device = None) -> None:
|
| 127 |
+
r"""Waits for all kernels in all streams on the CPU device to complete.
|
| 128 |
+
|
| 129 |
+
Args:
|
| 130 |
+
device (torch.device or int, optional): ignored, there's only one CPU device.
|
| 131 |
+
|
| 132 |
+
N.B. This function only exists to facilitate device-agnostic code.
|
| 133 |
+
"""
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
class Stream:
|
| 137 |
+
"""
|
| 138 |
+
N.B. This class only exists to facilitate device-agnostic code
|
| 139 |
+
"""
|
| 140 |
+
|
| 141 |
+
def __init__(self, priority: int = -1) -> None:
|
| 142 |
+
pass
|
| 143 |
+
|
| 144 |
+
def wait_stream(self, stream) -> None:
|
| 145 |
+
pass
|
| 146 |
+
|
| 147 |
+
def record_event(self) -> None:
|
| 148 |
+
pass
|
| 149 |
+
|
| 150 |
+
def wait_event(self, event) -> None:
|
| 151 |
+
pass
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
class Event:
|
| 155 |
+
def query(self) -> bool:
|
| 156 |
+
return True
|
| 157 |
+
|
| 158 |
+
def record(self, stream=None) -> None:
|
| 159 |
+
pass
|
| 160 |
+
|
| 161 |
+
def synchronize(self) -> None:
|
| 162 |
+
pass
|
| 163 |
+
|
| 164 |
+
def wait(self, stream=None) -> None:
|
| 165 |
+
pass
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
_default_cpu_stream = Stream()
|
| 169 |
+
_current_stream = _default_cpu_stream
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def current_stream(device: torch.types.Device = None) -> Stream:
|
| 173 |
+
r"""Returns the currently selected :class:`Stream` for a given device.
|
| 174 |
+
|
| 175 |
+
Args:
|
| 176 |
+
device (torch.device or int, optional): Ignored.
|
| 177 |
+
|
| 178 |
+
N.B. This function only exists to facilitate device-agnostic code
|
| 179 |
+
|
| 180 |
+
"""
|
| 181 |
+
return _current_stream
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
class StreamContext(AbstractContextManager):
|
| 185 |
+
r"""Context-manager that selects a given stream.
|
| 186 |
+
|
| 187 |
+
N.B. This class only exists to facilitate device-agnostic code
|
| 188 |
+
|
| 189 |
+
"""
|
| 190 |
+
|
| 191 |
+
cur_stream: Stream | None
|
| 192 |
+
|
| 193 |
+
def __init__(self, stream):
|
| 194 |
+
self.stream = stream
|
| 195 |
+
self.prev_stream = _default_cpu_stream
|
| 196 |
+
|
| 197 |
+
def __enter__(self):
|
| 198 |
+
cur_stream = self.stream
|
| 199 |
+
if cur_stream is None:
|
| 200 |
+
return
|
| 201 |
+
|
| 202 |
+
global _current_stream
|
| 203 |
+
self.prev_stream = _current_stream
|
| 204 |
+
_current_stream = cur_stream
|
| 205 |
+
|
| 206 |
+
def __exit__(self, type: Any, value: Any, traceback: Any) -> None:
|
| 207 |
+
cur_stream = self.stream
|
| 208 |
+
if cur_stream is None:
|
| 209 |
+
return
|
| 210 |
+
|
| 211 |
+
global _current_stream
|
| 212 |
+
_current_stream = self.prev_stream
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def stream(stream: Stream) -> AbstractContextManager:
|
| 216 |
+
r"""Wrapper around the Context-manager StreamContext that
|
| 217 |
+
selects a given stream.
|
| 218 |
+
|
| 219 |
+
N.B. This function only exists to facilitate device-agnostic code
|
| 220 |
+
"""
|
| 221 |
+
return StreamContext(stream)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def device_count() -> int:
|
| 225 |
+
r"""Returns number of CPU devices (not cores). Always 1.
|
| 226 |
+
|
| 227 |
+
N.B. This function only exists to facilitate device-agnostic code
|
| 228 |
+
"""
|
| 229 |
+
return 1
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def set_device(device: torch.types.Device) -> None:
|
| 233 |
+
r"""Sets the current device, in CPU we do nothing.
|
| 234 |
+
|
| 235 |
+
N.B. This function only exists to facilitate device-agnostic code
|
| 236 |
+
"""
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def current_device() -> str:
|
| 240 |
+
r"""Returns current device for cpu. Always 'cpu'.
|
| 241 |
+
|
| 242 |
+
N.B. This function only exists to facilitate device-agnostic code
|
| 243 |
+
"""
|
| 244 |
+
return "cpu"
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def is_initialized() -> bool:
|
| 248 |
+
r"""Returns True if the CPU is initialized. Always True.
|
| 249 |
+
|
| 250 |
+
N.B. This function only exists to facilitate device-agnostic code
|
| 251 |
+
"""
|
| 252 |
+
return True
|
western_ai/usa/pytorch/torch/cpu/amp/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# pyrefly: ignore [deprecated]
|
| 2 |
+
from .autocast_mode import autocast
|
| 3 |
+
from .grad_scaler import GradScaler
|
western_ai/usa/pytorch/torch/cpu/amp/autocast_mode.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import sys
|
| 3 |
+
from typing import Any
|
| 4 |
+
from typing_extensions import deprecated
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
__all__ = ["autocast"]
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@deprecated(
|
| 13 |
+
"`torch.cpu.amp.autocast(args...)` is deprecated. "
|
| 14 |
+
"Please use `torch.amp.autocast('cpu', args...)` instead.",
|
| 15 |
+
category=FutureWarning,
|
| 16 |
+
)
|
| 17 |
+
class autocast(torch.amp.autocast_mode.autocast):
|
| 18 |
+
r"""
|
| 19 |
+
See :class:`torch.autocast`.
|
| 20 |
+
``torch.cpu.amp.autocast(args...)`` is deprecated. Please use ``torch.amp.autocast("cpu", args...)`` instead.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
# TODO: remove this conditional once we stop supporting Python < 3.13
|
| 24 |
+
# Prior to Python 3.13, inspect.signature could not retrieve the correct
|
| 25 |
+
# signature information for classes decorated with @deprecated (unless
|
| 26 |
+
# the __new__ static method was explicitly defined);
|
| 27 |
+
#
|
| 28 |
+
# However, this issue has been fixed in Python 3.13 and later versions.
|
| 29 |
+
if sys.version_info < (3, 13):
|
| 30 |
+
|
| 31 |
+
def __new__(
|
| 32 |
+
cls,
|
| 33 |
+
enabled: bool = True,
|
| 34 |
+
dtype: torch.dtype = torch.bfloat16,
|
| 35 |
+
cache_enabled: bool = True,
|
| 36 |
+
):
|
| 37 |
+
return super().__new__(cls)
|
| 38 |
+
|
| 39 |
+
def __init_subclass__(cls):
|
| 40 |
+
pass
|
| 41 |
+
|
| 42 |
+
def __init__(
|
| 43 |
+
self,
|
| 44 |
+
enabled: bool = True,
|
| 45 |
+
dtype: torch.dtype = torch.bfloat16,
|
| 46 |
+
cache_enabled: bool = True,
|
| 47 |
+
):
|
| 48 |
+
if torch._jit_internal.is_scripting():
|
| 49 |
+
self._enabled = enabled
|
| 50 |
+
self.device = "cpu"
|
| 51 |
+
self.fast_dtype = dtype
|
| 52 |
+
return
|
| 53 |
+
super().__init__(
|
| 54 |
+
"cpu", enabled=enabled, dtype=dtype, cache_enabled=cache_enabled
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
def __enter__(self):
|
| 58 |
+
if torch._jit_internal.is_scripting():
|
| 59 |
+
return self
|
| 60 |
+
return super().__enter__()
|
| 61 |
+
|
| 62 |
+
# TODO: discuss a unified TorchScript-friendly API for autocast
|
| 63 |
+
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any): # type: ignore[override]
|
| 64 |
+
if torch._jit_internal.is_scripting():
|
| 65 |
+
return
|
| 66 |
+
return super().__exit__(exc_type, exc_val, exc_tb)
|
| 67 |
+
|
| 68 |
+
def __call__(self, func):
|
| 69 |
+
if torch._jit_internal.is_scripting():
|
| 70 |
+
return func
|
| 71 |
+
return super().__call__(func)
|
western_ai/usa/pytorch/torch/cpu/amp/grad_scaler.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing_extensions import deprecated
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
__all__ = ["GradScaler"]
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class GradScaler(torch.amp.GradScaler):
|
| 10 |
+
r"""
|
| 11 |
+
See :class:`torch.amp.GradScaler`.
|
| 12 |
+
``torch.cpu.amp.GradScaler(args...)`` is deprecated. Please use ``torch.amp.GradScaler("cpu", args...)`` instead.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
@deprecated(
|
| 16 |
+
"`torch.cpu.amp.GradScaler(args...)` is deprecated. "
|
| 17 |
+
"Please use `torch.amp.GradScaler('cpu', args...)` instead.",
|
| 18 |
+
category=FutureWarning,
|
| 19 |
+
)
|
| 20 |
+
def __init__(
|
| 21 |
+
self,
|
| 22 |
+
init_scale: float = 2.0**16,
|
| 23 |
+
growth_factor: float = 2.0,
|
| 24 |
+
backoff_factor: float = 0.5,
|
| 25 |
+
growth_interval: int = 2000,
|
| 26 |
+
enabled: bool = True,
|
| 27 |
+
) -> None:
|
| 28 |
+
super().__init__(
|
| 29 |
+
"cpu",
|
| 30 |
+
init_scale=init_scale,
|
| 31 |
+
growth_factor=growth_factor,
|
| 32 |
+
backoff_factor=backoff_factor,
|
| 33 |
+
growth_interval=growth_interval,
|
| 34 |
+
enabled=enabled,
|
| 35 |
+
)
|
western_ai/usa/pytorch/torch/csrc/CudaIPCTypes.cpp
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include <ATen/MapAllocator.h>
|
| 2 |
+
#include <c10/cuda/CUDAGuard.h>
|
| 3 |
+
#include <torch/csrc/CudaIPCTypes.h>
|
| 4 |
+
#include <atomic>
|
| 5 |
+
#include <map>
|
| 6 |
+
#include <mutex>
|
| 7 |
+
#include <string>
|
| 8 |
+
|
| 9 |
+
namespace torch {
|
| 10 |
+
|
| 11 |
+
namespace {
|
| 12 |
+
|
| 13 |
+
void warnProducerTerminatedBeforeSharedTensorsReleased() {
|
| 14 |
+
static bool warned = false;
|
| 15 |
+
if (!warned) {
|
| 16 |
+
LOG(WARNING)
|
| 17 |
+
<< "Producer process has been terminated before all shared CUDA tensors released. See Note [Sharing CUDA tensors]";
|
| 18 |
+
warned = true;
|
| 19 |
+
}
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
struct CudaIPCGlobalEntities {
|
| 23 |
+
// This class is used as a singleton (see cuda_ipc_global_entities)
|
| 24 |
+
// This variable is used to track its lifetime to avoid accessing it
|
| 25 |
+
// after it was destroyed which would lead to segmentation faults
|
| 26 |
+
// Note that a trivial type is used which doesn't suffer from construction
|
| 27 |
+
// and destruction order issues
|
| 28 |
+
static bool alive;
|
| 29 |
+
|
| 30 |
+
std::mutex ref_counters_mutex_;
|
| 31 |
+
std::atomic<int64_t> sync_events_used_{0};
|
| 32 |
+
std::map<std::string, std::shared_ptr<CudaIPCRefCountersFile>>
|
| 33 |
+
ref_counters_files_;
|
| 34 |
+
std::shared_ptr<CudaIPCRefCountersFile> next_available_ref_counters_file_;
|
| 35 |
+
CudaIPCSentDataLimbo CudaIPCSentDataLimbo_;
|
| 36 |
+
CudaIPCGlobalEntities() {
|
| 37 |
+
alive = true;
|
| 38 |
+
}
|
| 39 |
+
CudaIPCGlobalEntities(const CudaIPCGlobalEntities&) = delete;
|
| 40 |
+
CudaIPCGlobalEntities(CudaIPCGlobalEntities&&) = delete;
|
| 41 |
+
CudaIPCGlobalEntities& operator=(const CudaIPCGlobalEntities&) = delete;
|
| 42 |
+
CudaIPCGlobalEntities& operator=(CudaIPCGlobalEntities&&) = delete;
|
| 43 |
+
~CudaIPCGlobalEntities() {
|
| 44 |
+
alive = false;
|
| 45 |
+
CudaIPCSentDataLimbo_.collect();
|
| 46 |
+
safe_clean_current_file();
|
| 47 |
+
if (next_available_ref_counters_file_) {
|
| 48 |
+
warnProducerTerminatedBeforeSharedTensorsReleased();
|
| 49 |
+
}
|
| 50 |
+
}
|
| 51 |
+
void safe_clean_current_file() {
|
| 52 |
+
std::lock_guard<std::mutex> lock(ref_counters_mutex_);
|
| 53 |
+
if (next_available_ref_counters_file_ &&
|
| 54 |
+
next_available_ref_counters_file_->offsets_in_use() == 0) {
|
| 55 |
+
ref_counters_files_.erase(next_available_ref_counters_file_->handle());
|
| 56 |
+
next_available_ref_counters_file_.reset();
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
};
|
| 60 |
+
|
| 61 |
+
bool CudaIPCGlobalEntities::alive = false;
|
| 62 |
+
CudaIPCGlobalEntities cuda_ipc_global_entities;
|
| 63 |
+
|
| 64 |
+
CudaIPCSentDataLimbo::~CudaIPCSentDataLimbo() {
|
| 65 |
+
collect();
|
| 66 |
+
if (size() > 0) {
|
| 67 |
+
warnProducerTerminatedBeforeSharedTensorsReleased();
|
| 68 |
+
}
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
bool CudaIPCSentDataLimbo::collect() {
|
| 72 |
+
bool freed_memory = false;
|
| 73 |
+
std::vector<std::unique_ptr<CudaIPCSentData>> reset_blocks;
|
| 74 |
+
{ // Begin critical section to modify shared blocks
|
| 75 |
+
std::lock_guard<std::mutex> lock(limbo_mutex_);
|
| 76 |
+
std::vector<std::unique_ptr<CudaIPCSentData>> kept_blocks;
|
| 77 |
+
for (auto& sd : shared_blocks_) {
|
| 78 |
+
if (sd->counter_value() > 0) {
|
| 79 |
+
kept_blocks.push_back(std::move(sd));
|
| 80 |
+
} else {
|
| 81 |
+
freed_memory = true;
|
| 82 |
+
reset_blocks.push_back(std::move(sd));
|
| 83 |
+
}
|
| 84 |
+
}
|
| 85 |
+
shared_blocks_ = std::move(kept_blocks);
|
| 86 |
+
}
|
| 87 |
+
// Need to reset blocks out of the critical section here, otherwise it
|
| 88 |
+
// deadlocks.
|
| 89 |
+
for (auto& sd : reset_blocks) {
|
| 90 |
+
sd.reset();
|
| 91 |
+
}
|
| 92 |
+
return freed_memory;
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
void CudaIPCSentDataLimbo::add(std::unique_ptr<CudaIPCSentData> shared_block) {
|
| 96 |
+
std::lock_guard<std::mutex> lock(limbo_mutex_);
|
| 97 |
+
static bool warned = false;
|
| 98 |
+
if (shared_blocks_.size() > CUDA_IPC_WARN_AFTER_X_BLOCKS_IN_LIMBO &&
|
| 99 |
+
!warned) {
|
| 100 |
+
LOG(WARNING)
|
| 101 |
+
<< "Producer process tried to deallocate over "
|
| 102 |
+
<< CUDA_IPC_WARN_AFTER_X_BLOCKS_IN_LIMBO
|
| 103 |
+
<< " memory blocks referred by consumer processes. Deallocation might be significantly slowed down. "
|
| 104 |
+
<< "We assume it will never going to be the case, but if it is, please file but to https://github.com/pytorch/pytorch";
|
| 105 |
+
warned = true;
|
| 106 |
+
}
|
| 107 |
+
shared_blocks_.push_back(std::move(shared_block));
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
uint64_t CudaIPCSentDataLimbo::size() {
|
| 111 |
+
std::lock_guard<std::mutex> lock(limbo_mutex_);
|
| 112 |
+
return shared_blocks_.size();
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
void CudaIPCSentDataDelete(void* ptr) {
|
| 116 |
+
std::unique_ptr<CudaIPCSentData> sent_data(
|
| 117 |
+
static_cast<CudaIPCSentData*>(ptr));
|
| 118 |
+
if (!CudaIPCGlobalEntities::alive) {
|
| 119 |
+
return;
|
| 120 |
+
}
|
| 121 |
+
if (sent_data->counter_value() > 0) {
|
| 122 |
+
cuda_ipc_global_entities.CudaIPCSentDataLimbo_.add(std::move(sent_data));
|
| 123 |
+
}
|
| 124 |
+
cuda_ipc_global_entities.CudaIPCSentDataLimbo_.collect();
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
void ReturnRefCounter(const std::string& handle, uint64_t offset /* unused */) {
|
| 128 |
+
if (!CudaIPCGlobalEntities::alive) {
|
| 129 |
+
return;
|
| 130 |
+
}
|
| 131 |
+
std::lock_guard<std::mutex> lock(
|
| 132 |
+
cuda_ipc_global_entities.ref_counters_mutex_);
|
| 133 |
+
auto& map = cuda_ipc_global_entities.ref_counters_files_;
|
| 134 |
+
auto it = map.find(handle);
|
| 135 |
+
if (it != map.end()) {
|
| 136 |
+
it->second->return_offset(offset);
|
| 137 |
+
if (it->second->offsets_in_use() == 0 && !it->second->have_offsets()) {
|
| 138 |
+
map.erase(handle);
|
| 139 |
+
}
|
| 140 |
+
}
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
} // namespace
|
| 144 |
+
|
| 145 |
+
CudaIPCSentData::CudaIPCSentData(
|
| 146 |
+
std::string handle,
|
| 147 |
+
uint64_t offset,
|
| 148 |
+
uint64_t* counter_ptr,
|
| 149 |
+
at::Device device)
|
| 150 |
+
: handle_(std::move(handle)),
|
| 151 |
+
offset_(offset),
|
| 152 |
+
counter_ptr_(counter_ptr),
|
| 153 |
+
device_(device) {
|
| 154 |
+
#if !defined(USE_ROCM)
|
| 155 |
+
// CUDA have the unofficial limit on the number of recorded blocking
|
| 156 |
+
// interprocess events, to prevent using of all events, we are switching to
|
| 157 |
+
// StreamSync before limit reached.
|
| 158 |
+
//
|
| 159 |
+
// ```python
|
| 160 |
+
// import torch
|
| 161 |
+
// a = [ torch.cuda.Event(
|
| 162 |
+
// enable_timing=False, blocking=True, interprocess=True) for i in
|
| 163 |
+
// range(30000) ]
|
| 164 |
+
// [i.record() for i in a]
|
| 165 |
+
// ```
|
| 166 |
+
//
|
| 167 |
+
if (cuda_ipc_global_entities.sync_events_used_.load() <
|
| 168 |
+
CUDA_IPC_MAXIMUM_EVENTS_TO_USE) {
|
| 169 |
+
// TODO: More efficient would be to create event inside of main thread (at
|
| 170 |
+
// the moment of the queue.put). The reason this is more efficient is
|
| 171 |
+
// because the main thread may have queued extra work on the stream, which
|
| 172 |
+
// this event will consequently wait for (uselessly).
|
| 173 |
+
cuda_ipc_global_entities.sync_events_used_++;
|
| 174 |
+
C10_CUDA_CHECK(cudaEventCreateWithFlags(
|
| 175 |
+
&event_,
|
| 176 |
+
cudaEventDisableTiming | cudaEventInterprocess |
|
| 177 |
+
cudaEventBlockingSync));
|
| 178 |
+
C10_CUDA_CHECK(cudaEventRecord(
|
| 179 |
+
event_, c10::cuda::getCurrentCUDAStream(device.index())));
|
| 180 |
+
event_sync_required_ = true;
|
| 181 |
+
} else {
|
| 182 |
+
auto stream = c10::cuda::getCurrentCUDAStream(device.index());
|
| 183 |
+
at::cuda::stream_synchronize(stream);
|
| 184 |
+
event_ = nullptr;
|
| 185 |
+
event_sync_required_ = false;
|
| 186 |
+
}
|
| 187 |
+
#else
|
| 188 |
+
// cuIpcGetEventHandle with HIP is not supported, so we have to sync
|
| 189 |
+
// stream instead of passing event
|
| 190 |
+
auto stream = c10::cuda::getCurrentCUDAStream(device.index());
|
| 191 |
+
at::cuda::stream_synchronize(stream);
|
| 192 |
+
event_sync_required_ = false;
|
| 193 |
+
#endif
|
| 194 |
+
}
|
| 195 |
+
|
| 196 |
+
CudaIPCSentData::~CudaIPCSentData() {
|
| 197 |
+
if (!CudaIPCGlobalEntities::alive) {
|
| 198 |
+
original_ptr_.release_context();
|
| 199 |
+
}
|
| 200 |
+
ReturnRefCounter(handle_, offset_);
|
| 201 |
+
#if !defined(USE_ROCM)
|
| 202 |
+
try {
|
| 203 |
+
if (event_sync_required_) {
|
| 204 |
+
at::cuda::CUDAGuard device_guard(device_.index());
|
| 205 |
+
C10_CUDA_CHECK(cudaEventDestroy(event_));
|
| 206 |
+
if (!CudaIPCGlobalEntities::alive) {
|
| 207 |
+
return;
|
| 208 |
+
}
|
| 209 |
+
cuda_ipc_global_entities.sync_events_used_--;
|
| 210 |
+
}
|
| 211 |
+
// NOLINTNEXTLINE(bugprone-empty-catch)
|
| 212 |
+
} catch (...) { /* No throw */
|
| 213 |
+
}
|
| 214 |
+
#endif
|
| 215 |
+
}
|
| 216 |
+
|
| 217 |
+
uint64_t CudaIPCSentData::counter_value() {
|
| 218 |
+
return *counter_ptr_;
|
| 219 |
+
}
|
| 220 |
+
|
| 221 |
+
at::DataPtr GetNewRefCountedSentData(void* data, at::Device device) {
|
| 222 |
+
{
|
| 223 |
+
std::lock_guard<std::mutex> lock(
|
| 224 |
+
cuda_ipc_global_entities.ref_counters_mutex_);
|
| 225 |
+
if (!cuda_ipc_global_entities.next_available_ref_counters_file_) {
|
| 226 |
+
std::string ref_counter_handle = at::NewProcessWideShmHandle();
|
| 227 |
+
|
| 228 |
+
int flags =
|
| 229 |
+
at::ALLOCATOR_MAPPED_SHAREDMEM | at::ALLOCATOR_MAPPED_EXCLUSIVE;
|
| 230 |
+
at::DataPtr sptr = at::RefcountedMapAllocator::makeDataPtr(
|
| 231 |
+
ref_counter_handle.c_str(),
|
| 232 |
+
flags,
|
| 233 |
+
sizeof(int64_t) * CUDA_IPC_REF_COUNTER_FILE_SIZE,
|
| 234 |
+
nullptr);
|
| 235 |
+
auto rc = std::make_shared<CudaIPCRefCountersFile>(
|
| 236 |
+
ref_counter_handle, CUDA_IPC_REF_COUNTER_FILE_SIZE, std::move(sptr));
|
| 237 |
+
cuda_ipc_global_entities.ref_counters_files_[ref_counter_handle] = rc;
|
| 238 |
+
cuda_ipc_global_entities.next_available_ref_counters_file_ = rc;
|
| 239 |
+
}
|
| 240 |
+
}
|
| 241 |
+
cuda_ipc_global_entities.next_available_ref_counters_file_->set_counter(1);
|
| 242 |
+
auto sent_data = new CudaIPCSentData(
|
| 243 |
+
cuda_ipc_global_entities.next_available_ref_counters_file_->handle(),
|
| 244 |
+
cuda_ipc_global_entities.next_available_ref_counters_file_->get_offset(),
|
| 245 |
+
cuda_ipc_global_entities.next_available_ref_counters_file_->counter_ptr(),
|
| 246 |
+
device);
|
| 247 |
+
|
| 248 |
+
cuda_ipc_global_entities.next_available_ref_counters_file_->rotate_offset();
|
| 249 |
+
if (!cuda_ipc_global_entities.next_available_ref_counters_file_
|
| 250 |
+
->have_offsets()) {
|
| 251 |
+
cuda_ipc_global_entities.next_available_ref_counters_file_.reset();
|
| 252 |
+
}
|
| 253 |
+
return at::DataPtr(data, sent_data, CudaIPCSentDataDelete, device);
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
bool CudaIPCCollect() {
|
| 257 |
+
if (!CudaIPCGlobalEntities::alive) {
|
| 258 |
+
return true;
|
| 259 |
+
}
|
| 260 |
+
bool freed_memory = cuda_ipc_global_entities.CudaIPCSentDataLimbo_.collect();
|
| 261 |
+
if (cuda_ipc_global_entities.CudaIPCSentDataLimbo_.size() == 0) {
|
| 262 |
+
cuda_ipc_global_entities.safe_clean_current_file();
|
| 263 |
+
}
|
| 264 |
+
return freed_memory;
|
| 265 |
+
}
|
| 266 |
+
|
| 267 |
+
} // namespace torch
|
| 268 |
+
|
| 269 |
+
namespace c10 {
|
| 270 |
+
namespace {
|
| 271 |
+
REGISTER_FREE_MEMORY_CALLBACK("cuda_ipc_collect", CudaIPCCollectCallback)
|
| 272 |
+
}
|
| 273 |
+
} // namespace c10
|