luclima commited on
Commit
9d05fac
·
verified ·
1 Parent(s): c454bdd

Upload folder using huggingface_hub (part 39)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. western_ai/usa/pytorch/torch/CMakeLists.txt +568 -0
  2. western_ai/usa/pytorch/torch/autograd/_functions/replace.vim +3 -0
  3. western_ai/usa/pytorch/torch/autograd/_functions/tensor.py +72 -0
  4. western_ai/usa/pytorch/torch/autograd/_functions/utils.py +26 -0
  5. western_ai/usa/pytorch/torch/autograd/anomaly_mode.py +123 -0
  6. western_ai/usa/pytorch/torch/autograd/forward_ad.py +241 -0
  7. western_ai/usa/pytorch/torch/autograd/function.py +898 -0
  8. western_ai/usa/pytorch/torch/autograd/functional.py +1212 -0
  9. western_ai/usa/pytorch/torch/autograd/grad_mode.py +489 -0
  10. western_ai/usa/pytorch/torch/autograd/gradcheck.py +2324 -0
  11. western_ai/usa/pytorch/torch/autograd/graph.py +989 -0
  12. western_ai/usa/pytorch/torch/autograd/profiler.py +1799 -0
  13. western_ai/usa/pytorch/torch/autograd/profiler_legacy.py +327 -0
  14. western_ai/usa/pytorch/torch/autograd/profiler_util.py +1617 -0
  15. western_ai/usa/pytorch/torch/autograd/variable.py +15 -0
  16. western_ai/usa/pytorch/torch/backends/__init__.py +140 -0
  17. western_ai/usa/pytorch/torch/backends/_coreml/__init__.py +0 -0
  18. western_ai/usa/pytorch/torch/backends/_coreml/preprocess.py +154 -0
  19. western_ai/usa/pytorch/torch/backends/_nnapi/__init__.py +0 -0
  20. western_ai/usa/pytorch/torch/backends/_nnapi/prepare.py +208 -0
  21. western_ai/usa/pytorch/torch/backends/_nnapi/serializer.py +0 -0
  22. western_ai/usa/pytorch/torch/backends/cpu/__init__.py +21 -0
  23. western_ai/usa/pytorch/torch/backends/cuda/__init__.py +711 -0
  24. western_ai/usa/pytorch/torch/backends/cudnn/__init__.py +261 -0
  25. western_ai/usa/pytorch/torch/backends/cudnn/rnn.py +128 -0
  26. western_ai/usa/pytorch/torch/backends/cusparselt/__init__.py +57 -0
  27. western_ai/usa/pytorch/torch/backends/kleidiai/__init__.py +7 -0
  28. western_ai/usa/pytorch/torch/backends/mha/__init__.py +25 -0
  29. western_ai/usa/pytorch/torch/backends/miopen/__init__.py +50 -0
  30. western_ai/usa/pytorch/torch/backends/mkl/__init__.py +59 -0
  31. western_ai/usa/pytorch/torch/backends/mkldnn/__init__.py +139 -0
  32. western_ai/usa/pytorch/torch/backends/mps/__init__.py +60 -0
  33. western_ai/usa/pytorch/torch/backends/nnpack/__init__.py +32 -0
  34. western_ai/usa/pytorch/torch/backends/openmp/__init__.py +7 -0
  35. western_ai/usa/pytorch/torch/backends/opt_einsum/__init__.py +117 -0
  36. western_ai/usa/pytorch/torch/backends/python_native/__init__.py +387 -0
  37. western_ai/usa/pytorch/torch/backends/quantized/__init__.py +65 -0
  38. western_ai/usa/pytorch/torch/backends/xeon/__init__.py +0 -0
  39. western_ai/usa/pytorch/torch/backends/xeon/run_cpu.py +957 -0
  40. western_ai/usa/pytorch/torch/backends/xnnpack/__init__.py +29 -0
  41. western_ai/usa/pytorch/torch/compiler/__init__.py +984 -0
  42. western_ai/usa/pytorch/torch/compiler/_cache.py +342 -0
  43. western_ai/usa/pytorch/torch/compiler/config.py +344 -0
  44. western_ai/usa/pytorch/torch/contrib/__init__.py +0 -0
  45. western_ai/usa/pytorch/torch/contrib/_tensorboard_vis.py +152 -0
  46. western_ai/usa/pytorch/torch/cpu/__init__.py +252 -0
  47. western_ai/usa/pytorch/torch/cpu/amp/__init__.py +3 -0
  48. western_ai/usa/pytorch/torch/cpu/amp/autocast_mode.py +71 -0
  49. western_ai/usa/pytorch/torch/cpu/amp/grad_scaler.py +35 -0
  50. western_ai/usa/pytorch/torch/csrc/CudaIPCTypes.cpp +273 -0
western_ai/usa/pytorch/torch/CMakeLists.txt ADDED
@@ -0,0 +1,568 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # This file used to build libtorch.so.
2
+ # Now it only builds the Torch python bindings.
3
+
4
+ if(NOT CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO)
5
+ cmake_minimum_required(VERSION 3.27 FATAL_ERROR)
6
+ project(torch CXX C)
7
+ find_package(torch REQUIRED)
8
+ option(USE_CUDA "Use CUDA" ON)
9
+ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
10
+ endif()
11
+
12
+ if(NOT BUILD_PYTHON)
13
+ return()
14
+ endif()
15
+
16
+ set(TORCH_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
17
+ set(TORCH_ROOT "${TORCH_SRC_DIR}/..")
18
+
19
+ if(NOT TORCH_INSTALL_LIB_DIR)
20
+ set(TORCH_INSTALL_LIB_DIR lib)
21
+ endif()
22
+
23
+ if(MSVC)
24
+ set(LIBSHM_SUBDIR libshm_windows)
25
+ else()
26
+ set(LIBSHM_SUBDIR libshm)
27
+ endif()
28
+
29
+ set(LIBSHM_SRCDIR ${TORCH_SRC_DIR}/lib/${LIBSHM_SUBDIR})
30
+ add_subdirectory(${LIBSHM_SRCDIR})
31
+
32
+ # Generate files
33
+ set(TOOLS_PATH "${TORCH_ROOT}/tools")
34
+
35
+
36
+ set(TORCH_PYTHON_SRCS
37
+ ${GENERATED_CXX_PYTHON}
38
+ )
39
+ append_filelist("libtorch_python_core_sources" TORCH_PYTHON_SRCS)
40
+
41
+ # NB: This has to match the condition under which the JIT test directory
42
+ # is included (at the time of writing that's in caffe2/CMakeLists.txt).
43
+ if(BUILD_TEST)
44
+ add_definitions(-DBUILDING_TESTS)
45
+ list(APPEND TORCH_PYTHON_SRCS
46
+ ${TORCH_ROOT}/test/cpp/jit/torch_python_test.cpp
47
+ )
48
+ endif()
49
+
50
+ set(TORCH_PYTHON_INCLUDE_DIRECTORIES
51
+ ${PYTHON_INCLUDE_DIR}
52
+
53
+ ${TORCH_ROOT}
54
+ ${TORCH_ROOT}/aten/src
55
+
56
+ ${CMAKE_BINARY_DIR}
57
+ ${CMAKE_BINARY_DIR}/aten/src
58
+ ${CMAKE_BINARY_DIR}/caffe2/aten/src
59
+ ${CMAKE_BINARY_DIR}/third_party
60
+ ${CMAKE_BINARY_DIR}/third_party/onnx
61
+
62
+ ${TORCH_ROOT}/third_party/valgrind-headers
63
+
64
+ ${TORCH_ROOT}/third_party/gloo
65
+ ${TORCH_ROOT}/third_party/onnx
66
+ ${TORCH_ROOT}/third_party/flatbuffers/include
67
+ ${TORCH_ROOT}/third_party/kineto/libkineto/include
68
+ ${TORCH_ROOT}/third_party/cpp-httplib
69
+ ${TORCH_ROOT}/third_party/nlohmann/include
70
+
71
+ ${TORCH_SRC_DIR}/csrc
72
+ ${TORCH_SRC_DIR}/csrc/api/include
73
+ ${TORCH_SRC_DIR}/lib
74
+ ${TORCH_SRC_DIR}/standalone
75
+ )
76
+
77
+ list(APPEND TORCH_PYTHON_INCLUDE_DIRECTORIES ${LIBSHM_SRCDIR})
78
+
79
+ set(TORCH_PYTHON_LINK_LIBRARIES
80
+ Python::Module
81
+ pybind::pybind11
82
+ httplib
83
+ nlohmann
84
+ moodycamel
85
+ shm
86
+ fmt::fmt-header-only
87
+ ATEN_CPU_FILES_GEN_LIB)
88
+
89
+ if(USE_ASAN AND TARGET Sanitizer::address)
90
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::address)
91
+ endif()
92
+ # Skip UBSan on ROCm: see comment in cmake/Dependencies.cmake (UBSan + ASAN
93
+ # on ROCm Clang produces misaligned ASAN global metadata that aborts at
94
+ # startup).
95
+ if(USE_ASAN AND TARGET Sanitizer::undefined AND NOT USE_ROCM)
96
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::undefined)
97
+ endif()
98
+ if(USE_LSAN AND TARGET Sanitizer::leak)
99
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::leak)
100
+ endif()
101
+ if(USE_TSAN AND TARGET Sanitizer::thread)
102
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES Sanitizer::thread)
103
+ endif()
104
+
105
+ set(TORCH_PYTHON_COMPILE_DEFINITIONS)
106
+
107
+ set(TORCH_PYTHON_COMPILE_OPTIONS)
108
+
109
+ set(TORCH_PYTHON_LINK_FLAGS "")
110
+
111
+ if(MSVC)
112
+ string(APPEND TORCH_PYTHON_LINK_FLAGS " /NODEFAULTLIB:LIBCMT.LIB")
113
+ # Python headers use #pragma comment(lib, "pythonXX") which makes MSVC search
114
+ # for the library by bare name. Add the directory so the linker can find it.
115
+ foreach(_pylib IN LISTS Python_LIBRARIES)
116
+ if(_pylib MATCHES "python[0-9].*\\.lib$")
117
+ get_filename_component(_pylib_dir "${_pylib}" DIRECTORY)
118
+ string(APPEND TORCH_PYTHON_LINK_FLAGS " /LIBPATH:\"${_pylib_dir}\"")
119
+ break()
120
+ endif()
121
+ endforeach()
122
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES ${Python_LIBRARIES} onnx_library)
123
+ if(NOT CMAKE_BUILD_TYPE MATCHES "Release")
124
+ string(APPEND TORCH_PYTHON_LINK_FLAGS " /DEBUG:FULL")
125
+ endif()
126
+ elseif(APPLE)
127
+ string(APPEND TORCH_PYTHON_LINK_FLAGS " -undefined dynamic_lookup")
128
+ else()
129
+ list(APPEND TORCH_PYTHON_COMPILE_OPTIONS
130
+ -fno-strict-aliasing
131
+ -Wno-strict-aliasing)
132
+ endif()
133
+
134
+ if(USE_ITT)
135
+ list(APPEND TORCH_PYTHON_SRCS
136
+ ${TORCH_SRC_DIR}/csrc/itt.cpp
137
+ )
138
+ list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_ITT)
139
+ endif()
140
+
141
+ if(USE_CUDA)
142
+ include(${TORCH_ROOT}/cmake/public/cuda.cmake)
143
+ append_filelist("libtorch_python_cuda_core_sources" TORCH_PYTHON_SRCS)
144
+
145
+ list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUDA)
146
+ if(USE_CUDNN)
147
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::cudnn)
148
+ list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUDNN)
149
+ endif()
150
+ if(USE_CUSPARSELT)
151
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::cusparselt)
152
+ list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUSPARSELT)
153
+ endif()
154
+ if(USE_CUFILE)
155
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::cufile)
156
+ list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_CUFILE)
157
+ endif()
158
+
159
+ if(TARGET torch::nvtx3)
160
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::nvtx3)
161
+ else()
162
+ if(TARGET torch::nvtoolsext)
163
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES torch::nvtoolsext)
164
+ endif()
165
+ endif()
166
+ endif()
167
+
168
+ if(USE_ROCM)
169
+ append_filelist("libtorch_python_cuda_core_sources" TORCH_PYTHON_SRCS)
170
+
171
+ list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS
172
+ USE_ROCM
173
+ __HIP_PLATFORM_AMD__
174
+ )
175
+ if(NOT WIN32)
176
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES ${ROCM_ROCTX_LIB})
177
+ endif()
178
+ endif()
179
+
180
+ if(USE_XPU)
181
+ include(${TORCH_ROOT}/cmake/public/xpu.cmake)
182
+ append_filelist("libtorch_python_xpu_sources" TORCH_PYTHON_SRCS)
183
+ list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_XPU)
184
+ endif()
185
+
186
+ if(USE_CUDNN OR USE_ROCM)
187
+ list(APPEND TORCH_PYTHON_SRCS
188
+ ${TORCH_SRC_DIR}/csrc/cuda/shared/cudnn.cpp
189
+ )
190
+ if(USE_STATIC_CUDNN)
191
+ set_source_files_properties(
192
+ ${TORCH_SRC_DIR}/csrc/cuda/shared/cudnn.cpp
193
+ PROPERTIES COMPILE_DEFINITIONS "USE_STATIC_CUDNN"
194
+ )
195
+ endif()
196
+ endif()
197
+
198
+ if(USE_CUSPARSELT)
199
+ list(APPEND TORCH_PYTHON_SRCS ${TORCH_SRC_DIR}/csrc/cuda/shared/cusparselt.cpp)
200
+ endif()
201
+
202
+ if(USE_MPS)
203
+ list(APPEND TORCH_PYTHON_SRCS ${MPS_PYTHON_SRCS})
204
+ endif()
205
+
206
+ if(USE_VALGRIND AND NOT WIN32)
207
+ list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_VALGRIND)
208
+ endif()
209
+
210
+ add_custom_target(torch_python_stubs DEPENDS
211
+ "${TORCH_SRC_DIR}/_C/__init__.pyi"
212
+ "${TORCH_SRC_DIR}/_C/_VariableFunctions.pyi"
213
+ "${TORCH_SRC_DIR}/nn/functional.pyi"
214
+ "${TORCH_SRC_DIR}/utils/data/datapipes/datapipe.pyi"
215
+ )
216
+
217
+ file(GLOB_RECURSE torchgen_python CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/torchgen/*.py")
218
+ file(GLOB_RECURSE autograd_python CONFIGURE_DEPENDS "${TOOLS_PATH}/autograd/*.py")
219
+ file(GLOB_RECURSE pyi_python CONFIGURE_DEPENDS "${TOOLS_PATH}/pyi/*.py")
220
+ add_custom_command(
221
+ OUTPUT
222
+ "${TORCH_SRC_DIR}/_C/__init__.pyi"
223
+ "${TORCH_SRC_DIR}/_C/_VariableFunctions.pyi"
224
+ "${TORCH_SRC_DIR}/nn/functional.pyi"
225
+ COMMAND
226
+ "${Python_EXECUTABLE}" -mtools.pyi.gen_pyi
227
+ --native-functions-path "aten/src/ATen/native/native_functions.yaml"
228
+ --tags-path "aten/src/ATen/native/tags.yaml"
229
+ --deprecated-functions-path "tools/autograd/deprecated.yaml"
230
+ DEPENDS
231
+ "${TORCH_SRC_DIR}/_C/__init__.pyi.in"
232
+ "${TORCH_SRC_DIR}/_C/_VariableFunctions.pyi.in"
233
+ "${TORCH_SRC_DIR}/nn/functional.pyi.in"
234
+ "${TORCH_ROOT}/aten/src/ATen/native/native_functions.yaml"
235
+ "${TORCH_ROOT}/aten/src/ATen/native/tags.yaml"
236
+ "${TORCH_ROOT}/tools/autograd/deprecated.yaml"
237
+ "${TORCH_ROOT}/torch/_torch_docs.py"
238
+ "${TORCH_ROOT}/torch/_tensor_docs.py"
239
+ ${pyi_python}
240
+ ${autograd_python}
241
+ ${torchgen_python}
242
+ WORKING_DIRECTORY
243
+ "${TORCH_ROOT}"
244
+ )
245
+ file(GLOB_RECURSE datapipe_files CONFIGURE_DEPENDS "${TORCH_SRC_DIR}/utils/data/datapipes/*.py")
246
+ add_custom_command(
247
+ OUTPUT
248
+ "${TORCH_SRC_DIR}/utils/data/datapipes/datapipe.pyi"
249
+ COMMAND
250
+ ${CMAKE_COMMAND} -E env --modify PYTHONPATH=path_list_prepend:"${TORCH_ROOT}" --
251
+ "${Python_EXECUTABLE}" ${TORCH_SRC_DIR}/utils/data/datapipes/gen_pyi.py
252
+ DEPENDS
253
+ "${TORCH_SRC_DIR}/utils/data/datapipes/datapipe.pyi.in"
254
+ ${datapipe_files}
255
+ WORKING_DIRECTORY
256
+ "${TORCH_ROOT}"
257
+ )
258
+ if(USE_DISTRIBUTED)
259
+ if(WIN32)
260
+ append_filelist("libtorch_python_distributed_core_sources" TORCH_PYTHON_SRCS)
261
+ else()
262
+ append_filelist("libtorch_python_distributed_sources" TORCH_PYTHON_SRCS)
263
+ endif()
264
+ # Disable certain warnings for GCC-9.X
265
+ if(CMAKE_COMPILER_IS_GNUCXX)
266
+ set_source_files_properties(${TORCH_SRC_DIR}/csrc/distributed/autograd/init.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type")
267
+ set_source_files_properties(${TORCH_SRC_DIR}/csrc/distributed/rpc/testing/init.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type")
268
+ set_source_files_properties(${TORCH_SRC_DIR}/csrc/distributed/c10d/init.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type")
269
+ endif()
270
+ # NCCL is a private dependency of libtorch, but libtorch_python includes
271
+ # some private headers of libtorch, which in turn include NCCL. As a hacky
272
+ # alternative to making NCCL a public dependency of libtorch, we make it
273
+ # a private dependency of libtorch_python as well.
274
+ if(USE_NCCL)
275
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES __caffe2_nccl)
276
+ endif()
277
+ # Same for MPI.
278
+ if(USE_MPI)
279
+ list(APPEND TORCH_PYTHON_LINK_LIBRARIES MPI::MPI_CXX)
280
+ endif()
281
+ list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_C10D)
282
+
283
+ endif()
284
+
285
+ if(USE_NCCL AND NOT WIN32)
286
+ list(APPEND TORCH_PYTHON_SRCS
287
+ ${TORCH_SRC_DIR}/csrc/cuda/python_nccl.cpp)
288
+ list(APPEND TORCH_PYTHON_COMPILE_DEFINITIONS USE_NCCL)
289
+ endif()
290
+
291
+ # coreml
292
+ if(USE_COREML_DELEGATE)
293
+ list(APPEND TORCH_PYTHON_SRCS ${TORCH_SRC_DIR}/csrc/jit/backends/coreml/cpp/backend.cpp)
294
+ list(APPEND TORCH_PYTHON_SRCS ${TORCH_SRC_DIR}/csrc/jit/backends/coreml/cpp/preprocess.cpp)
295
+ endif()
296
+
297
+
298
+ add_library(torch_python SHARED ${TORCH_PYTHON_SRCS})
299
+ torch_compile_options(torch_python) # see cmake/public/utils.cmake
300
+ if(APPLE)
301
+ target_compile_options(torch_python PRIVATE
302
+ $<$<COMPILE_LANGUAGE:CXX>: -fvisibility=default>)
303
+ endif()
304
+
305
+ if(CAFFE2_USE_MKL AND BUILD_LIBTORCHLESS)
306
+
307
+ # Use the RPATH of the linked libraries
308
+ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
309
+ # we need to explicitly link caffe2::mkl in order to have the
310
+ # correct RPATH in torch_python for the split build
311
+ target_link_libraries(torch_python PRIVATE caffe2::mkl)
312
+ endif()
313
+
314
+ add_dependencies(torch_python onnx_proto)
315
+ # Avoid numpy for the DEPLOY build
316
+ if(USE_NUMPY)
317
+ target_link_libraries(torch_python PRIVATE Python::NumPy)
318
+ target_compile_definitions(torch_python PRIVATE USE_NUMPY)
319
+ endif()
320
+
321
+ if(USE_CUFILE AND NOT USE_ROCM)
322
+ target_compile_definitions(torch_python PRIVATE USE_CUFILE)
323
+ endif()
324
+
325
+ if(BUILD_LAZY_TS_BACKEND)
326
+ target_compile_definitions(torch_python PRIVATE BUILD_LAZY_TS_BACKEND)
327
+ endif()
328
+
329
+ if(HAVE_SOVERSION)
330
+ set_target_properties(torch_python PROPERTIES
331
+ VERSION ${TORCH_VERSION} SOVERSION ${TORCH_SOVERSION})
332
+ endif()
333
+
334
+ # in case of the split build we need to add compile definitions
335
+ if(BUILD_LIBTORCHLESS)
336
+
337
+ if(USE_UCC AND USE_C10D_UCC)
338
+ target_compile_definitions(torch_python PRIVATE USE_C10D_UCC)
339
+ endif()
340
+
341
+ if(USE_NCCL AND USE_C10D_NCCL)
342
+ target_compile_definitions(torch_python PRIVATE USE_C10D_NCCL)
343
+ endif()
344
+
345
+ if(USE_DISTRIBUTED)
346
+ target_compile_definitions(torch_python PRIVATE USE_DISTRIBUTED)
347
+ endif()
348
+
349
+ if(USE_MPI AND USE_C10D_MPI)
350
+ target_compile_definitions(torch_python PRIVATE USE_C10D_MPI)
351
+ endif()
352
+
353
+ if(USE_GLOO AND USE_C10D_GLOO)
354
+ target_compile_definitions(torch_python PRIVATE USE_C10D_GLOO)
355
+ endif()
356
+
357
+ if(NOT WIN32)
358
+ target_compile_definitions(torch_python PRIVATE USE_RPC)
359
+ endif()
360
+
361
+ if(USE_TENSORPIPE)
362
+ target_compile_definitions(torch_python PRIVATE USE_TENSORPIPE)
363
+ endif()
364
+
365
+ set(EXPERIMENTAL_SINGLE_THREAD_POOL "0" CACHE STRING
366
+ "Experimental option to use a single thread pool for inter- and intra-op parallelism")
367
+ if("${EXPERIMENTAL_SINGLE_THREAD_POOL}")
368
+ target_compile_definitions(torch_python PRIVATE "-DAT_EXPERIMENTAL_SINGLE_THREAD_POOL=1")
369
+ endif()
370
+
371
+ endif()
372
+
373
+ add_dependencies(torch_python torch_python_stubs)
374
+ add_dependencies(torch_python flatbuffers)
375
+
376
+
377
+ if(USE_PRECOMPILED_HEADERS)
378
+ target_precompile_headers(torch_python PRIVATE
379
+ "$<$<COMPILE_LANGUAGE:CXX>:ATen/ATen.h>")
380
+ endif()
381
+
382
+ # Required workaround for generated sources
383
+ # See https://samthursfield.wordpress.com/2015/11/21/cmake-dependencies-between-targets-and-files-and-custom-commands/#custom-commands-in-different-directories
384
+ add_dependencies(torch_python generate-torch-sources)
385
+ set_source_files_properties(
386
+ ${GENERATED_CXX_PYTHON}
387
+ PROPERTIES GENERATED TRUE
388
+ )
389
+
390
+ # Disable certain warnings for GCC-9.X
391
+ if(CMAKE_COMPILER_IS_GNUCXX)
392
+ set_source_files_properties(${TORCH_SRC_DIR}/csrc/Module.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type")
393
+ set_source_files_properties(${TORCH_SRC_DIR}/csrc/autograd/python_variable.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type")
394
+ endif()
395
+
396
+ # Preserve CUDA_GENCODE flags
397
+ if(USE_CUDA)
398
+ torch_cuda_get_nvcc_gencode_flag(_ARCH_FLAGS)
399
+ set_source_files_properties(${TORCH_SRC_DIR}/csrc/cuda/Module.cpp PROPERTIES COMPILE_FLAGS "-DCUDA_ARCH_FLAGS=\"${_ARCH_FLAGS_readable}\"")
400
+ endif()
401
+
402
+ # Preserve HIP arch flags
403
+ if(USE_ROCM)
404
+ string(REPLACE ";" " " PYTORCH_ROCM_ARCH_readable "${PYTORCH_ROCM_ARCH}")
405
+ set_source_files_properties(${TORCH_SRC_DIR}/csrc/cuda/Module.cpp PROPERTIES COMPILE_FLAGS "-DCUDA_ARCH_FLAGS=\"${PYTORCH_ROCM_ARCH_readable}\"")
406
+ endif()
407
+
408
+ # Preserve XPU arch flags
409
+ if(USE_XPU)
410
+ string(REPLACE "," " " _ARCH_FLAGS_readable "${TORCH_XPU_ARCH_LIST}")
411
+ set_source_files_properties(${TORCH_SRC_DIR}/csrc/xpu/Module.cpp PROPERTIES COMPILE_FLAGS "-DXPU_ARCH_FLAGS=\"${_ARCH_FLAGS_readable}\"")
412
+ endif()
413
+
414
+ target_compile_definitions(torch_python PRIVATE "-DTHP_BUILD_MAIN_LIB")
415
+
416
+ target_link_libraries(torch_python PRIVATE ${TORCH_LIB} ${TORCH_PYTHON_LINK_LIBRARIES})
417
+
418
+ target_compile_definitions(torch_python PRIVATE ${TORCH_PYTHON_COMPILE_DEFINITIONS})
419
+
420
+ target_compile_options(torch_python PRIVATE ${TORCH_PYTHON_COMPILE_OPTIONS})
421
+
422
+ target_include_directories(torch_python PUBLIC ${TORCH_PYTHON_INCLUDE_DIRECTORIES})
423
+
424
+ if(USE_UCC)
425
+ target_link_libraries(torch_python PRIVATE __caffe2_ucc)
426
+ target_compile_definitions(torch_python PRIVATE USE_UCC)
427
+ endif()
428
+
429
+ if(BUILD_ONEDNN_GRAPH)
430
+ target_compile_definitions(torch_python PRIVATE "-DBUILD_ONEDNN_GRAPH")
431
+ if(NOT BUILD_LIBTORCHLESS)
432
+ target_compile_definitions(torch_cpu PRIVATE "-DBUILD_ONEDNN_GRAPH")
433
+ endif()
434
+ endif()
435
+
436
+ if(${CMAKE_BUILD_TYPE} STREQUAL "RelWithAssert")
437
+ # Workaround numerous decret-without-a-gil warnings from JIT
438
+ # see https://github.com/pytorch/pytorch/issues/130073
439
+ target_compile_definitions(torch_python PRIVATE "-DPYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF")
440
+ endif()
441
+
442
+ if(NOT TORCH_PYTHON_LINK_FLAGS STREQUAL "")
443
+ set_target_properties(torch_python PROPERTIES LINK_FLAGS "${TORCH_PYTHON_LINK_FLAGS}")
444
+ endif()
445
+
446
+ install(TARGETS torch_python DESTINATION "${TORCH_INSTALL_LIB_DIR}")
447
+
448
+ # Build the torch._C Python extension module from the thin stub that
449
+ # forwards to torch_python. This was historically built by setuptools;
450
+ # building it here lets both setuptools and scikit-build-core use the
451
+ # same code path.
452
+ if(BUILD_PYTHON AND NOT BUILD_LIBTORCH_WHL)
453
+ Python_add_library(_C MODULE WITH_SOABI "${TORCH_SRC_DIR}/csrc/stub.c")
454
+ target_link_libraries(_C PRIVATE torch_python)
455
+ set_target_properties(_C PROPERTIES
456
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/torch"
457
+ # libtorch_python.so is installed to lib/ relative to _C, so use
458
+ # $ORIGIN/lib as the install RPATH (the global default is just $ORIGIN).
459
+ INSTALL_RPATH "${_rpath_portable_origin}/lib"
460
+ )
461
+ if(NOT WIN32)
462
+ set_target_properties(_C PROPERTIES
463
+ C_VISIBILITY_PRESET "default"
464
+ )
465
+ endif()
466
+ # Install at the install prefix root, which maps to torch/ in the wheel
467
+ # via wheel.install-dir = "torch". The import library (Windows _C.lib) goes
468
+ # alongside the other import libs in torch/lib/, matching the layout that
469
+ # setup.py used before _C moved to CMake.
470
+ install(TARGETS _C
471
+ RUNTIME DESTINATION "."
472
+ LIBRARY DESTINATION "."
473
+ ARCHIVE DESTINATION "lib"
474
+ )
475
+
476
+ # ---[ NCCL EP extension (torch._nccl_ep)
477
+ # Optional standalone extension holding the NCCL EP bindings, imported lazily
478
+ # by torch/distributed/_token_switch.py so libtorch_cuda never references
479
+ # ncclEp*. How it links libnccl_ep follows how torch links NCCL:
480
+ # - USE_SYSTEM_NCCL=OFF: __caffe2_nccl_ep is libnccl_ep.a, statically linked
481
+ # here -- self-contained (nccl* bind to torch_cuda's embedded NCCL; JIT
482
+ # headers baked from the submodule build dir; no nccl4py).
483
+ # - USE_SYSTEM_NCCL=ON: __caffe2_nccl_ep is libnccl_ep.so, NEEDED-linked here
484
+ # and resolved from the nccl4py wheel at runtime (RPATH to nccl/ep/lib; JIT
485
+ # headers also from nccl4py, so nothing is baked).
486
+ if(USE_NCCL_EP)
487
+ Python_add_library(_nccl_ep MODULE WITH_SOABI
488
+ "${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/nccl_ep.cu"
489
+ "${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/nccl_ep_pybind.cpp")
490
+ target_compile_definitions(_nccl_ep PRIVATE USE_NCCL_EP USE_C10D_NCCL)
491
+ # torch_python: pybind + Tensor/ProcessGroup casters; torch_cuda:
492
+ # ProcessGroupNCCL::getCommPtr, getNcclDataType, CUDA stream, embedded NCCL;
493
+ # __caffe2_nccl_ep: libnccl_ep (static .a or shared .so) + nccl_ep headers.
494
+ target_link_libraries(_nccl_ep PRIVATE
495
+ torch_python torch_cuda __caffe2_nccl_ep pybind::pybind11)
496
+ set_target_properties(_nccl_ep PROPERTIES
497
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/torch")
498
+ if(USE_SYSTEM_NCCL)
499
+ # Dynamic: resolve nccl4py's libnccl_ep.so (site-packages/nccl/ep/lib, one
500
+ # dir up from torch/ via $ORIGIN/..) at import.
501
+ set_target_properties(_nccl_ep PROPERTIES
502
+ INSTALL_RPATH "${_rpath_portable_origin}/lib;${_rpath_portable_origin}/../nccl/ep/lib")
503
+ else()
504
+ # Static: only torch's own libs needed; bake the in-tree JIT header dir so
505
+ # the extension self-configures NCCL_EP_HOME / NCCL_HOME at import.
506
+ target_compile_definitions(_nccl_ep PRIVATE
507
+ "NCCL_EP_JIT_HOME=\"${NCCL_EP_JIT_HOME}\"")
508
+ set_target_properties(_nccl_ep PROPERTIES
509
+ INSTALL_RPATH "${_rpath_portable_origin}/lib")
510
+ endif()
511
+ if(NOT WIN32)
512
+ set_target_properties(_nccl_ep PROPERTIES C_VISIBILITY_PRESET "default")
513
+ endif()
514
+ install(TARGETS _nccl_ep RUNTIME DESTINATION "." LIBRARY DESTINATION ".")
515
+ endif()
516
+ endif()
517
+
518
+ # Generate torch/version.py from the appropriate CMake cache variables.
519
+ if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
520
+ set(TORCH_VERSION_DEBUG 1)
521
+ else()
522
+ set(TORCH_VERSION_DEBUG 0)
523
+ endif()
524
+
525
+ set(CUDA_VERSION "")
526
+ if(CUDAToolkit_VERSION_MAJOR)
527
+ set(CUDA_VERSION "${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR}")
528
+ endif()
529
+
530
+ add_custom_target(
531
+ gen_torch_version ALL
532
+ "${Python_EXECUTABLE}" -m tools.generate_torch_version
533
+ --is-debug=${TORCH_VERSION_DEBUG}
534
+ --cuda-version=${CUDA_VERSION}
535
+ --hip-version=${HIP_VERSION_CLEAN}
536
+ --rocm-version=${ROCM_VERSION_DEV}
537
+ --xpu-version=${SYCL_COMPILER_VERSION}
538
+ BYPRODUCTS ${TORCH_SRC_DIR}/version.py
539
+ COMMENT "Regenerating version file..."
540
+ WORKING_DIRECTORY ${TORCH_ROOT}
541
+ )
542
+ add_dependencies(torch_python gen_torch_version)
543
+ # version.py is gitignored (generated), so scikit-build-core won't include it
544
+ # via Python package collection. Install it explicitly via cmake instead.
545
+ install(FILES "${TORCH_SRC_DIR}/version.py" DESTINATION ".")
546
+
547
+ # Skip building this library under MacOS, since it is currently failing to build on Mac
548
+ # Github issue #61930
549
+ if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
550
+ # Add Android Nnapi delegate library
551
+ add_library(nnapi_backend SHARED
552
+ ${TORCH_SRC_DIR}/csrc/jit/backends/nnapi/nnapi_backend_lib.cpp
553
+ ${TORCH_SRC_DIR}/csrc/jit/backends/nnapi/nnapi_backend_preprocess.cpp
554
+ )
555
+ # Pybind11 requires explicit linking of the torch_python library
556
+ if(BUILD_LIBTORCHLESS)
557
+ target_link_libraries(nnapi_backend PRIVATE ${TORCH_LIB})
558
+ else()
559
+ target_link_libraries(nnapi_backend PRIVATE torch)
560
+ endif()
561
+ target_link_libraries(nnapi_backend PRIVATE torch_python pybind::pybind11 fmt::fmt-header-only)
562
+ if(MSVC AND TORCH_PYTHON_LINK_FLAGS)
563
+ set_target_properties(nnapi_backend PROPERTIES LINK_FLAGS "${TORCH_PYTHON_LINK_FLAGS}")
564
+ endif()
565
+ endif()
566
+
567
+ set(TORCH_PYTHON_COMPILE_OPTIONS ${TORCH_PYTHON_COMPILE_OPTIONS} PARENT_SCOPE)
568
+ set(TORCH_PYTHON_LINK_FLAGS ${TORCH_PYTHON_LINK_FLAGS} PARENT_SCOPE)
western_ai/usa/pytorch/torch/autograd/_functions/replace.vim ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ %s/self/ctx/g
2
+ %s/\s\+def forward/ @staticmethod\r def forward/g
3
+ %s/\s\+def backward/ @staticmethod\r @once_differentiable\r def backward/g
western_ai/usa/pytorch/torch/autograd/_functions/tensor.py ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import operator
3
+ from functools import reduce
4
+ from typing_extensions import deprecated
5
+
6
+ import torch
7
+ import torch._utils
8
+ from torch.autograd.function import Function
9
+
10
+
11
+ class Type(Function):
12
+ @staticmethod
13
+ @deprecated(
14
+ "`torch.autograd._functions.Type` is deprecated as of PyTorch 2.1, "
15
+ "please use `torch.tensor.to(dtype=dtype)` instead.",
16
+ category=FutureWarning,
17
+ )
18
+ # pyrefly: ignore [bad-override]
19
+ def forward(ctx, i, dest_type):
20
+ ctx.input_type = type(i)
21
+ ctx.input_device = -1 if not i.is_cuda else i.get_device()
22
+ return i.type(dest_type)
23
+
24
+ @staticmethod
25
+ # pyrefly: ignore [bad-override]
26
+ def backward(ctx, grad_output):
27
+ if ctx.input_device == -1:
28
+ return grad_output.type(ctx.input_type), None
29
+ else:
30
+ with torch.accelerator.device_index(ctx.input_device):
31
+ return grad_output.type(ctx.input_type), None
32
+
33
+
34
+ # TODO: deprecate this
35
+ class Resize(Function):
36
+ @staticmethod
37
+ # pyrefly: ignore [bad-override]
38
+ def forward(ctx, tensor, sizes):
39
+ ctx.sizes = sizes
40
+ ctx.numel = reduce(operator.mul, sizes, 1)
41
+ if tensor.numel() != ctx.numel:
42
+ raise RuntimeError(
43
+ (
44
+ "requested resize to {} ({} elements in total), "
45
+ "but the given tensor has a size of {} ({} elements). "
46
+ "autograd's resize can only change the shape of a given "
47
+ "tensor, while preserving the number of elements. "
48
+ ).format(
49
+ "x".join(map(str, sizes)),
50
+ ctx.numel,
51
+ "x".join(map(str, tensor.size())),
52
+ tensor.numel(),
53
+ )
54
+ )
55
+ ctx.input_sizes = tensor.size()
56
+ if tensor.is_quantized:
57
+ tensor.copy_(tensor)
58
+ return tensor.contiguous().view(*sizes)
59
+ if tensor.is_contiguous():
60
+ result = tensor.new(tensor).contiguous().view(*sizes)
61
+ return result
62
+ else:
63
+ return tensor.contiguous().view(*sizes)
64
+
65
+ @staticmethod
66
+ # pyrefly: ignore [bad-override]
67
+ def backward(ctx, grad_output):
68
+ if grad_output.numel() != ctx.numel:
69
+ raise AssertionError(
70
+ f"Expected grad_output to have {ctx.numel} elements, but got {grad_output.numel()}"
71
+ )
72
+ return grad_output.contiguous().view(ctx.input_sizes), None
western_ai/usa/pytorch/torch/autograd/_functions/utils.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+
3
+
4
+ def maybe_view(tensor, size, check_same_size=True):
5
+ if check_same_size and tensor.size() == size:
6
+ return tensor
7
+ return tensor.contiguous().view(size)
8
+
9
+
10
+ def maybe_unexpand(tensor, old_size, check_same_size=True):
11
+ if check_same_size and tensor.size() == old_size:
12
+ return tensor
13
+ num_unsqueezed = tensor.dim() - len(old_size)
14
+ expanded_dims = [
15
+ dim
16
+ for dim, (expanded, original) in enumerate(
17
+ zip(tensor.size()[num_unsqueezed:], old_size)
18
+ )
19
+ if expanded != original
20
+ ]
21
+
22
+ for _ in range(num_unsqueezed):
23
+ tensor = tensor.sum(0, keepdim=False)
24
+ for dim in expanded_dims:
25
+ tensor = tensor.sum(dim, keepdim=True)
26
+ return tensor
western_ai/usa/pytorch/torch/autograd/anomaly_mode.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ r"""Autograd anomaly mode."""
3
+
4
+ import warnings
5
+
6
+ import torch
7
+
8
+
9
+ __all__ = ["detect_anomaly", "set_detect_anomaly"]
10
+
11
+
12
+ class detect_anomaly:
13
+ r"""Context-manager that enable anomaly detection for the autograd engine.
14
+
15
+ This does two things:
16
+
17
+ - Running the forward pass with detection enabled will allow the backward
18
+ pass to print the traceback of the forward operation that created the failing
19
+ backward function.
20
+ - If ``check_nan`` is ``True``, any backward computation that generate "nan"
21
+ value will raise an error. Default ``True``.
22
+
23
+ .. warning::
24
+ This mode should be enabled only for debugging as the different tests
25
+ will slow down your program execution.
26
+
27
+ Example:
28
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_ANOMALY)
29
+ >>> import torch
30
+ >>> from torch import autograd
31
+ >>> class MyFunc(autograd.Function):
32
+ ... @staticmethod
33
+ ... def forward(ctx, inp):
34
+ ... return inp.clone()
35
+ ...
36
+ ... @staticmethod
37
+ ... def backward(ctx, gO):
38
+ ... # Error during the backward pass
39
+ ... raise RuntimeError("Some error in backward")
40
+ ... return gO.clone()
41
+ >>> def run_fn(a):
42
+ ... out = MyFunc.apply(a)
43
+ ... return out.sum()
44
+ >>> inp = torch.rand(10, 10, requires_grad=True)
45
+ >>> out = run_fn(inp)
46
+ >>> out.backward()
47
+ Traceback (most recent call last):
48
+ File "<stdin>", line 1, in <module>
49
+ File "/your/pytorch/install/torch/_tensor.py", line 93, in backward
50
+ torch.autograd.backward(self, gradient, retain_graph, create_graph)
51
+ File "/your/pytorch/install/torch/autograd/__init__.py", line 90, in backward
52
+ allow_unreachable=True) # allow_unreachable flag
53
+ File "/your/pytorch/install/torch/autograd/function.py", line 76, in apply
54
+ return self._forward_cls.backward(self, *args)
55
+ File "<stdin>", line 8, in backward
56
+ RuntimeError: Some error in backward
57
+ >>> with autograd.detect_anomaly():
58
+ ... inp = torch.rand(10, 10, requires_grad=True)
59
+ ... out = run_fn(inp)
60
+ ... out.backward()
61
+ Traceback of forward call that caused the error:
62
+ File "tmp.py", line 53, in <module>
63
+ out = run_fn(inp)
64
+ File "tmp.py", line 44, in run_fn
65
+ out = MyFunc.apply(a)
66
+ Traceback (most recent call last):
67
+ File "<stdin>", line 4, in <module>
68
+ File "/your/pytorch/install/torch/_tensor.py", line 93, in backward
69
+ torch.autograd.backward(self, gradient, retain_graph, create_graph)
70
+ File "/your/pytorch/install/torch/autograd/__init__.py", line 90, in backward
71
+ allow_unreachable=True) # allow_unreachable flag
72
+ File "/your/pytorch/install/torch/autograd/function.py", line 76, in apply
73
+ return self._forward_cls.backward(self, *args)
74
+ File "<stdin>", line 8, in backward
75
+ RuntimeError: Some error in backward
76
+
77
+ """
78
+
79
+ def __init__(self, check_nan=True) -> None:
80
+ self.prev = torch.is_anomaly_enabled()
81
+ self.check_nan = check_nan
82
+ self.prev_check_nan = torch.is_anomaly_check_nan_enabled()
83
+ warnings.warn(
84
+ "Anomaly Detection has been enabled. "
85
+ "This mode will increase the runtime "
86
+ "and should only be enabled for debugging.",
87
+ stacklevel=2,
88
+ )
89
+
90
+ def __enter__(self) -> None:
91
+ torch.set_anomaly_enabled(True, self.check_nan)
92
+
93
+ def __exit__(self, *args: object) -> None:
94
+ torch.set_anomaly_enabled(self.prev, self.prev_check_nan)
95
+
96
+
97
+ class set_detect_anomaly:
98
+ r"""Context-manager that sets the anomaly detection for the autograd engine on or off.
99
+
100
+ ``set_detect_anomaly`` will enable or disable the autograd anomaly detection
101
+ based on its argument :attr:`mode`.
102
+ It can be used as a context-manager or as a function.
103
+
104
+ See ``detect_anomaly`` above for details of the anomaly detection behaviour.
105
+
106
+ Args:
107
+ mode (bool): Flag whether to enable anomaly detection (``True``),
108
+ or disable (``False``).
109
+ check_nan (bool): Flag whether to raise an error when the backward
110
+ generate "nan"
111
+
112
+ """
113
+
114
+ def __init__(self, mode: bool, check_nan: bool = True) -> None:
115
+ self.prev = torch.is_anomaly_enabled()
116
+ self.prev_check_nan = torch.is_anomaly_check_nan_enabled()
117
+ torch.set_anomaly_enabled(mode, check_nan)
118
+
119
+ def __enter__(self) -> None:
120
+ pass
121
+
122
+ def __exit__(self, *args: object) -> None:
123
+ torch.set_anomaly_enabled(self.prev, self.prev_check_nan)
western_ai/usa/pytorch/torch/autograd/forward_ad.py ADDED
@@ -0,0 +1,241 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import os
3
+ from typing import Any, NamedTuple
4
+
5
+ import torch
6
+
7
+ from .grad_mode import _DecoratorContextManager
8
+
9
+
10
+ __all__ = [
11
+ "UnpackedDualTensor",
12
+ "enter_dual_level",
13
+ "exit_dual_level",
14
+ "make_dual",
15
+ "unpack_dual",
16
+ "dual_level",
17
+ ]
18
+
19
+ # Global variable used to make the python API simpler to use
20
+ _current_level = -1
21
+
22
+
23
+ def enter_dual_level():
24
+ r"""Enter a new forward grad level.
25
+
26
+ This level can be used to make and unpack dual Tensors to compute
27
+ forward gradients.
28
+
29
+ This function also updates the current level that is used by default
30
+ by the other functions in this API.
31
+ """
32
+ from torch._functorch.predispatch import (
33
+ _enter_dual_level as _predispatch_enter_dual_level,
34
+ )
35
+
36
+ global _current_level
37
+ new_level = _predispatch_enter_dual_level()
38
+ if new_level != _current_level + 1:
39
+ raise RuntimeError(
40
+ "Entering a new forward AD level but the current level "
41
+ "is not valid. Make sure you did not modify it directly."
42
+ )
43
+ _current_level = new_level
44
+ return new_level
45
+
46
+
47
+ def exit_dual_level(*, level=None):
48
+ r"""Exit a forward grad level.
49
+
50
+ This function deletes all the gradients associated with this
51
+ level. Only deleting the latest entered level is allowed.
52
+
53
+ This function also updates the current level that is used by default
54
+ by the other functions in this API.
55
+ """
56
+ global _current_level
57
+ if level is None:
58
+ level = _current_level
59
+ if level != _current_level:
60
+ raise RuntimeError(
61
+ "Trying to exit a forward AD level that was not the last one "
62
+ "that was created. This is not supported."
63
+ )
64
+ from torch._functorch.predispatch import (
65
+ _exit_dual_level as _predispatch_exit_dual_level,
66
+ )
67
+
68
+ _predispatch_exit_dual_level(level=level)
69
+ _current_level = level - 1
70
+
71
+
72
+ def _maybe_load_decompositions():
73
+ if os.environ.get("PYTORCH_JIT", "1") == "1" and __debug__:
74
+ from torch._decomp import decompositions_for_jvp # noqa: F401
75
+
76
+
77
+ def make_dual(tensor, tangent, *, level=None):
78
+ r"""Associate a tensor value with its tangent to create a "dual tensor" for forward AD gradient computation.
79
+
80
+ The result is a new tensor aliased to :attr:`tensor` with :attr:`tangent` embedded
81
+ as an attribute as-is if it has the same storage layout or copied otherwise.
82
+ The tangent attribute can be recovered with :func:`unpack_dual`.
83
+
84
+ This function is backward differentiable.
85
+
86
+ Given a function `f` whose jacobian is `J`, it allows one to compute the Jacobian-vector product (`jvp`)
87
+ between `J` and a given vector `v` as follows.
88
+
89
+ Example::
90
+
91
+ >>> # xdoctest: +SKIP("Undefined variables")
92
+ >>> with dual_level():
93
+ ... inp = make_dual(x, v)
94
+ ... out = f(inp)
95
+ ... y, jvp = unpack_dual(out)
96
+
97
+ Please see the `forward-mode AD tutorial <https://pytorch.org/tutorials/intermediate/forward_ad_usage.html>`__
98
+ for detailed steps on how to use this API.
99
+
100
+ """
101
+ # See NOTE: [forward-mode AD decompositions mechanism]
102
+ #
103
+ # Import from torch._decomp import decompositions_for_jvp to register
104
+ # decompositions for jvp to the jit registry
105
+ #
106
+ # FIXME: We specify that __debug__ must be True because
107
+ # if python is run with -OO or -O flags (i.e., __debug__ is False), we encounter the
108
+ # following error:
109
+ #
110
+ # Return value was annotated as having type Tuple[NoneType, NoneType] but is actually of
111
+ # type Tuple[Tensor, Tensor]:
112
+ # File ".../torch/_decomp/__init__.py", line 1585
113
+ # else:
114
+ # buffer = z
115
+ # return min - torch.log1p(z), buffer
116
+ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE
117
+ _maybe_load_decompositions()
118
+
119
+ if level is None:
120
+ level = _current_level
121
+
122
+ if level < 0:
123
+ raise RuntimeError(
124
+ "Trying to create a dual Tensor for forward AD but no level "
125
+ "exists, make sure to enter_dual_level() first."
126
+ )
127
+ if not (tensor.is_floating_point() or tensor.is_complex()):
128
+ raise ValueError(
129
+ f"Expected primal to be floating point or complex, but got: {tensor.dtype}"
130
+ )
131
+ if not (tangent.is_floating_point() or tangent.is_complex()):
132
+ raise ValueError(
133
+ f"Expected tangent to be floating point or complex, but got: {tangent.dtype}"
134
+ )
135
+
136
+ from torch._functorch.predispatch import _make_dual as _predispatch_make_dual
137
+
138
+ return _predispatch_make_dual(tensor, tangent, level=level)
139
+
140
+
141
+ class UnpackedDualTensor(NamedTuple):
142
+ r"""Namedtuple returned by :func:`unpack_dual` containing the primal and tangent components of the dual tensor.
143
+
144
+ See :func:`unpack_dual` for more details.
145
+ """
146
+
147
+ primal: torch.Tensor
148
+ tangent: torch.Tensor | None
149
+
150
+
151
+ def unpack_dual(tensor, *, level=None):
152
+ r"""Unpack a "dual tensor" to get both its Tensor value and its forward AD gradient.
153
+
154
+ The result is a namedtuple ``(primal, tangent)`` where ``primal`` is a view of
155
+ :attr:`tensor`'s primal and ``tangent`` is :attr:`tensor`'s tangent as-is.
156
+ Neither of these tensors can be dual tensor of level :attr:`level`.
157
+
158
+ This function is backward differentiable.
159
+
160
+ Example::
161
+
162
+ >>> # xdoctest: +SKIP("Undefined variables")
163
+ >>> with dual_level():
164
+ ... inp = make_dual(x, x_t)
165
+ ... out = f(inp)
166
+ ... y, jvp = unpack_dual(out)
167
+ ... jvp = unpack_dual(out).tangent
168
+
169
+ Please see the `forward-mode AD tutorial <https://pytorch.org/tutorials/intermediate/forward_ad_usage.html>`__
170
+ for detailed steps on how to use this API.
171
+ """
172
+ if level is None:
173
+ level = _current_level
174
+
175
+ if level < 0:
176
+ return UnpackedDualTensor(tensor, None)
177
+
178
+ from torch._functorch.predispatch import _unpack_dual as _predispatch_unpack_dual
179
+
180
+ primal, dual = _predispatch_unpack_dual(tensor, level=level)
181
+
182
+ return UnpackedDualTensor(primal, dual)
183
+
184
+
185
+ class dual_level(_DecoratorContextManager):
186
+ r"""Context-manager for forward AD, where all forward AD computation must occur within the ``dual_level`` context.
187
+
188
+ .. Note::
189
+
190
+ The ``dual_level`` context appropriately enters and exit the dual level to
191
+ controls the current forward AD level, which is used by default by the other
192
+ functions in this API.
193
+
194
+ We currently don't plan to support nested ``dual_level`` contexts, however, so
195
+ only a single forward AD level is supported. To compute higher-order
196
+ forward grads, one can use :func:`torch.func.jvp`.
197
+
198
+ Example::
199
+
200
+ >>> # xdoctest: +SKIP("Undefined variables")
201
+ >>> x = torch.tensor([1])
202
+ >>> x_t = torch.tensor([1])
203
+ >>> with dual_level():
204
+ ... inp = make_dual(x, x_t)
205
+ ... # Do computations with inp
206
+ ... out = your_fn(inp)
207
+ ... _, grad = unpack_dual(out)
208
+ >>> grad is None
209
+ False
210
+ >>> # After exiting the level, the grad is deleted
211
+ >>> _, grad_after = unpack_dual(out)
212
+ >>> grad is None
213
+ True
214
+
215
+ Please see the `forward-mode AD tutorial <https://pytorch.org/tutorials/intermediate/forward_ad_usage.html>`__
216
+ for detailed steps on how to use this API.
217
+ """
218
+
219
+ def __enter__(self):
220
+ return enter_dual_level()
221
+
222
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
223
+ exit_dual_level()
224
+
225
+
226
+ # Private helper functions
227
+ _is_fwd_grad_enabled = torch._C._is_fwd_grad_enabled
228
+
229
+
230
+ # Private helper function to enable or disable fwd grad.
231
+ # If you're a user and want to use this, please file an issue to discuss the use case.
232
+ class _set_fwd_grad_enabled(_DecoratorContextManager):
233
+ def __init__(self, mode: bool) -> None:
234
+ self.prev = _is_fwd_grad_enabled()
235
+ torch._C._set_fwd_grad_enabled(mode)
236
+
237
+ def __enter__(self) -> None:
238
+ pass
239
+
240
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
241
+ torch._C._set_fwd_grad_enabled(self.prev)
western_ai/usa/pytorch/torch/autograd/function.py ADDED
@@ -0,0 +1,898 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import functools
3
+ import inspect
4
+ import itertools
5
+ import warnings
6
+ from collections import OrderedDict
7
+ from collections.abc import Callable
8
+ from typing import Any, Concatenate, TypeVar
9
+ from typing_extensions import deprecated, ParamSpec
10
+
11
+ import torch
12
+ import torch._C as _C
13
+ import torch._functorch as _functorch
14
+ import torch.utils.hooks as hooks
15
+ from torch._C import _functions
16
+ from torch._functorch.autograd_function import custom_function_call
17
+
18
+
19
+ __all__ = [
20
+ "FunctionCtx",
21
+ "BackwardCFunction",
22
+ "FunctionMeta",
23
+ "Function",
24
+ "once_differentiable",
25
+ "InplaceFunction",
26
+ "NestedIOFunction",
27
+ ]
28
+
29
+ # Unique id provider for each class inheriting from Function
30
+ # This is incremented in FunctionMeta during class definition
31
+ AUTOGRAD_FUNCTION_COUNTER = itertools.count()
32
+
33
+ _T = TypeVar("_T")
34
+ _R = TypeVar("_R")
35
+ _P = ParamSpec("_P")
36
+
37
+
38
+ # Formerly known as: _ContextMethodMixin
39
+ class FunctionCtx:
40
+ def save_for_backward(self, *tensors: torch.Tensor):
41
+ r"""Save given tensors for a future call to :func:`~Function.backward`.
42
+
43
+ ``save_for_backward`` should be called at most once, in either the
44
+ :func:`setup_context` or :func:`forward` methods, and only with tensors.
45
+
46
+ All tensors intended to be used in the backward pass should be saved
47
+ with ``save_for_backward`` (as opposed to directly on ``ctx``) to prevent
48
+ incorrect gradients and memory leaks, and enable the application of saved
49
+ tensor hooks. See :class:`torch.autograd.graph.saved_tensors_hooks`.
50
+ See :ref:`extending-autograd` for more details.
51
+
52
+ Note that if intermediary tensors, tensors that are neither inputs
53
+ nor outputs of :func:`forward`, are saved for backward, your custom Function
54
+ may not support double backward.
55
+ Custom Functions that do not support double backward should decorate their
56
+ :func:`backward` method with ``@once_differentiable`` so that performing
57
+ double backward raises an error. If you'd like to support double backward,
58
+ you can either recompute intermediaries based on the inputs during backward
59
+ or return the intermediaries as the outputs of the custom Function. See the
60
+ `double backward tutorial <https://pytorch.org/tutorials/intermediate/custom_function_double_backward_tutorial.html>`_
61
+ for more details.
62
+
63
+ In :func:`backward`, saved tensors can be accessed through the :attr:`saved_tensors`
64
+ attribute. Before returning them to the user, a check is made to ensure
65
+ they weren't used in any in-place operation that modified their content.
66
+
67
+ Arguments can also be ``None``. This is a no-op.
68
+
69
+ See :ref:`extending-autograd` for more details on how to use this method.
70
+
71
+ Example::
72
+
73
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
74
+ >>> class Func(Function):
75
+ >>> @staticmethod
76
+ >>> def forward(ctx, x: torch.Tensor, y: torch.Tensor, z: int):
77
+ >>> w = x * z
78
+ >>> out = x * y + y * z + w * y
79
+ >>> ctx.save_for_backward(x, y, w, out)
80
+ >>> ctx.z = z # z is not a tensor
81
+ >>> return out
82
+ >>>
83
+ >>> @staticmethod
84
+ >>> @once_differentiable
85
+ >>> def backward(ctx, grad_out):
86
+ >>> x, y, w, out = ctx.saved_tensors
87
+ >>> z = ctx.z
88
+ >>> gx = grad_out * (y + y * z)
89
+ >>> gy = grad_out * (x + z + w)
90
+ >>> gz = None
91
+ >>> return gx, gy, gz
92
+ >>>
93
+ >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double)
94
+ >>> b = torch.tensor(2., requires_grad=True, dtype=torch.double)
95
+ >>> c = 4
96
+ >>> d = Func.apply(a, b, c)
97
+
98
+ """
99
+ self.to_save = tensors
100
+
101
+ def save_for_forward(self, *tensors: torch.Tensor):
102
+ r"""Save given tensors for a future call to :func:`~Function.jvp`.
103
+
104
+ ``save_for_forward`` should be called at most once, in either the
105
+ :func:`setup_context` or :func:`forward` methods, and all arguments
106
+ should be tensors.
107
+
108
+ In :func:`jvp`, saved objects can be accessed through the :attr:`saved_tensors`
109
+ attribute.
110
+
111
+ Arguments can also be ``None``. This is a no-op.
112
+
113
+ See :ref:`extending-autograd` for more details on how to use this method.
114
+
115
+ Example::
116
+
117
+ >>> # xdoctest: +SKIP
118
+ >>> class Func(torch.autograd.Function):
119
+ >>> @staticmethod
120
+ >>> def forward(ctx, x: torch.Tensor, y: torch.Tensor, z: int):
121
+ >>> ctx.save_for_backward(x, y)
122
+ >>> ctx.save_for_forward(x, y)
123
+ >>> ctx.z = z
124
+ >>> return x * y * z
125
+ >>>
126
+ >>> @staticmethod
127
+ >>> def jvp(ctx, x_t, y_t, _):
128
+ >>> x, y = ctx.saved_tensors
129
+ >>> z = ctx.z
130
+ >>> return z * (y * x_t + x * y_t)
131
+ >>>
132
+ >>> @staticmethod
133
+ >>> def vjp(ctx, grad_out):
134
+ >>> x, y = ctx.saved_tensors
135
+ >>> z = ctx.z
136
+ >>> return z * grad_out * y, z * grad_out * x, None
137
+ >>>
138
+ >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double)
139
+ >>> t = torch.tensor(1., dtype=torch.double)
140
+ >>> b = torch.tensor(2., requires_grad=True, dtype=torch.double)
141
+ >>> c = 4
142
+ >>>
143
+ >>> with fwAD.dual_level():
144
+ >>> a_dual = fwAD.make_dual(a, t)
145
+ >>> d = Func.apply(a_dual, b, c)
146
+
147
+ """
148
+ for tensor in tensors:
149
+ if not (isinstance(tensor, torch.Tensor) or tensor is None):
150
+ raise AssertionError(
151
+ "save_for_forward expects all arguments to be tensors; you should "
152
+ "save non-tensors as attributes on ctx."
153
+ )
154
+
155
+ self.saved_for_forward = tensors
156
+
157
+ def mark_dirty(self, *args: torch.Tensor):
158
+ r"""Mark given tensors as modified in an in-place operation.
159
+
160
+ This should be called at most once, in either the :func:`setup_context`
161
+ or :func:`forward` methods, and all arguments should be inputs.
162
+
163
+ Every tensor that's been modified in-place in a call to :func:`forward`
164
+ should be given to this function, to ensure correctness of our checks.
165
+ It doesn't matter whether the function is called before or after
166
+ modification.
167
+
168
+ Examples::
169
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
170
+ >>> class Inplace(Function):
171
+ >>> @staticmethod
172
+ >>> def forward(ctx, x):
173
+ >>> x_npy = x.numpy() # x_npy shares storage with x
174
+ >>> x_npy += 1
175
+ >>> ctx.mark_dirty(x)
176
+ >>> return x
177
+ >>>
178
+ >>> @staticmethod
179
+ >>> @once_differentiable
180
+ >>> def backward(ctx, grad_output):
181
+ >>> return grad_output
182
+ >>>
183
+ >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double).clone()
184
+ >>> b = a * a
185
+ >>> Inplace.apply(a) # This would lead to wrong gradients!
186
+ >>> # but the engine would not know unless we mark_dirty
187
+ >>> # xdoctest: +SKIP
188
+ >>> b.backward() # RuntimeError: one of the variables needed for gradient
189
+ >>> # computation has been modified by an inplace operation
190
+
191
+ """
192
+ self.dirty_tensors = args
193
+
194
+ @deprecated(
195
+ "`mark_shared_storage` is deprecated. "
196
+ "Tensors with shared storages are automatically tracked. "
197
+ "Note that calls to `set_()` are not tracked",
198
+ category=FutureWarning,
199
+ )
200
+ def mark_shared_storage(self, *pairs):
201
+ pass
202
+
203
+ def mark_non_differentiable(self, *args: torch.Tensor):
204
+ r"""Mark outputs as non-differentiable.
205
+
206
+ This should be called at most once, in either the :func:`setup_context`
207
+ or :func:`forward` methods, and all arguments should be tensor outputs.
208
+
209
+ This will mark outputs as not requiring gradients, increasing the
210
+ efficiency of backward computation. You still need to accept a gradient
211
+ for each output in :meth:`~Function.backward`, but it's always going to
212
+ be a zero tensor with the same shape as the shape of a corresponding
213
+ output.
214
+
215
+ This is used e.g. for indices returned from a sort. See example::
216
+ >>> class Func(Function):
217
+ >>> @staticmethod
218
+ >>> def forward(ctx, x):
219
+ >>> sorted, idx = x.sort()
220
+ >>> ctx.mark_non_differentiable(idx)
221
+ >>> ctx.save_for_backward(x, idx)
222
+ >>> return sorted, idx
223
+ >>>
224
+ >>> @staticmethod
225
+ >>> @once_differentiable
226
+ >>> def backward(ctx, g1, g2): # still need to accept g2
227
+ >>> x, idx = ctx.saved_tensors
228
+ >>> grad_input = torch.zeros_like(x)
229
+ >>> grad_input.index_add_(0, idx, g1)
230
+ >>> return grad_input
231
+
232
+ """
233
+ self.non_differentiable = args
234
+
235
+ def set_materialize_grads(self, value: bool):
236
+ r"""Set whether to materialize grad tensors. Default is ``True``.
237
+
238
+ This should be called only from either the :func:`setup_context` or
239
+ :func:`forward` methods.
240
+
241
+ If ``True``, undefined grad tensors will be expanded to tensors full of zeros
242
+ prior to calling the :func:`backward` and :func:`jvp` methods.
243
+
244
+ Example::
245
+
246
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
247
+ >>> class SimpleFunc(Function):
248
+ >>> @staticmethod
249
+ >>> def forward(ctx, x):
250
+ >>> return x.clone(), x.clone()
251
+ >>>
252
+ >>> @staticmethod
253
+ >>> @once_differentiable
254
+ >>> def backward(ctx, g1, g2):
255
+ >>> return g1 + g2 # No check for None necessary
256
+ >>>
257
+ >>> # We modify SimpleFunc to handle non-materialized grad outputs
258
+ >>> class Func(Function):
259
+ >>> @staticmethod
260
+ >>> def forward(ctx, x):
261
+ >>> ctx.set_materialize_grads(False)
262
+ >>> ctx.save_for_backward(x)
263
+ >>> return x.clone(), x.clone()
264
+ >>>
265
+ >>> @staticmethod
266
+ >>> @once_differentiable
267
+ >>> def backward(ctx, g1, g2):
268
+ >>> x, = ctx.saved_tensors
269
+ >>> grad_input = torch.zeros_like(x)
270
+ >>> if g1 is not None: # We must check for None now
271
+ >>> grad_input += g1
272
+ >>> if g2 is not None:
273
+ >>> grad_input += g2
274
+ >>> return grad_input
275
+ >>>
276
+ >>> a = torch.tensor(1., requires_grad=True)
277
+ >>> b, _ = Func.apply(a) # induces g2 to be undefined
278
+
279
+ """
280
+ self.materialize_grads = value
281
+
282
+
283
+ # DO NOT USE: This is only defined to be able to load old serialized models
284
+ _ContextMethodMixin = FunctionCtx
285
+
286
+
287
+ class _HookMixin:
288
+ @staticmethod
289
+ def _register_hook(backward_hooks, hook):
290
+ if backward_hooks is None:
291
+ backward_hooks = OrderedDict()
292
+ handle = hooks.RemovableHandle(backward_hooks)
293
+ backward_hooks[handle.id] = hook
294
+ return backward_hooks, handle
295
+
296
+
297
+ class BackwardCFunction(_C._FunctionBase, FunctionCtx, _HookMixin):
298
+ r"""
299
+ This class is used for internal autograd work. Do not use.
300
+ """
301
+
302
+ def _get_user_fn(self):
303
+ backward_fn = self._forward_cls.backward # type: ignore[attr-defined]
304
+ vjp_fn = self._forward_cls.vjp # type: ignore[attr-defined]
305
+ if backward_fn is not Function.backward and vjp_fn is not Function.vjp:
306
+ raise RuntimeError(
307
+ "Implementing both 'backward' and 'vjp' for a custom "
308
+ "Function is not allowed. You should only implement one "
309
+ "of them."
310
+ )
311
+ return vjp_fn if vjp_fn is not Function.vjp else backward_fn
312
+
313
+ def apply(self, *args):
314
+ r"""
315
+ Apply method used when executing this Node during the backward.
316
+
317
+ Called by the autograd engine (non-boxed path) and by direct
318
+ grad_fn.apply() calls. When boxed_grads_call is True, boxes
319
+ grads into a mutable list before calling user's backward.
320
+ """
321
+ user_fn = self._get_user_fn()
322
+ fwd_cls = self._forward_cls # type: ignore[attr-defined] # pyrefly: ignore[missing-attribute]
323
+ if getattr(fwd_cls, "boxed_grads_call", False):
324
+ args = (list(args),)
325
+ return user_fn(self, *args)
326
+
327
+ def apply_boxed(self, *args):
328
+ r"""
329
+ Apply method called by the autograd engine when boxed_grads_call
330
+ is True. Grads arrive as a single mutable list argument, allowing
331
+ backward to free individual grads mid-execution.
332
+ """
333
+ return self._get_user_fn()(self, *args)
334
+
335
+ def apply_jvp(self, *args):
336
+ r"""
337
+ Apply method used when executing forward mode AD during the forward
338
+ """
339
+ # _forward_cls is defined by derived class
340
+ return self._forward_cls.jvp(self, *args) # type: ignore[attr-defined]
341
+
342
+ def _compiled_autograd_key(self):
343
+ return self._forward_cls._compiled_autograd_key(self) # type: ignore[attr-defined]
344
+
345
+
346
+ class FunctionMeta(type):
347
+ """Function metaclass.
348
+
349
+ This metaclass sets up the following properties:
350
+ _backward_cls: The Function class corresponding to the differentiated
351
+ version of this function (which is generated on the fly by this
352
+ metaclass).
353
+ """
354
+
355
+ def __init__(cls, name, bases, attrs):
356
+ backward_fn = type(
357
+ name + "Backward", (BackwardCFunction,), {"_forward_cls": cls}
358
+ )
359
+ backward_fn._autograd_function_id = next(AUTOGRAD_FUNCTION_COUNTER) # type: ignore[attr-defined]
360
+ cls._backward_cls = backward_fn
361
+
362
+ super().__init__(name, bases, attrs)
363
+
364
+
365
+ class _SingleLevelFunction(
366
+ _C._FunctionBase, FunctionCtx, _HookMixin, metaclass=FunctionMeta
367
+ ):
368
+ @staticmethod
369
+ def forward(*args: Any, **kwargs: Any) -> Any:
370
+ r"""Define the forward of the custom autograd Function.
371
+
372
+ This function is to be overridden by all subclasses.
373
+ There are two ways to define forward:
374
+
375
+ Usage 1 (Combined forward and ctx)::
376
+
377
+ @staticmethod
378
+ def forward(ctx: Any, *args: Any, **kwargs: Any) -> Any:
379
+ pass
380
+
381
+ - It must accept a context ctx as the first argument, followed by any
382
+ number of arguments (tensors or other types).
383
+ - See :ref:`combining-forward-context` for more details
384
+
385
+ Usage 2 (Separate forward and ctx)::
386
+
387
+ @staticmethod
388
+ def forward(*args: Any, **kwargs: Any) -> Any:
389
+ pass
390
+
391
+
392
+ @staticmethod
393
+ def setup_context(ctx: Any, inputs: Tuple[Any, ...], output: Any) -> None:
394
+ pass
395
+
396
+ - The forward no longer accepts a ctx argument.
397
+ - Instead, you must also override the :meth:`torch.autograd.Function.setup_context`
398
+ staticmethod to handle setting up the ``ctx`` object.
399
+ ``output`` is the output of the forward, ``inputs`` are a Tuple of inputs
400
+ to the forward.
401
+ - See :ref:`extending-autograd` for more details
402
+
403
+ The context can be used to store arbitrary data that can be then
404
+ retrieved during the backward pass. Tensors should not be stored
405
+ directly on `ctx` (though this is not currently enforced for
406
+ backward compatibility). Instead, tensors should be saved either with
407
+ :func:`ctx.save_for_backward` if they are intended to be used in
408
+ ``backward`` (equivalently, ``vjp``) or :func:`ctx.save_for_forward`
409
+ if they are intended to be used for in ``jvp``.
410
+ """
411
+ raise NotImplementedError(
412
+ "You must implement the forward function for custom autograd.Function."
413
+ )
414
+
415
+ @staticmethod
416
+ def setup_context(ctx: Any, inputs: tuple[Any, ...], output: Any) -> Any:
417
+ r"""There are two ways to define the forward pass of an autograd.Function.
418
+
419
+ Either:
420
+
421
+ 1. Override forward with the signature ``forward(ctx, *args, **kwargs)``.
422
+ ``setup_context`` is not overridden. Setting up the ctx for backward
423
+ happens inside the ``forward``.
424
+ 2. Override forward with the signature ``forward(*args, **kwargs)`` and
425
+ override ``setup_context``. Setting up the ctx for backward happens
426
+ inside ``setup_context`` (as opposed to inside the ``forward``)
427
+
428
+ See :meth:`torch.autograd.Function.forward` and :ref:`extending-autograd` for more details.
429
+ """
430
+ raise NotImplementedError("setup_context is not implemented.")
431
+
432
+ @staticmethod
433
+ def backward(ctx: Any, *grad_outputs: Any) -> Any:
434
+ r"""Define a formula for differentiating the operation with backward mode automatic differentiation.
435
+
436
+ This function is to be overridden by all subclasses.
437
+ (Defining this function is equivalent to defining the ``vjp`` function.)
438
+
439
+ It must accept a context :attr:`ctx` as the first argument, followed by
440
+ as many outputs as the :func:`forward` returned (None will be passed in
441
+ for non tensor outputs of the forward function),
442
+ and it should return as many tensors, as there were inputs to
443
+ :func:`forward`. Each argument is the gradient w.r.t the given output,
444
+ and each returned value should be the gradient w.r.t. the
445
+ corresponding input. If an input is not a Tensor or is a Tensor not
446
+ requiring grads, you can just pass None as a gradient for that input.
447
+
448
+ The context can be used to retrieve tensors saved during the forward
449
+ pass. It also has an attribute :attr:`ctx.needs_input_grad` as a tuple
450
+ of booleans representing whether each input needs gradient. E.g.,
451
+ :func:`backward` will have ``ctx.needs_input_grad[0] = True`` if the
452
+ first input to :func:`forward` needs gradient computed w.r.t. the
453
+ output.
454
+ """
455
+ raise NotImplementedError(
456
+ "You must implement either the backward or vjp method for "
457
+ "your custom autograd.Function to use it with backward "
458
+ "mode AD."
459
+ )
460
+
461
+ # vjp and backward are alias of each other
462
+ vjp = backward
463
+
464
+ """
465
+ Bool that specifies if PyTorch should clear saved tensors after the first
466
+ access to ``ctx.saved_tensors``. When set to True, accessing saved_tensors
467
+ clears the internal references, allowing the tensors to be cleared as soon
468
+ as the Tensor returned by saved_tensors is deleted.
469
+
470
+ This is useful for reducing memory pressure in backward passes when you
471
+ only need to access saved tensors once.
472
+
473
+ Default is False.
474
+ """
475
+ clear_saved_tensors_on_access = False
476
+
477
+ """
478
+ Bool that specifies if backward should receive grads as a single mutable
479
+ list argument instead of individual args in an immutable tuple. This allows
480
+ backward to free individual grads mid-execution by removing them from the
481
+ list, reducing peak memory.
482
+
483
+ When True, ``backward(ctx, grads)`` receives a single list instead of
484
+ ``backward(ctx, *grads)``.
485
+
486
+ Default is False.
487
+ """
488
+ boxed_grads_call = False
489
+
490
+ @staticmethod
491
+ def jvp(ctx: Any, *grad_inputs: Any) -> Any:
492
+ r"""Define a formula for differentiating the operation with forward mode automatic differentiation.
493
+
494
+ This function is to be overridden by all subclasses.
495
+ It must accept a context :attr:`ctx` as the first argument, followed by
496
+ as many inputs as the :func:`forward` got (None will be passed in
497
+ for non tensor inputs of the forward function),
498
+ and it should return as many tensors as there were outputs to
499
+ :func:`forward`. Each argument is the gradient w.r.t the given input,
500
+ and each returned value should be the gradient w.r.t. the
501
+ corresponding output. If an output is not a Tensor or the function is not
502
+ differentiable with respect to that output, you can just pass None as a
503
+ gradient for that input.
504
+
505
+ You can use the :attr:`ctx` object to pass any value from the forward to this
506
+ functions.
507
+ """
508
+ raise NotImplementedError(
509
+ "You must implement the jvp function for custom "
510
+ "autograd.Function to use it with forward mode AD."
511
+ )
512
+
513
+
514
+ class Function(_SingleLevelFunction):
515
+ r"""Base class to create custom `autograd.Function`.
516
+
517
+ To create a custom `autograd.Function`, subclass this class and implement
518
+ the :meth:`forward` and :meth:`backward` static methods. Then, to use your custom
519
+ op in the forward pass, call the class method ``apply``. Do not call
520
+ :meth:`forward` directly.
521
+
522
+ To ensure correctness and best performance, make sure you are calling the
523
+ correct methods on ``ctx`` and validating your backward function using
524
+ :func:`torch.autograd.gradcheck`.
525
+
526
+ See :ref:`extending-autograd` for more details on how to use this class.
527
+
528
+ Examples::
529
+
530
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
531
+ >>> class Exp(Function):
532
+ >>> @staticmethod
533
+ >>> def forward(ctx, i):
534
+ >>> result = i.exp()
535
+ >>> ctx.save_for_backward(result)
536
+ >>> return result
537
+ >>>
538
+ >>> @staticmethod
539
+ >>> def backward(ctx, grad_output):
540
+ >>> result, = ctx.saved_tensors
541
+ >>> return grad_output * result
542
+ >>>
543
+ >>> # Use it by calling the apply method:
544
+ >>> # xdoctest: +SKIP
545
+ >>> output = Exp.apply(input)
546
+ """
547
+
548
+ def __init__(self, *args, **kwargs):
549
+ warnings.warn(
550
+ f"{self.__class__} should not be instantiated. Methods on autograd functions "
551
+ "are all static, so you should invoke them on the class itself. "
552
+ "Instantiating an autograd function will raise an "
553
+ "error in a future version of PyTorch.",
554
+ DeprecationWarning,
555
+ stacklevel=2,
556
+ )
557
+
558
+ def __call__(self, *args, **kwargs):
559
+ raise RuntimeError(
560
+ "Legacy autograd function with non-static forward method is deprecated. "
561
+ "Please use new-style autograd function with static forward method. "
562
+ "(Example: https://pytorch.org/docs/stable/autograd.html#torch.autograd.Function)"
563
+ )
564
+
565
+ """
566
+ Bool that specifies if PyTorch should attempt to autogenerate
567
+ :func:`torch.vmap` support for this autograd.Function. You may set this to
568
+ True only if this autograd.Function's forward, backward, and jvp (if they
569
+ exist) are written using PyTorch operations; otherwise, please override
570
+ :meth:`torch.autograd.Function.vmap` to add support for :func:`torch.vmap`.
571
+
572
+ Please see :ref:`func-autograd-function` for more details.
573
+ """
574
+ generate_vmap_rule = False
575
+
576
+ @staticmethod
577
+ def vmap(info, in_dims, *args):
578
+ r"""Define the behavior for this autograd.Function underneath :func:`torch.vmap`.
579
+
580
+ For a :func:`torch.autograd.Function` to support
581
+ :func:`torch.vmap`, you must either override this static method, or set
582
+ ``generate_vmap_rule`` to ``True`` (you may not do both).
583
+
584
+ If you choose to override this staticmethod: it must accept
585
+
586
+ - an ``info`` object as the first argument. ``info.batch_size``
587
+ specifies the size of the dimension being vmapped over,
588
+ while ``info.randomness`` is the randomness option passed to
589
+ :func:`torch.vmap`.
590
+ - an ``in_dims`` tuple as the second argument.
591
+ For each arg in ``args``, ``in_dims`` has a corresponding
592
+ ``Optional[int]``. It is ``None`` if the arg is not a Tensor or if
593
+ the arg is not being vmapped over, otherwise, it is an integer
594
+ specifying what dimension of the Tensor is being vmapped over.
595
+ - ``*args``, which is the same as the args to :meth:`~Function.forward`.
596
+
597
+ The return of the vmap staticmethod is a tuple of ``(output, out_dims)``.
598
+ Similar to ``in_dims``, ``out_dims`` should be of the same structure as
599
+ ``output`` and contain one ``out_dim`` per output that specifies if the
600
+ output has the vmapped dimension and what index it is in.
601
+
602
+ Please see :ref:`func-autograd-function` for more details.
603
+ """
604
+ raise NotImplementedError(
605
+ "To use autograd.Function with vmap, you must either override the "
606
+ "vmap staticmethod or set generate_vmap_rule=True."
607
+ )
608
+
609
+ @classmethod
610
+ def apply(cls, *args, **kwargs):
611
+ def bind_default_args(func, *args, **kwargs):
612
+ signature = inspect.signature(func)
613
+ bound_args = signature.bind(*args, **kwargs)
614
+ bound_args.apply_defaults()
615
+
616
+ return bound_args.args, bound_args.kwargs
617
+
618
+ is_setup_ctx_defined = _is_setup_context_defined(cls.setup_context)
619
+ if is_setup_ctx_defined:
620
+ args, kwargs = bind_default_args(cls.forward, *args, **kwargs)
621
+
622
+ if not torch._C._are_functorch_transforms_active():
623
+ # See NOTE: [functorch vjp and autograd interaction]
624
+ args = _functorch.utils.unwrap_dead_wrappers(args)
625
+ return super().apply(*args, **kwargs) # type: ignore[misc]
626
+
627
+ if not is_setup_ctx_defined:
628
+ raise RuntimeError(
629
+ "In order to use an autograd.Function with functorch transforms "
630
+ "(vmap, grad, jvp, jacrev, ...), it must override the setup_context "
631
+ "staticmethod. For more details, please see "
632
+ "https://pytorch.org/docs/main/notes/extending.func.html"
633
+ )
634
+
635
+ return custom_function_call(cls, *args, **kwargs)
636
+
637
+ @staticmethod
638
+ def _compiled_autograd_key(ctx):
639
+ return (ctx._autograd_function_id,)
640
+
641
+
642
+ def _is_setup_context_defined(fn):
643
+ return fn != _SingleLevelFunction.setup_context
644
+
645
+
646
+ def once_differentiable(
647
+ fn: Callable[Concatenate[_T, _P], _R],
648
+ ) -> Callable[Concatenate[_T, _P], _R]:
649
+ @functools.wraps(fn)
650
+ def wrapper(ctx: _T, *args: _P.args, **kwargs: _P.kwargs) -> _R:
651
+ with torch.no_grad():
652
+ outputs = fn(ctx, *args, **kwargs)
653
+
654
+ if not torch.is_grad_enabled():
655
+ return outputs
656
+
657
+ # If any of the inputs have requires_grad=True, we force the outputs
658
+ # to have requires_grad=True but point to a grad_fn which throws an
659
+ # error message during (double) back-propagation.
660
+ # XXX: this is only an approximation of requires_grad - there's no way
661
+ # to figure out if fn didn't use ctx.saved_tensors and as a result
662
+ # some Tensors might require grad, even if no args do.
663
+ # Unfortunately, this leads to unexpected error messages ("no nodes
664
+ # require computing gradients"), but I don't have a better idea.
665
+ # These functions would raise an error in backward anyway.
666
+ requires_grad = any(
667
+ isinstance(arg, torch.Tensor) and arg.requires_grad for arg in args
668
+ )
669
+ if not requires_grad:
670
+ return outputs
671
+
672
+ if not isinstance(outputs, tuple):
673
+ outputs_ = (outputs,)
674
+ else:
675
+ outputs_ = outputs
676
+
677
+ err_fn = _functions.DelayedError(
678
+ b"trying to differentiate twice a function that was marked "
679
+ b"with @once_differentiable",
680
+ len(outputs_),
681
+ )
682
+
683
+ # Create aliases of each output that has requires_grad=True. We need
684
+ # at least one of the inputs to err_fn to require grad so that the
685
+ # output will have a grad_fn.
686
+ def fake_requires_grad(var):
687
+ if var is not None:
688
+ var = var.detach()
689
+ var.requires_grad = True
690
+ return var
691
+
692
+ return err_fn(*[fake_requires_grad(v) for v in outputs_]) # type: ignore[return-value]
693
+
694
+ return wrapper
695
+
696
+
697
+ class InplaceFunction(Function):
698
+ r"""
699
+ This class is here only for backward compatibility reasons.
700
+ Use :class:`Function` instead of this for any new use case.
701
+ """
702
+
703
+ def __init__(self, inplace=False):
704
+ super().__init__()
705
+ self.inplace = inplace
706
+
707
+
708
+ def _nested_map(condition, fn, condition_msg=None):
709
+ def _map(obj):
710
+ if condition(obj):
711
+ return fn(obj)
712
+ elif obj is None:
713
+ return None
714
+ elif isinstance(obj, (list, tuple)):
715
+ mapped = (_map(x) for x in obj)
716
+ if hasattr(obj, "_fields"):
717
+ # obj is namedtuple
718
+ return type(obj)(*mapped)
719
+ return type(obj)(mapped)
720
+ elif isinstance(obj, dict):
721
+ return {x: _map(obj[x]) for x in obj}
722
+ else:
723
+ raise ValueError(
724
+ "Auto nesting doesn't know how to process "
725
+ "an input object of type "
726
+ + torch.typename(obj)
727
+ + (
728
+ ". Accepted types: " + condition_msg + ", or lists/tuples of them"
729
+ if condition_msg
730
+ else ""
731
+ )
732
+ )
733
+
734
+ return _map
735
+
736
+
737
+ def _jit_unwrap_structured(obj):
738
+ if hasattr(obj, "_jit_unwrap"):
739
+ return obj._jit_unwrap()
740
+ return obj
741
+
742
+
743
+ def _iter_filter(condition, allow_unknown=False, condition_msg=None, conversion=None):
744
+ def _iter(obj):
745
+ if conversion is not None:
746
+ obj = conversion(obj)
747
+ if condition(obj):
748
+ yield obj
749
+ elif obj is None:
750
+ return
751
+ elif isinstance(obj, (list, tuple)):
752
+ for o in obj:
753
+ yield from _iter(o)
754
+ elif isinstance(obj, dict):
755
+ # We only accept primitive key types, so we needn't inspect them
756
+ for o in obj.values():
757
+ yield from _iter(o)
758
+ elif allow_unknown:
759
+ yield obj
760
+ else:
761
+ raise ValueError(
762
+ "Auto nesting doesn't know how to process "
763
+ "an input object of type "
764
+ + torch.typename(obj)
765
+ + (
766
+ ". Accepted types: " + condition_msg + ", or lists/tuples of them"
767
+ if condition_msg
768
+ else ""
769
+ )
770
+ )
771
+
772
+ return _iter
773
+
774
+
775
+ def _unflatten(input, proto):
776
+ # unflatten a list or tuple input into a nested list/tuple structure
777
+ # specified by proto
778
+ def unflatten_helper(input, proto):
779
+ res: list[torch.Tensor | None] = []
780
+ if hasattr(proto, "_jit_wrap"):
781
+ return proto._jit_wrap(input)
782
+ if not isinstance(proto, (list, tuple)):
783
+ return input[0], input[1:]
784
+ for e in proto:
785
+ if e is None:
786
+ res.append(e)
787
+ else:
788
+ res_e, input = unflatten_helper(input, e)
789
+ res.append(res_e)
790
+ return type(proto)(res), input
791
+
792
+ return unflatten_helper(input, proto)[0]
793
+
794
+
795
+ _iter_jit_values = _iter_filter(
796
+ lambda o: o is None or isinstance(o, torch._C.Value),
797
+ condition_msg="jit's Values or None",
798
+ )
799
+ _iter_tensors = _iter_filter(
800
+ lambda x: isinstance(x, torch.Tensor),
801
+ condition_msg="Tensors",
802
+ conversion=_jit_unwrap_structured,
803
+ )
804
+ _iter_tensors_permissive = _iter_filter(
805
+ lambda x: isinstance(x, torch.Tensor),
806
+ allow_unknown=True,
807
+ condition_msg="Tensors (permissive)",
808
+ )
809
+ _iter_None_tensors = _iter_filter(
810
+ lambda o: o is None or isinstance(o, torch.Tensor), condition_msg="Tensors or None"
811
+ )
812
+ _map_tensor_data = _nested_map(
813
+ lambda x: isinstance(x, torch.Tensor), lambda o: o.data, condition_msg="Tensors"
814
+ )
815
+
816
+
817
+ class NestedIOFunction(Function):
818
+ r"""
819
+ This class is here only for backward compatibility reasons.
820
+ Use :class:`Function` instead of this for any new use case.
821
+ """
822
+
823
+ # The 'type: ignore' statements are needed here because these functions are declared as '@staticmethod' in the
824
+ # superclass (Function) but are instance methods here, which mypy reports as incompatible.
825
+
826
+ def _do_forward(self, *input):
827
+ self._nested_input = input
828
+ flat_input = tuple(_iter_tensors(input))
829
+ flat_output = super()._do_forward(*flat_input) # type: ignore[misc]
830
+ nested_tensors = _unflatten(flat_output, self._nested_output)
831
+ return nested_tensors
832
+
833
+ def _do_backward(self, gradients, retain_variables):
834
+ self.retain_variables = retain_variables
835
+ result = super()._do_backward(gradients, retain_variables) # type: ignore[misc]
836
+ if not retain_variables:
837
+ del self._nested_output
838
+ del self._to_save_nested
839
+ return result
840
+
841
+ def backward(self, *gradients: Any) -> Any: # type: ignore[override]
842
+ r"""
843
+ Shared backward utility.
844
+ """
845
+ nested_gradients = _unflatten(gradients, self._nested_output)
846
+ result = self.backward_extended(*nested_gradients) # type: ignore[func-returns-value]
847
+ return tuple(_iter_None_tensors(result))
848
+
849
+ __call__ = _do_forward
850
+
851
+ def forward(self, *args: Any) -> Any: # type: ignore[override]
852
+ r"""
853
+ Shared forward utility.
854
+ """
855
+ nested_tensors = _map_tensor_data(self._nested_input)
856
+ result = self.forward_extended(*nested_tensors) # type: ignore[func-returns-value]
857
+ del self._nested_input
858
+ self._nested_output = result
859
+ return tuple(_iter_tensors(result))
860
+
861
+ def save_for_backward(self, *args: Any) -> None:
862
+ r"""
863
+ See :meth:`Function.save_for_backward`.
864
+ """
865
+ self.to_save = tuple(_iter_tensors(args))
866
+ self._to_save_nested = args
867
+
868
+ @property
869
+ def saved_tensors(self): # type: ignore[override]
870
+ r"""
871
+ See :meth:`Function.saved_tensors`.
872
+ """
873
+ flat_tensors = super().saved_tensors # type: ignore[misc]
874
+ return _unflatten(flat_tensors, self._to_save_nested)
875
+
876
+ def mark_dirty(self, *args: Any, **kwargs: Any) -> None:
877
+ r"""
878
+ See :meth:`Function.mark_dirty`.
879
+ """
880
+ self.dirty_tensors = tuple(_iter_tensors((args, kwargs)))
881
+
882
+ def mark_non_differentiable(self, *args: Any, **kwargs: Any) -> None:
883
+ r"""
884
+ See :meth:`Function.mark_non_differentiable`.
885
+ """
886
+ self.non_differentiable = tuple(_iter_tensors((args, kwargs)))
887
+
888
+ def forward_extended(self, *input: Any) -> None:
889
+ r"""
890
+ User defined forward.
891
+ """
892
+ raise NotImplementedError
893
+
894
+ def backward_extended(self, *grad_output: Any) -> None:
895
+ r"""
896
+ User defined backward.
897
+ """
898
+ raise NotImplementedError
western_ai/usa/pytorch/torch/autograd/functional.py ADDED
@@ -0,0 +1,1212 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+
3
+ import torch
4
+ from torch._vmap_internals import _vmap
5
+
6
+ from . import forward_ad as fwAD
7
+
8
+
9
+ __all__ = ["vjp", "jvp", "jacobian", "hessian", "hvp", "vhp"]
10
+
11
+ # Utility functions
12
+
13
+
14
+ def _as_tuple_nocheck(x):
15
+ if isinstance(x, tuple):
16
+ return x
17
+ elif isinstance(x, list):
18
+ return tuple(x)
19
+ else:
20
+ return (x,)
21
+
22
+
23
+ def _as_tuple(inp, arg_name=None, fn_name=None):
24
+ # Ensures that inp is a tuple of Tensors
25
+ # Returns whether or not the original inp was a tuple and the tupled version of the input
26
+ if arg_name is None and fn_name is None:
27
+ return _as_tuple_nocheck(inp)
28
+
29
+ is_inp_tuple = True
30
+ if not isinstance(inp, tuple):
31
+ inp = (inp,)
32
+ is_inp_tuple = False
33
+
34
+ for i, el in enumerate(inp):
35
+ if not isinstance(el, torch.Tensor):
36
+ if is_inp_tuple:
37
+ raise TypeError(
38
+ f"The {arg_name} given to {fn_name} must be either a Tensor or a tuple of Tensors but the"
39
+ f" value at index {i} has type {type(el)}."
40
+ )
41
+ else:
42
+ raise TypeError(
43
+ f"The {arg_name} given to {fn_name} must be either a Tensor or a tuple of Tensors but the"
44
+ f" given {arg_name} has type {type(el)}."
45
+ )
46
+
47
+ return is_inp_tuple, inp
48
+
49
+
50
+ def _tuple_postprocess(res, to_unpack):
51
+ # Unpacks a potentially nested tuple of Tensors
52
+ # to_unpack should be a single boolean or a tuple of two booleans.
53
+ # It is used to:
54
+ # - invert _as_tuple when res should match the inp given to _as_tuple
55
+ # - optionally remove nesting of two tuples created by multiple calls to _as_tuple
56
+ if isinstance(to_unpack, tuple):
57
+ if len(to_unpack) != 2:
58
+ raise AssertionError("Expected to_unpack tuple to have exactly 2 elements")
59
+ if not to_unpack[1]:
60
+ res = tuple(el[0] for el in res)
61
+ if not to_unpack[0]:
62
+ res = res[0]
63
+ else:
64
+ if not to_unpack:
65
+ res = res[0]
66
+ return res
67
+
68
+
69
+ def _grad_preprocess(inputs, create_graph, need_graph):
70
+ # Preprocess the inputs to make sure they require gradient
71
+ # inputs is a tuple of Tensors to preprocess
72
+ # create_graph specifies if the user wants gradients to flow back to the Tensors in inputs
73
+ # need_graph specifies if we internally want gradients to flow back to the Tensors in res
74
+ # Note that we *always* create a new Tensor object to be able to see the difference between
75
+ # inputs given as arguments and the same Tensors automatically captured by the user function.
76
+ # Check this issue for more details on how that can happen: https://github.com/pytorch/pytorch/issues/32576
77
+ res = []
78
+ for inp in inputs:
79
+ if create_graph and inp.requires_grad:
80
+ # Create at least a new Tensor object in a differentiable way
81
+ if not inp.is_sparse:
82
+ # Use .view_as() to get a shallow copy
83
+ res.append(inp.view_as(inp))
84
+ else:
85
+ # We cannot use view for sparse Tensors so we clone
86
+ res.append(inp.clone())
87
+ else:
88
+ res.append(inp.detach().requires_grad_(need_graph))
89
+ return tuple(res)
90
+
91
+
92
+ def _grad_postprocess(inputs, create_graph):
93
+ # Postprocess the generated Tensors to avoid returning Tensors with history when the user did not
94
+ # request it.
95
+ if isinstance(inputs[0], torch.Tensor):
96
+ if not create_graph:
97
+ return tuple(inp.detach() for inp in inputs)
98
+ else:
99
+ return inputs
100
+ else:
101
+ return tuple(_grad_postprocess(inp, create_graph) for inp in inputs)
102
+
103
+
104
+ def _validate_v(v, other, is_other_tuple):
105
+ # This assumes that other is the correct shape, and v should match
106
+ # Both are assumed to be tuples of Tensors
107
+ if len(other) != len(v):
108
+ if is_other_tuple:
109
+ raise RuntimeError(
110
+ f"v is a tuple of invalid length: should be {len(other)} but got {len(v)}."
111
+ )
112
+ else:
113
+ raise RuntimeError("The given v should contain a single Tensor.")
114
+
115
+ for idx, (el_v, el_other) in enumerate(zip(v, other)):
116
+ if el_v.size() != el_other.size():
117
+ prepend = ""
118
+ if is_other_tuple:
119
+ prepend = f"Entry {idx} in "
120
+ raise RuntimeError(
121
+ f"{prepend}v has invalid size: should be {el_other.size()} but got {el_v.size()}."
122
+ )
123
+
124
+
125
+ def _check_requires_grad(inputs, input_type, strict):
126
+ # Used to make all the necessary checks to raise nice errors in strict mode.
127
+ if not strict:
128
+ return
129
+
130
+ if input_type not in ["outputs", "grad_inputs", "jacobian", "hessian"]:
131
+ raise RuntimeError("Invalid input_type to _check_requires_grad")
132
+ for i, inp in enumerate(inputs):
133
+ if inp is None:
134
+ # This can only be reached for grad_inputs.
135
+ raise RuntimeError(
136
+ f"The output of the user-provided function is independent of input {i}."
137
+ " This is not allowed in strict mode."
138
+ )
139
+ if not inp.requires_grad:
140
+ if input_type == "hessian":
141
+ raise RuntimeError(
142
+ f"The hessian of the user-provided function with respect to input {i}"
143
+ " is independent of the input. This is not allowed in strict mode."
144
+ " You should ensure that your function is thrice differentiable and that"
145
+ " the hessian depends on the inputs."
146
+ )
147
+ elif input_type == "jacobian":
148
+ raise RuntimeError(
149
+ "While computing the hessian, found that the jacobian of the user-provided"
150
+ f" function with respect to input {i} is independent of the input. This is not"
151
+ " allowed in strict mode. You should ensure that your function is twice"
152
+ " differentiable and that the jacobian depends on the inputs (this would be"
153
+ " violated by a linear function for example)."
154
+ )
155
+ elif input_type == "grad_inputs":
156
+ raise RuntimeError(
157
+ f"The gradient with respect to input {i} is independent of the inputs of the"
158
+ " user-provided function. This is not allowed in strict mode."
159
+ )
160
+ else:
161
+ raise RuntimeError(
162
+ f"Output {i} of the user-provided function does not require gradients."
163
+ " The outputs must be computed in a differentiable manner from the input"
164
+ " when running in strict mode."
165
+ )
166
+
167
+
168
+ def _autograd_grad(
169
+ outputs,
170
+ inputs,
171
+ grad_outputs=None,
172
+ create_graph=False,
173
+ retain_graph=None,
174
+ is_grads_batched=False,
175
+ ):
176
+ # Version of autograd.grad that accepts `None` in outputs and do not compute gradients for them.
177
+ # This has the extra constraint that inputs has to be a tuple
178
+ if not isinstance(outputs, tuple):
179
+ raise AssertionError("Expected outputs to be a tuple")
180
+ if grad_outputs is None:
181
+ grad_outputs = (None,) * len(outputs)
182
+ if not isinstance(grad_outputs, tuple):
183
+ raise AssertionError("Expected grad_outputs to be a tuple")
184
+ if len(outputs) != len(grad_outputs):
185
+ raise AssertionError(
186
+ f"Expected outputs and grad_outputs to have the same length, "
187
+ f"but got {len(outputs)} and {len(grad_outputs)}"
188
+ )
189
+
190
+ new_outputs: tuple[torch.Tensor, ...] = ()
191
+ new_grad_outputs: tuple[torch.Tensor, ...] = ()
192
+ for out, grad_out in zip(outputs, grad_outputs):
193
+ if out is not None and out.requires_grad:
194
+ new_outputs += (out,)
195
+ # pyrefly: ignore [bad-assignment]
196
+ new_grad_outputs += (grad_out,)
197
+
198
+ if len(new_outputs) == 0:
199
+ # No differentiable output, we don't need to call the autograd engine
200
+ return (None,) * len(inputs)
201
+ else:
202
+ return torch.autograd.grad(
203
+ new_outputs,
204
+ inputs,
205
+ new_grad_outputs,
206
+ allow_unused=True,
207
+ create_graph=create_graph,
208
+ retain_graph=retain_graph,
209
+ is_grads_batched=is_grads_batched,
210
+ )
211
+
212
+
213
+ def _fill_in_zeros(grads, refs, strict, create_graph, stage):
214
+ # Used to detect None in the grads and depending on the flags, either replace them
215
+ # with Tensors full of 0s of the appropriate size based on the refs or raise an error.
216
+ # strict and create graph allow us to detect when it is appropriate to raise an error
217
+ # stage gives us information of which backward call we consider to give good error message
218
+ if stage not in ["back", "back_trick", "double_back", "double_back_trick"]:
219
+ raise RuntimeError(f"Invalid stage argument '{stage}' to _fill_in_zeros")
220
+
221
+ res: tuple[torch.Tensor, ...] = ()
222
+ for i, grads_i in enumerate(grads):
223
+ if grads_i is None:
224
+ if strict:
225
+ if stage == "back":
226
+ raise RuntimeError(
227
+ "The output of the user-provided function is independent of "
228
+ f"input {i}. This is not allowed in strict mode."
229
+ )
230
+ elif stage == "back_trick":
231
+ raise RuntimeError(
232
+ f"The gradient with respect to the input is independent of entry {i}"
233
+ " in the grad_outputs when using the double backward trick to compute"
234
+ " forward mode gradients. This is not allowed in strict mode."
235
+ )
236
+ elif stage == "double_back":
237
+ raise RuntimeError(
238
+ "The jacobian of the user-provided function is independent of "
239
+ f"input {i}. This is not allowed in strict mode."
240
+ )
241
+ else:
242
+ raise RuntimeError(
243
+ "The hessian of the user-provided function is independent of "
244
+ f"entry {i} in the grad_jacobian. This is not allowed in strict "
245
+ "mode as it prevents from using the double backward trick to "
246
+ "replace forward mode AD."
247
+ )
248
+
249
+ grads_i = torch.zeros_like(refs[i])
250
+ else:
251
+ if strict and create_graph and not grads_i.requires_grad:
252
+ if "double" not in stage:
253
+ raise RuntimeError(
254
+ "The jacobian of the user-provided function is independent of "
255
+ f"input {i}. This is not allowed in strict mode when create_graph=True."
256
+ )
257
+ else:
258
+ raise RuntimeError(
259
+ "The hessian of the user-provided function is independent of "
260
+ f"input {i}. This is not allowed in strict mode when create_graph=True."
261
+ )
262
+
263
+ res += (grads_i,)
264
+
265
+ return res
266
+
267
+
268
+ # Public API
269
+
270
+
271
+ def vjp(func, inputs, v=None, create_graph=False, strict=False):
272
+ r"""Compute the dot product between a vector ``v`` and the Jacobian of the given function at the point given by the inputs.
273
+
274
+ Args:
275
+ func (function): a Python function that takes Tensor inputs and returns
276
+ a tuple of Tensors or a Tensor.
277
+ inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
278
+ v (tuple of Tensors or Tensor): The vector for which the vector
279
+ Jacobian product is computed. Must be the same size as the output
280
+ of ``func``. This argument is optional when the output of ``func``
281
+ contains a single element and (if it is not provided) will be set
282
+ as a Tensor containing a single ``1``.
283
+ create_graph (bool, optional): If ``True``, both the output and result
284
+ will be computed in a differentiable way. Note that when ``strict``
285
+ is ``False``, the result can not require gradients or be
286
+ disconnected from the inputs. Defaults to ``False``.
287
+ strict (bool, optional): If ``True``, an error will be raised when we
288
+ detect that there exists an input such that all the outputs are
289
+ independent of it. If ``False``, we return a Tensor of zeros as the
290
+ vjp for said inputs, which is the expected mathematical value.
291
+ Defaults to ``False``.
292
+
293
+ Returns:
294
+ output (tuple): tuple with:
295
+ func_output (tuple of Tensors or Tensor): output of ``func(inputs)``
296
+
297
+ vjp (tuple of Tensors or Tensor): result of the dot product with
298
+ the same shape as the inputs.
299
+
300
+ Example:
301
+
302
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
303
+ >>> def exp_reducer(x):
304
+ ... return x.exp().sum(dim=1)
305
+ >>> inputs = torch.rand(4, 4)
306
+ >>> v = torch.ones(4)
307
+ >>> # xdoctest: +IGNORE_WANT("non-deterministic")
308
+ >>> vjp(exp_reducer, inputs, v)
309
+ (tensor([5.7817, 7.2458, 5.7830, 6.7782]),
310
+ tensor([[1.4458, 1.3962, 1.3042, 1.6354],
311
+ [2.1288, 1.0652, 1.5483, 2.5035],
312
+ [2.2046, 1.1292, 1.1432, 1.3059],
313
+ [1.3225, 1.6652, 1.7753, 2.0152]]))
314
+
315
+ >>> vjp(exp_reducer, inputs, v, create_graph=True)
316
+ (tensor([5.7817, 7.2458, 5.7830, 6.7782], grad_fn=<SumBackward1>),
317
+ tensor([[1.4458, 1.3962, 1.3042, 1.6354],
318
+ [2.1288, 1.0652, 1.5483, 2.5035],
319
+ [2.2046, 1.1292, 1.1432, 1.3059],
320
+ [1.3225, 1.6652, 1.7753, 2.0152]], grad_fn=<MulBackward0>))
321
+
322
+ >>> def adder(x, y):
323
+ ... return 2 * x + 3 * y
324
+ >>> inputs = (torch.rand(2), torch.rand(2))
325
+ >>> v = torch.ones(2)
326
+ >>> vjp(adder, inputs, v)
327
+ (tensor([2.4225, 2.3340]),
328
+ (tensor([2., 2.]), tensor([3., 3.])))
329
+ """
330
+ with torch.enable_grad():
331
+ is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "vjp")
332
+ inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True)
333
+
334
+ outputs = func(*inputs)
335
+ is_outputs_tuple, outputs = _as_tuple(
336
+ outputs, "outputs of the user-provided function", "vjp"
337
+ )
338
+ _check_requires_grad(outputs, "outputs", strict=strict)
339
+
340
+ if v is not None:
341
+ _, v = _as_tuple(v, "v", "vjp")
342
+ v = _grad_preprocess(v, create_graph=create_graph, need_graph=False)
343
+ _validate_v(v, outputs, is_outputs_tuple)
344
+ else:
345
+ if len(outputs) != 1 or outputs[0].nelement() != 1:
346
+ raise RuntimeError(
347
+ "The vector v can only be None if the "
348
+ "user-provided function returns "
349
+ "a single Tensor with a single element."
350
+ )
351
+
352
+ enable_grad = True if create_graph else torch.is_grad_enabled()
353
+ with torch.set_grad_enabled(enable_grad):
354
+ grad_res = _autograd_grad(outputs, inputs, v, create_graph=create_graph)
355
+ vjp = _fill_in_zeros(grad_res, inputs, strict, create_graph, "back")
356
+
357
+ # Cleanup objects and return them to the user
358
+ outputs = _grad_postprocess(outputs, create_graph)
359
+ vjp = _grad_postprocess(vjp, create_graph)
360
+
361
+ return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess(
362
+ vjp, is_inputs_tuple
363
+ )
364
+
365
+
366
+ def jvp(func, inputs, v=None, create_graph=False, strict=False):
367
+ r"""Compute the dot product between the Jacobian of the given function at the point given by the inputs and a vector ``v``.
368
+
369
+ Args:
370
+ func (function): a Python function that takes Tensor inputs and returns
371
+ a tuple of Tensors or a Tensor.
372
+ inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
373
+ v (tuple of Tensors or Tensor): The vector for which the Jacobian
374
+ vector product is computed. Must be the same size as the input of
375
+ ``func``. This argument is optional when the input to ``func``
376
+ contains a single element and (if it is not provided) will be set
377
+ as a Tensor containing a single ``1``.
378
+ create_graph (bool, optional): If ``True``, both the output and result
379
+ will be computed in a differentiable way. Note that when ``strict``
380
+ is ``False``, the result can not require gradients or be
381
+ disconnected from the inputs. Defaults to ``False``.
382
+ strict (bool, optional): If ``True``, an error will be raised when we
383
+ detect that there exists an input such that all the outputs are
384
+ independent of it. If ``False``, we return a Tensor of zeros as the
385
+ jvp for said inputs, which is the expected mathematical value.
386
+ Defaults to ``False``.
387
+
388
+ Returns:
389
+ output (tuple): tuple with:
390
+ func_output (tuple of Tensors or Tensor): output of ``func(inputs)``
391
+
392
+ jvp (tuple of Tensors or Tensor): result of the dot product with
393
+ the same shape as the output.
394
+
395
+ Note:
396
+ ``autograd.functional.jvp`` computes the jvp by using the backward of
397
+ the backward (sometimes called the double backwards trick). This is not
398
+ the most performant way of computing the jvp. Please consider using
399
+ :func:`torch.func.jvp` or the
400
+ :ref:`low-level forward-mode AD API <forward-mode-ad>` instead.
401
+
402
+ Example:
403
+
404
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
405
+ >>> def exp_reducer(x):
406
+ ... return x.exp().sum(dim=1)
407
+ >>> inputs = torch.rand(4, 4)
408
+ >>> v = torch.ones(4, 4)
409
+ >>> # xdoctest: +IGNORE_WANT("non-deterministic")
410
+ >>> jvp(exp_reducer, inputs, v)
411
+ (tensor([6.3090, 4.6742, 7.9114, 8.2106]),
412
+ tensor([6.3090, 4.6742, 7.9114, 8.2106]))
413
+
414
+ >>> jvp(exp_reducer, inputs, v, create_graph=True)
415
+ (tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=<SumBackward1>),
416
+ tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=<SqueezeBackward1>))
417
+
418
+ >>> def adder(x, y):
419
+ ... return 2 * x + 3 * y
420
+ >>> inputs = (torch.rand(2), torch.rand(2))
421
+ >>> v = (torch.ones(2), torch.ones(2))
422
+ >>> jvp(adder, inputs, v)
423
+ (tensor([2.2399, 2.5005]),
424
+ tensor([5., 5.]))
425
+
426
+ """
427
+ with torch.enable_grad():
428
+ is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jvp")
429
+ inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True)
430
+
431
+ if v is not None:
432
+ _, v = _as_tuple(v, "v", "jvp")
433
+ v = _grad_preprocess(v, create_graph=create_graph, need_graph=False)
434
+ _validate_v(v, inputs, is_inputs_tuple)
435
+ else:
436
+ if len(inputs) != 1 or inputs[0].nelement() != 1:
437
+ raise RuntimeError(
438
+ "The vector v can only be None if the input to "
439
+ "the user-provided function is a single Tensor "
440
+ "with a single element."
441
+ )
442
+
443
+ outputs = func(*inputs)
444
+ is_outputs_tuple, outputs = _as_tuple(
445
+ outputs, "outputs of the user-provided function", "jvp"
446
+ )
447
+ _check_requires_grad(outputs, "outputs", strict=strict)
448
+ # The backward is linear so the value of grad_outputs is not important as
449
+ # it won't appear in the double backward graph. We only need to ensure that
450
+ # it does not contain inf or nan.
451
+ grad_outputs = tuple(
452
+ torch.zeros_like(out, requires_grad=True) for out in outputs
453
+ )
454
+
455
+ grad_inputs = _autograd_grad(outputs, inputs, grad_outputs, create_graph=True)
456
+ _check_requires_grad(grad_inputs, "grad_inputs", strict=strict)
457
+
458
+ if create_graph:
459
+ with torch.enable_grad():
460
+ grad_res = _autograd_grad(
461
+ grad_inputs, grad_outputs, v, create_graph=create_graph
462
+ )
463
+ jvp = _fill_in_zeros(grad_res, outputs, strict, create_graph, "back_trick")
464
+ else:
465
+ grad_res = _autograd_grad(
466
+ grad_inputs, grad_outputs, v, create_graph=create_graph
467
+ )
468
+ jvp = _fill_in_zeros(grad_res, outputs, strict, create_graph, "back_trick")
469
+
470
+ # Cleanup objects and return them to the user
471
+ outputs = _grad_postprocess(outputs, create_graph)
472
+ jvp = _grad_postprocess(jvp, create_graph)
473
+
474
+ return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess(
475
+ jvp, is_outputs_tuple
476
+ )
477
+
478
+
479
+ def _construct_standard_basis_for(
480
+ tensors: tuple[torch.Tensor, ...], tensor_numels: tuple[int, ...]
481
+ ) -> tuple[torch.Tensor, ...]:
482
+ # This function:
483
+ # - constructs a N=sum(tensor_numels) standard basis. i.e. an NxN identity matrix.
484
+ # - Splits the identity matrix into chunks with each chunk size determined by `tensor_numels`.
485
+ # - Each chunk corresponds to one tensor. The chunk has the same dtype and
486
+ # device as the tensor
487
+ #
488
+ # For example, with tensor_numels = [1, 2, 1], this function returns:
489
+ # ( tensor([[1], tensor([[0, 0], tensor([[0],
490
+ # [0], [1, 0], [0],
491
+ # [0], [0, 1], [0],
492
+ # [0]]) , [0, 0]]) , [1]]) )
493
+ #
494
+ # Precondition: tensor_numels == tuple(tensor.numel() for tensor in tensors)
495
+ # Precondition: tensors always has at least one element.
496
+ #
497
+ # See NOTE: [Computing jacobian with vmap and grad for multiple tensors]
498
+ # for context behind this function. All the pre-conditions are guarded for
499
+ # in torch.autograd.functional.jacobian.
500
+ if len(tensors) != len(tensor_numels):
501
+ raise AssertionError(
502
+ f"Expected tensors and tensor_numels to have the same length, "
503
+ f"but got {len(tensors)} and {len(tensor_numels)}"
504
+ )
505
+ if len(tensors) == 0:
506
+ raise AssertionError("Expected at least one tensor")
507
+ total_numel = sum(tensor_numels)
508
+ chunks = tuple(
509
+ tensor.new_zeros(total_numel, tensor_numel)
510
+ for tensor, tensor_numel in zip(tensors, tensor_numels)
511
+ )
512
+ diag_start_idx = 0
513
+ for chunk, numel in zip(chunks, tensor_numels):
514
+ chunk.diagonal(diag_start_idx).fill_(1)
515
+ diag_start_idx -= numel
516
+ return chunks
517
+
518
+
519
+ def _jacfwd(func, inputs, strict=False, vectorize=False):
520
+ if strict:
521
+ raise RuntimeError(
522
+ "torch.autograd.functional.jacobian: `strict=True` "
523
+ 'and `strategy="forward-mode"` are not supported together (yet). '
524
+ "Please either set `strict=False` or "
525
+ '`strategy="reverse-mode"`.'
526
+ )
527
+ is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jacobian")
528
+ output_info = []
529
+
530
+ if vectorize:
531
+ # See NOTE: [Computing jacobian with vmap and grad for multiple outputs]
532
+ input_numels = tuple(input.numel() for input in inputs)
533
+
534
+ # Step 1: Prepare tangents
535
+ tangents = _construct_standard_basis_for(inputs, input_numels)
536
+
537
+ # Step 2: Compute vmap over computation with dual tensors
538
+ def jvp(tangents):
539
+ with fwAD.dual_level():
540
+ dual_inputs = tuple(
541
+ fwAD.make_dual(input, tangent.view_as(input))
542
+ for input, tangent in zip(inputs, tangents)
543
+ )
544
+ _is_outputs_tuple, dual_outputs = _as_tuple(
545
+ func(*dual_inputs), "outputs"
546
+ )
547
+ output_info.append(_is_outputs_tuple)
548
+ jv = []
549
+ primal_outs = []
550
+ for dual_out in dual_outputs:
551
+ primal, tangent = fwAD.unpack_dual(dual_out)
552
+ primal_outs.append(primal)
553
+ if tangent is not None:
554
+ jv.append(tangent)
555
+ else:
556
+ jv.append(torch.zeros_like(primal))
557
+ output_info.append(primal_outs)
558
+ return tuple(jv)
559
+
560
+ outputs_before_split = _vmap(jvp)(tangents)
561
+ is_outputs_tuple, outputs = output_info
562
+ # Step 3: for each of the output tangents, split along dim 0
563
+ jacobian_input_output = []
564
+ for jac_output_i, output_i in zip(outputs_before_split, outputs):
565
+ jacobian_output_i_output = []
566
+ for jac, input_j in zip(jac_output_i.split(input_numels, dim=0), inputs):
567
+ # We need to transpose the Jacobian because in forward AD, the
568
+ # batch dimension represents that of the inputs
569
+ jacobian_input_i_output_j = jac.permute(*range(1, jac.ndim), 0).reshape(
570
+ (*output_i.shape, *input_j.shape)
571
+ )
572
+
573
+ jacobian_output_i_output.append(jacobian_input_i_output_j)
574
+ jacobian_input_output.append(jacobian_output_i_output)
575
+
576
+ # Omit [Step 4] because everything is already transposed w/ forward AD
577
+ return _tuple_postprocess(
578
+ jacobian_input_output, (is_outputs_tuple, is_inputs_tuple)
579
+ )
580
+ else:
581
+ raise NotImplementedError(
582
+ "Computing Jacobian using forward-AD or forward-over-reverse Hessian is"
583
+ "only implemented for `vectorize=True`."
584
+ )
585
+
586
+
587
+ def jacobian(
588
+ func,
589
+ inputs,
590
+ create_graph=False,
591
+ strict=False,
592
+ vectorize=False,
593
+ strategy="reverse-mode",
594
+ ):
595
+ r"""Compute the Jacobian of a given function.
596
+
597
+ Args:
598
+ func (function): a Python function that takes Tensor inputs and returns
599
+ a tuple of Tensors or a Tensor.
600
+ inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
601
+ create_graph (bool, optional): If ``True``, the Jacobian will be
602
+ computed in a differentiable manner. Note that when ``strict`` is
603
+ ``False``, the result can not require gradients or be disconnected
604
+ from the inputs. Defaults to ``False``.
605
+ strict (bool, optional): If ``True``, an error will be raised when we
606
+ detect that there exists an input such that all the outputs are
607
+ independent of it. If ``False``, we return a Tensor of zeros as the
608
+ jacobian for said inputs, which is the expected mathematical value.
609
+ Defaults to ``False``.
610
+ vectorize (bool, optional): This feature is experimental.
611
+ Please consider using :func:`torch.func.jacrev` or
612
+ :func:`torch.func.jacfwd` instead if you are looking for something
613
+ less experimental and more performant.
614
+ When computing the jacobian, usually we invoke
615
+ ``autograd.grad`` once per row of the jacobian. If this flag is
616
+ ``True``, we perform only a single ``autograd.grad`` call with
617
+ ``batched_grad=True`` which uses the vmap prototype feature.
618
+ Though this should lead to performance improvements in many cases,
619
+ because this feature is still experimental, there may be performance
620
+ cliffs. See :func:`torch.autograd.grad`'s ``batched_grad`` parameter for
621
+ more information.
622
+ strategy (str, optional): Set to ``"forward-mode"`` or ``"reverse-mode"`` to
623
+ determine whether the Jacobian will be computed with forward or reverse
624
+ mode AD. Currently, ``"forward-mode"`` requires ``vectorized=True``.
625
+ Defaults to ``"reverse-mode"``. If ``func`` has more outputs than
626
+ inputs, ``"forward-mode"`` tends to be more performant. Otherwise,
627
+ prefer to use ``"reverse-mode"``.
628
+
629
+ Returns:
630
+ Jacobian (Tensor or nested tuple of Tensors): if there is a single
631
+ input and output, this will be a single Tensor containing the
632
+ Jacobian for the linearized inputs and output. If one of the two is
633
+ a tuple, then the Jacobian will be a tuple of Tensors. If both of
634
+ them are tuples, then the Jacobian will be a tuple of tuple of
635
+ Tensors where ``Jacobian[i][j]`` will contain the Jacobian of the
636
+ ``i``\th output and ``j``\th input and will have as size the
637
+ concatenation of the sizes of the corresponding output and the
638
+ corresponding input and will have same dtype and device as the
639
+ corresponding input. If strategy is ``forward-mode``, the dtype will be
640
+ that of the output; otherwise, the input.
641
+
642
+ Example:
643
+
644
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
645
+ >>> def exp_reducer(x):
646
+ ... return x.exp().sum(dim=1)
647
+ >>> inputs = torch.rand(2, 2)
648
+ >>> # xdoctest: +IGNORE_WANT("non-deterministic")
649
+ >>> jacobian(exp_reducer, inputs)
650
+ tensor([[[1.4917, 2.4352],
651
+ [0.0000, 0.0000]],
652
+ [[0.0000, 0.0000],
653
+ [2.4369, 2.3799]]])
654
+
655
+ >>> jacobian(exp_reducer, inputs, create_graph=True)
656
+ tensor([[[1.4917, 2.4352],
657
+ [0.0000, 0.0000]],
658
+ [[0.0000, 0.0000],
659
+ [2.4369, 2.3799]]], grad_fn=<ViewBackward>)
660
+
661
+ >>> def exp_adder(x, y):
662
+ ... return 2 * x.exp() + 3 * y
663
+ >>> inputs = (torch.rand(2), torch.rand(2))
664
+ >>> jacobian(exp_adder, inputs)
665
+ (tensor([[2.8052, 0.0000],
666
+ [0.0000, 3.3963]]),
667
+ tensor([[3., 0.],
668
+ [0., 3.]]))
669
+
670
+ >>> def linear_model(x):
671
+ ... W = torch.tensor([[2.0, -1.0], [0.0, 1.0]])
672
+ ... b = torch.tensor([1.0, 0.5])
673
+ ... return x @ W.T + b
674
+
675
+ >>> x = torch.randn(4, 2, requires_grad=True)
676
+ >>> jac = jacobian(linear_model, x, vectorize=True)
677
+ >>> jac.shape
678
+ torch.Size([4, 2, 4, 2])
679
+ """
680
+ if strategy not in ("forward-mode", "reverse-mode"):
681
+ raise AssertionError(
682
+ 'Expected strategy to be either "forward-mode" or "reverse-mode". Hint: If your '
683
+ 'function has more outputs than inputs, "forward-mode" tends to be more performant. '
684
+ 'Otherwise, prefer to use "reverse-mode".'
685
+ )
686
+ if strategy == "forward-mode":
687
+ if create_graph:
688
+ raise NotImplementedError(
689
+ "torch.autograd.functional.jacobian: `create_graph=True` "
690
+ 'and `strategy="forward-mode"` are not supported together (yet). '
691
+ "Please either set `create_graph=False` or "
692
+ '`strategy="reverse-mode"`.'
693
+ )
694
+ return _jacfwd(func, inputs, strict, vectorize)
695
+
696
+ with torch.enable_grad():
697
+ is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jacobian")
698
+ inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True)
699
+
700
+ outputs = func(*inputs)
701
+ is_outputs_tuple, outputs = _as_tuple(
702
+ outputs, "outputs of the user-provided function", "jacobian"
703
+ )
704
+ _check_requires_grad(outputs, "outputs", strict=strict)
705
+
706
+ if vectorize:
707
+ if strict:
708
+ raise RuntimeError(
709
+ "torch.autograd.functional.jacobian: `strict=True` "
710
+ "and `vectorized=True` are not supported together. "
711
+ "Please either set `strict=False` or "
712
+ "`vectorize=False`."
713
+ )
714
+ # NOTE: [Computing jacobian with vmap and grad for multiple outputs]
715
+ #
716
+ # Let's consider f(x) = (x**2, x.sum()) and let x = torch.randn(3).
717
+ # It turns out we can compute the jacobian of this function with a single
718
+ # call to autograd.grad by using vmap over the correct grad_outputs.
719
+ #
720
+ # Firstly, one way to compute the jacobian is to stack x**2 and x.sum()
721
+ # into a 4D vector. E.g., use g(x) = torch.stack([x**2, x.sum()])
722
+ #
723
+ # To get the first row of the jacobian, we call
724
+ # >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([1, 0, 0, 0]))
725
+ # To get the 2nd row of the jacobian, we call
726
+ # >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([0, 1, 0, 0]))
727
+ # and so on.
728
+ #
729
+ # Using vmap, we can vectorize all 4 of these computations into one by
730
+ # passing the standard basis for R^4 as the grad_output.
731
+ # vmap(partial(autograd.grad, g(x), x))(torch.eye(4)).
732
+ #
733
+ # Now, how do we compute the jacobian *without stacking the output*?
734
+ # We can just split the standard basis across the outputs. So to
735
+ # compute the jacobian of f(x), we'd use
736
+ # >>> autograd.grad(f(x), x, grad_outputs=_construct_standard_basis_for(...))
737
+ # The grad_outputs looks like the following:
738
+ # ( torch.tensor([[1, 0, 0],
739
+ # [0, 1, 0],
740
+ # [0, 0, 1],
741
+ # [0, 0, 0]]),
742
+ # torch.tensor([[0],
743
+ # [0],
744
+ # [0],
745
+ # [1]]) )
746
+ #
747
+ # But we're not done yet!
748
+ # >>> vmap(partial(autograd.grad(f(x), x, grad_outputs=...)))
749
+ # returns a Tensor of shape [4, 3]. We have to remember to split the
750
+ # jacobian of shape [4, 3] into two:
751
+ # - one of shape [3, 3] for the first output
752
+ # - one of shape [ 3] for the second output
753
+
754
+ # Step 1: Construct grad_outputs by splitting the standard basis
755
+ output_numels = tuple(output.numel() for output in outputs)
756
+ grad_outputs = _construct_standard_basis_for(outputs, output_numels)
757
+ flat_outputs = tuple(output.reshape(-1) for output in outputs)
758
+
759
+ # Step 2: Call vmap + autograd.grad
760
+ def vjp(grad_output):
761
+ vj = list(
762
+ _autograd_grad(
763
+ flat_outputs,
764
+ inputs,
765
+ grad_output,
766
+ create_graph=create_graph,
767
+ is_grads_batched=True,
768
+ )
769
+ )
770
+ for el_idx, vj_el in enumerate(vj):
771
+ if vj_el is not None:
772
+ continue
773
+ vj[el_idx] = torch.zeros_like(inputs[el_idx]).expand(
774
+ (sum(output_numels),) + inputs[el_idx].shape
775
+ )
776
+ return tuple(vj)
777
+
778
+ jacobians_of_flat_output = vjp(grad_outputs)
779
+
780
+ # Step 3: The returned jacobian is one big tensor per input. In this step,
781
+ # we split each Tensor by output.
782
+ jacobian_input_output = []
783
+ for jac_input_i, input_i in zip(jacobians_of_flat_output, inputs):
784
+ jacobian_input_i_output = []
785
+ for jac, output_j in zip(
786
+ jac_input_i.split(output_numels, dim=0), outputs
787
+ ):
788
+ jacobian_input_i_output_j = jac.view(output_j.shape + input_i.shape)
789
+ jacobian_input_i_output.append(jacobian_input_i_output_j)
790
+ jacobian_input_output.append(jacobian_input_i_output)
791
+
792
+ # Step 4: Right now, `jacobian` is a List[List[Tensor]].
793
+ # The outer List corresponds to the number of inputs,
794
+ # the inner List corresponds to the number of outputs.
795
+ # We need to exchange the order of these and convert to tuples
796
+ # before returning.
797
+ jacobian_output_input = tuple(zip(*jacobian_input_output))
798
+
799
+ jacobian_output_input = _grad_postprocess(
800
+ jacobian_output_input, create_graph
801
+ )
802
+ return _tuple_postprocess(
803
+ jacobian_output_input, (is_outputs_tuple, is_inputs_tuple)
804
+ )
805
+
806
+ jacobian: tuple[torch.Tensor, ...] = ()
807
+
808
+ for i, out in enumerate(outputs):
809
+ # mypy complains that expression and variable have different types due to the empty list
810
+ jac_i: tuple[list[torch.Tensor]] = tuple([] for _ in range(len(inputs))) # type: ignore[assignment]
811
+ for j in range(out.nelement()):
812
+ vj = _autograd_grad(
813
+ (out.reshape(-1)[j],),
814
+ inputs,
815
+ retain_graph=True,
816
+ create_graph=create_graph,
817
+ )
818
+
819
+ for el_idx, (jac_i_el, vj_el, inp_el) in enumerate(
820
+ zip(jac_i, vj, inputs)
821
+ ):
822
+ if vj_el is not None:
823
+ if strict and create_graph and not vj_el.requires_grad:
824
+ msg = (
825
+ "The jacobian of the user-provided function is "
826
+ f"independent of input {i}. This is not allowed in "
827
+ "strict mode when create_graph=True."
828
+ )
829
+ raise RuntimeError(msg)
830
+ jac_i_el.append(vj_el)
831
+ else:
832
+ if strict:
833
+ msg = (
834
+ f"Output {i} of the user-provided function is "
835
+ f"independent of input {el_idx}. This is not allowed in "
836
+ "strict mode."
837
+ )
838
+ raise RuntimeError(msg)
839
+ jac_i_el.append(torch.zeros_like(inp_el))
840
+
841
+ # pyrefly: ignore [bad-assignment]
842
+ jacobian += (
843
+ tuple(
844
+ torch.stack(jac_i_el, dim=0).view(
845
+ out.size() + inputs[el_idx].size() # type: ignore[operator]
846
+ )
847
+ for (el_idx, jac_i_el) in enumerate(jac_i)
848
+ ),
849
+ )
850
+
851
+ jacobian = _grad_postprocess(jacobian, create_graph)
852
+
853
+ return _tuple_postprocess(jacobian, (is_outputs_tuple, is_inputs_tuple))
854
+
855
+
856
+ def hessian(
857
+ func,
858
+ inputs,
859
+ create_graph=False,
860
+ strict=False,
861
+ vectorize=False,
862
+ outer_jacobian_strategy="reverse-mode",
863
+ ):
864
+ r"""Compute the Hessian of a given scalar function.
865
+
866
+ Args:
867
+ func (function): a Python function that takes Tensor inputs and returns
868
+ a Tensor with a single element.
869
+ inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
870
+ create_graph (bool, optional): If ``True``, the Hessian will be computed in
871
+ a differentiable manner. Note that when ``strict`` is ``False``, the result can not
872
+ require gradients or be disconnected from the inputs.
873
+ Defaults to ``False``.
874
+ strict (bool, optional): If ``True``, an error will be raised when we detect that there exists an input
875
+ such that all the outputs are independent of it. If ``False``, we return a Tensor of zeros as the
876
+ hessian for said inputs, which is the expected mathematical value.
877
+ Defaults to ``False``.
878
+ vectorize (bool, optional): This feature is experimental.
879
+ Please consider using :func:`torch.func.hessian`
880
+ instead if you are looking for something less experimental and more performant.
881
+ When computing the hessian, usually we invoke
882
+ ``autograd.grad`` once per row of the hessian. If this flag is
883
+ ``True``, we use the vmap prototype feature as the backend to
884
+ vectorize calls to ``autograd.grad`` so we only invoke it once
885
+ instead of once per row. This should lead to performance
886
+ improvements in many use cases, however, due to this feature
887
+ being incomplete, there may be performance cliffs. Please
888
+ use `torch._C._debug_only_display_vmap_fallback_warnings(True)`
889
+ to show any performance warnings and file us issues if
890
+ warnings exist for your use case. Defaults to ``False``.
891
+ outer_jacobian_strategy (str, optional): The Hessian is computed by
892
+ computing the Jacobian of a Jacobian. The inner Jacobian is always
893
+ computed in reverse-mode AD. Setting strategy to ``"forward-mode"``
894
+ or ``"reverse-mode"`` determines whether the outer Jacobian will be
895
+ computed with forward or reverse mode AD. Currently, computing the outer
896
+ Jacobian in ``"forward-mode"`` requires ``vectorized=True``. Defaults
897
+ to ``"reverse-mode"``.
898
+
899
+ Returns:
900
+ Hessian (Tensor or a tuple of tuple of Tensors): if there is a single input,
901
+ this will be a single Tensor containing the Hessian for the input.
902
+ If it is a tuple, then the Hessian will be a tuple of tuples where
903
+ ``Hessian[i][j]`` will contain the Hessian of the ``i``\th input
904
+ and ``j``\th input with size the sum of the size of the ``i``\th input plus
905
+ the size of the ``j``\th input. ``Hessian[i][j]`` will have the same
906
+ dtype and device as the corresponding ``i``\th input.
907
+
908
+ Example:
909
+
910
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
911
+ >>> def pow_reducer(x):
912
+ ... return x.pow(3).sum()
913
+ >>> inputs = torch.rand(2, 2)
914
+ >>> # xdoctest: +IGNORE_WANT("non-deterministic")
915
+ >>> hessian(pow_reducer, inputs)
916
+ tensor([[[[5.2265, 0.0000],
917
+ [0.0000, 0.0000]],
918
+ [[0.0000, 4.8221],
919
+ [0.0000, 0.0000]]],
920
+ [[[0.0000, 0.0000],
921
+ [1.9456, 0.0000]],
922
+ [[0.0000, 0.0000],
923
+ [0.0000, 3.2550]]]])
924
+
925
+ >>> hessian(pow_reducer, inputs, create_graph=True)
926
+ tensor([[[[5.2265, 0.0000],
927
+ [0.0000, 0.0000]],
928
+ [[0.0000, 4.8221],
929
+ [0.0000, 0.0000]]],
930
+ [[[0.0000, 0.0000],
931
+ [1.9456, 0.0000]],
932
+ [[0.0000, 0.0000],
933
+ [0.0000, 3.2550]]]], grad_fn=<ViewBackward>)
934
+
935
+
936
+ >>> def pow_adder_reducer(x, y):
937
+ ... return (2 * x.pow(2) + 3 * y.pow(2)).sum()
938
+ >>> inputs = (torch.rand(2), torch.rand(2))
939
+ >>> hessian(pow_adder_reducer, inputs)
940
+ ((tensor([[4., 0.],
941
+ [0., 4.]]),
942
+ tensor([[0., 0.],
943
+ [0., 0.]])),
944
+ (tensor([[0., 0.],
945
+ [0., 0.]]),
946
+ tensor([[6., 0.],
947
+ [0., 6.]])))
948
+ """
949
+ is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "hessian")
950
+ if outer_jacobian_strategy not in (
951
+ "forward-mode",
952
+ "reverse-mode",
953
+ ):
954
+ raise AssertionError(
955
+ 'Expected strategy to be either "forward-mode" or "reverse-mode".'
956
+ )
957
+
958
+ def ensure_single_output_function(*inp):
959
+ out = func(*inp)
960
+ is_out_tuple, t_out = _as_tuple(
961
+ out, "outputs of the user-provided function", "hessian"
962
+ )
963
+ _check_requires_grad(t_out, "outputs", strict=strict)
964
+
965
+ if is_out_tuple or not isinstance(out, torch.Tensor):
966
+ raise RuntimeError(
967
+ "The function given to hessian should return a single Tensor"
968
+ )
969
+
970
+ if out.nelement() != 1:
971
+ raise RuntimeError(
972
+ "The Tensor returned by the function given to hessian should contain a single element"
973
+ )
974
+
975
+ return out.squeeze()
976
+
977
+ def jac_func(*inp):
978
+ if outer_jacobian_strategy == "forward-mode":
979
+ # _grad_preprocess requires create_graph=True and input to require_grad
980
+ # or else the input will be detached
981
+ inp = tuple(t.requires_grad_(True) for t in inp)
982
+ jac = jacobian(ensure_single_output_function, inp, create_graph=True)
983
+ _check_requires_grad(jac, "jacobian", strict=strict)
984
+ return jac
985
+
986
+ res = jacobian(
987
+ jac_func,
988
+ inputs,
989
+ create_graph=create_graph,
990
+ strict=strict,
991
+ vectorize=vectorize,
992
+ strategy=outer_jacobian_strategy,
993
+ )
994
+ return _tuple_postprocess(res, (is_inputs_tuple, is_inputs_tuple))
995
+
996
+
997
+ def vhp(func, inputs, v=None, create_graph=False, strict=False):
998
+ r"""Compute the dot product between vector ``v`` and Hessian of a given scalar function at a specified point.
999
+
1000
+ Args:
1001
+ func (function): a Python function that takes Tensor inputs and returns
1002
+ a Tensor with a single element.
1003
+ inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
1004
+ v (tuple of Tensors or Tensor): The vector for which the vector Hessian
1005
+ product is computed. Must be the same size as the input of
1006
+ ``func``. This argument is optional when ``func``'s input contains
1007
+ a single element and (if it is not provided) will be set as a
1008
+ Tensor containing a single ``1``.
1009
+ create_graph (bool, optional): If ``True``, both the output and result
1010
+ will be computed in a differentiable way. Note that when ``strict``
1011
+ is ``False``, the result can not require gradients or be
1012
+ disconnected from the inputs.
1013
+ Defaults to ``False``.
1014
+ strict (bool, optional): If ``True``, an error will be raised when we
1015
+ detect that there exists an input such that all the outputs are
1016
+ independent of it. If ``False``, we return a Tensor of zeros as the
1017
+ vhp for said inputs, which is the expected mathematical value.
1018
+ Defaults to ``False``.
1019
+
1020
+ Returns:
1021
+ output (tuple): tuple with:
1022
+ func_output (tuple of Tensors or Tensor): output of ``func(inputs)``
1023
+
1024
+ vhp (tuple of Tensors or Tensor): result of the dot product with the
1025
+ same shape as the inputs.
1026
+
1027
+ Example:
1028
+
1029
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
1030
+ >>> def pow_reducer(x):
1031
+ ... return x.pow(3).sum()
1032
+ >>> inputs = torch.rand(2, 2)
1033
+ >>> v = torch.ones(2, 2)
1034
+ >>> # xdoctest: +IGNORE_WANT("non-deterministic")
1035
+ >>> vhp(pow_reducer, inputs, v)
1036
+ (tensor(0.5591),
1037
+ tensor([[1.0689, 1.2431],
1038
+ [3.0989, 4.4456]]))
1039
+ >>> vhp(pow_reducer, inputs, v, create_graph=True)
1040
+ (tensor(0.5591, grad_fn=<SumBackward0>),
1041
+ tensor([[1.0689, 1.2431],
1042
+ [3.0989, 4.4456]], grad_fn=<MulBackward0>))
1043
+ >>> def pow_adder_reducer(x, y):
1044
+ ... return (2 * x.pow(2) + 3 * y.pow(2)).sum()
1045
+ >>> inputs = (torch.rand(2), torch.rand(2))
1046
+ >>> v = (torch.zeros(2), torch.ones(2))
1047
+ >>> vhp(pow_adder_reducer, inputs, v)
1048
+ (tensor(4.8053),
1049
+ (tensor([0., 0.]),
1050
+ tensor([6., 6.])))
1051
+ """
1052
+ with torch.enable_grad():
1053
+ is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "vhp")
1054
+ inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True)
1055
+
1056
+ if v is not None:
1057
+ _, v = _as_tuple(v, "v", "vhp")
1058
+ v = _grad_preprocess(v, create_graph=create_graph, need_graph=False)
1059
+ _validate_v(v, inputs, is_inputs_tuple)
1060
+ else:
1061
+ if len(inputs) != 1 or inputs[0].nelement() != 1:
1062
+ raise RuntimeError(
1063
+ "The vector v can only be None if the input to the user-provided function "
1064
+ "is a single Tensor with a single element."
1065
+ )
1066
+ outputs = func(*inputs)
1067
+ is_outputs_tuple, outputs = _as_tuple(
1068
+ outputs, "outputs of the user-provided function", "vhp"
1069
+ )
1070
+ _check_requires_grad(outputs, "outputs", strict=strict)
1071
+
1072
+ if is_outputs_tuple or not isinstance(outputs[0], torch.Tensor):
1073
+ raise RuntimeError(
1074
+ "The function given to vhp should return a single Tensor"
1075
+ )
1076
+
1077
+ if outputs[0].nelement() != 1:
1078
+ raise RuntimeError(
1079
+ "The Tensor returned by the function given to vhp should contain a single element"
1080
+ )
1081
+
1082
+ jac = _autograd_grad(outputs, inputs, create_graph=True)
1083
+ _check_requires_grad(jac, "jacobian", strict=strict)
1084
+
1085
+ enable_grad = True if create_graph else torch.is_grad_enabled()
1086
+ with torch.set_grad_enabled(enable_grad):
1087
+ grad_res = _autograd_grad(jac, inputs, v, create_graph=create_graph)
1088
+ vhp = _fill_in_zeros(grad_res, inputs, strict, create_graph, "double_back")
1089
+
1090
+ outputs = _grad_postprocess(outputs, create_graph)
1091
+ vhp = _grad_postprocess(vhp, create_graph)
1092
+
1093
+ return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess(
1094
+ vhp, is_inputs_tuple
1095
+ )
1096
+
1097
+
1098
+ def hvp(func, inputs, v=None, create_graph=False, strict=False):
1099
+ r"""Compute the dot product between the scalar function's Hessian and a vector ``v`` at a specified point.
1100
+
1101
+ Args:
1102
+ func (function): a Python function that takes Tensor inputs and returns
1103
+ a Tensor with a single element.
1104
+ inputs (tuple of Tensors or Tensor): inputs to the function ``func``.
1105
+ v (tuple of Tensors or Tensor): The vector for which the Hessian vector
1106
+ product is computed. Must be the same size as the input of
1107
+ ``func``. This argument is optional when ``func``'s input contains
1108
+ a single element and (if it is not provided) will be set as a
1109
+ Tensor containing a single ``1``.
1110
+ create_graph (bool, optional): If ``True``, both the output and result will be
1111
+ computed in a differentiable way. Note that when ``strict`` is
1112
+ ``False``, the result can not require gradients or be disconnected
1113
+ from the inputs. Defaults to ``False``.
1114
+ strict (bool, optional): If ``True``, an error will be raised when we
1115
+ detect that there exists an input such that all the outputs are
1116
+ independent of it. If ``False``, we return a Tensor of zeros as the
1117
+ hvp for said inputs, which is the expected mathematical value.
1118
+ Defaults to ``False``.
1119
+ Returns:
1120
+ output (tuple): tuple with:
1121
+ func_output (tuple of Tensors or Tensor): output of ``func(inputs)``
1122
+
1123
+ hvp (tuple of Tensors or Tensor): result of the dot product with
1124
+ the same shape as the inputs.
1125
+
1126
+ Example:
1127
+
1128
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
1129
+ >>> def pow_reducer(x):
1130
+ ... return x.pow(3).sum()
1131
+ >>> inputs = torch.rand(2, 2)
1132
+ >>> v = torch.ones(2, 2)
1133
+ >>> # xdoctest: +IGNORE_WANT("non-deterministic")
1134
+ >>> hvp(pow_reducer, inputs, v)
1135
+ (tensor(0.1448),
1136
+ tensor([[2.0239, 1.6456],
1137
+ [2.4988, 1.4310]]))
1138
+
1139
+ >>> hvp(pow_reducer, inputs, v, create_graph=True)
1140
+ (tensor(0.1448, grad_fn=<SumBackward0>),
1141
+ tensor([[2.0239, 1.6456],
1142
+ [2.4988, 1.4310]], grad_fn=<MulBackward0>))
1143
+
1144
+
1145
+ >>> def pow_adder_reducer(x, y):
1146
+ ... return (2 * x.pow(2) + 3 * y.pow(2)).sum()
1147
+ >>> inputs = (torch.rand(2), torch.rand(2))
1148
+ >>> v = (torch.zeros(2), torch.ones(2))
1149
+ >>> hvp(pow_adder_reducer, inputs, v)
1150
+ (tensor(2.3030),
1151
+ (tensor([0., 0.]),
1152
+ tensor([6., 6.])))
1153
+
1154
+ Note:
1155
+
1156
+ This function is significantly slower than `vhp` due to backward mode AD constraints.
1157
+ If your functions is twice continuously differentiable, then hvp = vhp.t(). So if you
1158
+ know that your function satisfies this condition, you should use vhp instead that is
1159
+ much faster with the current implementation.
1160
+
1161
+ """
1162
+ with torch.enable_grad():
1163
+ is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "hvp")
1164
+ inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True)
1165
+
1166
+ if v is not None:
1167
+ _, v = _as_tuple(v, "v", "hvp")
1168
+ v = _grad_preprocess(v, create_graph=create_graph, need_graph=False)
1169
+ _validate_v(v, inputs, is_inputs_tuple)
1170
+ else:
1171
+ if len(inputs) != 1 or inputs[0].nelement() != 1:
1172
+ raise RuntimeError(
1173
+ "The vector v can only be None if the input to the user-provided function "
1174
+ "is a single Tensor with a single element."
1175
+ )
1176
+ outputs = func(*inputs)
1177
+ is_outputs_tuple, outputs = _as_tuple(
1178
+ outputs, "outputs of the user-provided function", "hvp"
1179
+ )
1180
+ _check_requires_grad(outputs, "outputs", strict=strict)
1181
+
1182
+ if is_outputs_tuple or not isinstance(outputs[0], torch.Tensor):
1183
+ raise RuntimeError(
1184
+ "The function given to hvp should return a single Tensor"
1185
+ )
1186
+
1187
+ if outputs[0].nelement() != 1:
1188
+ raise RuntimeError(
1189
+ "The Tensor returned by the function given to hvp should contain a single element"
1190
+ )
1191
+
1192
+ jac = _autograd_grad(outputs, inputs, create_graph=True)
1193
+ _check_requires_grad(jac, "jacobian", strict=strict)
1194
+
1195
+ grad_jac = tuple(torch.zeros_like(inp, requires_grad=True) for inp in inputs)
1196
+
1197
+ double_back = _autograd_grad(jac, inputs, grad_jac, create_graph=True)
1198
+ _check_requires_grad(jac, "hessian", strict=strict)
1199
+
1200
+ enable_grad = True if create_graph else torch.is_grad_enabled()
1201
+ with torch.set_grad_enabled(enable_grad):
1202
+ grad_res = _autograd_grad(double_back, grad_jac, v, create_graph=create_graph)
1203
+ hvp = _fill_in_zeros(
1204
+ grad_res, inputs, strict, create_graph, "double_back_trick"
1205
+ )
1206
+
1207
+ outputs = _grad_postprocess(outputs, create_graph)
1208
+ hvp = _grad_postprocess(hvp, create_graph)
1209
+
1210
+ return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess(
1211
+ hvp, is_inputs_tuple
1212
+ )
western_ai/usa/pytorch/torch/autograd/grad_mode.py ADDED
@@ -0,0 +1,489 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ from typing import Any
3
+
4
+ import torch
5
+ from torch.utils._contextlib import (
6
+ _DecoratorContextManager,
7
+ _NoParamDecoratorContextManager,
8
+ F,
9
+ )
10
+
11
+
12
+ __all__ = [
13
+ "no_grad",
14
+ "enable_grad",
15
+ "set_grad_enabled",
16
+ "inference_mode",
17
+ "set_multithreading_enabled",
18
+ "enforce_grad_layout_policy",
19
+ ]
20
+
21
+
22
+ class no_grad(_NoParamDecoratorContextManager):
23
+ r"""Context-manager that disables gradient calculation.
24
+
25
+ Disabling gradient calculation is useful for inference, when you are sure
26
+ that you will not call :meth:`Tensor.backward()`. It will reduce memory
27
+ consumption for computations that would otherwise have `requires_grad=True`.
28
+
29
+ In this mode, the result of every computation will have
30
+ `requires_grad=False`, even when the inputs have `requires_grad=True`.
31
+ There is an exception! All factory functions, or functions that create
32
+ a new Tensor and take a requires_grad kwarg, will NOT be affected by
33
+ this mode.
34
+
35
+ This context manager is thread local; it will not affect computation
36
+ in other threads.
37
+
38
+ Also functions as a decorator.
39
+
40
+ .. note::
41
+ No-grad is one of several mechanisms that can enable or
42
+ disable gradients locally see :ref:`locally-disable-grad-doc` for
43
+ more information on how they compare.
44
+
45
+ .. note::
46
+ This API does not apply to :ref:`forward-mode AD <forward-mode-ad>`.
47
+ If you want to disable forward AD for a computation, you can unpack
48
+ your dual tensors.
49
+
50
+ Example::
51
+ >>> # xdoctest: +SKIP
52
+ >>> x = torch.tensor([1.], requires_grad=True)
53
+ >>> with torch.no_grad():
54
+ ... y = x * 2
55
+ >>> y.requires_grad
56
+ False
57
+ >>> @torch.no_grad()
58
+ ... def doubler(x):
59
+ ... return x * 2
60
+ >>> z = doubler(x)
61
+ >>> z.requires_grad
62
+ False
63
+ >>> @torch.no_grad()
64
+ ... def tripler(x):
65
+ ... return x * 3
66
+ >>> z = tripler(x)
67
+ >>> z.requires_grad
68
+ False
69
+ >>> # factory function exception
70
+ >>> with torch.no_grad():
71
+ ... a = torch.nn.Parameter(torch.rand(10))
72
+ >>> a.requires_grad
73
+ True
74
+ """
75
+
76
+ def __init__(self) -> None:
77
+ if not torch._jit_internal.is_scripting():
78
+ super().__init__()
79
+ self.prev = False
80
+
81
+ def __enter__(self) -> None:
82
+ self.prev = torch.is_grad_enabled()
83
+ torch.set_grad_enabled(False)
84
+
85
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
86
+ torch.set_grad_enabled(self.prev)
87
+
88
+
89
+ class enable_grad(_NoParamDecoratorContextManager):
90
+ r"""Context-manager that enables gradient calculation.
91
+
92
+ Enables gradient calculation, if it has been disabled via :class:`~no_grad`
93
+ or :class:`~set_grad_enabled`.
94
+
95
+ This context manager is thread local; it will not affect computation
96
+ in other threads.
97
+
98
+ Also functions as a decorator.
99
+
100
+ .. note::
101
+ enable_grad is one of several mechanisms that can enable or
102
+ disable gradients locally see :ref:`locally-disable-grad-doc` for
103
+ more information on how they compare.
104
+
105
+ .. note::
106
+ This API does not apply to :ref:`forward-mode AD <forward-mode-ad>`.
107
+
108
+ Example::
109
+ >>> # xdoctest: +SKIP
110
+ >>> x = torch.tensor([1.], requires_grad=True)
111
+ >>> with torch.no_grad():
112
+ ... with torch.enable_grad():
113
+ ... y = x * 2
114
+ >>> y.requires_grad
115
+ True
116
+ >>> y.backward()
117
+ >>> x.grad
118
+ tensor([2.])
119
+ >>> @torch.enable_grad()
120
+ ... def doubler(x):
121
+ ... return x * 2
122
+ >>> with torch.no_grad():
123
+ ... z = doubler(x)
124
+ >>> z.requires_grad
125
+ True
126
+ >>> @torch.enable_grad()
127
+ ... def tripler(x):
128
+ ... return x * 3
129
+ >>> with torch.no_grad():
130
+ ... z = tripler(x)
131
+ >>> z.requires_grad
132
+ True
133
+
134
+ """
135
+
136
+ def __enter__(self) -> None:
137
+ self.prev = torch.is_grad_enabled()
138
+ torch._C._set_grad_enabled(True)
139
+
140
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
141
+ torch._C._set_grad_enabled(self.prev)
142
+
143
+
144
+ class set_grad_enabled(_DecoratorContextManager):
145
+ r"""Context-manager that sets gradient calculation on or off.
146
+
147
+ ``set_grad_enabled`` will enable or disable grads based on its argument :attr:`mode`.
148
+ It can be used as a context-manager or as a function.
149
+
150
+ This context manager is thread local; it will not affect computation
151
+ in other threads.
152
+
153
+ Args:
154
+ mode (bool): Flag whether to enable grad (``True``), or disable
155
+ (``False``). This can be used to conditionally enable
156
+ gradients.
157
+
158
+ .. note::
159
+ set_grad_enabled is one of several mechanisms that can enable or
160
+ disable gradients locally see :ref:`locally-disable-grad-doc` for
161
+ more information on how they compare.
162
+
163
+ .. note::
164
+ This API does not apply to :ref:`forward-mode AD <forward-mode-ad>`.
165
+
166
+ Example::
167
+ >>> # xdoctest: +SKIP
168
+ >>> x = torch.tensor([1.], requires_grad=True)
169
+ >>> is_train = False
170
+ >>> with torch.set_grad_enabled(is_train):
171
+ ... y = x * 2
172
+ >>> y.requires_grad
173
+ False
174
+ >>> _ = torch.set_grad_enabled(True)
175
+ >>> y = x * 2
176
+ >>> y.requires_grad
177
+ True
178
+ >>> _ = torch.set_grad_enabled(False)
179
+ >>> y = x * 2
180
+ >>> y.requires_grad
181
+ False
182
+
183
+ """
184
+
185
+ def __init__(self, mode: bool) -> None:
186
+ self.prev = torch.is_grad_enabled()
187
+ self.mode = mode
188
+ torch._C._set_grad_enabled(mode)
189
+
190
+ def __call__(self, orig_func: F) -> F:
191
+ torch._C._set_grad_enabled(self.prev)
192
+ return super().__call__(orig_func)
193
+
194
+ def __enter__(self) -> None:
195
+ torch._C._set_grad_enabled(self.mode)
196
+
197
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
198
+ torch._C._set_grad_enabled(self.prev)
199
+
200
+ def __str__(self) -> str:
201
+ return f"{torch.typename(self)}(mode={self.mode})"
202
+
203
+ def __repr__(self) -> str:
204
+ return str(self)
205
+
206
+ def clone(self) -> "set_grad_enabled":
207
+ r"""
208
+ Create a copy of this class
209
+ """
210
+ return self.__class__(self.mode)
211
+
212
+
213
+ class inference_mode(_DecoratorContextManager):
214
+ r"""Context manager that enables or disables inference mode.
215
+
216
+ InferenceMode is analogous to :class:`~no_grad` and should be used
217
+ when you are certain your operations will not interact with autograd
218
+ (e.g., during data loading or model evaluation). Compared to
219
+ :class:`~no_grad`, it removes additional overhead by disabling view
220
+ tracking and version counter bumps. It is also more restrictive, in
221
+ that tensors created in this mode cannot be used in computations
222
+ recorded by autograd.
223
+
224
+ This context manager is thread-local; it does not affect computation
225
+ in other threads.
226
+
227
+ Also functions as a decorator.
228
+
229
+ .. note::
230
+ Inference mode is one of several mechanisms that can locally enable
231
+ or disable gradients. See :ref:`locally-disable-grad-doc` for a
232
+ comparison. If avoiding the use of tensors created in inference mode
233
+ in autograd-tracked regions is difficult, consider benchmarking your
234
+ code with and without inference mode to weigh the performance benefits
235
+ against the trade-offs. You can always use :class:`~no_grad` instead.
236
+
237
+ .. note::
238
+ Unlike some other mechanisms that locally enable or disable grad,
239
+ entering inference_mode also disables :ref:`forward-mode AD <forward-mode-ad>`.
240
+
241
+ .. warning::
242
+ `inference_mode` does NOT automatically set the model to evaluation mode.
243
+ For proper inference behavior (e.g., disabling dropout, using running statistics
244
+ in batch normalization), you must explicitly set your model to evaluation mode using
245
+ `model.eval()` in addition to using this context manager.
246
+
247
+ Args:
248
+ mode (bool or function): Either a boolean flag to enable or disable
249
+ inference mode, or a Python function to decorate with inference
250
+ mode enabled.
251
+
252
+ Example::
253
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
254
+ >>> import torch
255
+ >>> x = torch.ones(1, 2, 3, requires_grad=True)
256
+ >>> with torch.inference_mode():
257
+ ... y = x * x
258
+ >>> y.requires_grad
259
+ False
260
+ >>> # xdoctest: +SKIP("want string isn't quite right")
261
+ >>> y._version
262
+ Traceback (most recent call last):
263
+ File "<stdin>", line 1, in <module>
264
+ RuntimeError: Inference tensors do not track version counter.
265
+ >>> @torch.inference_mode()
266
+ ... def func(x):
267
+ ... return x * x
268
+ >>> out = func(x)
269
+ >>> out.requires_grad
270
+ False
271
+ >>> @torch.inference_mode()
272
+ ... def doubler(x):
273
+ ... return x * 2
274
+ >>> out = doubler(x)
275
+ >>> out.requires_grad
276
+ False
277
+
278
+ """
279
+
280
+ def __init__(self, mode: bool = True) -> None:
281
+ if not torch._jit_internal.is_scripting():
282
+ super().__init__()
283
+ self.mode = mode
284
+
285
+ def __new__(cls, mode=True):
286
+ if isinstance(mode, bool):
287
+ return super().__new__(cls)
288
+ return cls()(mode)
289
+
290
+ def __enter__(self) -> None:
291
+ self._inference_mode_context = torch._C._InferenceMode(self.mode)
292
+ self._inference_mode_context.__enter__()
293
+
294
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
295
+ self._inference_mode_context.__exit__(exc_type, exc_value, traceback)
296
+
297
+ def clone(self) -> "inference_mode":
298
+ r"""
299
+ Create a copy of this class
300
+ """
301
+ return self.__class__(self.mode)
302
+
303
+
304
+ def _enter_inference_mode(mode):
305
+ mode_context = torch._C._InferenceMode(mode)
306
+ mode_context.__enter__()
307
+ return mode_context
308
+
309
+
310
+ def _exit_inference_mode(mode):
311
+ mode.__exit__(None, None, None)
312
+
313
+
314
+ class set_multithreading_enabled(_DecoratorContextManager):
315
+ r"""Context-manager that enables or disables multithreaded backward.
316
+
317
+ Ordinarily, when :ref:`accelerator<accelerators>` devices are in use,
318
+ the backward pass runs on device-specific worker threads. The engine
319
+ creates these threads based on the number of available devices and
320
+ reuses them across iterations.
321
+
322
+ When ``mode=False``, the backward pass runs on the calling thread
323
+ instead. ``mode=True`` restores the default behavior.
324
+
325
+ This can be used as a context-manager or as a function. It is
326
+ thread-local and will not affect computation in other threads.
327
+
328
+ Args:
329
+ mode (bool): Whether to enable multithreaded backward (``True``,
330
+ default) or disable (``False``).
331
+
332
+ .. note::
333
+ This API does not apply to :ref:`forward-mode AD <forward-mode-ad>`,
334
+ which never uses multithreading.
335
+
336
+ """
337
+
338
+ def __init__(self, mode: bool) -> None:
339
+ self.prev = torch._C._is_multithreading_enabled()
340
+ torch._C._set_multithreading_enabled(mode)
341
+ self.mode = mode
342
+
343
+ def __enter__(self) -> None:
344
+ pass
345
+
346
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
347
+ torch._C._set_multithreading_enabled(self.prev)
348
+
349
+ def clone(self) -> "set_multithreading_enabled":
350
+ r"""
351
+ Create a copy of this class
352
+ """
353
+ return self.__class__(self.mode)
354
+
355
+
356
+ class enforce_grad_layout_policy(_DecoratorContextManager):
357
+ r"""Context-manager that controls the gradient layout policy enforcement.
358
+
359
+ The gradient layout contract ensures that accumulated gradients have
360
+ strides matching their corresponding parameters (for non-overlapping
361
+ dense parameters) or are row-major contiguous (otherwise). This aids
362
+ performance in optimizers and distributed reducers.
363
+
364
+ When ``enable=False``, the autograd engine relaxes this enforcement:
365
+
366
+ - Stealable gradients whose layout does *not* match the parameter can
367
+ still be stolen directly (avoiding an extra copy).
368
+ - The "grad and param do not obey the gradient layout contract" warning
369
+ is suppressed.
370
+
371
+ The logic that *creates* a brand-new gradient (e.g., cloning into the
372
+ correct layout when the gradient is not stealable) is **not** affected
373
+ by this flag.
374
+
375
+ This can be used as a context-manager or as a function. It is
376
+ thread-local and will not affect computation in other threads.
377
+
378
+ Args:
379
+ enable (bool): Whether to enforce the gradient layout contract
380
+ (``True``, default) or relax enforcement (``False``).
381
+
382
+ Example::
383
+ >>> # xdoctest: +SKIP
384
+ >>> import torch
385
+ >>> p = torch.empty(2, 3, 4).permute(2, 0, 1).requires_grad_()
386
+ >>> with torch.autograd.enforce_grad_layout_policy(False):
387
+ ... (p * 2).sum().backward()
388
+ ... # p.grad may now have the same strides as the incoming
389
+ ... # gradient rather than being forced to match p's strides.
390
+ """
391
+
392
+ def __init__(self, enable: bool = True) -> None:
393
+ self.prev = torch._C._is_grad_layout_enforcement_enabled()
394
+ torch._C._set_grad_layout_enforcement_enabled(enable)
395
+ self.mode = enable
396
+
397
+ def __call__(self, orig_func: F) -> F:
398
+ torch._C._set_grad_layout_enforcement_enabled(self.prev)
399
+ return super().__call__(orig_func)
400
+
401
+ def __enter__(self) -> None:
402
+ pass
403
+
404
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
405
+ torch._C._set_grad_layout_enforcement_enabled(self.prev)
406
+
407
+ def clone(self) -> "enforce_grad_layout_policy":
408
+ r"""
409
+ Create a copy of this class
410
+ """
411
+ return self.__class__(self.mode)
412
+
413
+
414
+ class _force_original_view_tracking(_DecoratorContextManager):
415
+ r"""Context-manager that sets whether or not to always enable view-replay in autograd.
416
+
417
+ ``set_view_replay_enabled`` will enable or disable view-replay based on its argument :attr:`mode`.
418
+ It can be used as a context-manager or as a function.
419
+
420
+ This context manager is thread local; it will not affect computation
421
+ in other threads.
422
+
423
+ When a tensor view is mutated, the autograd engine needs to decide whether or not
424
+ to regenerate the "updated view" by either replaying the chain of views from the updated base,
425
+ or with a single call to as_strided.
426
+
427
+ If set_view_replay_enabled is set to True, then autograd will always use view replay.
428
+ Otherwise, it will fall back to its existing logic.
429
+
430
+ Args:
431
+ mode (bool): Flag whether to enable view-replay (``True``), or disable
432
+ (``False``).
433
+
434
+ """
435
+
436
+ def __init__(self, mode: bool) -> None:
437
+ self.prev = torch._C._is_view_replay_enabled()
438
+ self.mode = mode
439
+ torch._C._set_view_replay_enabled(mode)
440
+
441
+ def __call__(self, orig_func: F) -> F:
442
+ torch._C._set_view_replay_enabled(self.prev)
443
+ return super().__call__(orig_func)
444
+
445
+ def __enter__(self) -> None:
446
+ torch._C._set_view_replay_enabled(self.mode)
447
+
448
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
449
+ torch._C._set_view_replay_enabled(self.prev)
450
+
451
+ def clone(self):
452
+ return self.__class__(self.mode)
453
+
454
+
455
+ class _unsafe_preserve_version_counter(_DecoratorContextManager):
456
+ r"""DO NOT USE THIS UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING.
457
+
458
+ This context manager can lead to arbitrary silent-correctness issues in any other part of your code
459
+ (even the ones not touched directly by the context manager)!
460
+
461
+ Ordinarily, autograd will track mutations to tensors by incrementing its `._version` attribute.
462
+ This is generally important for correctness, as for example, mutating a tensor that autograd has saved
463
+ for the backwards pass can result in incorrect gradients, and autograd uses the version counter to detect
464
+ and error out in this situation.
465
+
466
+ However, there are rare instances where it might be useful to hide mutations from autograd. For example:
467
+ if a tensor is very large, and you'd like to free its memory by storing it elsewhere, and re-populate
468
+ the tensor right before it is needed by autograd.
469
+
470
+ Args:
471
+ tensor (torch.Tensor): the tensor in question, that you would like to preserve the version counter of.
472
+
473
+ .. note::
474
+ This API does not apply to :ref:`forward-mode AD <forward-mode-ad>`.
475
+
476
+ """
477
+
478
+ def __init__(self, tensors: torch.Tensor | tuple[torch.Tensor, ...]) -> None:
479
+ self.tensors = (tensors,) if isinstance(tensors, torch.Tensor) else tensors
480
+ if not isinstance(self.tensors, tuple):
481
+ raise AssertionError("Expected tensors to be a tuple")
482
+ self.prev_versions = tuple(t._version for t in self.tensors)
483
+
484
+ def __enter__(self) -> None:
485
+ pass
486
+
487
+ # pyrefly: ignore [bad-override]
488
+ def __exit__(self, *args) -> None:
489
+ torch._C._autograd._unsafe_set_version_counter(self.tensors, self.prev_versions)
western_ai/usa/pytorch/torch/autograd/gradcheck.py ADDED
@@ -0,0 +1,2324 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import collections
3
+ import functools
4
+ import warnings
5
+ from collections.abc import Callable, Iterable
6
+ from itertools import product
7
+ from typing_extensions import deprecated
8
+
9
+ import torch
10
+ import torch.testing
11
+
12
+ # pyrefly: ignore [deprecated]
13
+ from torch._vmap_internals import _vmap, vmap
14
+ from torch.overrides import is_tensor_like
15
+ from torch.types import _TensorOrOptionalTensors, _TensorOrTensors
16
+
17
+
18
+ # Note: `get_*_jacobian` functions are added here even though we didn't intend to make them public
19
+ # since they have been exposed from before we added `__all__` and we already maintain BC for them
20
+ # We should eventually deprecate them and remove them from `__all__`
21
+ __all__ = [
22
+ "gradcheck",
23
+ "gradgradcheck",
24
+ "GradcheckError",
25
+ "get_numerical_jacobian",
26
+ "get_analytical_jacobian",
27
+ "get_numerical_jacobian_wrt_specific_input",
28
+ ]
29
+
30
+
31
+ class GradcheckError(RuntimeError):
32
+ r"""Error raised by :func:`gradcheck` and :func:`gradgradcheck`."""
33
+
34
+
35
+ def _is_sparse_compressed_tensor(obj: torch.Tensor):
36
+ return obj.layout in {
37
+ torch.sparse_csr,
38
+ torch.sparse_csc,
39
+ torch.sparse_bsr,
40
+ torch.sparse_bsc,
41
+ }
42
+
43
+
44
+ def _is_sparse_any_tensor(obj: torch.Tensor):
45
+ return _is_sparse_compressed_tensor(obj) or obj.layout is torch.sparse_coo
46
+
47
+
48
+ def _is_float_or_complex_tensor(obj):
49
+ return is_tensor_like(obj) and (obj.is_floating_point() or obj.is_complex())
50
+
51
+
52
+ def _allocate_jacobians_with_inputs(
53
+ input_tensors: tuple, numel_output
54
+ ) -> tuple[torch.Tensor, ...]:
55
+ # Makes zero-filled tensors from inputs. If `numel_output` is not None, for
56
+ # each tensor in `input_tensors`, returns a new zero-filled tensor with height
57
+ # of `t.numel` and width of `numel_output`. Otherwise, for each tensor, returns
58
+ # a 1-d tensor with size `(t.numel,)`. Each new tensor will be strided and have
59
+ # the same dtype and device as those of the corresponding input.
60
+ out: list[torch.Tensor] = [
61
+ t.new_zeros((t.numel(), numel_output), layout=torch.strided)
62
+ for t in input_tensors
63
+ if _is_float_or_complex_tensor(t) and t.requires_grad
64
+ ]
65
+ return tuple(out)
66
+
67
+
68
+ def _allocate_jacobians_with_outputs(
69
+ output_tensors: tuple, numel_input, dtype=None, device=None
70
+ ) -> tuple[torch.Tensor, ...]:
71
+ # Makes zero-filled tensors from outputs. If `dim` is not None, for each tensor
72
+ # in `output_tensors`, returns a new zero-filled tensor with height of `dim` and
73
+ # width of `t.numel`. Otherwise, for each tensor, returns a 1-d tensor with size
74
+ # (t.numel,).
75
+ options = {"dtype": dtype, "device": device, "layout": torch.strided}
76
+ out: list[torch.Tensor] = [
77
+ t.new_zeros((numel_input, t.numel()), **options)
78
+ for t in output_tensors
79
+ if _is_float_or_complex_tensor(t)
80
+ ]
81
+ return tuple(out)
82
+
83
+
84
+ def _iter_tensors(
85
+ x: torch.Tensor | Iterable[torch.Tensor], only_requiring_grad: bool = False
86
+ ) -> Iterable[torch.Tensor]:
87
+ if is_tensor_like(x):
88
+ # mypy doesn't narrow type of `x` to torch.Tensor
89
+ if x.requires_grad or not only_requiring_grad: # type: ignore[union-attr]
90
+ yield x # type: ignore[misc]
91
+ elif isinstance(x, collections.abc.Iterable) and not isinstance(x, str):
92
+ for elem in x:
93
+ yield from _iter_tensors(elem, only_requiring_grad)
94
+
95
+
96
+ def _densify(x):
97
+ # return a copy of sparse x with all unspecified elements
98
+ # "replaced" with zero-valued elements
99
+ if isinstance(x, (list, tuple)):
100
+ return type(x)(map(_densify, x))
101
+ elif not is_tensor_like(x) or x.layout in {torch.strided, torch._mkldnn}: # type: ignore[attr-defined] # no attr _mkldnn
102
+ return x
103
+ elif x.layout is torch.sparse_coo:
104
+ device = x.device
105
+ indices_dtype = x._indices().dtype
106
+ tmp = torch.ones(x.shape[: x.sparse_dim()], dtype=torch.int8, device=device)
107
+ indices = tmp.nonzero().t().to(dtype=indices_dtype)
108
+ values = torch.zeros(
109
+ (tmp.numel(), *x.shape[x.sparse_dim() :]), dtype=x.dtype, device=device
110
+ )
111
+ x_coalesced = x.detach().coalesce()
112
+ if x_coalesced.numel() > 0:
113
+ stride = tmp.stride()
114
+ flat_indices = (
115
+ x_coalesced.indices()
116
+ .mul(
117
+ torch.tensor(stride, dtype=indices_dtype, device=device).unsqueeze(
118
+ 1
119
+ )
120
+ )
121
+ .sum(0)
122
+ )
123
+ values[flat_indices] = x_coalesced.values()
124
+ return (
125
+ torch.sparse_coo_tensor(indices, values, x.shape)
126
+ ._coalesced_(True)
127
+ .requires_grad_(x.requires_grad)
128
+ )
129
+ elif _is_sparse_compressed_tensor(x):
130
+ blocksize = (
131
+ x.values().shape[1:3]
132
+ if x.layout in {torch.sparse_bsr, torch.sparse_bsc}
133
+ else None
134
+ )
135
+ compressed_indices = (
136
+ x.crow_indices()
137
+ if x.layout in {torch.sparse_csr, torch.sparse_bsr}
138
+ else x.ccol_indices()
139
+ )
140
+ # We'll use intermediate sparse COO for simplicity
141
+ r = _densify(x.detach().to_sparse(layout=torch.sparse_coo)).to_sparse(
142
+ layout=x.layout, blocksize=blocksize
143
+ )
144
+ # Check that all elements are specified also after `to_sparse` op:
145
+ dense_numel = r.values().numel() // max(1, r.values().shape[0])
146
+ batch_numel = compressed_indices.numel() // compressed_indices.shape[-1]
147
+ sparse_numel = r.numel() // max(1, dense_numel * batch_numel)
148
+ if sparse_numel != r._nnz():
149
+ raise AssertionError(
150
+ f"{x.layout} densify failed: expected nnz={sparse_numel} but got {r._nnz()}"
151
+ )
152
+ return r.requires_grad_(x.requires_grad)
153
+ elif _is_sparse_any_tensor(x):
154
+ raise NotImplementedError(x.layout)
155
+ return x
156
+
157
+
158
+ def _iter_tensor(x_tensor):
159
+ # (Only used for slow gradcheck) Returns a generator that yields the following
160
+ # elements at each iteration:
161
+ # 1) a tensor: the same tensor is returned across all iterations. The tensor
162
+ # is not the same as the original x_tensor as given as input - it is
163
+ # prepared so that it can be modified in-place. Depending on whether the
164
+ # input tensor is strided, sparse, or dense, the returned tensor may or may
165
+ # not share storage with x_tensor.
166
+ # 2) a tuple of indices that can be used with advanced indexing (yielded in
167
+ # dictionary order)
168
+ # 3) flattened index that will be used to index into the Jacobian tensor
169
+ #
170
+ # For a tensor t with size (2, 2), _iter_tensor yields:
171
+ # `x, (0, 0), 0`, `x, (0, 1), 1`, `x, (1, 0), 2`, `x, (1, 1), 3`
172
+ #
173
+ # where x is the t.data of the original tensor. Perturbing the entry of x
174
+ # at index (1, 1) yields the 3rd column of the overall Jacobian matrix.
175
+ if _is_sparse_any_tensor(x_tensor):
176
+
177
+ def get_stride(size):
178
+ dim = len(size)
179
+ tmp = 1
180
+ stride = [0] * dim
181
+ for i in reversed(range(dim)):
182
+ stride[i] = tmp
183
+ tmp *= size[i]
184
+ return stride
185
+
186
+ x_nnz = x_tensor._nnz()
187
+ x_size = list(x_tensor.size())
188
+ if x_tensor.layout is torch.sparse_coo:
189
+ x_indices = x_tensor._indices().t()
190
+ x_values = x_tensor._values()
191
+ elif x_tensor.layout is torch.sparse_csr:
192
+ x_indices = torch._convert_indices_from_csr_to_coo(
193
+ x_tensor.crow_indices(), x_tensor.col_indices()
194
+ ).t()
195
+ x_values = x_tensor.values()
196
+ elif x_tensor.layout is torch.sparse_csc:
197
+ x_indices = torch._convert_indices_from_csr_to_coo(
198
+ x_tensor.ccol_indices(), x_tensor.row_indices(), transpose=True
199
+ ).t()
200
+ x_values = x_tensor.values()
201
+ elif x_tensor.layout is torch.sparse_bsr:
202
+ x_block_values = x_tensor.values()
203
+ x_blocksize = x_block_values.size()[1:3]
204
+ x_indices = (
205
+ torch._convert_indices_from_csr_to_coo(
206
+ x_tensor.crow_indices(), x_tensor.col_indices()
207
+ )
208
+ .repeat_interleave(x_blocksize[0] * x_blocksize[1], 1)
209
+ .mul_(torch.tensor(x_blocksize, device=x_tensor.device).reshape(2, 1))
210
+ .add_(
211
+ torch.stack(
212
+ torch.where(torch.ones(x_blocksize, device=x_tensor.device))
213
+ ).repeat(1, x_nnz)
214
+ )
215
+ .t()
216
+ )
217
+ x_values = x_block_values.flatten(0, 2)
218
+ x_nnz = x_values.size(0)
219
+ elif x_tensor.layout is torch.sparse_bsc:
220
+ x_block_values = x_tensor.values()
221
+ x_blocksize = x_block_values.size()[1:3]
222
+ x_indices = (
223
+ torch._convert_indices_from_csr_to_coo(
224
+ x_tensor.ccol_indices(), x_tensor.row_indices(), transpose=True
225
+ )
226
+ .repeat_interleave(x_blocksize[0] * x_blocksize[1], 1)
227
+ .mul_(torch.tensor(x_blocksize, device=x_tensor.device).reshape(2, 1))
228
+ .add_(
229
+ torch.stack(
230
+ torch.where(torch.ones(x_blocksize, device=x_tensor.device))
231
+ ).repeat(1, x_nnz)
232
+ )
233
+ .t()
234
+ )
235
+ x_values = x_block_values.flatten(0, 2)
236
+ x_nnz = x_values.size(0)
237
+ else:
238
+ raise NotImplementedError(f"_iter_tensor for {x_tensor.layout} input")
239
+ x_stride = get_stride(x_size)
240
+ # Use .data here to get around the version check
241
+ x_values = x_values.data
242
+ for i in range(x_nnz):
243
+ x_value = x_values[i]
244
+ for x_idx in product(*[range(m) for m in x_values.size()[1:]]):
245
+ indices = x_indices[i].tolist() + list(x_idx)
246
+ d_idx = sum(indices[k] * x_stride[k] for k in range(len(x_size)))
247
+ yield x_value, x_idx, d_idx
248
+ elif x_tensor.layout == torch._mkldnn: # type: ignore[attr-defined]
249
+ for d_idx, x_idx in enumerate(product(*[range(m) for m in x_tensor.size()])):
250
+ # this is really inefficient, but without indexing implemented, there's
251
+ # not really a better way than converting back and forth
252
+ x_tensor_dense = x_tensor.to_dense()
253
+ yield x_tensor_dense, x_idx, d_idx
254
+ else:
255
+ # Use .data here to get around the version check
256
+ x_tensor = x_tensor.data
257
+ for d_idx, x_idx in enumerate(product(*[range(m) for m in x_tensor.size()])):
258
+ yield x_tensor, x_idx, d_idx
259
+
260
+
261
+ def _get_numerical_jacobian(
262
+ fn, inputs, outputs=None, target=None, eps=1e-3, is_forward_ad=False
263
+ ) -> list[tuple[torch.Tensor, ...]]:
264
+ """Compute the numerical Jacobian of `fn(inputs)` with respect to `target`.
265
+
266
+ If not specified, targets are the input. Returns M * N Jacobians where N is the
267
+ number of tensors in target that require grad and M is the number of non-integral
268
+ outputs.
269
+
270
+ Args:
271
+ fn: the function to compute the jacobian for
272
+ inputs: inputs to `fn`
273
+ outputs: provide precomputed outputs to avoid one extra invocation of fn
274
+ target: the Tensors wrt whom Jacobians are calculated (default=`inputs`)
275
+ eps: the magnitude of the perturbation during finite differencing
276
+ (default=`1e-3`)
277
+ is_forward_ad: if this numerical jacobian is computed to be checked wrt
278
+ forward AD gradients (this is used for error checking only)
279
+
280
+ Returns:
281
+ A list of M N-tuples of tensors
282
+
283
+ Note that `target` may not even be part of `input` to `fn`, so please be
284
+ **very careful** in this to not clone `target`.
285
+ """
286
+ jacobians: list[tuple[torch.Tensor, ...]] = []
287
+ if outputs is None:
288
+ outputs = _as_tuple(fn(*_as_tuple(inputs)))
289
+ if not is_forward_ad and any(o.is_complex() for o in outputs):
290
+ raise ValueError(
291
+ "Expected output to be non-complex. get_numerical_jacobian no "
292
+ "longer supports functions that return complex outputs."
293
+ )
294
+ if target is None:
295
+ target = inputs
296
+ inp_indices = [
297
+ i for i, a in enumerate(target) if is_tensor_like(a) and a.requires_grad
298
+ ]
299
+ for inp, inp_idx in zip(_iter_tensors(target, True), inp_indices):
300
+ jacobians += [
301
+ get_numerical_jacobian_wrt_specific_input(
302
+ fn,
303
+ inp_idx,
304
+ inputs,
305
+ outputs,
306
+ eps,
307
+ input=inp,
308
+ is_forward_ad=is_forward_ad,
309
+ )
310
+ ]
311
+ return jacobians
312
+
313
+
314
+ @deprecated(
315
+ "`get_numerical_jacobian` was part of PyTorch's private API and not "
316
+ "meant to be exposed. We are deprecating it and it will be removed "
317
+ "in a future version of PyTorch. If you have a specific use for "
318
+ "this or feature request for this to be a stable API, please file "
319
+ "us an issue at https://github.com/pytorch/pytorch/issues/new",
320
+ category=FutureWarning,
321
+ )
322
+ def get_numerical_jacobian(fn, inputs, target=None, eps=1e-3, grad_out=1.0):
323
+ """Compute the numerical Jacobian for a given fn and its inputs.
324
+
325
+ This is a Deprecated API.
326
+
327
+ Args:
328
+ fn: the function to compute the Jacobian for (must take inputs as a tuple)
329
+ inputs: input to `fn`
330
+ target: the Tensors wrt whom Jacobians are calculated (default=`input`)
331
+ eps: the magnitude of the perturbation during finite differencing
332
+ (default=`1e-3`)
333
+ grad_out: defaults to 1.0.
334
+
335
+ Returns:
336
+ A list of Jacobians of `fn` (restricted to its first output) with respect to
337
+ each input or target, if provided.
338
+
339
+ Note that `target` may not even be part of `input` to `fn`, so please be
340
+ **very careful** in this to not clone `target`.
341
+ """
342
+ if (
343
+ grad_out != 1.0
344
+ ): # grad_out param is only kept for backward compatibility reasons
345
+ raise ValueError(
346
+ "Expected grad_out to be 1.0. get_numerical_jacobian no longer "
347
+ "supports values of grad_out != 1.0."
348
+ )
349
+
350
+ def fn_pack_inps(*inps):
351
+ return fn(inps)
352
+
353
+ jacobians = _get_numerical_jacobian(fn_pack_inps, inputs, None, target, eps)
354
+
355
+ return tuple(jacobian_for_each_output[0] for jacobian_for_each_output in jacobians)
356
+
357
+
358
+ def _compute_numerical_gradient(fn, entry, v, norm_v, nbhd_checks_fn):
359
+ # Computes numerical directional derivative as finite difference
360
+ # of function `fn` at input `entry`, perturbed by vector `v`.
361
+ if _is_sparse_compressed_tensor(entry):
362
+ # sparse compressed tensors don't implement sub/add/copy_
363
+ # yet. However, in non-masked semantics context entry and v
364
+ # have the same sparse indices ...
365
+ if entry.layout != v.layout:
366
+ raise AssertionError(
367
+ f"Expected entry and v to have the same layout, but got {entry.layout} and {v.layout}"
368
+ )
369
+ if entry._nnz() != v._nnz():
370
+ raise AssertionError(
371
+ f"Expected entry and v to have the same nnz, but got {entry._nnz()} and {v._nnz()} "
372
+ f"with entry shape {entry.shape}"
373
+ )
374
+ # ... the finite differencing can be performed on values only:
375
+ entry = entry.values()
376
+ v = v.values()
377
+ # we'll detach to avoid backward computations that sparse
378
+ # tensors have limited support for.
379
+ entry = entry.detach()
380
+
381
+ orig = entry.clone()
382
+ entry.copy_(orig - v)
383
+ outa = fn()
384
+ entry.copy_(orig + v)
385
+ outb = fn()
386
+ entry.copy_(orig)
387
+
388
+ def compute(a, b):
389
+ nbhd_checks_fn(a, b)
390
+ ret = (b - a) / (2 * norm_v) # use central difference approx
391
+ return ret.detach().reshape(-1)
392
+
393
+ return tuple(compute(a, b) for (a, b) in zip(outa, outb))
394
+
395
+
396
+ def _compute_numerical_jvps_wrt_specific_input(
397
+ jvp_fn, delta, input_is_complex, is_forward_ad=False
398
+ ) -> list[torch.Tensor]:
399
+ # Computing the jacobian only works for real delta
400
+ # For details on the algorithm used here, refer:
401
+ # Section 3.5.3 https://arxiv.org/pdf/1701.00392.pdf
402
+ # s = fn(z) where z = x for real valued input
403
+ # and z = x + yj for complex valued input
404
+ jvps: list[torch.Tensor] = []
405
+ ds_dx_tup = jvp_fn(delta[0] if isinstance(delta, tuple) else delta)
406
+
407
+ if input_is_complex: # C -> R
408
+ ds_dy_tup = (
409
+ jvp_fn(delta[1] * 1j) if isinstance(delta, tuple) else jvp_fn(delta * 1j)
410
+ )
411
+ for ds_dx, ds_dy in zip(ds_dx_tup, ds_dy_tup):
412
+ if ds_dx.is_complex():
413
+ raise AssertionError("Expected ds_dx to be real-valued, not complex")
414
+ # conjugate wirtinger derivative
415
+ conj_w_d = ds_dx + ds_dy * 1j
416
+ jvps.append(conj_w_d)
417
+ else:
418
+ for ds_dx in ds_dx_tup: # R -> R or (R -> C for the forward AD case)
419
+ if not is_forward_ad and ds_dx.is_complex():
420
+ raise AssertionError("Expected ds_dx to be real-valued, not complex.")
421
+ jvps.append(ds_dx)
422
+ return jvps
423
+
424
+
425
+ def _combine_jacobian_cols(
426
+ jacobians_cols: dict[int, list[torch.Tensor]], outputs, input, numel
427
+ ) -> tuple[torch.Tensor, ...]:
428
+ # jacobian_cols maps column_idx -> output_idx -> single column of jacobian Tensor
429
+ # we return a list that maps output_idx -> full jacobian Tensor
430
+ jacobians = _allocate_jacobians_with_outputs(
431
+ outputs, numel, dtype=input.dtype if input.dtype.is_complex else None
432
+ )
433
+ for i, jacobian in enumerate(jacobians):
434
+ for k, v in jacobians_cols.items():
435
+ jacobian[k] = v[i]
436
+ return jacobians
437
+
438
+
439
+ def _prepare_input(
440
+ input: torch.Tensor, maybe_perturbed_input: torch.Tensor | None, fast_mode=False
441
+ ) -> torch.Tensor:
442
+ # Prepares the inputs to be passed into the function while including the new
443
+ # modified input.
444
+ if input.layout == torch._mkldnn: # type: ignore[attr-defined] # no attr _mkldnn
445
+ # Convert back to mkldnn
446
+ if maybe_perturbed_input is not None:
447
+ return maybe_perturbed_input.to_mkldnn()
448
+ else:
449
+ return input
450
+ elif _is_sparse_any_tensor(input):
451
+ if fast_mode and maybe_perturbed_input is not None:
452
+ # entry is already a "cloned" version of the original tensor
453
+ # thus changes to entry are not reflected in the input
454
+ return maybe_perturbed_input
455
+ else:
456
+ return input
457
+ else:
458
+ # We cannot use entry (input.data) if we want gradgrad to work because
459
+ # fn (in the gradgrad case) needs to compute grad wrt input
460
+ return input
461
+
462
+
463
+ def _check_outputs_same_dtype_and_shape(output1, output2, eps, idx=None) -> None:
464
+ # Check that the returned outputs don't have different dtype or shape when you
465
+ # perturb the input
466
+ on_index = f"on index {idx} " if idx is not None else ""
467
+ if output1.shape != output2.shape:
468
+ raise AssertionError(
469
+ f"Expected `func` to return outputs with the same shape"
470
+ f" when inputs are perturbed {on_index}by {eps}, but got:"
471
+ f" shapes {output1.shape} and {output2.shape}."
472
+ )
473
+ if output1.dtype != output2.dtype:
474
+ raise AssertionError(
475
+ f"Expected `func` to return outputs with the same dtype"
476
+ f" when inputs are perturbed {on_index}by {eps}, but got:"
477
+ f" dtypes {output1.dtype} and {output2.dtype}."
478
+ )
479
+
480
+
481
+ def get_numerical_jacobian_wrt_specific_input(
482
+ fn, input_idx, inputs, outputs, eps, input=None, is_forward_ad=False
483
+ ) -> tuple[torch.Tensor, ...]:
484
+ # Computes the numerical jacobians wrt to a single input. Returns N jacobian
485
+ # tensors, where N is the number of outputs. We use a dictionary for
486
+ # jacobian_cols because indices aren't necessarily consecutive for sparse inputs
487
+ # When we perturb only a single element of the input tensor at a time, the jvp
488
+ # is equivalent to a single col of the Jacobian matrix of fn.
489
+ jacobian_cols: dict[int, list[torch.Tensor]] = {}
490
+ input = inputs[input_idx] if input is None else input
491
+ if not input.requires_grad:
492
+ raise AssertionError("Expected input to have requires_grad=True")
493
+ for x, idx, d_idx in _iter_tensor(input):
494
+ wrapped_fn = _with_prepare_inputs(fn, inputs, input_idx, x)
495
+ input_to_perturb = x[idx]
496
+ nbhd_checks_fn = functools.partial(
497
+ _check_outputs_same_dtype_and_shape, idx=idx, eps=eps
498
+ )
499
+ jvp_fn = _get_numerical_jvp_fn(
500
+ wrapped_fn, input_to_perturb, eps, nbhd_checks_fn
501
+ )
502
+ jacobian_cols[d_idx] = _compute_numerical_jvps_wrt_specific_input(
503
+ jvp_fn, eps, x.is_complex(), is_forward_ad
504
+ )
505
+ return _combine_jacobian_cols(jacobian_cols, outputs, input, input.numel())
506
+
507
+
508
+ def _get_analytical_jacobian_forward_ad(
509
+ fn, inputs, outputs, *, check_grad_dtypes=False, all_u=None
510
+ ) -> tuple[tuple[torch.Tensor, ...], ...]:
511
+ """Compute the analytical Jacobian using forward mode AD of `fn(inputs)` using forward mode AD with respect to `target`.
512
+
513
+ Return N * M Jacobians where N is the number of tensors in target that require grad and
514
+ M is the number of non-integral outputs.
515
+ Contrary to other functions here, this function requires "inputs" to actually be used by the function.
516
+ The computed value is expected to be wrong if the function captures the inputs by side effect instead of
517
+ using the passed ones (many torch.nn tests do this).
518
+
519
+ Args:
520
+ fn: the function to compute the jacobian for
521
+ inputs: inputs to `fn`
522
+ outputs: provide precomputed outputs to avoid one extra invocation of fn
523
+ check_grad_dtypes: if True, will check that the gradient dtype are valid
524
+ all_u (optional): if provided, the Jacobian will be right multiplied with this vector
525
+
526
+ Returns:
527
+ A tuple of M N-tuples of tensors
528
+ """
529
+ # To avoid early import issues
530
+ fwAD = torch.autograd.forward_ad
531
+
532
+ tensor_inputs = tuple(i for i in inputs if is_tensor_like(i) and i.requires_grad)
533
+
534
+ if any(i.is_complex() for i in tensor_inputs):
535
+ raise ValueError(
536
+ "Expected inputs to be non-complex for _get_analytical_jacobian_forward_ad."
537
+ )
538
+
539
+ if all_u:
540
+ jacobians = tuple(
541
+ _allocate_jacobians_with_outputs(outputs, 1) for i in tensor_inputs
542
+ )
543
+ else:
544
+ jacobians = tuple(
545
+ _allocate_jacobians_with_outputs(outputs, i.numel()) for i in tensor_inputs
546
+ )
547
+
548
+ with fwAD.dual_level():
549
+ fw_grads = []
550
+ dual_inputs = []
551
+ for inp in inputs:
552
+ if is_tensor_like(inp) and inp.requires_grad:
553
+ if inp.layout == torch._mkldnn: # type: ignore[attr-defined]
554
+ raise ValueError(
555
+ "MKLDNN inputs are not support for forward AD gradcheck."
556
+ )
557
+
558
+ inp = fwAD.make_dual(inp.detach(), torch.zeros_like(inp))
559
+ # If inp is a differentiable view, the dual might not be the tangent given to
560
+ # make_dual, so read it explicitly from the dual tensor
561
+ fw_grads.append(fwAD.unpack_dual(inp)[1])
562
+ dual_inputs.append(inp)
563
+
564
+ if all_u:
565
+ # Do the full reduction in one pass
566
+ # To be consistent with numerical evaluation, we actually compute one reduction per input
567
+ for i, (fw_grad, u) in enumerate(zip(fw_grads, all_u)):
568
+ fw_grad.copy_(u.view_as(fw_grad))
569
+ raw_outputs = _as_tuple(fn(*dual_inputs))
570
+ dual_outputs = filter(_is_float_or_complex_tensor, raw_outputs)
571
+ for index_o, d_o in enumerate(dual_outputs):
572
+ val, res = fwAD.unpack_dual(d_o)
573
+ if (
574
+ check_grad_dtypes
575
+ and res is not None
576
+ and val.is_complex() != res.is_complex()
577
+ ):
578
+ raise GradcheckError("Forward AD gradient has dtype mismatch.")
579
+
580
+ # Remove extra dimension of size 1 corresponding to the reduced input
581
+ jacobians[i][index_o].squeeze_(0)
582
+ if res is None:
583
+ jacobians[i][index_o].zero_()
584
+ else:
585
+ jacobians[i][index_o].copy_(res.reshape(-1))
586
+ fw_grad.zero_()
587
+ else:
588
+ # Reconstruct the full Jacobian column by column
589
+ for i, fw_grad in enumerate(fw_grads):
590
+ for lin_idx, grad_idx in enumerate(
591
+ product(*[range(m) for m in fw_grad.size()])
592
+ ):
593
+ fw_grad[grad_idx] = 1.0
594
+ raw_outputs = _as_tuple(fn(*dual_inputs))
595
+ dual_outputs = filter(_is_float_or_complex_tensor, raw_outputs)
596
+ for index_o, d_o in enumerate(dual_outputs):
597
+ val, res = fwAD.unpack_dual(d_o)
598
+ if (
599
+ check_grad_dtypes
600
+ and res is not None
601
+ and val.is_complex() != res.is_complex()
602
+ ):
603
+ raise GradcheckError(
604
+ "Forward AD gradient has dtype mismatch."
605
+ )
606
+
607
+ if res is None:
608
+ jacobians[i][index_o][lin_idx].zero_()
609
+ else:
610
+ jacobians[i][index_o][lin_idx].copy_(res.reshape(-1))
611
+ fw_grad[grad_idx] = 0.0
612
+
613
+ return jacobians
614
+
615
+
616
+ def _get_input_to_perturb(input):
617
+ # Prepare the input so that it can be modified in-place and do certain
618
+ # operations that require the tensor to have strides. If fast_mode=False,
619
+ # _iter_tensor would handle the below cases:
620
+ if input.layout == torch._mkldnn: # type: ignore[attr-defined] # no attr _mkldnn
621
+ # Convert to dense so we can perform operations that require strided tensors
622
+ input_to_perturb = input.to_dense()
623
+ elif _is_sparse_any_tensor(input):
624
+ # Clone because input may require grad, and copy_ calls resize_,
625
+ # which is not allowed for .data
626
+ input_to_perturb = input.clone()
627
+ else:
628
+ input_to_perturb = input.data
629
+ return input_to_perturb
630
+
631
+
632
+ def _with_prepare_inputs(fn, inputs, input_idx, input_to_perturb, fast_mode=False):
633
+ # Wraps `fn` so that its inputs are already supplied
634
+ def wrapped_fn():
635
+ inp = tuple(
636
+ _prepare_input(a, input_to_perturb if i == input_idx else None, fast_mode)
637
+ if is_tensor_like(a)
638
+ else a
639
+ for i, a in enumerate(_as_tuple(inputs))
640
+ )
641
+ return tuple(a.clone() for a in _as_tuple(fn(*inp)))
642
+
643
+ return wrapped_fn
644
+
645
+
646
+ def _get_numerical_jvp_fn(wrapped_fn, input_to_perturb, eps, nbhd_checks_fn):
647
+ # Wraps jvp_fn so that certain arguments are already supplied
648
+ def jvp_fn(delta):
649
+ return _compute_numerical_gradient(
650
+ wrapped_fn, input_to_perturb, delta, eps, nbhd_checks_fn
651
+ )
652
+
653
+ return jvp_fn
654
+
655
+
656
+ def _reshape_tensor_or_tuple(u, shape):
657
+ # We don't need to reshape when input corresponding to u is sparse
658
+ if isinstance(u, tuple):
659
+ if not _is_sparse_any_tensor(u[0]):
660
+ return (u[0].reshape(shape), u[1].reshape(shape))
661
+ else:
662
+ if not _is_sparse_any_tensor(u):
663
+ return u.reshape(shape)
664
+ return u
665
+
666
+
667
+ def _mul_tensor_or_tuple(u, k):
668
+ if isinstance(u, tuple):
669
+ return (k * u[0], k * u[1])
670
+ else:
671
+ return k * u
672
+
673
+
674
+ def _get_numerical_jvp_wrt_specific_input(
675
+ fn, input_idx, inputs, u, eps, is_forward_ad=False
676
+ ) -> list[torch.Tensor]:
677
+ input = inputs[input_idx]
678
+ input_to_perturb = _get_input_to_perturb(input)
679
+ wrapped_fn = _with_prepare_inputs(fn, inputs, input_idx, input_to_perturb, True)
680
+ nbhd_checks_fn = functools.partial(_check_outputs_same_dtype_and_shape, eps=eps)
681
+ jvp_fn = _get_numerical_jvp_fn(wrapped_fn, input_to_perturb, eps, nbhd_checks_fn)
682
+ u = _reshape_tensor_or_tuple(u, input_to_perturb.shape)
683
+ u = _mul_tensor_or_tuple(u, eps)
684
+ return _compute_numerical_jvps_wrt_specific_input(
685
+ jvp_fn, u, input.is_complex(), is_forward_ad
686
+ )
687
+
688
+
689
+ def _get_numerical_vJu(
690
+ fn, inputs, inp_indices, func_out, all_u, all_v, eps, is_forward_ad
691
+ ):
692
+ # Note that all_v can also be None, in that case, this function only computes Ju.
693
+ reduced_jacobians: list[list[torch.Tensor]] = []
694
+ for inp_idx, u in zip(inp_indices, all_u):
695
+ all_Ju = _get_numerical_jvp_wrt_specific_input(
696
+ fn, inp_idx, inputs, u, eps, is_forward_ad
697
+ )
698
+ # Filter out the Ju for non floating point outputs
699
+ filtered_Ju = []
700
+ func_out = _as_tuple(func_out)
701
+ if len(all_Ju) != len(func_out):
702
+ raise AssertionError(
703
+ f"Expected all_Ju and func_out to have the same length, "
704
+ f"but got {len(all_Ju)} and {len(func_out)}"
705
+ )
706
+ for Ju, output in zip(all_Ju, func_out):
707
+ if _is_float_or_complex_tensor(output):
708
+ filtered_Ju.append(Ju)
709
+ else:
710
+ # TODO: handle the other Ju
711
+ pass
712
+ if all_v is not None:
713
+ jacobian_scalars: list[torch.Tensor] = []
714
+ for v, Ju in zip(all_v, filtered_Ju):
715
+ jacobian_scalars.append(_dot_with_type_promotion(v, Ju))
716
+ reduced_jacobians.append(jacobian_scalars)
717
+ else:
718
+ reduced_jacobians.append(filtered_Ju)
719
+ return reduced_jacobians
720
+
721
+
722
+ def _check_jacobians_equal(j1, j2, atol):
723
+ # Check whether the max difference between two Jacobian tensors are within some
724
+ # tolerance `atol`.
725
+ for j1_x, j2_x in zip(j1, j2):
726
+ if j1_x.numel() != 0 and (j1_x - j2_x).abs().max() > atol:
727
+ return False
728
+ return True
729
+
730
+
731
+ def _stack_and_check_tensors(
732
+ list_of_list_of_tensors, inputs, numel_outputs
733
+ ) -> tuple[tuple[torch.Tensor, ...], bool, bool]:
734
+ # For the ith tensor in the inner list checks whether it has the same size and
735
+ # dtype as the ith differentiable input.
736
+ out_jacobians = _allocate_jacobians_with_inputs(inputs, numel_outputs)
737
+ diff_input_list = list(_iter_tensors(inputs, True))
738
+ correct_grad_sizes = True
739
+ correct_grad_types = True
740
+ for i, tensor_list in enumerate(list_of_list_of_tensors):
741
+ inp = diff_input_list[i]
742
+ out_jacobian = out_jacobians[i]
743
+ for j, tensor in enumerate(tensor_list):
744
+ if tensor is not None and tensor.size() != inp.size():
745
+ correct_grad_sizes = False
746
+ elif tensor is not None and tensor.dtype != inp.dtype:
747
+ correct_grad_types = False
748
+ if tensor is None:
749
+ out_jacobian[:, j].zero_()
750
+ else:
751
+ dense = tensor.to_dense() if tensor.layout != torch.strided else tensor
752
+ if out_jacobian[:, j].numel() != dense.numel():
753
+ raise AssertionError(
754
+ f"Expected out_jacobian column to have {dense.numel()} elements, "
755
+ f"but got {out_jacobian[:, j].numel()}"
756
+ )
757
+ out_jacobian[:, j] = dense.reshape(-1)
758
+ return out_jacobians, correct_grad_sizes, correct_grad_types
759
+
760
+
761
+ FAILED_NONDET_MSG = """\n
762
+ NOTE: If your op relies on non-deterministic operations i.e., it is listed here:
763
+ https://pytorch.org/docs/stable/generated/torch.use_deterministic_algorithms.html
764
+ this failure might be expected.
765
+
766
+ If you are adding a new operator, please file an issue and then use one of the
767
+ workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck.
768
+ If the test
769
+ - manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
770
+ with `nondet_tol=<tol>` as a keyword argument.
771
+ - is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
772
+ to have `gradcheck_nondet_tol=<tol>`.
773
+ - is a Module test (e.g., in common_nn.py), then modify the corresponding
774
+ module_test entry to have `gradcheck_nondet_tol=<tol>`
775
+ """
776
+
777
+
778
+ def _check_analytical_jacobian_attributes(
779
+ inputs, output, nondet_tol, check_grad_dtypes, fast_mode=False, v=None
780
+ ) -> tuple[torch.Tensor, ...]:
781
+ # This is used by both fast and slow mode:
782
+ # - For slow mode, vjps[i][j] is the jth row of the Jacobian wrt the ith
783
+ # input.
784
+ # - For fast mode, vjps[i][0] is a linear combination of the rows
785
+ # of the Jacobian wrt the ith input
786
+ diff_input_list = list(_iter_tensors(inputs, True))
787
+
788
+ def vjp_fn(grad_output):
789
+ return torch.autograd.grad(
790
+ output, diff_input_list, grad_output, retain_graph=True, allow_unused=True
791
+ )
792
+
793
+ # Compute everything twice to check for nondeterminism (which we call reentrancy)
794
+ if fast_mode:
795
+ vjps1 = _get_analytical_vjps_wrt_specific_output(vjp_fn, output.clone(), v)
796
+ vjps2 = _get_analytical_vjps_wrt_specific_output(vjp_fn, output.clone(), v)
797
+ else:
798
+ vjps1 = _compute_analytical_jacobian_rows(vjp_fn, output.clone())
799
+ vjps2 = _compute_analytical_jacobian_rows(vjp_fn, output.clone())
800
+
801
+ output_numel = output.numel() if not fast_mode else 1
802
+ jacobians1, types_ok, sizes_ok = _stack_and_check_tensors(
803
+ vjps1, inputs, output_numel
804
+ )
805
+ jacobians2, _, _ = _stack_and_check_tensors(vjps2, inputs, output_numel)
806
+ reentrant = _check_jacobians_equal(jacobians1, jacobians2, nondet_tol)
807
+
808
+ if not types_ok and check_grad_dtypes:
809
+ raise GradcheckError("Gradient has dtype mismatch")
810
+ if not sizes_ok:
811
+ raise GradcheckError("Analytical gradient has incorrect size")
812
+ if not reentrant:
813
+ raise GradcheckError(
814
+ "Backward is not reentrant, i.e., running backward with "
815
+ "same input and grad_output multiple times gives different values, "
816
+ "although analytical gradient matches numerical gradient."
817
+ f"The tolerance for nondeterminism was {nondet_tol}." + FAILED_NONDET_MSG
818
+ )
819
+ return jacobians1
820
+
821
+
822
+ def _get_analytical_vJu_backward_mode(
823
+ inputs, outputs, nondet_tol, check_grad_dtypes, all_v, all_u
824
+ ):
825
+ reduced_jacobians: list[list[torch.Tensor]] = []
826
+ for output, v in zip(outputs, all_v):
827
+ all_vJ = _check_analytical_jacobian_attributes(
828
+ inputs, output, nondet_tol, check_grad_dtypes, fast_mode=True, v=v
829
+ )
830
+ jacobian_scalars: list[torch.Tensor] = []
831
+ for vJ, u in zip(all_vJ, all_u):
832
+ # Why do we need squeeze here? vJ is a 2-d tensor so that we can reuse
833
+ # the error checking logic from slow mode
834
+ vJ = vJ.T.squeeze(0)
835
+ if vJ.is_complex(): # C -> R
836
+ tv = torch.view_as_real(vJ.resolve_conj())
837
+ tr = tv.select(-1, 0)
838
+ ti = tv.select(-1, 1)
839
+ jacobian_scalars.append(tr.dot(u[0]) + 1j * ti.dot(u[1]))
840
+ else: # R -> R
841
+ jacobian_scalars.append(vJ.dot(u))
842
+ reduced_jacobians.append(jacobian_scalars)
843
+ return reduced_jacobians
844
+
845
+
846
+ @deprecated(
847
+ "`get_analytical_jacobian` was part of PyTorch's private API and not "
848
+ "meant to be exposed. We are deprecating it and it will be removed "
849
+ "in a future version of PyTorch. If you have a specific use for "
850
+ "this or feature request for this to be a stable API, please file "
851
+ "us an issue at https://github.com/pytorch/pytorch/issues/new",
852
+ category=FutureWarning,
853
+ )
854
+ def get_analytical_jacobian(inputs, output, nondet_tol=0.0, grad_out=1.0):
855
+ # Replicates the behavior of the old get_analytical_jacobian before the refactor
856
+ # This shares much of its code with _check_analytical_jacobian_attributes
857
+ if (
858
+ grad_out != 1.0
859
+ ): # grad_out param is only kept for backward compatibility reasons
860
+ raise ValueError(
861
+ "Expected grad_out to be 1.0. get_analytical_jacobian no longer "
862
+ "supports values of grad_out != 1.0."
863
+ )
864
+ if output.is_complex():
865
+ raise ValueError(
866
+ "Expected output to be non-complex. get_analytical_jacobian no "
867
+ "longer supports functions that return complex outputs."
868
+ )
869
+ diff_input_list = list(_iter_tensors(inputs, True))
870
+
871
+ def vjp_fn(grad_output):
872
+ return torch.autograd.grad(
873
+ output, diff_input_list, grad_output, retain_graph=True, allow_unused=True
874
+ )
875
+
876
+ # Compute everything twice to check for nondeterminism (which we call reentrancy)
877
+ vjps1 = _compute_analytical_jacobian_rows(vjp_fn, output.clone())
878
+ vjps2 = _compute_analytical_jacobian_rows(vjp_fn, output.clone())
879
+
880
+ output_numel = output.numel()
881
+ jacobians1, types_ok, sizes_ok = _stack_and_check_tensors(
882
+ vjps1, inputs, output_numel
883
+ )
884
+ jacobians2, _, _ = _stack_and_check_tensors(vjps2, inputs, output_numel)
885
+ reentrant = _check_jacobians_equal(jacobians1, jacobians2, nondet_tol)
886
+
887
+ return jacobians1, reentrant, sizes_ok, types_ok
888
+
889
+
890
+ def _get_analytical_jacobian(inputs, outputs, input_idx, output_idx):
891
+ # Computes the analytical Jacobian in slow mode for a single input-output pair.
892
+ # Forgoes performing checks on dtype, shape, and reentrancy.
893
+ jacobians = _check_analytical_jacobian_attributes(
894
+ inputs, outputs[output_idx], nondet_tol=float("inf"), check_grad_dtypes=False
895
+ )
896
+ return jacobians[input_idx]
897
+
898
+
899
+ def _compute_analytical_jacobian_rows(
900
+ vjp_fn, sample_output
901
+ ) -> list[list[torch.Tensor | None]]:
902
+ # Computes Jacobian row-by-row by projecting `vjp_fn` = v^T J on standard basis
903
+ # vectors: vjp_fn(e) = e^T J is a corresponding row of the Jacobian.
904
+ # NB: this function does not assume vjp_fn(v) to return tensors with the same
905
+ # number of elements for different v. This is checked when we later combine the
906
+ # rows into a single tensor.
907
+ grad_out_base = torch.zeros_like(
908
+ sample_output, memory_format=torch.legacy_contiguous_format
909
+ )
910
+ flat_grad_out = grad_out_base.view(-1)
911
+ # jacobians_rows[i][j] is the Jacobian jth row for the ith input
912
+ jacobians_rows: list[list[torch.Tensor | None]] = []
913
+ for j in range(flat_grad_out.numel()):
914
+ flat_grad_out.zero_()
915
+ flat_grad_out[j] = 1.0 # projection for jth row of Jacobian
916
+ grad_inputs = vjp_fn(grad_out_base)
917
+ for i, d_x in enumerate(grad_inputs):
918
+ if j == 0:
919
+ jacobians_rows.append([])
920
+ jacobians_rows[i] += [
921
+ d_x.clone() if isinstance(d_x, torch.Tensor) else None
922
+ ]
923
+ return jacobians_rows
924
+
925
+
926
+ def _get_analytical_vjps_wrt_specific_output(
927
+ vjp_fn, sample_output, v
928
+ ) -> list[list[torch.Tensor | None]]:
929
+ grad_inputs = vjp_fn(v.reshape(sample_output.shape))
930
+ vjps: list[list[torch.Tensor | None]] = [
931
+ [vjp.clone() if isinstance(vjp, torch.Tensor) else None] for vjp in grad_inputs
932
+ ]
933
+ return vjps
934
+
935
+
936
+ def _check_inputs(tupled_inputs) -> bool:
937
+ # Make sure that gradients are saved for at least one input
938
+ any_input_requiring_grad = False
939
+ for idx, inp in enumerate(tupled_inputs):
940
+ if is_tensor_like(inp) and inp.requires_grad:
941
+ if not (inp.dtype == torch.float64 or inp.dtype == torch.complex128):
942
+ warnings.warn(
943
+ f"Input #{idx} requires gradient and "
944
+ "is not a double precision floating point or complex. "
945
+ "This check will likely fail if all the inputs are "
946
+ "not of double precision floating point or complex. ",
947
+ stacklevel=2,
948
+ )
949
+ if inp.is_sparse:
950
+ content = inp._values()
951
+ elif _is_sparse_compressed_tensor(inp):
952
+ content = inp.values()
953
+ else:
954
+ content = inp
955
+ # TODO: To cover more problematic cases, replace stride = 0 check with
956
+ # "any overlap in memory" once we have a proper function to check it.
957
+ if content.layout is not torch._mkldnn: # type: ignore[attr-defined]
958
+ if not all(
959
+ st > 0 or sz <= 1
960
+ for st, sz in zip(content.stride(), content.size())
961
+ ):
962
+ raise RuntimeError(
963
+ f"The {idx}th input has a dimension with stride 0. gradcheck only "
964
+ "supports inputs that are non-overlapping to be able to "
965
+ "compute the numerical gradients correctly. You should call "
966
+ ".contiguous on the input before passing it to gradcheck."
967
+ )
968
+ any_input_requiring_grad = True
969
+
970
+ if not any_input_requiring_grad:
971
+ raise ValueError(
972
+ "gradcheck expects at least one input tensor to require gradient, "
973
+ "but none of them have requires_grad=True."
974
+ )
975
+ return True
976
+
977
+
978
+ def _check_outputs(outputs) -> None:
979
+ if any(_is_sparse_any_tensor(t) for t in outputs if isinstance(t, torch.Tensor)):
980
+ # it is easier to call to_dense() on the sparse output than
981
+ # to modify analytical jacobian
982
+ raise ValueError(
983
+ "Sparse output is not supported at gradcheck yet. "
984
+ "Please call to_dense(masked_grad=...) on the output of fn for gradcheck."
985
+ )
986
+ if any(t.layout == torch._mkldnn for t in outputs if isinstance(t, torch.Tensor)): # type: ignore[attr-defined]
987
+ raise ValueError(
988
+ "MKLDNN output is not supported at gradcheck yet. "
989
+ "Please call to_dense(masked_grad=...) on the output of fn for gradcheck."
990
+ )
991
+
992
+
993
+ def _check_no_differentiable_outputs(
994
+ func, inputs, func_out, eps, *, is_forward_ad
995
+ ) -> bool:
996
+ # When there are no differentiable outputs, numerical gradient for a function is
997
+ # expected to be zero.
998
+ jacobians_all_inputs_outputs = _get_numerical_jacobian(
999
+ func, inputs, func_out, eps=eps, is_forward_ad=is_forward_ad
1000
+ )
1001
+ for jacobians_all_outputs_and_fixed_input in jacobians_all_inputs_outputs:
1002
+ for jacobian in jacobians_all_outputs_and_fixed_input:
1003
+ if torch.ne(jacobian, 0).sum() > 0:
1004
+ raise GradcheckError(
1005
+ "Numerical gradient for function expected to be zero"
1006
+ )
1007
+ return True
1008
+
1009
+
1010
+ def _check_no_differentiable_outputs_fast(
1011
+ func, func_out, all_inputs, inputs_indices, all_u, eps, nondet_tol
1012
+ ):
1013
+ for inp_idx, u in zip(inputs_indices, all_u):
1014
+ jvps = _get_numerical_jvp_wrt_specific_input(func, inp_idx, all_inputs, u, eps)
1015
+ for jvp in jvps:
1016
+ if jvp.numel() == 0:
1017
+ continue
1018
+ if (jvp - torch.zeros_like(jvp)).abs().max() > nondet_tol:
1019
+ raise GradcheckError(
1020
+ "Numerical gradient for function expected to be zero"
1021
+ )
1022
+ return True
1023
+
1024
+
1025
+ FAILED_BATCHED_GRAD_MSG = """
1026
+ gradcheck or gradgradcheck failed while testing batched gradient computation.
1027
+ This could have been invoked in a number of ways (via a test that calls
1028
+ gradcheck/gradgradcheck directly or via an autogenerated test).
1029
+
1030
+ If you are adding a new operator, please file an issue and then use one of the
1031
+ workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck.
1032
+ If the test
1033
+ - manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
1034
+ with `check_batched_grad=False` as a keyword argument.
1035
+ - is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
1036
+ to have `check_batched_grad=False` and/or `check_batched_gradgrad=False`.
1037
+
1038
+ If you're modifying an existing operator that supports batched grad computation,
1039
+ or wish to make a new operator work with batched grad computation, please read
1040
+ the following.
1041
+
1042
+ To compute batched grads (e.g., jacobians, hessians), we vmap over the backward
1043
+ computation. The most common failure case is if there is a 'vmap-incompatible
1044
+ operation' in the backward pass. Please see
1045
+ NOTE: [How to write vmap-compatible backward formulas]
1046
+ in the codebase for an explanation of how to fix this.
1047
+ """.strip()
1048
+
1049
+ FAILED_BATCHED_GRAD_MSG_FWD_AD = """
1050
+ gradcheck failed while testing batched gradient computation with forward-mode AD.
1051
+ This test is enabled automatically when both `check_batched_grad=True`
1052
+ and `check_forward_ad=True`, but can be disabled in the following ways
1053
+ dependong on how the test was invoked (via a test that calls gradcheck
1054
+ directly or via an autogenerated test).
1055
+
1056
+ If you are adding a new operator, please file an issue and then use one of the
1057
+ workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck.
1058
+ If the test
1059
+ - manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
1060
+ with `check_batched_forward_grad=False` as a keyword argument.
1061
+ - is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
1062
+ to have `check_batched_forward_grad=False`
1063
+ """
1064
+
1065
+
1066
+ def _get_failed_batched_grad_test_msg(
1067
+ output_idx, input_idx, res, exp, is_forward_ad=False
1068
+ ):
1069
+ return f"""
1070
+ For output {output_idx} and input {input_idx}:
1071
+
1072
+ {FAILED_BATCHED_GRAD_MSG_FWD_AD if is_forward_ad else FAILED_BATCHED_GRAD_MSG}
1073
+
1074
+ Got:
1075
+ {res}
1076
+
1077
+ Expected:
1078
+ {exp}
1079
+ """.strip()
1080
+
1081
+
1082
+ def _test_batched_grad_forward_ad(func, inputs) -> bool:
1083
+ fwAD = torch.autograd.forward_ad # To avoid early import issues (do we need this?)
1084
+ if not isinstance(inputs, tuple):
1085
+ raise AssertionError("Expected inputs to be a tuple")
1086
+
1087
+ for input_idx, current_input in enumerate(inputs):
1088
+ if not (is_tensor_like(current_input) and current_input.requires_grad):
1089
+ continue
1090
+
1091
+ def jvp(tangent: torch.Tensor):
1092
+ with fwAD.dual_level():
1093
+ dual = fwAD.make_dual(current_input.detach(), tangent)
1094
+ inputs_with_dual = tuple(
1095
+ dual
1096
+ if idx == input_idx
1097
+ else (inp.detach() if is_tensor_like(inp) else inp)
1098
+ for idx, inp in enumerate(inputs)
1099
+ )
1100
+ dual_outputs = _as_tuple(func(*inputs_with_dual))
1101
+ ret = []
1102
+ for dual_output in dual_outputs:
1103
+ if dual_output is None:
1104
+ continue
1105
+ primal_out, tangent_out = fwAD.unpack_dual(dual_output)
1106
+ if tangent_out is not None:
1107
+ ret.append(tangent_out)
1108
+ else:
1109
+ ret.append(
1110
+ torch.zeros(
1111
+ [], dtype=primal_out.dtype, device=primal_out.device
1112
+ ).expand(primal_out.shape)
1113
+ )
1114
+ return tuple(ret)
1115
+
1116
+ if not _is_float_or_complex_tensor(current_input):
1117
+ continue
1118
+
1119
+ tangents = [torch.randn_like(current_input) for _ in range(2)]
1120
+ expected = [jvp(t) for t in tangents]
1121
+ expected = [torch.stack(shards) for shards in zip(*expected)]
1122
+
1123
+ try:
1124
+ result = _vmap(jvp)(torch.stack(tangents))
1125
+ except RuntimeError as ex:
1126
+ # Rethrow to provide a better error message
1127
+ raise GradcheckError(
1128
+ f"While computing batched gradients, got: {ex}\n\n{FAILED_BATCHED_GRAD_MSG_FWD_AD}"
1129
+ ) from ex
1130
+
1131
+ for input_idx, (res, exp) in enumerate(zip(result, expected)):
1132
+ if torch.allclose(res, exp):
1133
+ continue
1134
+ raise GradcheckError(
1135
+ _get_failed_batched_grad_test_msg(
1136
+ input_idx, input_idx, res, exp, is_forward_ad=True
1137
+ )
1138
+ )
1139
+ return True
1140
+
1141
+
1142
+ def _test_batched_grad(input, output, output_idx) -> bool:
1143
+ # NB: _test_batched_grad compares two autograd.grad invocations with a single
1144
+ # vmap(autograd.grad) invocation. It's not exactly a "gradcheck" in the
1145
+ # sense that we're not comparing an analytical jacobian with a numeric one,
1146
+ # but it is morally similar (we could have computed a full analytic jac
1147
+ # via vmap, but that is potentially slow)
1148
+ diff_input_list = list(_iter_tensors(input, True))
1149
+ grad = functools.partial(
1150
+ torch.autograd.grad,
1151
+ output,
1152
+ diff_input_list,
1153
+ retain_graph=True,
1154
+ allow_unused=True,
1155
+ )
1156
+
1157
+ def vjp(v):
1158
+ results = grad(v)
1159
+ results = tuple(
1160
+ grad
1161
+ if grad is not None
1162
+ else torch.zeros([], dtype=inp.dtype, device=inp.device).expand(inp.shape)
1163
+ for grad, inp in zip(results, diff_input_list)
1164
+ )
1165
+ return results
1166
+
1167
+ grad_outputs = [torch.randn_like(output) for _ in range(2)]
1168
+
1169
+ expected = [vjp(gO) for gO in grad_outputs]
1170
+ expected = [torch.stack(shards) for shards in zip(*expected)]
1171
+
1172
+ # Squash warnings since these are expected to happen in most cases
1173
+ # NB: this doesn't work for CUDA tests: https://github.com/pytorch/pytorch/issues/50209
1174
+ with warnings.catch_warnings():
1175
+ warnings.filterwarnings("ignore", message="There is a performance drop")
1176
+ warnings.filterwarnings("ignore", message="Please use `torch.vmap`")
1177
+ try:
1178
+ result = vmap(vjp)(torch.stack(grad_outputs))
1179
+ except RuntimeError as ex:
1180
+ # It's OK that we're not raising the error at the correct callsite.
1181
+ # That's because the callsite is always going to inside the Python
1182
+ # autograd.grad instead of the C++ traceback of what line in the
1183
+ # backward formula
1184
+ raise GradcheckError(
1185
+ f"While computing batched gradients, got: {ex}\n\n{FAILED_BATCHED_GRAD_MSG}"
1186
+ ) from ex
1187
+
1188
+ for input_idx, (res, exp) in enumerate(zip(result, expected)):
1189
+ if torch.allclose(res, exp):
1190
+ continue
1191
+ raise GradcheckError(
1192
+ _get_failed_batched_grad_test_msg(output_idx, input_idx, res, exp)
1193
+ )
1194
+ return True
1195
+
1196
+
1197
+ def _test_backward_mul_by_grad_output(outputs, inputs, masked) -> bool:
1198
+ # Tests that backward is multiplied by grad_output
1199
+ diff_input_list: list[torch.Tensor] = list(_iter_tensors(inputs, True))
1200
+ if not diff_input_list:
1201
+ raise GradcheckError("no Tensors requiring grad found in input")
1202
+ grads_input = torch.autograd.grad(
1203
+ outputs,
1204
+ diff_input_list,
1205
+ [
1206
+ torch.zeros_like(o, memory_format=torch.legacy_contiguous_format)
1207
+ for o in outputs
1208
+ ],
1209
+ allow_unused=True,
1210
+ )
1211
+ for gi, di in zip(grads_input, diff_input_list):
1212
+ if gi is None:
1213
+ continue
1214
+ if isinstance(gi, torch.Tensor) and gi.layout != torch.strided:
1215
+ if gi.layout != di.layout:
1216
+ raise GradcheckError(
1217
+ "grad is incorrect layout ("
1218
+ + str(gi.layout)
1219
+ + " is not "
1220
+ + str(di.layout)
1221
+ + ")"
1222
+ )
1223
+ if _is_sparse_any_tensor(gi):
1224
+ sparse_kind = str(gi.layout).replace("torch.", "").replace("_coo", "")
1225
+ if gi.sparse_dim() != di.sparse_dim():
1226
+ raise GradcheckError(
1227
+ f"grad is {sparse_kind} tensor, but has incorrect sparse_dim"
1228
+ f" {gi.sparse_dim()}, expected {di.sparse_dim()}"
1229
+ )
1230
+ if gi.dense_dim() != di.dense_dim():
1231
+ raise GradcheckError(
1232
+ f"grad is {sparse_kind} tensor, but has incorrect dense_dim"
1233
+ f" {gi.dense_dim()}, expected {di.dense_dim()}"
1234
+ )
1235
+ gi = gi.to_dense()
1236
+ di = di.to_dense()
1237
+ if masked:
1238
+ if not torch.allclose(gi, torch.zeros_like(gi)):
1239
+ raise GradcheckError("backward not multiplied by grad_output")
1240
+ elif not gi.eq(0).all():
1241
+ raise GradcheckError("backward not multiplied by grad_output")
1242
+ if gi.dtype != di.dtype:
1243
+ raise GradcheckError("grad is incorrect type")
1244
+ if gi.device != di.device:
1245
+ raise GradcheckError("grad is incorrect device")
1246
+ if gi.size() != di.size():
1247
+ raise GradcheckError("grad is incorrect size")
1248
+ return True
1249
+
1250
+
1251
+ def _test_undefined_forward_mode(func, outputs, inputs):
1252
+ fwAD = torch.autograd.forward_ad
1253
+
1254
+ _inp_tensors_idx, inp_tensors = _get_inp_tensors(inputs)
1255
+ _all_v, all_u, _all_u_dense = _make_vectors(
1256
+ inp_tensors, outputs, use_forward_ad=True
1257
+ )
1258
+
1259
+ with fwAD.dual_level():
1260
+ fw_grads = []
1261
+ dual_inputs = []
1262
+ tensor_indices = set()
1263
+ for i, inp in enumerate(inputs):
1264
+ if is_tensor_like(inp) and inp.requires_grad:
1265
+ if inp.layout == torch._mkldnn: # type: ignore[attr-defined]
1266
+ raise ValueError(
1267
+ "MKLDNN inputs are not support for forward AD gradcheck."
1268
+ )
1269
+
1270
+ inp = fwAD.make_dual(inp.detach(), torch.zeros_like(inp))
1271
+ # If inp is a differentiable view, the dual might not be the tangent given to
1272
+ # make_dual, so read it explicitly from the dual tensor
1273
+ fw_grads.append(fwAD.unpack_dual(inp)[1])
1274
+ tensor_indices.add(i)
1275
+ dual_inputs.append(inp)
1276
+
1277
+ for fw_grad, u in zip(fw_grads, all_u):
1278
+ fw_grad.copy_(u.view_as(fw_grad))
1279
+
1280
+ for idx, inp in enumerate(inputs):
1281
+ if idx not in tensor_indices:
1282
+ continue
1283
+ dual_inp_obj = dual_inputs[idx]
1284
+
1285
+ # case 1 (Materialized Zero Tensor Tangent)
1286
+ dual_inputs[idx] = fwAD.make_dual(inp.detach(), torch.zeros_like(inp))
1287
+ raw_outputs = _as_tuple(func(*dual_inputs))
1288
+ dual_outputs1 = filter(_is_float_or_complex_tensor, raw_outputs)
1289
+
1290
+ # case 2 (Efficient Zero Tensor Tangent since we don't make a dual object and pass a regular tensor)
1291
+ dual_inputs[idx] = inp.detach()
1292
+ raw_outputs = _as_tuple(func(*dual_inputs))
1293
+ dual_outputs2 = filter(_is_float_or_complex_tensor, raw_outputs)
1294
+
1295
+ # reset
1296
+ dual_inputs[idx] = dual_inp_obj
1297
+
1298
+ for index_o, (d_o1, d_o2) in enumerate(zip(dual_outputs1, dual_outputs2)):
1299
+ _val1, res1 = fwAD.unpack_dual(d_o1)
1300
+ _val2, res2 = fwAD.unpack_dual(d_o2)
1301
+
1302
+ if not (res1 is None or res2 is None):
1303
+ if not torch.allclose(res1, res2):
1304
+ raise GradcheckError(
1305
+ "Mismatch in tangent values for output with index: ",
1306
+ index_o,
1307
+ " when input: ",
1308
+ inp,
1309
+ " has an undefined tangent value. ",
1310
+ " Got: ",
1311
+ res1,
1312
+ " but expected: ",
1313
+ res2,
1314
+ )
1315
+ return True
1316
+
1317
+
1318
+ def _test_undefined_backward_mode(func, outputs, inputs) -> bool:
1319
+ diff_input_list: list[torch.Tensor] = list(_iter_tensors(inputs, True))
1320
+ if not diff_input_list:
1321
+ raise GradcheckError("no Tensors requiring grad found in input")
1322
+
1323
+ def warn_bc_breaking():
1324
+ warnings.warn(
1325
+ "Backwards compatibility: New undefined gradient support checking "
1326
+ "feature is enabled by default, but it may break existing callers "
1327
+ "of this function. If this is true for you, you can call this "
1328
+ 'function with "check_undefined_grad=False" to disable the feature',
1329
+ stacklevel=2,
1330
+ )
1331
+
1332
+ def check_undefined_grad_support(output_to_check):
1333
+ grads_output = [
1334
+ torch.zeros_like(o, memory_format=torch.legacy_contiguous_format)
1335
+ for o in output_to_check
1336
+ ]
1337
+ try:
1338
+ grads_input = torch.autograd.grad(
1339
+ output_to_check, diff_input_list, grads_output, allow_unused=True
1340
+ )
1341
+ except RuntimeError as e:
1342
+ warn_bc_breaking()
1343
+ raise GradcheckError(
1344
+ "Expected backward function to handle undefined output grads. "
1345
+ 'Please look at "Notes about undefined output gradients" in '
1346
+ '"tools/autograd/derivatives.yaml"'
1347
+ ) from e
1348
+
1349
+ for gi in grads_input:
1350
+ if (gi is not None) and (not gi.eq(0).all()):
1351
+ warn_bc_breaking()
1352
+ raise GradcheckError(
1353
+ "Expected all input grads to be undefined or zero when all output grads are undefined "
1354
+ 'or zero. Please look at "Notes about undefined output gradients" in '
1355
+ '"tools/autograd/derivatives.yaml"'
1356
+ )
1357
+ return True
1358
+
1359
+ # All backward functions must work properly if all output grads are undefined
1360
+ outputs_to_check = [
1361
+ [
1362
+ torch._C._functions.UndefinedGrad()(o)
1363
+ for o in _differentiable_outputs(func(*inputs))
1364
+ # This check filters out Tensor-likes that aren't instances of Tensor.
1365
+ if isinstance(o, torch.Tensor)
1366
+ ]
1367
+ ]
1368
+
1369
+ # If there are multiple output grads, we should be able to undef one at a time without error
1370
+ if len(outputs_to_check[0]) > 1:
1371
+ for undef_grad_idx in range(len(outputs)):
1372
+ output_to_check = _differentiable_outputs(func(*inputs))
1373
+ outputs_to_check.append(
1374
+ [
1375
+ torch._C._functions.UndefinedGrad()(o)
1376
+ if idx == undef_grad_idx
1377
+ else o
1378
+ for idx, o in enumerate(output_to_check)
1379
+ ]
1380
+ )
1381
+
1382
+ return all(check_undefined_grad_support(output) for output in outputs_to_check)
1383
+
1384
+
1385
+ def _as_tuple(x):
1386
+ if isinstance(x, tuple):
1387
+ return x
1388
+ elif isinstance(x, list):
1389
+ return tuple(x)
1390
+ else:
1391
+ return (x,)
1392
+
1393
+
1394
+ def _differentiable_outputs(x):
1395
+ return tuple(o for o in _as_tuple(x) if o.requires_grad)
1396
+
1397
+
1398
+ def _get_notallclose_msg(
1399
+ analytical,
1400
+ numerical,
1401
+ output_idx,
1402
+ input_idx,
1403
+ complex_indices,
1404
+ test_imag=False,
1405
+ is_forward_ad=False,
1406
+ ) -> str:
1407
+ out_is_complex = (
1408
+ (not is_forward_ad) and complex_indices and output_idx in complex_indices
1409
+ )
1410
+ inp_is_complex = is_forward_ad and complex_indices and input_idx in complex_indices
1411
+ part = "imaginary" if test_imag else "real"
1412
+ element = "inputs" if is_forward_ad else "outputs"
1413
+ prefix = (
1414
+ ""
1415
+ if not (out_is_complex or inp_is_complex)
1416
+ else f"While considering the {part} part of complex {element} only, "
1417
+ )
1418
+ mode = "computed with forward mode " if is_forward_ad else ""
1419
+ return (
1420
+ prefix
1421
+ + f"Jacobian {mode}mismatch for output {output_idx:d} with respect to input {input_idx:d},\n"
1422
+ f"numerical:{numerical}\nanalytical:{analytical}\n"
1423
+ )
1424
+
1425
+
1426
+ def _transpose(matrix_of_tensors):
1427
+ # returns list of tuples
1428
+ return list(zip(*matrix_of_tensors))
1429
+
1430
+
1431
+ def _real_and_imag_output(fn):
1432
+ # returns new functions real(fn), and imag(fn) where real(fn) and imag(fn) behave the same as
1433
+ # the original fn, except torch.real or torch.imag are applied to the complex outputs
1434
+ def apply_to_c_outs(fn, fn_to_apply):
1435
+ def wrapped_fn(*inputs):
1436
+ outs = _as_tuple(fn(*inputs))
1437
+ return tuple(fn_to_apply(o) if o.is_complex() else o for o in outs)
1438
+
1439
+ return wrapped_fn
1440
+
1441
+ return apply_to_c_outs(fn, torch.real), apply_to_c_outs(fn, torch.imag)
1442
+
1443
+
1444
+ def _real_and_imag_input(fn, complex_inp_indices, tupled_inputs):
1445
+ # returns new functions that take real inputs instead of complex inputs as
1446
+ # (x, y) -> fn(x + y * 1j). And it computes: inp -> fn(inp + y * 1j) and inp -> fn(x + inp * 1j).
1447
+ # In each case, the other part is considered constant.
1448
+ # We do not use 0 for the constant here to make sure we always call the user function with a valid input.
1449
+ def apply_to_c_inps(fn, fn_to_apply):
1450
+ def wrapped_fn(*inputs):
1451
+ new_inputs = list(inputs)
1452
+ for should_be_complex in complex_inp_indices:
1453
+ new_inputs[should_be_complex] = fn_to_apply(
1454
+ new_inputs[should_be_complex], tupled_inputs[should_be_complex]
1455
+ )
1456
+ return _as_tuple(fn(*new_inputs))
1457
+
1458
+ return wrapped_fn
1459
+
1460
+ real_fn = apply_to_c_inps(fn, lambda inp, orig: inp + orig.imag * 1j)
1461
+ imag_fn = apply_to_c_inps(fn, lambda inp, orig: orig.real + inp * 1j)
1462
+ return real_fn, imag_fn
1463
+
1464
+
1465
+ def _gradcheck_real_imag(
1466
+ gradcheck_fn,
1467
+ func,
1468
+ func_out,
1469
+ tupled_inputs,
1470
+ outputs,
1471
+ eps,
1472
+ rtol,
1473
+ atol,
1474
+ check_grad_dtypes,
1475
+ check_forward_ad,
1476
+ check_backward_ad,
1477
+ nondet_tol,
1478
+ check_undefined_grad,
1479
+ ):
1480
+ complex_out_indices = [i for i, o in enumerate(outputs) if o.is_complex()]
1481
+ has_any_complex_output = any(o.is_complex() for o in _as_tuple(func_out))
1482
+ if check_backward_ad:
1483
+ if has_any_complex_output:
1484
+ real_fn, imag_fn = _real_and_imag_output(func)
1485
+
1486
+ imag_func_out = imag_fn(*tupled_inputs)
1487
+ imag_outputs = _differentiable_outputs(imag_func_out)
1488
+ gradcheck_fn(
1489
+ imag_fn,
1490
+ imag_func_out,
1491
+ tupled_inputs,
1492
+ imag_outputs,
1493
+ eps,
1494
+ rtol,
1495
+ atol,
1496
+ check_grad_dtypes,
1497
+ nondet_tol,
1498
+ complex_indices=complex_out_indices,
1499
+ test_imag=True,
1500
+ )
1501
+
1502
+ real_func_out = real_fn(*tupled_inputs)
1503
+ real_outputs = _differentiable_outputs(real_func_out)
1504
+ gradcheck_fn(
1505
+ real_fn,
1506
+ real_func_out,
1507
+ tupled_inputs,
1508
+ real_outputs,
1509
+ eps,
1510
+ rtol,
1511
+ atol,
1512
+ check_grad_dtypes,
1513
+ nondet_tol,
1514
+ complex_indices=complex_out_indices,
1515
+ )
1516
+ else:
1517
+ gradcheck_fn(
1518
+ func,
1519
+ func_out,
1520
+ tupled_inputs,
1521
+ outputs,
1522
+ eps,
1523
+ rtol,
1524
+ atol,
1525
+ check_grad_dtypes,
1526
+ nondet_tol,
1527
+ )
1528
+
1529
+ if check_forward_ad:
1530
+ complex_inp_indices = [
1531
+ i
1532
+ for i, inp in enumerate(tupled_inputs)
1533
+ if is_tensor_like(inp) and inp.is_complex()
1534
+ ]
1535
+ if complex_inp_indices:
1536
+ real_fn, imag_fn = _real_and_imag_input(
1537
+ func, complex_inp_indices, tupled_inputs
1538
+ )
1539
+
1540
+ imag_inputs = [
1541
+ inp.imag if is_tensor_like(inp) and inp.is_complex() else inp
1542
+ for inp in tupled_inputs
1543
+ ]
1544
+ imag_func_out = imag_fn(*imag_inputs)
1545
+ diff_imag_func_out = _differentiable_outputs(imag_func_out)
1546
+ gradcheck_fn(
1547
+ imag_fn,
1548
+ imag_func_out,
1549
+ imag_inputs,
1550
+ diff_imag_func_out,
1551
+ eps,
1552
+ rtol,
1553
+ atol,
1554
+ check_grad_dtypes,
1555
+ nondet_tol,
1556
+ complex_indices=complex_inp_indices,
1557
+ test_imag=True,
1558
+ use_forward_ad=True,
1559
+ )
1560
+
1561
+ real_inputs = [
1562
+ inp.real if is_tensor_like(inp) and inp.is_complex() else inp
1563
+ for inp in tupled_inputs
1564
+ ]
1565
+ real_func_out = real_fn(*real_inputs)
1566
+ diff_real_func_out = _differentiable_outputs(real_func_out)
1567
+ gradcheck_fn(
1568
+ real_fn,
1569
+ real_func_out,
1570
+ real_inputs,
1571
+ diff_real_func_out,
1572
+ eps,
1573
+ rtol,
1574
+ atol,
1575
+ check_grad_dtypes,
1576
+ nondet_tol,
1577
+ complex_indices=complex_inp_indices,
1578
+ use_forward_ad=True,
1579
+ )
1580
+ if check_undefined_grad:
1581
+ _test_undefined_forward_mode(imag_fn, imag_func_out, imag_inputs)
1582
+ _test_undefined_forward_mode(real_fn, real_func_out, real_inputs)
1583
+ else:
1584
+ gradcheck_fn(
1585
+ func,
1586
+ func_out,
1587
+ tupled_inputs,
1588
+ outputs,
1589
+ eps,
1590
+ rtol,
1591
+ atol,
1592
+ check_grad_dtypes,
1593
+ nondet_tol,
1594
+ use_forward_ad=True,
1595
+ )
1596
+ if check_undefined_grad:
1597
+ _test_undefined_forward_mode(func, outputs, tupled_inputs)
1598
+
1599
+
1600
+ def _slow_gradcheck(
1601
+ func,
1602
+ func_out,
1603
+ tupled_inputs,
1604
+ outputs,
1605
+ eps,
1606
+ rtol,
1607
+ atol,
1608
+ check_grad_dtypes,
1609
+ nondet_tol,
1610
+ *,
1611
+ use_forward_ad=False,
1612
+ complex_indices=None,
1613
+ test_imag=False,
1614
+ masked=False,
1615
+ ):
1616
+ func_out = _as_tuple(func_out)
1617
+ if not outputs:
1618
+ return _check_no_differentiable_outputs(
1619
+ func, tupled_inputs, func_out, eps=eps, is_forward_ad=use_forward_ad
1620
+ )
1621
+ tupled_inputs_numerical = tupled_inputs if masked else _densify(tupled_inputs)
1622
+
1623
+ numerical = _transpose(
1624
+ _get_numerical_jacobian(
1625
+ func,
1626
+ tupled_inputs_numerical,
1627
+ func_out,
1628
+ eps=eps,
1629
+ is_forward_ad=use_forward_ad,
1630
+ )
1631
+ )
1632
+ # Note: [numerical vs analytical output length]
1633
+ # The numerical path returns jacobian quantity for all outputs, even if requires_grad of that
1634
+ # output is False. This behavior is necessary for _check_no_differentiable_outputs to work.
1635
+ numerical = [nj for o, nj in zip(func_out, numerical) if o.requires_grad]
1636
+ if use_forward_ad:
1637
+ analytical_forward = _get_analytical_jacobian_forward_ad(
1638
+ func, tupled_inputs, func_out, check_grad_dtypes=check_grad_dtypes
1639
+ )
1640
+
1641
+ for i, n_per_out in enumerate(numerical):
1642
+ for j, n in enumerate(n_per_out):
1643
+ a = analytical_forward[j][i]
1644
+ if not _allclose_with_type_promotion(a, n.to(a.device), rtol, atol):
1645
+ raise GradcheckError(
1646
+ _get_notallclose_msg(
1647
+ a, n, i, j, complex_indices, test_imag, is_forward_ad=True
1648
+ )
1649
+ )
1650
+ else:
1651
+ for i, o in enumerate(outputs):
1652
+ analytical = _check_analytical_jacobian_attributes(
1653
+ tupled_inputs, o, nondet_tol, check_grad_dtypes
1654
+ )
1655
+
1656
+ for j, (a, n) in enumerate(zip(analytical, numerical[i])):
1657
+ if not _allclose_with_type_promotion(a, n.to(a.device), rtol, atol):
1658
+ raise GradcheckError(
1659
+ _get_notallclose_msg(a, n, i, j, complex_indices, test_imag)
1660
+ )
1661
+
1662
+ return True
1663
+
1664
+
1665
+ def _dot_with_type_promotion(u, v):
1666
+ if u.dim() != 1 or v.dim() != 1:
1667
+ raise AssertionError(
1668
+ f"Expected u and v to be 1D tensors, but got dims {u.dim()} and {v.dim()}"
1669
+ )
1670
+ return (u * v).sum()
1671
+
1672
+
1673
+ def _allclose_with_type_promotion(a, b, rtol, atol):
1674
+ promoted_type = torch.promote_types(a.dtype, b.dtype)
1675
+ a = a.to(dtype=promoted_type)
1676
+ b = b.to(dtype=promoted_type)
1677
+ return torch.allclose(a, b, rtol, atol)
1678
+
1679
+
1680
+ def _to_real_dtype(dtype):
1681
+ if dtype == torch.complex128:
1682
+ return torch.float64
1683
+ elif dtype == torch.complex64:
1684
+ return torch.float32
1685
+ else:
1686
+ return dtype
1687
+
1688
+
1689
+ def _vec_from_tensor(x, generator, downcast_complex=False):
1690
+ # Create a random vector with the same number of elements as x and the same
1691
+ # dtype/device. If x is complex and downcast_complex is False, we create a
1692
+ # complex tensor with only real component.
1693
+ if x.layout == torch.sparse_coo:
1694
+ # For sparse, create a random sparse vec with random values in the same
1695
+ # indices. Make sure size is set so that it isn't inferred to be smaller.
1696
+ x_values = x._values()
1697
+ dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype
1698
+ values = (
1699
+ torch.rand(x_values.numel(), generator=generator)
1700
+ .to(dtype=dtype, device=x.device)
1701
+ .view(x_values.shape)
1702
+ )
1703
+ values /= values.norm()
1704
+ vec = torch.sparse_coo_tensor(x._indices(), values, x.size(), device=x.device)
1705
+ elif _is_sparse_compressed_tensor(x):
1706
+ if x.layout in {torch.sparse_csr, torch.sparse_bsr}:
1707
+ compressed_indices, plain_indices = x.crow_indices(), x.col_indices()
1708
+ else:
1709
+ compressed_indices, plain_indices = x.ccol_indices(), x.row_indices()
1710
+ x_values = x.values()
1711
+ dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype
1712
+ values = (
1713
+ torch.rand(x_values.numel(), generator=generator)
1714
+ .to(dtype=dtype, device=x.device)
1715
+ .view(x_values.shape)
1716
+ )
1717
+ values /= values.norm()
1718
+ vec = torch.sparse_compressed_tensor(
1719
+ compressed_indices,
1720
+ plain_indices,
1721
+ values,
1722
+ x.size(),
1723
+ layout=x.layout,
1724
+ device=x.device,
1725
+ )
1726
+ else:
1727
+ dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype
1728
+ vec = torch.rand(x.numel(), generator=generator).to(
1729
+ dtype=dtype, device=x.device
1730
+ )
1731
+ vec /= vec.norm()
1732
+ return vec
1733
+
1734
+
1735
+ def _get_inp_tensors(tupled_inputs):
1736
+ inp_idx_tup = [
1737
+ (i, t)
1738
+ for i, t in enumerate(tupled_inputs)
1739
+ if is_tensor_like(t) and t.requires_grad
1740
+ ]
1741
+ return [tup[0] for tup in inp_idx_tup], [tup[1] for tup in inp_idx_tup]
1742
+
1743
+
1744
+ def _adjusted_atol(atol, u, v):
1745
+ # In slow gradcheck, we compare A and B element-wise, i.e., for some a, b we
1746
+ # allow: |a - b| < atol + rtol * b. But since we now compare q1 = v^T A u and
1747
+ # q2 = v^T B u, we must allow |q1 - q2| < v^T E u + rtol * v^T B u, where E is
1748
+ # the correctly sized matrix in which each entry is atol.
1749
+ #
1750
+ # We see that atol needs to be scaled by v^T M u (where M is an all-ones M x N
1751
+ # matrix): v^T M u = \sum_{i} \sum_{j} u[i] * v[j] = sum(u) * sum(v).
1752
+
1753
+ # For the case of complex inputs, u has re and im. components: u = (ur, ui).
1754
+ # Let q = a - b = (qr, qi) in the above notation, eg. q is the difference between analytic and numerical
1755
+ # Jacobians. Then the transformed tolerance checks being done in the torch.allclose ops
1756
+ # are of form abs(Re{q_r u_r - q_i u_i}) < atol * (abs(u_r) + abs(u_i)), and equivalently
1757
+ # for imaginary components abs(Im{q_r u_r - q_i ui}) < atol * (abs(u_r) + abs(u_i)).
1758
+ # Since u is drawn randomly non-negative, the end effect is a factor
1759
+ # (sum(u_r) + sum(u_i)) * sum(v) increase in atol for complex inputs, eg.
1760
+ # a statistical factor of 2 as compared to the real case.
1761
+
1762
+ sum_v = 1.0 if v is None else v.sum()
1763
+
1764
+ if isinstance(u, tuple):
1765
+ # case of complex input
1766
+ ur, ui = u[0], u[1]
1767
+ sum_ur = ur.sum()
1768
+ sum_ui = ui.sum()
1769
+ complex_modified_atol = atol * (float(sum_ur) + float(sum_ui)) * float(sum_v)
1770
+ return complex_modified_atol
1771
+
1772
+ # case of real input
1773
+ sum_u = u.sum()
1774
+ modified_atol = atol * float(sum_u) * float(sum_v)
1775
+ return modified_atol
1776
+
1777
+
1778
+ FAST_FAIL_SLOW_OK_MSG = """
1779
+ Fast gradcheck failed but element-wise differences are small. This means that the
1780
+ test might've passed in slow_mode!
1781
+
1782
+ If you are adding a new operator, please file an issue and then use one of the
1783
+ workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck:
1784
+
1785
+ If the test
1786
+ - manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
1787
+ with `fast_mode=False` as a keyword argument.
1788
+ - is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
1789
+ to have `gradcheck_fast_mode=False`
1790
+ - is a Module test (e.g., in common_nn.py), then modify the corresponding
1791
+ module_test entry to have `gradcheck_fast_mode=False`
1792
+ """.strip()
1793
+
1794
+
1795
+ def _run_slow_mode_and_get_error(
1796
+ func, tupled_inputs, outputs, input_idx, output_idx, rtol, atol, eps, is_forward_ad
1797
+ ):
1798
+ # Compute jacobians in slow mode for better error message
1799
+ slow_numerical = _get_numerical_jacobian(
1800
+ func, tupled_inputs, outputs, eps=eps, is_forward_ad=is_forward_ad
1801
+ )[input_idx][output_idx]
1802
+ if is_forward_ad:
1803
+
1804
+ def new_fn(inp):
1805
+ new_inputs = list(tupled_inputs)
1806
+ new_inputs[input_idx] = inp
1807
+ return _as_tuple(func(*new_inputs))[output_idx]
1808
+
1809
+ slow_analytical = _get_analytical_jacobian_forward_ad(
1810
+ new_fn, (tupled_inputs[input_idx],), (outputs[output_idx],)
1811
+ )[0][0]
1812
+ else:
1813
+ slow_analytical = _get_analytical_jacobian(
1814
+ tupled_inputs, outputs, input_idx, output_idx
1815
+ )
1816
+
1817
+ # Assume jacobians are non-empty and have the same shape
1818
+ slow_max_diff = (slow_numerical - slow_analytical).abs().max()
1819
+
1820
+ slow_allclose = torch.allclose(slow_analytical, slow_numerical, rtol, atol)
1821
+ msg = (
1822
+ "\nThe above quantities relating the numerical and analytical jacobians are computed \n"
1823
+ "in fast mode. See: https://github.com/pytorch/pytorch/issues/53876 for more background \n"
1824
+ "about fast mode. Below, we recompute numerical and analytical jacobians in slow mode:\n\n"
1825
+ f"Numerical:\n {slow_numerical}\n"
1826
+ f"Analytical:\n{slow_analytical}\n\n"
1827
+ f"The max per-element difference (slow mode) is: {slow_max_diff}.\n"
1828
+ )
1829
+ if slow_allclose:
1830
+ # Slow gradcheck would've passed!
1831
+ msg += FAST_FAIL_SLOW_OK_MSG
1832
+ return msg
1833
+
1834
+
1835
+ def _to_flat_dense_if_sparse(tensor):
1836
+ if _is_sparse_any_tensor(tensor):
1837
+ return tensor.to_dense().reshape(-1)
1838
+ else:
1839
+ return tensor
1840
+
1841
+
1842
+ def _make_vectors(inp_tensors, outputs, *, use_forward_ad):
1843
+ # Use our own generator to avoid messing with the user's RNG state
1844
+ g_cpu = torch.Generator()
1845
+
1846
+ def _vec_from_tensor_cpu(*args):
1847
+ # Default allocate all tensors on CPU, so they are on the same device as the generator
1848
+ # even if the user specified a default device
1849
+ with torch.device("cpu"):
1850
+ return _vec_from_tensor(*args)
1851
+
1852
+ all_u = []
1853
+ all_u_dense = []
1854
+ for inp in inp_tensors:
1855
+ ur = _vec_from_tensor_cpu(inp, g_cpu, True)
1856
+ ur_dense = _to_flat_dense_if_sparse(ur)
1857
+ if inp.is_complex():
1858
+ ui = _vec_from_tensor_cpu(inp, g_cpu, True)
1859
+ all_u.append((ur, ui))
1860
+ ui_dense = _to_flat_dense_if_sparse(ui)
1861
+ all_u_dense.append((ur_dense, ui_dense))
1862
+ else:
1863
+ all_u.append(ur)
1864
+ all_u_dense.append(ur_dense)
1865
+ all_v = (
1866
+ None
1867
+ if use_forward_ad
1868
+ else [_vec_from_tensor_cpu(out, g_cpu) for out in outputs]
1869
+ )
1870
+ return all_v, all_u, all_u_dense
1871
+
1872
+
1873
+ def _check_analytical_numerical_equal(
1874
+ all_analytical,
1875
+ all_numerical,
1876
+ complex_indices,
1877
+ tupled_inputs,
1878
+ outputs,
1879
+ func,
1880
+ all_v,
1881
+ all_u,
1882
+ rtol,
1883
+ atol,
1884
+ eps,
1885
+ test_imag,
1886
+ *,
1887
+ is_forward_ad=False,
1888
+ ):
1889
+ for i, all_numerical_for_input_i in enumerate(all_numerical):
1890
+ for j, n in enumerate(all_numerical_for_input_i):
1891
+ # Forward AD generates the transpose of what this function expects
1892
+ if is_forward_ad:
1893
+ a = all_analytical[i][j]
1894
+ else:
1895
+ a = all_analytical[j][i]
1896
+ n = n.to(device=a.device)
1897
+ updated_atol = _adjusted_atol(atol, all_u[i], all_v[j] if all_v else None)
1898
+ if not _allclose_with_type_promotion(a, n.to(a.device), rtol, updated_atol):
1899
+ jacobians_str = _run_slow_mode_and_get_error(
1900
+ func, tupled_inputs, outputs, i, j, rtol, atol, eps, is_forward_ad
1901
+ )
1902
+ raise GradcheckError(
1903
+ _get_notallclose_msg(
1904
+ a, n, j, i, complex_indices, test_imag, is_forward_ad
1905
+ )
1906
+ + jacobians_str
1907
+ )
1908
+
1909
+
1910
+ def _fast_gradcheck(
1911
+ func,
1912
+ func_out,
1913
+ inputs,
1914
+ outputs,
1915
+ eps,
1916
+ rtol,
1917
+ atol,
1918
+ check_grad_dtypes,
1919
+ nondet_tol,
1920
+ *,
1921
+ use_forward_ad=False,
1922
+ complex_indices=None,
1923
+ test_imag=False,
1924
+ masked=False,
1925
+ ):
1926
+ # See https://github.com/pytorch/pytorch/issues/53876 for details
1927
+ inp_tensors_idx, inp_tensors = _get_inp_tensors(inputs)
1928
+ # Backward mode computes v^T * J (VJP)
1929
+ # Since we computed J * u (JVP) through finite difference method, we perform an equality check
1930
+ # between VJP * u, v * JVP
1931
+ # ----
1932
+ # Forward mode computes J * u (JVP)
1933
+ # Since we already compute JVP through finite difference method,
1934
+ # we don't need v for correctness check here as asserted below
1935
+ all_v, all_u, all_u_dense = _make_vectors(
1936
+ inp_tensors, outputs, use_forward_ad=use_forward_ad
1937
+ )
1938
+
1939
+ inputs_numerical, all_u_numerical, all_v_numerical = (
1940
+ (inputs, all_u, all_v) if masked else _densify((inputs, all_u, all_v))
1941
+ )
1942
+
1943
+ numerical_vJu = _get_numerical_vJu(
1944
+ func,
1945
+ inputs_numerical,
1946
+ inp_tensors_idx,
1947
+ func_out,
1948
+ all_u_numerical,
1949
+ all_v_numerical,
1950
+ eps,
1951
+ is_forward_ad=use_forward_ad,
1952
+ )
1953
+ # TODO: replicate https://github.com/pytorch/pytorch/pull/77743 for fast gradcheck as well
1954
+ if use_forward_ad:
1955
+ if all_v is not None:
1956
+ raise AssertionError("Expected all_v to be None.")
1957
+ analytical_vJu = _get_analytical_jacobian_forward_ad(
1958
+ func,
1959
+ inputs,
1960
+ _as_tuple(func_out),
1961
+ all_u=all_u,
1962
+ check_grad_dtypes=check_grad_dtypes,
1963
+ )
1964
+ else:
1965
+ if not outputs:
1966
+ _check_no_differentiable_outputs_fast(
1967
+ func, func_out, inputs, inp_tensors_idx, all_u, eps, nondet_tol
1968
+ )
1969
+
1970
+ analytical_vJu = _get_analytical_vJu_backward_mode(
1971
+ inputs, outputs, nondet_tol, check_grad_dtypes, all_v, all_u_dense
1972
+ )
1973
+
1974
+ _check_analytical_numerical_equal(
1975
+ analytical_vJu,
1976
+ numerical_vJu,
1977
+ complex_indices,
1978
+ inputs,
1979
+ outputs,
1980
+ func,
1981
+ all_v,
1982
+ all_u,
1983
+ rtol,
1984
+ atol,
1985
+ eps,
1986
+ test_imag,
1987
+ is_forward_ad=use_forward_ad,
1988
+ )
1989
+
1990
+ return True
1991
+
1992
+
1993
+ # Note [VarArg of Tensors]
1994
+ # ~~~~~~~~~~~~~~~~~~~~~~~~
1995
+ # 'func' accepts a vararg of tensors, which isn't expressible in the type system at the moment.
1996
+ # If https://mypy.readthedocs.io/en/latest/additional_features.html?highlight=callable#extended-callable-types is accepted,
1997
+ # the '...' first argument of Callable can be replaced with VarArg(Tensor).
1998
+ # For now, we permit any input.
1999
+ def gradcheck(
2000
+ func: Callable[..., _TensorOrTensors], # See Note [VarArg of Tensors]
2001
+ inputs: _TensorOrTensors,
2002
+ *,
2003
+ eps: float = 1e-6,
2004
+ atol: float = 1e-5,
2005
+ rtol: float = 1e-3,
2006
+ raise_exception: bool = True,
2007
+ nondet_tol: float = 0.0,
2008
+ check_undefined_grad: bool = True,
2009
+ check_grad_dtypes: bool = False,
2010
+ check_batched_grad: bool = False,
2011
+ check_batched_forward_grad: bool = False,
2012
+ check_forward_ad: bool = False,
2013
+ check_backward_ad: bool = True,
2014
+ fast_mode: bool = False,
2015
+ masked: bool | None = None,
2016
+ ) -> bool:
2017
+ r"""Check gradients computed via small finite differences against analytical
2018
+ gradients wrt tensors in :attr:`inputs` that are of floating point or complex type
2019
+ and with ``requires_grad=True``.
2020
+
2021
+ The check between numerical and analytical gradients uses :func:`~torch.allclose`.
2022
+
2023
+ For most of the complex functions we consider for optimization purposes, no notion of
2024
+ Jacobian exists. Instead, gradcheck verifies if the numerical and analytical values of
2025
+ the Wirtinger and Conjugate Wirtinger derivatives are consistent. Because the gradient
2026
+ computation is done under the assumption that the overall function has a real-valued
2027
+ output, we treat functions with complex output in a special way. For these functions,
2028
+ gradcheck is applied to two real-valued functions corresponding to taking the real
2029
+ components of the complex outputs for the first, and taking the imaginary components
2030
+ of the complex outputs for the second. For more details, check out
2031
+ :ref:`complex_autograd-doc`.
2032
+
2033
+ .. note::
2034
+ The default values are designed for :attr:`input` of double precision.
2035
+ This check will likely fail if :attr:`input` is of less precision, e.g.,
2036
+ ``FloatTensor``.
2037
+
2038
+ .. note::
2039
+ Gradcheck may fail when evaluated on non-differentiable points
2040
+ because the numerically computed gradients via finite differencing may differ
2041
+ those computed analytically (not necessarily because either is incorrect).
2042
+ For more context, see :ref:`non-differentiable-func-grad`.
2043
+
2044
+ .. warning::
2045
+ If any checked tensor in :attr:`input` has overlapping memory, i.e.,
2046
+ different indices pointing to the same memory address (e.g., from
2047
+ :func:`torch.Tensor.expand`), this check will likely fail because the numerical
2048
+ gradients computed by point perturbation at such indices will change
2049
+ values at all other indices that share the same memory address.
2050
+
2051
+ Args:
2052
+ func (function): a Python function that takes Tensor inputs and returns
2053
+ a Tensor or a tuple of Tensors
2054
+ inputs (tuple of Tensor or Tensor): inputs to the function
2055
+ eps (float, optional): perturbation for finite differences
2056
+ atol (float, optional): absolute tolerance
2057
+ rtol (float, optional): relative tolerance
2058
+ raise_exception (bool, optional): indicating whether to raise an exception if
2059
+ the check fails. The exception gives more information about the
2060
+ exact nature of the failure. This is helpful when debugging gradchecks.
2061
+ nondet_tol (float, optional): tolerance for non-determinism. When running
2062
+ identical inputs through the differentiation, the results must either match
2063
+ exactly (default, 0.0) or be within this tolerance.
2064
+ check_undefined_grad (bool, optional): if ``True``, check if undefined output grads
2065
+ are supported and treated as zeros, for ``Tensor`` outputs.
2066
+ check_batched_grad (bool, optional): if ``True``, check if we can compute
2067
+ batched gradients using prototype vmap support. Defaults to False.
2068
+ check_batched_forward_grad (bool, optional): if ``True``, checks if we can compute
2069
+ batched forward gradients using forward ad and prototype vmap support. Defaults to ``False``.
2070
+ check_forward_ad (bool, optional): if ``True``, check that the gradients computed with forward
2071
+ mode AD match the numerical ones. Defaults to ``False``.
2072
+ check_backward_ad (bool, optional): if ``False``, do not perform any checks that rely on
2073
+ backward mode AD to be implemented. Defaults to ``True``.
2074
+ fast_mode (bool, optional): Fast mode for gradcheck and gradgradcheck is currently only
2075
+ implemented for R to R functions. If none of the inputs and outputs are complex
2076
+ a faster implementation of gradcheck that no longer computes the entire jacobian
2077
+ is run; otherwise, we fall back to the slow implementation.
2078
+ masked (bool, optional): if ``True``, the gradients of unspecified elements of
2079
+ sparse tensors are ignored. Defaults to ``False``.
2080
+ Returns:
2081
+ ``True`` if all differences satisfy allclose condition
2082
+
2083
+ """
2084
+ if not (check_forward_ad or check_backward_ad):
2085
+ raise AssertionError(
2086
+ "Expected at least one of check_forward_ad or check_backward_ad to be True"
2087
+ )
2088
+ if check_batched_grad and not check_backward_ad:
2089
+ raise AssertionError(
2090
+ "Setting check_batched_grad=True requires check_backward_ad to be True"
2091
+ )
2092
+ if check_batched_forward_grad and not check_forward_ad:
2093
+ raise AssertionError(
2094
+ "Setting check_batched_forward_grad=True requires check_forward_ad to be True"
2095
+ )
2096
+ args = locals().copy()
2097
+ args.pop("raise_exception")
2098
+ if not raise_exception:
2099
+ try:
2100
+ return _gradcheck_helper(**args)
2101
+ except GradcheckError:
2102
+ return False
2103
+ else:
2104
+ return _gradcheck_helper(**args)
2105
+
2106
+
2107
+ def _gradcheck_helper(
2108
+ func,
2109
+ inputs,
2110
+ eps,
2111
+ atol,
2112
+ rtol,
2113
+ nondet_tol,
2114
+ check_undefined_grad,
2115
+ check_grad_dtypes,
2116
+ check_batched_grad,
2117
+ check_batched_forward_grad,
2118
+ check_forward_ad,
2119
+ check_backward_ad,
2120
+ fast_mode,
2121
+ masked,
2122
+ ):
2123
+ tupled_inputs = _as_tuple(inputs)
2124
+ _check_inputs(tupled_inputs)
2125
+
2126
+ func_out = func(*tupled_inputs)
2127
+ outputs = _differentiable_outputs(func_out)
2128
+ _check_outputs(outputs)
2129
+
2130
+ gradcheck_fn = functools.partial(
2131
+ _fast_gradcheck if fast_mode else _slow_gradcheck, masked=masked
2132
+ )
2133
+ _gradcheck_real_imag(
2134
+ gradcheck_fn,
2135
+ func,
2136
+ func_out,
2137
+ tupled_inputs,
2138
+ outputs,
2139
+ eps,
2140
+ rtol,
2141
+ atol,
2142
+ check_grad_dtypes,
2143
+ check_forward_ad=check_forward_ad,
2144
+ check_backward_ad=check_backward_ad,
2145
+ nondet_tol=nondet_tol,
2146
+ check_undefined_grad=check_undefined_grad,
2147
+ )
2148
+
2149
+ if check_batched_forward_grad:
2150
+ _test_batched_grad_forward_ad(func, tupled_inputs)
2151
+
2152
+ # Short circuit because remaining tests rely on backward AD to be implemented
2153
+ if not check_backward_ad:
2154
+ return True
2155
+
2156
+ for i, o in enumerate(outputs):
2157
+ if check_batched_grad:
2158
+ _test_batched_grad(tupled_inputs, o, i)
2159
+
2160
+ _test_backward_mul_by_grad_output(outputs, tupled_inputs, masked)
2161
+
2162
+ if check_undefined_grad and check_backward_ad:
2163
+ _test_undefined_backward_mode(func, outputs, tupled_inputs)
2164
+ return True
2165
+
2166
+
2167
+ def gradgradcheck(
2168
+ func: Callable[..., _TensorOrTensors], # See Note [VarArg of Tensors]
2169
+ inputs: _TensorOrTensors,
2170
+ grad_outputs: _TensorOrOptionalTensors | None = None,
2171
+ *,
2172
+ eps: float = 1e-6,
2173
+ atol: float = 1e-5,
2174
+ rtol: float = 1e-3,
2175
+ gen_non_contig_grad_outputs: bool = False,
2176
+ raise_exception: bool = True,
2177
+ nondet_tol: float = 0.0,
2178
+ check_undefined_grad: bool = True,
2179
+ check_grad_dtypes: bool = False,
2180
+ check_batched_grad: bool = False,
2181
+ check_fwd_over_rev: bool = False,
2182
+ check_rev_over_rev: bool = True,
2183
+ fast_mode: bool = False,
2184
+ masked: bool = False,
2185
+ ) -> bool:
2186
+ r"""Check gradients of gradients computed via small finite differences
2187
+ against analytical gradients wrt tensors in :attr:`inputs` and
2188
+ :attr:`grad_outputs` that are of floating point or complex type and with
2189
+ ``requires_grad=True``.
2190
+
2191
+ This function checks that backpropagating through the gradients computed
2192
+ to the given :attr:`grad_outputs` are correct.
2193
+
2194
+ The check between numerical and analytical gradients uses :func:`~torch.allclose`.
2195
+
2196
+ .. note::
2197
+ The default values are designed for :attr:`input` and
2198
+ :attr:`grad_outputs` of double precision. This check will likely fail if
2199
+ they are of less precision, e.g., ``FloatTensor``.
2200
+
2201
+ .. warning::
2202
+ If any checked tensor in :attr:`input` and :attr:`grad_outputs` has
2203
+ overlapping memory, i.e., different indices pointing to the same memory
2204
+ address (e.g., from :func:`torch.Tensor.expand`), this check will likely fail
2205
+ because the numerical gradients computed by point perturbation at such
2206
+ indices will change values at all other indices that share the same
2207
+ memory address.
2208
+
2209
+ Args:
2210
+ func (function): a Python function that takes Tensor inputs and returns
2211
+ a Tensor or a tuple of Tensors
2212
+ inputs (tuple of Tensor or Tensor): inputs to the function
2213
+ grad_outputs (tuple of [Tensor or None] or Tensor, optional): The gradients with
2214
+ respect to the function's outputs.
2215
+ eps (float, optional): perturbation for finite differences
2216
+ atol (float, optional): absolute tolerance
2217
+ rtol (float, optional): relative tolerance
2218
+ gen_non_contig_grad_outputs (bool, optional): if :attr:`grad_outputs` is
2219
+ ``None`` and :attr:`gen_non_contig_grad_outputs` is ``True``, the
2220
+ randomly generated gradient outputs are made to be noncontiguous
2221
+ raise_exception (bool, optional): indicating whether to raise an exception if
2222
+ the check fails. The exception gives more information about the
2223
+ exact nature of the failure. This is helpful when debugging gradchecks.
2224
+ nondet_tol (float, optional): tolerance for non-determinism. When running
2225
+ identical inputs through the differentiation, the results must either match
2226
+ exactly (default, 0.0) or be within this tolerance. Note that a small amount
2227
+ of nondeterminism in the gradient will lead to larger inaccuracies in
2228
+ the second derivative.
2229
+ check_undefined_grad (bool, optional): if True, check if undefined output grads
2230
+ are supported and treated as zeros
2231
+ check_batched_grad (bool, optional): if True, check if we can compute
2232
+ batched gradients using prototype vmap support. Defaults to False.
2233
+ fast_mode (bool, optional): if True, run a faster implementation of gradgradcheck that
2234
+ no longer computes the entire jacobian.
2235
+ masked (bool, optional): if True, the gradients of unspecified elements of
2236
+ sparse tensors are ignored (default, False).
2237
+ Returns:
2238
+ True if all differences satisfy allclose condition
2239
+ """
2240
+ if not (check_fwd_over_rev or check_rev_over_rev):
2241
+ raise AssertionError(
2242
+ "Expected at least one of check_fwd_over_rev or check_rev_over_rev to be True"
2243
+ )
2244
+ if check_undefined_grad and not check_rev_over_rev:
2245
+ raise AssertionError(
2246
+ "Setting check_undefined_grad=True requires check_rev_over_rev to be True"
2247
+ )
2248
+ if check_batched_grad and not check_rev_over_rev:
2249
+ raise AssertionError(
2250
+ "Setting check_batched_grad=True requires check_rev_over_rev to be True"
2251
+ )
2252
+ # TODO: do we want to test this too?
2253
+ # assert not (check_batched_forward_grad and not check_fwd_over_rev), (
2254
+ # "Setting check_batched_forward_grad=True requires check_fwd_over_rev to be True")
2255
+ tupled_inputs = _as_tuple(inputs)
2256
+
2257
+ if grad_outputs is None:
2258
+ # If grad_outputs is not specified, create random Tensors of the same shape, type, and device as the outputs
2259
+
2260
+ outputs = _differentiable_outputs(func(*tupled_inputs))
2261
+ tupled_grad_outputs = tuple(
2262
+ torch.testing.make_tensor(
2263
+ x.shape,
2264
+ dtype=x.dtype
2265
+ if x.is_floating_point() or x.is_complex()
2266
+ else torch.double,
2267
+ device=x.device,
2268
+ low=-1,
2269
+ high=1,
2270
+ requires_grad=True,
2271
+ noncontiguous=gen_non_contig_grad_outputs,
2272
+ )
2273
+ for x in outputs
2274
+ )
2275
+ else:
2276
+ tupled_grad_outputs = _as_tuple(grad_outputs)
2277
+
2278
+ num_outputs = len(tupled_grad_outputs)
2279
+
2280
+ # NB: We need to save the requires_grad information about the inputs here because gradcheck detaches inputs
2281
+ # before running forward mode AD
2282
+ diff_input_args_indices = {
2283
+ i for i, x in enumerate(tupled_inputs) if is_tensor_like(x) and x.requires_grad
2284
+ }
2285
+ diff_grad_output_indices = {
2286
+ i for i, x in enumerate(tupled_grad_outputs) if x.requires_grad
2287
+ }
2288
+
2289
+ def new_func(*args):
2290
+ # Restore the requires_grad information
2291
+ input_args = tuple(
2292
+ x.requires_grad_() if i in diff_input_args_indices else x
2293
+ for i, x in enumerate(args[:-num_outputs])
2294
+ )
2295
+ outputs = _differentiable_outputs(func(*input_args))
2296
+ grad_outputs = tuple(
2297
+ x.requires_grad_() if i in diff_grad_output_indices else x
2298
+ for i, x in enumerate(args[-num_outputs:])
2299
+ )
2300
+ diff_input_args = tuple(
2301
+ x for i, x in enumerate(input_args) if i in diff_input_args_indices
2302
+ )
2303
+ grad_inputs = torch.autograd.grad(
2304
+ outputs, diff_input_args, grad_outputs, create_graph=True, allow_unused=True
2305
+ )
2306
+ grad_inputs = tuple(g for g in grad_inputs if g is not None)
2307
+ return grad_inputs
2308
+
2309
+ return gradcheck(
2310
+ new_func,
2311
+ tupled_inputs + tupled_grad_outputs,
2312
+ eps=eps,
2313
+ atol=atol,
2314
+ rtol=rtol,
2315
+ raise_exception=raise_exception,
2316
+ nondet_tol=nondet_tol,
2317
+ check_undefined_grad=check_undefined_grad,
2318
+ check_grad_dtypes=check_grad_dtypes,
2319
+ check_batched_grad=check_batched_grad,
2320
+ fast_mode=fast_mode,
2321
+ check_forward_ad=check_fwd_over_rev,
2322
+ check_backward_ad=check_rev_over_rev,
2323
+ masked=masked,
2324
+ )
western_ai/usa/pytorch/torch/autograd/graph.py ADDED
@@ -0,0 +1,989 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import abc
2
+ import contextlib
3
+ import contextvars
4
+ import functools
5
+ import logging
6
+ import threading
7
+ from collections import defaultdict, deque
8
+ from collections.abc import (
9
+ Callable,
10
+ Generator,
11
+ Iterable,
12
+ Iterator,
13
+ MutableMapping,
14
+ Sequence,
15
+ )
16
+ from typing import (
17
+ Any,
18
+ cast,
19
+ Literal,
20
+ NamedTuple,
21
+ Optional,
22
+ TYPE_CHECKING,
23
+ TypeAlias,
24
+ Union,
25
+ )
26
+ from weakref import WeakKeyDictionary, WeakValueDictionary
27
+
28
+ import torch
29
+ from torch.autograd.variable import Variable
30
+ from torch.utils._python_dispatch import TorchDispatchMode
31
+ from torch.utils.hooks import RemovableHandle
32
+
33
+
34
+ if TYPE_CHECKING:
35
+ from torch._ops import OpOverload
36
+
37
+
38
+ __all__ = [
39
+ "saved_tensors_hooks",
40
+ "save_on_cpu",
41
+ "disable_saved_tensors_hooks",
42
+ "register_multi_grad_hook",
43
+ "allow_mutation_on_saved_tensors",
44
+ "Node",
45
+ "GradientEdge",
46
+ "get_gradient_edge",
47
+ "increment_version",
48
+ "region_activation_memory_budget",
49
+ "set_warn_on_accumulate_grad_stream_mismatch",
50
+ "set_override_stale_capture_stream",
51
+ ]
52
+
53
+
54
+ log = logging.getLogger(__name__)
55
+
56
+
57
+ class Node(abc.ABC):
58
+ @abc.abstractmethod
59
+ def name(self) -> str:
60
+ r"""Return the name.
61
+
62
+ Example::
63
+
64
+ >>> import torch
65
+ >>> a = torch.tensor([0., 0., 0.], requires_grad=True)
66
+ >>> b = a.clone()
67
+ >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node)
68
+ >>> print(b.grad_fn.name())
69
+ CloneBackward0
70
+ """
71
+ raise NotImplementedError
72
+
73
+ @property
74
+ @abc.abstractmethod
75
+ def next_functions(self) -> tuple[tuple[Optional["Node"], int], ...]:
76
+ raise NotImplementedError
77
+
78
+ @abc.abstractmethod
79
+ def metadata(self) -> dict:
80
+ r"""Return the metadata."""
81
+ raise NotImplementedError
82
+
83
+ @abc.abstractmethod
84
+ def _sequence_nr(self) -> int:
85
+ raise NotImplementedError
86
+
87
+ @property
88
+ @abc.abstractmethod
89
+ def _input_metadata(self) -> list[Any]:
90
+ raise NotImplementedError
91
+
92
+ @abc.abstractmethod
93
+ def _register_hook_dict(self, tensor: torch.Tensor) -> None:
94
+ raise NotImplementedError
95
+
96
+ @abc.abstractmethod
97
+ def register_hook(self, fn: Callable[..., Any]) -> RemovableHandle:
98
+ r"""Register a backward hook.
99
+
100
+ The hook will be called every time a gradient with respect to the
101
+ Node is computed. The hook should have the following signature::
102
+
103
+ hook(grad_inputs: Tuple[Tensor], grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None
104
+
105
+
106
+ The hook should not modify its argument, but it can optionally return
107
+ a new gradient which will be used in place of :attr:`grad_inputs`.
108
+
109
+ This function returns a handle with a method ``handle.remove()``
110
+ that removes the hook from the module.
111
+
112
+ .. note::
113
+ See :ref:`backward-hooks-execution` for more information on how when this hook
114
+ is executed, and how its execution is ordered relative to other hooks.
115
+
116
+ .. note::
117
+ In the rare case where the hook is registered while the Node has already
118
+ begun execution, there is no longer any guarantee on :attr:`grad_outputs`
119
+ content (it might be as usual or empty depending on other factors). The
120
+ hook can still optionally return a new gradient to be used in place of
121
+ :attr:`grad_inputs` independent of :attr:`grad_outputs`.
122
+
123
+ Example::
124
+
125
+ >>> import torch
126
+ >>> a = torch.tensor([0., 0., 0.], requires_grad=True)
127
+ >>> b = a.clone()
128
+ >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node)
129
+ >>> handle = b.grad_fn.register_hook(lambda gI, gO: (gO[0] * 2,))
130
+ >>> b.sum().backward(retain_graph=True)
131
+ >>> print(a.grad)
132
+ tensor([2., 2., 2.])
133
+ >>> handle.remove() # Removes the hook
134
+ >>> a.grad = None
135
+ >>> b.sum().backward(retain_graph=True)
136
+ >>> print(a.grad)
137
+ tensor([1., 1., 1.])
138
+ """
139
+ raise NotImplementedError
140
+
141
+ @abc.abstractmethod
142
+ def register_prehook(self, fn: Callable[..., Any]) -> RemovableHandle:
143
+ r"""Register a backward pre-hook.
144
+
145
+ The hook will be called every time a gradient with respect to the
146
+ Node is computed. The hook should have the following signature::
147
+
148
+ hook(grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None
149
+
150
+ The hook should not modify its argument, but it can optionally return
151
+ a new gradient which will be used in place of :attr:`grad_outputs`.
152
+
153
+ This function returns a handle with a method ``handle.remove()``
154
+ that removes the hook from the module.
155
+
156
+ .. note::
157
+ See :ref:`backward-hooks-execution` for more information on how when this hook
158
+ is executed, and how its execution is ordered relative to other hooks.
159
+
160
+ Example::
161
+
162
+ >>> a = torch.tensor([0., 0., 0.], requires_grad=True)
163
+ >>> b = a.clone()
164
+ >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node)
165
+ >>> handle = b.grad_fn.register_prehook(lambda gI: (gI[0] * 2,))
166
+ >>> b.sum().backward(retain_graph=True)
167
+ >>> print(a.grad)
168
+ tensor([2., 2., 2.])
169
+ >>> handle.remove()
170
+ >>> a.grad = None
171
+ >>> b.sum().backward(retain_graph=True)
172
+ >>> print(a.grad)
173
+ tensor([1., 1., 1.])
174
+ """
175
+ raise NotImplementedError
176
+
177
+ @classmethod
178
+ def __subclasshook__(cls, subclass: type) -> bool:
179
+ if cls is Node and (
180
+ (
181
+ subclass is not None
182
+ and subclass is getattr(torch._C._functions, subclass.__name__, None)
183
+ )
184
+ or issubclass(subclass, torch.autograd.function.BackwardCFunction)
185
+ ):
186
+ return True
187
+ return NotImplemented
188
+
189
+
190
+ def _get_grad_fn_or_grad_acc(t: Union[torch.Tensor, "GradientEdge"]) -> Node:
191
+ if isinstance(t, GradientEdge):
192
+ return t.node
193
+ if t.requires_grad and t.grad_fn is None:
194
+ with torch.enable_grad():
195
+ node = t.view_as(t).grad_fn.next_functions[0][0] # type: ignore[union-attr]
196
+ else:
197
+ node = t.grad_fn
198
+ if node is None:
199
+ raise AssertionError("Expected gradient function to be set")
200
+ return node
201
+
202
+
203
+ class GradientEdge(NamedTuple):
204
+ """Object representing a given gradient edge within the autograd graph.
205
+
206
+ To get the gradient edge where a given Tensor gradient will be computed,
207
+ you can do ``edge = autograd.graph.get_gradient_edge(tensor)``.
208
+ """
209
+
210
+ node: Node
211
+ output_nr: int
212
+ # This token can be used to ensure the graph stays alive when it cannot be
213
+ # done via the node field
214
+ ownership_token: Node | None = None
215
+
216
+
217
+ def get_gradient_edge(tensor: torch.Tensor) -> GradientEdge:
218
+ """Get the gradient edge for computing the gradient of the given Tensor.
219
+
220
+ In particular, it is equivalent to call
221
+ ``g = autograd.grad(loss, input)`` and ``g = autograd.grad(loss, get_gradient_edge(input))``.
222
+ """
223
+ if not tensor.requires_grad:
224
+ raise RuntimeError(
225
+ "It is not possible to get the gradient edge for a Tensor "
226
+ "that does not require gradients",
227
+ )
228
+ grad_fn = _get_grad_fn_or_grad_acc(tensor)
229
+
230
+ # Python-based Node are owned by the C++ side meaning the python grad_fn
231
+ # object we hold here does NOT keep the C++ graph alive.
232
+ # Create an ownership token by creating a new C++ node that own the graph
233
+ # we care about here.
234
+ token = None
235
+ if isinstance(grad_fn, torch._C._FunctionBase):
236
+ with torch.enable_grad():
237
+ token = tensor.view_as(tensor).grad_fn
238
+
239
+ # Note that output_nr default to 0 which is the right value
240
+ # for the AccumulateGrad node.
241
+ # pyrefly: ignore [bad-argument-type]
242
+ return GradientEdge(grad_fn, tensor.output_nr, ownership_token=token)
243
+
244
+
245
+ def increment_version(tensor: torch.Tensor | Iterable[torch.Tensor]) -> None:
246
+ """Update autograd metadata tracking whether the given Tensor was modified in place.
247
+
248
+ This is to enable more accurate error checking within the autograd engine.
249
+ It is already done automatically by PyTorch functions and within custom Function
250
+ when mark_dirty() is called appropriately so you only need to call this explicitly
251
+ if you are doing inplace operation on the Tensor data in a way that Pytorch doesn't
252
+ know about. For example a custom kernel that reads the Tensor data_ptr and modifies
253
+ the memory inplace based on this pointer. Can accept either a tensor, or a list of tensors.
254
+
255
+ Note that incrementing the version counter multiple times for a single inplace operation
256
+ is not problematic.
257
+
258
+ Note that if you pass in tensor constructed under torch.inference_mode(),
259
+ we will not bump its version counter (because your tensor does not have one).
260
+ """
261
+ if isinstance(tensor, torch.Tensor):
262
+ tensor = (tensor,)
263
+ torch._C._increment_version(tensor)
264
+
265
+
266
+ class saved_tensors_hooks:
267
+ """Context-manager that sets a pair of pack / unpack hooks for saved tensors.
268
+
269
+ Use this context-manager to define how intermediary results of an operation
270
+ should be packed before saving, and unpacked on retrieval.
271
+
272
+ In that context, the ``pack_hook`` function will be called every time an
273
+ operation saves a tensor for backward (this includes intermediary results
274
+ saved using
275
+ :func:`~torch.autograd.function._ContextMethodMixin.save_for_backward` but
276
+ also those recorded by a PyTorch-defined operation). The output of
277
+ ``pack_hook`` is then stored in the computation graph instead of the
278
+ original tensor.
279
+
280
+ The ``unpack_hook`` is called when the saved tensor needs to be accessed,
281
+ namely when executing :func:`torch.Tensor.backward()` or
282
+ :func:`torch.autograd.grad()`. It takes as argument the *packed* object
283
+ returned by ``pack_hook`` and should return a tensor which has the same
284
+ content as the original tensor (passed as input to the corresponding
285
+ ``pack_hook``).
286
+
287
+ The hooks should have the following signatures:
288
+
289
+ pack_hook(tensor: Tensor) -> Any
290
+
291
+ unpack_hook(Any) -> Tensor
292
+
293
+ where the return value of ``pack_hook`` is a valid input to ``unpack_hook``.
294
+
295
+ In general, you want ``unpack_hook(pack_hook(t))`` to be equal to ``t`` in terms
296
+ of value, size, dtype and device.
297
+
298
+ Example::
299
+
300
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
301
+ >>> def pack_hook(x):
302
+ ... print("Packing", x)
303
+ ... return x.detach()
304
+ >>>
305
+ >>> def unpack_hook(x):
306
+ ... print("Unpacking", x)
307
+ ... return x
308
+ >>>
309
+ >>> a = torch.ones(5, requires_grad=True)
310
+ >>> b = torch.ones(5, requires_grad=True) * 2
311
+ >>> with torch.autograd.graph.saved_tensors_hooks(pack_hook, unpack_hook):
312
+ ... y = a * b
313
+ Packing tensor([1., 1., 1., 1., 1.], requires_grad=True)
314
+ Packing tensor([2., 2., 2., 2., 2.], grad_fn=<MulBackward0>)
315
+ >>> y.sum().backward()
316
+ Unpacking tensor([1., 1., 1., 1., 1.], requires_grad=True)
317
+ Unpacking tensor([2., 2., 2., 2., 2.], grad_fn=<MulBackward0>)
318
+
319
+ .. warning ::
320
+ Performing an inplace operation on the input to either hooks may lead
321
+ to undefined behavior.
322
+
323
+ .. warning ::
324
+ Only one pair of hooks is allowed at a time. When recursively nesting this
325
+ context-manager, only the inner-most pair of hooks will be applied.
326
+
327
+ .. warning ::
328
+ To avoid reference cycle, the return value of ``pack_hook`` cannot hold a
329
+ reference to the input tensor. For example, use `lambda x: x.detach()`
330
+ instead of `lambda x: x` as the pack hook.
331
+ """
332
+
333
+ def __init__(
334
+ self,
335
+ pack_hook: Callable[[torch.Tensor], Any],
336
+ unpack_hook: Callable[[Any], torch.Tensor],
337
+ ) -> None:
338
+ self.pack_hook = pack_hook
339
+ self.unpack_hook = unpack_hook
340
+
341
+ def __enter__(self) -> None:
342
+ torch._C._autograd._push_saved_tensors_default_hooks(
343
+ self.pack_hook, self.unpack_hook
344
+ )
345
+
346
+ def __exit__(self, *args: object) -> None:
347
+ torch._C._autograd._pop_saved_tensors_default_hooks()
348
+
349
+
350
+ class save_on_cpu(saved_tensors_hooks):
351
+ """Context manager under which tensors saved by the forward pass will be stored on cpu, then retrieved for backward.
352
+
353
+ When performing operations within this context manager, intermediary
354
+ results saved in the graph during the forward pass will be moved to CPU,
355
+ then copied back to the original device when needed for the backward pass.
356
+ If the graph was already on CPU, no tensor copy is performed.
357
+
358
+ Use this context-manager to trade compute for GPU memory usage (e.g.
359
+ when your model doesn't fit in GPU memory during training).
360
+
361
+ .. warning::
362
+
363
+ When ``pin_memory=True``, the GPU to CPU copy during packing is
364
+ asynchronous. Accessing saved tensors on CPU (e.g. via
365
+ ``grad_fn._saved_self``) before the CUDA stream has finished may
366
+ yield incorrect data. Call :func:`torch.cuda.synchronize` first
367
+ if you need to read them.
368
+
369
+ Args:
370
+ pin_memory (bool): If ``True`` tensors will be saved to CPU pinned memory
371
+ during packing and copied to GPU asynchronously during both
372
+ packing and unpacking.
373
+ Defaults to ``False``.
374
+ Also see :ref:`cuda-memory-pinning`.
375
+
376
+
377
+ Example::
378
+
379
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA)
380
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD)
381
+ >>> a = torch.randn(5, requires_grad=True, device="cuda")
382
+ >>> b = torch.randn(5, requires_grad=True, device="cuda")
383
+ >>> c = torch.randn(5, requires_grad=True, device="cuda")
384
+ >>>
385
+ >>> def f(a, b, c):
386
+ ... prod_1 = a * b # a and b are saved on GPU
387
+ ... with torch.autograd.graph.save_on_cpu():
388
+ ... prod_2 = prod_1 * c # prod_1 and c are saved on CPU
389
+ ... y = prod_2 * a # prod_2 and a are saved on GPU
390
+ ... return y
391
+ >>>
392
+ >>> y = f(a, b, c)
393
+ >>> del a, b, c # for illustration only
394
+ >>> # the content of a, b, and prod_2 are still alive on GPU
395
+ >>> # the content of prod_1 and c only live on CPU
396
+ >>> y.sum().backward() # all CPU tensors are moved back to GPU, for backward
397
+ >>> # all intermediary tensors are released (deleted) after the call to backward
398
+ """
399
+
400
+ def __init__(self, pin_memory: bool = False, device_type: str = "cuda") -> None:
401
+ device_module = getattr(torch, device_type, torch.cuda)
402
+
403
+ def pack_to_cpu(tensor: torch.Tensor) -> tuple[torch.device, torch.Tensor]:
404
+ if not pin_memory:
405
+ return (tensor.device, tensor.cpu())
406
+ is_pinnable = device_module.is_available() and not tensor.is_sparse
407
+ packed = torch.empty(
408
+ tensor.size(),
409
+ dtype=tensor.dtype,
410
+ layout=tensor.layout,
411
+ pin_memory=is_pinnable,
412
+ )
413
+ packed.copy_(tensor, non_blocking=is_pinnable)
414
+ return (tensor.device, packed)
415
+
416
+ def unpack_from_cpu(packed: tuple[torch.device, torch.Tensor]) -> torch.Tensor:
417
+ device, tensor = packed
418
+ return tensor.to(device, non_blocking=pin_memory)
419
+
420
+ super().__init__(pack_to_cpu, unpack_from_cpu)
421
+
422
+
423
+ @contextlib.contextmanager
424
+ def disable_saved_tensors_hooks(error_message: str) -> Generator[None, None, None]:
425
+ """Context-manager that disables the saved tensors default hooks feature.
426
+
427
+ Useful for if you are creating a feature that does not work with saved
428
+ tensors default hooks.
429
+
430
+ Args:
431
+ error_message (str): When saved tensors default hooks are used when they
432
+ have been disabled, a RuntimeError with this
433
+ error message gets raised.
434
+
435
+ Example::
436
+
437
+ >>> # xdoctest: +SKIP(failing)
438
+ >>> message = "saved tensors default hooks are disabled"
439
+ >>> with torch.autograd.graph.disable_saved_tensors_hooks(message):
440
+ ... # Raises RuntimeError: saved tensors default hooks are disabled
441
+ ... with torch.autograd.graph.save_on_cpu():
442
+ ... pass
443
+ """
444
+ maybe_prev_message = None
445
+ try:
446
+ maybe_prev_message = (
447
+ torch._C._autograd._saved_tensors_hooks_get_disabled_error_message()
448
+ )
449
+ torch._C._autograd._saved_tensors_hooks_disable(error_message)
450
+ yield
451
+ finally:
452
+ # See NOTE: [disabled_error_message invariant]
453
+ if maybe_prev_message is None:
454
+ torch._C._autograd._saved_tensors_hooks_enable()
455
+ else:
456
+ torch._C._autograd._saved_tensors_hooks_disable(maybe_prev_message)
457
+
458
+
459
+ def region_activation_memory_budget(
460
+ budget: float,
461
+ ) -> contextlib.AbstractContextManager[None]:
462
+ r"""Context-manager that sets the activation memory budget for the region of
463
+ a compiled forward traced under it.
464
+
465
+ .. warning::
466
+ This is a prototype feature and is subject to change.
467
+
468
+ Under :func:`torch.compile`, the min-cut partitioner chooses which
469
+ activations to save versus recompute in the backward pass to stay under a
470
+ memory budget. ``budget`` is a ratio in ``[0, 1]``: ``0.0`` corresponds to
471
+ the activation memory from applying activation checkpointing to the full
472
+ region, and ``1.0`` corresponds to the activation memory from the default
473
+ runtime-optimized strategy. So ``0.4`` would result in a strategy that saves
474
+ 40% of the activations compared to the default strategy. It solves a 0-1
475
+ knapsack to find the minimum recompute necessary to stay below the budget.
476
+ This overrides the global ``torch._functorch.config.activation_memory_budget``
477
+ for the annotated region.
478
+
479
+ .. note::
480
+ Today the partitioner only supports a single budget per compiled graph,
481
+ so the annotation must cover every forward op in the graph (a partial
482
+ annotation is rejected rather than silently applied graph-wide), and all
483
+ annotated nodes must agree on the budget. To use different budgets for
484
+ different parts of a model, separate them with a graph break (e.g.
485
+ ``torch._dynamo.graph_break()``) so each part becomes its own graph.
486
+
487
+ This only has an effect under :func:`torch.compile`; using it outside of a
488
+ compiled region raises a ``RuntimeError``.
489
+
490
+ Args:
491
+ budget (float): Activation memory budget ratio in ``[0, 1]``.
492
+
493
+ Example::
494
+
495
+ >>> # xdoctest: +SKIP
496
+ >>> with torch.autograd.graph.region_activation_memory_budget(0.0):
497
+ ... x = layer(x) # recompute this region's activations in backward
498
+ """
499
+ import torch.fx.traceback as fx_traceback
500
+
501
+ if isinstance(budget, bool) or not isinstance(budget, (int, float)):
502
+ raise TypeError(
503
+ f"torch.autograd.graph.region_activation_memory_budget: expects a "
504
+ f"float, got {type(budget).__name__}"
505
+ )
506
+ if not 0.0 <= budget <= 1.0:
507
+ raise ValueError(
508
+ f"torch.autograd.graph.region_activation_memory_budget: must be in "
509
+ f"[0, 1], got {budget}"
510
+ )
511
+ # The budget only takes effect when read back by the partitioner during
512
+ # compilation. Dynamo folds torch.compiler.is_compiling() to True while
513
+ # tracing this (inlined) call, so this guard only fires in eager mode.
514
+ if not torch.compiler.is_compiling():
515
+ raise RuntimeError(
516
+ "torch.autograd.graph.region_activation_memory_budget can only be "
517
+ "used inside a torch.compile region; it has no effect in eager mode."
518
+ )
519
+ return fx_traceback.annotate(
520
+ {fx_traceback.MEMORY_BUDGET_ANNOTATION_KEY: float(budget)}
521
+ )
522
+
523
+
524
+ def set_warn_on_accumulate_grad_stream_mismatch(enabled: bool) -> None:
525
+ """Whether to warn when the AccumulateGrad node's stream does not match the stream
526
+ of the node that produced the incoming gradient.
527
+ """
528
+ return torch._C._set_warn_on_accumulate_grad_stream_mismatch(enabled)
529
+
530
+
531
+ def set_override_stale_capture_stream(enabled: bool) -> None:
532
+ """Control behavior when autograd detects a stale non-capturing stream during
533
+ CUDA graph capture.
534
+
535
+ During CUDA graph capture, autograd nodes may reference a stale stream
536
+ that is not part of the capture. With the flag disabled (the
537
+ process-initial state), autograd raises a ``RuntimeError`` when the stale
538
+ stream is the default stream (stream 0), because this case always
539
+ invalidates the capture: ``cudaStreamWaitEvent`` on the default stream
540
+ pulls a non-capturing stream into the graph. For non-default stale streams
541
+ the stream reference is left unchanged; the capture will succeed if the
542
+ user has joined the stream into the capture (e.g. via
543
+ ``capture_stream.wait_stream(stale_stream)``) and will otherwise fail with
544
+ a CUDA runtime error.
545
+
546
+ When ``enabled=True``, any stale non-capturing stream (default or
547
+ non-default) is automatically overridden with the producer's capturing
548
+ stream, allowing the capture to proceed. This is a process-global setting
549
+ and is not thread-local.
550
+
551
+ Args:
552
+ enabled (bool): If ``True``, override stale non-capturing streams with
553
+ the producer's capturing stream during CUDA graph capture. If
554
+ ``False`` (the process-initial state), raise an error only when the
555
+ stale stream is the default stream (stream 0); other stale streams
556
+ are left unchanged.
557
+ """
558
+ return torch._C._set_override_stale_capture_stream(enabled)
559
+
560
+
561
+ class _MultiHandle(RemovableHandle):
562
+ handles: tuple[RemovableHandle, ...]
563
+
564
+ def __init__(self, handles: tuple[RemovableHandle, ...]) -> None:
565
+ self.handles = handles
566
+
567
+ def remove(self) -> None:
568
+ for handle in self.handles:
569
+ handle.remove()
570
+
571
+ def __getstate__(self) -> tuple[RemovableHandle, ...]:
572
+ return self.handles
573
+
574
+ def __setstate__(self, state: tuple[RemovableHandle, ...]) -> None:
575
+ self.handles = state
576
+
577
+
578
+ def register_multi_grad_hook(
579
+ tensors: Sequence[torch.Tensor],
580
+ fn: Callable[[Sequence[torch.Tensor | None]], None]
581
+ | Callable[[torch.Tensor], None],
582
+ *,
583
+ mode: Literal["all", "any"] = "all",
584
+ ) -> RemovableHandle:
585
+ r"""Register a multi-grad backward hook.
586
+
587
+ There are two supported modes: ``"all"`` and ``"any"``.
588
+
589
+ Under the ``"all"`` mode, the hook will be called after gradients with respect to every tensor in
590
+ :attr:`tensors` have been computed. If a tensor is in :attr:`tensors` but
591
+ is not part of the graph, or if a tensor is not needed to compute the gradients
592
+ for any ``inputs`` specified for the current ``.backward()`` or ``.grad()`` call,
593
+ this tensor will be ignored and the hook will not wait for its gradient to be
594
+ computed.
595
+
596
+ After every non-ignored tensor's gradient has been computed, :attr:`fn` will be
597
+ called with those gradients. ``None`` will be passed for tensors that did not
598
+ have their gradients computed.
599
+
600
+ Under the ``"any"`` mode, the hook will be called after the first gradient
601
+ with respect to a tensor in :attr:`tensors` has been computed. The hook
602
+ will be called with that gradient as its argument.
603
+
604
+ The hook should not modify its arguments.
605
+
606
+ This function returns a handle with a method ``handle.remove()`` that removes the hook.
607
+
608
+ .. note::
609
+ See :ref:`backward-hooks-execution` for more information on how when this hook
610
+ is executed, and how its execution is ordered relative to other hooks.
611
+
612
+ Example::
613
+
614
+ >>> import torch
615
+ >>>
616
+ >>> a = torch.rand(2, 3, requires_grad=True)
617
+ >>> b = torch.rand(2, 3, requires_grad=True)
618
+ >>> c = a * b
619
+ >>> d = a * b
620
+ >>>
621
+ >>> def fn(grads):
622
+ ... print([g is not None for g in grads])
623
+ ...
624
+ >>> torch.autograd.graph.register_multi_grad_hook((a, b, c, d), fn)
625
+ >>>
626
+ >>> c.sum().backward(retain_graph=True)
627
+ [True, True, True, False]
628
+ >>> c.sum().backward(inputs=(a,), retain_graph=True)
629
+ [True, False, True, False]
630
+ >>>
631
+ """
632
+ supported_modes = ("all", "any")
633
+ lock = threading.Lock()
634
+
635
+ if mode not in supported_modes:
636
+ raise ValueError(f"Expects mode to be one of {supported_modes} but got {mode}")
637
+
638
+ if mode == "all":
639
+ count: dict[int, int] = {}
640
+ nb_calls = None
641
+ buffer: dict[int, list[torch.Tensor | None]] = {}
642
+
643
+ grad_fns = list(map(_get_grad_fn_or_grad_acc, tensors))
644
+ len_tensors = len(tensors)
645
+
646
+ def get_inner_hook(idx: int) -> Callable[[torch.Tensor], None]:
647
+ def inner_hook(grad: torch.Tensor) -> None:
648
+ nonlocal count, nb_calls, buffer, fn
649
+ id = torch._C._current_graph_task_id()
650
+ if id == -1:
651
+ raise AssertionError(
652
+ "expected this hook to be called inside a backward call"
653
+ )
654
+ count[id] = count.get(id, 0)
655
+ # pyrefly: ignore [unsupported-operation]
656
+ buffer[id] = buffer.get(id, [None] * len_tensors)
657
+
658
+ with lock:
659
+ curr_count, count[id] = count[id], count[id] + 1
660
+
661
+ if curr_count == 0:
662
+ # On the first call, compute the actual nb_calls and buffer
663
+ nb_calls = sum(
664
+ map(torch._C._will_engine_execute_node, grad_fns)
665
+ )
666
+
667
+ buffer[id][idx] = grad
668
+
669
+ if nb_calls is None:
670
+ raise AssertionError("Expected nb_calls to be set")
671
+ if curr_count == nb_calls - 1:
672
+ fn = cast(Callable[[Sequence[torch.Tensor | None]], None], fn)
673
+ fn(buffer[id])
674
+ del count[id]
675
+ del buffer[id]
676
+
677
+ return inner_hook
678
+
679
+ handles = tuple(
680
+ t.register_hook(get_inner_hook(i)) for i, t in enumerate(tensors)
681
+ )
682
+ elif mode == "any":
683
+ fn = cast(Callable[[torch.Tensor], None], fn)
684
+ ran_hook: dict[int, bool] = defaultdict(bool)
685
+
686
+ @functools.wraps(fn)
687
+ def wrapped_fn(grad: torch.Tensor) -> None:
688
+ nonlocal ran_hook
689
+ id = torch._C._current_graph_task_id()
690
+ if id == -1:
691
+ raise AssertionError(
692
+ "expected this hook to be called inside a backward call"
693
+ )
694
+ with lock:
695
+ prev, ran_hook[id] = ran_hook[id], True
696
+ if prev:
697
+ return
698
+ fn(grad)
699
+
700
+ handles = tuple(
701
+ tensor.register_hook(wrapped_fn)
702
+ for tensor in tensors
703
+ if tensor.requires_grad
704
+ )
705
+
706
+ return _MultiHandle(handles) # type: ignore[possibly-undefined]
707
+
708
+
709
+ # NOTE [Allow mutation on tensors saved for backward]
710
+ #
711
+ # 1. Tensor gets saved for backward
712
+ # - remember the python object id and the version of the tensor
713
+ # - remember aliasing information (data_ptr of base + version)
714
+ # - save the original so we control its lifetime
715
+ # 2. Any time a tensor gets in-placed
716
+ # - for each tensor aliased to it:
717
+ # - check using its object id and version to see if it has been saved
718
+ # - if it has been saved, clone it
719
+ # - delete the reference to the original
720
+ # 3. during backward
721
+ # - if the clone exists, the tensor must've been modified in-place
722
+ _allow_mutation_on_saved_tensors_enabled: bool = False
723
+
724
+
725
+ _TID: TypeAlias = tuple[int, int, int]
726
+ _SID: TypeAlias = tuple[int, int]
727
+
728
+
729
+ def _get_tid(tensor: torch.Tensor) -> _TID:
730
+ # FIXME: This is almost definitely a bug.
731
+ if isinstance(
732
+ tensor,
733
+ (
734
+ torch._subclasses.fake_tensor.FakeTensor,
735
+ torch._subclasses.functional_tensor.FunctionalTensor,
736
+ ),
737
+ ):
738
+ data_ptr = 0
739
+ else:
740
+ data_ptr = tensor.data_ptr()
741
+ return (id(tensor), data_ptr, tensor._version)
742
+
743
+
744
+ def _get_sid(tensor: torch.Tensor) -> _SID:
745
+ # FIXME: This is almost definitely a bug.
746
+ if isinstance(
747
+ tensor,
748
+ (
749
+ torch._subclasses.fake_tensor.FakeTensor,
750
+ torch._subclasses.functional_tensor.FunctionalTensor,
751
+ ),
752
+ ):
753
+ data_ptr = 0
754
+ else:
755
+ data_ptr = tensor.data_ptr()
756
+ return (data_ptr, tensor._version)
757
+
758
+
759
+ class _Handle:
760
+ pass
761
+
762
+
763
+ class _swap_with_cloned(saved_tensors_hooks):
764
+ def __init__(self, ctx: "_AllowMutationOnSavedContext") -> None:
765
+ def pack_hook(tensor: torch.Tensor) -> _Handle:
766
+ tid = _get_tid(tensor)
767
+ sid = _get_sid(tensor)
768
+ # Tensors saved for backward have an entry in _tid_to_weakhandle
769
+ handle: _Handle | None = None
770
+
771
+ # Save aliasing information
772
+ ctx.sid_to_tid[sid].add(tid)
773
+
774
+ # NB: The same tensor (of the same version) can be saved multiple times
775
+ if tid not in ctx.tid_to_weakhandle:
776
+ handle = _Handle()
777
+ ctx.tid_to_weakhandle[tid] = handle
778
+ ctx.original[handle] = tensor
779
+ else:
780
+ # Store an additional strong reference to the handle
781
+ handle = ctx.tid_to_weakhandle[tid]
782
+ return handle
783
+
784
+ def unpack_hook(handle: _Handle) -> torch.Tensor:
785
+ error_msg = (
786
+ "Trying to backward outside of the 'allow_mutation_on_saved_tensors' context"
787
+ "in which the graph was originally recorded."
788
+ )
789
+ if not _allow_mutation_on_saved_tensors_enabled:
790
+ raise AssertionError(error_msg)
791
+ if handle in ctx.cloned:
792
+ res = ctx.cloned[handle]
793
+ else:
794
+ if handle not in ctx.original:
795
+ raise AssertionError(error_msg)
796
+ res = ctx.original[handle]
797
+ return res
798
+
799
+ super().__init__(pack_hook, unpack_hook)
800
+
801
+
802
+ class _CloneArgBeforeMutateMode(TorchDispatchMode):
803
+ def __init__(self, ctx: "_AllowMutationOnSavedContext") -> None:
804
+ self.ctx = ctx
805
+
806
+ def __torch_dispatch__(
807
+ self,
808
+ func: "OpOverload",
809
+ types: Iterable[type],
810
+ args: tuple[Any, ...] = (),
811
+ kwargs: dict[Any, Any] | None = None,
812
+ ) -> Any:
813
+ kwargs = kwargs or {}
814
+
815
+ def maybe_clone(t: torch.Tensor) -> None:
816
+ tid = _get_tid(t)
817
+ sid = _get_sid(t)
818
+ ctx = self.ctx
819
+ if sid in ctx.sid_to_tid:
820
+ for tid in ctx.sid_to_tid[sid]:
821
+ if tid not in ctx.tid_to_weakhandle:
822
+ # We know that if tid is in sid_to_tid, then it must also be in
823
+ # tid_to_weakhandle. However, it is possible for the tensor to be
824
+ # saved at one point, but cleared by backward before it is modified
825
+ # in-place. Consider the following example:
826
+ #
827
+ # >>> a = torch.randn(2, 3, requires_grad=True).clone()
828
+ # >>> out = (a**2).sum()
829
+ # >>> out.backward()
830
+ # >>> a.sin_()
831
+ continue
832
+ handle = ctx.tid_to_weakhandle[tid]
833
+ if handle in ctx.cloned:
834
+ # The same exact tensor has been cloned already
835
+ continue
836
+ ctx.cloned[handle] = ctx.original[handle].clone()
837
+ del ctx.original[handle]
838
+
839
+ for idx, arg in enumerate(func._schema.arguments):
840
+ if arg.alias_info is not None and arg.alias_info.is_write:
841
+ if arg.is_out:
842
+ maybe_clone(kwargs["out"])
843
+ elif isinstance(args[idx], list):
844
+ # Foreach case. (Possible optimization: if most of the
845
+ # tensors need to be cloned, use a for each clone?)
846
+ for t in args[idx]:
847
+ maybe_clone(t)
848
+ else:
849
+ maybe_clone(args[idx])
850
+
851
+ return func(*args, **kwargs)
852
+
853
+
854
+ class _AllowMutationOnSavedContext:
855
+ def __init__(self) -> None:
856
+ self.cloned: MutableMapping[_Handle, torch.Tensor] = WeakKeyDictionary()
857
+ self.original: MutableMapping[_Handle, torch.Tensor] = WeakKeyDictionary()
858
+ self.tid_to_weakhandle: MutableMapping[_TID, _Handle] = WeakValueDictionary()
859
+ self.sid_to_tid: dict[_SID, set[_TID]] = defaultdict(set)
860
+
861
+ def clear(self) -> None:
862
+ self.cloned.clear()
863
+ self.original.clear()
864
+ self.tid_to_weakhandle.clear()
865
+ self.sid_to_tid.clear()
866
+
867
+
868
+ @contextlib.contextmanager
869
+ def allow_mutation_on_saved_tensors() -> Generator[
870
+ _AllowMutationOnSavedContext, None, None
871
+ ]:
872
+ """Context manager under which mutating tensors saved for backward is allowed.
873
+
874
+ Under this context manager, tensors saved for backward are cloned on mutation,
875
+ so the original version can still be used during backward. Normally, mutating a tensor
876
+ saved for backward will result in an error raised when it's used during backward.
877
+
878
+ To ensure the correct behavior, both the forward and backward should be run under
879
+ the same context manager.
880
+
881
+ Returns:
882
+ An _AllowMutationOnSavedContext object storing the state managed by this
883
+ context manager. This object can be useful for debugging purposes. The state
884
+ managed by the context manager is automatically cleared upon exiting.
885
+
886
+ Example::
887
+
888
+ >>> import torch
889
+ >>> with torch.autograd.graph.allow_mutation_on_saved_tensors():
890
+ ... # forward
891
+ ... a = torch.ones(2, 3, requires_grad=True)
892
+ ... b = a.clone()
893
+ ... out = (b**2).sum()
894
+ ... b.sin_()
895
+ ... # backward
896
+ ... out.sum().backward()
897
+ ...
898
+ tensor([[0.8415, 0.8415, 0.8415],
899
+ [0.8415, 0.8415, 0.8415]], grad_fn=<SinBackward0>)
900
+ """
901
+ global _allow_mutation_on_saved_tensors_enabled
902
+
903
+ ctx = _AllowMutationOnSavedContext()
904
+
905
+ with _swap_with_cloned(ctx), _CloneArgBeforeMutateMode(ctx):
906
+ try:
907
+ if _allow_mutation_on_saved_tensors_enabled:
908
+ raise RuntimeError(
909
+ "allow_mutation_on_saved_tensors contexts cannot be nested"
910
+ )
911
+ _allow_mutation_on_saved_tensors_enabled = True
912
+ yield ctx
913
+ finally:
914
+ ctx.clear()
915
+ _allow_mutation_on_saved_tensors_enabled = False
916
+
917
+
918
+ def _register_logging_hooks_on_whole_graph(
919
+ t_outputs: Sequence[torch.Tensor | GradientEdge],
920
+ ) -> Callable[[], None]:
921
+ grad_fns = list(map(_get_grad_fn_or_grad_acc, t_outputs))
922
+
923
+ def iter_graph(roots: list[Node]) -> Iterator[Node]:
924
+ if not roots:
925
+ return
926
+ seen: set[Node] = set()
927
+ q: deque[Node] = deque()
928
+ for node in roots:
929
+ if node is not None:
930
+ seen.add(node)
931
+ q.append(node)
932
+
933
+ while q:
934
+ node = q.popleft()
935
+ for fn, _ in node.next_functions:
936
+ if fn in seen or fn is None:
937
+ continue
938
+ seen.add(fn)
939
+ q.append(fn)
940
+
941
+ yield node
942
+
943
+ def fmt(t: torch.Tensor | None) -> str:
944
+ # Avoid circular import
945
+ from torch.utils._dtype_abbrs import dtype_abbrs
946
+
947
+ if t is None:
948
+ return "None"
949
+ return f"{dtype_abbrs[t.dtype]}[{', '.join(map(str, t.shape))}]"
950
+
951
+ def prehook(grad_outputs: Sequence[torch.Tensor | None]) -> None:
952
+ node = torch._C._current_autograd_node()
953
+ grad_outputs_str = f"[{','.join(fmt(t) for t in grad_outputs)}]"
954
+ log_str = f"Executing: {node} with grad_outputs: {grad_outputs_str}"
955
+ log.debug(log_str)
956
+
957
+ handles = [node.register_prehook(prehook) for node in iter_graph(grad_fns)]
958
+
959
+ def unregister_hooks() -> None:
960
+ for handle in handles:
961
+ handle.remove()
962
+
963
+ return unregister_hooks
964
+
965
+
966
+ def _engine_run_backward(
967
+ t_outputs: Sequence[torch.Tensor | GradientEdge],
968
+ *args: Any,
969
+ **kwargs: Any,
970
+ ) -> tuple[torch.Tensor, ...]:
971
+ attach_logging_hooks = log.getEffectiveLevel() <= logging.DEBUG
972
+ if attach_logging_hooks:
973
+ unregister_hooks = _register_logging_hooks_on_whole_graph(t_outputs)
974
+
975
+ # Need to save the context so compiler config will be visible in device threads
976
+ torch._C._stash_obj_in_tls("context", contextvars.copy_context())
977
+
978
+ try:
979
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
980
+ t_outputs, *args, **kwargs
981
+ ) # Calls into the C++ engine to run the backward pass
982
+ finally:
983
+ if attach_logging_hooks:
984
+ unregister_hooks() # type: ignore[possibly-undefined]
985
+ # Erase rather than overwrite-with-None so the thread_local map is
986
+ # truly empty. SafePyObject's destructor needs the GIL; if a thread
987
+ # exits while a SafePyObject is still in its thread_local,
988
+ # __call_tls_dtors fires the destructor → take_gil → deadlock.
989
+ torch._C._remove_obj_from_tls("context")
western_ai/usa/pytorch/torch/autograd/profiler.py ADDED
@@ -0,0 +1,1799 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import ast
3
+ import copy
4
+ import json
5
+ import logging
6
+ import re
7
+ import uuid
8
+ from bisect import bisect_left
9
+ from collections import defaultdict
10
+ from collections.abc import Iterable
11
+ from dataclasses import dataclass
12
+ from time import perf_counter_ns
13
+ from typing import Any, Optional
14
+ from warnings import warn
15
+
16
+
17
+ log = logging.getLogger(__name__)
18
+
19
+ import torch
20
+ import torch.cuda
21
+ from torch._C import _get_privateuse1_backend_name
22
+ from torch._C._profiler import _ExperimentalConfig
23
+ from torch.autograd import (
24
+ _disable_profiler,
25
+ _enable_profiler,
26
+ _kineto_step,
27
+ _prepare_profiler,
28
+ _ProfilerResult,
29
+ _supported_activities,
30
+ _toggle_collection_dynamic,
31
+ DeviceType,
32
+ kineto_available,
33
+ ProfilerActivity,
34
+ ProfilerConfig,
35
+ ProfilerState,
36
+ )
37
+ from torch.autograd.profiler_util import (
38
+ _filter_name,
39
+ _filter_stack_entry,
40
+ _rewrite_name,
41
+ EventList,
42
+ FunctionEvent,
43
+ MEMORY_EVENT_NAME,
44
+ MemRecordsAcc,
45
+ OUT_OF_MEMORY_EVENT_NAME,
46
+ )
47
+ from torch.futures import Future
48
+
49
+
50
+ __all__ = [
51
+ "profile",
52
+ "record_function",
53
+ "emit_itt",
54
+ "emit_nvtx",
55
+ "load_nvprof",
56
+ "EnforceUnique",
57
+ "parse_nvprof_trace",
58
+ "KinetoStepTracker",
59
+ "EventList",
60
+ "FunctionEvent",
61
+ "MemRecordsAcc",
62
+ ]
63
+
64
+ try:
65
+ # Available in Python >= 3.2
66
+ from contextlib import ContextDecorator as _ContextDecorator
67
+ except ImportError:
68
+ import functools
69
+
70
+ class _ContextDecorator: # type: ignore[no-redef]
71
+ def __enter__(self):
72
+ raise NotImplementedError
73
+
74
+ def __exit__(self, exc_type, exc_val, exc_tb):
75
+ raise NotImplementedError
76
+
77
+ def __call__(self, func):
78
+ @functools.wraps(func)
79
+ def wrapped(*args, **kwargs):
80
+ with self:
81
+ return func(*args, **kwargs)
82
+
83
+ return wrapped
84
+
85
+
86
+ # global python state - whether profiler is currently enabled
87
+ # useful for fast python checks to reduce latency
88
+ _is_profiler_enabled: bool = False
89
+
90
+ # https://github.com/pytorch/kineto/blob/a054a4be0db117c579a21747debf19c863631f26/libkineto/src/output_json.cpp#L559
91
+ # Kernel and CUDA runtime API events are connected by ac2g; forward and backward
92
+ # ATen ops are connected by fwdbwd.
93
+ _profile_flow_types = {"fwdbwd", "ac2g"}
94
+
95
+
96
+ def _set_is_profiler_enabled(enable: bool):
97
+ global _is_profiler_enabled
98
+ _is_profiler_enabled = enable
99
+
100
+
101
+ def _run_on_profiler_start():
102
+ _set_is_profiler_enabled(True)
103
+
104
+
105
+ def _run_on_profiler_stop():
106
+ _set_is_profiler_enabled(False)
107
+
108
+
109
+ @dataclass
110
+ class _ProfilerStats:
111
+ "Profiler timing and stats used by developers to catch issues/regressions"
112
+
113
+ profiling_window_duration_sec: float = 0
114
+ number_of_events: int = 0
115
+ profiler_prepare_call_duration_us: int = 0
116
+ profiler_enable_call_duration_us: int = 0
117
+ profiler_disable_call_duration_us: int = 0
118
+ parse_kineto_call_duration_us: int = 0
119
+ function_events_build_tree_call_duration_us: int = 0
120
+
121
+
122
+ class _EventItem:
123
+ def __init__(self, event):
124
+ self.e = event
125
+ self.start, self.end = self._interval_of()
126
+
127
+ def _interval_of(self):
128
+ start = self.e["ts"]
129
+ duration = self.e.get("dur", 0)
130
+ return start, start + duration
131
+
132
+
133
+ def _find_events_covered_in(
134
+ events: list[dict[str, Any]],
135
+ top_level_events: list[dict[str, Any]],
136
+ ):
137
+ """Find events covered in top level events.
138
+
139
+ Returns a dict mapping top-level event uid to covered event uids.
140
+ """
141
+ top_level_by_tid = defaultdict(list)
142
+ for event in top_level_events:
143
+ top_level_by_tid[event["tid"]].append(_EventItem(event))
144
+ for tid in top_level_by_tid:
145
+ top_level_by_tid[tid] = sorted(top_level_by_tid[tid], key=lambda x: x.start)
146
+ starts_by_tid = {
147
+ tid: [event.start for event in event_items]
148
+ for tid, event_items in top_level_by_tid.items()
149
+ }
150
+ top_level_id_to_events = defaultdict(set)
151
+
152
+ for event in events:
153
+ if event.get("cat") not in {
154
+ "cuda_runtime",
155
+ "cuLaunchKernel",
156
+ "cpu_op",
157
+ "cuda_driver",
158
+ "gpu_memset",
159
+ "python_function",
160
+ }:
161
+ continue
162
+ if "CallFrom" in event.get("args", {}):
163
+ continue
164
+ tid = event.get("tid")
165
+ if tid not in top_level_by_tid:
166
+ continue
167
+ item = _EventItem(event)
168
+ event_items = top_level_by_tid[tid]
169
+ idx = bisect_left(starts_by_tid[tid], item.start) - 1
170
+ if idx < 0 or idx >= len(event_items):
171
+ continue
172
+ top_level_item = event_items[idx]
173
+ if top_level_item.start <= item.start and item.end <= top_level_item.end:
174
+ top_level_id_to_events[top_level_item.e["uid"]].add(event["uid"])
175
+ return top_level_id_to_events
176
+
177
+
178
+ class profile:
179
+ """Context manager that manages autograd profiler state and holds a summary of results.
180
+
181
+ .. note::
182
+ This is the backend, most people should use :mod:`torch.profiler` instead.
183
+
184
+ Under the hood it just records events of functions being executed in C++ and
185
+ exposes those events to Python. You can wrap any code into it and it will
186
+ only report runtime of PyTorch functions.
187
+ Note: profiler is thread local and is automatically propagated into the async tasks
188
+
189
+ Args:
190
+ enabled (bool, optional): Setting this to False makes this context manager a no-op.
191
+
192
+ use_cuda (bool, optional): Enables timing of CUDA events as well
193
+ using the cudaEvent API. (will be deprecated)
194
+
195
+ use_device (str, optional): Enables timing of device events.
196
+ Adds approximately 4us of overhead to each tensor operation when use cuda.
197
+ The valid devices options are 'cuda', 'xpu', 'mtia' and 'privateuseone'.
198
+
199
+ record_shapes (bool, optional): If shapes recording is set, information
200
+ about input dimensions will be collected. This allows one to see which
201
+ dimensions have been used under the hood and further group by them
202
+ using prof.key_averages(group_by_input_shape=True). Please note that
203
+ shape recording might skew your profiling data. It is recommended to
204
+ use separate runs with and without shape recording to validate the timing.
205
+ Most likely the skew will be negligible for bottom most events (in a case
206
+ of nested function calls). But for higher level functions the total
207
+ self cpu time might be artificially increased because of the shape
208
+ collection.
209
+
210
+ with_flops (bool, optional): If with_flops is set, the profiler will estimate
211
+ the FLOPs (floating point operations) value using the operator's input shape.
212
+ This allows one to estimate the hardware performance. Currently,
213
+ this option only works for the matrix multiplication and 2D convolution operators.
214
+
215
+ profile_memory (bool, optional): track tensor memory allocation/deallocation.
216
+
217
+ with_stack (bool, optional): record source information (file and line number) for the ops.
218
+
219
+ with_modules (bool): record module hierarchy (including function names)
220
+ corresponding to the callstack of the op. e.g. If module A's forward call's
221
+ module B's forward which contains an aten::add op,
222
+ then aten::add's module hierarchy is A.B
223
+ Note that this support exist, at the moment, only for TorchScript models
224
+ and not eager mode models.
225
+
226
+ use_kineto (bool, optional): experimental, enable profiling with Kineto profiler.
227
+
228
+ use_cpu (bool, optional): profile CPU events; setting to ``False`` requires
229
+ ``use_kineto=True`` and can be used to lower the overhead for GPU-only profiling.
230
+
231
+ experimental_config (_ExperimentalConfig) : A set of experimental options
232
+ used by profiler libraries like Kineto. Note, backward compatibility is not guaranteed.
233
+
234
+ acc_events (bool): Enable the accumulation of FunctionEvents across multiple profiling cycles
235
+
236
+ post_processing_timeout_s (float): Optional timeout in seconds for post-processing profiler
237
+ results. In this context, post-processing happens after the profiling itself has finished.
238
+ If specified, event parsing will stop after this duration and return partial results. Useful
239
+ for handling large traces that may take too long to process.
240
+
241
+
242
+ .. warning::
243
+ Enabling memory profiling or source attribution incurs additional profiler
244
+ overhead
245
+
246
+ .. warning::
247
+ This context managers should not be called recursively, i.e. no nested
248
+ instances are allowed
249
+
250
+ .. warning::
251
+ Due to some CUDA multiprocessing limitations (see :ref:`multiprocessing-cuda-note`),
252
+ one cannot use the profiler with ``use_device = 'cuda'`` to benchmark
253
+ DataLoaders with ``num_workers > 0``. If you wish to benchmark data loading,
254
+ please use ``use_device = None`` or ``num_workers = 0``.
255
+
256
+ Example:
257
+ >>> # xdoctest: +SKIP
258
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER)
259
+ >>> x = torch.randn((1, 1), requires_grad=True)
260
+ >>> with torch.autograd.profiler.profile() as prof:
261
+ >>> for _ in range(100): # any normal python code, really!
262
+ >>> y = x ** 2
263
+ >>> y.backward()
264
+ >>> # NOTE: some columns were removed for brevity
265
+ >>> print(prof.key_averages().table(sort_by="self_cpu_time_total"))
266
+ ----------------------------------- --------------- --------------- ---------------
267
+ Name Self CPU total CPU time avg Number of Calls
268
+ ----------------------------------- --------------- --------------- ---------------
269
+ mul 32.048ms 32.048ms 200
270
+ pow 27.041ms 27.041ms 200
271
+ PowBackward0 9.727ms 55.483ms 100
272
+ torch::autograd::AccumulateGrad 9.148ms 9.148ms 100
273
+ torch::autograd::GraphRoot 691.816us 691.816us 100
274
+ ----------------------------------- --------------- --------------- ---------------
275
+
276
+ """
277
+
278
+ def __init__(
279
+ self,
280
+ enabled=True,
281
+ *,
282
+ use_cuda=False, # Deprecated
283
+ use_device=None,
284
+ record_shapes=False,
285
+ with_flops=False,
286
+ profile_memory=False,
287
+ with_stack=False,
288
+ with_modules=False,
289
+ use_kineto=False,
290
+ use_cpu=True,
291
+ experimental_config=None,
292
+ acc_events=False,
293
+ custom_trace_id_callback=None,
294
+ post_processing_timeout_s: float | None = None,
295
+ activity_filters: dict[ProfilerActivity, set[str]] | None = None,
296
+ ):
297
+ self.enabled: bool = enabled
298
+ if not self.enabled:
299
+ return
300
+ self.use_cuda = use_cuda
301
+ if self.use_cuda:
302
+ warn(
303
+ "The attribute `use_cuda` will be deprecated soon, "
304
+ "please use ``use_device = 'cuda'`` instead.",
305
+ FutureWarning,
306
+ stacklevel=2,
307
+ )
308
+ self.use_device: str | None = "cuda"
309
+ else:
310
+ self.use_device = use_device
311
+ # TODO Consider changing _function_events into data structure with size cap
312
+ self._function_events: EventList | None = None
313
+ self._old_function_events: EventList | None = None
314
+ # Function event processing is done lazily
315
+ self._needs_processing = False
316
+ self.entered = False
317
+ self.record_shapes = record_shapes
318
+ self.with_flops = with_flops
319
+ self.record_shapes |= self.with_flops
320
+ self.profile_memory = profile_memory
321
+ self.with_stack = with_stack
322
+ self.with_modules = with_modules
323
+ self.use_cpu = use_cpu
324
+ self.acc_events = acc_events
325
+ if experimental_config is None:
326
+ experimental_config = _ExperimentalConfig()
327
+ if experimental_config.trace_only and with_stack:
328
+ warn(
329
+ "trace_only=True is incompatible with with_stack=True "
330
+ "(stack traces require event post-processing). "
331
+ "Disabling trace_only."
332
+ )
333
+ experimental_config = copy.copy(experimental_config)
334
+ experimental_config.trace_only = False
335
+ if (
336
+ experimental_config.profiler_metrics
337
+ or experimental_config.profiler_measure_per_kernel
338
+ ):
339
+ warn(
340
+ "profiler_metrics and profiler_measure_per_kernel are deprecated "
341
+ "and ignored. These options will be removed in a future release.",
342
+ FutureWarning,
343
+ stacklevel=2,
344
+ )
345
+ self.experimental_config = experimental_config
346
+ self.kineto_results: _ProfilerResult | None = None
347
+ self.profiling_start_time_ns = 0
348
+ self.profiling_end_time_ns = 0
349
+ self._stats = _ProfilerStats()
350
+ self.custom_trace_id_callback = custom_trace_id_callback
351
+ self.post_processing_timeout_s = post_processing_timeout_s
352
+ self.activity_filters = activity_filters or {}
353
+ self.trace_id = ""
354
+ if not self.use_cpu:
355
+ if not use_kineto:
356
+ raise AssertionError(
357
+ "Device-only events supported only with Kineto (use_kineto=True)"
358
+ )
359
+
360
+ if self.use_device is not None:
361
+ VALID_DEVICE_OPTIONS = ["cuda", "xpu", "mtia", "hpu"]
362
+ if _get_privateuse1_backend_name() != "privateuseone":
363
+ VALID_DEVICE_OPTIONS.append(_get_privateuse1_backend_name())
364
+ if self.use_device not in VALID_DEVICE_OPTIONS:
365
+ warn(
366
+ f"The {self.use_device} is not a valid device option.", stacklevel=2
367
+ )
368
+ self.use_device = None
369
+
370
+ if self.use_device == "cuda" and not torch.cuda.is_available():
371
+ warn("CUDA is not available, disabling CUDA profiling", stacklevel=2)
372
+ self.use_cuda = False
373
+ self.use_device = None
374
+
375
+ if self.use_device == "xpu" and not torch.xpu.is_available():
376
+ warn("XPU is not available, disabling XPU profiling", stacklevel=2)
377
+ self.use_device = None
378
+
379
+ if self.use_device == "hpu" and not (
380
+ hasattr(torch, "hpu") and torch.hpu.is_available()
381
+ ):
382
+ warn("HPU is not available, disabling HPU profiling", stacklevel=2)
383
+ self.use_device = None
384
+
385
+ self.kineto_activities = set()
386
+ if self.use_cpu:
387
+ self.kineto_activities.add(ProfilerActivity.CPU)
388
+
389
+ self.profiler_kind = ProfilerState.KINETO
390
+ if self.use_device == "cuda":
391
+ if not use_kineto or ProfilerActivity.CUDA not in _supported_activities():
392
+ if not self.use_cpu:
393
+ raise AssertionError("Legacy CUDA profiling requires use_cpu=True")
394
+ self.profiler_kind = ProfilerState.KINETO_GPU_FALLBACK
395
+ else:
396
+ self.kineto_activities.add(ProfilerActivity.CUDA)
397
+ elif self.use_device == "xpu":
398
+ if not (use_kineto and ProfilerActivity.XPU in _supported_activities()):
399
+ raise AssertionError(
400
+ "Legacy XPU profiling is not supported. Requires use_kineto=True on XPU devices."
401
+ )
402
+ self.kineto_activities.add(ProfilerActivity.XPU)
403
+ elif self.use_device == "mtia":
404
+ if not (use_kineto and ProfilerActivity.MTIA in _supported_activities()):
405
+ raise AssertionError(
406
+ "Legacy MTIA profiling is not supported. Requires use_kineto=True on MTIA devices."
407
+ )
408
+ self.kineto_activities.add(ProfilerActivity.MTIA)
409
+ elif self.use_device == "hpu":
410
+ if not (use_kineto and ProfilerActivity.HPU in _supported_activities()):
411
+ raise AssertionError(
412
+ "Legacy HPU profiling is not supported. Requires use_kineto=True on HPU devices."
413
+ )
414
+ self.kineto_activities.add(ProfilerActivity.HPU)
415
+ elif self.use_device is not None and self.use_device != "privateuseone":
416
+ if use_kineto:
417
+ # Native tracing mode: use KINETO_PRIVATEUSE1 with registered IActivityProfiler
418
+ self.profiler_kind = ProfilerState.KINETO_PRIVATEUSE1
419
+ if ProfilerActivity.PrivateUse1 in _supported_activities():
420
+ self.kineto_activities.add(ProfilerActivity.PrivateUse1)
421
+ else:
422
+ # Marker-only mode: use fallback state
423
+ if not self.use_cpu:
424
+ raise AssertionError(
425
+ "Legacy privateuse1 profiling requires use_cpu=True"
426
+ )
427
+ self.profiler_kind = ProfilerState.KINETO_PRIVATEUSE1_FALLBACK
428
+
429
+ if len(self.kineto_activities) == 0:
430
+ raise AssertionError("No activities specified for the profiler")
431
+
432
+ if (
433
+ self.post_processing_timeout_s is not None
434
+ and self.post_processing_timeout_s < 0
435
+ ):
436
+ raise ValueError("post_processing_timeout_s must be non-negative")
437
+
438
+ def default_trace_id(self):
439
+ # Generate a UUID
440
+ uuid_raw = uuid.uuid4()
441
+
442
+ return f"{uuid_raw.int:032X}"
443
+
444
+ def create_trace_id(self):
445
+ if self.custom_trace_id_callback:
446
+ return self.custom_trace_id_callback()
447
+ return self.default_trace_id()
448
+
449
+ def config(self, create_trace_id=False):
450
+ # only need to generate new trace id upon prepare trace not start trace
451
+ if create_trace_id:
452
+ trace_id = self.create_trace_id()
453
+ self.trace_id = trace_id
454
+ return ProfilerConfig(
455
+ self.profiler_kind,
456
+ self.record_shapes,
457
+ self.profile_memory,
458
+ self.with_stack,
459
+ self.with_flops,
460
+ self.with_modules,
461
+ self.experimental_config,
462
+ self.trace_id,
463
+ )
464
+
465
+ def __enter__(self):
466
+ if not self.enabled:
467
+ return
468
+ if self.entered:
469
+ raise RuntimeError("Profiler context manager is not reentrant")
470
+ self._prepare_trace()
471
+ self._start_trace()
472
+ return self
473
+
474
+ def _prepare_trace(self):
475
+ self.entered = True
476
+ t0 = perf_counter_ns()
477
+ _prepare_profiler(
478
+ self.config(create_trace_id=True),
479
+ self.kineto_activities,
480
+ activity_filter=self.activity_filters,
481
+ )
482
+ t1 = perf_counter_ns()
483
+ self._stats.profiler_prepare_call_duration_us = int((t1 - t0) / 1000)
484
+
485
+ def _start_trace(self):
486
+ self.entered = True
487
+ _run_on_profiler_start()
488
+ t0 = perf_counter_ns()
489
+ _enable_profiler(self.config(create_trace_id=False), self.kineto_activities)
490
+ t1 = perf_counter_ns()
491
+ self._stats.profiler_enable_call_duration_us = int((t1 - t0) / 1000)
492
+ self.profiling_start_time_ns = t1
493
+
494
+ def __exit__(self, exc_type, exc_val, exc_tb):
495
+ if not self.enabled:
496
+ return
497
+ if self.use_device and hasattr(torch, self.use_device):
498
+ device_module = getattr(torch, self.use_device)
499
+ if hasattr(device_module, "synchronize"):
500
+ device_module.synchronize()
501
+
502
+ if self._function_events and self.acc_events:
503
+ self._old_function_events = self._function_events
504
+ self._function_events = None
505
+ self._needs_processing = True
506
+
507
+ t0 = perf_counter_ns()
508
+
509
+ self.kineto_results = _disable_profiler()
510
+ t1 = perf_counter_ns()
511
+ self._stats.profiler_disable_call_duration_us = int((t1 - t0) / 1000)
512
+ self.profiling_end_time_ns = t0
513
+
514
+ _run_on_profiler_stop()
515
+
516
+ self._stats.profiling_window_duration_sec = (
517
+ (self.profiling_end_time_ns - self.profiling_start_time_ns) * 1.0 / 1e9
518
+ )
519
+
520
+ # If we plan to accumulate events we should post process the function events
521
+ # right away to retain the state across multiple start/stop calls
522
+ if self.acc_events and not self.experimental_config.trace_only:
523
+ self._ensure_function_events()
524
+ return False
525
+
526
+ def __repr__(self):
527
+ if self._needs_processing:
528
+ self._ensure_function_events()
529
+ if self._function_events is None:
530
+ return "<unfinished torch.autograd.profile>"
531
+ return repr(self._function_events)
532
+
533
+ def __str__(self):
534
+ if self._needs_processing:
535
+ self._ensure_function_events()
536
+ if self._function_events is None:
537
+ return "<unfinished torch.autograd.profile>"
538
+ return str(self._function_events)
539
+
540
+ def _ensure_function_events(self):
541
+ """Process function events lazily if required"""
542
+ if self.experimental_config.trace_only:
543
+ raise RuntimeError(
544
+ "events() is not available when trace_only=True in "
545
+ "ExperimentalConfig. Use export_chrome_trace() instead."
546
+ )
547
+ if self._function_events is not None:
548
+ return
549
+ self._needs_processing = False
550
+
551
+ t0 = perf_counter_ns()
552
+ parsed_results = []
553
+ if self.kineto_results:
554
+ parsed_results = self._parse_kineto_results(
555
+ self.kineto_results, timeout_s=self.post_processing_timeout_s
556
+ )
557
+ t1 = perf_counter_ns()
558
+ self._stats.parse_kineto_call_duration_us = int((t1 - t0) / 1000)
559
+
560
+ self._function_events = EventList(
561
+ parsed_results,
562
+ use_device=self.use_device,
563
+ profile_memory=self.profile_memory,
564
+ with_flops=self.with_flops,
565
+ )
566
+ t0 = perf_counter_ns()
567
+ self._function_events._build_tree()
568
+ t1 = perf_counter_ns()
569
+ self._stats.function_events_build_tree_call_duration_us = int((t1 - t0) / 1000)
570
+ self._stats.number_of_events = len(self._function_events)
571
+
572
+ if self._old_function_events and self.acc_events:
573
+ for evt in self._old_function_events:
574
+ self._function_events.append(evt)
575
+ self._old_function_events = None
576
+
577
+ @property
578
+ def function_events(self):
579
+ if self._function_events is None or self._needs_processing:
580
+ self._ensure_function_events()
581
+ return self._function_events
582
+
583
+ def table(
584
+ self,
585
+ sort_by=None,
586
+ row_limit=100,
587
+ max_src_column_width=75,
588
+ max_name_column_width=55,
589
+ max_shapes_column_width=80,
590
+ header=None,
591
+ top_level_events_only=False,
592
+ ):
593
+ self._ensure_function_events()
594
+ if self._function_events is None:
595
+ raise AssertionError("Expected profiling results")
596
+ return self._function_events.table(
597
+ sort_by=sort_by,
598
+ row_limit=row_limit,
599
+ max_src_column_width=max_src_column_width,
600
+ max_name_column_width=max_name_column_width,
601
+ max_shapes_column_width=max_shapes_column_width,
602
+ header=header,
603
+ top_level_events_only=top_level_events_only,
604
+ )
605
+
606
+ table.__doc__ = EventList.table.__doc__
607
+
608
+ def _assign_uniq_id_to_event(self, trace):
609
+ if trace.get("uid_assigned"):
610
+ return {event["uid"]: event for event in trace["traceEvents"]}
611
+ uid_2_events = {}
612
+ for uid, event in enumerate(trace["traceEvents"]):
613
+ event["uid"] = uid
614
+ uid_2_events[uid] = event
615
+ trace["uid_assigned"] = True
616
+ return uid_2_events
617
+
618
+ def _build_flow_mapping(self, trace, flow_events):
619
+ """Build src/dst event mappings from profiler flow events."""
620
+ if not flow_events:
621
+ return {}, {}
622
+
623
+ flow_pair: dict[int, list[int | None]] = {}
624
+ prev_event = None
625
+ for event in trace["traceEvents"]:
626
+ if (
627
+ event.get("name") not in _profile_flow_types
628
+ and event.get("cat") not in _profile_flow_types
629
+ ):
630
+ prev_event = event
631
+ continue
632
+ if event.get("ph") == "s":
633
+ if prev_event is None:
634
+ continue
635
+ pair = flow_pair.setdefault(int(event["id"]), [None, None])
636
+ pair[0] = prev_event["uid"]
637
+ elif event.get("ph") == "f":
638
+ if prev_event is None:
639
+ continue
640
+ pair = flow_pair.setdefault(int(event["id"]), [None, None])
641
+ pair[1] = prev_event["uid"]
642
+ prev_event = event
643
+
644
+ src2dst = {}
645
+ dst2src = {}
646
+ for src, dst in flow_pair.values():
647
+ if src is None or dst is None:
648
+ continue
649
+ src2dst[src] = dst
650
+ dst2src[dst] = src
651
+ return src2dst, dst2src
652
+
653
+ def _maybe_triton_call(self, kernel_name):
654
+ return kernel_name.startswith("triton_")
655
+
656
+ def _has_kernel_info(self, compile_info, graph_key, kernel_name):
657
+ graph_info = compile_info.get(graph_key, {})
658
+ if kernel_name in graph_info:
659
+ return True
660
+ kernel_prefix = kernel_name + ":"
661
+ return any(name.startswith(kernel_prefix) for name in graph_info)
662
+
663
+ def _maybe_extern_call(self, event, compile_info, fwd_key, bw_keys, need_bw):
664
+ kernel_name = event["name"]
665
+ return not self._has_kernel_info(compile_info, fwd_key, kernel_name) and (
666
+ not need_bw
667
+ or all(
668
+ not self._has_kernel_info(compile_info, bw_key, kernel_name)
669
+ for bw_key in bw_keys
670
+ )
671
+ )
672
+
673
+ def add_to_chrome_trace(self, origin_trace):
674
+ """Add Inductor kernel stack information to a Chrome trace."""
675
+ from torch._inductor.debug import get_kernel_information_jsons
676
+
677
+ compile_linenos = get_kernel_information_jsons()
678
+ if not compile_linenos:
679
+ return origin_trace
680
+
681
+ trace = origin_trace
682
+ uid_2_events = self._assign_uniq_id_to_event(trace)
683
+
684
+ real_events = []
685
+ flow_events = []
686
+ compiled_events = []
687
+ extern_events = []
688
+ for event in trace["traceEvents"]:
689
+ name = event.get("name", "")
690
+ cat = event.get("cat", "")
691
+ if name in _profile_flow_types or cat in _profile_flow_types:
692
+ flow_events.append(event)
693
+ continue
694
+
695
+ real_events.append(event)
696
+ if cat == "cpu_op" and (
697
+ name == "CompiledFunctionBackward" or "Torch-Compiled Region" in name
698
+ ):
699
+ compiled_events.append(event)
700
+ if (
701
+ name
702
+ and "extern_kernels" in name
703
+ and cat
704
+ in (
705
+ "cpu_op",
706
+ "python_function",
707
+ )
708
+ ):
709
+ extern_events.append(event)
710
+
711
+ compiled_events = sorted(compiled_events, key=lambda x: _EventItem(x).start)
712
+ compiled_event_items = [_EventItem(event) for event in compiled_events]
713
+ compiled_event_starts = [item.start for item in compiled_event_items]
714
+ ops_in_compile_region = _find_events_covered_in(real_events, compiled_events)
715
+ ops_in_extern_region = _find_events_covered_in(real_events, extern_events)
716
+ src2dst, dst2src = self._build_flow_mapping(trace, flow_events)
717
+ kernel_uids_by_external_id = defaultdict(list)
718
+ for event in real_events:
719
+ if event.get("cat") != "kernel":
720
+ continue
721
+ external_id = event.get("args", {}).get("External id")
722
+ if external_id is not None:
723
+ kernel_uids_by_external_id[external_id].append(event["uid"])
724
+
725
+ def _related_compile_region(compile_event):
726
+ src_event = uid_2_events[dst2src[compile_event["uid"]]]
727
+ src_item = _EventItem(src_event)
728
+ idx = bisect_left(compiled_event_starts, src_item.start) - 1
729
+ # Compiled regions are expected to be nearly nested around their
730
+ # source events, so the nearest preceding region usually matches.
731
+ while idx >= 0:
732
+ region_item = compiled_event_items[idx]
733
+ if (
734
+ src_item.start > region_item.start
735
+ and src_item.end < region_item.end
736
+ ):
737
+ return region_item.e
738
+ idx -= 1
739
+ raise ValueError(f"Cannot find compile region for {compile_event}")
740
+
741
+ def _parse_compile_region_key(key):
742
+ try:
743
+ parsed_key = ast.literal_eval(key)
744
+ except (SyntaxError, ValueError):
745
+ return None
746
+ if (
747
+ not isinstance(parsed_key, tuple)
748
+ or len(parsed_key) != 2
749
+ or not isinstance(parsed_key[0], str)
750
+ or not isinstance(parsed_key[1], bool)
751
+ ):
752
+ return None
753
+ return parsed_key
754
+
755
+ def _backward_graph_keys(compile_info, compile_name):
756
+ bw_graph_key = str((compile_name, True))
757
+ keys = [bw_graph_key]
758
+ if bw_graph_key in compile_info:
759
+ return keys
760
+
761
+ prefix = compile_name + "_"
762
+ for key, info in compile_info.items():
763
+ if not info:
764
+ continue
765
+ parsed_key = _parse_compile_region_key(key)
766
+ if parsed_key is None:
767
+ continue
768
+ graph_name, is_backward = parsed_key
769
+ if is_backward and graph_name.startswith(prefix):
770
+ keys.append(key)
771
+ return keys
772
+
773
+ def _stack_from_kernel_info(kernel_info):
774
+ if isinstance(kernel_info, dict):
775
+ return kernel_info.get("stack_traces")
776
+ return kernel_info
777
+
778
+ def _stack_for_kernel(compile_info, graph_key, kernel_name):
779
+ graph_info = compile_info.get(graph_key, {})
780
+ kernel_info = graph_info.get(kernel_name)
781
+ if kernel_info is None:
782
+ kernel_prefix = kernel_name + ":"
783
+ for name, info in graph_info.items():
784
+ if name.startswith(kernel_prefix):
785
+ kernel_info = info
786
+ break
787
+ return _stack_from_kernel_info(kernel_info)
788
+
789
+ def _single_stack_for_graph(compile_info, graph_key):
790
+ stacks = [
791
+ stack
792
+ for stack in (
793
+ _stack_from_kernel_info(kernel_info)
794
+ for kernel_info in compile_info.get(graph_key, {}).values()
795
+ )
796
+ if stack is not None
797
+ ]
798
+ if len(stacks) == 1:
799
+ return stacks[0]
800
+ return None
801
+
802
+ def _assign_stack(event, compile_info, fwd_key, bw_keys, kernel_name):
803
+ for bw_key in bw_keys:
804
+ stack = _stack_for_kernel(compile_info, bw_key, kernel_name)
805
+ if stack is not None:
806
+ event.setdefault("args", {})["stack"] = stack
807
+ return True
808
+ stack = _stack_for_kernel(compile_info, fwd_key, kernel_name)
809
+ if stack is not None:
810
+ event.setdefault("args", {})["stack"] = stack
811
+ return True
812
+ return False
813
+
814
+ def _assign_single_stack(event, compile_info, fwd_key, bw_keys):
815
+ for bw_key in bw_keys:
816
+ stack = _single_stack_for_graph(compile_info, bw_key)
817
+ if stack is not None:
818
+ event.setdefault("args", {})["stack"] = stack
819
+ return True
820
+ stack = _single_stack_for_graph(compile_info, fwd_key)
821
+ if stack is not None:
822
+ event.setdefault("args", {})["stack"] = stack
823
+ return True
824
+ return False
825
+
826
+ def _kernel_events_for_op(op_id):
827
+ if op_id in src2dst:
828
+ return [uid_2_events[src2dst[op_id]]]
829
+ external_id = uid_2_events[op_id].get("args", {}).get("External id")
830
+ if external_id is None:
831
+ return []
832
+ return [
833
+ uid_2_events[kernel_uid]
834
+ for kernel_uid in kernel_uids_by_external_id.get(external_id, [])
835
+ ]
836
+
837
+ def _extern_kernel_name(name):
838
+ match = re.search(r"\b(extern\w+)\s*,", name)
839
+ return match.group(1) if match else name
840
+
841
+ for compile_func_id, scoped_ops in ops_in_compile_region.items():
842
+ compile_event = uid_2_events[compile_func_id]
843
+ compile_name = compile_event["name"]
844
+ need_bw = "Backward" in compile_name
845
+
846
+ if need_bw:
847
+ try:
848
+ compile_name = _related_compile_region(compile_event)["name"]
849
+ except ValueError:
850
+ continue
851
+
852
+ fw_graph_key = str((compile_name, False))
853
+ bw_graph_keys = _backward_graph_keys(compile_linenos, compile_name)
854
+ if fw_graph_key not in compile_linenos and all(
855
+ bw_graph_key not in compile_linenos for bw_graph_key in bw_graph_keys
856
+ ):
857
+ continue
858
+
859
+ for op_id in scoped_ops:
860
+ kernel_events = _kernel_events_for_op(op_id)
861
+ if not kernel_events:
862
+ continue
863
+ for kernel_event in kernel_events:
864
+ assigned_stack = False
865
+ warn_if_unrecognized = False
866
+ if self._maybe_extern_call(
867
+ kernel_event,
868
+ compile_linenos,
869
+ fw_graph_key,
870
+ bw_graph_keys,
871
+ need_bw,
872
+ ):
873
+ for extern_call, related_ops in ops_in_extern_region.items():
874
+ if op_id in related_ops:
875
+ extern_event = uid_2_events[extern_call]
876
+ assigned_stack = _assign_stack(
877
+ kernel_event,
878
+ compile_linenos,
879
+ fw_graph_key,
880
+ bw_graph_keys,
881
+ _extern_kernel_name(extern_event.get("name", "")),
882
+ )
883
+ if assigned_stack:
884
+ break
885
+ if not assigned_stack:
886
+ assigned_stack = _assign_single_stack(
887
+ kernel_event,
888
+ compile_linenos,
889
+ fw_graph_key,
890
+ bw_graph_keys,
891
+ )
892
+ elif not self._maybe_triton_call(kernel_event.get("name", "")):
893
+ warn_if_unrecognized = True
894
+ assigned_stack = _assign_single_stack(
895
+ kernel_event,
896
+ compile_linenos,
897
+ fw_graph_key,
898
+ bw_graph_keys,
899
+ )
900
+ else:
901
+ assigned_stack = _assign_stack(
902
+ kernel_event,
903
+ compile_linenos,
904
+ fw_graph_key,
905
+ bw_graph_keys,
906
+ kernel_event.get("name", ""),
907
+ )
908
+ if not assigned_stack and warn_if_unrecognized:
909
+ log.warning(
910
+ "Kernel %s cannot be recognized as a custom kernel or triton kernel. "
911
+ "Please try profile with stack=True.",
912
+ kernel_event.get("name", ""),
913
+ )
914
+ return trace
915
+
916
+ def export_chrome_trace(self, path, metadata=None, use_python_export=False):
917
+ """
918
+ Exports the collected trace in Chrome JSON format. If kineto is enabled, only
919
+ last cycle in schedule is exported.
920
+ """
921
+ if use_python_export and kineto_available():
922
+ from torch.profiler._chrome_trace_export import (
923
+ export_chrome_trace as _export,
924
+ )
925
+
926
+ _export(self.kineto_results, path, metadata) # type: ignore[union-attr]
927
+ elif kineto_available():
928
+ self.kineto_results.save(path) # type: ignore[union-attr]
929
+ else:
930
+ self._ensure_function_events()
931
+ return self._function_events.export_chrome_trace(path) # type: ignore[union-attr]
932
+
933
+ import torch._inductor.config as inductor_config
934
+
935
+ if inductor_config.trace.provenance_tracking_to_timeline:
936
+ from torch._inductor.debug import get_kernel_information_jsons
937
+
938
+ try:
939
+ # Kineto owns the initial serialization path. Re-read the exported
940
+ # trace here so timeline provenance stays decoupled from Kineto's
941
+ # JSON writer and remains a Python-only post-processing step.
942
+ with open(path) as f:
943
+ trace = json.load(f)
944
+
945
+ num_events = len(trace.get("traceEvents", []))
946
+ max_events = inductor_config.trace.provenance_tracking_max_events
947
+ if max_events > 0 and num_events > max_events:
948
+ log.warning(
949
+ "Skipping provenance tracking: trace has %d events "
950
+ "(exceeds limit of %d). Set TORCH_COMPILE_DEBUG_MAX_EVENTS=0 "
951
+ "to disable this protection or increase the limit.",
952
+ num_events,
953
+ max_events,
954
+ )
955
+ return
956
+
957
+ if not inductor_config.triton.unique_kernel_names:
958
+ log.warning(
959
+ "Profiling trace does not contain Triton kernel stack traces "
960
+ "because TORCHINDUCTOR_UNIQUE_KERNEL_NAMES=0."
961
+ )
962
+ if inductor_config.cpp_wrapper:
963
+ log.warning(
964
+ "Profiling trace does not contain compiled kernel stack traces "
965
+ "because cpp_wrapper is enabled."
966
+ )
967
+ return
968
+ try:
969
+ log.info("Add stack trace to compiled kernel.")
970
+ trace = self.add_to_chrome_trace(trace)
971
+ with open(path, "w") as f:
972
+ json.dump(trace, f, indent=1)
973
+ except MemoryError:
974
+ log.error(
975
+ "MemoryError during add_to_chrome_trace. "
976
+ "Try increasing TORCH_COMPILE_DEBUG_MAX_EVENTS or disable provenance tracking."
977
+ )
978
+ raise
979
+ except Exception:
980
+ log.exception("Failed to add stack trace to compiled kernel")
981
+ finally:
982
+ get_kernel_information_jsons().clear()
983
+
984
+ export_chrome_trace.__doc__ = EventList.export_chrome_trace.__doc__
985
+
986
+ def export_stacks(self, path: str, metric: str = "self_cpu_time_total"):
987
+ self._ensure_function_events()
988
+ if self._function_events is None:
989
+ raise AssertionError("Expected profiling results")
990
+ if not self.with_stack:
991
+ raise AssertionError("export_stacks() requires with_stack=True")
992
+ return self._function_events.export_stacks(path, metric)
993
+
994
+ def toggle_collection_dynamic(
995
+ self, enabled: bool, activities: Iterable[ProfilerActivity]
996
+ ):
997
+ """
998
+ Toggles the collection of activities for the current profiler instance.
999
+ """
1000
+ return _toggle_collection_dynamic(enabled, set(activities))
1001
+
1002
+ def key_averages(
1003
+ self,
1004
+ group_by_input_shape=False,
1005
+ group_by_stack_n=0,
1006
+ group_by_overload_name=False,
1007
+ include_python_functions=False,
1008
+ ):
1009
+ self._ensure_function_events()
1010
+ if self._function_events is None:
1011
+ raise AssertionError("Expected profiling results")
1012
+ return self._function_events.key_averages(
1013
+ group_by_input_shape,
1014
+ group_by_stack_n,
1015
+ group_by_overload_name,
1016
+ include_python_functions,
1017
+ )
1018
+
1019
+ key_averages.__doc__ = EventList.key_averages.__doc__
1020
+
1021
+ def total_average(self):
1022
+ self._ensure_function_events()
1023
+ if self._function_events is None:
1024
+ raise AssertionError("Expected profiling results")
1025
+ return self._function_events.total_average()
1026
+
1027
+ total_average.__doc__ = EventList.total_average.__doc__
1028
+
1029
+ @property
1030
+ def self_cpu_time_total(self):
1031
+ """Returns total time spent on CPU.
1032
+
1033
+ The total time is a sum of all self times across all the events.
1034
+ """
1035
+ self._ensure_function_events()
1036
+ if self._function_events is None:
1037
+ raise AssertionError("Expected profiling results")
1038
+ return self._function_events.self_cpu_time_total
1039
+
1040
+ def _parse_kineto_results(
1041
+ self, result: _ProfilerResult, timeout_s: float | None = None
1042
+ ):
1043
+ # result.events() has most of the events - PyTorch op-level and device-level events
1044
+
1045
+ timeout_ns = int(timeout_s * 1e9) if timeout_s is not None else None
1046
+ result_events = result.events()
1047
+ if timeout_ns is not None and timeout_ns < 0:
1048
+ raise ValueError("timeout_s must be non-negative")
1049
+ start_time_ns = perf_counter_ns()
1050
+ timed_out = False
1051
+
1052
+ def _check_timeout() -> bool:
1053
+ """Check if timeout has been exceeded. Returns True if timed out."""
1054
+ nonlocal timed_out
1055
+ if timeout_ns is not None and not timed_out:
1056
+ elapsed_ns = perf_counter_ns() - start_time_ns
1057
+ if elapsed_ns >= timeout_ns:
1058
+ timed_out = True
1059
+ return timed_out
1060
+
1061
+ trace_start_ns = result.trace_start_ns()
1062
+ mem_records = [
1063
+ [evt, False] for evt in result_events if evt.name() == MEMORY_EVENT_NAME
1064
+ ]
1065
+ oom_records = [
1066
+ evt for evt in result_events if evt.name() == OUT_OF_MEMORY_EVENT_NAME
1067
+ ]
1068
+ mem_records_acc = MemRecordsAcc(mem_records)
1069
+
1070
+ def _cpu_memory_usage(mem_record):
1071
+ return (
1072
+ mem_record.nbytes()
1073
+ if mem_record.device_type()
1074
+ in [DeviceType.CPU, DeviceType.MKLDNN, DeviceType.IDEEP]
1075
+ else 0
1076
+ )
1077
+
1078
+ def _device_memory_usage(mem_record):
1079
+ return (
1080
+ mem_record.nbytes()
1081
+ if mem_record.device_type()
1082
+ in [
1083
+ DeviceType.CUDA,
1084
+ DeviceType.PrivateUse1,
1085
+ DeviceType.HIP,
1086
+ DeviceType.XPU,
1087
+ ]
1088
+ else 0
1089
+ )
1090
+
1091
+ # Create and return FunctionEvent list, which contains all function events
1092
+ # Here 2 function events are created:
1093
+ # all_function_events contains all events associated with each kineto event from result
1094
+ all_function_events = []
1095
+ # frontend_function_events contains the events in aten or torch frontend level,
1096
+ # whose correlation id is 0
1097
+ frontend_function_events = []
1098
+ device_corr_map: dict[int, list[FunctionEvent]] = {}
1099
+ max_evt_id = 0
1100
+ for kineto_event in result_events:
1101
+ if _check_timeout():
1102
+ break
1103
+
1104
+ if (
1105
+ _filter_name(kineto_event.name())
1106
+ or getattr(kineto_event, "is_hidden_event", lambda: False)()
1107
+ ):
1108
+ continue
1109
+ rel_start_ns = kineto_event.start_ns() - trace_start_ns
1110
+ rel_end_ns = kineto_event.end_ns() - trace_start_ns
1111
+
1112
+ cpu_memory_usage = 0
1113
+ device_memory_usage = 0
1114
+ if kineto_event.device_type() == DeviceType.CPU:
1115
+ # find the corresponding memory allocation events
1116
+ for mem_record in mem_records_acc.in_interval(
1117
+ kineto_event.start_ns(), kineto_event.end_ns()
1118
+ ):
1119
+ cpu_memory_usage += _cpu_memory_usage(mem_record[0])
1120
+ device_memory_usage += _device_memory_usage(mem_record[0])
1121
+ mem_record[1] = True
1122
+
1123
+ is_async = kineto_event.is_async() or (
1124
+ kineto_event.start_thread_id() != kineto_event.end_thread_id()
1125
+ )
1126
+
1127
+ fe = FunctionEvent(
1128
+ id=kineto_event.correlation_id(),
1129
+ name=_rewrite_name(name=kineto_event.name(), with_wildcard=True),
1130
+ overload_name=kineto_event.overload_name(),
1131
+ trace_name=_rewrite_name(name=kineto_event.name(), with_wildcard=False),
1132
+ thread=kineto_event.start_thread_id(),
1133
+ start_us=rel_start_ns / 1000,
1134
+ end_us=rel_end_ns / 1000,
1135
+ fwd_thread=kineto_event.fwd_thread_id(),
1136
+ input_shapes=kineto_event.shapes(),
1137
+ concrete_inputs=kineto_event.concrete_inputs(),
1138
+ kwinputs=kineto_event.kwinputs(),
1139
+ stack=[
1140
+ entry
1141
+ for entry in kineto_event.stack()
1142
+ if _filter_stack_entry(entry)
1143
+ ],
1144
+ scope=kineto_event.scope(),
1145
+ use_device=self.use_device,
1146
+ cpu_memory_usage=cpu_memory_usage,
1147
+ device_memory_usage=device_memory_usage,
1148
+ is_async=is_async,
1149
+ sequence_nr=kineto_event.sequence_nr(),
1150
+ device_type=kineto_event.device_type(),
1151
+ device_index=kineto_event.device_index(),
1152
+ device_resource_id=kineto_event.device_resource_id(),
1153
+ flops=kineto_event.flops(),
1154
+ is_user_annotation=kineto_event.is_user_annotation(),
1155
+ is_python_function=kineto_event.is_python_function(),
1156
+ activity_type=kineto_event.activity_type(),
1157
+ metadata_json=kineto_event.metadata_json(),
1158
+ extra_meta=kineto_event.extra_meta() or None,
1159
+ flow_id=kineto_event.flow_id(),
1160
+ flow_type=kineto_event.flow_type(),
1161
+ flow_start=kineto_event.flow_start(),
1162
+ external_id=kineto_event.external_id(),
1163
+ linked_correlation_id=kineto_event.linked_correlation_id(),
1164
+ structured_input_shapes=kineto_event.structured_input_shapes(),
1165
+ structured_input_strides=kineto_event.structured_input_strides(),
1166
+ input_dtypes=kineto_event.dtypes(),
1167
+ python_id=kineto_event.python_id(),
1168
+ python_parent_id=kineto_event.python_parent_id(),
1169
+ python_module_id=kineto_event.python_module_id(),
1170
+ )
1171
+ max_evt_id = max(max_evt_id, fe.id)
1172
+ if fe.device_type == DeviceType.CPU and not fe.is_async:
1173
+ if self.use_device == _get_privateuse1_backend_name():
1174
+ privateuse1_time = kineto_event.privateuse1_elapsed_us()
1175
+ if privateuse1_time > 0:
1176
+ fe.append_kernel(fe.name, fe.device_index, privateuse1_time)
1177
+ fe.is_legacy = True
1178
+ elif self.use_device == "cuda":
1179
+ # Check if we have CUDA time as a fallback
1180
+ cuda_time = kineto_event.cuda_elapsed_us()
1181
+ if cuda_time > 0:
1182
+ fe.append_kernel(fe.name, fe.device_index, cuda_time)
1183
+ fe.is_legacy = True
1184
+ all_function_events.append(fe)
1185
+ corr_id = kineto_event.linked_correlation_id()
1186
+ if corr_id > 0:
1187
+ if corr_id not in device_corr_map:
1188
+ device_corr_map[corr_id] = []
1189
+ device_corr_map[corr_id].append(fe)
1190
+ elif corr_id == 0:
1191
+ frontend_function_events.append(fe)
1192
+ else:
1193
+ raise RuntimeError(
1194
+ f"Got negative correlation id {corr_id} in profiler post processing"
1195
+ )
1196
+
1197
+ # associate device kernels and device runtime (CPU) with CPU events
1198
+ for fe in frontend_function_events:
1199
+ if (
1200
+ fe.device_type == DeviceType.CPU
1201
+ and not fe.is_async
1202
+ and fe.id in device_corr_map
1203
+ ):
1204
+ for f_evt in device_corr_map[fe.id]:
1205
+ if f_evt.device_type in [
1206
+ DeviceType.CUDA,
1207
+ DeviceType.PrivateUse1,
1208
+ DeviceType.XPU,
1209
+ ]:
1210
+ fe.append_kernel(
1211
+ f_evt.name,
1212
+ f_evt.device_index,
1213
+ f_evt.time_range.end - f_evt.time_range.start,
1214
+ )
1215
+ elif f_evt.device_type == DeviceType.CPU:
1216
+ # make sure that 'thread' of a CPU Kineto (e.g. Device Runtime) event is associated
1217
+ # with the 'thread' of the corresponding linked PyTorch event to properly track
1218
+ # parents and children
1219
+ f_evt.thread = fe.thread
1220
+
1221
+ def _create_function_event_for_memory_events(evt):
1222
+ rel_start_ns = evt.start_ns() - trace_start_ns
1223
+ fe = FunctionEvent(
1224
+ id=max_evt_id,
1225
+ name=evt.name(),
1226
+ overload_name="",
1227
+ trace_name=None, # not outputting in the trace
1228
+ thread=evt.start_thread_id(),
1229
+ start_us=rel_start_ns / 1000,
1230
+ end_us=rel_start_ns / 1000, # no duration
1231
+ fwd_thread=evt.start_thread_id(),
1232
+ input_shapes=[],
1233
+ stack=[],
1234
+ scope=0, # RecordScope::FUNCTION
1235
+ use_device=self.use_device,
1236
+ cpu_memory_usage=_cpu_memory_usage(evt),
1237
+ device_memory_usage=_device_memory_usage(evt),
1238
+ is_async=False,
1239
+ sequence_nr=-1,
1240
+ device_type=DeviceType.CPU,
1241
+ device_index=0,
1242
+ )
1243
+ return fe
1244
+
1245
+ # output top-level memory events
1246
+ for mem_record in mem_records:
1247
+ if _check_timeout():
1248
+ break
1249
+
1250
+ if not mem_record[1]:
1251
+ max_evt_id += 1
1252
+ fe = _create_function_event_for_memory_events(mem_record[0])
1253
+ all_function_events.append(fe)
1254
+
1255
+ for oom_record in oom_records:
1256
+ if _check_timeout():
1257
+ break
1258
+
1259
+ max_evt_id += 1
1260
+ fe = _create_function_event_for_memory_events(oom_record)
1261
+ all_function_events.append(fe)
1262
+
1263
+ if timed_out:
1264
+ log.warning(
1265
+ "Profiler _parse_kineto_results timed out after %.3f seconds, "
1266
+ "returning partial results with %d events",
1267
+ timeout_s,
1268
+ len(all_function_events),
1269
+ )
1270
+
1271
+ all_function_events.sort(
1272
+ key=lambda evt: [evt.time_range.start, -evt.time_range.end]
1273
+ )
1274
+ return all_function_events
1275
+
1276
+
1277
+ # Set by torch.profiler to the active cupti_monitor ProfilerObserver while a session is
1278
+ # running (None otherwise). record_function routes regions to it via push/pop_annotation.
1279
+ # Held as an opaque object -- NOT imported from the cupti package -- so record_function never
1280
+ # pulls in the cupti chain on a non-cupti run, and there is a single "is a session active"
1281
+ # signal (this reference) rather than a separate flag.
1282
+ _active_cupti_profiler_observer: Any = None
1283
+
1284
+
1285
+ def _set_active_cupti_profiler_observer(observer: Any) -> None:
1286
+ global _active_cupti_profiler_observer
1287
+ _active_cupti_profiler_observer = observer
1288
+
1289
+
1290
+ class record_function(_ContextDecorator): # pyrefly: ignore [invalid-inheritance]
1291
+ """Context manager/function decorator that adds a label to a code block/function when running autograd profiler.
1292
+ Label will only appear if CPU activity tracing is enabled.
1293
+
1294
+ It is useful when tracing the code profile.
1295
+
1296
+ Args:
1297
+ name (str): Label assigned to the block of code.
1298
+ node_id (int): ID of node, for distributed profiling. Unset in
1299
+ non-distributed cases.
1300
+
1301
+ Example:
1302
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER)
1303
+ >>> x = torch.randn((1, 1), requires_grad=True)
1304
+ >>> with torch.autograd.profiler.profile() as prof:
1305
+ ... y = x**2
1306
+ ... with torch.autograd.profiler.record_function(
1307
+ ... "label-z"
1308
+ ... ): # label the block
1309
+ ... z = y**3
1310
+ ... y.backward()
1311
+ >>> # xdoctest: +IGNORE_WANT
1312
+ >>> # NOTE: some columns were removed for brevity
1313
+ >>> print(prof.key_averages().table(sort_by="self_cpu_time_total"))
1314
+ ----------------------------------- --------------- --------------- ---------------
1315
+ Name Self CPU total % CPU time avg Number of Calls
1316
+ ----------------------------------- --------------- --------------- ---------------
1317
+ pow 60.77% 47.470us 3
1318
+ mul 21.73% 25.465us 2
1319
+ PowBackward0 12.03% 121.891us 1
1320
+ torch::autograd::AccumulateGrad 2.70% 6.324us 1
1321
+ label-z 2.13% 12.421us 1
1322
+ torch::autograd::GraphRoot 0.64% 1.503us 1
1323
+ ----------------------------------- --------------- --------------- ---------------
1324
+ Self CPU time total: 234.344us
1325
+ CUDA time total: 0.000us
1326
+
1327
+ """
1328
+
1329
+ def __init__(self, name: str, args: str | None = None):
1330
+ self.name: str = name
1331
+ self.args: str | None = args
1332
+ # Whether or not we should run record function's end callbacks when exiting.
1333
+ self.run_callbacks_on_exit: bool = True
1334
+ # TODO: TorchScript ignores standard type annotation here
1335
+ # self.record: Optional["torch.classes.profiler._RecordFunction"] = None
1336
+ self.record = torch.jit.annotate(
1337
+ # pyrefly: ignore [not-a-type]
1338
+ Optional["torch.classes.profiler._RecordFunction"],
1339
+ None,
1340
+ )
1341
+ self._cupti_monitor_external_id: int | None = None
1342
+
1343
+ def __enter__(self):
1344
+ self.record = torch.ops.profiler._record_function_enter_new(
1345
+ self.name, self.args
1346
+ )
1347
+ # Route the region to the active cupti_monitor observer, if any. The reference is
1348
+ # None unless a cupti_monitor profile is running, so a non-cupti run never touches
1349
+ # the cupti chain. Guarded by is_scripting() (the global access doesn't compile under
1350
+ # TorchScript), and the global is read inside the guard so it is dead-code-eliminated.
1351
+ if not torch.jit.is_scripting():
1352
+ observer = _active_cupti_profiler_observer
1353
+ if observer is not None:
1354
+ self._cupti_monitor_external_id = observer.push_annotation(self.name)
1355
+ return self
1356
+
1357
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any):
1358
+ if not torch.jit.is_scripting():
1359
+ if self._cupti_monitor_external_id is not None:
1360
+ observer = _active_cupti_profiler_observer
1361
+ if observer is not None:
1362
+ observer.pop_annotation()
1363
+ self._cupti_monitor_external_id = None
1364
+ if not self.run_callbacks_on_exit:
1365
+ return
1366
+
1367
+ # Local variable is needed by TorchScript to refine Optional[T] to T
1368
+ record = self.record
1369
+ if record is None:
1370
+ raise AssertionError("Expected record to be set")
1371
+
1372
+ # TODO: Too slow with __torch_function__ handling enabled
1373
+ # See https://github.com/pytorch/pytorch/issues/76410
1374
+ if not torch.jit.is_scripting():
1375
+ with torch._C.DisableTorchFunctionSubclass():
1376
+ torch.ops.profiler._record_function_exit._RecordFunction(record)
1377
+ else:
1378
+ torch.ops.profiler._record_function_exit(record)
1379
+
1380
+ def _call_end_callbacks_on_future(self, fut: Future[Any]) -> Future[Any]:
1381
+ """Use for profiling async calls that return a future.
1382
+
1383
+ Calling this function will extend recording beyond this scope, until the future is
1384
+ satisfied. It is useful for profiling the end to end time of asynchronous calls.
1385
+ This function should only be called once to attach the callback onto the future, and
1386
+ will throw if called multiple times.
1387
+
1388
+ Args:
1389
+ fut: (torch._C.Future): future for which to schedule
1390
+ callback for.
1391
+
1392
+ Returns:
1393
+ A future that completes with the value of the passed in future when
1394
+ the profiling callbacks have ran.
1395
+
1396
+ """
1397
+ # Throw if we have already attached a callback onto the future.
1398
+ if not self.run_callbacks_on_exit:
1399
+ raise RuntimeError("_call_end_callbacks_on_future can only be called once.")
1400
+
1401
+ # We are scheduling to run this RecordFunction's end callbacks when the
1402
+ # passed in future completes, so don't run end callbacks on exit.
1403
+ self.run_callbacks_on_exit = False
1404
+
1405
+ # Local variable is needed by TorchScript to refine Optional[T] to T
1406
+ record = self.record
1407
+ if record is None:
1408
+ raise AssertionError("Expected record to be set")
1409
+
1410
+ # TODO: Too slow with __torch_function__ handling enabled
1411
+ # See https://github.com/pytorch/pytorch/issues/76410
1412
+ if not torch.jit.is_scripting():
1413
+ with torch._C.DisableTorchFunctionSubclass():
1414
+ profiled_future = (
1415
+ torch.ops.profiler._call_end_callbacks_on_jit_fut._RecordFunction(
1416
+ record, fut
1417
+ )
1418
+ )
1419
+ else:
1420
+ profiled_future = torch.ops.profiler._call_end_callbacks_on_jit_fut(
1421
+ record, fut
1422
+ )
1423
+ return profiled_future
1424
+
1425
+
1426
+ class emit_itt:
1427
+ """Context manager that makes every autograd operation emit an ITT range.
1428
+
1429
+ It is useful when running the program under Intel(R) VTune Profiler::
1430
+
1431
+ vtune <--vtune-flags> <regular command here>
1432
+
1433
+ The Instrumentation and Tracing Technology (ITT) API enables your application to generate and
1434
+ control the collection of trace data during its execution across different Intel tools.
1435
+ This context manager is to annotate Intel(R) VTune Profiling trace. With help of this context manager,
1436
+ you will be able to see labeled ranges in Intel(R) VTune Profiler GUI.
1437
+
1438
+ .. warning:
1439
+ This context manager should not be called recursively, i.e. at most one
1440
+ instance should be enabled at any given time.
1441
+
1442
+ Args:
1443
+ enabled (bool, optional): Setting ``enabled=False`` makes this context manager a no-op.
1444
+ Default: ``True``.
1445
+ record_shapes (bool, optional): If ``record_shapes=True``, the itt range wrapping
1446
+ each autograd op will append information about the sizes of Tensor arguments received
1447
+ by that op, in the following format:
1448
+ ``[[arg0.size(0), arg0.size(1), ...], [arg1.size(0), arg1.size(1), ...], ...]``
1449
+ Non-tensor arguments will be represented by ``[]``.
1450
+ Arguments will be listed in the order they are received by the backend op.
1451
+ Please note that this order may not match the order in which those arguments were passed
1452
+ on the Python side. Also note that shape recording may increase the overhead of itt range creation.
1453
+ Default: ``False``
1454
+
1455
+ Example:
1456
+ >>> # xdoctest: +SKIP("Undefined variables")
1457
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER)
1458
+ >>> with torch.autograd.profiler.emit_itt():
1459
+ ... model(x)
1460
+
1461
+ """
1462
+
1463
+ def __init__(self, enabled=True, record_shapes=False):
1464
+ self.enabled = enabled
1465
+ self.entered = False
1466
+ self.record_shapes = record_shapes
1467
+
1468
+ def __enter__(self):
1469
+ if not self.enabled:
1470
+ return
1471
+ if self.entered:
1472
+ raise RuntimeError("ITT annotation context manager is not reentrant")
1473
+ self.entered = True
1474
+ _run_on_profiler_start()
1475
+ _enable_profiler(
1476
+ ProfilerConfig(
1477
+ ProfilerState.ITT,
1478
+ self.record_shapes,
1479
+ False,
1480
+ False,
1481
+ False,
1482
+ False,
1483
+ _ExperimentalConfig(),
1484
+ ),
1485
+ set(),
1486
+ )
1487
+ return self
1488
+
1489
+ def __exit__(self, exc_type, exc_val, exc_tb):
1490
+ if not self.enabled:
1491
+ return
1492
+ _disable_profiler()
1493
+ _run_on_profiler_stop()
1494
+ return False
1495
+
1496
+
1497
+ class emit_nvtx:
1498
+ """Context manager that makes every autograd operation emit an NVTX range.
1499
+
1500
+ It is useful when running the program under nvprof::
1501
+
1502
+ nvprof --profile-from-start off -o trace_name.prof -- <regular command here>
1503
+
1504
+ Unfortunately, there's no way to force nvprof to flush the data it collected
1505
+ to disk, so for CUDA profiling one has to use this context manager to annotate
1506
+ nvprof traces and wait for the process to exit before inspecting them.
1507
+ Then, either NVIDIA Visual Profiler (nvvp) can be used to visualize the timeline, or
1508
+ :func:`torch.autograd.profiler.load_nvprof` can load the results for inspection
1509
+ e.g. in Python REPL.
1510
+
1511
+ .. warning:
1512
+ This context manager should not be called recursively, i.e. at most one
1513
+ instance should be enabled at any given time.
1514
+
1515
+ Args:
1516
+ enabled (bool, optional): Setting ``enabled=False`` makes this context manager a no-op.
1517
+ Default: ``True``.
1518
+ record_shapes (bool, optional): If ``record_shapes=True``, the nvtx range wrapping
1519
+ each autograd op will append information about the sizes of Tensor arguments received
1520
+ by that op, in the following format:
1521
+ ``[[arg0.size(0), arg0.size(1), ...], [arg1.size(0), arg1.size(1), ...], ...]``
1522
+ Non-tensor arguments will be represented by ``[]``.
1523
+ Arguments will be listed in the order they are received by the backend op.
1524
+ Please note that this order may not match the order in which those arguments were passed
1525
+ on the Python side. Also note that shape recording may increase the overhead of nvtx range creation.
1526
+ Default: ``False``
1527
+
1528
+ Example:
1529
+ >>> # xdoctest: +SKIP("undefined variables")
1530
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER)
1531
+ >>> with torch.cuda.profiler.profile():
1532
+ ... model(x) # Warmup CUDA memory allocator and profiler
1533
+ ... with torch.autograd.profiler.emit_nvtx():
1534
+ ... model(x)
1535
+
1536
+ **Forward-backward correlation**
1537
+
1538
+ When viewing a profile created using :class:`emit_nvtx` in the Nvidia Visual Profiler,
1539
+ correlating each backward-pass op with the corresponding forward-pass op can be difficult.
1540
+ To ease this task, :class:`emit_nvtx` appends sequence number information to the ranges it
1541
+ generates.
1542
+
1543
+ During the forward pass, each function range is decorated with ``seq=<N>``. ``seq`` is a running
1544
+ counter, incremented each time a new backward Function object is created and stashed for backward.
1545
+ Thus, the ``seq=<N>`` annotation associated with each forward function range tells you that
1546
+ if a backward Function object is created by this forward function,
1547
+ the backward object will receive sequence number N.
1548
+ During the backward pass, the top-level range wrapping each C++ backward Function's
1549
+ ``apply()`` call is decorated with ``stashed seq=<M>``. ``M`` is the sequence number that
1550
+ the backward object was created with. By comparing ``stashed seq`` numbers in backward with ``seq``
1551
+ numbers in forward, you can track down which forward op created each backward Function.
1552
+
1553
+ Any functions executed during the backward pass are also decorated with ``seq=<N>``. During
1554
+ default backward (with ``create_graph=False``) this information is irrelevant, and in fact,
1555
+ ``N`` may simply be 0 for all such functions. Only the top-level ranges associated with
1556
+ backward Function objects' ``apply()`` methods are useful, as a way to correlate these Function
1557
+ objects with the earlier forward pass.
1558
+
1559
+ **Double-backward**
1560
+
1561
+ If, on the other hand, a backward pass with ``create_graph=True`` is underway (in other words,
1562
+ if you are setting up for a double-backward), each function's execution during backward
1563
+ is given a nonzero, useful ``seq=<N>``. Those functions may themselves create Function objects
1564
+ to be executed later during double-backward, just as the original functions in the forward pass did.
1565
+ The relationship between backward and double-backward is conceptually the same as the relationship
1566
+ between forward and backward: The functions still emit current-sequence-number-tagged ranges,
1567
+ the Function objects they create still stash those sequence numbers, and during the eventual
1568
+ double-backward, the Function objects' ``apply()`` ranges are still tagged with ``stashed seq``
1569
+ numbers, which can be compared to `seq` numbers from the backward pass.
1570
+
1571
+ .. warning:
1572
+ The sequence number is thread-local, and some forward functions don't create an associated
1573
+ backward Function object (instead delegating that to sub-functions further down the call chain).
1574
+ For these reasons, the correspondence of stashed sequence numbers in
1575
+ backward Function ``apply()`` ranges with `seq` numbers in forward-pass ranges is
1576
+ not guaranteed to be 1 to 1. The sequence numbers alone may not be enough to fully
1577
+ disambiguate which forward function created which
1578
+ backward Function object. You may need to make a judgment based on analytic knowledge of what
1579
+ the expected correspondence should be.
1580
+ """
1581
+
1582
+ def __init__(self, enabled=True, record_shapes=False):
1583
+ self.enabled = enabled
1584
+ self.entered = False
1585
+ self.record_shapes = record_shapes
1586
+
1587
+ def __enter__(self):
1588
+ if not self.enabled:
1589
+ return
1590
+ if self.entered:
1591
+ raise RuntimeError("NVTX annotation context manager is not reentrant")
1592
+ self.entered = True
1593
+ torch.cuda.synchronize()
1594
+ _run_on_profiler_start()
1595
+ _enable_profiler(
1596
+ ProfilerConfig(
1597
+ ProfilerState.NVTX,
1598
+ self.record_shapes,
1599
+ False,
1600
+ False,
1601
+ False,
1602
+ False,
1603
+ _ExperimentalConfig(),
1604
+ ),
1605
+ set(),
1606
+ )
1607
+ return self
1608
+
1609
+ def __exit__(self, exc_type, exc_val, exc_tb):
1610
+ if not self.enabled:
1611
+ return
1612
+ torch.cuda.synchronize()
1613
+ _disable_profiler()
1614
+ _run_on_profiler_stop()
1615
+ return False
1616
+
1617
+
1618
+ def load_nvprof(path):
1619
+ """Open an nvprof trace file and parses autograd annotations.
1620
+
1621
+ Args:
1622
+ path (str): path to nvprof trace
1623
+ """
1624
+ return EventList(parse_nvprof_trace(path))
1625
+
1626
+
1627
+ class EnforceUnique:
1628
+ """Raises an error if a key is seen more than once."""
1629
+
1630
+ def __init__(self):
1631
+ self.seen = set()
1632
+
1633
+ def see(self, *key):
1634
+ r"""
1635
+ Observe a key and raise an error if it is seen multiple times.
1636
+ """
1637
+ if key in self.seen:
1638
+ raise RuntimeError("duplicate key: " + str(key))
1639
+ self.seen.add(key)
1640
+
1641
+
1642
+ def parse_nvprof_trace(path):
1643
+ import sqlite3
1644
+
1645
+ conn = sqlite3.connect(path)
1646
+ conn.row_factory = sqlite3.Row
1647
+
1648
+ # Parse strings table
1649
+ strings = {}
1650
+ for r in conn.execute("SELECT _id_ as id, value FROM StringTable"):
1651
+ strings[r["id"]] = torch._C._demangle(r["value"])
1652
+
1653
+ # First, find all functions and create FunctionEvents for them
1654
+ marker_query = """
1655
+ SELECT
1656
+ start.id AS marker_id, start.name, start.timestamp AS start_time, end.timestamp AS end_time
1657
+ FROM
1658
+ CUPTI_ACTIVITY_KIND_MARKER AS start INNER JOIN CUPTI_ACTIVITY_KIND_MARKER AS end
1659
+ ON start.id = end.id
1660
+ WHERE
1661
+ start.name != 0 AND end.name = 0
1662
+ """
1663
+ functions = []
1664
+ functions_map = {}
1665
+ unique = EnforceUnique()
1666
+ for row in conn.execute(marker_query):
1667
+ unique.see(row["marker_id"])
1668
+ evt = FunctionEvent(
1669
+ id=row["marker_id"],
1670
+ node_id=0, # missing a node_id when calling FunctionEvent. This is just to ensure
1671
+ # that pytorch doesn't crash when creating a FunctionEvent() object
1672
+ name=strings[row["name"]],
1673
+ start_us=row["start_time"],
1674
+ end_us=row["end_time"],
1675
+ thread=0,
1676
+ ) # TODO: find in sqlite database
1677
+ functions.append(evt)
1678
+ functions_map[evt.id] = evt
1679
+
1680
+ # Now, correlate all kernels with FunctionEvents
1681
+ kernel_query = """
1682
+ SELECT
1683
+ start.id AS marker_id, start.name, start.timestamp, end.timestamp,
1684
+ runtime._id_ AS runtime_id, runtime.cbid, runtime.start AS runtime_start, runtime.end AS runtime_end,
1685
+ kernel.start AS kernel_start, kernel.end AS kernel_end, kernel.name AS kernel_name
1686
+ FROM
1687
+ CUPTI_ACTIVITY_KIND_MARKER AS start
1688
+ INNER JOIN CUPTI_ACTIVITY_KIND_MARKER AS end
1689
+ ON start.id = end.id
1690
+ INNER JOIN CUPTI_ACTIVITY_KIND_RUNTIME as runtime
1691
+ ON (start.timestamp < runtime.start AND runtime.end < end.timestamp)
1692
+ INNER JOIN CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL AS kernel
1693
+ ON kernel.correlationId = runtime.correlationId
1694
+ """
1695
+ unique = EnforceUnique()
1696
+ for row in conn.execute(kernel_query):
1697
+ unique.see(row["marker_id"], row["runtime_id"])
1698
+ # 211 is cudaKernelLaunch for cuda >= 9.2
1699
+ if row["cbid"] != 211:
1700
+ raise AssertionError(f"Expected cbid to be 211, but got {row['cbid']}")
1701
+ evt = functions_map[row["marker_id"]]
1702
+ evt.append_kernel(
1703
+ row["kernel_name"], 0, row["kernel_end"] - row["kernel_start"]
1704
+ )
1705
+
1706
+ functions.sort(key=lambda evt: evt.time_range.start)
1707
+ return functions
1708
+
1709
+
1710
+ class KinetoStepTracker:
1711
+ """Provides an abstraction for incrementing the step count globally.
1712
+
1713
+ Previously, we only had one place to mark that a step() has occurred
1714
+ in the program via pytorch profiler step(). We will now add step hooks
1715
+ in the Optimizer class https://github.com/pytorch/pytorch/issues/88446
1716
+
1717
+ - This could mean programs that already call profiler.step() every
1718
+ iteration can end up double incrementing step count.
1719
+ - If a model uses multiple optimizers we can also have double or more
1720
+ counting of the step.
1721
+
1722
+ We fix this by adding a layer of abstraction before calling step()
1723
+ to the kineto library. The idea is to maintain steps per requester in a dict:
1724
+
1725
+ .. code-block::
1726
+
1727
+ {
1728
+ "ProfilerStep": 100, # triggered by profiler step() call
1729
+ "Optimizer1Step": 100, # Optimizer 1 or 2 are just examples, could be SGD, Adam etc
1730
+ "Optimizer2Step": 100,
1731
+ }
1732
+
1733
+ To figure out the global step count just take the max of dict values (100).
1734
+
1735
+ If one of the count increments the max will go up.
1736
+
1737
+ .. code-block::
1738
+
1739
+ {
1740
+ "ProfilerStep": 100,
1741
+ "Optimizer1Step": 101, # Optimizer1 got incremented first say
1742
+ "Optimizer2Step": 100,
1743
+ }
1744
+
1745
+ Then global step count is 101
1746
+ We only call the kineto step() function when global count increments.
1747
+
1748
+ NOTE: Please do not use the KinetoStepTracker in modules beside the Optimizer
1749
+ for now. The result could be incorrect increments of the step count.
1750
+ """
1751
+
1752
+ _current_step = 0
1753
+ _step_dict: dict[str, int] = defaultdict(int)
1754
+
1755
+ @classmethod
1756
+ def init_step_count(cls, requester: str):
1757
+ r"""
1758
+ Initialize for a given requester.
1759
+ """
1760
+ cls._step_dict[requester] = cls._current_step
1761
+
1762
+ @classmethod
1763
+ def erase_step_count(cls, requester: str) -> bool:
1764
+ r"""
1765
+ Remove a given requester.
1766
+ """
1767
+ return cls._step_dict.pop(requester, None) is not None
1768
+
1769
+ @classmethod
1770
+ def increment_step(cls, requester: str) -> int:
1771
+ """Increments the step count for the requester.
1772
+
1773
+ Additionally if the max over all step counts has incremented then
1774
+ trigger the _kineto_step() returns global step count
1775
+ """
1776
+ if requester not in cls._step_dict:
1777
+ cls.init_step_count(requester)
1778
+ cls._step_dict[requester] += 1
1779
+
1780
+ new_step = max(cls._step_dict.values())
1781
+ if new_step > cls._current_step:
1782
+ delta = new_step - cls._current_step
1783
+ if delta > 1:
1784
+ warn(
1785
+ "Profiler step count has increased more than 1 - "
1786
+ f"current_step = {cls._current_step} step dict = {cls._step_dict}",
1787
+ stacklevel=2,
1788
+ )
1789
+ for _ in range(delta):
1790
+ _kineto_step()
1791
+ cls._current_step = new_step
1792
+ return cls._current_step
1793
+
1794
+ @classmethod
1795
+ def current_step(cls) -> int:
1796
+ r"""
1797
+ Get the latest step for any requester
1798
+ """
1799
+ return cls._current_step
western_ai/usa/pytorch/torch/autograd/profiler_legacy.py ADDED
@@ -0,0 +1,327 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import itertools
3
+ import warnings
4
+ from typing_extensions import deprecated
5
+
6
+ import torch
7
+ import torch.cuda
8
+ from torch.autograd import (
9
+ _disable_profiler_legacy,
10
+ _enable_profiler_legacy,
11
+ DeviceType,
12
+ ProfilerConfig,
13
+ ProfilerState,
14
+ )
15
+ from torch.autograd.profiler_util import (
16
+ _filter_name,
17
+ _filter_stack_entry,
18
+ _rewrite_name,
19
+ EventList,
20
+ FunctionEvent,
21
+ MEMORY_EVENT_NAME,
22
+ )
23
+
24
+
25
+ __all__ = ["profile"]
26
+
27
+
28
+ @deprecated(
29
+ "`torch.autograd.profiler_legacy.profile` is deprecated and will be removed in a future release. "
30
+ "Please use `torch.profiler` instead.",
31
+ category=None, # TODO: change to `FutureWarning`
32
+ )
33
+ class profile:
34
+ """DEPRECATED: use torch.profiler instead."""
35
+
36
+ def __init__(
37
+ self,
38
+ enabled=True,
39
+ *,
40
+ use_cuda=False,
41
+ record_shapes=False,
42
+ with_flops=False,
43
+ profile_memory=False,
44
+ with_stack=False,
45
+ with_modules=False,
46
+ ):
47
+ self.enabled: bool = enabled
48
+ if not self.enabled:
49
+ return
50
+ self.use_cuda = use_cuda
51
+ self.function_events = None
52
+ self.entered = False
53
+ self.record_shapes = record_shapes
54
+ self.with_flops = with_flops
55
+ self.record_shapes |= self.with_flops
56
+ self.profile_memory = profile_memory
57
+ self.with_stack = with_stack
58
+ self.with_modules = with_modules
59
+
60
+ if self.use_cuda and not torch.cuda.is_available():
61
+ warnings.warn(
62
+ "CUDA is not available, disabling CUDA profiling",
63
+ stacklevel=2,
64
+ )
65
+ self.use_cuda = False
66
+
67
+ if self.use_cuda:
68
+ self.profiler_kind = ProfilerState.CUDA
69
+ else:
70
+ self.profiler_kind = ProfilerState.CPU
71
+
72
+ def config(self):
73
+ return ProfilerConfig(
74
+ self.profiler_kind,
75
+ self.record_shapes,
76
+ self.profile_memory,
77
+ self.with_stack,
78
+ self.with_flops,
79
+ self.with_modules,
80
+ # avoid exposing _ExperimentalConfig this in legacy public API
81
+ torch._C._profiler._ExperimentalConfig(),
82
+ )
83
+
84
+ def __enter__(self):
85
+ if not self.enabled:
86
+ return
87
+ if self.entered:
88
+ raise RuntimeError("Profiler context manager is not reentrant")
89
+ self.entered = True
90
+ self._start_trace()
91
+ return self
92
+
93
+ def _start_trace(self):
94
+ _enable_profiler_legacy(self.config())
95
+
96
+ def __exit__(self, exc_type, exc_val, exc_tb):
97
+ if not self.enabled:
98
+ return
99
+ if self.use_cuda:
100
+ torch.cuda.synchronize()
101
+
102
+ records = _disable_profiler_legacy()
103
+ parsed_results = _parse_legacy_records(records)
104
+ # pyrefly: ignore [bad-assignment]
105
+ self.function_events = EventList(
106
+ parsed_results,
107
+ use_device="cuda" if self.use_cuda else None,
108
+ profile_memory=self.profile_memory,
109
+ with_flops=self.with_flops,
110
+ )
111
+ # pyrefly: ignore [missing-attribute]
112
+ self.function_events._build_tree()
113
+ return False
114
+
115
+ def __repr__(self):
116
+ if self.function_events is None:
117
+ return "<unfinished profiler_legacy.profile>"
118
+ return repr(self.function_events)
119
+
120
+ def __str__(self):
121
+ if self.function_events is None:
122
+ return "<unfinished profile.profiler_legacy.profile>"
123
+ return str(self.function_events)
124
+
125
+ def _check_finish(self):
126
+ if self.function_events is None:
127
+ raise RuntimeError("Profiler didn't finish running")
128
+
129
+ def table(
130
+ self,
131
+ sort_by=None,
132
+ row_limit=100,
133
+ max_src_column_width=75,
134
+ max_name_column_width=55,
135
+ max_shapes_column_width=80,
136
+ header=None,
137
+ top_level_events_only=False,
138
+ ):
139
+ self._check_finish()
140
+ if self.function_events is None:
141
+ raise AssertionError("Expected profiling results")
142
+ return self.function_events.table(
143
+ sort_by=sort_by,
144
+ row_limit=row_limit,
145
+ max_src_column_width=max_src_column_width,
146
+ max_name_column_width=max_name_column_width,
147
+ max_shapes_column_width=max_shapes_column_width,
148
+ header=header,
149
+ top_level_events_only=top_level_events_only,
150
+ )
151
+
152
+ table.__doc__ = EventList.table.__doc__
153
+
154
+ def export_chrome_trace(self, path):
155
+ self._check_finish()
156
+ if self.function_events is None:
157
+ raise AssertionError("Expected profiling results")
158
+ return self.function_events.export_chrome_trace(path)
159
+
160
+ export_chrome_trace.__doc__ = EventList.export_chrome_trace.__doc__
161
+
162
+ def export_stacks(self, path: str, metric: str = "self_cpu_time_total"):
163
+ self._check_finish()
164
+ if self.function_events is None:
165
+ raise AssertionError("Expected profiling results")
166
+ if not self.with_stack:
167
+ raise AssertionError("export_stacks() requires with_stack=True")
168
+ return self.function_events.export_stacks(path, metric)
169
+
170
+ def key_averages(self, group_by_input_shape=False, group_by_stack_n=0):
171
+ self._check_finish()
172
+ if self.function_events is None:
173
+ raise AssertionError("Expected profiling results")
174
+ return self.function_events.key_averages(group_by_input_shape, group_by_stack_n)
175
+
176
+ key_averages.__doc__ = EventList.key_averages.__doc__
177
+
178
+ def total_average(self):
179
+ self._check_finish()
180
+ if self.function_events is None:
181
+ raise AssertionError("Expected profiling results")
182
+ return self.function_events.total_average()
183
+
184
+ total_average.__doc__ = EventList.total_average.__doc__
185
+
186
+ @property
187
+ def self_cpu_time_total(self):
188
+ """Return CPU time as the sum of self times across all events."""
189
+ self._check_finish()
190
+ if self.function_events is None:
191
+ raise AssertionError("Expected profiling results")
192
+ return self.function_events.self_cpu_time_total
193
+
194
+
195
+ def _parse_legacy_records(thread_records):
196
+ def _get_record_key(record):
197
+ """Return a tuple for correlating start and end records in `_parse_legacy_records`."""
198
+ return (record.handle(), record.node_id())
199
+
200
+ start_record = None
201
+ functions = []
202
+
203
+ # '__start_profile' is not guaranteed to be first, so we must find it here
204
+ for record in itertools.chain.from_iterable(thread_records):
205
+ name = record.name()
206
+ if start_record is None and name == "__start_profile":
207
+ start_record = record
208
+
209
+ if start_record is None or start_record.is_remote():
210
+ raise AssertionError("Expected a valid local start_record")
211
+
212
+ for thread_record_list in thread_records:
213
+ # accumulated memory allocations per handle
214
+ cpu_memory_allocs = {}
215
+ cuda_memory_allocs = {}
216
+ # ranges per handle
217
+ range_starts = {}
218
+
219
+ filtered_handles = set()
220
+ prev_record = None
221
+ for record in thread_record_list:
222
+ record_key = _get_record_key(record)
223
+ if _filter_name(record.name()) or record_key in filtered_handles:
224
+ filtered_handles.add(record_key)
225
+ continue
226
+
227
+ if record.kind() == "push":
228
+ # workaround to reduce double logging from operator
229
+ # wrappers and redispatch
230
+ if prev_record is not None:
231
+ duplicate = (
232
+ prev_record.name() == record.name()
233
+ and prev_record.kind() == record.kind()
234
+ and prev_record.node_id() == record.node_id()
235
+ )
236
+ if duplicate:
237
+ filtered_handles.add(record_key)
238
+ continue
239
+
240
+ range_starts[record_key] = record
241
+ cpu_memory_allocs[record_key] = 0
242
+ cuda_memory_allocs[record_key] = 0
243
+ elif record.kind() == "pop":
244
+ if record_key not in range_starts:
245
+ raise AssertionError(
246
+ f"Expected record with key {record_key} to exist in range_starts. "
247
+ "This means that the pop event did not have a corresponding push."
248
+ )
249
+
250
+ start = range_starts[record_key]
251
+
252
+ cpu_memory_usage = cpu_memory_allocs[record_key]
253
+ cuda_memory_usage = cuda_memory_allocs[record_key]
254
+ is_async = start.is_async() or (start.thread_id() != record.thread_id())
255
+ is_remote_event = record.is_remote()
256
+ start_flops = start.flops()
257
+
258
+ fe = FunctionEvent(
259
+ id=record.handle(),
260
+ node_id=record.node_id(),
261
+ name=_rewrite_name(name=start.name(), with_wildcard=True),
262
+ trace_name=_rewrite_name(name=start.name(), with_wildcard=False),
263
+ thread=start.thread_id(),
264
+ start_us=start_record.cpu_elapsed_us(start),
265
+ end_us=start_record.cpu_elapsed_us(record),
266
+ fwd_thread=start.fwd_thread_id(),
267
+ input_shapes=start.shapes(),
268
+ stack=[
269
+ entry for entry in start.stack() if _filter_stack_entry(entry)
270
+ ],
271
+ scope=start.scope(),
272
+ use_device="cuda" if start.has_cuda() else None,
273
+ cpu_memory_usage=cpu_memory_usage,
274
+ device_memory_usage=cuda_memory_usage,
275
+ is_async=is_async,
276
+ is_remote=is_remote_event,
277
+ sequence_nr=start.sequence_nr(),
278
+ device_type=DeviceType.CPU,
279
+ is_legacy=True,
280
+ flops=start_flops,
281
+ )
282
+ # note: async events have only cpu total time
283
+ if not is_async and start.has_cuda():
284
+ duration = start.cuda_elapsed_us(record)
285
+ if duration > 0:
286
+ fe.append_kernel(start.name(), start.device(), duration)
287
+ functions.append(fe)
288
+ del range_starts[record_key]
289
+ del cpu_memory_allocs[record_key]
290
+ del cuda_memory_allocs[record_key]
291
+ elif record.kind() == "memory_alloc":
292
+ num_open_handles_cpu = len(cpu_memory_allocs)
293
+ num_open_handles_cuda = len(cuda_memory_allocs)
294
+ if num_open_handles_cpu != num_open_handles_cuda:
295
+ raise AssertionError(
296
+ f"Expected CPU and CUDA memory allocation handles to match, "
297
+ f"but got {num_open_handles_cpu} CPU and {num_open_handles_cuda} CUDA"
298
+ )
299
+ for handle in cpu_memory_allocs:
300
+ cpu_memory_allocs[handle] += record.cpu_memory_usage()
301
+ for handle in cuda_memory_allocs:
302
+ cuda_memory_allocs[handle] += record.cuda_memory_usage()
303
+ if num_open_handles_cpu == 0:
304
+ # output event as a top-level memory event
305
+ fe = FunctionEvent(
306
+ id=0,
307
+ name=MEMORY_EVENT_NAME,
308
+ trace_name=None,
309
+ thread=0,
310
+ start_us=0,
311
+ end_us=0,
312
+ stack=[],
313
+ cpu_memory_usage=record.cpu_memory_usage(),
314
+ device_memory_usage=record.cuda_memory_usage(),
315
+ is_legacy=True,
316
+ )
317
+ functions.append(fe)
318
+ prev_record = record
319
+
320
+ # Sort functions by start time then by end time ascending.
321
+ # This ensures that--in the case of nested events which
322
+ # have the same start time (which may happen due to the
323
+ # granularity of the given clock tick)--we always show
324
+ # the outermost nested call first. This adds stability
325
+ # in how FunctionEvents appear
326
+ functions.sort(key=lambda evt: [evt.time_range.start, -evt.time_range.end])
327
+ return functions
western_ai/usa/pytorch/torch/autograd/profiler_util.py ADDED
@@ -0,0 +1,1617 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import bisect
3
+ import itertools
4
+ import json
5
+ import math
6
+ from collections import defaultdict, namedtuple
7
+ from collections.abc import Callable
8
+ from operator import attrgetter
9
+ from typing import Any, NamedTuple
10
+ from typing_extensions import deprecated
11
+
12
+ import torch
13
+ from torch.autograd import DeviceType
14
+
15
+
16
+ __all__ = [
17
+ "EventList",
18
+ "EventMetadata",
19
+ "FormattedTimesMixin",
20
+ "Interval",
21
+ "Kernel",
22
+ "FunctionEvent",
23
+ "FunctionEventAvg",
24
+ "StringTable",
25
+ "MemRecordsAcc",
26
+ ]
27
+
28
+
29
+ class EventList(list):
30
+ """A list of profiling events with helper methods for analysis and visualization.
31
+
32
+ EventList extends the standard Python list to provide specialized methods for
33
+ working with profiling events (FunctionEvent or FunctionEventAvg objects).
34
+ It includes utilities for aggregating statistics, formatting output tables,
35
+ and exporting profiling data.
36
+
37
+ This class is typically returned by profiler methods and should not be
38
+ instantiated directly by users.
39
+
40
+ Args:
41
+ *args: Standard list arguments.
42
+ use_device (str, optional): Device type for profiling ("cuda", "xpu", etc.).
43
+ profile_memory (bool, optional): Whether memory profiling was enabled. Default: False.
44
+ with_flops (bool, optional): Whether to include FLOP counts. Default: False.
45
+
46
+ Attributes:
47
+ _use_device (str): Device type being profiled.
48
+ _profile_memory (bool): Whether memory profiling is enabled.
49
+ _with_flops (bool): Whether FLOP counting is enabled.
50
+ _tree_built (bool): Whether the event tree structure has been built.
51
+
52
+ Key Methods:
53
+ table(...): Format events as a table string for display.
54
+ export_chrome_trace(path): Export to Chrome tracing format.
55
+ export_stacks(path, metric): Export stack traces with metrics.
56
+ key_averages(...): Compute averaged statistics grouped by operation name.
57
+ total_average(): Compute aggregate totals across all events (sums, not averages).
58
+
59
+ Properties:
60
+ self_cpu_time_total: Sum of self CPU time across all events.
61
+
62
+ Example::
63
+
64
+ import torch
65
+ from torch.profiler import profile, ProfilerActivity
66
+
67
+ with profile(activities=[ProfilerActivity.CPU]) as prof:
68
+ x = torch.randn(100, 100)
69
+ y = torch.matmul(x, x)
70
+
71
+ # EventList is returned by prof.events()
72
+ events = prof.events()
73
+
74
+ # Display as formatted table
75
+ print(
76
+ events.table(
77
+ sort_by="cpu_time_total", row_limit=20, top_level_events_only=False
78
+ )
79
+ )
80
+
81
+ # Export to Chrome tracing format
82
+ events.export_chrome_trace("trace.json")
83
+
84
+ # Get averaged statistics
85
+ avg_events = events.key_averages()
86
+ print(avg_events.table())
87
+
88
+ # Export stack traces
89
+ events.export_stacks("stacks.txt", "self_cpu_time_total")
90
+
91
+ See Also:
92
+ - :class:`FunctionEvent`: Individual profiling event
93
+ - :class:`FunctionEventAvg`: Averaged profiling statistics
94
+ - :meth:`table`: Format events as a readable table
95
+ - :meth:`key_averages`: Aggregate events by operation name
96
+ """
97
+
98
+ def __init__(self, *args, **kwargs):
99
+ use_device = kwargs.pop("use_device", None)
100
+ profile_memory = kwargs.pop("profile_memory", False)
101
+ with_flops = kwargs.pop("with_flops", False)
102
+ # pyrefly: ignore [not-iterable]
103
+ super().__init__(*args, **kwargs)
104
+ self._use_device = use_device
105
+ self._profile_memory = profile_memory
106
+ self._tree_built = False
107
+ self._with_flops = with_flops
108
+
109
+ def _build_tree(self):
110
+ self._populate_cpu_children()
111
+ self._remove_dup_nodes()
112
+ self._set_backward_stacktraces()
113
+ self._tree_built = True
114
+
115
+ def __str__(self):
116
+ return self.table()
117
+
118
+ def _remove_dup_nodes(self):
119
+ while True:
120
+ to_delete = set()
121
+
122
+ for idx in range(len(self)):
123
+ if (
124
+ self[idx].cpu_parent is not None
125
+ and self[idx].cpu_parent.name == self[idx].name
126
+ and len(self[idx].cpu_parent.cpu_children) == 1
127
+ ):
128
+ self[idx].cpu_parent.cpu_children = self[idx].cpu_children
129
+ self[idx].cpu_parent.kernels = self[idx].kernels # lift kernels up
130
+ for ch in self[idx].cpu_children:
131
+ ch.cpu_parent = self[idx].cpu_parent
132
+ to_delete.add(idx)
133
+ if len(to_delete) == 0:
134
+ break
135
+
136
+ new_evts = [ev for ind, ev in enumerate(self) if ind not in to_delete]
137
+
138
+ self.clear()
139
+
140
+ self.extend(new_evts)
141
+
142
+ def _populate_cpu_children(self):
143
+ """Populate child events into each underlying FunctionEvent object.
144
+
145
+ One event is a child of another if [s1, e1) is inside [s2, e2). Where
146
+ s1 and e1 would be start and end of the child event's interval. And
147
+ s2 and e2 start and end of the parent event's interval
148
+
149
+ Example: In event list [[0, 10], [1, 3], [3, 4]] would have make [0, 10]
150
+ be a parent of two other intervals.
151
+
152
+ If for any reason two intervals intersect only partially, this function
153
+ will not record a parent child relationship between then.
154
+ """
155
+ # Some events can be async (i.e. start and end on different threads),
156
+ # since it's generally undefined how to attribute children ranges to
157
+ # async ranges, we do not use them when calculating nested ranges and stats
158
+ sync_events = [
159
+ evt
160
+ for evt in self
161
+ if not evt.is_async and evt.device_type == DeviceType.CPU
162
+ ]
163
+ events = sorted(
164
+ sync_events,
165
+ key=attrgetter("thread"),
166
+ )
167
+ # Group by both thread and node_id, so that events that happen to have
168
+ # the same thread_id but are from different nodes aren't incorrectly
169
+ # grouped together.
170
+ threads = itertools.groupby(
171
+ events, key=lambda event: (event.thread, event.node_id)
172
+ )
173
+
174
+ # For each thread we keep a stack of current nested parents.
175
+ # We maintain the invariant that each interval is a subset of all other
176
+ # intervals lower in the stack.
177
+ #
178
+ # First we sort the intervals by their start time. Then we iterate over them.
179
+ # Every time we see a new interval we remove several parents from
180
+ # the top until we restore the invariant. Then parent child relationship
181
+ # if recorded if the stack is not empty.
182
+ # Finally we add new interval to the list
183
+ #
184
+ # Algorithm has O(N * log(N)) complexity where N is number of
185
+ # intervals
186
+ for _thread_id, thread_events in threads:
187
+ thread_events_ = sorted(
188
+ thread_events,
189
+ key=lambda event: [event.time_range.start, -event.time_range.end],
190
+ )
191
+ current_events: list[FunctionEvent] = []
192
+ for event in thread_events_:
193
+ while len(current_events) > 0:
194
+ parent = current_events[-1]
195
+ if (
196
+ event.time_range.start >= parent.time_range.end
197
+ or event.time_range.end > parent.time_range.end
198
+ ):
199
+ # this can't be a parent
200
+ current_events.pop()
201
+ else:
202
+ parent.append_cpu_child(event)
203
+ if event.cpu_parent is not None:
204
+ raise AssertionError(
205
+ f"There is already a CPU parent event for {event.key}"
206
+ )
207
+ event.set_cpu_parent(parent)
208
+ break
209
+
210
+ current_events.append(event)
211
+
212
+ def _set_backward_stacktraces(self):
213
+ def bw_parent(evt):
214
+ if evt is None:
215
+ return None
216
+ elif evt.scope == 1: # BACKWARD_FUNCTION
217
+ return evt
218
+ else:
219
+ return bw_parent(evt.cpu_parent)
220
+
221
+ fwd_stacks = {}
222
+ for evt in self:
223
+ if bw_parent(evt) is None and evt.stack is not None:
224
+ t = (evt.sequence_nr, evt.thread)
225
+ if t not in fwd_stacks:
226
+ fwd_stacks[t] = evt.stack
227
+
228
+ for evt in self:
229
+ p = bw_parent(evt)
230
+ if p is not None:
231
+ if p.fwd_thread is None:
232
+ raise AssertionError(
233
+ "Expected fwd_thread to be set for backward parent"
234
+ )
235
+ t = (p.sequence_nr, p.fwd_thread)
236
+ evt.stack = fwd_stacks.get(t, [])
237
+
238
+ @property
239
+ def self_cpu_time_total(self):
240
+ return sum(event.self_cpu_time_total for event in self)
241
+
242
+ def table(
243
+ self,
244
+ sort_by=None,
245
+ row_limit=100,
246
+ max_src_column_width=75,
247
+ max_name_column_width=55,
248
+ max_shapes_column_width=80,
249
+ header=None,
250
+ top_level_events_only=False,
251
+ time_unit=None,
252
+ ):
253
+ """Print an EventList as a nicely formatted table.
254
+
255
+ Args:
256
+ sort_by (str, optional): Attribute used to sort entries. By default
257
+ they are printed in the same order as they were registered.
258
+ Valid keys include: ``cpu_time``, ``cuda_time``, ``xpu_time``,
259
+ ``cpu_time_total``, ``cuda_time_total``, ``xpu_time_total``,
260
+ ``cpu_memory_usage``, ``cuda_memory_usage``, ``xpu_memory_usage``,
261
+ ``self_cpu_memory_usage``, ``self_cuda_memory_usage``,
262
+ ``self_xpu_memory_usage``, ``count``.
263
+ top_level_events_only(bool, optional): Boolean flag to determine the
264
+ selection of events to display. If true, the profiler will only
265
+ display events at top level like top-level invocation of python
266
+ `lstm`, python `add` or other functions, nested events like low-level
267
+ cpu/cuda/xpu ops events are omitted for profiler result readability.
268
+ time_unit(str, optional): A time unit to be used for all values in the
269
+ table. Valid options are: ``s``, ``ms`` and ``us``.
270
+
271
+ Returns:
272
+ A string containing the table.
273
+ """
274
+ return _build_table(
275
+ self,
276
+ sort_by=sort_by,
277
+ row_limit=row_limit,
278
+ max_src_column_width=max_src_column_width,
279
+ max_name_column_width=max_name_column_width,
280
+ max_shapes_column_width=max_shapes_column_width,
281
+ header=header,
282
+ profile_memory=self._profile_memory,
283
+ with_flops=self._with_flops,
284
+ top_level_events_only=top_level_events_only,
285
+ time_unit=time_unit,
286
+ )
287
+
288
+ def export_chrome_trace(self, path):
289
+ """Export an EventList as a Chrome tracing tools file.
290
+
291
+ The checkpoint can be later loaded and inspected under ``chrome://tracing`` URL.
292
+
293
+ Args:
294
+ path (str): Path where the trace will be written.
295
+ """
296
+ import os
297
+
298
+ device_name = "cuda" if not self._use_device else self._use_device
299
+ with open(path, "w") as f:
300
+ next_id = 0
301
+ # Use file IO over using json.dump since JSON dumping is very slow and
302
+ # this technique is proven to give a 4x speedup.
303
+ f.write("[")
304
+ for evt in self:
305
+ if evt.trace_name is None:
306
+ continue
307
+ f.write(
308
+ '{{"name": "{}", '
309
+ '"ph": "X", '
310
+ '"ts": {}, '
311
+ '"dur": {}, '
312
+ '"tid": {}, '
313
+ '"pid": "CPU functions", '
314
+ '"args": {{}}}}, '.format(
315
+ evt.trace_name,
316
+ evt.time_range.start,
317
+ evt.time_range.elapsed_us(),
318
+ evt.thread
319
+ if not evt.is_remote
320
+ else f'" node_id:{evt.node_id}, thread_id:{evt.thread} "',
321
+ )
322
+ )
323
+ for _ in evt.kernels:
324
+ # 's' and 'f' draw Flow arrows from
325
+ # the CPU launch to the GPU kernel
326
+ f.write(
327
+ f'{{"name": "{evt.trace_name}", '
328
+ '"ph": "s", '
329
+ f'"ts": {evt.time_range.start}, '
330
+ f'"tid": {evt.thread}, '
331
+ '"pid": "CPU functions", '
332
+ f'"id": {next_id}, '
333
+ f'"cat": "cpu_to_{device_name}", '
334
+ '"args": {}}, '
335
+ )
336
+ # Note: use torch.profiler to get device kernel trace
337
+ next_id += 1
338
+ if len(self) > 0:
339
+ # remove trailing whitespace and comma
340
+ f.seek(f.tell() - 2, os.SEEK_SET)
341
+ f.truncate()
342
+ f.write("]")
343
+
344
+ def supported_export_stacks_metrics(self):
345
+ return [
346
+ "self_cpu_time_total",
347
+ "self_cuda_time_total",
348
+ "self_xpu_time_total",
349
+ "self_privateuse1_time_total",
350
+ ]
351
+
352
+ def export_stacks(self, path: str, metric: str):
353
+ if metric not in self.supported_export_stacks_metrics():
354
+ raise ValueError(
355
+ "metric should be one of: "
356
+ + str(self.supported_export_stacks_metrics())
357
+ )
358
+ translate_table = str.maketrans(" ;\t\n", "____")
359
+ with open(path, "w") as f:
360
+ for evt in self:
361
+ if evt.stack and len(evt.stack) > 0:
362
+ metric_value = getattr(
363
+ evt,
364
+ metric.replace("cuda", "device")
365
+ .replace("xpu", "device")
366
+ .replace("privateuse1", "device"),
367
+ )
368
+ if int(metric_value) > 0:
369
+ stack_str = ""
370
+ for entry in reversed(evt.stack):
371
+ stack_str += entry.translate(translate_table)
372
+ stack_str += ";"
373
+ stack_str = stack_str[:-1] + " " + str(int(metric_value))
374
+ f.write(stack_str + "\n")
375
+
376
+ def key_averages(
377
+ self,
378
+ group_by_input_shapes=False,
379
+ group_by_stack_n=0,
380
+ group_by_overload_name=False,
381
+ include_python_functions=False,
382
+ ):
383
+ """Averages all function events over their keys.
384
+
385
+ Args:
386
+ group_by_input_shapes: group entries by
387
+ (event name, input shapes) rather than just event name.
388
+ This is useful to see which input shapes contribute to the runtime
389
+ the most and may help with size-specific optimizations or
390
+ choosing the best candidates for quantization (aka fitting a roof line)
391
+
392
+ group_by_stack_n: group by top n stack trace entries
393
+
394
+ group_by_overload_name: Differentiate operators by their overload name e.g. aten::add.Tensor
395
+ and aten::add.out will be aggregated separately
396
+
397
+ include_python_functions: include Python function events (e.g. individual
398
+ Python callsite entries captured with ``with_stack=True``) in the
399
+ averages. By default these are excluded because they tend to appear as
400
+ misleading hotspots (e.g. ``threading.py: wait``) that obscure the
401
+ real operator-level breakdown. Set to ``True`` to restore the raw
402
+ per-callsite view.
403
+
404
+ Returns:
405
+ An EventList containing FunctionEventAvg objects.
406
+ """
407
+ if not self._tree_built:
408
+ raise AssertionError(
409
+ "Expected tree to be built before calling key_averages"
410
+ )
411
+ stats: dict[tuple[str, ...], FunctionEventAvg] = defaultdict(FunctionEventAvg)
412
+
413
+ def get_key(
414
+ event, group_by_input_shapes, group_by_stack_n, group_by_overload_name
415
+ ) -> tuple[str, ...]:
416
+ key = [
417
+ str(event.key),
418
+ str(event.node_id),
419
+ str(event.device_type),
420
+ str(event.is_legacy),
421
+ str(event.is_user_annotation),
422
+ ]
423
+ if group_by_overload_name:
424
+ key.append(evt.overload_name)
425
+ if group_by_input_shapes:
426
+ key.append(str(event.input_shapes))
427
+ if group_by_stack_n > 0:
428
+ key += event.stack[:group_by_stack_n]
429
+ return tuple(key)
430
+
431
+ for evt in self:
432
+ if evt.is_python_function and not include_python_functions:
433
+ continue
434
+ stats[
435
+ get_key(
436
+ evt, group_by_input_shapes, group_by_stack_n, group_by_overload_name
437
+ )
438
+ ].add(evt)
439
+
440
+ avg_list = EventList(
441
+ stats.values(),
442
+ use_device=self._use_device,
443
+ profile_memory=self._profile_memory,
444
+ with_flops=self._with_flops,
445
+ )
446
+ for evt in avg_list:
447
+ evt.stack = evt.stack[:group_by_stack_n]
448
+ if not group_by_input_shapes:
449
+ evt.input_shapes = ""
450
+ if not group_by_overload_name:
451
+ evt.overload_name = ""
452
+ return avg_list
453
+
454
+ def total_average(self):
455
+ """Compute aggregate statistics across all events.
456
+
457
+ Accumulates statistics from all events into a single FunctionEventAvg object.
458
+ This is primarily useful for computing total metrics (total CPU time, total
459
+ memory usage, etc.) across the entire profiling session, regardless of
460
+ operation type.
461
+
462
+ Note:
463
+ This sums up times and counts across ALL different operations, so the
464
+ "average" metrics (like cpu_time) represent the average time per operation
465
+ call across the entire session, mixing all operation types together.
466
+ For per-operation averages, use :meth:`key_averages` instead.
467
+
468
+ Returns:
469
+ FunctionEventAvg: A single aggregate object with key="Total" containing
470
+ accumulated statistics.
471
+
472
+ """
473
+ total_stat = FunctionEventAvg()
474
+ for evt in self:
475
+ total_stat += evt
476
+ total_stat.key = None
477
+ total_stat.key = "Total"
478
+ return total_stat
479
+
480
+
481
+ def _format_time(time_us):
482
+ """Define how to format time in FunctionEvent."""
483
+ US_IN_SECOND = 1000.0 * 1000.0
484
+ US_IN_MS = 1000.0
485
+ if time_us >= US_IN_SECOND:
486
+ return f"{time_us / US_IN_SECOND:.3f}s"
487
+ if time_us >= US_IN_MS:
488
+ return f"{time_us / US_IN_MS:.3f}ms"
489
+ return f"{time_us:.3f}us"
490
+
491
+
492
+ def _format_time_share(time_us, total_time_us):
493
+ """Define how to format time in FunctionEvent."""
494
+ if total_time_us == 0:
495
+ if time_us != 0:
496
+ raise AssertionError(f"Expected time_us == 0 but got {time_us}")
497
+ return "NaN"
498
+ return f"{time_us * 100.0 / total_time_us:.2f}%"
499
+
500
+
501
+ def _format_memory(nbytes):
502
+ """Return a formatted memory size string."""
503
+ KB = 1024
504
+ MB = 1024 * KB
505
+ GB = 1024 * MB
506
+ if abs(nbytes) >= GB:
507
+ return f"{nbytes * 1.0 / GB:.2f} GB"
508
+ elif abs(nbytes) >= MB:
509
+ return f"{nbytes * 1.0 / MB:.2f} MB"
510
+ elif abs(nbytes) >= KB:
511
+ return f"{nbytes * 1.0 / KB:.2f} KB"
512
+ else:
513
+ return str(nbytes) + " B"
514
+
515
+
516
+ def _attr_formatter(name):
517
+ return property(lambda self: _format_time(getattr(self, name)))
518
+
519
+
520
+ class FormattedTimesMixin:
521
+ """Helpers for FunctionEvent and FunctionEventAvg.
522
+
523
+ The subclass should define `*_time_total` and `count` attributes.
524
+ """
525
+
526
+ cpu_time_str = _attr_formatter("cpu_time")
527
+ device_time_str = _attr_formatter("device_time")
528
+ cpu_time_total_str = _attr_formatter("cpu_time_total")
529
+ device_time_total_str = _attr_formatter("device_time_total")
530
+ self_cpu_time_total_str = _attr_formatter("self_cpu_time_total")
531
+ self_device_time_total_str = _attr_formatter("self_device_time_total")
532
+
533
+ @property
534
+ def cpu_time(self):
535
+ return 0.0 if self.count == 0 else 1.0 * self.cpu_time_total / self.count # type: ignore[attr-defined]
536
+
537
+ @property
538
+ def device_time(self):
539
+ return 0.0 if self.count == 0 else 1.0 * self.device_time_total / self.count # type: ignore[attr-defined]
540
+
541
+ @property
542
+ @deprecated(
543
+ "`cuda_time` is deprecated, please use `device_time` instead.",
544
+ category=FutureWarning,
545
+ )
546
+ def cuda_time(self): # To be deprecated
547
+ return self.device_time
548
+
549
+
550
+ class Interval:
551
+ def __init__(self, start, end):
552
+ self.start = start
553
+ self.end = end
554
+
555
+ def elapsed_us(self):
556
+ r"""
557
+ Returns the length of the interval
558
+ """
559
+ return self.end - self.start
560
+
561
+
562
+ Kernel = namedtuple("Kernel", ["name", "device", "duration"])
563
+
564
+
565
+ class EventMetadata(NamedTuple):
566
+ # Kernel fields
567
+ registers_per_thread: int | None
568
+ shared_memory: int | None
569
+ grid: list[int] | None
570
+ block: list[int] | None
571
+ priority: int | None
572
+ blocks_per_sm: float | None
573
+ warps_per_sm: float | None
574
+ occupancy: dict[str, Any] | None
575
+ est_occupancy_pct: float | None
576
+ queued: int | None
577
+ graph_id: int | None
578
+ graph_node_id: int | None
579
+ stream: int | None
580
+ context: int | None
581
+ channel: int | None
582
+ channel_type: int | None
583
+ # Memory fields
584
+ bytes: int | None
585
+ bandwidth_gb_s: float | None
586
+ # NCCL fields
587
+ collective_name: str | None
588
+ dtype: str | None
589
+ in_msg_nelems: int | None
590
+ out_msg_nelems: int | None
591
+ in_split_size: str | None
592
+ out_split_size: str | None
593
+ global_rank_start: int | None
594
+ global_rank_stride: int | None
595
+ group_size: int | None
596
+ process_group_name: str | None
597
+ process_group_desc: str | None
598
+ group_ranks: str | None
599
+ rank: int | None
600
+ src_rank: int | None
601
+ dst_rank: int | None
602
+ seq: int | None
603
+ is_async: bool | None
604
+
605
+
606
+ def _to_str(v: str) -> str:
607
+ return v.strip('"')
608
+
609
+
610
+ def _to_bool(v: str) -> bool:
611
+ return v in ("1", "true")
612
+
613
+
614
+ # Kineto key → (EventMetadata field name, converter from string)
615
+ _EVENT_METADATA_KEYS: dict[str, tuple[str, Callable[[str], Any]]] = {
616
+ "registers per thread": ("registers_per_thread", int),
617
+ "shared memory": ("shared_memory", int),
618
+ "grid": ("grid", json.loads), # list[int]
619
+ "block": ("block", json.loads), # list[int]
620
+ "priority": ("priority", int),
621
+ "blocks per SM": ("blocks_per_sm", float),
622
+ "warps per SM": ("warps_per_sm", float),
623
+ "est. achieved occupancy %": ("est_occupancy_pct", float),
624
+ "occupancy": ("occupancy", json.loads), # dict[str, Any]
625
+ "queued": ("queued", int),
626
+ "graph id": ("graph_id", int),
627
+ "graph node id": ("graph_node_id", int),
628
+ "stream": ("stream", int),
629
+ "context": ("context", int),
630
+ "channel": ("channel", int),
631
+ "channel_type": ("channel_type", int),
632
+ "bytes": ("bytes", int),
633
+ "memory bandwidth (GB/s)": ("bandwidth_gb_s", float),
634
+ "Collective name": ("collective_name", _to_str),
635
+ "dtype": ("dtype", _to_str),
636
+ "In msg nelems": ("in_msg_nelems", int),
637
+ "Out msg nelems": ("out_msg_nelems", int),
638
+ "In split size": ("in_split_size", _to_str),
639
+ "Out split size": ("out_split_size", _to_str),
640
+ "Global rank start": ("global_rank_start", int),
641
+ "Global rank stride": ("global_rank_stride", int),
642
+ "Group size": ("group_size", int),
643
+ "Process Group Name": ("process_group_name", _to_str),
644
+ "Process Group Description": ("process_group_desc", _to_str),
645
+ "Process Group Ranks": ("group_ranks", _to_str),
646
+ "Rank": ("rank", int),
647
+ "Src Rank": ("src_rank", int),
648
+ "Dst Rank": ("dst_rank", int),
649
+ "Seq": ("seq", int),
650
+ "Is asynchronized op": ("is_async", _to_bool),
651
+ }
652
+
653
+
654
+ def _build_metadata(extra_meta):
655
+ fields: dict[str, Any] = {}
656
+ any_populated = False
657
+ for kineto_key, (field_name, convert) in _EVENT_METADATA_KEYS.items():
658
+ v = extra_meta.get(kineto_key)
659
+ if v is not None:
660
+ fields[field_name] = convert(v)
661
+ any_populated = True
662
+ else:
663
+ fields[field_name] = None
664
+ return EventMetadata(**fields) if any_populated else None
665
+
666
+
667
+ class FunctionEvent(FormattedTimesMixin):
668
+ """Profiling information about a single function.
669
+
670
+ FunctionEvent records the execution of a single operation during profiling.
671
+ These events are obtained from the profiler/kineto and contain detailed
672
+ timing and memory usage information.
673
+
674
+ .. note::
675
+ FunctionEvent objects are typically created by the profiler/kineto and should not
676
+ be instantiated directly by users. Access them through the profiler's output.
677
+
678
+ Attributes:
679
+ id (int): Unique identifier for this event.
680
+ node_id (int): Node identifier for distributed profiling (-1 if not applicable).
681
+ name (str): Name of the profiled function/operator.
682
+ overload_name (str): Overload name for the operator (requires _ExperimentalConfig(capture_overload_names=True) set).
683
+ trace_name (str): Same as name, just changes ProfilerStep* to ProfilerStep#
684
+ time_range (Interval): Time interval containing start and end timestamps in microseconds.
685
+ thread (int): Thread ID where the operation started.
686
+ fwd_thread (int): Thread ID of the corresponding forward operation.
687
+ kernels (List[Kernel]): List of device kernels launched by this operation.
688
+ count (int): Number of times this event was called (usually 1).
689
+ cpu_children (List[FunctionEvent]): Direct CPU child operations.
690
+ cpu_parent (FunctionEvent): Direct CPU parent operation.
691
+ input_shapes (List[List[int]]): Shapes of input tensors (requires record_shapes=True).
692
+ For plain tensor inputs, each entry is a list of dimensions (e.g. ``[16, 16]``).
693
+ TensorList inputs are represented as an empty list ``[]``; use
694
+ ``structured_input_shapes`` to get per-element shapes for TensorList inputs.
695
+ concrete_inputs (List[Any]): Concrete input values (requires record_shapes=true).
696
+ kwinputs (Dict[str, Any]): Keyword arguments (requires record_shapes=true).
697
+ stack (List[str]): Python stack trace where the operation was called (requires with_stack=true).
698
+ scope (int): at::RecordScope identifier (0=forward, 1=backward, etc.).
699
+ use_device (str): Device type being profiled ("cuda", "xpu", etc.).
700
+ cpu_memory_usage (int): CPU memory allocated in bytes.
701
+ device_memory_usage (int): Device memory allocated in bytes.
702
+ is_async (bool): Whether this is an asynchronous operation.
703
+ is_remote (bool): Whether this operation occurred on a remote node.
704
+ sequence_nr (int): Sequence number for autograd operations.
705
+ device_type (DeviceType): Type of device (CPU, CUDA, XPU, PrivateUse1, etc.).
706
+ device_index (int): Index of the device (e.g., GPU 0, 1, 2).
707
+ device_resource_id (int): Resource ID on the device (ie. stream ID).
708
+ is_legacy (bool): Whether this is from the legacy profiler.
709
+ flops (int): Estimated floating point operations.
710
+ is_user_annotation (bool): Whether this is a user-annotated region.
711
+ metadata_json (str): Deprecated. Use event_metadata instead.
712
+ event_metadata (EventMetadata): Additional metadata in structured format.
713
+ structured_input_shapes (List[List[int] | List[List[int]]]): Like ``input_shapes``
714
+ but distinguishes TensorList inputs. Plain tensor inputs are ``List[int]``;
715
+ TensorList inputs are ``List[List[int]]`` containing one shape per tensor in the list.
716
+ Matches the ``"Input Dims"`` field in the Chrome trace JSON.
717
+ structured_input_strides (List[List[int] | List[List[int]]]): Strides of input
718
+ tensors in the same format as ``structured_input_shapes`` (requires
719
+ record_shapes=True).
720
+
721
+ Properties:
722
+ cpu_time_total (float): Total CPU time in microseconds.
723
+ device_time_total (float): Total device (CUDA/XPU/etc) time in microseconds.
724
+ self_cpu_time_total (float): CPU time excluding child operations.
725
+ self_device_time_total (float): Device time excluding child operations.
726
+ self_cpu_memory_usage (int): CPU memory usage excluding child operations.
727
+ self_device_memory_usage (int): Device memory usage excluding child operations.
728
+ cpu_time (float): Average CPU time per call.
729
+ device_time (float): Average device time per call.
730
+ key (str): Key used for grouping events (usually same as name).
731
+
732
+ See Also:
733
+ - :class:`torch.profiler.profile`: Context manager for profiling
734
+ - :class:`EventList`: List container for FunctionEvent objects with helper methods
735
+ - :class:`FunctionEventAvg`: Averaged statistics over multiple FunctionEvent objects
736
+ """
737
+
738
+ def __init__(
739
+ self,
740
+ id,
741
+ name,
742
+ thread,
743
+ start_us,
744
+ end_us,
745
+ overload_name=None,
746
+ fwd_thread=None,
747
+ input_shapes=None,
748
+ stack=None,
749
+ scope=0,
750
+ use_device=None,
751
+ cpu_memory_usage=0,
752
+ device_memory_usage=0,
753
+ is_async=False,
754
+ is_remote=False,
755
+ sequence_nr=-1,
756
+ node_id=-1,
757
+ device_type=DeviceType.CPU,
758
+ device_index=0,
759
+ device_resource_id=None,
760
+ is_legacy=False,
761
+ flops=None,
762
+ trace_name=None,
763
+ concrete_inputs=None,
764
+ kwinputs=None,
765
+ is_user_annotation=False,
766
+ is_python_function=False,
767
+ activity_type=None,
768
+ metadata_json=None,
769
+ flow_id=None,
770
+ flow_type=None,
771
+ flow_start=None,
772
+ external_id=0,
773
+ linked_correlation_id=0,
774
+ extra_meta=None,
775
+ structured_input_shapes=None,
776
+ structured_input_strides=None,
777
+ input_dtypes=None,
778
+ python_id=-1,
779
+ python_parent_id=-1,
780
+ python_module_id=-1,
781
+ ):
782
+ self.id: int = id
783
+ self.node_id: int = node_id
784
+ self.name: str = name
785
+ # pyrefly: ignore [bad-assignment]
786
+ self.overload_name: str = overload_name
787
+ # pyrefly: ignore [bad-assignment]
788
+ self.trace_name: str = trace_name
789
+ self.time_range: Interval = Interval(start_us, end_us)
790
+ self.thread: int = thread
791
+ self.fwd_thread: int | None = fwd_thread
792
+ self.kernels: list[Kernel] = []
793
+ self.count: int = 1
794
+ self.cpu_children: list[FunctionEvent] = []
795
+ self.cpu_parent: FunctionEvent | None = None
796
+ # pyrefly: ignore [bad-assignment]
797
+ self.input_shapes: tuple[int, ...] = input_shapes
798
+ # pyrefly: ignore [bad-assignment]
799
+ self.concrete_inputs: list[Any] = concrete_inputs
800
+ # pyrefly: ignore [bad-assignment]
801
+ self.kwinputs: dict[str, Any] = kwinputs
802
+ # pyrefly: ignore [bad-assignment]
803
+ self.stack: list = stack
804
+ self.scope: int = scope
805
+ self.use_device: str | None = use_device
806
+ self.cpu_memory_usage: int = cpu_memory_usage
807
+ self.device_memory_usage: int = device_memory_usage
808
+ self.is_async: bool = is_async
809
+ self.is_remote: bool = is_remote
810
+ self.sequence_nr: int = sequence_nr
811
+ self.device_type: DeviceType = device_type
812
+ self.device_index: int = device_index
813
+ self.device_resource_id: int = (
814
+ thread if device_resource_id is None else device_resource_id
815
+ )
816
+ self.is_legacy: bool = is_legacy
817
+ self.flops: int | None = flops
818
+ self.is_user_annotation: bool | None = is_user_annotation
819
+ self.is_python_function: bool = is_python_function
820
+ self.activity_type: str | None = activity_type
821
+ self.self_cpu_percent = -1
822
+ self.total_cpu_percent = -1
823
+ self.total_device_percent = -1
824
+ self._metadata_json = metadata_json
825
+ self.flow_id: int | None = flow_id
826
+ self.flow_type: int | None = flow_type
827
+ self.flow_start: bool | None = flow_start
828
+ self.external_id: int = external_id
829
+ self.linked_correlation_id: int = linked_correlation_id
830
+ self.event_metadata: EventMetadata | None = (
831
+ _build_metadata(extra_meta) if extra_meta else None
832
+ )
833
+ # pyrefly: ignore [bad-assignment]
834
+ self.structured_input_shapes: list = structured_input_shapes
835
+ # pyrefly: ignore [bad-assignment]
836
+ self.structured_input_strides: list = structured_input_strides
837
+ # pyrefly: ignore [bad-assignment]
838
+ self.input_dtypes: list[str] = input_dtypes
839
+ self.python_id: int = python_id
840
+ self.python_parent_id: int = python_parent_id
841
+ self.python_module_id: int = python_module_id
842
+
843
+ def append_kernel(self, name, device, duration):
844
+ if self.device_type != DeviceType.CPU:
845
+ raise AssertionError("Expected device_type to be CPU")
846
+ self.kernels.append(Kernel(name, device, duration))
847
+
848
+ def append_cpu_child(self, child):
849
+ """Append a CPU child of type FunctionEvent.
850
+
851
+ One is supposed to append only direct children to the event to have
852
+ correct self cpu time being reported.
853
+ """
854
+ if self.device_type != DeviceType.CPU:
855
+ raise AssertionError("Expected device_type to be CPU")
856
+ if not isinstance(child, FunctionEvent):
857
+ raise AssertionError("Expected child to be a FunctionEvent")
858
+ if child.device_type != DeviceType.CPU:
859
+ raise AssertionError("Expected child device_type to be CPU")
860
+ self.cpu_children.append(child)
861
+
862
+ def set_cpu_parent(self, parent):
863
+ """Set the immediate CPU parent of type FunctionEvent.
864
+
865
+ One profiling FunctionEvent should have only one CPU parent such that
866
+ the child's range interval is completely inside the parent's. We use
867
+ this connection to determine the event is from top-level op or not.
868
+ """
869
+ if self.device_type != DeviceType.CPU:
870
+ raise AssertionError("Expected device_type to be CPU")
871
+ if not isinstance(parent, FunctionEvent):
872
+ raise AssertionError("Expected parent to be a FunctionEvent")
873
+ if parent.device_type != DeviceType.CPU:
874
+ raise AssertionError("Expected parent device_type to be CPU")
875
+ self.cpu_parent = parent
876
+
877
+ # Note: async events don't have children, are not used when computing 'self'
878
+ # metrics of other events, have only total cpu time
879
+ @property
880
+ def self_cpu_memory_usage(self):
881
+ if self.is_async or self.device_type != DeviceType.CPU:
882
+ return 0
883
+ return self.cpu_memory_usage - sum(
884
+ child.cpu_memory_usage for child in self.cpu_children
885
+ )
886
+
887
+ @property
888
+ def self_device_memory_usage(self):
889
+ if self.is_async or self.device_type != DeviceType.CPU:
890
+ return 0
891
+ return self.device_memory_usage - sum(
892
+ child.device_memory_usage for child in self.cpu_children
893
+ )
894
+
895
+ @property
896
+ @deprecated(
897
+ "`metadata_json` is deprecated. Use `event_metadata` instead.",
898
+ category=FutureWarning,
899
+ )
900
+ def metadata_json(self):
901
+ return self._metadata_json
902
+
903
+ @property
904
+ @deprecated(
905
+ "`self_cuda_memory_usage` is deprecated. Use `self_device_memory_usage` instead.",
906
+ category=FutureWarning,
907
+ )
908
+ def self_cuda_memory_usage(self): # To be deprecated
909
+ return self.self_device_memory_usage
910
+
911
+ @property
912
+ def cpu_time_total(self):
913
+ if self.device_type == DeviceType.CPU:
914
+ return self.time_range.elapsed_us()
915
+ else:
916
+ return 0
917
+
918
+ @property
919
+ def self_cpu_time_total(self):
920
+ if self.is_async or self.device_type != DeviceType.CPU:
921
+ return 0
922
+ return self.cpu_time_total - sum(
923
+ child.cpu_time_total for child in self.cpu_children
924
+ )
925
+
926
+ @property
927
+ def device_time_total(self):
928
+ if self.is_async or not self.use_device:
929
+ return 0
930
+ if self.device_type == DeviceType.CPU:
931
+ if not self.is_legacy:
932
+ # account for the kernels in the children ops
933
+ return sum(kinfo.duration for kinfo in self.kernels) + sum(
934
+ ch.device_time_total for ch in self.cpu_children
935
+ )
936
+ else:
937
+ # each legacy cpu events has a single (fake) kernel
938
+ return sum(kinfo.duration for kinfo in self.kernels)
939
+ else:
940
+ if self.device_type not in [
941
+ DeviceType.CUDA,
942
+ DeviceType.PrivateUse1,
943
+ DeviceType.MTIA,
944
+ DeviceType.HPU,
945
+ DeviceType.XPU,
946
+ ]:
947
+ raise AssertionError(
948
+ f"Expected device_type to be CUDA, PrivateUse1, MTIA, HPU or XPU, but got {self.device_type}"
949
+ )
950
+ return self.time_range.elapsed_us()
951
+
952
+ @property
953
+ @deprecated(
954
+ "`cuda_time_total` is deprecated. Use `device_time_total` instead.",
955
+ category=FutureWarning,
956
+ )
957
+ def cuda_time_total(self): # To be deprecated
958
+ return self.device_time_total
959
+
960
+ @property
961
+ def self_device_time_total(self):
962
+ if self.is_async or not self.use_device:
963
+ return 0
964
+ if self.device_type == DeviceType.CPU:
965
+ return self.device_time_total - sum(
966
+ child.device_time_total for child in self.cpu_children
967
+ )
968
+ else:
969
+ if self.device_type not in [
970
+ DeviceType.CUDA,
971
+ DeviceType.PrivateUse1,
972
+ DeviceType.MTIA,
973
+ DeviceType.HPU,
974
+ DeviceType.XPU,
975
+ ]:
976
+ raise AssertionError(
977
+ f"Expected device_type to be CUDA, PrivateUse1, MTIA, HPU or XPU, but got {self.device_type}"
978
+ )
979
+ return self.device_time_total
980
+
981
+ @property
982
+ @deprecated(
983
+ "`self_cuda_time_total` is deprecated. Use `self_device_time_total` instead.",
984
+ category=FutureWarning,
985
+ )
986
+ def self_cuda_time_total(self): # To be deprecated
987
+ return self.self_device_time_total
988
+
989
+ @property
990
+ def key(self):
991
+ return self.name
992
+
993
+ def __repr__(self):
994
+ device_name = self.use_device
995
+ device_time = self.device_time_str
996
+ device_memory_usage = self.device_memory_usage
997
+ return (
998
+ f"<FunctionEvent id={self.id} name={self.name} overload_name={self.overload_name} "
999
+ f"device_type={self.device_type} node_id={self.node_id} cpu_time={self.cpu_time_str} "
1000
+ f"start_us={self.time_range.start} end_us={self.time_range.end} "
1001
+ f"cpu_children={str([child.id for child in self.cpu_children])} {device_name}_time={device_time} "
1002
+ f"name={self.name} thread={self.thread} input_shapes={str(self.input_shapes)} "
1003
+ f"cpu_memory_usage={self.cpu_memory_usage} {device_name}_memory_usage={device_memory_usage} "
1004
+ f"is_async={self.is_async} is_remote={self.is_remote} seq_nr={self.sequence_nr} is_legacy={self.is_legacy}>"
1005
+ )
1006
+
1007
+
1008
+ class FunctionEventAvg(FormattedTimesMixin):
1009
+ """Averaged profiling statistics over multiple FunctionEvent objects.
1010
+
1011
+ FunctionEventAvg aggregates statistics from multiple FunctionEvent objects
1012
+ with the same key (typically same operation name). This is useful for getting
1013
+ average performance metrics across multiple invocations of the same operation.
1014
+
1015
+ This class is typically created by calling :meth:`EventList.key_averages()` on
1016
+ a profiler's event list.
1017
+
1018
+ Attributes:
1019
+ key (str): Grouping key for the events (typically operation name).
1020
+ count (int): Total number of events aggregated.
1021
+ node_id (int): Node identifier for distributed profiling (-1 if not applicable).
1022
+ is_async (bool): Whether the operations are asynchronous.
1023
+ is_remote (bool): Whether the operations occurred on a remote node.
1024
+ use_device (str): Device type being profiled ("cuda", "xpu", etc.).
1025
+ cpu_time_total (int): Accumulated total CPU time in microseconds.
1026
+ device_time_total (int): Accumulated total device time in microseconds.
1027
+ self_cpu_time_total (int): Accumulated self CPU time (excluding children) in microseconds.
1028
+ self_device_time_total (int): Accumulated self device time (excluding children) in microseconds.
1029
+ input_shapes (List[List[int]]): Input tensor shapes (requires record_shapes=true).
1030
+ overload_name (str): Operator overload name (requires _ExperimentalConfig(capture_overload_names=True) set).
1031
+ stack (List[str]): Python stack trace where the operation was called (requires with_stack=true).
1032
+ scope (int): at::RecordScope identifier (0=forward, 1=backward, etc.).
1033
+ cpu_memory_usage (int): Accumulated CPU memory usage in bytes.
1034
+ device_memory_usage (int): Accumulated device memory usage in bytes.
1035
+ self_cpu_memory_usage (int): Accumulated self CPU memory usage in bytes.
1036
+ self_device_memory_usage (int): Accumulated self device memory usage in bytes.
1037
+ cpu_children (List[FunctionEvent]): CPU child events.
1038
+ cpu_parent (FunctionEvent): CPU parent event.
1039
+ device_type (DeviceType): Type of device (CPU, CUDA, XPU, PrivateUse1, etc.).
1040
+ is_legacy (bool): Whether from legacy profiler.
1041
+ flops (int): Total floating point operations.
1042
+ is_user_annotation (bool): Whether this is a user-annotated region.
1043
+
1044
+ Properties:
1045
+ cpu_time (float): Average CPU time per invocation.
1046
+ device_time (float): Average device time per invocation.
1047
+
1048
+ See Also:
1049
+ - :class:`EventList.key_averages`: Method that creates FunctionEventAvg objects
1050
+ - :class:`FunctionEvent`: Individual profiling event
1051
+ - :class:`EventList`: Container for profiling events
1052
+ """
1053
+
1054
+ def __init__(self) -> None:
1055
+ self.key: str | None = None
1056
+ self.count: int = 0
1057
+ self.node_id: int = 0
1058
+ self.is_async: bool = False
1059
+ self.is_remote: bool = False
1060
+ self.use_device: str | None = None
1061
+ self.cpu_time_total: int = 0
1062
+ self.device_time_total: int = 0
1063
+ self.self_cpu_time_total: int = 0
1064
+ self.self_device_time_total: int = 0
1065
+ self.input_shapes: list[list[int]] | None = None
1066
+ self.overload_name: str | None = None
1067
+ self.stack: list | None = None
1068
+ self.scope: int | None = None
1069
+ self.cpu_memory_usage: int = 0
1070
+ self.device_memory_usage: int = 0
1071
+ self.self_cpu_memory_usage: int = 0
1072
+ self.self_device_memory_usage: int = 0
1073
+ self.cpu_children: list[FunctionEvent] | None = None
1074
+ self.cpu_parent: FunctionEvent | None = None
1075
+ self.device_type: DeviceType = DeviceType.CPU
1076
+ self.is_legacy: bool = False
1077
+ self.flops: int = 0
1078
+
1079
+ def add(self, other):
1080
+ if self.key is None:
1081
+ # First function being recorded as part of FunctionEventAvg, propagate
1082
+ # fields.
1083
+ self.key = other.key
1084
+ self.node_id = other.node_id
1085
+ self.is_async = other.is_async
1086
+ self.is_remote = other.is_remote
1087
+ self.cpu_parent = other.cpu_parent
1088
+ self.cpu_children = other.cpu_children
1089
+
1090
+ self.overload_name = other.overload_name
1091
+ self.input_shapes = other.input_shapes
1092
+ self.stack = other.stack
1093
+ self.scope = other.scope
1094
+ self.device_type = other.device_type
1095
+ self.is_legacy = other.is_legacy
1096
+ self.use_device = other.use_device
1097
+ self.is_user_annotation = other.is_user_annotation
1098
+
1099
+ if not isinstance(other, (FunctionEvent, FunctionEventAvg)):
1100
+ raise AssertionError(
1101
+ "Expected other to be a FunctionEvent or FunctionEventAvg"
1102
+ )
1103
+ if other.key != self.key:
1104
+ raise AssertionError(
1105
+ f"Expected keys to match, but got {other.key} vs {self.key}"
1106
+ )
1107
+
1108
+ self.cpu_time_total += other.cpu_time_total
1109
+ self.device_time_total += other.device_time_total
1110
+ self.self_cpu_time_total += other.self_cpu_time_total
1111
+ self.self_device_time_total += other.self_device_time_total
1112
+ self.cpu_memory_usage += other.cpu_memory_usage
1113
+ self.device_memory_usage += other.device_memory_usage
1114
+ self.self_cpu_memory_usage += other.self_cpu_memory_usage
1115
+ self.self_device_memory_usage += other.self_device_memory_usage
1116
+ self.count += other.count
1117
+ if self.flops is None:
1118
+ # pyrefly: ignore [bad-assignment]
1119
+ self.flops = other.flops
1120
+ elif other.flops is not None:
1121
+ self.flops += other.flops
1122
+ return self
1123
+
1124
+ def __iadd__(self, other):
1125
+ return self.add(other)
1126
+
1127
+ def __repr__(self):
1128
+ device_name = "cuda" if not self.use_device else self.use_device
1129
+ self_device_time = self.self_device_time_total_str
1130
+ device_time = self.device_time_str
1131
+ device_memory = self.device_memory_usage
1132
+ return (
1133
+ f"<FunctionEventAvg key={self.key} self_cpu_time={self.self_cpu_time_total_str} cpu_time={self.cpu_time_str} "
1134
+ f" self_{device_name}_time={self_device_time} {device_name}_time={device_time} input_shapes={str(self.input_shapes)} "
1135
+ f"cpu_memory_usage={self.cpu_memory_usage} {device_name}_memory_usage={device_memory}>"
1136
+ )
1137
+
1138
+
1139
+ class StringTable(defaultdict):
1140
+ def __missing__(self, key):
1141
+ # manage cases like 't' (demangled to 'unsigned short') separately,
1142
+ # for now simply check the length to avoid unexpected results for
1143
+ # the short sequences
1144
+ self[key] = torch._C._demangle(key) if len(key) > 1 else key
1145
+ return self[key]
1146
+
1147
+
1148
+ class MemRecordsAcc:
1149
+ """Acceleration structure for accessing mem_records in interval."""
1150
+
1151
+ def __init__(self, mem_records):
1152
+ self._mem_records = mem_records
1153
+ self._start_nses: list[int] = []
1154
+ self._indices: list[int] = []
1155
+ if len(mem_records) > 0:
1156
+ tmp = sorted([(r[0].start_ns(), i) for i, r in enumerate(mem_records)])
1157
+ self._start_nses, self._indices = zip(*tmp) # type: ignore[assignment]
1158
+
1159
+ def in_interval(self, start_ns, end_ns):
1160
+ r"""
1161
+ Return all records in the given interval
1162
+ """
1163
+ start_idx = bisect.bisect_left(self._start_nses, start_ns)
1164
+ end_idx = bisect.bisect_right(self._start_nses, end_ns)
1165
+ for i in range(start_idx, end_idx):
1166
+ yield self._mem_records[self._indices[i]]
1167
+
1168
+
1169
+ def _filter_stack_entry(entry):
1170
+ filtered_entries = [
1171
+ ("autograd/__init__", "_make_grads"),
1172
+ ("autograd/__init__", "backward"),
1173
+ ("torch/tensor", "backward"),
1174
+ ("_internal/common_utils", "prof_callable"),
1175
+ ("_internal/common_utils", "prof_func_call"),
1176
+ ("_internal/common_utils", "prof_meth_call"),
1177
+ ]
1178
+ return all(not (f[0] in entry and f[1] in entry) for f in filtered_entries)
1179
+
1180
+
1181
+ MEMORY_EVENT_NAME = "[memory]"
1182
+ OUT_OF_MEMORY_EVENT_NAME = "[OutOfMemory]"
1183
+
1184
+
1185
+ def _filter_name(name):
1186
+ # ignoring the following utility ops
1187
+ filtered_out_names = [
1188
+ MEMORY_EVENT_NAME, # used only for the top-level memory events
1189
+ OUT_OF_MEMORY_EVENT_NAME,
1190
+ "profiler::_record_function_enter",
1191
+ "profiler::_record_function_enter_new",
1192
+ "profiler::_record_function_exit",
1193
+ "aten::is_leaf",
1194
+ "aten::output_nr",
1195
+ "aten::_version",
1196
+ ]
1197
+ return name in filtered_out_names
1198
+
1199
+
1200
+ # Demangles and optionally rewrites the provided event name,
1201
+ # with_wildcard - whether to replace certain numbered event names
1202
+ # with a wildcard name to aggregate them together in the profiler table
1203
+ # output
1204
+ def _rewrite_name(name, with_wildcard=False):
1205
+ string_table = StringTable()
1206
+ name = string_table[name]
1207
+ if with_wildcard:
1208
+ if name.startswith("ProfilerStep#"):
1209
+ name = "ProfilerStep*"
1210
+ return name
1211
+
1212
+
1213
+ def _build_table(
1214
+ events,
1215
+ sort_by=None,
1216
+ header=None,
1217
+ row_limit=100,
1218
+ max_src_column_width=75,
1219
+ max_name_column_width=55,
1220
+ max_shapes_column_width=80,
1221
+ with_flops=False,
1222
+ profile_memory=False,
1223
+ top_level_events_only=False,
1224
+ time_unit=None,
1225
+ ):
1226
+ """Print a summary of events (which can be a list of FunctionEvent or FunctionEventAvg)."""
1227
+ if len(events) == 0:
1228
+ return ""
1229
+
1230
+ has_device_time = any(event.self_device_time_total > 0 for event in events)
1231
+ has_device_mem = any(event.self_device_memory_usage > 0 for event in events)
1232
+ use_device = events[0].use_device
1233
+ # Running on PrivateUse1 device with profiler but not enable
1234
+ # ProfilerActivity.PrivateUse1 can also catch privateuse1 memory usage.
1235
+ # Here only need to check has_privateuse1_time if not use_device.
1236
+ if not use_device and has_device_time:
1237
+ raise RuntimeError("use_device is None, but there is device performance data.")
1238
+
1239
+ has_input_shapes = any(
1240
+ (event.input_shapes is not None and len(event.input_shapes) > 0)
1241
+ for event in events
1242
+ )
1243
+
1244
+ has_overload_names = any(
1245
+ (event.overload_name is not None and len(event.overload_name) > 0)
1246
+ for event in events
1247
+ )
1248
+
1249
+ if sort_by is not None:
1250
+ events = EventList(
1251
+ sorted(
1252
+ events,
1253
+ key=lambda evt: getattr(
1254
+ evt,
1255
+ sort_by.replace("cuda", "device")
1256
+ .replace("xpu", "device")
1257
+ .replace("privateuse1", "device"),
1258
+ ),
1259
+ reverse=True,
1260
+ ),
1261
+ use_device=use_device,
1262
+ profile_memory=profile_memory,
1263
+ with_flops=with_flops,
1264
+ )
1265
+
1266
+ name_column_width = max(len(evt.key) for evt in events) + 4
1267
+ if max_name_column_width is not None:
1268
+ name_column_width = min(name_column_width, max_name_column_width)
1269
+
1270
+ shapes_column_width = max(len(str(evt.input_shapes)) for evt in events) + 4
1271
+ if max_shapes_column_width is not None:
1272
+ shapes_column_width = min(shapes_column_width, max_shapes_column_width)
1273
+
1274
+ DEFAULT_COLUMN_WIDTH = 12
1275
+ flops_column_width = DEFAULT_COLUMN_WIDTH
1276
+
1277
+ src_column_width = None
1278
+ stacks = [
1279
+ evt.stack for evt in events if evt.stack is not None and len(evt.stack) > 0
1280
+ ]
1281
+ has_stack = len(stacks) > 0
1282
+ if has_stack:
1283
+ src_column_width = (
1284
+ max(max(len(entry) for entry in stack) for stack in stacks) + 4
1285
+ )
1286
+ if max_src_column_width is not None:
1287
+ src_column_width = min(src_column_width, max_src_column_width)
1288
+
1289
+ headers = ["Name"]
1290
+ if has_overload_names:
1291
+ headers.append("Overload Name")
1292
+ headers += [
1293
+ "Self CPU %",
1294
+ "Self CPU",
1295
+ "CPU total %",
1296
+ "CPU total",
1297
+ "CPU time avg",
1298
+ ]
1299
+
1300
+ device_name = use_device.upper() if use_device is not None else "None"
1301
+ if has_device_time:
1302
+ headers.extend(
1303
+ [
1304
+ f"Self {device_name}",
1305
+ f"Self {device_name} %",
1306
+ f"{device_name} total",
1307
+ f"{device_name} time avg",
1308
+ ]
1309
+ )
1310
+ if profile_memory:
1311
+ headers.extend(
1312
+ [
1313
+ "CPU Mem",
1314
+ "Self CPU Mem",
1315
+ ]
1316
+ )
1317
+ if use_device and has_device_mem:
1318
+ headers.extend(
1319
+ [
1320
+ f"{device_name} Mem",
1321
+ f"Self {device_name} Mem",
1322
+ ]
1323
+ )
1324
+ headers.append("# of Calls")
1325
+ # Only append Node ID if any event has a valid (>= 0) Node ID
1326
+ append_node_id = any(evt.node_id != -1 for evt in events)
1327
+ if append_node_id:
1328
+ headers.append("Node ID")
1329
+
1330
+ SPACING_SIZE = 2
1331
+ row_format_lst = [""]
1332
+ header_sep_lst = [""]
1333
+ line_length_lst = [-SPACING_SIZE]
1334
+
1335
+ def add_column(padding, text_dir=">"):
1336
+ row_format_lst[0] += (
1337
+ "{: " + text_dir + str(padding) + "}" + (" " * SPACING_SIZE)
1338
+ )
1339
+ header_sep_lst[0] += "-" * padding + (" " * SPACING_SIZE)
1340
+ line_length_lst[0] += padding + SPACING_SIZE
1341
+
1342
+ def auto_scale_flops(flops):
1343
+ flop_headers = [
1344
+ "FLOPs",
1345
+ "KFLOPs",
1346
+ "MFLOPs",
1347
+ "GFLOPs",
1348
+ "TFLOPs",
1349
+ "PFLOPs",
1350
+ ]
1351
+ if flops <= 0:
1352
+ raise AssertionError(f"Expected flops to be positive, but got {flops}")
1353
+ # pyrefly: ignore [no-matching-overload]
1354
+ log_flops = max(0, min(math.log10(flops) / 3, float(len(flop_headers) - 1)))
1355
+ if not (log_flops >= 0 and log_flops < len(flop_headers)):
1356
+ raise AssertionError(
1357
+ f"Expected log_flops to be in range [0, {len(flop_headers)}), but got {log_flops}"
1358
+ )
1359
+ return (pow(10, (math.floor(log_flops) * -3.0)), flop_headers[int(log_flops)])
1360
+
1361
+ add_column(name_column_width)
1362
+ if has_overload_names:
1363
+ add_column(name_column_width)
1364
+ for _ in headers[1 + has_overload_names :]:
1365
+ add_column(DEFAULT_COLUMN_WIDTH)
1366
+
1367
+ if has_input_shapes:
1368
+ headers.append("Input Shapes")
1369
+ add_column(shapes_column_width)
1370
+
1371
+ if has_stack:
1372
+ headers.append("Source Location")
1373
+ add_column(src_column_width, text_dir="<")
1374
+
1375
+ if with_flops:
1376
+ # Auto-scaling of flops header
1377
+ raw_flops = [evt.flops for evt in events if evt.flops > 0]
1378
+ if len(raw_flops) != 0:
1379
+ (flops_scale, flops_header) = auto_scale_flops(min(raw_flops))
1380
+ headers.append(f"Total {flops_header}")
1381
+ add_column(flops_column_width)
1382
+ else:
1383
+ with_flops = False # can't find any valid flops
1384
+
1385
+ row_format = row_format_lst[0]
1386
+ header_sep = header_sep_lst[0]
1387
+ line_length = line_length_lst[0]
1388
+ add_column = None # type: ignore[assignment]
1389
+
1390
+ result = []
1391
+
1392
+ def append(s):
1393
+ result.append(s)
1394
+ result.append("\n") # Yes, newline after the end as well
1395
+
1396
+ sum_self_cpu_time_total = 0
1397
+ sum_self_device_time_total = 0
1398
+ for evt in events:
1399
+ sum_self_cpu_time_total += evt.self_cpu_time_total
1400
+ if evt.device_type == DeviceType.CPU and evt.is_legacy:
1401
+ # in legacy profiler, kernel info is stored in cpu events
1402
+ sum_self_device_time_total += evt.self_device_time_total
1403
+ elif (
1404
+ evt.device_type
1405
+ in [
1406
+ DeviceType.CUDA,
1407
+ DeviceType.PrivateUse1,
1408
+ DeviceType.MTIA,
1409
+ DeviceType.XPU,
1410
+ ]
1411
+ and not evt.is_user_annotation
1412
+ ):
1413
+ # in kineto profiler, there're events with the correct device type (e.g. CUDA)
1414
+ sum_self_device_time_total += evt.self_device_time_total
1415
+
1416
+ # Actual printing
1417
+ if header is not None:
1418
+ append("=" * line_length)
1419
+ append(header)
1420
+ if top_level_events_only:
1421
+ append("=" * line_length)
1422
+ append("This report only display top-level ops statistics")
1423
+ append(header_sep)
1424
+ append(row_format.format(*headers))
1425
+
1426
+ append(header_sep)
1427
+
1428
+ def trim_path(path, src_column_width):
1429
+ if len(path) > src_column_width:
1430
+ offset = len(path) - src_column_width
1431
+ path = path[offset:]
1432
+ if len(path) > 3:
1433
+ path = "..." + path[3:]
1434
+ return path
1435
+
1436
+ def override_time_unit(time_us, default_str, time_unit):
1437
+ US_IN_SECOND = 1000.0 * 1000.0
1438
+ US_IN_MS = 1000.0
1439
+ if time_unit == "s":
1440
+ return f"{time_us / US_IN_SECOND:.3f}s"
1441
+ elif time_unit == "ms":
1442
+ return f"{time_us / US_IN_MS:.3f}ms"
1443
+ elif time_unit == "us":
1444
+ return f"{time_us:.3f}us"
1445
+ else:
1446
+ return default_str
1447
+
1448
+ event_limit = 0
1449
+ for evt in events:
1450
+ if event_limit == row_limit:
1451
+ break
1452
+ if top_level_events_only and evt.cpu_parent is not None:
1453
+ continue
1454
+ else:
1455
+ event_limit += 1
1456
+ name = evt.key
1457
+ if max_name_column_width is not None and len(name) >= max_name_column_width - 3:
1458
+ name = name[: (max_name_column_width - 3)] + "..."
1459
+
1460
+ evt.self_cpu_percent = _format_time_share(
1461
+ evt.self_cpu_time_total, sum_self_cpu_time_total
1462
+ )
1463
+ evt.total_cpu_percent = (
1464
+ _format_time_share(evt.cpu_time_total, sum_self_cpu_time_total)
1465
+ if not evt.is_async
1466
+ else 0
1467
+ )
1468
+
1469
+ row_values = [name]
1470
+ if has_overload_names:
1471
+ overload_name = evt.overload_name
1472
+ if (
1473
+ max_name_column_width is not None
1474
+ and len(overload_name) >= max_name_column_width - 3
1475
+ ):
1476
+ overload_name = overload_name[: (max_name_column_width - 3)] + "..."
1477
+ row_values += [overload_name]
1478
+ row_values += [
1479
+ # Self CPU total %, 0 for async events.
1480
+ evt.self_cpu_percent,
1481
+ override_time_unit(
1482
+ evt.self_cpu_time_total, evt.self_cpu_time_total_str, time_unit
1483
+ ), # Self CPU total
1484
+ # CPU total %, 0 for async events.
1485
+ evt.total_cpu_percent,
1486
+ override_time_unit(
1487
+ evt.cpu_time_total, evt.cpu_time_total_str, time_unit
1488
+ ), # CPU total
1489
+ override_time_unit(
1490
+ evt.cpu_time, evt.cpu_time_str, time_unit
1491
+ ), # CPU time avg
1492
+ ]
1493
+ if has_device_time:
1494
+ evt.total_device_percent = _format_time_share(
1495
+ evt.self_device_time_total, sum_self_device_time_total
1496
+ )
1497
+ row_values.extend(
1498
+ [
1499
+ override_time_unit(
1500
+ evt.self_device_time_total,
1501
+ evt.self_device_time_total_str,
1502
+ time_unit,
1503
+ ),
1504
+ # device time total %
1505
+ evt.total_device_percent,
1506
+ override_time_unit(
1507
+ evt.device_time_total, evt.device_time_total_str, time_unit
1508
+ ),
1509
+ override_time_unit(
1510
+ evt.device_time, evt.device_time_str, time_unit
1511
+ ), # device time avg
1512
+ ]
1513
+ )
1514
+ if profile_memory:
1515
+ row_values.extend(
1516
+ [
1517
+ # CPU Mem Total
1518
+ _format_memory(evt.cpu_memory_usage),
1519
+ # Self CPU Mem Total
1520
+ _format_memory(evt.self_cpu_memory_usage),
1521
+ ]
1522
+ )
1523
+ if use_device and has_device_mem:
1524
+ row_values.extend(
1525
+ [
1526
+ # Device Mem Total
1527
+ _format_memory(evt.device_memory_usage),
1528
+ # Self Device Mem Total
1529
+ _format_memory(evt.self_device_memory_usage),
1530
+ ]
1531
+ )
1532
+ row_values.append(
1533
+ evt.count, # Number of calls
1534
+ )
1535
+
1536
+ if append_node_id:
1537
+ row_values.append(evt.node_id)
1538
+ if has_input_shapes:
1539
+ row_values.append(str(evt.input_shapes)[:shapes_column_width])
1540
+ if with_flops:
1541
+ if evt.flops <= 0:
1542
+ row_values.append("--")
1543
+ else:
1544
+ row_values.append(f"{evt.flops * flops_scale:8.3f}") # type: ignore[possibly-undefined]
1545
+ if has_stack:
1546
+ src_field = ""
1547
+ if len(evt.stack) > 0:
1548
+ src_field = trim_path(evt.stack[0], src_column_width)
1549
+ row_values.append(src_field)
1550
+ append(row_format.format(*row_values))
1551
+
1552
+ if has_stack:
1553
+ empty_headers = [""] * (len(headers) - 1)
1554
+ for entry in evt.stack[1:]:
1555
+ append(
1556
+ row_format.format(
1557
+ *(empty_headers + [trim_path(entry, src_column_width)])
1558
+ )
1559
+ )
1560
+ empty_headers.append("")
1561
+ append(row_format.format(*empty_headers))
1562
+
1563
+ append(header_sep)
1564
+ append(
1565
+ f"Self CPU time total: {override_time_unit(sum_self_cpu_time_total, _format_time(sum_self_cpu_time_total), time_unit)}"
1566
+ )
1567
+ if has_device_time:
1568
+ append(
1569
+ f"Self {use_device.upper() if use_device is not None else 'None'} "
1570
+ f"time total: {override_time_unit(sum_self_device_time_total, _format_time(sum_self_device_time_total), time_unit)}"
1571
+ )
1572
+ return "".join(result)
1573
+
1574
+
1575
+ # Collect all events with stack traces and format them canonically
1576
+ def _canonicalize_profiler_events(events):
1577
+ """
1578
+ Extract and format all events with stack traces in a canonical way
1579
+ for deterministic testing.
1580
+ """
1581
+ events_with_traces = []
1582
+
1583
+ for event in events:
1584
+ # Extract relevant fields
1585
+ event_name = event.get("name", "")
1586
+ node_name = event["args"].get("node_name", "")
1587
+ stack_trace = event["args"].get("stack_trace", "")
1588
+
1589
+ # Get the last non-empty line of the stack trace that is actual source,
1590
+ # not a caret-marker line. Python 3.11+ appends "^^^^"/"~~~~" indicator
1591
+ # lines below the source when a FrameSummary has colno/end_colno set
1592
+ # (e.g. dynamo-generated stack traces); those lines must be skipped so
1593
+ # we still surface the source code line in canonicalized output.
1594
+ lines = [s.strip() for s in stack_trace.split("\n") if s.strip()]
1595
+ source_lines = [s for s in lines if not set(s).issubset({"^", "~", " "})]
1596
+ stack_trace = source_lines[-1] if source_lines else ""
1597
+
1598
+ events_with_traces.append(
1599
+ {
1600
+ "event_name": event_name[:30],
1601
+ "node_name": node_name,
1602
+ "stack_trace": stack_trace,
1603
+ "start_time": event.get("ts", 0),
1604
+ }
1605
+ )
1606
+
1607
+ # Sort by node_name for deterministic ordering
1608
+ events_with_traces.sort(key=lambda x: x["start_time"])
1609
+
1610
+ # Format as a string
1611
+ lines: list[str] = []
1612
+ for evt in events_with_traces:
1613
+ lines.append(
1614
+ f"event={evt['event_name']} node={evt['node_name']} stack_trace={evt['stack_trace']}"
1615
+ )
1616
+
1617
+ return "\n".join(lines)
western_ai/usa/pytorch/torch/autograd/variable.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import torch
3
+ from torch._C import _ImperativeEngine as ImperativeEngine
4
+
5
+
6
+ __all__ = ["VariableMeta", "Variable"]
7
+
8
+
9
+ class VariableMeta(type):
10
+ def __instancecheck__(cls, other):
11
+ return isinstance(other, torch.Tensor)
12
+
13
+
14
+ class Variable(torch._C._LegacyVariableBase, metaclass=VariableMeta): # type: ignore[misc]
15
+ _execution_engine = ImperativeEngine()
western_ai/usa/pytorch/torch/backends/__init__.py ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import sys
3
+ import types
4
+ from contextlib import contextmanager
5
+
6
+ import torch
7
+
8
+
9
+ # The idea for this parameter is that we forbid bare assignment
10
+ # to torch.backends.<cudnn|mkldnn>.enabled and friends when running our
11
+ # test suite, where it's very easy to forget to undo the change
12
+ # later.
13
+ __allow_nonbracketed_mutation_flag = True
14
+
15
+
16
+ def disable_global_flags():
17
+ global __allow_nonbracketed_mutation_flag
18
+ __allow_nonbracketed_mutation_flag = False
19
+
20
+
21
+ def flags_frozen():
22
+ return not __allow_nonbracketed_mutation_flag
23
+
24
+
25
+ @contextmanager
26
+ def __allow_nonbracketed_mutation():
27
+ global __allow_nonbracketed_mutation_flag
28
+ old = __allow_nonbracketed_mutation_flag
29
+ __allow_nonbracketed_mutation_flag = True
30
+ try:
31
+ yield
32
+ finally:
33
+ __allow_nonbracketed_mutation_flag = old
34
+
35
+
36
+ class ContextProp:
37
+ def __init__(self, getter, setter):
38
+ self.getter = getter
39
+ self.setter = setter
40
+
41
+ def __get__(self, obj, objtype):
42
+ return self.getter()
43
+
44
+ def __set__(self, obj, val):
45
+ if not flags_frozen():
46
+ self.setter(val)
47
+ else:
48
+ raise RuntimeError(
49
+ f"not allowed to set {obj.__name__} flags "
50
+ "after disable_global_flags; please use flags() context manager instead"
51
+ )
52
+
53
+
54
+ class PropModule(types.ModuleType):
55
+ def __init__(self, m, name):
56
+ super().__init__(name)
57
+ self.m = m
58
+
59
+ def __getattr__(self, attr):
60
+ return self.m.__getattribute__(attr)
61
+
62
+
63
+ class _FP32Precision:
64
+ def __init__(self, backend, op):
65
+ self.backend = backend
66
+ self.op = op
67
+
68
+ def __setattr__(self, name, value):
69
+ if name == "fp32_precision":
70
+ torch._C._set_fp32_precision_setter(self.backend, self.op, value)
71
+ elif name in ("backend", "op"):
72
+ super().__setattr__(name, value)
73
+ else:
74
+ raise AttributeError("Unknown attribute " + name)
75
+
76
+ def __getattr__(self, name):
77
+ if name == "fp32_precision":
78
+ return torch._C._get_fp32_precision_getter(self.backend, self.op)
79
+ else:
80
+ raise AttributeError("Unknown attribute " + name)
81
+
82
+
83
+ def set_flags(_fp32_precision="none"):
84
+ orig_flags = (torch._C._get_fp32_precision_getter("generic", "all"),)
85
+ if _fp32_precision is not None:
86
+ torch._C._set_fp32_precision_setter("generic", "all", _fp32_precision)
87
+ return orig_flags
88
+
89
+
90
+ @contextmanager
91
+ def flags(fp32_precision="none"):
92
+ with __allow_nonbracketed_mutation():
93
+ orig_flags = set_flags(fp32_precision)
94
+ try:
95
+ yield
96
+ finally:
97
+ with __allow_nonbracketed_mutation():
98
+ set_flags(*orig_flags)
99
+
100
+
101
+ def _get_fp32_precision_getter(backend, op):
102
+ def inner():
103
+ return torch._C._get_fp32_precision_getter(backend, op)
104
+
105
+ return inner
106
+
107
+
108
+ def _set_fp32_precision_setter(backend, op):
109
+ def inner(precision):
110
+ return torch._C._set_fp32_precision_setter(backend, op, precision)
111
+
112
+ return inner
113
+
114
+
115
+ class GenericModule(PropModule):
116
+ fp32_precision = ContextProp(
117
+ _get_fp32_precision_getter("generic", "all"),
118
+ _set_fp32_precision_setter("generic", "all"),
119
+ )
120
+
121
+
122
+ sys.modules[__name__] = GenericModule(sys.modules[__name__], __name__)
123
+
124
+ from torch.backends import (
125
+ cpu as cpu,
126
+ cuda as cuda,
127
+ cudnn as cudnn,
128
+ cusparselt as cusparselt,
129
+ kleidiai as kleidiai,
130
+ mha as mha,
131
+ miopen as miopen,
132
+ mkl as mkl,
133
+ mkldnn as mkldnn,
134
+ mps as mps,
135
+ nnpack as nnpack,
136
+ openmp as openmp,
137
+ opt_einsum as opt_einsum,
138
+ python_native as python_native,
139
+ quantized as quantized,
140
+ )
western_ai/usa/pytorch/torch/backends/_coreml/__init__.py ADDED
File without changes
western_ai/usa/pytorch/torch/backends/_coreml/preprocess.py ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import hashlib
3
+ import json
4
+
5
+ import coremltools as ct # type: ignore[import]
6
+ from coremltools.converters.mil.input_types import TensorType # type: ignore[import]
7
+ from coremltools.converters.mil.mil import types # type: ignore[import]
8
+ from coremltools.models.neural_network import quantization_utils # type: ignore[import]
9
+
10
+ import torch
11
+
12
+
13
+ CT_METADATA_VERSION = "com.github.apple.coremltools.version"
14
+ CT_METADATA_SOURCE = "com.github.apple.coremltools.source"
15
+
16
+
17
+ class ScalarType:
18
+ Float = 0
19
+ Double = 1
20
+ Int = 2
21
+ Long = 3
22
+ Undefined = 4
23
+
24
+
25
+ # Supported Tensor types in coremltools:
26
+ # https://github.com/apple/coremltools/blob/main/coremltools/converters/mil/frontend/torch/converter.py#L28
27
+ torch_to_mil_types = {
28
+ ScalarType.Float: types.fp32,
29
+ ScalarType.Double: types.fp64,
30
+ ScalarType.Int: types.int32,
31
+ ScalarType.Long: types.int64,
32
+ }
33
+
34
+
35
+ class CoreMLComputeUnit:
36
+ CPU = "cpuOnly"
37
+ CPUAndGPU = "cpuAndGPU"
38
+ ALL = "all"
39
+
40
+
41
+ class CoreMLQuantizationMode:
42
+ LINEAR = "linear"
43
+ LINEAR_SYMMETRIC = "linear_symmetric"
44
+ NONE = "none"
45
+
46
+
47
+ def TensorSpec(shape, dtype=ScalarType.Float):
48
+ return (shape, dtype)
49
+
50
+
51
+ def CompileSpec(
52
+ inputs,
53
+ outputs,
54
+ backend=CoreMLComputeUnit.CPU,
55
+ allow_low_precision=True,
56
+ quantization_mode=CoreMLQuantizationMode.NONE,
57
+ mlmodel_export_path=None,
58
+ convert_to=None,
59
+ ):
60
+ return (
61
+ inputs,
62
+ outputs,
63
+ backend,
64
+ allow_low_precision,
65
+ quantization_mode,
66
+ mlmodel_export_path,
67
+ convert_to,
68
+ )
69
+
70
+
71
+ def _check_enumerated_shape(shape):
72
+ for s in shape:
73
+ if not isinstance(s, (list, tuple)):
74
+ return False
75
+ return True
76
+
77
+
78
+ def _convert_to_mil_type(shape, dtype, name: str):
79
+ mil_shape = shape
80
+ if _check_enumerated_shape(shape):
81
+ mil_shape = ct.EnumeratedShapes(shape)
82
+ ml_type = TensorType(shape=mil_shape, dtype=torch_to_mil_types[dtype])
83
+ ml_type.name = name
84
+ return ml_type
85
+
86
+
87
+ def preprocess(script_module: torch._C.ScriptObject, compile_spec: dict[str, tuple]):
88
+ spec = compile_spec["forward"]
89
+ (
90
+ input_specs,
91
+ output_specs,
92
+ backend,
93
+ allow_low_precision,
94
+ quantization_mode,
95
+ mlmodel_export_path,
96
+ convert_to,
97
+ ) = spec
98
+ mil_inputs = []
99
+ inputs = []
100
+ for index, input in enumerate(input_specs):
101
+ shape, dtype = input
102
+ name = "input_" + str(index)
103
+ inputs.append([name, str(dtype), str(shape)])
104
+ ml_type = _convert_to_mil_type(shape, dtype, name)
105
+ mil_inputs.append(ml_type)
106
+ model = torch.jit.RecursiveScriptModule._construct(script_module, lambda x: None)
107
+ mlmodel = ct.convert(model, inputs=mil_inputs, convert_to=convert_to)
108
+
109
+ if quantization_mode != CoreMLQuantizationMode.NONE:
110
+ quant_model_spec = quantization_utils.quantize_weights(
111
+ mlmodel, nbits=8, quantization_mode=quantization_mode
112
+ )
113
+ mlmodel = ct.models.MLModel(quant_model_spec)
114
+
115
+ spec = mlmodel.get_spec()
116
+ if len(spec.description.output) != len(output_specs): # type: ignore[attr-defined]
117
+ raise AssertionError(
118
+ f"Number of outputs in spec ({len(spec.description.output)}) " # type: ignore[attr-defined]
119
+ f"does not match output_specs ({len(output_specs)})"
120
+ )
121
+ outputs = []
122
+ for index, output in enumerate(output_specs):
123
+ shape, dtype = output
124
+ name = spec.description.output[index].name # type: ignore[attr-defined]
125
+ outputs.append([name, str(dtype), str(shape)])
126
+ mlmodel = ct.models.model.MLModel(spec)
127
+ print(mlmodel)
128
+
129
+ if mlmodel_export_path is not None:
130
+ print(f"Saving CoreML .mlmodel file to {mlmodel_export_path}")
131
+ mlmodel.save(mlmodel_export_path)
132
+
133
+ config = {
134
+ "spec_ver": str(spec.specificationVersion), # type: ignore[attr-defined]
135
+ "backend": backend,
136
+ "allow_low_precision": str(allow_low_precision),
137
+ }
138
+ metadata = {
139
+ "coremltool_ver": mlmodel.user_defined_metadata[CT_METADATA_VERSION],
140
+ "torch_ver": mlmodel.user_defined_metadata[CT_METADATA_SOURCE],
141
+ }
142
+ coreml_compile_spec = {
143
+ "inputs": inputs,
144
+ "outputs": outputs,
145
+ "config": config,
146
+ "metadata": metadata,
147
+ }
148
+ mlmodel = spec.SerializeToString() # type: ignore[attr-defined]
149
+
150
+ return {
151
+ "model": mlmodel,
152
+ "hash": str(hashlib.sha256(mlmodel).hexdigest()),
153
+ "extra": json.dumps(coreml_compile_spec),
154
+ }
western_ai/usa/pytorch/torch/backends/_nnapi/__init__.py ADDED
File without changes
western_ai/usa/pytorch/torch/backends/_nnapi/prepare.py ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-decorators
2
+ # mypy: allow-untyped-defs
3
+
4
+ from typing import Optional
5
+
6
+ import torch
7
+ from torch.backends._nnapi.serializer import _NnapiSerializer
8
+
9
+
10
+ ANEURALNETWORKS_PREFER_LOW_POWER = 0
11
+ ANEURALNETWORKS_PREFER_FAST_SINGLE_ANSWER = 1
12
+ ANEURALNETWORKS_PREFER_SUSTAINED_SPEED = 2
13
+
14
+
15
+ class NnapiModule(torch.nn.Module):
16
+ """Torch Module that wraps an NNAPI Compilation.
17
+
18
+ This module handles preparing the weights, initializing the
19
+ NNAPI TorchBind object, and adjusting the memory formats
20
+ of all inputs and outputs.
21
+ """
22
+
23
+ # _nnapi.Compilation is defined
24
+ comp: Optional[torch.classes._nnapi.Compilation] # type: ignore[name-defined] # noqa: UP045
25
+ weights: list[torch.Tensor]
26
+ out_templates: list[torch.Tensor]
27
+
28
+ def __init__(
29
+ self,
30
+ shape_compute_module: torch.nn.Module,
31
+ ser_model: torch.Tensor,
32
+ weights: list[torch.Tensor],
33
+ inp_mem_fmts: list[int],
34
+ out_mem_fmts: list[int],
35
+ compilation_preference: int,
36
+ relax_f32_to_f16: bool,
37
+ ):
38
+ super().__init__()
39
+ self.shape_compute_module = shape_compute_module
40
+ self.ser_model = ser_model
41
+ self.weights = weights
42
+ self.inp_mem_fmts = inp_mem_fmts
43
+ self.out_mem_fmts = out_mem_fmts
44
+ self.out_templates = []
45
+ self.comp = None
46
+ self.compilation_preference = compilation_preference
47
+ self.relax_f32_to_f16 = relax_f32_to_f16
48
+
49
+ @torch.jit.export
50
+ def init(self, args: list[torch.Tensor]):
51
+ if self.comp is not None:
52
+ raise AssertionError("comp must be None before initialization")
53
+ self.out_templates = self.shape_compute_module.prepare(self.ser_model, args) # type: ignore[operator]
54
+ self.weights = [w.contiguous() for w in self.weights]
55
+ comp = torch.classes._nnapi.Compilation()
56
+ comp.init2(
57
+ self.ser_model,
58
+ self.weights,
59
+ self.compilation_preference,
60
+ self.relax_f32_to_f16,
61
+ )
62
+
63
+ self.comp = comp
64
+
65
+ def forward(self, args: list[torch.Tensor]) -> list[torch.Tensor]:
66
+ if self.comp is None:
67
+ self.init(args)
68
+ comp = self.comp
69
+ if comp is None:
70
+ raise AssertionError("comp must not be None")
71
+ outs = [torch.empty_like(out) for out in self.out_templates]
72
+
73
+ if len(args) != len(self.inp_mem_fmts):
74
+ raise AssertionError(
75
+ f"args length {len(args)} != inp_mem_fmts length {len(self.inp_mem_fmts)}"
76
+ )
77
+ fixed_args = []
78
+ for idx in range(len(args)):
79
+ fmt = self.inp_mem_fmts[idx]
80
+ # These constants match the values in DimOrder in serializer.py
81
+ # TODO: See if it's possible to use those directly.
82
+ if fmt == 0:
83
+ fixed_args.append(args[idx].contiguous())
84
+ elif fmt == 1:
85
+ fixed_args.append(args[idx].permute(0, 2, 3, 1).contiguous())
86
+ else:
87
+ raise ValueError("Invalid mem_fmt")
88
+ comp.run(fixed_args, outs)
89
+ if len(outs) != len(self.out_mem_fmts):
90
+ raise AssertionError(
91
+ f"outs length {len(outs)} != out_mem_fmts length {len(self.out_mem_fmts)}"
92
+ )
93
+ for idx in range(len(self.out_templates)):
94
+ fmt = self.out_mem_fmts[idx]
95
+ # These constants match the values in DimOrder in serializer.py
96
+ # TODO: See if it's possible to use those directly.
97
+ if fmt in (0, 2):
98
+ pass
99
+ elif fmt == 1:
100
+ outs[idx] = outs[idx].permute(0, 3, 1, 2)
101
+ else:
102
+ raise ValueError("Invalid mem_fmt")
103
+ return outs
104
+
105
+
106
+ def convert_model_to_nnapi(
107
+ model,
108
+ inputs,
109
+ serializer=None,
110
+ return_shapes=None,
111
+ use_int16_for_qint16=False,
112
+ compilation_preference=ANEURALNETWORKS_PREFER_SUSTAINED_SPEED,
113
+ relax_f32_to_f16=False,
114
+ ):
115
+ (
116
+ shape_compute_module,
117
+ ser_model_tensor,
118
+ used_weights,
119
+ inp_mem_fmts,
120
+ out_mem_fmts,
121
+ retval_count,
122
+ ) = process_for_nnapi(
123
+ model, inputs, serializer, return_shapes, use_int16_for_qint16
124
+ )
125
+
126
+ nnapi_model = NnapiModule(
127
+ shape_compute_module,
128
+ ser_model_tensor,
129
+ used_weights,
130
+ inp_mem_fmts,
131
+ out_mem_fmts,
132
+ compilation_preference,
133
+ relax_f32_to_f16,
134
+ )
135
+
136
+ class NnapiInterfaceWrapper(torch.nn.Module):
137
+ """NNAPI list-ifying and de-list-ifying wrapper.
138
+
139
+ NNAPI always expects a list of inputs and provides a list of outputs.
140
+ This module allows us to accept inputs as separate arguments.
141
+ It returns results as either a single tensor or tuple,
142
+ matching the original module.
143
+ """
144
+
145
+ def __init__(self, mod):
146
+ super().__init__()
147
+ self.mod = mod
148
+
149
+ wrapper_model_py = NnapiInterfaceWrapper(nnapi_model)
150
+ wrapper_model = torch.jit.script(wrapper_model_py)
151
+ # TODO: Maybe make these names match the original.
152
+ arg_list = ", ".join(f"arg_{idx}" for idx in range(len(inputs)))
153
+ if retval_count < 0:
154
+ ret_expr = "retvals[0]"
155
+ else:
156
+ ret_expr = "".join(f"retvals[{idx}], " for idx in range(retval_count))
157
+ wrapper_model.define(
158
+ f"def forward(self, {arg_list}):\n"
159
+ f" retvals = self.mod([{arg_list}])\n"
160
+ f" return {ret_expr}\n"
161
+ )
162
+ return wrapper_model
163
+
164
+
165
+ def process_for_nnapi(
166
+ model, inputs, serializer=None, return_shapes=None, use_int16_for_qint16=False
167
+ ):
168
+ model = torch.jit.freeze(model)
169
+
170
+ if isinstance(inputs, torch.Tensor):
171
+ inputs = [inputs]
172
+
173
+ serializer = serializer or _NnapiSerializer(
174
+ config=None, use_int16_for_qint16=use_int16_for_qint16
175
+ )
176
+ (
177
+ ser_model,
178
+ used_weights,
179
+ inp_mem_fmts,
180
+ out_mem_fmts,
181
+ shape_compute_lines,
182
+ retval_count,
183
+ ) = serializer.serialize_model(model, inputs, return_shapes)
184
+ ser_model_tensor = torch.tensor(ser_model, dtype=torch.int32)
185
+
186
+ # We have to create a new class here every time this function is called
187
+ # because module.define adds a method to the *class*, not the instance.
188
+ class ShapeComputeModule(torch.nn.Module):
189
+ """Code-gen-ed module for tensor shape computation.
190
+
191
+ module.prepare will mutate ser_model according to the computed operand
192
+ shapes, based on the shapes of args. Returns a list of output templates.
193
+ """
194
+
195
+ shape_compute_module = torch.jit.script(ShapeComputeModule())
196
+ real_shape_compute_lines = [
197
+ "def prepare(self, ser_model: torch.Tensor, args: List[torch.Tensor]) -> List[torch.Tensor]:\n",
198
+ ] + [f" {line}\n" for line in shape_compute_lines]
199
+ shape_compute_module.define("".join(real_shape_compute_lines))
200
+
201
+ return (
202
+ shape_compute_module,
203
+ ser_model_tensor,
204
+ used_weights,
205
+ inp_mem_fmts,
206
+ out_mem_fmts,
207
+ retval_count,
208
+ )
western_ai/usa/pytorch/torch/backends/_nnapi/serializer.py ADDED
The diff for this file is too large to render. See raw diff
 
western_ai/usa/pytorch/torch/backends/cpu/__init__.py ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+
4
+ __all__ = [
5
+ "get_cpu_capability",
6
+ ]
7
+
8
+
9
+ def get_cpu_capability() -> str:
10
+ r"""Return cpu capability as a string value.
11
+
12
+ Possible values:
13
+ - "DEFAULT"
14
+ - "VSX"
15
+ - "Z VECTOR"
16
+ - "NO AVX"
17
+ - "AVX2"
18
+ - "AVX512"
19
+ - "SVE256"
20
+ """
21
+ return torch._C._get_cpu_capability()
western_ai/usa/pytorch/torch/backends/cuda/__init__.py ADDED
@@ -0,0 +1,711 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import contextlib
3
+ from typing import Any
4
+ from typing_extensions import deprecated
5
+
6
+ import torch
7
+
8
+
9
+ __all__ = [
10
+ "is_built",
11
+ "cuFFTPlanCacheAttrContextProp",
12
+ "cuFFTPlanCache",
13
+ "cuFFTPlanCacheManager",
14
+ "cuBLASModule",
15
+ "preferred_linalg_library",
16
+ "preferred_blas_library",
17
+ "cublas_workspace_size",
18
+ "cublaslt_workspace_size",
19
+ "blas_workspace_size",
20
+ "preferred_rocm_fa_library",
21
+ "is_ck_sdpa_available",
22
+ "cufft_plan_cache",
23
+ "matmul",
24
+ "SDPAParams",
25
+ "enable_cudnn_sdp",
26
+ "cudnn_sdp_enabled",
27
+ "enable_flash_sdp",
28
+ "flash_sdp_enabled",
29
+ "enable_mem_efficient_sdp",
30
+ "mem_efficient_sdp_enabled",
31
+ "math_sdp_enabled",
32
+ "enable_math_sdp",
33
+ "allow_fp16_bf16_reduction_math_sdp",
34
+ "fp16_bf16_reduction_math_sdp_allowed",
35
+ "is_flash_attention_available",
36
+ "can_use_flash_attention",
37
+ "can_use_efficient_attention",
38
+ "can_use_cudnn_attention",
39
+ "sdp_kernel",
40
+ ]
41
+
42
+
43
+ def is_built():
44
+ r"""
45
+ Return whether PyTorch is built with CUDA support.
46
+
47
+ Note that this doesn't necessarily mean CUDA is available; just that if this PyTorch
48
+ binary were run on a machine with working CUDA drivers and devices, we would be able to use it.
49
+ """
50
+ return torch._C._has_cuda
51
+
52
+
53
+ class cuFFTPlanCacheAttrContextProp:
54
+ # Like regular ContextProp, but uses the `.device_index` attribute from the
55
+ # calling object as the first argument to the getter and setter.
56
+ def __init__(self, getter, setter):
57
+ self.getter = getter
58
+ self.setter = setter
59
+
60
+ def __get__(self, obj, objtype):
61
+ return self.getter(obj.device_index)
62
+
63
+ def __set__(self, obj, val):
64
+ if isinstance(self.setter, str):
65
+ raise RuntimeError(self.setter)
66
+ self.setter(obj.device_index, val)
67
+
68
+
69
+ class cuFFTPlanCache:
70
+ r"""
71
+ Represent a specific plan cache for a specific `device_index`.
72
+
73
+ The attributes `size` and `max_size`, and method `clear`, can fetch and/ or
74
+ change properties of the C++ cuFFT plan cache.
75
+ """
76
+
77
+ def __init__(self, device_index):
78
+ self.device_index = device_index
79
+
80
+ size = cuFFTPlanCacheAttrContextProp(
81
+ torch._cufft_get_plan_cache_size,
82
+ ".size is a read-only property showing the number of plans currently in the "
83
+ "cache. To change the cache capacity, set cufft_plan_cache.max_size.",
84
+ )
85
+
86
+ max_size = cuFFTPlanCacheAttrContextProp(
87
+ torch._cufft_get_plan_cache_max_size, torch._cufft_set_plan_cache_max_size
88
+ )
89
+
90
+ def clear(self):
91
+ return torch._cufft_clear_plan_cache(self.device_index)
92
+
93
+
94
+ class cuFFTPlanCacheManager:
95
+ r"""
96
+ Represent all cuFFT plan caches, return the cuFFTPlanCache for a given device when indexed.
97
+
98
+ Finally, this object, when used directly as a `cuFFTPlanCache` object (e.g.,
99
+ setting the `.max_size`) attribute, the current device's cuFFT plan cache is
100
+ used.
101
+ """
102
+
103
+ __initialized = False
104
+
105
+ def __init__(self):
106
+ self.caches = []
107
+ self.__initialized = True
108
+
109
+ def __getitem__(self, device):
110
+ index = torch.cuda._utils._get_device_index(device)
111
+ if index < 0 or index >= torch.cuda.device_count():
112
+ raise RuntimeError(
113
+ f"cufft_plan_cache: expected 0 <= device index < {torch.cuda.device_count()}, but got "
114
+ f"device with index {index}"
115
+ )
116
+ if len(self.caches) == 0:
117
+ self.caches.extend(
118
+ cuFFTPlanCache(index) for index in range(torch.cuda.device_count())
119
+ )
120
+ return self.caches[index]
121
+
122
+ def __getattr__(self, name):
123
+ return getattr(self[torch.cuda.current_device()], name)
124
+
125
+ def __setattr__(self, name, value):
126
+ if self.__initialized:
127
+ return setattr(self[torch.cuda.current_device()], name, value)
128
+ else:
129
+ return super().__setattr__(name, value)
130
+
131
+
132
+ class cuBLASModule:
133
+ @staticmethod
134
+ def _parse_reduction_setting(value: Any, attr_name: str) -> tuple[bool, bool]:
135
+ def _ensure_bool(obj: Any, which: str) -> bool:
136
+ if isinstance(obj, bool):
137
+ return obj
138
+ raise TypeError(
139
+ f"{attr_name} expects a bool for {which}, but got {type(obj)!r}"
140
+ )
141
+
142
+ if isinstance(value, bool):
143
+ return value, True
144
+ if isinstance(value, (list, tuple)):
145
+ if not value:
146
+ raise TypeError(f"{attr_name} expects at least one boolean argument")
147
+ if len(value) > 2:
148
+ raise TypeError(f"{attr_name} expects at most two boolean arguments")
149
+ allow_reduced_precision = _ensure_bool(value[0], "allow_reduced_precision")
150
+ if len(value) == 1:
151
+ return allow_reduced_precision, True
152
+ allow_splitk = _ensure_bool(value[1], "allow_splitk")
153
+ return allow_reduced_precision, allow_splitk
154
+ raise TypeError(
155
+ f"{attr_name} expects a bool or a tuple/list of bools, but got {type(value)!r}"
156
+ )
157
+
158
+ def __getattr__(self, name):
159
+ if name == "allow_tf32":
160
+ return torch._C._get_cublas_allow_tf32()
161
+ elif name == "allow_fp16_reduced_precision_reduction":
162
+ allow_reduced_precision, _ = (
163
+ torch._C._get_cublas_allow_fp16_reduced_precision_reduction()
164
+ )
165
+ return allow_reduced_precision
166
+ elif name == "allow_fp16_reduced_precision_reduction_split_k":
167
+ _, allow_splitk = (
168
+ torch._C._get_cublas_allow_fp16_reduced_precision_reduction()
169
+ )
170
+ return allow_splitk
171
+ elif name == "allow_bf16_reduced_precision_reduction":
172
+ allow_reduced_precision, _ = (
173
+ torch._C._get_cublas_allow_bf16_reduced_precision_reduction()
174
+ )
175
+ return allow_reduced_precision
176
+ elif name == "allow_bf16_reduced_precision_reduction_split_k":
177
+ _, allow_splitk = (
178
+ torch._C._get_cublas_allow_bf16_reduced_precision_reduction()
179
+ )
180
+ return allow_splitk
181
+ elif name == "allow_fp16_accumulation":
182
+ return torch._C._get_cublas_allow_fp16_accumulation()
183
+ elif name == "fp32_precision":
184
+ return torch._C._get_fp32_precision_getter("cuda", "matmul")
185
+ raise AttributeError("Unknown attribute " + name)
186
+
187
+ def __setattr__(self, name, value):
188
+ if name == "allow_tf32":
189
+ return torch._C._set_cublas_allow_tf32(value)
190
+ elif name == "allow_fp16_reduced_precision_reduction":
191
+ allow_reduced_precision, allow_splitk = self._parse_reduction_setting(
192
+ value, "allow_fp16_reduced_precision_reduction"
193
+ )
194
+ return torch._C._set_cublas_allow_fp16_reduced_precision_reduction(
195
+ allow_reduced_precision,
196
+ allow_splitk,
197
+ )
198
+ elif name == "allow_bf16_reduced_precision_reduction":
199
+ allow_reduced_precision, allow_splitk = self._parse_reduction_setting(
200
+ value, "allow_bf16_reduced_precision_reduction"
201
+ )
202
+ return torch._C._set_cublas_allow_bf16_reduced_precision_reduction(
203
+ allow_reduced_precision,
204
+ allow_splitk,
205
+ )
206
+ elif name == "allow_fp16_accumulation":
207
+ return torch._C._set_cublas_allow_fp16_accumulation(value)
208
+ elif name == "fp32_precision":
209
+ return torch._C._set_fp32_precision_setter("cuda", "matmul", value)
210
+ raise AttributeError("Unknown attribute " + name)
211
+
212
+
213
+ _LinalgBackends = {
214
+ "default": torch._C._LinalgBackend.Default,
215
+ "cusolver": torch._C._LinalgBackend.Cusolver,
216
+ "magma": torch._C._LinalgBackend.Magma,
217
+ }
218
+ _LinalgBackends_str = ", ".join(_LinalgBackends.keys())
219
+
220
+
221
+ def preferred_linalg_library(
222
+ backend: None | str | torch._C._LinalgBackend = None,
223
+ ) -> torch._C._LinalgBackend:
224
+ r"""
225
+ Override the heuristic PyTorch uses to choose between cuSOLVER and MAGMA for CUDA linear algebra operations.
226
+
227
+ .. warning:: This flag is experimental and subject to change.
228
+
229
+ When PyTorch runs a CUDA linear algebra operation it often uses the cuSOLVER or MAGMA libraries,
230
+ and if both are available it decides which to use with a heuristic.
231
+ This flag (a :class:`str`) allows overriding those heuristics.
232
+
233
+ * If `"cusolver"` is set then cuSOLVER will be used wherever possible.
234
+ * If `"magma"` is set then MAGMA will be used wherever possible.
235
+ * If `"default"` (the default) is set then heuristics will be used to pick between
236
+ cuSOLVER and MAGMA if both are available.
237
+ * When no input is given, this function returns the currently preferred library.
238
+ * User may use the environment variable TORCH_LINALG_PREFER_CUSOLVER=1 to set the preferred library to cuSOLVER
239
+ globally.
240
+ This flag only sets the initial value of the preferred library and the preferred library
241
+ may still be overridden by this function call later in your script.
242
+
243
+ Note: When a library is preferred other libraries may still be used if the preferred library
244
+ doesn't implement the operation(s) called.
245
+ This flag may achieve better performance if PyTorch's heuristic library selection is incorrect
246
+ for your application's inputs.
247
+
248
+ Currently supported linalg operators:
249
+
250
+ * :func:`torch.linalg.inv`
251
+ * :func:`torch.linalg.inv_ex`
252
+ * :func:`torch.linalg.cholesky`
253
+ * :func:`torch.linalg.cholesky_ex`
254
+ * :func:`torch.cholesky_solve`
255
+ * :func:`torch.cholesky_inverse`
256
+ * :func:`torch.linalg.lu_factor`
257
+ * :func:`torch.linalg.lu`
258
+ * :func:`torch.linalg.lu_solve`
259
+ * :func:`torch.linalg.qr`
260
+ * :func:`torch.linalg.eigh`
261
+ * :func:`torch.linalg.eighvals`
262
+ * :func:`torch.linalg.svd`
263
+ * :func:`torch.linalg.svdvals`
264
+ """
265
+ if backend is None:
266
+ pass
267
+ elif isinstance(backend, str):
268
+ if backend not in _LinalgBackends:
269
+ raise RuntimeError(
270
+ f"Unknown input value. Choose from: {_LinalgBackends_str}."
271
+ )
272
+ torch._C._set_linalg_preferred_backend(_LinalgBackends[backend])
273
+ elif isinstance(backend, torch._C._LinalgBackend):
274
+ torch._C._set_linalg_preferred_backend(backend)
275
+ else:
276
+ raise RuntimeError("Unknown input value type.")
277
+
278
+ return torch._C._get_linalg_preferred_backend()
279
+
280
+
281
+ _BlasBackends = {
282
+ "default": torch._C._BlasBackend.Default,
283
+ "cublas": torch._C._BlasBackend.Cublas,
284
+ "hipblas": torch._C._BlasBackend.Cublas, # alias
285
+ "cublaslt": torch._C._BlasBackend.Cublaslt,
286
+ "hipblaslt": torch._C._BlasBackend.Cublaslt, # alias
287
+ "ck": torch._C._BlasBackend.Ck,
288
+ }
289
+ _BlasBackends_str = ", ".join(_BlasBackends.keys())
290
+
291
+
292
+ def preferred_blas_library(
293
+ backend: None | str | torch._C._BlasBackend = None,
294
+ ) -> torch._C._BlasBackend:
295
+ r"""
296
+ Override the library PyTorch uses for BLAS operations. Choose between cuBLAS, cuBLASLt, and CK [ROCm-only].
297
+
298
+ .. warning:: This flag is experimental and subject to change.
299
+
300
+ When PyTorch runs a CUDA BLAS operation it defaults to cuBLAS even if both cuBLAS and cuBLASLt are available.
301
+ For PyTorch built for ROCm, hipBLAS, hipBLASLt, and CK may offer different performance.
302
+ This flag (a :class:`str`) allows overriding which BLAS library to use.
303
+
304
+ * If `"cublas"` is set then cuBLAS will be used wherever possible.
305
+ * If `"cublaslt"` is set then cuBLASLt will be used wherever possible.
306
+ * If `"ck"` is set then CK will be used wherever possible.
307
+ * If `"default"` (the default) is set then heuristics will be used to pick between the other options.
308
+ * When no input is given, this function returns the currently preferred library.
309
+ * User may use the environment variable TORCH_BLAS_PREFER_CUBLASLT=1 to set the preferred library to cuBLASLt
310
+ globally.
311
+ This flag only sets the initial value of the preferred library and the preferred library
312
+ may still be overridden by this function call later in your script.
313
+
314
+ Note: When a library is preferred other libraries may still be used if the preferred library
315
+ doesn't implement the operation(s) called.
316
+ This flag may achieve better performance if PyTorch's library selection is incorrect
317
+ for your application's inputs.
318
+
319
+ """
320
+ if backend is None:
321
+ pass
322
+ elif isinstance(backend, str):
323
+ if backend not in _BlasBackends:
324
+ raise RuntimeError(
325
+ f"Unknown input value. Choose from: {_BlasBackends_str}."
326
+ )
327
+ torch._C._set_blas_preferred_backend(_BlasBackends[backend])
328
+ elif isinstance(backend, torch._C._BlasBackend):
329
+ torch._C._set_blas_preferred_backend(backend)
330
+ else:
331
+ raise RuntimeError("Unknown input value type.")
332
+
333
+ return torch._C._get_blas_preferred_backend()
334
+
335
+
336
+ def cublas_workspace_size(size: None | int = None) -> int:
337
+ r"""Query or set the cuBLAS workspace size in bytes.
338
+
339
+ When called with no arguments, returns the current workspace size.
340
+ When called with a size argument, sets the workspace size and returns the new value.
341
+ Setting the workspace size will take precedence over the CUBLAS_WORKSPACE_CONFIG environment variable.
342
+ Changes take effect lazily: only handles used after the change get new workspaces.
343
+
344
+ Args:
345
+ size (int, optional): workspace size in bytes. Must be non-negative.
346
+
347
+ Returns:
348
+ int: the current (or newly set) workspace size in bytes.
349
+ """
350
+ if size is not None:
351
+ torch._C._cuda_setCublasWorkspaceSize(size)
352
+ return torch._C._cuda_getCublasWorkspaceSize()
353
+
354
+
355
+ def cublaslt_workspace_size(size: None | int = None) -> int:
356
+ r"""Query or set the cuBLASLt workspace size in bytes.
357
+
358
+ When called with no arguments, returns the current workspace size.
359
+ When called with a size argument, sets the workspace size and returns the new value.
360
+ Setting the workspace size will take precedence over the CUBLASLT_WORKSPACE_SIZE environment variable.
361
+ Changes take effect lazily: only handles used after the change get new workspaces.
362
+
363
+ Args:
364
+ size (int, optional): workspace size in bytes. Must be non-negative.
365
+
366
+ Returns:
367
+ int: the current (or newly set) workspace size in bytes.
368
+ """
369
+ if size is not None:
370
+ torch._C._cuda_setCublasLtWorkspaceSize(size)
371
+ return torch._C._cuda_getCublasLtWorkspaceSize()
372
+
373
+
374
+ def blas_workspace_size(
375
+ size: None | int = None,
376
+ backend: None | str | torch._C._BlasBackend = None,
377
+ ) -> int:
378
+ r"""Query or set the BLAS workspace size for a given backend.
379
+
380
+ Convenience wrapper that dispatches to :func:`cublas_workspace_size` or
381
+ :func:`cublaslt_workspace_size` depending on the backend.
382
+
383
+ When *backend* is ``None`` the current :func:`preferred_blas_library` is
384
+ used. ``Default`` is resolved to the platform's default backend (cuBLAS
385
+ on NVIDIA, potentially hipBLASLt on supported ROCm architectures).
386
+
387
+ .. note::
388
+
389
+ When ``TORCH_CUBLASLT_UNIFIED_WORKSPACE`` is enabled (the default on
390
+ open-source CUDA builds), the cuBLASLt workspace is capped at the
391
+ cuBLAS workspace size and physically reuses the same allocation.
392
+ Setting a large cuBLASLt workspace via this function will therefore
393
+ *not* increase memory beyond the cuBLAS workspace size.
394
+
395
+ .. note::
396
+
397
+ Setting the workspace size for the cublas backend will take precedence
398
+ over the CUBLAS_WORKSPACE_CONFIG environment variable, and setting the
399
+ workspace size for the cublaslt backend will take precedence over the
400
+ CUBLASLT_WORKSPACE_SIZE environment variable.
401
+
402
+ Args:
403
+ size (int, optional): workspace size in bytes. Must be non-negative.
404
+ When omitted the current size is returned without modification.
405
+ backend (str | torch._C._BlasBackend, optional): which backend's
406
+ workspace to query/set. Accepts the same strings as
407
+ :func:`preferred_blas_library` (e.g. ``"cublas"``, ``"cublaslt"``).
408
+
409
+ Returns:
410
+ int: the current (or newly set) workspace size in bytes.
411
+
412
+ Raises:
413
+ RuntimeError: if the resolved backend is CK (no workspace concept).
414
+ """
415
+ if backend is None:
416
+ resolved = preferred_blas_library()
417
+ elif isinstance(backend, str):
418
+ if backend not in _BlasBackends:
419
+ raise RuntimeError(
420
+ f"Unknown backend string. Choose from: {_BlasBackends_str}."
421
+ )
422
+ resolved = _BlasBackends[backend]
423
+ elif isinstance(backend, torch._C._BlasBackend):
424
+ resolved = backend
425
+ else:
426
+ raise RuntimeError("Unknown backend type.")
427
+
428
+ if resolved == torch._C._BlasBackend.Default:
429
+ resolved = torch._C._get_blas_default_backend()
430
+
431
+ if resolved == torch._C._BlasBackend.Ck:
432
+ raise RuntimeError("CK backend does not use a workspace.")
433
+
434
+ if resolved == torch._C._BlasBackend.Cublaslt:
435
+ return cublaslt_workspace_size(size)
436
+ return cublas_workspace_size(size)
437
+
438
+
439
+ _ROCmFABackends = {
440
+ "default": torch._C._ROCmFABackend.Default,
441
+ "aotriton": torch._C._ROCmFABackend.AOTriton,
442
+ "ck": torch._C._ROCmFABackend.Ck,
443
+ }
444
+ _ROCmFABackends_str = ", ".join(_ROCmFABackends.keys())
445
+
446
+
447
+ from torch._C import _SDPAParams as SDPAParams, _SDPBackend as SDPBackend
448
+
449
+
450
+ def preferred_rocm_fa_library(
451
+ backend: None | str | torch._C._ROCmFABackend = None,
452
+ ) -> torch._C._ROCmFABackend:
453
+ r"""
454
+ [ROCm-only]
455
+ Override the backend PyTorch uses in ROCm environments for Flash Attention. Choose between AOTriton and CK
456
+
457
+ .. warning:: This flag is experimental and subject to change.
458
+
459
+ When Flash Attention is enabled and desired, PyTorch defaults to using AOTriton as the backend.
460
+ This flag (a :class:`str`) allows users to override this backend to use composable_kernel
461
+
462
+ * If `"default"` is set then the default backend will be used wherever possible. Currently AOTriton.
463
+ * If `"aotriton"` is set then AOTriton will be used wherever possible.
464
+ * If `"ck"` is set then CK will be used wherever possible.
465
+ * When no input is given, this function returns the currently preferred library.
466
+ * User may use the environment variable TORCH_ROCM_FA_PREFER_CK=1 to set the preferred library to CK
467
+ globally.
468
+
469
+ Note: When a library is preferred other libraries may still be used if the preferred library
470
+ doesn't implement the operation(s) called.
471
+ This flag may achieve better performance if PyTorch's library selection is incorrect
472
+ for your application's inputs.
473
+ """
474
+ if backend is None:
475
+ pass
476
+ elif isinstance(backend, str):
477
+ if backend not in _ROCmFABackends:
478
+ raise RuntimeError(
479
+ f"Unknown input value. Choose from: {_ROCmFABackends_str}."
480
+ )
481
+ torch._C._set_rocm_fa_preferred_backend(_ROCmFABackends[backend])
482
+ elif isinstance(backend, torch._C._ROCmFABackend):
483
+ torch._C._set_rocm_fa_preferred_backend(backend)
484
+ else:
485
+ raise ValueError(f"Unknown input value. Choose from: {_ROCmFABackends_str}.")
486
+
487
+ return torch._C._get_rocm_fa_preferred_backend()
488
+
489
+
490
+ # Set the __module__ attribute
491
+ SDPAParams.__module__ = "torch.backends.cuda"
492
+ SDPAParams.__name__ = "SDPAParams"
493
+
494
+
495
+ def is_ck_sdpa_available() -> bool:
496
+ r"""
497
+ .. warning:: This flag is beta and subject to change.
498
+
499
+ Returns whether composable_kernel may be used as the backend for
500
+ scaled-dot-product-attention.
501
+ """
502
+ # pyrefly: ignore [missing-attribute]
503
+ return torch._C._is_ck_sdpa_available()
504
+
505
+
506
+ def flash_sdp_enabled():
507
+ r"""
508
+ .. warning:: This flag is beta and subject to change.
509
+
510
+ Returns whether flash scaled dot product attention is enabled or not.
511
+ """
512
+ return torch._C._get_flash_sdp_enabled()
513
+
514
+
515
+ def enable_flash_sdp(enabled: bool):
516
+ r"""
517
+ .. warning:: This flag is beta and subject to change.
518
+
519
+ Enables or disables flash scaled dot product attention.
520
+ """
521
+ torch._C._set_sdp_use_flash(enabled)
522
+
523
+
524
+ def mem_efficient_sdp_enabled():
525
+ r"""
526
+ .. warning:: This flag is beta and subject to change.
527
+
528
+ Returns whether memory efficient scaled dot product attention is enabled or not.
529
+ """
530
+ return torch._C._get_mem_efficient_sdp_enabled()
531
+
532
+
533
+ def enable_mem_efficient_sdp(enabled: bool):
534
+ r"""
535
+ .. warning:: This flag is beta and subject to change.
536
+
537
+ Enables or disables memory efficient scaled dot product attention.
538
+ """
539
+ torch._C._set_sdp_use_mem_efficient(enabled)
540
+
541
+
542
+ def math_sdp_enabled():
543
+ r"""
544
+ .. warning:: This flag is beta and subject to change.
545
+
546
+ Returns whether math scaled dot product attention is enabled or not.
547
+ """
548
+ return torch._C._get_math_sdp_enabled()
549
+
550
+
551
+ def enable_math_sdp(enabled: bool):
552
+ r"""
553
+ .. warning:: This flag is beta and subject to change.
554
+
555
+ Enables or disables math scaled dot product attention.
556
+ """
557
+ torch._C._set_sdp_use_math(enabled)
558
+
559
+
560
+ def allow_fp16_bf16_reduction_math_sdp(enabled: bool):
561
+ r"""
562
+ .. warning:: This flag is beta and subject to change.
563
+
564
+ Enables or disables fp16/bf16 reduction in math scaled dot product attention.
565
+ """
566
+ torch._C._set_math_sdp_allow_fp16_bf16_reduction(enabled)
567
+
568
+
569
+ def fp16_bf16_reduction_math_sdp_allowed():
570
+ r"""
571
+ .. warning:: This flag is beta and subject to change.
572
+
573
+ Returns whether fp16/bf16 reduction in math scaled dot product attention is enabled or not.
574
+ """
575
+ return torch._C._get_math_sdp_allow_fp16_bf16_reduction()
576
+
577
+
578
+ def is_flash_attention_available() -> bool:
579
+ r"""Check if PyTorch was built with FlashAttention for scaled_dot_product_attention.
580
+
581
+ Returns:
582
+ True if FlashAttention is built and available; otherwise, False.
583
+
584
+ Note:
585
+ This function is dependent on a CUDA-enabled build of PyTorch. It will return False
586
+ in non-CUDA environments.
587
+ """
588
+ return torch._C._is_flash_attention_available()
589
+
590
+
591
+ def can_use_flash_attention(params: SDPAParams, debug: bool = False) -> bool:
592
+ r"""Check if FlashAttention can be utilized in scaled_dot_product_attention.
593
+
594
+ Args:
595
+ params: An instance of SDPAParams containing the tensors for query,
596
+ key, value, an optional attention mask, dropout rate, and
597
+ a flag indicating if the attention is causal.
598
+ debug: Whether to logging.warn debug information as to why FlashAttention could not be run.
599
+ Defaults to False.
600
+
601
+ Returns:
602
+ True if FlashAttention can be used with the given parameters; otherwise, False.
603
+
604
+ Note:
605
+ This function is dependent on a CUDA-enabled build of PyTorch. It will return False
606
+ in non-CUDA environments.
607
+ """
608
+ return torch._C._can_use_flash_attention(params, debug)
609
+
610
+
611
+ def can_use_efficient_attention(params: SDPAParams, debug: bool = False) -> bool:
612
+ r"""Check if efficient_attention can be utilized in scaled_dot_product_attention.
613
+
614
+ Args:
615
+ params: An instance of SDPAParams containing the tensors for query,
616
+ key, value, an optional attention mask, dropout rate, and
617
+ a flag indicating if the attention is causal.
618
+ debug: Whether to logging.warn with information as to why efficient_attention could not be run.
619
+ Defaults to False.
620
+
621
+ Returns:
622
+ True if efficient_attention can be used with the given parameters; otherwise, False.
623
+
624
+ Note:
625
+ This function is dependent on a CUDA-enabled build of PyTorch. It will return False
626
+ in non-CUDA environments.
627
+ """
628
+ return torch._C._can_use_mem_efficient_attention(params, debug)
629
+
630
+
631
+ def can_use_cudnn_attention(params: SDPAParams, debug: bool = False) -> bool:
632
+ r"""Check if cudnn_attention can be utilized in scaled_dot_product_attention.
633
+
634
+ Args:
635
+ params: An instance of SDPAParams containing the tensors for query,
636
+ key, value, an optional attention mask, dropout rate, and
637
+ a flag indicating if the attention is causal.
638
+ debug: Whether to logging.warn with information as to why cuDNN attention could not be run.
639
+ Defaults to False.
640
+
641
+ Returns:
642
+ True if cuDNN can be used with the given parameters; otherwise, False.
643
+
644
+ Note:
645
+ This function is dependent on a CUDA-enabled build of PyTorch. It will return False
646
+ in non-CUDA environments.
647
+ """
648
+ return torch._C._can_use_cudnn_attention(params, debug)
649
+
650
+
651
+ def cudnn_sdp_enabled():
652
+ r"""
653
+ .. warning:: This flag is beta and subject to change.
654
+
655
+ Returns whether cuDNN scaled dot product attention is enabled or not.
656
+ """
657
+ return torch._C._get_cudnn_sdp_enabled()
658
+
659
+
660
+ def enable_cudnn_sdp(enabled: bool):
661
+ r"""
662
+ .. warning:: This flag is beta and subject to change.
663
+
664
+ Enables or disables cuDNN scaled dot product attention.
665
+ """
666
+ torch._C._set_sdp_use_cudnn(enabled)
667
+
668
+
669
+ @contextlib.contextmanager
670
+ @deprecated(
671
+ (
672
+ "`torch.backends.cuda.sdp_kernel()` is deprecated. "
673
+ "In the future, this context manager will be removed. "
674
+ "Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, "
675
+ "with updated signature."
676
+ ),
677
+ category=FutureWarning,
678
+ )
679
+ def sdp_kernel(
680
+ enable_flash: bool = True,
681
+ enable_math: bool = True,
682
+ enable_mem_efficient: bool = True,
683
+ enable_cudnn: bool = True,
684
+ ):
685
+ r"""
686
+ .. warning:: This flag is beta and subject to change.
687
+
688
+ This context manager can be used to temporarily enable or disable any of the three backends for scaled dot product attention.
689
+ Upon exiting the context manager, the previous state of the flags will be restored.
690
+ """
691
+ from torch.nn.attention import sdpa_kernel
692
+
693
+ backend_list = []
694
+ if enable_flash:
695
+ backend_list.append(SDPBackend.FLASH_ATTENTION)
696
+ if enable_mem_efficient:
697
+ backend_list.append(SDPBackend.EFFICIENT_ATTENTION)
698
+ if enable_math:
699
+ backend_list.append(SDPBackend.MATH)
700
+ if enable_cudnn:
701
+ backend_list.append(SDPBackend.CUDNN_ATTENTION)
702
+
703
+ with sdpa_kernel(backend_list) as context:
704
+ try:
705
+ yield context
706
+ finally:
707
+ pass
708
+
709
+
710
+ cufft_plan_cache = cuFFTPlanCacheManager()
711
+ matmul = cuBLASModule()
western_ai/usa/pytorch/torch/backends/cudnn/__init__.py ADDED
@@ -0,0 +1,261 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import os
3
+ import sys
4
+ import warnings
5
+ from contextlib import contextmanager
6
+ from typing import Optional
7
+
8
+ import torch
9
+ from torch.backends import (
10
+ __allow_nonbracketed_mutation,
11
+ _FP32Precision,
12
+ _get_fp32_precision_getter,
13
+ _set_fp32_precision_setter,
14
+ ContextProp,
15
+ PropModule,
16
+ )
17
+
18
+ from . import rnn
19
+
20
+
21
+ try:
22
+ from torch._C import _cudnn
23
+ except ImportError:
24
+ _cudnn = None # type: ignore[assignment]
25
+
26
+ # Write:
27
+ #
28
+ # torch.backends.cudnn.enabled = False
29
+ #
30
+ # to globally disable CuDNN/MIOpen
31
+
32
+ __cudnn_version: int | None = None
33
+
34
+ if _cudnn is not None:
35
+
36
+ def _init():
37
+ global __cudnn_version
38
+ if __cudnn_version is None:
39
+ # pyrefly: ignore [missing-attribute]
40
+ __cudnn_version = _cudnn.getVersionInt()
41
+ # pyrefly: ignore [missing-attribute]
42
+ runtime_version = _cudnn.getRuntimeVersion()
43
+ # pyrefly: ignore [missing-attribute]
44
+ compile_version = _cudnn.getCompileVersion()
45
+ runtime_major, runtime_minor, _ = runtime_version
46
+ compile_major, compile_minor, _ = compile_version
47
+ # Different major versions are always incompatible
48
+ # Starting with cuDNN 7, minor versions are backwards-compatible
49
+ # Not sure about MIOpen (ROCm), so always do a strict check
50
+ if runtime_major != compile_major:
51
+ cudnn_compatible = False
52
+ # pyrefly: ignore [missing-attribute]
53
+ elif runtime_major < 7 or not _cudnn.is_cuda:
54
+ cudnn_compatible = runtime_minor == compile_minor
55
+ else:
56
+ cudnn_compatible = runtime_minor >= compile_minor
57
+ if not cudnn_compatible:
58
+ if os.environ.get("PYTORCH_SKIP_CUDNN_COMPATIBILITY_CHECK", "0") == "1":
59
+ return True
60
+ base_error_msg = (
61
+ f"cuDNN version incompatibility: "
62
+ f"PyTorch was compiled against {compile_version} "
63
+ f"but found runtime version {runtime_version}. "
64
+ f"PyTorch already comes bundled with cuDNN. "
65
+ f"One option to resolving this error is to ensure PyTorch "
66
+ f"can find the bundled cuDNN. "
67
+ )
68
+
69
+ if "LD_LIBRARY_PATH" in os.environ:
70
+ ld_library_path = os.environ.get("LD_LIBRARY_PATH", "")
71
+ if any(
72
+ substring in ld_library_path for substring in ["cuda", "cudnn"]
73
+ ):
74
+ raise RuntimeError(
75
+ f"{base_error_msg}"
76
+ f"Looks like your LD_LIBRARY_PATH contains incompatible version of cudnn. "
77
+ f"Please either remove it from the path or install cudnn {compile_version}"
78
+ )
79
+ else:
80
+ raise RuntimeError(
81
+ f"{base_error_msg}"
82
+ f"one possibility is that there is a "
83
+ f"conflicting cuDNN in LD_LIBRARY_PATH."
84
+ )
85
+ else:
86
+ raise RuntimeError(base_error_msg)
87
+ # Check if cuDNN version is compatible with available CUDA devices
88
+ if torch.cuda.is_available() and not torch.version.hip:
89
+ min_cc = min(
90
+ [
91
+ torch.cuda.get_device_capability(i)
92
+ for i in range(torch.cuda.device_count())
93
+ ]
94
+ )
95
+ if __cudnn_version >= 91100 and min_cc < (7, 5):
96
+ raise RuntimeError(
97
+ f"cuDNN version {__cudnn_version} is not compatible with devices with SM < 7.5. "
98
+ f"Please install a version of PyTorch with a compatible cuDNN version. "
99
+ f"https://github.com/pytorch/pytorch/blob/main/RELEASE.md#release-compatibility-matrix"
100
+ )
101
+
102
+ return True
103
+
104
+ else:
105
+
106
+ def _init():
107
+ return False
108
+
109
+
110
+ def version():
111
+ """Return the version of cuDNN."""
112
+ if not _init():
113
+ return None
114
+ return __cudnn_version
115
+
116
+
117
+ CUDNN_TENSOR_DTYPES = {
118
+ torch.half,
119
+ torch.float,
120
+ torch.double,
121
+ }
122
+
123
+
124
+ def is_available():
125
+ r"""Return a bool indicating if CUDNN is currently available."""
126
+ return torch._C._has_cudnn
127
+
128
+
129
+ def is_acceptable(tensor):
130
+ if not torch._C._get_cudnn_enabled():
131
+ return False
132
+ if tensor.device.type != "cuda" or tensor.dtype not in CUDNN_TENSOR_DTYPES:
133
+ return False
134
+ if not is_available():
135
+ warnings.warn(
136
+ "PyTorch was compiled without cuDNN/MIOpen support. To use cuDNN/MIOpen, rebuild "
137
+ "PyTorch making sure the library is visible to the build system.",
138
+ stacklevel=2,
139
+ )
140
+ return False
141
+ if not _init():
142
+ warnings.warn(
143
+ "cuDNN/MIOpen library not found. Check your {libpath}".format(
144
+ libpath={"darwin": "DYLD_LIBRARY_PATH", "win32": "PATH"}.get(
145
+ sys.platform, "LD_LIBRARY_PATH"
146
+ )
147
+ ),
148
+ stacklevel=2,
149
+ )
150
+ return False
151
+ return True
152
+
153
+
154
+ def set_flags(
155
+ _enabled=None,
156
+ _benchmark=None,
157
+ _benchmark_limit=None,
158
+ _deterministic=None,
159
+ _allow_tf32=None,
160
+ _fp32_precision="none",
161
+ _depthwise_kernel=None,
162
+ ):
163
+ orig_flags = (
164
+ torch._C._get_cudnn_enabled(),
165
+ torch._C._get_cudnn_benchmark(),
166
+ None if not is_available() else torch._C._cuda_get_cudnn_benchmark_limit(),
167
+ torch._C._get_cudnn_deterministic(),
168
+ torch._C._get_cudnn_allow_tf32(),
169
+ torch._C._get_fp32_precision_getter("cuda", "all"),
170
+ torch._C._get_cudnn_depthwise_kernel(),
171
+ )
172
+ if _enabled is not None:
173
+ torch._C._set_cudnn_enabled(_enabled)
174
+ if _benchmark is not None:
175
+ torch._C._set_cudnn_benchmark(_benchmark)
176
+ if _benchmark_limit is not None and is_available():
177
+ torch._C._cuda_set_cudnn_benchmark_limit(_benchmark_limit)
178
+ if _deterministic is not None:
179
+ torch._C._set_cudnn_deterministic(_deterministic)
180
+ if _allow_tf32 is not None:
181
+ torch._C._set_cudnn_allow_tf32(_allow_tf32)
182
+ if _fp32_precision is not None:
183
+ torch._C._set_fp32_precision_setter("cuda", "all", _fp32_precision)
184
+ if _depthwise_kernel is not None:
185
+ torch._C._set_cudnn_depthwise_kernel(_depthwise_kernel)
186
+ return orig_flags
187
+
188
+
189
+ @contextmanager
190
+ def flags(
191
+ enabled=False,
192
+ benchmark=False,
193
+ benchmark_limit=10,
194
+ deterministic=False,
195
+ allow_tf32=True,
196
+ fp32_precision="none",
197
+ depthwise_kernel="auto",
198
+ ):
199
+ with __allow_nonbracketed_mutation():
200
+ orig_flags = set_flags(
201
+ enabled,
202
+ benchmark,
203
+ benchmark_limit,
204
+ deterministic,
205
+ allow_tf32,
206
+ fp32_precision,
207
+ depthwise_kernel,
208
+ )
209
+ try:
210
+ yield
211
+ finally:
212
+ # recover the previous values
213
+ with __allow_nonbracketed_mutation():
214
+ set_flags(*orig_flags)
215
+
216
+
217
+ # The magic here is to allow us to intercept code like this:
218
+ #
219
+ # torch.backends.<cudnn|mkldnn>.enabled = True
220
+
221
+
222
+ class CudnnModule(PropModule):
223
+ enabled = ContextProp(torch._C._get_cudnn_enabled, torch._C._set_cudnn_enabled)
224
+ deterministic = ContextProp(
225
+ torch._C._get_cudnn_deterministic, torch._C._set_cudnn_deterministic
226
+ )
227
+ benchmark = ContextProp(
228
+ torch._C._get_cudnn_benchmark, torch._C._set_cudnn_benchmark
229
+ )
230
+ benchmark_limit = None
231
+ if is_available():
232
+ benchmark_limit = ContextProp(
233
+ torch._C._cuda_get_cudnn_benchmark_limit,
234
+ torch._C._cuda_set_cudnn_benchmark_limit,
235
+ )
236
+ allow_tf32 = ContextProp(
237
+ torch._C._get_cudnn_allow_tf32, torch._C._set_cudnn_allow_tf32
238
+ )
239
+ conv = _FP32Precision("cuda", "conv")
240
+ fp32_precision = ContextProp(
241
+ _get_fp32_precision_getter("cuda", "all"),
242
+ _set_fp32_precision_setter("cuda", "all"),
243
+ )
244
+ depthwise_kernel = ContextProp(
245
+ torch._C._get_cudnn_depthwise_kernel,
246
+ torch._C._set_cudnn_depthwise_kernel,
247
+ )
248
+
249
+
250
+ # This is the sys.modules replacement trick, see
251
+ # https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273
252
+ sys.modules[__name__] = CudnnModule(sys.modules[__name__], __name__)
253
+
254
+ # Add type annotation for the replaced module
255
+ enabled: bool
256
+ deterministic: bool
257
+ benchmark: bool
258
+ allow_tf32: bool
259
+ fp32_precision: str
260
+ benchmark_limit: int
261
+ depthwise_kernel: str
western_ai/usa/pytorch/torch/backends/cudnn/rnn.py ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import sys
3
+
4
+ import torch._C
5
+ import torch.cuda
6
+ from torch.backends import (
7
+ _get_fp32_precision_getter,
8
+ _set_fp32_precision_setter,
9
+ PropModule,
10
+ )
11
+
12
+
13
+ try:
14
+ from torch._C import _cudnn
15
+ except ImportError:
16
+ # Uses of all the functions below should be guarded by torch.backends.cudnn.is_available(),
17
+ # so it's safe to not emit any checks here.
18
+ _cudnn = None # type: ignore[assignment]
19
+
20
+
21
+ def get_cudnn_mode(mode):
22
+ if mode == "RNN_RELU":
23
+ # pyrefly: ignore [missing-attribute]
24
+ return int(_cudnn.RNNMode.rnn_relu)
25
+ elif mode == "RNN_TANH":
26
+ # pyrefly: ignore [missing-attribute]
27
+ return int(_cudnn.RNNMode.rnn_tanh)
28
+ elif mode == "LSTM":
29
+ # pyrefly: ignore [missing-attribute]
30
+ return int(_cudnn.RNNMode.lstm)
31
+ elif mode == "GRU":
32
+ # pyrefly: ignore [missing-attribute]
33
+ return int(_cudnn.RNNMode.gru)
34
+ else:
35
+ raise ValueError(f"Unknown mode: {mode}")
36
+
37
+
38
+ # NB: We don't actually need this class anymore (in fact, we could serialize the
39
+ # dropout state for even better reproducibility), but it is kept for backwards
40
+ # compatibility for old models.
41
+ class Unserializable:
42
+ def __init__(self, inner):
43
+ self.inner = inner
44
+
45
+ def get(self):
46
+ return self.inner
47
+
48
+ def __getstate__(self):
49
+ return "<unserializable>"
50
+
51
+ def __setstate__(self, state):
52
+ self.inner = None
53
+
54
+
55
+ # we would like to use ContextProp from backends here but the
56
+ # frozen flags appears to be overzealous
57
+ class ContextProp:
58
+ def __init__(self, getter, setter):
59
+ self.getter = getter
60
+ self.setter = setter
61
+
62
+ def __get__(self, obj, objtype):
63
+ return self.getter()
64
+
65
+ def __set__(self, obj, val):
66
+ self.setter(val)
67
+
68
+
69
+ def init_dropout_state(dropout, train, dropout_seed, dropout_state):
70
+ dropout_desc_name = "desc_" + str(torch.cuda.current_device())
71
+ dropout_p = dropout if train else 0
72
+ if (dropout_desc_name not in dropout_state) or (
73
+ dropout_state[dropout_desc_name].get() is None
74
+ ):
75
+ if dropout_p == 0:
76
+ dropout_state[dropout_desc_name] = Unserializable(None)
77
+ else:
78
+ dropout_state[dropout_desc_name] = Unserializable(
79
+ torch._cudnn_init_dropout_state( # type: ignore[call-arg]
80
+ dropout_p,
81
+ train,
82
+ dropout_seed,
83
+ # pyrefly: ignore [unexpected-keyword]
84
+ self_ty=torch.uint8,
85
+ device=torch.device("cuda"),
86
+ )
87
+ )
88
+ dropout_ts = dropout_state[dropout_desc_name].get()
89
+ return dropout_ts
90
+
91
+
92
+ class CudnnRNNModule(PropModule):
93
+ def __init__(self, m, name):
94
+ super().__init__(m, name)
95
+ self.m.Unserializable = Unserializable
96
+ self.m.get_cudnn_mode = get_cudnn_mode
97
+ self.m.init_dropout_state = init_dropout_state
98
+
99
+ @staticmethod
100
+ def init_dropout_state(dropout, train, dropout_seed, dropout_state):
101
+ dropout_desc_name = "desc_" + str(torch.cuda.current_device())
102
+ dropout_p = dropout if train else 0
103
+ if (dropout_desc_name not in dropout_state) or (
104
+ dropout_state[dropout_desc_name].get() is None
105
+ ):
106
+ if dropout_p == 0:
107
+ dropout_state[dropout_desc_name] = Unserializable(None)
108
+ else:
109
+ dropout_state[dropout_desc_name] = Unserializable(
110
+ torch._cudnn_init_dropout_state( # type: ignore[call-arg]
111
+ dropout_p,
112
+ train,
113
+ dropout_seed,
114
+ # pyrefly: ignore [unexpected-keyword]
115
+ self_ty=torch.uint8,
116
+ device=torch.device("cuda"),
117
+ )
118
+ )
119
+ dropout_ts = dropout_state[dropout_desc_name].get()
120
+ return dropout_ts
121
+
122
+ fp32_precision = ContextProp(
123
+ _get_fp32_precision_getter("cuda", "rnn"),
124
+ _set_fp32_precision_setter("cuda", "rnn"),
125
+ )
126
+
127
+
128
+ sys.modules[__name__] = CudnnRNNModule(sys.modules[__name__], __name__)
western_ai/usa/pytorch/torch/backends/cusparselt/__init__.py ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional
2
+
3
+ import torch
4
+
5
+
6
+ __all__ = [
7
+ "version",
8
+ "is_available",
9
+ "get_max_alg_id",
10
+ ]
11
+
12
+ try:
13
+ from torch._C import _cusparselt
14
+ except ImportError:
15
+ _cusparselt = None # type: ignore[assignment]
16
+
17
+ __cusparselt_version: int | None = None
18
+ __MAX_ALG_ID: int | None = None
19
+
20
+ if _cusparselt is not None:
21
+
22
+ def _init() -> bool:
23
+ global __cusparselt_version
24
+ global __MAX_ALG_ID
25
+ if __cusparselt_version is None:
26
+ # pyrefly: ignore [missing-attribute]
27
+ __cusparselt_version = _cusparselt.getVersionInt()
28
+ if __cusparselt_version == 400:
29
+ __MAX_ALG_ID = 4
30
+ elif __cusparselt_version == 502:
31
+ __MAX_ALG_ID = 5
32
+ elif __cusparselt_version == 602:
33
+ __MAX_ALG_ID = 37
34
+ return True
35
+
36
+ else:
37
+
38
+ def _init() -> bool:
39
+ return False
40
+
41
+
42
+ def version() -> int | None:
43
+ """Return the version of cuSPARSELt"""
44
+ if not _init():
45
+ return None
46
+ return __cusparselt_version
47
+
48
+
49
+ def is_available() -> bool:
50
+ r"""Return a bool indicating if cuSPARSELt is currently available."""
51
+ return torch._C._has_cusparselt
52
+
53
+
54
+ def get_max_alg_id() -> int | None:
55
+ if not _init():
56
+ return None
57
+ return __MAX_ALG_ID
western_ai/usa/pytorch/torch/backends/kleidiai/__init__.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import torch
3
+
4
+
5
+ def is_available():
6
+ r"""Return whether PyTorch is built with KleidiAI support."""
7
+ return torch._C._has_kleidiai
western_ai/usa/pytorch/torch/backends/mha/__init__.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Config options to enable/disable C++ kernel for nn.functional.MHA
2
+ # and nn.TransformerEncoder
3
+ import torch
4
+
5
+
6
+ _is_fastpath_enabled: bool = True
7
+
8
+
9
+ def get_fastpath_enabled() -> bool:
10
+ """Returns whether fast path for TransformerEncoder and MultiHeadAttention
11
+ is enabled, or ``True`` if jit is scripting.
12
+
13
+ .. note::
14
+ The fastpath might not be run even if ``get_fastpath_enabled`` returns
15
+ ``True`` unless all conditions on inputs are met.
16
+ """
17
+ if not torch.jit.is_scripting():
18
+ return _is_fastpath_enabled
19
+ return True
20
+
21
+
22
+ def set_fastpath_enabled(value: bool) -> None:
23
+ """Sets whether fast path is enabled"""
24
+ global _is_fastpath_enabled
25
+ _is_fastpath_enabled = value
western_ai/usa/pytorch/torch/backends/miopen/__init__.py ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import sys
3
+ from contextlib import contextmanager
4
+
5
+ import torch
6
+ from torch.backends import __allow_nonbracketed_mutation, ContextProp, PropModule
7
+
8
+
9
+ def set_flags(
10
+ _immediate=None,
11
+ ):
12
+ orig_flags = (torch._C._get_miopen_immediate(),)
13
+ if _immediate is not None:
14
+ torch._C._set_miopen_immediate(_immediate)
15
+ return orig_flags
16
+
17
+
18
+ @contextmanager
19
+ def flags(
20
+ immediate=False,
21
+ ):
22
+ with __allow_nonbracketed_mutation():
23
+ orig_flags = set_flags(
24
+ immediate,
25
+ )
26
+ try:
27
+ yield
28
+ finally:
29
+ # recover the previous values
30
+ with __allow_nonbracketed_mutation():
31
+ set_flags(*orig_flags)
32
+
33
+
34
+ # The magic here is to allow us to intercept code like this:
35
+ #
36
+ # torch.backends.<miopen|mkldnn>.immediate = True
37
+
38
+
39
+ class MiopenModule(PropModule):
40
+ immediate = ContextProp(
41
+ torch._C._get_miopen_immediate, torch._C._set_miopen_immediate
42
+ )
43
+
44
+
45
+ # This is the sys.modules replacement trick, see
46
+ # https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273
47
+ sys.modules[__name__] = MiopenModule(sys.modules[__name__], __name__)
48
+
49
+ # Add type annotation for the replaced module
50
+ immediate: bool
western_ai/usa/pytorch/torch/backends/mkl/__init__.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import torch
3
+
4
+
5
+ def is_available():
6
+ r"""Return whether PyTorch is built with MKL support."""
7
+ return torch._C.has_mkl
8
+
9
+
10
+ VERBOSE_OFF = 0
11
+ VERBOSE_ON = 1
12
+
13
+
14
+ class verbose:
15
+ """
16
+ On-demand oneMKL verbosing functionality.
17
+
18
+ To make it easier to debug performance issues, oneMKL can dump verbose
19
+ messages containing execution information like duration while executing
20
+ the kernel. The verbosing functionality can be invoked via an environment
21
+ variable named `MKL_VERBOSE`. However, this methodology dumps messages in
22
+ all steps. Those are a large amount of verbose messages. Moreover, for
23
+ investigating the performance issues, generally taking verbose messages
24
+ for one single iteration is enough. This on-demand verbosing functionality
25
+ makes it possible to control scope for verbose message dumping. In the
26
+ following example, verbose messages will be dumped out for the second
27
+ inference only.
28
+
29
+ .. highlight:: python
30
+ .. code-block:: python
31
+
32
+ import torch
33
+
34
+ model(data)
35
+ with torch.backends.mkl.verbose(torch.backends.mkl.VERBOSE_ON):
36
+ model(data)
37
+
38
+ Args:
39
+ level: Verbose level
40
+ - ``VERBOSE_OFF``: Disable verbosing
41
+ - ``VERBOSE_ON``: Enable verbosing
42
+ """
43
+
44
+ def __init__(self, enable):
45
+ self.enable = enable
46
+
47
+ def __enter__(self):
48
+ if self.enable == VERBOSE_OFF:
49
+ return
50
+ st = torch._C._verbose.mkl_set_verbose(self.enable)
51
+ if not st:
52
+ raise AssertionError(
53
+ "Failed to set MKL into verbose mode. Please consider to disable this verbose scope."
54
+ )
55
+ return self
56
+
57
+ def __exit__(self, exc_type, exc_val, exc_tb):
58
+ torch._C._verbose.mkl_set_verbose(VERBOSE_OFF)
59
+ return False
western_ai/usa/pytorch/torch/backends/mkldnn/__init__.py ADDED
@@ -0,0 +1,139 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import sys
3
+ from contextlib import contextmanager
4
+ from typing import TYPE_CHECKING
5
+
6
+ import torch
7
+ from torch.backends import (
8
+ __allow_nonbracketed_mutation,
9
+ _FP32Precision,
10
+ _get_fp32_precision_getter,
11
+ _set_fp32_precision_setter,
12
+ ContextProp,
13
+ PropModule,
14
+ )
15
+
16
+
17
+ def is_available():
18
+ r"""Return whether PyTorch is built with MKL-DNN support."""
19
+ return torch._C._has_mkldnn
20
+
21
+
22
+ def is_acl_available():
23
+ r"""Return whether PyTorch is built with MKL-DNN + ACL support."""
24
+ # pyrefly: ignore [missing-attribute]
25
+ return torch._C._has_mkldnn_acl
26
+
27
+
28
+ VERBOSE_OFF = 0
29
+ VERBOSE_ON = 1
30
+ VERBOSE_ON_CREATION = 2
31
+
32
+
33
+ class verbose:
34
+ """
35
+ On-demand oneDNN (former MKL-DNN) verbosing functionality.
36
+
37
+ To make it easier to debug performance issues, oneDNN can dump verbose
38
+ messages containing information like kernel size, input data size and
39
+ execution duration while executing the kernel. The verbosing functionality
40
+ can be invoked via an environment variable named `DNNL_VERBOSE`. However,
41
+ this methodology dumps messages in all steps. Those are a large amount of
42
+ verbose messages. Moreover, for investigating the performance issues,
43
+ generally taking verbose messages for one single iteration is enough.
44
+ This on-demand verbosing functionality makes it possible to control scope
45
+ for verbose message dumping. In the following example, verbose messages
46
+ will be dumped out for the second inference only.
47
+
48
+ .. highlight:: python
49
+ .. code-block:: python
50
+
51
+ import torch
52
+
53
+ model(data)
54
+ with torch.backends.mkldnn.verbose(torch.backends.mkldnn.VERBOSE_ON):
55
+ model(data)
56
+
57
+ Args:
58
+ level: Verbose level
59
+ - ``VERBOSE_OFF``: Disable verbosing
60
+ - ``VERBOSE_ON``: Enable verbosing
61
+ - ``VERBOSE_ON_CREATION``: Enable verbosing, including oneDNN kernel creation
62
+ """
63
+
64
+ def __init__(self, level):
65
+ self.level = level
66
+
67
+ def __enter__(self):
68
+ if self.level == VERBOSE_OFF:
69
+ return
70
+ st = torch._C._verbose.mkldnn_set_verbose(self.level)
71
+ if not st:
72
+ raise AssertionError(
73
+ "Failed to set MKLDNN into verbose mode. Please consider to disable this verbose scope."
74
+ )
75
+ return self
76
+
77
+ def __exit__(self, exc_type, exc_val, exc_tb):
78
+ torch._C._verbose.mkldnn_set_verbose(VERBOSE_OFF)
79
+ return False
80
+
81
+
82
+ def set_flags(
83
+ _enabled=None, _deterministic=None, _allow_tf32=None, _fp32_precision="none"
84
+ ):
85
+ orig_flags = (
86
+ torch._C._get_mkldnn_enabled(),
87
+ torch._C._get_mkldnn_deterministic(),
88
+ torch._C._get_onednn_allow_tf32(),
89
+ torch._C._get_fp32_precision_getter("mkldnn", "all"),
90
+ )
91
+ if _enabled is not None:
92
+ torch._C._set_mkldnn_enabled(_enabled)
93
+ if _deterministic is not None:
94
+ torch._C._set_mkldnn_deterministic(_deterministic)
95
+ if _allow_tf32 is not None:
96
+ torch._C._set_onednn_allow_tf32(_allow_tf32)
97
+ if _fp32_precision is not None:
98
+ torch._C._set_fp32_precision_setter("mkldnn", "all", _fp32_precision)
99
+ return orig_flags
100
+
101
+
102
+ @contextmanager
103
+ def flags(enabled=False, deterministic=False, allow_tf32=True, fp32_precision="none"):
104
+ with __allow_nonbracketed_mutation():
105
+ orig_flags = set_flags(enabled, deterministic, allow_tf32, fp32_precision)
106
+ try:
107
+ yield
108
+ finally:
109
+ with __allow_nonbracketed_mutation():
110
+ set_flags(*orig_flags)
111
+
112
+
113
+ class MkldnnModule(PropModule):
114
+ def is_available(self):
115
+ return is_available()
116
+
117
+ enabled = ContextProp(torch._C._get_mkldnn_enabled, torch._C._set_mkldnn_enabled)
118
+ deterministic = ContextProp(
119
+ torch._C._get_mkldnn_deterministic, torch._C._set_mkldnn_deterministic
120
+ )
121
+ allow_tf32 = ContextProp(
122
+ torch._C._get_onednn_allow_tf32, torch._C._set_onednn_allow_tf32
123
+ )
124
+ matmul = _FP32Precision("mkldnn", "matmul")
125
+ conv = _FP32Precision("mkldnn", "conv")
126
+ rnn = _FP32Precision("mkldnn", "rnn")
127
+ fp32_precision = ContextProp(
128
+ _get_fp32_precision_getter("mkldnn", "all"),
129
+ _set_fp32_precision_setter("generic", "all"),
130
+ )
131
+
132
+
133
+ if TYPE_CHECKING:
134
+ enabled: ContextProp
135
+ deterministic: ContextProp
136
+ allow_tf32: ContextProp
137
+ fp32_precision: str
138
+
139
+ sys.modules[__name__] = MkldnnModule(sys.modules[__name__], __name__)
western_ai/usa/pytorch/torch/backends/mps/__init__.py ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from functools import lru_cache as _lru_cache
2
+ from typing import Optional, TYPE_CHECKING
3
+
4
+ import torch
5
+ from torch.library import Library as _Library
6
+
7
+
8
+ __all__ = [
9
+ "get_core_count",
10
+ "get_name",
11
+ "is_built",
12
+ "is_available",
13
+ "is_macos13_or_newer",
14
+ "is_macos_or_newer",
15
+ ]
16
+
17
+
18
+ def is_built() -> bool:
19
+ r"""Return whether PyTorch is built with MPS support.
20
+
21
+ Note that this doesn't necessarily mean MPS is available; just that
22
+ if this PyTorch binary were run a machine with working MPS drivers
23
+ and devices, we would be able to use it.
24
+ """
25
+ return torch._C._has_mps
26
+
27
+
28
+ @_lru_cache
29
+ def is_available() -> bool:
30
+ r"""Return a bool indicating if MPS is currently available."""
31
+ return torch._C._mps_is_available()
32
+
33
+
34
+ @_lru_cache
35
+ def is_macos_or_newer(major: int, minor: int) -> bool:
36
+ r"""Return a bool indicating whether MPS is running on given MacOS or newer."""
37
+ return torch._C._mps_is_on_macos_or_newer(major, minor)
38
+
39
+
40
+ @_lru_cache
41
+ def is_macos13_or_newer(minor: int = 0) -> bool:
42
+ r"""Return a bool indicating whether MPS is running on MacOS 13 or newer."""
43
+ return torch._C._mps_is_on_macos_or_newer(13, minor)
44
+
45
+
46
+ @_lru_cache
47
+ def get_name() -> str:
48
+ r"""Return Metal device name"""
49
+ return torch._C._mps_get_name()
50
+
51
+
52
+ @_lru_cache
53
+ def get_core_count() -> int:
54
+ r"""Return GPU core count.
55
+
56
+ According to the documentation, one core is comprised of 16 Execution Units.
57
+ One execution Unit has 8 ALUs.
58
+ And one ALU can run 24 threads, i.e. one core is capable of executing 3072 threads concurrently.
59
+ """
60
+ return torch._C._mps_get_core_count()
western_ai/usa/pytorch/torch/backends/nnpack/__init__.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ from contextlib import contextmanager
3
+
4
+ import torch
5
+ from torch.backends import __allow_nonbracketed_mutation, ContextProp, PropModule
6
+
7
+
8
+ __all__ = ["is_available", "flags", "set_flags"]
9
+
10
+
11
+ def is_available():
12
+ r"""Return whether PyTorch is built with NNPACK support."""
13
+ return torch._nnpack_available()
14
+
15
+
16
+ def set_flags(_enabled):
17
+ r"""Set if nnpack is enabled globally"""
18
+ orig_flags = (torch._C._get_nnpack_enabled(),)
19
+ torch._C._set_nnpack_enabled(_enabled)
20
+ return orig_flags
21
+
22
+
23
+ @contextmanager
24
+ def flags(enabled=False):
25
+ r"""Context manager for setting if nnpack is enabled globally"""
26
+ with __allow_nonbracketed_mutation():
27
+ orig_flags = set_flags(enabled)
28
+ try:
29
+ yield
30
+ finally:
31
+ with __allow_nonbracketed_mutation():
32
+ set_flags(orig_flags[0])
western_ai/usa/pytorch/torch/backends/openmp/__init__.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import torch
3
+
4
+
5
+ def is_available():
6
+ r"""Return whether PyTorch is built with OpenMP support."""
7
+ return torch._C.has_openmp
western_ai/usa/pytorch/torch/backends/opt_einsum/__init__.py ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import sys
3
+ import warnings
4
+ from contextlib import contextmanager
5
+ from functools import lru_cache as _lru_cache
6
+ from typing import Any
7
+
8
+ from torch.backends import __allow_nonbracketed_mutation, ContextProp, PropModule
9
+
10
+
11
+ try:
12
+ import opt_einsum as _opt_einsum # type: ignore[import]
13
+ except ImportError:
14
+ _opt_einsum = None
15
+
16
+
17
+ @_lru_cache
18
+ def is_available() -> bool:
19
+ r"""Return a bool indicating if opt_einsum is currently available.
20
+
21
+ You must install opt-einsum in order for torch to automatically optimize einsum. To
22
+ make opt-einsum available, you can install it along with torch: ``pip install torch[opt-einsum]``
23
+ or by itself: ``pip install opt-einsum``. If the package is installed, torch will import
24
+ it automatically and use it accordingly. Use this function to check whether opt-einsum
25
+ was installed and properly imported by torch.
26
+ """
27
+ return _opt_einsum is not None
28
+
29
+
30
+ def get_opt_einsum() -> Any:
31
+ r"""Return the opt_einsum package if opt_einsum is currently available, else None."""
32
+ return _opt_einsum
33
+
34
+
35
+ def _set_enabled(_enabled: bool) -> None:
36
+ if not is_available() and _enabled:
37
+ raise ValueError(
38
+ f"opt_einsum is not available, so setting `enabled` to {_enabled} will not reap "
39
+ "the benefits of calculating an optimal path for einsum. torch.einsum will "
40
+ "fall back to contracting from left to right. To enable this optimal path "
41
+ "calculation, please install opt-einsum."
42
+ )
43
+ global enabled
44
+ enabled = _enabled
45
+
46
+
47
+ def _get_enabled() -> bool:
48
+ return enabled
49
+
50
+
51
+ def _set_strategy(_strategy: str) -> None:
52
+ if not is_available():
53
+ raise ValueError(
54
+ f"opt_einsum is not available, so setting `strategy` to {_strategy} will not be meaningful. "
55
+ "torch.einsum will bypass path calculation and simply contract from left to right. "
56
+ "Please install opt_einsum or unset `strategy`."
57
+ )
58
+ if not enabled:
59
+ raise ValueError(
60
+ f"opt_einsum is not enabled, so setting a `strategy` to {_strategy} will not be meaningful. "
61
+ "torch.einsum will bypass path calculation and simply contract from left to right. "
62
+ "Please set `enabled` to `True` as well or unset `strategy`."
63
+ )
64
+ if _strategy not in ["auto", "greedy", "optimal"]:
65
+ raise ValueError(
66
+ f"`strategy` must be one of the following: [auto, greedy, optimal] but is {_strategy}"
67
+ )
68
+ global strategy
69
+ strategy = _strategy
70
+
71
+
72
+ def _get_strategy() -> str:
73
+ # pyrefly: ignore [bad-return]
74
+ return strategy
75
+
76
+
77
+ def set_flags(_enabled=None, _strategy=None):
78
+ orig_flags = (enabled, None if not is_available() else strategy)
79
+ if _enabled is not None:
80
+ _set_enabled(_enabled)
81
+ if _strategy is not None:
82
+ _set_strategy(_strategy)
83
+ return orig_flags
84
+
85
+
86
+ @contextmanager
87
+ def flags(enabled=None, strategy=None):
88
+ with __allow_nonbracketed_mutation():
89
+ orig_flags = set_flags(enabled, strategy)
90
+ try:
91
+ yield
92
+ finally:
93
+ # recover the previous values
94
+ with __allow_nonbracketed_mutation():
95
+ set_flags(*orig_flags)
96
+
97
+
98
+ # The magic here is to allow us to intercept code like this:
99
+ #
100
+ # torch.backends.opt_einsum.enabled = True
101
+
102
+
103
+ class OptEinsumModule(PropModule):
104
+ global enabled
105
+ enabled = ContextProp(_get_enabled, _set_enabled)
106
+ global strategy
107
+ strategy = None
108
+ if is_available():
109
+ strategy = ContextProp(_get_strategy, _set_strategy)
110
+
111
+
112
+ # This is the sys.modules replacement trick, see
113
+ # https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273
114
+ sys.modules[__name__] = OptEinsumModule(sys.modules[__name__], __name__)
115
+
116
+ enabled = bool(is_available())
117
+ strategy = "auto" if is_available() else None
western_ai/usa/pytorch/torch/backends/python_native/__init__.py ADDED
@@ -0,0 +1,387 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ User-facing API for controlling DSL operation overrides.
3
+
4
+ The torch.backends.python_native module provides control over DSL (Domain Specific Language)
5
+ operation overrides defined in torch._native. This allows users to selectively enable or disable
6
+ high-performance implementations from various DSLs like Triton and CuteDSL.
7
+
8
+ The module supports both coarse-grained control (entire DSLs) and fine-grained control
9
+ (individual operations or dispatch keys). All control operations support context managers
10
+ for temporary state changes.
11
+
12
+ Example usage::
13
+
14
+ import torch.backends.python_native as pn
15
+
16
+ # DSL-level control
17
+ pn.triton.enabled = False # Disable all triton ops
18
+ pn.cutedsl.enabled = True # Enable all cutedsl ops
19
+
20
+ # Individual operation control
21
+ pn.disable_operations("scaled_mm") # Disable specific op across all DSLs
22
+ pn.enable_operations("scaled_mm") # Re-enable specific op
23
+
24
+ # Context manager support
25
+ with pn.triton.disabled():
26
+ result = some_computation() # Triton ops disabled here
27
+
28
+ # Query capabilities
29
+ print(pn.available_dsls) # ['triton', 'cutedsl']
30
+ print(pn.get_dsl_operations("triton")) # Operations for triton
31
+ """
32
+
33
+ import functools
34
+ import sys
35
+ import types
36
+ from contextlib import contextmanager
37
+
38
+ from torch.backends import ContextProp, flags_frozen, PropModule
39
+
40
+
41
+ @contextmanager
42
+ def _preserve_filter_state():
43
+ """Context manager to save and restore registry filter state."""
44
+ filter_state = _get_filter_state()
45
+
46
+ # Save original state
47
+ original_state = (
48
+ set(filter_state._dsl_names),
49
+ set(filter_state._op_symbols),
50
+ set(filter_state._dispatch_keys),
51
+ )
52
+
53
+ try:
54
+ yield filter_state
55
+ finally:
56
+ # Restore original state
57
+ filter_state._dsl_names.clear()
58
+ filter_state._op_symbols.clear()
59
+ filter_state._dispatch_keys.clear()
60
+
61
+ filter_state._dsl_names.update(original_state[0])
62
+ filter_state._op_symbols.update(original_state[1])
63
+ filter_state._dispatch_keys.update(original_state[2])
64
+
65
+
66
+ def _get_dsl_registry():
67
+ """Lazy import to avoid circular imports."""
68
+ from torch._native.dsl_registry import dsl_registry
69
+
70
+ return dsl_registry
71
+
72
+
73
+ def _get_registry_functions():
74
+ """Lazy import of registry functions."""
75
+ from torch._native.registry import (
76
+ _filter_state,
77
+ _graphs,
78
+ deregister_op_overrides,
79
+ reenable_op_overrides,
80
+ )
81
+
82
+ return deregister_op_overrides, reenable_op_overrides, _graphs, _filter_state
83
+
84
+
85
+ def _get_filter_state():
86
+ """Direct access to filter state."""
87
+ return _get_registry_functions()[3]
88
+
89
+
90
+ def _get_dsl_module(dsl_name: str):
91
+ """Get the registered DSL module for direct control.
92
+
93
+ Uses the DSL registry to dynamically look up DSL modules instead of
94
+ hard-coding the mapping. This makes the function automatically extensible
95
+ for new DSLs without code changes.
96
+
97
+ Args:
98
+ dsl_name (str): Name of the DSL to retrieve.
99
+
100
+ Returns:
101
+ DSLModuleProtocol: The registered DSL module.
102
+
103
+ Raises:
104
+ ValueError: If the DSL is not registered.
105
+ """
106
+ registry = _get_dsl_registry()
107
+
108
+ # Use the public API to get the DSL module
109
+ dsl_module = registry.get_dsl_module(dsl_name)
110
+ if dsl_module is not None:
111
+ return dsl_module
112
+ else:
113
+ raise ValueError(
114
+ f"Unknown DSL: {dsl_name}. Available DSLs: {registry.list_all_dsls()}"
115
+ )
116
+
117
+
118
+ class DSLController:
119
+ """Controller for a specific DSL."""
120
+
121
+ def __init__(self, dsl_name: str):
122
+ self._dsl_name = dsl_name
123
+
124
+ @property
125
+ def name(self) -> str:
126
+ return self._dsl_name
127
+
128
+ @property
129
+ def available(self) -> bool:
130
+ """Check if DSL runtime is available."""
131
+ registry = _get_dsl_registry()
132
+ return registry.is_dsl_available(self._dsl_name)
133
+
134
+ @property
135
+ def version(self):
136
+ """Get DSL version."""
137
+ registry = _get_dsl_registry()
138
+ return registry.get_dsl_version(self._dsl_name)
139
+
140
+ @property
141
+ def enabled(self) -> bool:
142
+ """Check if DSL is currently enabled."""
143
+ filter_state = _get_filter_state()
144
+ return self._dsl_name not in filter_state._dsl_names
145
+
146
+ @enabled.setter
147
+ def enabled(self, value: bool):
148
+ """Enable or disable the DSL."""
149
+ if flags_frozen():
150
+ raise RuntimeError(
151
+ f"not allowed to set {self._dsl_name} DSL flags "
152
+ "after disable_global_flags; please use flags() context manager instead"
153
+ )
154
+ if value:
155
+ self.enable()
156
+ else:
157
+ self.disable()
158
+
159
+ def disable(self):
160
+ """Disable all operations for this DSL."""
161
+ dsl_module = _get_dsl_module(self._dsl_name)
162
+ dsl_module.deregister_op_overrides()
163
+
164
+ def enable(self):
165
+ """Re-enable all operations for this DSL."""
166
+ reenable_op_overrides = _get_registry_functions()[1]
167
+ reenable_op_overrides(enable_dsl_names=self._dsl_name)
168
+
169
+ @contextmanager
170
+ def disabled(self):
171
+ """Context manager to temporarily disable DSL."""
172
+ original_state = self.enabled
173
+ try:
174
+ self.disable()
175
+ yield
176
+ finally:
177
+ if original_state:
178
+ self.enable()
179
+
180
+ def __repr__(self):
181
+ status = "available" if self.available else "unavailable"
182
+ enabled_status = "enabled" if self.enabled else "disabled"
183
+ return f"DSLController({self._dsl_name}, {status}, {enabled_status})"
184
+
185
+
186
+ class PythonNativeModule(PropModule):
187
+ """Main module for python_native DSL control."""
188
+
189
+ def __init__(self, original_module):
190
+ super().__init__(original_module, original_module.__name__)
191
+
192
+ @property
193
+ def available_dsls(self) -> list[str]:
194
+ """Get list of available DSLs."""
195
+ registry = _get_dsl_registry()
196
+ result = registry.list_available_dsls()
197
+ return list(result) if not isinstance(result, list) else result
198
+
199
+ @property
200
+ def all_dsls(self) -> list[str]:
201
+ """Get list of all registered DSLs."""
202
+ registry = _get_dsl_registry()
203
+ result = registry.list_all_dsls()
204
+ return list(result) if not isinstance(result, list) else result
205
+
206
+ def get_dsl_operations(self, dsl_name: str) -> list[str]:
207
+ """Get list of operations registered by a specific DSL.
208
+
209
+ Args:
210
+ dsl_name (str): Name of the DSL to query (e.g., 'triton', 'cutedsl').
211
+
212
+ Returns:
213
+ list[str]: Sorted list of operation names registered by the DSL.
214
+
215
+ Example::
216
+
217
+ ops = torch.backends.python_native.get_dsl_operations("triton")
218
+ print(ops) # ['triton_to_mxfp8_dim0', ...]
219
+ """
220
+ from torch._native.registry import get_dsl_operations
221
+
222
+ return get_dsl_operations(dsl_name)
223
+
224
+ def disable_operations(self, *op_symbols: str):
225
+ """Disable specific operations across all DSLs.
226
+
227
+ Args:
228
+ *op_symbols (str): Names of operations to disable.
229
+
230
+ Example::
231
+
232
+ # Disable scaled matrix multiply across all DSLs
233
+ torch.backends.python_native.disable_operations("scaled_mm")
234
+
235
+ # Disable multiple operations
236
+ torch.backends.python_native.disable_operations(
237
+ "scaled_mm", "flash_attention"
238
+ )
239
+ """
240
+ deregister_op_overrides = _get_registry_functions()[0]
241
+ deregister_op_overrides(disable_op_symbols=list(op_symbols))
242
+
243
+ def enable_operations(self, *op_symbols: str):
244
+ """Re-enable specific operations across all DSLs.
245
+
246
+ Args:
247
+ *op_symbols (str): Names of operations to re-enable.
248
+
249
+ Example::
250
+
251
+ # Re-enable previously disabled operations
252
+ torch.backends.python_native.enable_operations(
253
+ "scaled_mm", "flash_attention"
254
+ )
255
+ """
256
+ reenable_op_overrides = _get_registry_functions()[1]
257
+ reenable_op_overrides(enable_op_symbols=list(op_symbols))
258
+
259
+ def disable_dispatch_keys(self, *dispatch_keys: str):
260
+ """Disable operations at specific dispatch keys.
261
+
262
+ Args:
263
+ *dispatch_keys (str): Dispatch keys to disable (e.g., 'CUDA', 'CPU').
264
+
265
+ Example::
266
+
267
+ # Disable all native operations on CUDA
268
+ torch.backends.python_native.disable_dispatch_keys("CUDA")
269
+ """
270
+ deregister_op_overrides = _get_registry_functions()[0]
271
+ deregister_op_overrides(disable_dispatch_keys=list(dispatch_keys))
272
+
273
+ def enable_dispatch_keys(self, *dispatch_keys: str):
274
+ """Re-enable operations at specific dispatch keys.
275
+
276
+ Args:
277
+ *dispatch_keys (str): Dispatch keys to re-enable (e.g., 'CUDA', 'CPU').
278
+
279
+ Example::
280
+
281
+ # Re-enable native operations on CUDA
282
+ torch.backends.python_native.enable_dispatch_keys("CUDA")
283
+ """
284
+ reenable_op_overrides = _get_registry_functions()[1]
285
+ reenable_op_overrides(enable_dispatch_keys=list(dispatch_keys))
286
+
287
+ @contextmanager
288
+ def operations_disabled(self, *op_symbols: str):
289
+ """Context manager to temporarily disable operations.
290
+
291
+ Args:
292
+ *op_symbols (str): Names of operations to temporarily disable.
293
+
294
+ Example::
295
+
296
+ with torch.backends.python_native.operations_disabled("scaled_mm"):
297
+ # scaled_mm is disabled across all DSLs
298
+ result = model(input)
299
+ # scaled_mm is automatically re-enabled here
300
+ """
301
+ filter_state = _get_filter_state()
302
+ previously_disabled_ops = {
303
+ op for op in op_symbols if op in filter_state._op_symbols
304
+ }
305
+
306
+ self.disable_operations(*op_symbols)
307
+ try:
308
+ yield
309
+ finally:
310
+ # Only re-enable operations that weren't already disabled
311
+ ops_to_reenable = [
312
+ op for op in op_symbols if op not in previously_disabled_ops
313
+ ]
314
+ if ops_to_reenable:
315
+ self.enable_operations(*ops_to_reenable)
316
+
317
+ @functools.lru_cache(maxsize=16) # noqa: B019
318
+ def _get_dsl_controller(self, name: str) -> "DSLController":
319
+ """Get or create a DSL controller (cached)."""
320
+ return DSLController(name)
321
+
322
+ def _get_registry_functions(self):
323
+ """Expose registry functions for testing."""
324
+ return _get_registry_functions()
325
+
326
+ def is_operation_disabled(self, op_symbol: str) -> bool:
327
+ """Check if an operation is currently disabled."""
328
+ filter_state = _get_filter_state()
329
+ return op_symbol in filter_state._op_symbols
330
+
331
+ def is_dsl_disabled(self, dsl_name: str) -> bool:
332
+ """Check if a DSL is currently disabled."""
333
+ filter_state = _get_filter_state()
334
+ return dsl_name in filter_state._dsl_names
335
+
336
+ def __getattr__(self, name: str):
337
+ """Dynamic attribute access for DSL controllers."""
338
+ # Skip dunder attributes to avoid triggering DSL registry lookups
339
+ # during torch initialization. inspect.getmodule() calls
340
+ # hasattr(module, '__file__') which would otherwise cause a circular
341
+ # import through _get_dsl_registry() while torch is still loading.
342
+ if name.startswith("__") and name.endswith("__"):
343
+ raise AttributeError(f"module '{self.__name__}' has no attribute '{name}'")
344
+
345
+ if name in self.all_dsls:
346
+ return self._get_dsl_controller(name)
347
+
348
+ # Expose private functions for testing
349
+ if name == "_get_dsl_module":
350
+ return _get_dsl_module
351
+ if name == "_get_registry_functions":
352
+ return self._get_registry_functions
353
+ if name == "_get_filter_state":
354
+ return _get_filter_state
355
+
356
+ raise AttributeError(f"module '{self.__name__}' has no attribute '{name}'")
357
+
358
+ def __dir__(self):
359
+ """Return available attributes including DSL names."""
360
+ attrs = set(super().__dir__())
361
+ attrs.update(
362
+ {
363
+ "available_dsls",
364
+ "all_dsls",
365
+ "get_dsl_operations",
366
+ "disable_operations",
367
+ "enable_operations",
368
+ "disable_dispatch_keys",
369
+ "enable_dispatch_keys",
370
+ "operations_disabled",
371
+ "is_operation_disabled",
372
+ "is_dsl_disabled",
373
+ }
374
+ )
375
+
376
+ # Add DSL names
377
+ try:
378
+ attrs.update(self.all_dsls)
379
+ except Exception:
380
+ # If registry not available yet, skip DSL names
381
+ pass
382
+
383
+ return sorted(attrs)
384
+
385
+
386
+ # Replace the current module with our enhanced version
387
+ sys.modules[__name__] = PythonNativeModule(sys.modules[__name__])
western_ai/usa/pytorch/torch/backends/quantized/__init__.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import sys
3
+ import types
4
+
5
+ import torch
6
+
7
+
8
+ # This function should correspond to the enums present in c10/core/QEngine.h
9
+ def _get_qengine_id(qengine: str) -> int:
10
+ if qengine == "none" or qengine == "" or qengine is None:
11
+ ret = 0
12
+ elif qengine == "fbgemm":
13
+ ret = 1
14
+ elif qengine == "qnnpack":
15
+ ret = 2
16
+ elif qengine == "onednn":
17
+ ret = 3
18
+ elif qengine == "x86":
19
+ ret = 4
20
+ else:
21
+ ret = -1
22
+ raise RuntimeError(f"{qengine} is not a valid value for quantized engine")
23
+ return ret
24
+
25
+
26
+ # This function should correspond to the enums present in c10/core/QEngine.h
27
+ def _get_qengine_str(qengine: int) -> str:
28
+ all_engines = {0: "none", 1: "fbgemm", 2: "qnnpack", 3: "onednn", 4: "x86"}
29
+ return all_engines.get(qengine, "*undefined")
30
+
31
+
32
+ class _QEngineProp:
33
+ def __get__(self, obj, objtype) -> str:
34
+ return _get_qengine_str(torch._C._get_qengine())
35
+
36
+ def __set__(self, obj, val: str) -> None:
37
+ torch._C._set_qengine(_get_qengine_id(val))
38
+
39
+
40
+ class _SupportedQEnginesProp:
41
+ def __get__(self, obj, objtype) -> list[str]:
42
+ qengines = torch._C._supported_qengines()
43
+ return [_get_qengine_str(qe) for qe in qengines]
44
+
45
+ def __set__(self, obj, val) -> None:
46
+ raise RuntimeError("Assignment not supported")
47
+
48
+
49
+ class QuantizedEngine(types.ModuleType):
50
+ def __init__(self, m, name):
51
+ super().__init__(name)
52
+ self.m = m
53
+
54
+ def __getattr__(self, attr):
55
+ return self.m.__getattribute__(attr)
56
+
57
+ engine = _QEngineProp()
58
+ supported_engines = _SupportedQEnginesProp()
59
+
60
+
61
+ # This is the sys.modules replacement trick, see
62
+ # https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273
63
+ sys.modules[__name__] = QuantizedEngine(sys.modules[__name__], __name__)
64
+ engine: str
65
+ supported_engines: list[str]
western_ai/usa/pytorch/torch/backends/xeon/__init__.py ADDED
File without changes
western_ai/usa/pytorch/torch/backends/xeon/run_cpu.py ADDED
@@ -0,0 +1,957 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ """
3
+ This is a script for launching PyTorch inference on Intel(R) Xeon(R) Scalable Processors with optimal configurations.
4
+
5
+ Single instance inference, multi-instance inference are enabled.
6
+
7
+ Note: term "instance" here doesn't refer to a cloud instance. This script is executed as a single process. It invokes
8
+ multiple "instances" which are formed from multiple threads for each. "instance" is kind of group of threads in this
9
+ context.
10
+
11
+ Illustrated as below:
12
+
13
+ ::
14
+
15
+ +-----------------------------+----------------------+-------+
16
+ | process | thread | core |
17
+ +=============================+======================+=======+
18
+ | torch.backends.xeon.run_cpu | instance 0: thread 0 | 0 |
19
+ | | thread 1 | 1 |
20
+ | +----------------------+-------+
21
+ | | instance 1: thread 0 | 2 |
22
+ | | thread 1 | 3 |
23
+ | +----------------------+-------+
24
+ | | ... | ... |
25
+ | +----------------------+-------+
26
+ | | instance N: thread 0 | M |
27
+ | | thread 1 | M+1 |
28
+ +-----------------------------+----------------------+-------+
29
+
30
+ To get the peak performance on Intel(R) Xeon(R) Scalable Processors, the script optimizes the configuration of thread and memory
31
+ management. For thread management, the script configures thread affinity and the preload of Intel OMP library.
32
+ For memory management, it configures NUMA binding and preload optimized memory allocation library (e.g. tcmalloc, jemalloc).
33
+
34
+ Environment variables that will be set by this script:
35
+
36
+ +------------------+-------------------------------------------------------------------------------------------------+
37
+ | Environ Variable | Value |
38
+ +==================+=================================================================================================+
39
+ | LD_PRELOAD | Depending on knobs you set, <lib>/libiomp5.so, <lib>/libjemalloc.so, <lib>/libtcmalloc.so might |
40
+ | | be appended to LD_PRELOAD. |
41
+ +------------------+-------------------------------------------------------------------------------------------------+
42
+ | KMP_AFFINITY | If libiomp5.so is preloaded, KMP_AFFINITY could be set to "granularity=fine,compact,1,0". |
43
+ +------------------+-------------------------------------------------------------------------------------------------+
44
+ | KMP_BLOCKTIME | If libiomp5.so is preloaded, KMP_BLOCKTIME is set to "1". |
45
+ +------------------+-------------------------------------------------------------------------------------------------+
46
+ | OMP_NUM_THREADS | value of ncores_per_instance |
47
+ +------------------+-------------------------------------------------------------------------------------------------+
48
+ | MALLOC_CONF | If libjemalloc.so is preloaded, MALLOC_CONF will be set to |
49
+ | | "oversize_threshold:1,background_thread:true,metadata_thp:auto". |
50
+ +------------------+-------------------------------------------------------------------------------------------------+
51
+
52
+ *Note*: This script respects environment variables set preliminarily. I.e. If you set the environment variables
53
+ mentioned above before running the script, the script will not overwrite the values in the script.
54
+
55
+ How to use this module:
56
+ ~~~~~~~~~~~~~~~~~~~~~~~
57
+
58
+ Single instance inference
59
+ -------------------------
60
+
61
+ 1. Run single-instance inference on a single node with all CPU nodes.
62
+
63
+ ::
64
+
65
+ python -m torch.backends.xeon.run_cpu --throughput-mode script.py args
66
+
67
+ 2. Run single-instance inference on a single CPU node.
68
+
69
+ ::
70
+
71
+ python -m torch.backends.xeon.run_cpu --node-id 1 script.py args
72
+
73
+ Multi-instance inference
74
+ ------------------------
75
+
76
+ 1. Multi-instance
77
+ By default this tool runs one process per node. If you want to set the instance numbers and core per instance,
78
+ --ninstances and --ncores-per-instance should be set.
79
+
80
+ ::
81
+
82
+ python -m torch.backends.xeon.run_cpu -- python_script args
83
+
84
+ eg: on an Intel(R) Xeon(R) Scalable Processor with 14 instance, 4 cores per instance
85
+
86
+ ::
87
+
88
+ python -m torch.backends.xeon.run_cpu --ninstances 14 --ncores-per-instance 4 python_script args
89
+
90
+ 2. Run single-instance inference among multiple instances.
91
+ By default, runs all ninstances. If you want to independently run a single instance among ninstances, specify rank.
92
+
93
+ eg: run 0th instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance (i.e., numactl -C 0-27)
94
+
95
+ ::
96
+
97
+ python -m torch.backends.xeon.run_cpu --ninstances 2 --rank 0 python_script args
98
+
99
+ eg: run 1st instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance (i.e., numactl -C 28-55)
100
+
101
+ ::
102
+
103
+ python -m torch.backends.xeon.run_cpu --ninstances 2 --rank 1 python_script args
104
+
105
+ eg: run 0th instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance, 2 cores per instance,
106
+ first four cores (i.e., numactl -C 0-1)
107
+
108
+ ::
109
+
110
+ python -m torch.backends.xeon.run_cpu --core-list "0, 1, 2, 3" --ninstances 2 --ncores-per-instance 2
111
+ --rank 0 python_script args
112
+
113
+ 3. To look up what optional arguments this module offers:
114
+
115
+ ::
116
+
117
+ python -m torch.backends.xeon.run_cpu --help
118
+
119
+ Memory allocator
120
+ ----------------
121
+
122
+ "--enable-tcmalloc" and "--enable-jemalloc" can be used to enable different memory allocator.
123
+
124
+ """
125
+
126
+ import glob
127
+ import logging
128
+ import os
129
+ import platform
130
+ import re
131
+ import subprocess
132
+ import sys
133
+ from argparse import ArgumentParser, RawTextHelpFormatter, REMAINDER
134
+ from os.path import expanduser
135
+
136
+ from torch.distributed.elastic.multiprocessing import (
137
+ DefaultLogsSpecs as _DefaultLogsSpecs,
138
+ start_processes,
139
+ Std,
140
+ )
141
+
142
+
143
+ format_str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
144
+ logging.basicConfig(level=logging.INFO, format=format_str)
145
+ logger = logging.getLogger(__name__)
146
+
147
+
148
+ class _CPUinfo:
149
+ """Get CPU information, such as cores list and NUMA information."""
150
+
151
+ def __init__(self, test_input=""):
152
+ self.cpuinfo = []
153
+ if platform.system() in ["Windows", "Darwin"]:
154
+ raise RuntimeError(f"{platform.system()} is not supported!!!")
155
+ elif platform.system() == "Linux":
156
+ # Sample output of: `lscpu --parse=CPU,Core,Socket,Node`
157
+ #
158
+ # # The following is the parsable format, which can be fed to other
159
+ # # programs. Each different item in every column has a unique ID
160
+ # # starting from zero.
161
+ # # CPU,Core,Socket,Node
162
+ # 0,0,0,0
163
+ # 1,1,0,0
164
+ # ...
165
+ if test_input == "":
166
+ lscpu_cmd = ["lscpu", "--parse=CPU,Core,Socket,Node"]
167
+ lscpu_info = subprocess.check_output(
168
+ lscpu_cmd, universal_newlines=True
169
+ ).split("\n")
170
+ else:
171
+ lscpu_info = test_input.split("\n")
172
+
173
+ # Get information about cpu, core, socket and node
174
+ for line in lscpu_info:
175
+ pattern = r"^([\d]+,[\d]+,[\d]+,[\d]?)"
176
+ regex_out = re.search(pattern, line)
177
+ if regex_out:
178
+ self.cpuinfo.append(regex_out.group(1).strip().split(","))
179
+
180
+ # physical cores := core column in lscpu output
181
+ # logical cores := cPU column in lscpu output
182
+ self.node_nums = int(max(line[3] for line in self.cpuinfo)) + 1
183
+ self.node_physical_cores: list[list[int]] = [] # node_id is index
184
+ self.node_logical_cores: list[list[int]] = [] # node_id is index
185
+ self.physical_core_node_map = {} # physical core to numa node id
186
+ self.logical_core_node_map = {} # logical core to numa node id
187
+
188
+ for node_id in range(self.node_nums):
189
+ cur_node_physical_core = []
190
+ cur_node_logical_core = []
191
+ for cpuinfo in self.cpuinfo:
192
+ nid = cpuinfo[3] if cpuinfo[3] != "" else "0"
193
+ if node_id == int(nid):
194
+ if int(cpuinfo[1]) not in cur_node_physical_core:
195
+ cur_node_physical_core.append(int(cpuinfo[1]))
196
+ self.physical_core_node_map[int(cpuinfo[1])] = int(node_id)
197
+ cur_node_logical_core.append(int(cpuinfo[0]))
198
+ self.logical_core_node_map[int(cpuinfo[0])] = int(node_id)
199
+ self.node_physical_cores.append(cur_node_physical_core)
200
+ self.node_logical_cores.append(cur_node_logical_core)
201
+
202
+ def _physical_core_nums(self):
203
+ return len(self.node_physical_cores) * len(self.node_physical_cores[0])
204
+
205
+ def _logical_core_nums(self):
206
+ return len(self.node_logical_cores) * len(self.node_logical_cores[0])
207
+
208
+ def get_node_physical_cores(self, node_id):
209
+ if node_id < 0 or node_id > self.node_nums - 1:
210
+ raise ValueError(
211
+ f"Invalid node id: {node_id}. Valid node ids: {list(range(len(self.node_physical_cores)))}"
212
+ )
213
+ return self.node_physical_cores[node_id]
214
+
215
+ def get_node_logical_cores(self, node_id):
216
+ if node_id < 0 or node_id > self.node_nums - 1:
217
+ raise ValueError(
218
+ f"Invalid node id: {node_id}. Valid node ids: {list(range(len(self.node_physical_cores)))}"
219
+ )
220
+ return self.node_logical_cores[node_id]
221
+
222
+ def get_all_physical_cores(self):
223
+ all_cores = []
224
+ for cores in self.node_physical_cores:
225
+ all_cores.extend(cores)
226
+ return all_cores
227
+
228
+ def get_all_logical_cores(self):
229
+ all_cores = []
230
+ for cores in self.node_logical_cores:
231
+ all_cores.extend(cores)
232
+ return all_cores
233
+
234
+ def numa_aware_check(self, core_list):
235
+ """
236
+ Check whether all cores in core_list are in the same NUMA node.
237
+
238
+ Cross NUMA will reduce performance.
239
+ We strongly advice to not use cores on different nodes.
240
+ """
241
+ cores_numa_map = self.logical_core_node_map
242
+ numa_ids = []
243
+ for core in core_list:
244
+ numa_id = cores_numa_map[core]
245
+ if numa_id not in numa_ids:
246
+ numa_ids.append(numa_id)
247
+ if len(numa_ids) > 1:
248
+ logger.warning(
249
+ "Numa Aware: cores:%s on different NUMA nodes:%s. To avoid \
250
+ this behavior, please use --ncores-per-instance knob to make sure number of cores is divisible by --ncores-per-\
251
+ instance. Alternatively, please use --skip-cross-node-cores knob.",
252
+ core_list,
253
+ numa_ids,
254
+ )
255
+ if len(numa_ids) == 0:
256
+ raise RuntimeError(
257
+ "invalid number of NUMA nodes; please make sure numa_ids >= 1"
258
+ )
259
+ return numa_ids
260
+
261
+
262
+ class _Launcher:
263
+ r"""Class for launcher."""
264
+
265
+ msg_lib_notfound = (
266
+ f"Unable to find the {{0}} library file lib{{1}}.so in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib \
267
+ or /.local/lib/ or /usr/local/lib/ or /usr/local/lib64/ or /usr/lib or /usr/lib64 or \
268
+ {expanduser('~')}/.local/lib/ so the LD_PRELOAD environment variable will not be set."
269
+ )
270
+
271
+ def __init__(self) -> None:
272
+ self.cpuinfo = _CPUinfo()
273
+
274
+ def add_lib_preload(self, lib_type):
275
+ """Enable TCMalloc/JeMalloc/intel OpenMP."""
276
+ library_paths = []
277
+ if "CONDA_PREFIX" in os.environ:
278
+ library_paths.append(f"{os.environ['CONDA_PREFIX']}/lib")
279
+ if "VIRTUAL_ENV" in os.environ:
280
+ library_paths.append(f"{os.environ['VIRTUAL_ENV']}/lib")
281
+
282
+ library_paths += [
283
+ f"{expanduser('~')}/.local/lib",
284
+ "/usr/local/lib",
285
+ "/usr/local/lib64",
286
+ "/usr/lib",
287
+ "/usr/lib64",
288
+ ]
289
+
290
+ lib_find = False
291
+ lib_set = False
292
+ for item in os.getenv("LD_PRELOAD", "").split(":"):
293
+ if item.endswith(f"lib{lib_type}.so"):
294
+ lib_set = True
295
+ break
296
+ if not lib_set:
297
+ for lib_path in library_paths:
298
+ # pyrefly: ignore [unbound-name]
299
+ library_file = os.path.join(lib_path, f"lib{lib_type}.so")
300
+ matches = glob.glob(library_file)
301
+ if len(matches) > 0:
302
+ # pyrefly: ignore [unbound-name]
303
+ ld_preloads = [f"{matches[0]}", os.getenv("LD_PRELOAD", "")]
304
+ # pyrefly: ignore [unbound-name]
305
+ os.environ["LD_PRELOAD"] = os.pathsep.join(
306
+ # pyrefly: ignore [unbound-name]
307
+ [p.strip(os.pathsep) for p in ld_preloads if p]
308
+ )
309
+ lib_find = True
310
+ break
311
+ return lib_set or lib_find
312
+
313
+ def is_numactl_available(self):
314
+ numactl_available = False
315
+ try:
316
+ cmd = ["numactl", "-C", "0", "-m", "0", "hostname"]
317
+ r = subprocess.run(
318
+ cmd,
319
+ env=os.environ,
320
+ stdout=subprocess.DEVNULL,
321
+ stderr=subprocess.DEVNULL,
322
+ check=False,
323
+ )
324
+ if r.returncode == 0:
325
+ numactl_available = True
326
+ except Exception:
327
+ pass
328
+ return numactl_available
329
+
330
+ def set_memory_allocator(
331
+ self, enable_tcmalloc=True, enable_jemalloc=False, use_default_allocator=False
332
+ ):
333
+ """
334
+ Enable TCMalloc/JeMalloc with LD_PRELOAD and set configuration for JeMalloc.
335
+
336
+ By default, PTMalloc will be used for PyTorch, but TCMalloc and JeMalloc can get better
337
+ memory reuse and reduce page fault to improve performance.
338
+ """
339
+ if enable_tcmalloc and enable_jemalloc:
340
+ raise RuntimeError(
341
+ "Unable to enable TCMalloc and JEMalloc at the same time."
342
+ )
343
+
344
+ if enable_tcmalloc:
345
+ find_tc = self.add_lib_preload(lib_type="tcmalloc")
346
+ if not find_tc:
347
+ msg = f'{self.msg_lib_notfound} you can use "conda install -c conda-forge gperftools" to install {{0}}'
348
+ logger.warning(msg.format("TCmalloc", "tcmalloc"))
349
+ else:
350
+ logger.info("Use TCMalloc memory allocator")
351
+
352
+ elif enable_jemalloc:
353
+ find_je = self.add_lib_preload(lib_type="jemalloc")
354
+ if not find_je:
355
+ msg = f'{self.msg_lib_notfound} you can use "conda install -c conda-forge jemalloc" to install {{0}}'
356
+ logger.warning(msg.format("Jemalloc", "jemalloc"))
357
+ else:
358
+ logger.info("Use JeMalloc memory allocator")
359
+ self.set_env(
360
+ "MALLOC_CONF",
361
+ "oversize_threshold:1,background_thread:true,metadata_thp:auto",
362
+ )
363
+
364
+ elif use_default_allocator:
365
+ pass
366
+
367
+ else:
368
+ find_tc = self.add_lib_preload(lib_type="tcmalloc")
369
+ if find_tc:
370
+ logger.info("Use TCMalloc memory allocator")
371
+ return
372
+ find_je = self.add_lib_preload(lib_type="jemalloc")
373
+ if find_je:
374
+ logger.info("Use JeMalloc memory allocator")
375
+ return
376
+ logger.warning(
377
+ """Neither TCMalloc nor JeMalloc is found in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib
378
+ or /.local/lib/ or /usr/local/lib/ or /usr/local/lib64/ or /usr/lib or /usr/lib64 or
379
+ %s/.local/lib/ so the LD_PRELOAD environment variable will not be set.
380
+ This may drop the performance""",
381
+ expanduser("~"),
382
+ )
383
+
384
+ def log_env_var(self, env_var_name=""):
385
+ if env_var_name in os.environ:
386
+ logger.info("%s=%s", env_var_name, os.environ[env_var_name])
387
+
388
+ def set_env(self, env_name, env_value):
389
+ if not env_value:
390
+ logger.warning("%s is None", env_name)
391
+ if env_name not in os.environ:
392
+ os.environ[env_name] = env_value
393
+ elif os.environ[env_name] != env_value:
394
+ logger.warning(
395
+ "Overriding value with the one set in environment variable: %s. \
396
+ Value applied: %s. Value ignored: %s",
397
+ env_name,
398
+ os.environ[env_name],
399
+ env_value,
400
+ )
401
+ self.log_env_var(env_name)
402
+
403
+ # set_kmp_affinity is used to control whether to set KMP_AFFINITY or not.
404
+ # In scenario that use all cores on all nodes, including logical cores, setting KMP_AFFINITY disables logical cores.
405
+ # In this case, KMP_AFFINITY should not be set.
406
+ def set_multi_thread_and_allocator(
407
+ self,
408
+ ncores_per_instance,
409
+ disable_iomp=False,
410
+ set_kmp_affinity=True,
411
+ enable_tcmalloc=True,
412
+ enable_jemalloc=False,
413
+ use_default_allocator=False,
414
+ ):
415
+ """
416
+ Set multi-thread configuration and enable Intel openMP and TCMalloc/JeMalloc.
417
+
418
+ By default, GNU openMP and PTMalloc are used in PyTorch. but Intel openMP and TCMalloc/JeMalloc are better alternatives
419
+ to get performance benefit.
420
+ """
421
+ self.set_memory_allocator(
422
+ enable_tcmalloc, enable_jemalloc, use_default_allocator
423
+ )
424
+ self.set_env("OMP_NUM_THREADS", str(ncores_per_instance))
425
+ if not disable_iomp:
426
+ find_iomp = self.add_lib_preload(lib_type="iomp5")
427
+ if not find_iomp:
428
+ msg = f'{self.msg_lib_notfound} you can use "conda install mkl" to install {{0}}'
429
+ logger.warning(msg.format("iomp", "iomp5"))
430
+ else:
431
+ logger.info("Using Intel OpenMP")
432
+ if set_kmp_affinity:
433
+ self.set_env("KMP_AFFINITY", "granularity=fine,compact,1,0")
434
+ self.set_env("KMP_BLOCKTIME", "1")
435
+ self.log_env_var("LD_PRELOAD")
436
+
437
+ r"""
438
+ Launcher for single instance and multi-instance
439
+ """
440
+
441
+ def launch(self, args):
442
+ cores = []
443
+ set_kmp_affinity = True
444
+ enable_taskset = False
445
+ if args.core_list: # user specify what cores will be used by params
446
+ cores = [int(x) for x in args.core_list.split(",")]
447
+ if args.ncores_per_instance == -1:
448
+ raise RuntimeError(
449
+ 'please specify the "--ncores-per-instance" if you have pass the --core-list params'
450
+ )
451
+ elif (
452
+ args.ninstances > 1
453
+ and args.ncores_per_instance * args.ninstances < len(cores)
454
+ ):
455
+ logger.warning(
456
+ "only first %s cores will be used, \
457
+ but you specify %s cores in core_list",
458
+ args.ncores_per_instance * args.ninstances,
459
+ len(cores),
460
+ )
461
+ else:
462
+ args.ninstances = len(cores) // args.ncores_per_instance
463
+
464
+ else:
465
+ if args.use_logical_core:
466
+ if args.node_id != -1:
467
+ cores = self.cpuinfo.get_node_logical_cores(args.node_id)
468
+ else:
469
+ cores = self.cpuinfo.get_all_logical_cores()
470
+ # When using all cores on all nodes, including logical cores,
471
+ # setting KMP_AFFINITY disables logical cores. Thus, KMP_AFFINITY should not be set.
472
+ set_kmp_affinity = False
473
+ else:
474
+ if args.node_id != -1:
475
+ cores = self.cpuinfo.get_node_physical_cores(args.node_id)
476
+ else:
477
+ cores = self.cpuinfo.get_all_physical_cores()
478
+ if (
479
+ not args.multi_instance
480
+ and args.ninstances == -1
481
+ and args.ncores_per_instance == -1
482
+ ):
483
+ args.ninstances = 1
484
+ args.ncores_per_instance = len(cores)
485
+ elif (
486
+ args.multi_instance
487
+ and args.ninstances == -1
488
+ and args.ncores_per_instance == -1
489
+ ):
490
+ args.throughput_mode = True
491
+ elif args.ncores_per_instance == -1 and args.ninstances != -1:
492
+ if args.ninstances > len(cores):
493
+ raise RuntimeError(
494
+ f"there are {len(cores)} total cores but you specify {args.ninstances} ninstances; \
495
+ please make sure ninstances <= total_cores)"
496
+ )
497
+ else:
498
+ args.ncores_per_instance = len(cores) // args.ninstances
499
+ elif args.ncores_per_instance != -1 and args.ninstances == -1:
500
+ if not args.skip_cross_node_cores:
501
+ args.ninstances = len(cores) // args.ncores_per_instance
502
+ else:
503
+ ncore_per_node = len(self.cpuinfo.node_physical_cores[0])
504
+ num_leftover_cores = ncore_per_node % args.ncores_per_instance
505
+ if args.ncores_per_instance > ncore_per_node:
506
+ # too many ncores_per_instance to skip cross-node cores
507
+ logger.warning(
508
+ "there are %s core(s) per socket, but you specify %s ncores_per_instance and \
509
+ skip_cross_node_cores. Please make sure --ncores-per-instance < core(s) per \
510
+ socket",
511
+ ncore_per_node,
512
+ args.ncores_per_instance,
513
+ )
514
+ sys.exit(-1)
515
+ elif num_leftover_cores == 0:
516
+ # aren't any cross-node cores
517
+ logger.info(
518
+ "--skip-cross-node-cores is set, but there are no cross-node cores."
519
+ )
520
+ args.ninstances = len(cores) // args.ncores_per_instance
521
+ else:
522
+ # skip cross-node cores
523
+ if args.ninstances != -1:
524
+ logger.warning(
525
+ "--skip-cross-node-cores is exclusive to --ninstances. --ninstances \
526
+ won't take effect even if it is set explicitly."
527
+ )
528
+
529
+ i = 1
530
+ leftover_cores = set()
531
+ while ncore_per_node * i <= len(cores):
532
+ leftover_cores.update(
533
+ cores[
534
+ ncore_per_node * i
535
+ - num_leftover_cores : ncore_per_node * i
536
+ ]
537
+ )
538
+ i += 1
539
+ cores = list(set(cores) - leftover_cores)
540
+ if len(cores) % args.ncores_per_instance != 0:
541
+ raise AssertionError(
542
+ f"Number of cores ({len(cores)}) must be divisible by "
543
+ f"ncores_per_instance ({args.ncores_per_instance})"
544
+ )
545
+ args.ninstances = len(cores) // args.ncores_per_instance
546
+ else:
547
+ if args.ninstances * args.ncores_per_instance > len(cores):
548
+ raise RuntimeError(
549
+ "Please make sure ninstances * ncores_per_instance <= total_cores"
550
+ )
551
+ if args.latency_mode:
552
+ logger.warning(
553
+ "--latency-mode is exclusive to --ninstances, --ncores-per-instance, --node-id and \
554
+ --use-logical-core. They won't take effect even they are set explicitly."
555
+ )
556
+ args.ncores_per_instance = 4
557
+ cores = self.cpuinfo.get_all_physical_cores()
558
+ args.ninstances = len(cores) // args.ncores_per_instance
559
+
560
+ if args.throughput_mode:
561
+ logger.warning(
562
+ "--throughput-mode is exclusive to --ninstances, --ncores-per-instance, --node-id and \
563
+ --use-logical-core. They won't take effect even they are set explicitly."
564
+ )
565
+ args.ninstances = self.cpuinfo.node_nums
566
+ cores = self.cpuinfo.get_all_physical_cores()
567
+ args.ncores_per_instance = len(cores) // args.ninstances
568
+
569
+ if args.ninstances > 1 and args.rank != -1:
570
+ logger.info(
571
+ "assigning %s cores for instance %s",
572
+ args.ncores_per_instance,
573
+ args.rank,
574
+ )
575
+
576
+ if not args.disable_numactl:
577
+ numactl_available = self.is_numactl_available()
578
+ if not numactl_available:
579
+ if not args.disable_taskset:
580
+ logger.warning(
581
+ "Core binding with numactl is not available. Disabling numactl and using taskset instead. \
582
+ This may affect performance in multi-socket system; please use numactl if memory binding is needed."
583
+ )
584
+ args.disable_numactl = True
585
+ enable_taskset = True
586
+ else:
587
+ logger.warning(
588
+ "Core binding with numactl is not available, and --disable_taskset is set. \
589
+ Please unset --disable_taskset to use taskset instead of numactl."
590
+ )
591
+ sys.exit(-1)
592
+
593
+ if not args.disable_taskset:
594
+ enable_taskset = True
595
+
596
+ self.set_multi_thread_and_allocator(
597
+ args.ncores_per_instance,
598
+ args.disable_iomp,
599
+ set_kmp_affinity,
600
+ args.enable_tcmalloc,
601
+ args.enable_jemalloc,
602
+ args.use_default_allocator,
603
+ )
604
+ entrypoint = ""
605
+ launch_args = {}
606
+ launch_envs: dict[int, dict] = {}
607
+ launch_tee = {}
608
+ # check whether is launched from torchrun with --nproc-per-node <num workers>
609
+ local_size = int(os.environ.get("LOCAL_WORLD_SIZE", 1))
610
+ local_rank = int(os.environ.get("LOCAL_RANK", 0))
611
+ for i in range(args.ninstances):
612
+ cmd = []
613
+ cur_process_cores = ""
614
+ if not args.disable_numactl or enable_taskset:
615
+ if not args.disable_numactl:
616
+ cmd = ["numactl"]
617
+ elif enable_taskset:
618
+ cmd = ["taskset"]
619
+ cores = sorted(cores)
620
+ if (
621
+ args.rank == -1
622
+ ): # sequentially assign ncores_per_instance to ninstances
623
+ core_list = cores[
624
+ i * args.ncores_per_instance : (i + 1)
625
+ * args.ncores_per_instance
626
+ ]
627
+ else: # assign ncores_per_instance from rank
628
+ core_list = cores[
629
+ args.rank * args.ncores_per_instance : (args.rank + 1)
630
+ * args.ncores_per_instance
631
+ ]
632
+
633
+ core_ranges: list[dict] = []
634
+ if local_size > 1:
635
+ total_num_cores = len(core_list)
636
+ cores_per_rank = total_num_cores // local_size
637
+ if cores_per_rank < 1:
638
+ raise AssertionError(
639
+ f"At least one core needs to be assigned to each rank, "
640
+ f"got {total_num_cores} cores for {local_size} ranks"
641
+ )
642
+ core_list = core_list[
643
+ cores_per_rank * local_rank : cores_per_rank * (local_rank + 1)
644
+ ]
645
+ for core in core_list:
646
+ if len(core_ranges) == 0:
647
+ range_elem = {"start": core, "end": core}
648
+ core_ranges.append(range_elem)
649
+ else:
650
+ if core - core_ranges[-1]["end"] == 1:
651
+ core_ranges[-1]["end"] = core
652
+ else:
653
+ range_elem = {"start": core, "end": core}
654
+ core_ranges.append(range_elem)
655
+ for r in core_ranges:
656
+ cur_process_cores = f"{cur_process_cores}{r['start']}-{r['end']},"
657
+ cur_process_cores = cur_process_cores[:-1]
658
+ if not args.disable_numactl:
659
+ numa_params = f"-C {cur_process_cores} "
660
+ numa_ids = ",".join(
661
+ [
662
+ str(numa_id)
663
+ for numa_id in self.cpuinfo.numa_aware_check(core_list)
664
+ ]
665
+ )
666
+ numa_params += f"-m {numa_ids}"
667
+ cmd.extend(numa_params.split())
668
+ elif enable_taskset:
669
+ taskset_params = f"-c {cur_process_cores} "
670
+ cmd.extend(taskset_params.split())
671
+ with_python = not args.no_python
672
+ if with_python:
673
+ cmd.append(sys.executable)
674
+ cmd.append("-u")
675
+ if args.module:
676
+ cmd.append("-m")
677
+ cmd.append(args.program)
678
+ cmd.extend(args.program_args)
679
+ cmd_s = " ".join(cmd)
680
+ logger.info(cmd_s)
681
+ if entrypoint == "":
682
+ entrypoint = cmd[0]
683
+ del cmd[0]
684
+ launch_args[i] = tuple(cmd)
685
+ launch_envs[i] = {}
686
+ launch_tee[i] = Std.ALL
687
+
688
+ if args.rank != -1: # launches single instance, rank, only
689
+ break
690
+
691
+ ctx = start_processes(
692
+ name=args.log_file_prefix,
693
+ entrypoint=entrypoint,
694
+ args=launch_args,
695
+ envs=launch_envs,
696
+ logs_specs=_DefaultLogsSpecs(log_dir=args.log_path, tee=launch_tee),
697
+ )
698
+ ctx.wait()
699
+
700
+
701
+ def _add_memory_allocator_params(parser):
702
+ group = parser.add_argument_group("Memory Allocator Parameters")
703
+ # allocator control
704
+ group.add_argument(
705
+ "--enable-tcmalloc",
706
+ "--enable_tcmalloc",
707
+ action="store_true",
708
+ default=False,
709
+ help="Enable tcmalloc allocator",
710
+ )
711
+ group.add_argument(
712
+ "--enable-jemalloc",
713
+ "--enable_jemalloc",
714
+ action="store_true",
715
+ default=False,
716
+ help="Enable jemalloc allocator",
717
+ )
718
+ group.add_argument(
719
+ "--use-default-allocator",
720
+ "--use_default_allocator",
721
+ action="store_true",
722
+ default=False,
723
+ help="Use default memory allocator",
724
+ )
725
+
726
+
727
+ def _add_multi_instance_params(parser):
728
+ group = parser.add_argument_group("Multi-instance Parameters")
729
+ # multi-instance control
730
+ group.add_argument(
731
+ "--ncores-per-instance",
732
+ "--ncores_per_instance",
733
+ metavar="\b",
734
+ default=-1,
735
+ type=int,
736
+ help="Cores per instance",
737
+ )
738
+ group.add_argument(
739
+ "--ninstances",
740
+ metavar="\b",
741
+ default=-1,
742
+ type=int,
743
+ help="For multi-instance, you should give the cores number you used for per instance.",
744
+ )
745
+ group.add_argument(
746
+ "--skip-cross-node-cores",
747
+ "--skip_cross_node_cores",
748
+ action="store_true",
749
+ default=False,
750
+ help="If specified --ncores-per-instance, skips cross-node cores.",
751
+ )
752
+ group.add_argument(
753
+ "--rank",
754
+ metavar="\b",
755
+ default="-1",
756
+ type=int,
757
+ help="Specify instance index to assign ncores_per_instance for rank; \
758
+ otherwise ncores_per_instance will be assigned sequentially to ninstances. Please refer to \
759
+ https://github.com/intel/intel-extension-for-pytorch/blob/master/docs/tutorials/performance_tuning/launch_script.md",
760
+ )
761
+ group.add_argument(
762
+ "--latency-mode",
763
+ "--latency_mode",
764
+ action="store_true",
765
+ default=False,
766
+ help="By default 4 core per instance and use all physical cores",
767
+ )
768
+ group.add_argument(
769
+ "--throughput-mode",
770
+ "--throughput_mode",
771
+ action="store_true",
772
+ default=False,
773
+ help="By default one instance per node and use all physical cores",
774
+ )
775
+ group.add_argument(
776
+ "--node-id",
777
+ "--node_id",
778
+ metavar="\b",
779
+ default=-1,
780
+ type=int,
781
+ help="node id for multi-instance, by default all nodes will be used",
782
+ )
783
+ group.add_argument(
784
+ "--use-logical-core",
785
+ "--use_logical_core",
786
+ action="store_true",
787
+ default=False,
788
+ help="Whether only use physical cores",
789
+ )
790
+ group.add_argument(
791
+ "--disable-numactl",
792
+ "--disable_numactl",
793
+ action="store_true",
794
+ default=False,
795
+ help="Disable numactl",
796
+ )
797
+ group.add_argument(
798
+ "--disable-taskset",
799
+ "--disable_taskset",
800
+ action="store_true",
801
+ default=False,
802
+ help="Disable taskset",
803
+ )
804
+ group.add_argument(
805
+ "--core-list",
806
+ "--core_list",
807
+ metavar="\b",
808
+ default=None,
809
+ type=str,
810
+ help='Specify the core list as "core_id, core_id, ....", otherwise, all the cores will be used.',
811
+ )
812
+ group.add_argument(
813
+ "--log-path",
814
+ "--log_path",
815
+ metavar="\b",
816
+ default="",
817
+ type=str,
818
+ help="The log file directory. Default path is "
819
+ ", which means disable logging to files.",
820
+ )
821
+ group.add_argument(
822
+ "--log-file-prefix",
823
+ "--log_file_prefix",
824
+ metavar="\b",
825
+ default="run",
826
+ type=str,
827
+ help="log file prefix",
828
+ )
829
+
830
+
831
+ def _add_kmp_iomp_params(parser):
832
+ group = parser.add_argument_group("IOMP Parameters")
833
+ group.add_argument(
834
+ "--disable-iomp",
835
+ "--disable_iomp",
836
+ action="store_true",
837
+ default=False,
838
+ help="By default, we use Intel OpenMP and libiomp5.so will be add to LD_PRELOAD",
839
+ )
840
+
841
+
842
+ def create_args(parser=None):
843
+ """
844
+ Parse the command line options.
845
+
846
+ @retval ArgumentParser
847
+ """
848
+ # pyrefly: ignore [missing-attribute]
849
+ parser.add_argument(
850
+ "--multi-instance",
851
+ "--multi_instance",
852
+ action="store_true",
853
+ default=False,
854
+ help="Enable multi-instance, by default one instance per node",
855
+ )
856
+
857
+ # pyrefly: ignore [missing-attribute]
858
+ parser.add_argument(
859
+ "-m",
860
+ "--module",
861
+ default=False,
862
+ action="store_true",
863
+ help="Changes each process to interpret the launch script "
864
+ "as a python module, executing with the same behavior as"
865
+ '"python -m".',
866
+ )
867
+
868
+ # pyrefly: ignore [missing-attribute]
869
+ parser.add_argument(
870
+ "--no-python",
871
+ "--no_python",
872
+ default=False,
873
+ action="store_true",
874
+ help='Do not prepend the --program script with "python" - just exec '
875
+ "it directly. Useful when the script is not a Python script.",
876
+ )
877
+
878
+ _add_memory_allocator_params(parser)
879
+ _add_kmp_iomp_params(parser)
880
+
881
+ _add_multi_instance_params(parser)
882
+ # positional
883
+ # pyrefly: ignore [missing-attribute]
884
+ parser.add_argument(
885
+ "program",
886
+ type=str,
887
+ help="The full path to the program/script to be launched. "
888
+ "followed by all the arguments for the script",
889
+ )
890
+
891
+ # rest from the training program
892
+ # pyrefly: ignore [missing-attribute]
893
+ parser.add_argument("program_args", nargs=REMAINDER)
894
+
895
+
896
+ def main(args):
897
+ env_before = set(os.environ.keys())
898
+ if platform.system() in ["Windows", "Darwin"]:
899
+ raise RuntimeError(f"{platform.system()} is not supported!!!")
900
+
901
+ if args.log_path:
902
+ os.makedirs(args.log_path, exist_ok=True)
903
+ else:
904
+ args.log_path = os.devnull
905
+
906
+ if args.latency_mode and args.throughput_mode:
907
+ raise RuntimeError(
908
+ "Either args.latency_mode or args.throughput_mode should be set"
909
+ )
910
+
911
+ if not args.no_python and not args.program.endswith(".py"):
912
+ raise RuntimeError(
913
+ 'For non Python script, you should use "--no-python" parameter.'
914
+ )
915
+
916
+ # Verify LD_PRELOAD
917
+ if "LD_PRELOAD" in os.environ:
918
+ lst_valid = []
919
+ tmp_ldpreload = os.environ["LD_PRELOAD"]
920
+ for item in tmp_ldpreload.split(":"):
921
+ matches = glob.glob(item)
922
+ if len(matches) > 0:
923
+ lst_valid.append(item)
924
+ else:
925
+ logger.warning("%s doesn't exist. Removing it from LD_PRELOAD.", item)
926
+ if len(lst_valid) > 0:
927
+ os.environ["LD_PRELOAD"] = ":".join(lst_valid)
928
+ else:
929
+ os.environ["LD_PRELOAD"] = ""
930
+
931
+ launcher = _Launcher()
932
+ launcher.launch(args)
933
+ for x in sorted(set(os.environ.keys()) - env_before):
934
+ logger.debug("%s=%s", x, os.environ[x])
935
+
936
+
937
+ if __name__ == "__main__":
938
+ parser = ArgumentParser(
939
+ description="This is a script for launching PyTorch inference on Intel(R) Xeon(R) Scalable "
940
+ "Processors with optimal configurations. Single instance inference, "
941
+ "multi-instance inference are enable. To get the peak performance on Intel(R) "
942
+ "Xeon(R) Scalable Processors, the script optimizes the configuration "
943
+ "of thread and memory management. For thread management, the script configures thread "
944
+ "affinity and the preload of Intel OMP library. For memory management, it configures "
945
+ "NUMA binding and preload optimized memory allocation library (e.g. tcmalloc, jemalloc) "
946
+ "\n################################# Basic usage ############################# \n"
947
+ "\n 1. single instance\n"
948
+ "\n >>> python -m torch.backends.xeon.run_cpu python_script args \n"
949
+ "\n2. multi-instance \n"
950
+ "\n >>> python -m torch.backends.xeon.run_cpu --ninstances xxx "
951
+ "--ncores-per-instance xx python_script args\n"
952
+ "\n############################################################################# \n",
953
+ formatter_class=RawTextHelpFormatter,
954
+ )
955
+ create_args(parser)
956
+ args = parser.parse_args()
957
+ main(args)
western_ai/usa/pytorch/torch/backends/xnnpack/__init__.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import sys
3
+ import types
4
+
5
+ import torch
6
+
7
+
8
+ class _XNNPACKEnabled:
9
+ def __get__(self, obj, objtype):
10
+ return torch._C._is_xnnpack_enabled()
11
+
12
+ def __set__(self, obj, val):
13
+ raise RuntimeError("Assignment not supported")
14
+
15
+
16
+ class XNNPACKEngine(types.ModuleType):
17
+ def __init__(self, m, name):
18
+ super().__init__(name)
19
+ self.m = m
20
+
21
+ def __getattr__(self, attr):
22
+ return self.m.__getattribute__(attr)
23
+
24
+ enabled = _XNNPACKEnabled()
25
+
26
+
27
+ # This is the sys.modules replacement trick, see
28
+ # https://stackoverflow.com/questions/2447353/getattr-on-a-module/7668273#7668273
29
+ sys.modules[__name__] = XNNPACKEngine(sys.modules[__name__], __name__)
western_ai/usa/pytorch/torch/compiler/__init__.py ADDED
@@ -0,0 +1,984 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import contextlib
3
+ import io
4
+ from collections.abc import Callable
5
+ from typing import Any, TYPE_CHECKING, TypeVar
6
+ from typing_extensions import ParamSpec
7
+
8
+ import torch
9
+ from torch._higher_order_ops.invoke_subgraph import NestedCompileRegionOptions
10
+
11
+ from . import config
12
+ from ._cache import CacheInfo
13
+
14
+
15
+ if TYPE_CHECKING:
16
+ from torch._dynamo.eval_frame import StanceStr
17
+
18
+
19
+ __all__ = [
20
+ "compile",
21
+ "config",
22
+ "assume_constant_result",
23
+ "reset",
24
+ "nonstrict_trace",
25
+ "allow_in_graph",
26
+ "substitute_in_graph",
27
+ "list_backends",
28
+ "disable",
29
+ "set_default_backend",
30
+ "get_default_backend",
31
+ "set_stance",
32
+ "set_enable_guard_collectives",
33
+ "cudagraph_mark_step_begin",
34
+ "load_compiled_function",
35
+ "wrap_numpy",
36
+ "is_compiling",
37
+ "is_dynamo_compiling",
38
+ "is_exporting",
39
+ "save_cache_artifacts",
40
+ "load_cache_artifacts",
41
+ "keep_portable_guards_unsafe",
42
+ "skip_guard_on_inbuilt_nn_modules_unsafe",
43
+ "skip_guard_on_all_nn_modules_unsafe",
44
+ "keep_tensor_guards_unsafe",
45
+ "skip_guard_on_globals_unsafe",
46
+ "skip_all_guards_unsafe",
47
+ "nested_compile_region",
48
+ ]
49
+
50
+
51
+ _P = ParamSpec("_P")
52
+ _R = TypeVar("_R")
53
+ FuncType = Callable[..., Any]
54
+ F = TypeVar("F", bound=FuncType)
55
+
56
+
57
+ def compile(*args, **kwargs):
58
+ """
59
+ See :func:`torch.compile` for details on the arguments for this function.
60
+ """
61
+ # pyrefly: ignore [not-iterable]
62
+ return torch.compile(*args, **kwargs)
63
+
64
+
65
+ def reset() -> None:
66
+ """
67
+ Reset the in-process compiler state.
68
+
69
+ This function clears Dynamo's in-memory compilation caches and related
70
+ process-local state used by :func:`torch.compile`. It does not delete
71
+ filesystem caches, such as Inductor's disk cache.
72
+ """
73
+ import torch._dynamo
74
+
75
+ torch._dynamo.reset()
76
+
77
+
78
+ def allow_in_graph(fn):
79
+ """
80
+ Tells the compiler frontend (Dynamo) to skip symbolic introspection of the function
81
+ and instead directly write it to the graph when encountered.
82
+
83
+ If you are using :func:`torch.compile` (with backend="inductor" (the default)), or
84
+ :func:`torch.export.export`, and trying to black-box a Python function throughout
85
+ all tracing, do not use this API.
86
+ Instead, please create a custom operator (see `PyTorch Custom Operators Landing Page
87
+ <https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html>`_)
88
+
89
+ .. warning::
90
+
91
+ If you're a typical torch.compile user (e.g. you're applying torch.compile to
92
+ a model to make it run faster), you probably don't want to use this function.
93
+ :func:`allow_in_graph` is a footgun because it skips the compiler frontend
94
+ (Dynamo) that is responsible for doing safety checks (graph breaks, handling
95
+ closures, etc). Incorrect usage will lead to difficult-to-debug silent
96
+ incorrectness issues.
97
+
98
+ Given a Python function with no allow_in_graph decorator, regular execution
99
+ of torch.compile traces through the function. :func:`allow_in_graph` changes
100
+ it so that the frontend does not trace inside the function, but the compiler
101
+ backend still traces through it. Compare this to custom operators, which
102
+ treats a function as a black box throughout the torch.compile stack. The following
103
+ table compares these mechanisms.
104
+
105
+ +------------------------+-----------------------+--------------------------------+
106
+ | Mechanism | Frontend (Dynamo) | Backend (AOTAutograd+Inductor) |
107
+ +========================+=======================+================================+
108
+ | no decorator | trace inside | trace inside |
109
+ +------------------------+-----------------------+--------------------------------+
110
+ | allow_in_graph | opaque callable | trace inside |
111
+ +------------------------+-----------------------+--------------------------------+
112
+ | custom op | opaque callable | opaque callable |
113
+ +------------------------+-----------------------+--------------------------------+
114
+
115
+ One common use case for :func:`allow_in_graph()` is as an escape hatch for the compiler
116
+ frontend: if you know the function works w.r.t. to the downstream components of the
117
+ compilation stack (AOTAutograd and Inductor) but there is a Dynamo bug that prevents it from
118
+ symbolically introspecting the function properly (or if your code is in C/C++ and
119
+ therefore cannot be introspected with Dynamo), then one can decorate said function
120
+ with :func:`allow_in_graph` to bypass Dynamo.
121
+
122
+ We require that ``fn`` adhere to the following restrictions. Failure to adhere
123
+ results in undefined behavior:
124
+
125
+ - The inputs to ``fn`` must be Proxy-able types in the FX graph. Valid types include:
126
+ Tensor/int/bool/float/None/List[Tensor?]/List[int?]/List[float?]
127
+ Tuple[Tensor?, ...]/Tuple[int?, ...]/Tuple[float?, ...]/torch.dtype/torch.device
128
+ - The outputs to ``fn`` must be Proxy-able types in the FX graph (see previous bullet)
129
+ - all Tensors used inside of ``fn`` must be passed directly as inputs to ``fn``
130
+ (as opposed to being captured variables).
131
+
132
+ See also :func:`nonstrict_trace()`, which has slightly fewer restrictions on the inputs.
133
+
134
+ Args:
135
+ fn: A callable representing the function to be included in the graph.
136
+ If ``fn`` is a list or tuple of callables it recursively applies
137
+ :func:`allow_in_graph()` to each function and returns a new list or
138
+ tuple containing the modified functions.
139
+
140
+ Example::
141
+
142
+ torch.compiler.allow_in_graph(my_custom_function)
143
+
144
+
145
+ @torch.compile(...)
146
+ def fn(x):
147
+ x = torch.add(x, 1)
148
+ x = my_custom_function(x)
149
+ x = torch.add(x, 1)
150
+ return x
151
+
152
+
153
+ fn(...)
154
+
155
+ Will capture a single graph containing ``my_custom_function()``.
156
+
157
+ """
158
+ import torch._dynamo
159
+
160
+ return torch._dynamo.allow_in_graph(fn)
161
+
162
+
163
+ def nonstrict_trace(traceable_fn: Callable[_P, _R]) -> Callable[_P, _R]:
164
+ """
165
+ Decorator to mark a function as nonstrict-traceable for dynamo.
166
+
167
+ A nonstrict-traced function appears as an opaque call in the dynamo graph.
168
+ Dynamo does not trace into the function body (hence the "nonstrict"), but
169
+ aot_autograd will trace into it.
170
+
171
+ This is similar to ``allow_in_graph`` but with enhanced support for:
172
+ - User-defined classes as inputs (must be registered with pytree)
173
+ - ``nn.Module`` as input arguments (parameters and buffers are tracked for autograd)
174
+ - Global/captured tensors treated as constants (assumed not updated during execution)
175
+
176
+ Note:
177
+ - With ``backend="eager"``, the original Python function runs directly.
178
+ With ``backend="aot_eager"``, the graph traced by aot_autograd runs.
179
+ With ``backend="inductor"``, the traced graph is compiled with inductor.
180
+
181
+ - Training is supported: you can call ``.backward()`` on outputs and gradients
182
+ will flow through the nonstrict-traced function.
183
+
184
+ Dangerous patterns (may cause silent incorrectness):
185
+ - Side effects between nonstric_trace'd fn and compiled region: The function should
186
+ not depend on variables mutated by other code inside the compiled function, and code
187
+ after the call should not depend on mutations made by it.
188
+
189
+ - Implicit inputs (closures/globals): Tensors captured from enclosing scopes
190
+ are treated as constants. Gradients will NOT flow back to them. Pass tensors
191
+ as explicit arguments if gradients are needed.
192
+
193
+ Restrictions:
194
+ - Both inputs and outputs must use pytree-compatible types. User-defined classes
195
+ must be registered via :func:`torch.utils._pytree.register_pytree_node`,
196
+ :func:`torch.utils._pytree.register_dataclass`, or
197
+ :func:`torch.utils._pytree.register_constant`. Tensors, Python primitives (int, float, bool, str),
198
+ symbolic types (SymInt, SymFloat, SymBool), and built-in containers (list,
199
+ tuple, dict) are already handled by default.
200
+ - Primitive values and container structure are specialized per call site:
201
+ each call site expects the same primitives and structure on every execution.
202
+
203
+ Example::
204
+
205
+ >>> import torch
206
+ >>> @torch.compiler.nonstrict_trace
207
+ ... def traced_forward(model, x):
208
+ ... # It's OK to have dynamo graph break within nonstrict_trace region
209
+ ... torch._dynamo.graph_break()
210
+ ... return model(x) + x
211
+ ...
212
+ >>> class MyModule(torch.nn.Module):
213
+ ... def __init__(self):
214
+ ... super().__init__()
215
+ ... self.inner = torch.nn.Linear(10, 10)
216
+ ...
217
+ ... def forward(self, x):
218
+ ... return traced_forward(self.inner, x)
219
+ ...
220
+ >>> # Compile and run
221
+ >>> model = MyModule()
222
+ >>> opt_model = torch.compile(model, backend="aot_eager", fullgraph=True)
223
+ >>> out = opt_model(torch.randn(10, 10))
224
+ >>> out.sum().backward() # Gradients flow through traced_forward
225
+
226
+ """
227
+ import torch._dynamo
228
+
229
+ return torch._dynamo.nonstrict_trace(traceable_fn)
230
+
231
+
232
+ def substitute_in_graph(
233
+ original_fn: Callable[_P, _R],
234
+ *,
235
+ can_constant_fold_through: bool = False,
236
+ skip_signature_check: bool = False,
237
+ ) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]:
238
+ """
239
+ Register a polyfill handler for a function, usually a C function from the C extension, to be
240
+ used in place of the original function when inlining the original function in the graph.
241
+
242
+ .. note::
243
+
244
+ The polyfill handler is only used when inlining the original function. It is not used when
245
+ the original function is called directly. In the eager mode, the decorated function calls
246
+ the performant C function rather than the polyfill handler.
247
+
248
+ The polyfill handler is a function that will be called in place of the original function when
249
+ inlining the original function. The polyfill handler should have the same signature and the same
250
+ behavior as the original function.
251
+
252
+ Args:
253
+ original_fn (callable): The original function, usually a C function, to register a polyfill
254
+ handler for.
255
+ can_constant_fold_through (bool, optional): Whether the polyfill handler can be constant
256
+ folded through. That is, if the polyfill handler is a pure function and its arguments
257
+ are constant, the result of the polyfill handler can be constant folded during the
258
+ compilation. Defaults to ``False``.
259
+ skip_signature_check (bool, optional): Whether to skip the signature check between the
260
+ original function and the polyfill handler. Defaults to ``False``.
261
+
262
+ Returns:
263
+ A decorator that registers the polyfill handler for the original function.
264
+
265
+ Example::
266
+
267
+ >>> import operator
268
+ >>> operator.indexOf([1, 2, 3, 4, 5], 3)
269
+ 2
270
+ >>> torch.compile(operator.indexOf, fullgraph=True)([1, 2, 3, 4, 5], 3)
271
+ ... # xdoctest: +SKIP("Long tracebacks")
272
+ Traceback (most recent call last):
273
+ ...
274
+ torch._dynamo.exc.Unsupported: ...
275
+
276
+ >>> @torch.compiler.substitute_in_graph(operator.indexOf)
277
+ ... def indexOf(a, b, /):
278
+ ... for i, item in enumerate(a):
279
+ ... if item is b or item == b:
280
+ ... return i
281
+ ... raise ValueError("sequence.index(x): x not in sequence")
282
+ >>>
283
+ >>> torch.compile(operator.indexOf, fullgraph=True)([1, 2, 3, 4, 5], 3)
284
+ 2
285
+ """
286
+ import torch._dynamo
287
+
288
+ return torch._dynamo.substitute_in_graph(
289
+ original_fn,
290
+ can_constant_fold_through=can_constant_fold_through,
291
+ skip_signature_check=skip_signature_check,
292
+ )
293
+
294
+
295
+ def list_backends(exclude_tags=("debug", "experimental")) -> list[str]:
296
+ """
297
+ Return valid strings that can be passed to `torch.compile(..., backend="name")`.
298
+
299
+ Args:
300
+ exclude_tags(optional): A tuple of strings representing tags to exclude.
301
+ """
302
+ import torch._dynamo
303
+
304
+ return torch._dynamo.list_backends(exclude_tags)
305
+
306
+
307
+ def assume_constant_result(fn):
308
+ """
309
+ This function is used to mark a function `fn` as having a constant result.
310
+ This allows the compiler to optimize away your function.
311
+ Returns The same function `fn`
312
+
313
+ Args:
314
+ fn: The function to be marked as having a constant result.
315
+
316
+ .. warning::
317
+ `assume_constant_result` can, if invalid, cause safety and soundness issues, :func:`torch.compile`
318
+ will not attempt to validate whether the constant assumption is true or not
319
+
320
+ """
321
+ import torch._dynamo
322
+
323
+ return torch._dynamo.assume_constant_result(fn)
324
+
325
+
326
+ def disable(fn=None, recursive=True, *, reason=None):
327
+ """
328
+ This function provides a decorator to disable compilation on a function.
329
+ It also provides the option of recursively disabling called functions.
330
+
331
+ Args:
332
+ fn (optional): The function to disable
333
+ recursive (optional): A boolean value indicating whether the disabling should be recursive.
334
+ reason (optional): A string value indicating the reason for disabling the function.
335
+ """
336
+ import torch._dynamo
337
+
338
+ return torch._dynamo.disable(fn, recursive, reason=reason)
339
+
340
+
341
+ def set_default_backend(backend: str | Callable[..., Any] | None) -> None:
342
+ """Set the default backend for ``torch.compile`` when no ``backend`` argument is specified.
343
+
344
+ Passing ``None`` resets the default back to ``"inductor"``.
345
+
346
+ Args:
347
+ backend: A backend name (string), a callable backend, or ``None``.
348
+
349
+ Example::
350
+
351
+ >>> torch.compiler.set_default_backend("eager")
352
+ >>> torch.compiler.get_default_backend()
353
+ 'eager'
354
+ >>> torch.compiler.set_default_backend(None) # reset
355
+ >>> torch.compiler.get_default_backend()
356
+ 'inductor'
357
+ """
358
+ from torch._dynamo.backends.registry import set_default_backend
359
+
360
+ set_default_backend(backend)
361
+
362
+
363
+ def get_default_backend() -> str | Callable[..., Any]:
364
+ """Return the current default backend for ``torch.compile``.
365
+
366
+ Returns:
367
+ The current default backend (string or callable). Initially ``"inductor"``.
368
+ """
369
+ from torch._dynamo.backends.registry import get_default_backend
370
+
371
+ return get_default_backend()
372
+
373
+
374
+ def set_stance(
375
+ stance: "StanceStr" = "default",
376
+ *,
377
+ skip_guard_eval_unsafe: bool = False,
378
+ force_backend: str | Callable[..., Any] | None = None,
379
+ ):
380
+ """
381
+ Set the current stance of the compiler.
382
+ Can be used as a function, context manager, or decorator.
383
+ Do not use this function inside a `torch.compile` region - an error will be raised otherwise.
384
+
385
+ .. code-block:: python
386
+
387
+ @torch.compile
388
+ def foo(x): ...
389
+
390
+
391
+ @torch.compiler.set_stance("force_eager")
392
+ def bar():
393
+ # will not be compiled
394
+ foo(...)
395
+
396
+
397
+ bar()
398
+
399
+ with torch.compiler.set_stance("force_eager"):
400
+ # will also not be compiled
401
+ foo(...)
402
+
403
+ torch.compiler.set_stance("force_eager")
404
+ # will also not be compiled
405
+ foo(...)
406
+ torch.compiler.set_stance("default")
407
+
408
+ # will be compiled
409
+ foo(...)
410
+
411
+ Args:
412
+ stance: The stance to set the compiler to. Valid values are:
413
+
414
+ - "default": The default stance, used for normal compilation.
415
+ - "force_eager": Ignore all `torch.compile` directives.
416
+ - "eager_on_recompile": Run code eagerly when a recompile is necessary.
417
+ If there is cached compiled code valid for the input, it will still be used.
418
+ - "fail_on_recompile": Raise an error when recompiling a function.
419
+ - "eager_then_compile": Run the first invocation in eager mode, then compile on
420
+ subsequent calls. This is beneficial for dynamic shapes as it allows inferring
421
+ dynamism from the first two invocations instead of wasting a static compile on
422
+ the first invocation.
423
+ - "aot_eager_then_compile": Run the first invocation with AOT eager to get memory
424
+ benefits from activation checkpointing, then compile on subsequent calls. Like
425
+ eager_then_compile, this improves handling of dynamic shapes by avoiding an
426
+ initial static compile.
427
+
428
+
429
+ skip_guard_eval_unsafe: A flag to run only differentiating guards.
430
+ CAUTION - This flag is unsafe and should only be used if your setup
431
+ meets the following conditions.
432
+
433
+ torch.compile uses a guard system to support recompilations and
434
+ choose which compiled artifact to run at runtime. These guards,
435
+ though efficient, add some overhead, which may impact performance in
436
+ scenarios where you need to optimize for minimal guard processing
437
+ time. This API enables you to disable guard evaluation, assuming
438
+ that you have warmed up the compiled model with a sufficient variety
439
+ of inputs. This assumption means that, after the warmup phase, no
440
+ further recompilations will be necessary. If this assumption fails,
441
+ there is a risk of silently producing incorrect results (hence the
442
+ term "unsafe" in the API name).
443
+
444
+ force_backend: If `stance` is "default", this argument can be used to force `torch.compile`
445
+ to use a specific backend. Otherwise, an error is raised.
446
+ """
447
+ import torch._dynamo
448
+
449
+ return torch._dynamo.set_stance(
450
+ stance,
451
+ skip_guard_eval_unsafe=skip_guard_eval_unsafe,
452
+ force_backend=force_backend,
453
+ )
454
+
455
+
456
+ # forbid in graph
457
+ set_stance._dynamo_forbidden = True # type: ignore[attr-defined]
458
+
459
+
460
+ def set_enable_guard_collectives(enabled: bool):
461
+ """
462
+ Enables use of collectives *during* guard evaluation to synchronize behavior
463
+ across ranks. This is expensive: we have to issue a collective every time
464
+ we enter a compiled code region, even if no rank actually would need to
465
+ compile. This can help prevent NCCL hangs by ensuring that we never have a
466
+ situation where one rank starts recompiling while other ranks don't compile;
467
+ it is especially useful in conjunction with enable_compiler_collectives
468
+ where such a situation would immediately cause a hang (as it is necessary
469
+ for all ranks to compile at the same time to run compiler collectives). Like
470
+ compiler collectives, you can only run this on SPMD programs; you will hang
471
+ otherwise. Note that a guard collective is only issued if there is any
472
+ compiled code to guard on; if this is the first time we encounter a frame or
473
+ the frame is skipped, we don't issue collectives.
474
+
475
+ Returns the previous setting of enabled.
476
+ """
477
+ from torch._C._dynamo.eval_frame import set_guard_complete_hook
478
+ from torch._dynamo.eval_frame import guard_collectives_hook
479
+
480
+ if enabled:
481
+ return set_guard_complete_hook(guard_collectives_hook) is not None # type: ignore[arg-type]
482
+ else:
483
+ return set_guard_complete_hook(None) is not None
484
+
485
+
486
+ set_enable_guard_collectives._dynamo_forbidden = True # type: ignore[attr-defined]
487
+
488
+
489
+ def cudagraph_mark_step_begin():
490
+ """
491
+ Indicates that a new iteration of inference or training is about to begin.
492
+
493
+ CUDA Graphs will free tensors of a prior iteration. A new iteration is started on each invocation of
494
+ torch.compile, so long as there is not a pending backward that has not been called.
495
+
496
+ If that heuristic is wrong, such as in the following example, manually mark it with this api.
497
+
498
+ .. code-block:: python
499
+
500
+ @torch.compile(mode="reduce-overhead")
501
+ def rand_foo():
502
+ return torch.rand([4], device="cuda")
503
+
504
+
505
+ for _ in range(5):
506
+ torch.compiler.cudagraph_mark_step_begin()
507
+ rand_foo() + rand_foo()
508
+
509
+ For more details, see `torch.compiler_cudagraph_trees <https://docs.pytorch.org/docs/main/user_guide/torch_compiler/torch.compiler_cudagraph_trees.html>`__ # noqa: B950
510
+ """
511
+ from torch._inductor import cudagraph_trees
512
+
513
+ cudagraph_trees.mark_step_begin()
514
+
515
+
516
+ def wrap_numpy(fn):
517
+ r"""Decorator that turns a function from ``np.ndarray``\ s to ``np.ndarray``\ s into a function
518
+ from ``torch.Tensor``\ s to ``torch.Tensor``\ s.
519
+
520
+ It is designed to be used with :func:`torch.compile` with ``fullgraph=True``. It allows you to
521
+ compile a NumPy function as if it were a PyTorch function. This allows you to run NumPy code
522
+ on CUDA or compute its gradients.
523
+
524
+ .. note::
525
+
526
+ This decorator does not work without :func:`torch.compile`.
527
+
528
+ Example::
529
+
530
+ >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA)
531
+ >>> # Compile a NumPy function as a Tensor -> Tensor function
532
+ >>> @torch.compile(fullgraph=True)
533
+ >>> @torch.compiler.wrap_numpy
534
+ >>> def fn(a: np.ndarray):
535
+ >>> return np.sum(a * a)
536
+ >>> # Execute the NumPy function using Tensors on CUDA and compute the gradients
537
+ >>> x = torch.arange(6, dtype=torch.float32, device="cuda", requires_grad=True)
538
+ >>> out = fn(x)
539
+ >>> out.backward()
540
+ >>> print(x.grad)
541
+ tensor([ 0., 2., 4., 6., 8., 10.], device='cuda:0')
542
+ """
543
+ from torch._dynamo.external_utils import wrap_numpy as wrap
544
+
545
+ return wrap(fn)
546
+
547
+
548
+ _is_compiling_flag: bool = False
549
+ _is_exporting_flag: bool = False
550
+ _is_non_strict_tracing_flag: bool = False
551
+
552
+
553
+ def is_compiling() -> bool:
554
+ """
555
+ Indicates whether a graph is executed/traced as part of torch.compile() or torch.export().
556
+
557
+ Note that there are 2 other related flags that should deprecated eventually:
558
+ * torch._dynamo.external_utils.is_compiling()
559
+ * torch._utils.is_compiling()
560
+
561
+ Example::
562
+
563
+ >>> def forward(self, x):
564
+ >>> if not torch.compiler.is_compiling():
565
+ >>> pass # ...logic that is not needed in a compiled/traced graph...
566
+ >>>
567
+ >>> # ...rest of the function...
568
+ """
569
+ if torch.jit.is_scripting():
570
+ return False
571
+ else:
572
+ return _is_compiling_flag
573
+
574
+
575
+ def _is_non_strict_tracing() -> bool:
576
+ """
577
+ Indicates whether we are inside a non-strict make_fx-based tracing session.
578
+ """
579
+ return _is_non_strict_tracing_flag
580
+
581
+
582
+ @contextlib.contextmanager
583
+ def _non_strict_tracing_context():
584
+ """Context manager that sets the non-strict tracing flag."""
585
+ global _is_non_strict_tracing_flag
586
+ old = _is_non_strict_tracing_flag
587
+ try:
588
+ _is_non_strict_tracing_flag = True
589
+ yield
590
+ finally:
591
+ _is_non_strict_tracing_flag = old
592
+
593
+
594
+ @contextlib.contextmanager
595
+ def _compile_session_context():
596
+ """Context manager that sets _is_compiling_flag for the duration of a
597
+ torch.compile session.
598
+ """
599
+ global _is_compiling_flag
600
+ old = _is_compiling_flag
601
+ try:
602
+ _is_compiling_flag = True
603
+ yield
604
+ finally:
605
+ _is_compiling_flag = old
606
+
607
+
608
+ @contextlib.contextmanager
609
+ def _patch_autograd_grad():
610
+ """Patch autograd.grad for non-strict make_fx tracing.
611
+
612
+ This patch installs autograd hooks so traced backward nodes preserve
613
+ stack trace, seq_nr, and autograd_backward metadata before delegating to
614
+ the real torch.autograd.grad.
615
+ """
616
+ import functools
617
+
618
+ import torch.autograd
619
+ from torch._dynamo.utils import warn_once
620
+ from torch._functorch._aot_autograd.logging_utils import (
621
+ setup_stacktrace_preservation_hooks_from_tensors,
622
+ )
623
+
624
+ warn_once(
625
+ "torch.compiler._patch_autograd_grad() is deprecated; "
626
+ "use torch.compiler._patch_engine_backward() instead."
627
+ )
628
+ # TODO: Remove this helper once Titan no longer depends on it.
629
+
630
+ _orig_grad = torch.autograd.grad
631
+
632
+ @functools.wraps(_orig_grad)
633
+ def _patched_grad(outputs, inputs, *args, **kwargs):
634
+ if not _is_non_strict_tracing():
635
+ raise AssertionError(
636
+ "_patch_autograd_grad() must be used under "
637
+ "_non_strict_tracing_context()"
638
+ )
639
+
640
+ setup_stacktrace_preservation_hooks_from_tensors(outputs)
641
+ return _orig_grad(outputs, inputs, *args, **kwargs)
642
+
643
+ torch.autograd.grad = _patched_grad
644
+ try:
645
+ yield
646
+ finally:
647
+ torch.autograd.grad = _orig_grad
648
+
649
+
650
+ @contextlib.contextmanager
651
+ def _patch_engine_backward():
652
+ """Patch _engine_run_backward for non-strict make_fx tracing.
653
+
654
+ This patch installs autograd hooks so traced backward nodes preserve
655
+ stack trace, seq_nr, and autograd_backward metadata before delegating to
656
+ the real autograd engine entrypoint used by backward().
657
+ """
658
+ import functools
659
+
660
+ import torch.autograd
661
+ import torch.autograd.graph
662
+ from torch._functorch._aot_autograd.logging_utils import (
663
+ setup_stacktrace_preservation_hooks_from_tensors,
664
+ )
665
+
666
+ _orig_engine_run_backward = torch.autograd.graph._engine_run_backward
667
+
668
+ @functools.wraps(_orig_engine_run_backward)
669
+ def _patched_engine_backward(outputs, *args, **kwargs):
670
+ if not _is_non_strict_tracing():
671
+ raise AssertionError(
672
+ "_patch_engine_backward() must be used under "
673
+ "_non_strict_tracing_context()"
674
+ )
675
+
676
+ setup_stacktrace_preservation_hooks_from_tensors(outputs)
677
+ return _orig_engine_run_backward(outputs, *args, **kwargs)
678
+
679
+ torch.autograd.graph._engine_run_backward = _patched_engine_backward
680
+ torch.autograd._engine_run_backward = _patched_engine_backward
681
+ try:
682
+ yield
683
+ finally:
684
+ torch.autograd.graph._engine_run_backward = _orig_engine_run_backward
685
+ torch.autograd._engine_run_backward = _orig_engine_run_backward
686
+
687
+
688
+ def is_dynamo_compiling() -> bool:
689
+ """
690
+ Indicates whether a graph is traced via TorchDynamo.
691
+
692
+ It's stricter than is_compiling() flag, as it would only be set to True when
693
+ TorchDynamo is used.
694
+
695
+ Example::
696
+
697
+ >>> def forward(self, x):
698
+ >>> if not torch.compiler.is_dynamo_compiling():
699
+ >>> pass # ...logic that is not needed in a TorchDynamo-traced graph...
700
+ >>>
701
+ >>> # ...rest of the function...
702
+ """
703
+ return False
704
+
705
+
706
+ def is_exporting() -> bool:
707
+ """
708
+ Indicates whether we're under exporting.
709
+
710
+ It's stricter than is_compiling() flag, as it would only be set to True when
711
+ torch.export is used.
712
+
713
+ Example::
714
+
715
+ >>> def forward(self, x):
716
+ >>> if not torch.compiler.is_exporting():
717
+ >>> pass # ...logic that is not needed in export...
718
+ >>>
719
+ >>> # ...rest of the function...
720
+ """
721
+ return _is_exporting_flag
722
+
723
+
724
+ def save_cache_artifacts() -> tuple[bytes, CacheInfo] | None:
725
+ """
726
+ Serializes all the cache artifacts that were created during the compilation
727
+
728
+ Example:
729
+
730
+ - Execute torch.compile
731
+ - Call torch.compiler.save_cache_artifacts()
732
+ """
733
+ from ._cache import CacheArtifactManager
734
+
735
+ if torch._dynamo.config.caching_precompile:
736
+ from torch._dynamo.precompile_context import PrecompileContext
737
+
738
+ PrecompileContext.save_to_dynamo_cache()
739
+
740
+ return CacheArtifactManager.serialize()
741
+
742
+
743
+ def load_cache_artifacts(serialized_artifacts: bytes) -> CacheInfo | None:
744
+ """
745
+ Hot loads cache artifacts that were previously serialized via
746
+ save_cache_artifacts
747
+
748
+ Example:
749
+
750
+ # From a previous invocation
751
+ artifacts = torch.compiler.save_cache_artifacts()
752
+
753
+ torch.compiler.load_cache_artifacts(artifacts[0])
754
+ """
755
+ from ._cache import CacheArtifactManager, CacheInfo
756
+
757
+ artifacts = CacheArtifactManager.deserialize(serialized_artifacts)
758
+ if artifacts is not None:
759
+ return CacheArtifactManager.populate_caches(artifacts)
760
+ return None
761
+
762
+
763
+ def keep_portable_guards_unsafe(guard_entries):
764
+ """
765
+ A common function to only keep guards that can be used in both Python and non-Python environments.
766
+ This includes:
767
+ - Tensor metadata and dynamic shape information.
768
+ - Global contexts state (e.g. autocast, no_grad, etc.)
769
+
770
+ This is unsafe to use by default.
771
+ To use this API, use guard_filter_fn argument while calling torch.compile
772
+
773
+ >> opt_mod = torch.compile(
774
+ >> mod,
775
+ >> options={"guard_filter_fn": torch.compiler.keep_global_context_and_tensor_guards_unsafe},
776
+ >> )
777
+ """
778
+ return [
779
+ (
780
+ g.guard_type in ("GLOBAL_STATE", "SHAPE_ENV")
781
+ or (g.guard_type == "TENSOR_MATCH" and not g.is_global)
782
+ )
783
+ for g in guard_entries
784
+ ]
785
+
786
+
787
+ def skip_guard_on_inbuilt_nn_modules_unsafe(guard_entries):
788
+ """
789
+ A common function to skip guards on the inbuilt nn modules like
790
+ torch.nn.Linear. This is unsafe to use by default. But for majority of
791
+ torch.compile users, the model code does not modify the inbuilt nn module
792
+ attributes. They can benefit from reduction in guard latency overhead using
793
+ this API.
794
+
795
+ To use this API, use guard_filter_fn argument while calling torch.compile
796
+
797
+ >> opt_mod = torch.compile(
798
+ >> mod,
799
+ >> options={"guard_filter_fn": torch.compiler.skip_guard_on_all_nn_modules_unsafe},
800
+ >> )
801
+ """
802
+ return [
803
+ not entry.orig_guard.source.is_unspecialized_builtin_nn_module()
804
+ for entry in guard_entries
805
+ ]
806
+
807
+
808
+ def skip_guard_on_all_nn_modules_unsafe(guard_entries):
809
+ """
810
+ A common function to skip guards on all nn modules, both user defined as
811
+ well inbuilt nn modules (like torch.nn.Linear). This is unsafe to use by
812
+ default. But for majority of torch.compile users, the model code does not
813
+ modify the nn module attributes. They can benefit from reduction in guard
814
+ latency overhead using this API.
815
+
816
+ To use this API, use guard_filter_fn argument while calling torch.compile
817
+
818
+ >> opt_mod = torch.compile(
819
+ >> mod,
820
+ >> options={"guard_filter_fn": torch.compiler.skip_guard_on_all_nn_modules_unsafe},
821
+ >> )
822
+ """
823
+
824
+ return [
825
+ not entry.orig_guard.source.is_unspecialized_nn_module()
826
+ for entry in guard_entries
827
+ ]
828
+
829
+
830
+ def keep_tensor_guards_unsafe(guard_entries, keep_parameters=False):
831
+ """
832
+ A common function to keep tensor guards on all tensors. This is unsafe to
833
+ use by default. But if you don't expect any changes in the model code, you
834
+ can just keep the tensor guards.
835
+
836
+
837
+ >> opt_mod = torch.compile(
838
+ >> mod,
839
+ >> options={"guard_filter_fn": torch.compiler.keep_tensor_guards},
840
+ >> )
841
+ """
842
+
843
+ keep_flags = []
844
+ for entry in guard_entries:
845
+ if entry.guard_type == "TENSOR_MATCH":
846
+ if not isinstance(entry.value, torch.nn.Parameter):
847
+ keep_flags.append(True)
848
+ elif keep_parameters:
849
+ keep_flags.append(True)
850
+ else:
851
+ keep_flags.append(False)
852
+ else:
853
+ keep_flags.append(False)
854
+ return keep_flags
855
+
856
+
857
+ def skip_guard_on_globals_unsafe(guard_entries):
858
+ """
859
+ A common function to skip guards on all globals. This is unsafe to use by
860
+ default. But if you don't expect any changes in the globals, you can just
861
+ keep the tensor guards.
862
+
863
+ >> opt_mod = torch.compile(
864
+ >> mod,
865
+ >> options={"guard_filter_fn": torch.compiler.skip_guard_on_globals},
866
+ >> )
867
+ """
868
+
869
+ return [not entry.is_global for entry in guard_entries]
870
+
871
+
872
+ def skip_all_guards_unsafe(guard_entries):
873
+ """
874
+ A function for skipping all guards on a compiled function.
875
+
876
+ WARNING: This function will drop all the safety guarantees from Dynamo
877
+ compiled function. Use this with caution.
878
+
879
+ To use this API, use guard_filter_fn argument while calling torch.compile
880
+
881
+ >> opt_mod = torch.compile(
882
+ >> mod,
883
+ >> options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe},
884
+ >> )
885
+ """
886
+ return [False for entry in guard_entries]
887
+
888
+
889
+ def nested_compile_region(
890
+ fn=None,
891
+ *,
892
+ options: NestedCompileRegionOptions | None = None,
893
+ max_reuse_entries: int = 8,
894
+ reuse_hash_fn=None,
895
+ ):
896
+ """
897
+ Tells **``torch.compile``** that the marked set of operations forms a nested
898
+ compile region (which is often repeated in the full model) whose code can be
899
+ compiled once and safely reused. ``nested_compile_region`` can also be used
900
+ as a decorator.
901
+
902
+ During **``torch.compile``** tracing, the compiler applies *hierarchical
903
+ compilation* with ``nested_compile_region``: it emits optimized code for the
904
+ marked region the first time it is encountered and re-emits (or "stamps
905
+ out") the previously compiled code on every subsequent invocation. This can
906
+ substantially reduce overall compile time for deeply-stacked,
907
+ structurally-identical components such as the transformer layers of a
908
+ large-language-model (LLM).
909
+
910
+ Outside a ``torch.compile`` context—i.e., in standard eager execution—the
911
+ call is a no-op, so existing workflows remain unaffected.
912
+
913
+ Note that ``nested_compile_region`` **does not** promise that a region will
914
+ be compiled exactly once. If the compiler detects that new input conditions
915
+ (shape, dtype, device, stride, globals etc.) make the cached version invalid
916
+ to reuse, it will transparently re-compile the region. Using it is
917
+ therefore *safe*: correctness is always preserved, and you pay the extra
918
+ compilation cost only when required.
919
+
920
+ Args:
921
+ fn: The function to wrap
922
+ options: Optional backend to use for compiling the subgraph.
923
+ Warning: this is an experimental feature under development and
924
+ not ready for use yet.
925
+ max_reuse_entries: Maximum number of reuse cache entries per function
926
+ before raising an error. If this limit is hit, guards keep failing
927
+ across invocations and hierarchical compilation is not effective.
928
+ reuse_hash_fn: Optional callable that takes the same ``*args, **kwargs``
929
+ as the wrapped function and returns an integer hash key. When
930
+ provided, Dynamo traces this function to obtain a constant integer
931
+ and uses it as the cache key for subgraph reuse, bypassing the
932
+ automatic fingerprint/guard machinery. Two calls that produce the
933
+ same hash key reuse the same cached subgraph. The hash function
934
+ must be fully traceable (no graph breaks) and must return a
935
+ constant integer.
936
+ """
937
+
938
+ if options is not None:
939
+ from torch._dynamo import config as dynamo_config
940
+
941
+ if not dynamo_config.enable_invoke_subgraph_regional_compile:
942
+ raise RuntimeError(
943
+ "nested_compile_region config is an experimental feature for testing only."
944
+ )
945
+
946
+ from torch._higher_order_ops.invoke_subgraph import (
947
+ mark_compile_region as _mark_compile_region,
948
+ )
949
+
950
+ return _mark_compile_region(
951
+ fn,
952
+ options=options,
953
+ max_reuse_entries=max_reuse_entries,
954
+ reuse_hash_fn=reuse_hash_fn,
955
+ )
956
+
957
+
958
+ def load_compiled_function(
959
+ file: io.IOBase,
960
+ *,
961
+ f_globals: dict[str, object] | None = None,
962
+ external_data: dict[str, Any] | None = None,
963
+ ) -> Callable[..., Any]:
964
+ """
965
+ Load an aot-compiled function from a file.
966
+
967
+ .. warning::
968
+
969
+ This API is currently experimental and subject to change.
970
+
971
+ Args:
972
+ file: A file-like object containing the serialized compiled function.
973
+ f_globals: Optional global scope enclosing the compiled function.
974
+ external_data: Optional data to be loaded into the runtime environment
975
+ of the compiled function. This should contain the same
976
+ data as AOTCompileResult.external_data returned from save_compiled_function() call.
977
+
978
+ Returns:
979
+ A torch-compiled function with compilation preloaded from disk.
980
+ """
981
+ from torch._dynamo.aot_compile import AOTCompiledFunction
982
+
983
+ data = file.read()
984
+ return AOTCompiledFunction.deserialize(data, f_globals, external_data)
western_ai/usa/pytorch/torch/compiler/_cache.py ADDED
@@ -0,0 +1,342 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import dataclasses
3
+ import logging
4
+ from abc import ABC, abstractmethod
5
+ from collections import defaultdict
6
+ from collections.abc import Generator
7
+ from contextlib import contextmanager
8
+ from itertools import chain
9
+ from typing import Any
10
+
11
+ from torch.utils._appending_byte_serializer import (
12
+ AppendingByteSerializer,
13
+ BytesReader,
14
+ BytesWriter,
15
+ )
16
+ from torch.utils._ordered_set import OrderedSet
17
+
18
+
19
+ log = logging.getLogger(__name__)
20
+
21
+
22
+ @dataclasses.dataclass(frozen=True)
23
+ class CacheArtifact(ABC):
24
+ """
25
+ Data for each cache artifact that will be serialized and deserialized
26
+ """
27
+
28
+ key: str
29
+ content: bytes = dataclasses.field(repr=False) # Do not display potential binary
30
+
31
+ @staticmethod
32
+ def serialize(writer: BytesWriter, cls: "CacheArtifact") -> None:
33
+ writer.write_str(cls.key)
34
+ writer.write_bytes(cls.content)
35
+
36
+ @staticmethod
37
+ def deserialize(artifact_type: str, reader: BytesReader) -> "CacheArtifact":
38
+ key = reader.read_str()
39
+ content = reader.read_bytes()
40
+ return CacheArtifactFactory.create(artifact_type, key, content)
41
+
42
+ @staticmethod
43
+ def encode(content: Any) -> bytes:
44
+ if not isinstance(content, bytes):
45
+ raise AssertionError(f"Expected bytes, got {type(content)}")
46
+ return content
47
+
48
+ @abstractmethod
49
+ def populate_cache(self) -> None:
50
+ pass
51
+
52
+ @staticmethod
53
+ def type() -> str:
54
+ """
55
+ Returns the type of the artifact. Must be unique across all CacheArtifact classes.
56
+
57
+ CacheArtifactFactory.register will add property method to CacheInfo based on this (def {type}_artifacts)
58
+ that returns all artifacts for specific cache.
59
+ """
60
+ raise RuntimeError("CacheArtifact is an abstract class, please use a subclass")
61
+
62
+
63
+ class CacheArtifactFactory:
64
+ """
65
+ Factory for creating CacheArtifact objects based on their type
66
+ """
67
+
68
+ _artifact_types: dict[str, type[CacheArtifact]] = {}
69
+
70
+ @classmethod
71
+ def register(cls, artifact_cls: type[CacheArtifact]) -> type[CacheArtifact]:
72
+ artifact_type_key = artifact_cls.type()
73
+ if artifact_cls.type() in cls._artifact_types:
74
+ raise AssertionError(
75
+ f"Artifact of type={artifact_type_key} already registered in mega-cache artifact factory"
76
+ )
77
+ cls._artifact_types[artifact_type_key] = artifact_cls
78
+ setattr(
79
+ CacheInfo,
80
+ f"{artifact_type_key}_artifacts",
81
+ property(lambda self: self.artifacts[artifact_type_key]),
82
+ )
83
+ return artifact_cls
84
+
85
+ @classmethod
86
+ def _get_artifact_type(cls, artifact_type_key: str) -> type[CacheArtifact]:
87
+ if artifact_type_key not in cls._artifact_types:
88
+ raise AssertionError(
89
+ f"Artifact of type={artifact_type_key} not registered in mega-cache artifact factory"
90
+ )
91
+ return cls._artifact_types[artifact_type_key]
92
+
93
+ @classmethod
94
+ def create(cls, artifact_type_key: str, key: str, content: bytes) -> CacheArtifact:
95
+ artifact_cls = cls._get_artifact_type(artifact_type_key)
96
+ # pyrefly: ignore [bad-instantiation]
97
+ return artifact_cls(key, content)
98
+
99
+ @classmethod
100
+ def encode_create(
101
+ cls, artifact_type_key: str, key: str, content: Any
102
+ ) -> CacheArtifact:
103
+ artifact_cls = cls._get_artifact_type(artifact_type_key)
104
+ # pyrefly: ignore [bad-instantiation]
105
+ return artifact_cls(key, artifact_cls.encode(content))
106
+
107
+
108
+ @dataclasses.dataclass
109
+ class CacheInfo:
110
+ """
111
+ Return value of serialization and deserialization for the purpose of
112
+ instrumentation
113
+ """
114
+
115
+ artifacts: defaultdict[str, list[str]] = dataclasses.field(
116
+ default_factory=lambda: defaultdict(list)
117
+ )
118
+
119
+ # Methods set by CacheArtifactFactory.register based on CacheArtifact.type()
120
+ @property
121
+ def inductor_artifacts(self) -> list[str]: # type: ignore[empty-body]
122
+ ...
123
+
124
+ @property
125
+ def autotune_artifacts(self) -> list[str]: # type: ignore[empty-body]
126
+ ...
127
+
128
+ @property
129
+ def aot_autograd_artifacts(self) -> list[str]: # type: ignore[empty-body]
130
+ ...
131
+
132
+ @property
133
+ def pgo_artifacts(self) -> list[str]: # type: ignore[empty-body]
134
+ ...
135
+
136
+ @property
137
+ def precompile_artifacts(self) -> list[str]: # type: ignore[empty-body]
138
+ ...
139
+
140
+ def add(self, artifact: CacheArtifact) -> None:
141
+ self.artifacts[artifact.type()].append(artifact.key)
142
+
143
+ def clear(self) -> None:
144
+ self.artifacts.clear()
145
+
146
+ def empty(self) -> bool:
147
+ return not self.artifacts
148
+
149
+
150
+ def _serialize_single_cache(
151
+ writer: BytesWriter, cls: "tuple[str, list[CacheArtifact]]"
152
+ ) -> None:
153
+ writer.write_str(cls[0])
154
+ writer.write_uint64(len(cls[1]))
155
+ for artifact in cls[1]:
156
+ CacheArtifact.serialize(writer, artifact)
157
+
158
+
159
+ def _deserialize_single_cache(
160
+ reader: BytesReader,
161
+ ) -> "tuple[str, list[CacheArtifact]]":
162
+ artifacts = []
163
+ artifact_type_key = reader.read_str()
164
+ num_artifacts = reader.read_uint64()
165
+ for _ in range(num_artifacts):
166
+ artifacts.append(CacheArtifact.deserialize(artifact_type_key, reader))
167
+
168
+ return artifact_type_key, artifacts
169
+
170
+
171
+ CacheArtifactsResult = dict[str, list[CacheArtifact]]
172
+
173
+
174
+ @dataclasses.dataclass(frozen=True)
175
+ class CacheArtifactRecorder:
176
+ """
177
+ Shared entry point for cache implementations to record artifacts.
178
+ """
179
+
180
+ artifact_type: str
181
+ key: str
182
+
183
+ def record(self, content: Any) -> None:
184
+ CacheArtifactManager.record_artifact(self.artifact_type, self.key, content)
185
+
186
+ def record_if_present(self, content: Any | None) -> None:
187
+ if content is not None:
188
+ self.record(content)
189
+
190
+
191
+ class CacheArtifactManager:
192
+ """
193
+ Lightweight manager class for collecting and processing cache artifacts for
194
+ hot loading
195
+
196
+ Intended Lifecycle:
197
+ - Execute code via torch.compile, this will call
198
+ CacheArtifactRecorder.record on each cache artifact
199
+ - Call CacheArtifactManager.serialize to convert all the cache artifacts
200
+ to portable format
201
+ - Call CacheArtifactManager.deserialize to hot load the cache artifacts on
202
+ a potentially different process
203
+
204
+ NOTE: There's no FB/FC guarantees, results of cache artifacts will not be
205
+ used unless code version matches.
206
+ """
207
+
208
+ # Protected by the compile_lock
209
+ _new_cache_artifacts: CacheArtifactsResult = defaultdict(list)
210
+ # Keep a separate seen artifacts list to make avoid unnecessary duplicates
211
+ # This list will not be cleared between serialize() calls
212
+ _seen_artifacts: OrderedSet[CacheArtifact] = OrderedSet()
213
+ # When serialize() is called, artifacts are transferred from _cache_artifacts to
214
+ # internal data structure of the _serializer
215
+ # This allows us to only pay the cost of serialization if serialize() is called
216
+ _serializer: AppendingByteSerializer[tuple[str, list[CacheArtifact]]] = (
217
+ AppendingByteSerializer(serialize_fn=_serialize_single_cache)
218
+ )
219
+ _cache_info: CacheInfo = CacheInfo()
220
+
221
+ @classmethod
222
+ def clear(cls) -> None:
223
+ cls._new_cache_artifacts.clear()
224
+ cls._seen_artifacts.clear()
225
+ cls._serializer.clear()
226
+ cls._cache_info.clear()
227
+
228
+ @classmethod
229
+ @contextmanager
230
+ def with_fresh_cache(cls) -> Generator[None, None, None]:
231
+ original_new_cache_artifacts = cls._new_cache_artifacts
232
+ original_seen_artifacts = cls._seen_artifacts
233
+ original_serializer = cls._serializer
234
+ original_cache_info = cls._cache_info
235
+
236
+ cls._new_cache_artifacts = defaultdict(list)
237
+ cls._seen_artifacts = OrderedSet()
238
+ cls._serializer = AppendingByteSerializer(serialize_fn=_serialize_single_cache)
239
+ cls._cache_info = cls._cache_info.__class__()
240
+ try:
241
+ yield
242
+ finally:
243
+ cls._new_cache_artifacts = original_new_cache_artifacts
244
+ cls._seen_artifacts = original_seen_artifacts
245
+ cls._serializer = original_serializer
246
+ cls._cache_info = original_cache_info
247
+
248
+ @classmethod
249
+ def record_artifact(
250
+ cls,
251
+ artifact_type: str,
252
+ key: str,
253
+ content: Any,
254
+ ) -> None:
255
+ """
256
+ Called from each caching operation to record the artifact in this
257
+ "mega" list
258
+ """
259
+ artifact = CacheArtifactFactory.encode_create(artifact_type, key, content)
260
+ if artifact in cls._seen_artifacts:
261
+ return
262
+ log.debug("Recording %s", artifact)
263
+ cls._new_cache_artifacts[artifact_type].append(artifact)
264
+ cls._seen_artifacts.add(artifact)
265
+
266
+ @classmethod
267
+ def need_serialize(cls) -> bool:
268
+ """
269
+ Have we seen new artifacts since last serialize call?
270
+ """
271
+ return len(cls._new_cache_artifacts) != 0
272
+
273
+ @classmethod
274
+ def serialize(cls) -> tuple[bytes, CacheInfo] | None:
275
+ """
276
+ Converts the "mega" list into portable format
277
+ """
278
+ for artifact in chain(*cls._new_cache_artifacts.values()):
279
+ log.debug("saving: %s", artifact)
280
+ cls._cache_info.add(artifact)
281
+
282
+ if cls._cache_info.empty():
283
+ # If there are no artifacts, don't just return bytes with
284
+ # version.
285
+ return None
286
+
287
+ try:
288
+ # We deep copy cls._cache_info since later compilations
289
+ # can keep adding to cache_info
290
+ info = copy.deepcopy(cls._cache_info)
291
+ cls._serializer.extend(cls._new_cache_artifacts.items())
292
+ artifact_bytes = cls._serializer.to_bytes()
293
+ cls._new_cache_artifacts.clear()
294
+ return artifact_bytes, info
295
+ except Exception:
296
+ log.warning("Failed to pickle cache artifacts", exc_info=True)
297
+ return None
298
+
299
+ @staticmethod
300
+ def deserialize(serialized_artifacts: bytes) -> CacheArtifactsResult | None:
301
+ """
302
+ Converts the portable format back into CacheArtifacts
303
+ """
304
+ try:
305
+ CacheArtifactManager._ensure_cache_artifacts_registered()
306
+ artifacts = dict(
307
+ AppendingByteSerializer.to_list(
308
+ serialized_artifacts,
309
+ deserialize_fn=_deserialize_single_cache,
310
+ )
311
+ )
312
+ except Exception:
313
+ log.warning("Failed to un-pickle cache artifacts", exc_info=True)
314
+ return None
315
+
316
+ return artifacts
317
+
318
+ @staticmethod
319
+ def populate_caches(artifacts: CacheArtifactsResult) -> CacheInfo:
320
+ info = CacheInfo()
321
+ for artifact in chain(*artifacts.values()):
322
+ log.debug("writing: %s", artifact)
323
+ info.add(artifact)
324
+ artifact.populate_cache()
325
+
326
+ return info
327
+
328
+ @classmethod
329
+ def _ensure_cache_artifacts_registered(cls) -> None:
330
+ """When deserializing caches in fresh process, we need to ensure that all
331
+ cache artifacts are registered in the cache registry. This is done by
332
+ simply importing all the cache artifacts already wrapped with register call.
333
+ """
334
+ from torch._dynamo.package import PrecompileCacheArtifact # noqa: F401
335
+ from torch._dynamo.pgo import PGOCacheArtifact # noqa: F401
336
+ from torch._functorch._aot_autograd.autograd_cache import ( # noqa: F401
337
+ AOTAutogradCacheArtifact,
338
+ )
339
+ from torch._inductor.codecache import InductorCacheArtifact # noqa: F401
340
+ from torch._inductor.runtime.autotune_cache import ( # noqa: F401
341
+ AutotuneCacheArtifact,
342
+ )
western_ai/usa/pytorch/torch/compiler/config.py ADDED
@@ -0,0 +1,344 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ This is the top-level configuration module for the compiler, containing
3
+ cross-cutting configuration options that affect all parts of the compiler
4
+ stack.
5
+
6
+ You may also be interested in the per-component configuration modules, which
7
+ contain configuration options that affect only a specific part of the compiler:
8
+
9
+ * :mod:`torch._dynamo.config`
10
+ * :mod:`torch._inductor.config`
11
+ * :mod:`torch._functorch.config`
12
+ * :mod:`torch.fx.experimental.config`
13
+ """
14
+
15
+ import sys
16
+
17
+ from torch.utils._config_module import Config, install_config_module
18
+
19
+
20
+ __all__ = [
21
+ "job_id",
22
+ "dynamic_shapes",
23
+ "assume_static_by_default",
24
+ "automatic_dynamic_shapes",
25
+ "recompile_limit",
26
+ "accumulated_recompile_limit",
27
+ "verbose",
28
+ "capture_scalar_outputs",
29
+ "capture_dynamic_output_shape_ops",
30
+ "log_file_name",
31
+ "fail_on_recompile_limit_hit",
32
+ "allow_unspec_int_on_nn_module",
33
+ "skip_tensor_guards_with_matching_dict_tags",
34
+ "enable_cpp_symbolic_shape_guards",
35
+ "wrap_top_frame",
36
+ "reorderable_logging_functions",
37
+ "force_disable_caches",
38
+ "compile_on_one_rank",
39
+ ]
40
+
41
+
42
+ # NB: Docblocks go UNDER variable definitions! Use spacing to make the
43
+ # grouping clear.
44
+
45
+ # FB-internal note: you do NOT have to specify this explicitly specify this if
46
+ # you run on MAST, we will automatically default this to
47
+ # mast:MAST_JOB_NAME:MAST_JOB_VERSION.
48
+ job_id: str | None = Config(
49
+ env_name_default=["TORCH_COMPILE_JOB_ID", "TORCH_COMPILE_STICKY_PGO_KEY"],
50
+ default=None,
51
+ )
52
+ """
53
+ Semantically, this should be an identifier that uniquely identifies, e.g., a
54
+ training job. You might have multiple attempts of the same job, e.g., if it was
55
+ preempted or needed to be restarted, but each attempt should be running
56
+ substantially the same workload with the same distributed topology. You can
57
+ set this by environment variable with :envvar:`TORCH_COMPILE_JOB_ID`.
58
+
59
+ Operationally, this controls the effect of profile-guided optimization related
60
+ persistent state. PGO state can affect how we perform compilation across
61
+ multiple invocations of PyTorch, e.g., the first time you run your program we
62
+ may compile twice as we discover what inputs are dynamic, and then PGO will
63
+ save this state so subsequent invocations only need to compile once, because
64
+ they remember it is dynamic. This profile information, however, is sensitive
65
+ to what workload you are running, so we require you to tell us that two jobs
66
+ are *related* (i.e., are the same workload) before we are willing to reuse
67
+ this information. Notably, PGO does nothing (even if explicitly enabled)
68
+ unless a valid ``job_id`` is available. In some situations, PyTorch can
69
+ configured to automatically compute a ``job_id`` based on the environment it
70
+ is running in.
71
+
72
+ Profiles are always collected on a per rank basis, so different ranks may have
73
+ different profiles. If you know your workload is truly SPMD, you can run with
74
+ :data:`torch._dynamo.config.enable_compiler_collectives` to ensure nodes get
75
+ consistent profiles across all ranks.
76
+ """
77
+
78
+ pgo_extra_read_key: str | None = Config(
79
+ env_name_default="TORCH_COMPILE_STICKY_PGO_READ", default=None
80
+ )
81
+ pgo_extra_write_key: str | None = Config(
82
+ env_name_default="TORCH_COMPILE_STICKY_PGO_WRITE", default=None
83
+ )
84
+ """
85
+ Additional read/write keys for PGO.
86
+ Write key: Besides writing to the default local/remote PGO state, this also writes to the specified key.
87
+ Read key: Besides reading from the default state, this also reads from the specified key (if written to before)
88
+ and merges it with the default state.
89
+ """
90
+
91
+
92
+ cache_key_tag: str = Config(env_name_default="TORCH_COMPILE_CACHE_KEY_TAG", default="")
93
+ """
94
+ Tag to be included in the cache key generation for all torch compile caching.
95
+ A common use case for such a tag is to break caches.
96
+ """
97
+
98
+ force_disable_caches: bool = Config(
99
+ justknob="pytorch/remote_cache:force_disable_caches",
100
+ env_name_force=[
101
+ "TORCHINDUCTOR_FORCE_DISABLE_CACHES",
102
+ "TORCH_COMPILE_FORCE_DISABLE_CACHES",
103
+ ],
104
+ default=False,
105
+ )
106
+ """
107
+ Force disables all caching -- This will take precedence over and override any other caching flag
108
+ """
109
+
110
+ compile_on_one_rank: bool = Config(
111
+ default=False,
112
+ env_name_default=[
113
+ "TORCH_COMPILE_ON_ONE_RANK",
114
+ "TORCH_DISTRIBUTED_COMPILE_ON_ONE_RANK",
115
+ ],
116
+ )
117
+ """
118
+ When enabled, device- and rank-specific values (devices, process groups) are computed at
119
+ runtime via custom ops rather than baked in at compile time, so a graph can be compiled on
120
+ one rank and run on all ranks. Read across the stack: make_fx, inductor, and distributed.
121
+ """
122
+
123
+ dynamic_sources: str = Config(
124
+ env_name_default="TORCH_COMPILE_DYNAMIC_SOURCES", default=""
125
+ )
126
+ r"""
127
+ Comma delimited list of sources that should be marked as dynamic. Primarily useful for large
128
+ models with graph breaks where you need intermediate tensors and ints to be marked dynamic.
129
+
130
+ Each entry is either an exact source name or a Python regex (regex applies only to the
131
+ source name, not to the dim) matched with ``re.match``. An optional ``:N`` suffix restricts
132
+ the entry to dim ``N`` of the matched tensor; without the suffix all dims are marked dynamic.
133
+ ``N`` must be a literal non-negative integer. To target multiple dims of the same source,
134
+ list it multiple times. Examples::
135
+
136
+ L['x'] # all dims of x dynamic (current behavior)
137
+ L['x']:0 # only dim 0 of x dynamic
138
+ L['x']:0, L['x']:2 # dims 0 and 2 of x dynamic
139
+
140
+ Regex examples (``re.match`` anchors at the start, not the end)::
141
+
142
+ L\['x.*'\] # all sources whose name starts with L['x (e.g. L['x'], L['x1'])
143
+ L\['x.*'\]:0 # dim 0 dynamic for every source matching L['x.*
144
+ .* # mark every source dynamic (all dims)
145
+ .*:0 # mark dim 0 dynamic for every tensor source
146
+
147
+ The ``:N`` suffix is ignored for non-tensor (int/scalar) sources, which have no dim.
148
+
149
+ Entries in this list are dominant over all other flags dynamic=False, force_nn_module_property_static_shapes
150
+ and force_parameter_static_shapes.
151
+ """
152
+
153
+ dynamic_values: str = Config(
154
+ env_name_default="TORCH_COMPILE_DYNAMIC_VALUES", default=""
155
+ )
156
+ """
157
+ Comma delimited list of integer values that should cause any matching int source or
158
+ tensor dim to be marked dynamic.
159
+
160
+ This is useful for JIT workflows with graph breaks where you don't know up-front which
161
+ sources need to be dynamic, but you can warm up the model with a distinctive "sentinel"
162
+ value (e.g. an odd batch size like 1111) and have any int or tensor dim that equals one
163
+ of those values be marked dynamic automatically. This is needed because with graph
164
+ breaks we do not propagate dynamic properties through eager code, so an int or tensor
165
+ derived in eager between two compiled regions cannot inherit the dynamic-ness of its
166
+ source — matching on the sentinel value lets us recover it on the next compiled region.
167
+
168
+ Example::
169
+
170
+ TORCH_COMPILE_DYNAMIC_VALUES="1111,2222"
171
+
172
+ Pick sentinel values unlikely to occur as legitimate shapes/ints in your
173
+ workload (e.g. 1111, 2222) — matching is purely by value, so any int source
174
+ or tensor dim equal to a listed value anywhere in the program will be
175
+ marked dynamic.
176
+
177
+ unlike ``dynamic_sources``, ``dynamic_values`` does NOT override
178
+ ``force_parameter_static_shapes`` or ``force_nn_module_property_static_shapes``.
179
+ """
180
+
181
+ unbacked_sources: str = Config(
182
+ env_name_default="TORCH_COMPILE_UNBACKED_SOURCES", default=""
183
+ )
184
+ """
185
+ Comma delimited list of sources that should be marked as unbacked. Primarily useful for large
186
+ models with graph breaks where you need intermediate tensors marked unbacked.
187
+
188
+ Supports the same ``:N`` per-dim suffix syntax as ``dynamic_sources``; see that config's
189
+ docstring for details.
190
+
191
+ Entries in this list are dominant over all other flags dynamic=False, force_nn_module_property_static_shapes
192
+ and force_parameter_static_shapes.
193
+ """
194
+
195
+ # force a python GC before recording cudagraphs
196
+ force_cudagraph_gc: bool = Config(env_name_default="TORCH_CUDAGRAPH_GC", default=False)
197
+ """
198
+ If True (the backward-compatible behavior) then gc.collect() before recording
199
+ any cudagraph.
200
+ """
201
+
202
+
203
+ # Cross-cutting configuration options that affect the entire compilation pipeline
204
+
205
+ dynamic_shapes: bool = Config(alias="torch._dynamo.config.dynamic_shapes")
206
+ """
207
+ Controls whether the compilation pipeline supports dynamic tensor shapes.
208
+ When enabled, the compiler can handle tensors with varying dimensions across
209
+ different invocations. This is a cross-cutting setting that affects shape
210
+ inference, guard generation, and code generation across the entire compilation
211
+ stack.
212
+ """
213
+
214
+ assume_static_by_default: bool = Config(
215
+ alias="torch._dynamo.config.assume_static_by_default"
216
+ )
217
+ """
218
+ When enabled, all tensor dimensions are assumed to be static unless explicitly
219
+ marked as dynamic or detected as changing. This compilation-wide behavior affects
220
+ how the entire stack handles shape specialization and can improve performance
221
+ for static workloads.
222
+ """
223
+
224
+ automatic_dynamic_shapes: bool = Config(
225
+ alias="torch._dynamo.config.automatic_dynamic_shapes"
226
+ )
227
+ """
228
+ Enables automatic detection and handling of dynamic shapes. When a tensor's
229
+ shape changes between compilations, the system automatically marks those
230
+ dimensions as dynamic rather than requiring manual specification. This
231
+ cross-cutting optimization improves the user experience by reducing recompilations.
232
+ """
233
+
234
+ recompile_limit: int = Config(alias="torch._dynamo.config.recompile_limit")
235
+ """
236
+ Maximum number of recompilations allowed for a single function before falling
237
+ back to eager execution. This compilation performance control prevents excessive
238
+ recompilation overhead that can degrade overall performance.
239
+ """
240
+
241
+ accumulated_recompile_limit: int = Config(
242
+ alias="torch._dynamo.config.accumulated_recompile_limit"
243
+ )
244
+ """
245
+ Global limit on total recompilations across all compiled functions to prevent
246
+ runaway recompilation scenarios. This safeguard protects against compilation
247
+ performance issues that could affect the entire program.
248
+ """
249
+
250
+ verbose: bool = Config(alias="torch._dynamo.config.verbose")
251
+ """
252
+ Enables verbose debugging output for Dynamo. When enabled, provides detailed
253
+ information about Dynamo's compilation decisions, optimizations, and potential
254
+ issues.
255
+ """
256
+
257
+
258
+ # TorchDynamo-specific configuration options
259
+
260
+ capture_scalar_outputs: bool = Config(
261
+ alias="torch._dynamo.config.capture_scalar_outputs"
262
+ )
263
+ """
264
+ Controls whether TorchDynamo captures operations that return scalar values (like .item())
265
+ into the FX graph. When disabled, these operations cause graph breaks. This is a
266
+ TorchDynamo-specific tracing behavior that affects how the tracer handles
267
+ scalar-returning operations.
268
+ """
269
+
270
+ capture_dynamic_output_shape_ops: bool = Config(
271
+ alias="torch._dynamo.config.capture_dynamic_output_shape_ops"
272
+ )
273
+ """
274
+ Controls whether TorchDynamo captures operations with dynamic output shapes (like
275
+ nonzero, unique) into the FX graph. When disabled, these operations cause graph breaks.
276
+ This is a TorchDynamo-specific setting for handling operations with unpredictable
277
+ output shapes during tracing.
278
+ """
279
+
280
+ log_file_name: str | None = Config(alias="torch._dynamo.config.log_file_name")
281
+ """
282
+ Specifies a file path for TorchDynamo-specific logging output. When set, internal
283
+ TorchDynamo debug information is written to this file rather than stdout. This is
284
+ useful for debugging TorchDynamo's internal tracing behavior.
285
+ """
286
+
287
+ fail_on_recompile_limit_hit: bool = Config(
288
+ alias="torch._dynamo.config.fail_on_recompile_limit_hit"
289
+ )
290
+ """
291
+ Raises a hard error when recompile limits are exceeded instead of falling back
292
+ to eager execution. This is useful for detecting excessive recompilation in
293
+ performance-critical deployments where you want to ensure compilation overhead
294
+ is kept under control.
295
+ """
296
+
297
+ allow_unspec_int_on_nn_module: bool = Config(
298
+ alias="torch._dynamo.config.allow_unspec_int_on_nn_module"
299
+ )
300
+ """
301
+ Allows integer attributes of nn.Module instances to be unspecialized through
302
+ the dynamic shape mechanism. By default, TorchDynamo specializes on all integer
303
+ module attributes, but this can cause excessive recompilation when integers
304
+ like step counters change frequently.
305
+ """
306
+
307
+ skip_tensor_guards_with_matching_dict_tags: bool = Config(
308
+ alias="torch._dynamo.config.skip_tensor_guards_with_matching_dict_tags"
309
+ )
310
+ """
311
+ Optimizes guard generation by treating tensors as immutable when they are
312
+ dictionary values with consistent dictionary tags across invocations. This
313
+ reduces guard overhead for tensors stored in persistent data structures.
314
+ """
315
+
316
+ enable_cpp_symbolic_shape_guards: bool = Config(
317
+ alias="torch._dynamo.config.enable_cpp_symbolic_shape_guards"
318
+ )
319
+ """
320
+ Uses C++ implementation for symbolic shape guard evaluation to improve performance.
321
+ The C++ guard manager can significantly speed up guard checking for symbolic shapes
322
+ in shape-polymorphic compilations.
323
+ """
324
+
325
+ wrap_top_frame: bool = Config(alias="torch._dynamo.config.wrap_top_frame")
326
+ """
327
+ Wraps the top-level decorated function/module in a frame wrapper to ensure
328
+ nn.Module hooks are compiled within the same frame as the main function. This
329
+ improves compilation coverage for models that rely on hooks.
330
+ """
331
+
332
+ reorderable_logging_functions: set = Config(
333
+ alias="torch._dynamo.config.reorderable_logging_functions"
334
+ )
335
+ """
336
+ A set of logging functions that can be reordered to execute after the compiled
337
+ portion of the graph, allowing larger graphs to be captured. Functions in this
338
+ set will have their execution deferred to avoid graph breaks, though this may
339
+ affect the timing of log output. In particular, mutated values will not be logged
340
+ at the right time, leading to incorrect logging.
341
+ """
342
+
343
+
344
+ install_config_module(sys.modules[__name__])
western_ai/usa/pytorch/torch/contrib/__init__.py ADDED
File without changes
western_ai/usa/pytorch/torch/contrib/_tensorboard_vis.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import time
3
+ from collections import defaultdict
4
+ from functools import partial
5
+
6
+ import torch
7
+
8
+
9
+ # Unfortunately it doesn't seem as if there was any way to get TensorBoard to do
10
+ # anything without having TF installed, and so this file has a hard dependency on it
11
+ # as well. It really is a debugging tool, so it doesn't matter.
12
+ try:
13
+ from tensorflow.core.framework import graph_pb2
14
+ from tensorflow.core.util import event_pb2
15
+ from tensorflow.python.summary.writer.writer import FileWriter
16
+ except ImportError:
17
+ raise ImportError(
18
+ "TensorBoard visualization of GraphExecutors requires having "
19
+ "TensorFlow installed"
20
+ ) from None
21
+
22
+
23
+ def dump_tensorboard_summary(graph_executor, logdir):
24
+ with FileWriter(logdir) as w:
25
+ pb_graph = visualize(graph_executor)
26
+ evt = event_pb2.Event(
27
+ wall_time=time.time(), graph_def=pb_graph.SerializeToString()
28
+ )
29
+ w.add_event(evt)
30
+
31
+
32
+ def visualize(graph, name_prefix="", pb_graph=None, executors_it=None):
33
+ """Visualizes an independent graph, or a graph executor."""
34
+ value_map = {}
35
+ pb_graph = pb_graph or graph_pb2.GraphDef()
36
+
37
+ if isinstance(graph, torch._C.GraphExecutorState):
38
+ visualize_graph_executor(
39
+ graph, name_prefix, pb_graph, partial(visualize, pb_graph=pb_graph)
40
+ )
41
+ return pb_graph
42
+
43
+ # Set up an input node
44
+ pb_graph.node.add(op="input", name=name_prefix + "input")
45
+ for i, value in enumerate(graph.param_node().outputs()):
46
+ value_map[value.unique()] = name_prefix + "input:" + str(i)
47
+
48
+ visualize_rec(graph, value_map, name_prefix, pb_graph, executors_it)
49
+
50
+ # Gather all outputs
51
+ return_node = pb_graph.node.add(op="output", name=name_prefix + "output")
52
+ for value in graph.return_node().inputs():
53
+ return_node.input.append(value_map[value.unique()])
54
+
55
+ return pb_graph
56
+
57
+
58
+ def visualize_graph_executor(state, name_prefix, pb_graph, inline_graph):
59
+ """Append the state of a given GraphExecutor to the graph protobuf.
60
+
61
+ Args:
62
+ state (GraphExecutor or GraphExecutorState): GraphExecutor to display.
63
+ name_prefix (str): Name prefix of the containing subgraph.
64
+ pb_graph (GraphDef): graph to append to.
65
+ inline_graph (Callable): a function that handles setting up a value_map,
66
+ so that some graphs in here can be inlined. This is necessary, because
67
+ this will simply be `visualize` for the top-level GraphExecutor,
68
+ or `inline_graph` for all nested ones.
69
+
70
+ The signature should look like (Graph, name_prefix) -> ().
71
+ It will be called exactly once.
72
+
73
+ The strategy is to embed all different configurations as independent subgraphs,
74
+ while inlining the original graph as the one that actually produces the values.
75
+ """
76
+ if state.autograd_fallback_graph is not None:
77
+ visualize(
78
+ graph=state.autograd_fallback_graph,
79
+ name_prefix=name_prefix + "autograd_fallback/",
80
+ pb_graph=pb_graph,
81
+ executors_it=iter(state.autograd_fallback.executors()),
82
+ )
83
+
84
+ for i, (arg_spec, plan) in enumerate(state.execution_plans.items()):
85
+ subgraph_name = name_prefix + f"plan{i}/"
86
+
87
+ # Create a disconnected node that will keep information regarding the input
88
+ # types of this trace. This is unfortunately a bit too verbose to be included
89
+ # in the subgraph name.
90
+ input_kinds = pb_graph.node.add(op="INPUT_KIND", name=subgraph_name)
91
+ input_kinds.attr["inputs"].s = repr(arg_spec).encode("ascii")
92
+
93
+ visualize(plan.graph, subgraph_name, pb_graph, iter(plan.code.executors()))
94
+
95
+ # Show gradient as an independent subgraph of this plan
96
+ if plan.grad_executor is not None:
97
+ grad_subgraph_name = subgraph_name + "grad/"
98
+ visualize(plan.grad_executor, grad_subgraph_name, pb_graph)
99
+
100
+ return inline_graph(state.graph, name_prefix + "original/")
101
+
102
+
103
+ def visualize_rec(graph, value_map, name_prefix, pb_graph, executors_it=None):
104
+ """Recursive part of visualize (basically skips setting up the input and output nodes)."""
105
+
106
+ def inline_graph(subgraph, name, node):
107
+ rec_value_map = {
108
+ inp.unique(): value_map[val.unique()]
109
+ for inp, val in zip(subgraph.inputs(), node.inputs())
110
+ }
111
+ visualize_rec(
112
+ graph=subgraph, value_map=rec_value_map, name_prefix=name, pb_graph=pb_graph
113
+ )
114
+ for out, val in zip(subgraph.outputs(), node.outputs()):
115
+ value_map[val.unique()] = rec_value_map[out.unique()]
116
+
117
+ op_id_counter: defaultdict[str, int] = defaultdict(int)
118
+
119
+ def name_for(node):
120
+ kind = node.kind()[node.kind().index("::") + 2 :]
121
+ op_id_counter[kind] += 1
122
+ return kind, name_prefix + kind + "_" + str(op_id_counter[kind])
123
+
124
+ def add_fusion_group(node):
125
+ op, name = name_for(node)
126
+ inline_graph(node.g("Subgraph"), name + "/", node)
127
+
128
+ def add_graph_executor(node):
129
+ op, name = name_for(node)
130
+ if executors_it is None:
131
+ add_node(node)
132
+ else:
133
+ ge = next(executors_it)
134
+ visualize_graph_executor(
135
+ ge, name + "/", pb_graph, partial(inline_graph, node=node)
136
+ )
137
+
138
+ def add_node(node):
139
+ if node.kind() == "prim::FusionGroup":
140
+ return add_fusion_group(node)
141
+ elif node.kind() == "prim::GraphExecutor":
142
+ return add_graph_executor(node)
143
+ op, name = name_for(node)
144
+ pb_node = pb_graph.node.add(op=op, name=name)
145
+ for value in node.inputs():
146
+ pb_node.input.append(value_map[value.unique()])
147
+ # TODO: handle attrs
148
+ for i, value in enumerate(node.outputs()):
149
+ value_map[value.unique()] = name + ":" + str(i)
150
+
151
+ for node in graph.nodes():
152
+ add_node(node)
western_ai/usa/pytorch/torch/cpu/__init__.py ADDED
@@ -0,0 +1,252 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ r"""
3
+ This package implements abstractions found in ``torch.cuda``
4
+ to facilitate writing device-agnostic code.
5
+ """
6
+
7
+ from collections.abc import Mapping
8
+ from contextlib import AbstractContextManager
9
+ from functools import lru_cache
10
+ from types import MappingProxyType
11
+ from typing import Any
12
+
13
+ import torch
14
+
15
+ from .. import device as _device
16
+ from . import amp
17
+
18
+
19
+ __all__ = [
20
+ "is_available",
21
+ "is_initialized",
22
+ "synchronize",
23
+ "current_device",
24
+ "current_stream",
25
+ "stream",
26
+ "set_device",
27
+ "device_count",
28
+ "Stream",
29
+ "StreamContext",
30
+ "Event",
31
+ "get_capabilities",
32
+ ]
33
+
34
+
35
+ @lru_cache(None)
36
+ def get_capabilities() -> Mapping[str, Any]:
37
+ """
38
+ Returns an immutable mapping of CPU capabilities detected at runtime.
39
+
40
+ This function queries the CPU for supported instruction sets and features
41
+ using cpuinfo. The result is cached after the first call for efficiency.
42
+
43
+ The returned mapping contains architecture-specific capabilities:
44
+
45
+ For x86/x86_64:
46
+ - SSE family: sse, sse2, sse3, ssse3, sse4_1, sse4_2, sse4a
47
+ - AVX family: avx, avx2, avx_vnni
48
+ - AVX-512 family: avx512_f, avx512_cd, avx512_dq, avx512_bw, avx512_vl,
49
+ avx512_ifma, avx512_vbmi, avx512_vbmi2, avx512_bitalg, avx512_vpopcntdq,
50
+ avx512_vnni, avx512_bf16, avx512_fp16, avx512_vp2intersect,
51
+ avx512_4vnniw, avx512_4fmaps
52
+ - AVX10 family: avx10_1, avx10_2
53
+ - AVX-VNNI-INT: avx_vnni_int8, avx_vnni_int16, avx_ne_convert
54
+ - AMX: amx_bf16, amx_tile, amx_int8, amx_fp16
55
+ - FMA: fma3, fma4
56
+ - Other: f16c, bmi, bmi2, popcnt, lzcnt, aes, sha, clflush, clflushopt, clwb
57
+
58
+ For ARM64:
59
+ - SIMD: neon, fp16_arith, bf16, i8mm, dot
60
+ - SVE: sve, sve2, sve_bf16, sve_max_length (when supported)
61
+ - SME: sme, sme2, sme_max_length (when supported)
62
+ - Other: atomics, fhm, rdm, crc32, aes, sha1, sha2, pmull
63
+
64
+ Common to all architectures:
65
+ - architecture: string identifying the CPU architecture
66
+
67
+ Returns:
68
+ MappingProxyType: An immutable mapping where keys are capability names
69
+ (e.g., 'avx2', 'sve') and values are booleans indicating
70
+ support, or integers for properties like vector lengths.
71
+
72
+ Example:
73
+ >>> caps = torch.cpu.get_capabilities()
74
+ >>> if caps.get("avx2", False):
75
+ ... print("AVX2 is supported")
76
+ >>> print(f"Architecture: {caps['architecture']}")
77
+ """
78
+ return MappingProxyType(torch._C._cpu._get_cpu_capability())
79
+
80
+
81
+ def _is_avx2_supported() -> bool:
82
+ r"""Returns a bool indicating if CPU supports AVX2."""
83
+ return get_capabilities().get("avx2", False)
84
+
85
+
86
+ def _is_avx512_supported() -> bool:
87
+ r"""Returns a bool indicating if CPU supports AVX512."""
88
+ return get_capabilities().get("avx512_f", False)
89
+
90
+
91
+ def _is_avx512_bf16_supported() -> bool:
92
+ r"""Returns a bool indicating if CPU supports AVX512_BF16."""
93
+ return get_capabilities().get("avx512_bf16", False)
94
+
95
+
96
+ def _is_vnni_supported() -> bool:
97
+ r"""Returns a bool indicating if CPU supports VNNI."""
98
+ # Note: Currently, it only checks avx512_vnni, will add the support of avx2_vnni later.
99
+ return get_capabilities().get("avx512_vnni", False)
100
+
101
+
102
+ def _is_amx_tile_supported() -> bool:
103
+ r"""Returns a bool indicating if CPU supports AMX_TILE."""
104
+ return get_capabilities().get("amx_tile", False)
105
+
106
+
107
+ def _is_amx_fp16_supported() -> bool:
108
+ r"""Returns a bool indicating if CPU supports AMX FP16."""
109
+ return get_capabilities().get("amx_fp16", False)
110
+
111
+
112
+ def _init_amx() -> bool:
113
+ r"""Initializes AMX instructions."""
114
+ return torch._C._cpu._init_amx()
115
+
116
+
117
+ def is_available() -> bool:
118
+ r"""Returns a bool indicating if CPU is currently available.
119
+
120
+ N.B. This function only exists to facilitate device-agnostic code
121
+
122
+ """
123
+ return True
124
+
125
+
126
+ def synchronize(device: torch.types.Device = None) -> None:
127
+ r"""Waits for all kernels in all streams on the CPU device to complete.
128
+
129
+ Args:
130
+ device (torch.device or int, optional): ignored, there's only one CPU device.
131
+
132
+ N.B. This function only exists to facilitate device-agnostic code.
133
+ """
134
+
135
+
136
+ class Stream:
137
+ """
138
+ N.B. This class only exists to facilitate device-agnostic code
139
+ """
140
+
141
+ def __init__(self, priority: int = -1) -> None:
142
+ pass
143
+
144
+ def wait_stream(self, stream) -> None:
145
+ pass
146
+
147
+ def record_event(self) -> None:
148
+ pass
149
+
150
+ def wait_event(self, event) -> None:
151
+ pass
152
+
153
+
154
+ class Event:
155
+ def query(self) -> bool:
156
+ return True
157
+
158
+ def record(self, stream=None) -> None:
159
+ pass
160
+
161
+ def synchronize(self) -> None:
162
+ pass
163
+
164
+ def wait(self, stream=None) -> None:
165
+ pass
166
+
167
+
168
+ _default_cpu_stream = Stream()
169
+ _current_stream = _default_cpu_stream
170
+
171
+
172
+ def current_stream(device: torch.types.Device = None) -> Stream:
173
+ r"""Returns the currently selected :class:`Stream` for a given device.
174
+
175
+ Args:
176
+ device (torch.device or int, optional): Ignored.
177
+
178
+ N.B. This function only exists to facilitate device-agnostic code
179
+
180
+ """
181
+ return _current_stream
182
+
183
+
184
+ class StreamContext(AbstractContextManager):
185
+ r"""Context-manager that selects a given stream.
186
+
187
+ N.B. This class only exists to facilitate device-agnostic code
188
+
189
+ """
190
+
191
+ cur_stream: Stream | None
192
+
193
+ def __init__(self, stream):
194
+ self.stream = stream
195
+ self.prev_stream = _default_cpu_stream
196
+
197
+ def __enter__(self):
198
+ cur_stream = self.stream
199
+ if cur_stream is None:
200
+ return
201
+
202
+ global _current_stream
203
+ self.prev_stream = _current_stream
204
+ _current_stream = cur_stream
205
+
206
+ def __exit__(self, type: Any, value: Any, traceback: Any) -> None:
207
+ cur_stream = self.stream
208
+ if cur_stream is None:
209
+ return
210
+
211
+ global _current_stream
212
+ _current_stream = self.prev_stream
213
+
214
+
215
+ def stream(stream: Stream) -> AbstractContextManager:
216
+ r"""Wrapper around the Context-manager StreamContext that
217
+ selects a given stream.
218
+
219
+ N.B. This function only exists to facilitate device-agnostic code
220
+ """
221
+ return StreamContext(stream)
222
+
223
+
224
+ def device_count() -> int:
225
+ r"""Returns number of CPU devices (not cores). Always 1.
226
+
227
+ N.B. This function only exists to facilitate device-agnostic code
228
+ """
229
+ return 1
230
+
231
+
232
+ def set_device(device: torch.types.Device) -> None:
233
+ r"""Sets the current device, in CPU we do nothing.
234
+
235
+ N.B. This function only exists to facilitate device-agnostic code
236
+ """
237
+
238
+
239
+ def current_device() -> str:
240
+ r"""Returns current device for cpu. Always 'cpu'.
241
+
242
+ N.B. This function only exists to facilitate device-agnostic code
243
+ """
244
+ return "cpu"
245
+
246
+
247
+ def is_initialized() -> bool:
248
+ r"""Returns True if the CPU is initialized. Always True.
249
+
250
+ N.B. This function only exists to facilitate device-agnostic code
251
+ """
252
+ return True
western_ai/usa/pytorch/torch/cpu/amp/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # pyrefly: ignore [deprecated]
2
+ from .autocast_mode import autocast
3
+ from .grad_scaler import GradScaler
western_ai/usa/pytorch/torch/cpu/amp/autocast_mode.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import sys
3
+ from typing import Any
4
+ from typing_extensions import deprecated
5
+
6
+ import torch
7
+
8
+
9
+ __all__ = ["autocast"]
10
+
11
+
12
+ @deprecated(
13
+ "`torch.cpu.amp.autocast(args...)` is deprecated. "
14
+ "Please use `torch.amp.autocast('cpu', args...)` instead.",
15
+ category=FutureWarning,
16
+ )
17
+ class autocast(torch.amp.autocast_mode.autocast):
18
+ r"""
19
+ See :class:`torch.autocast`.
20
+ ``torch.cpu.amp.autocast(args...)`` is deprecated. Please use ``torch.amp.autocast("cpu", args...)`` instead.
21
+ """
22
+
23
+ # TODO: remove this conditional once we stop supporting Python < 3.13
24
+ # Prior to Python 3.13, inspect.signature could not retrieve the correct
25
+ # signature information for classes decorated with @deprecated (unless
26
+ # the __new__ static method was explicitly defined);
27
+ #
28
+ # However, this issue has been fixed in Python 3.13 and later versions.
29
+ if sys.version_info < (3, 13):
30
+
31
+ def __new__(
32
+ cls,
33
+ enabled: bool = True,
34
+ dtype: torch.dtype = torch.bfloat16,
35
+ cache_enabled: bool = True,
36
+ ):
37
+ return super().__new__(cls)
38
+
39
+ def __init_subclass__(cls):
40
+ pass
41
+
42
+ def __init__(
43
+ self,
44
+ enabled: bool = True,
45
+ dtype: torch.dtype = torch.bfloat16,
46
+ cache_enabled: bool = True,
47
+ ):
48
+ if torch._jit_internal.is_scripting():
49
+ self._enabled = enabled
50
+ self.device = "cpu"
51
+ self.fast_dtype = dtype
52
+ return
53
+ super().__init__(
54
+ "cpu", enabled=enabled, dtype=dtype, cache_enabled=cache_enabled
55
+ )
56
+
57
+ def __enter__(self):
58
+ if torch._jit_internal.is_scripting():
59
+ return self
60
+ return super().__enter__()
61
+
62
+ # TODO: discuss a unified TorchScript-friendly API for autocast
63
+ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any): # type: ignore[override]
64
+ if torch._jit_internal.is_scripting():
65
+ return
66
+ return super().__exit__(exc_type, exc_val, exc_tb)
67
+
68
+ def __call__(self, func):
69
+ if torch._jit_internal.is_scripting():
70
+ return func
71
+ return super().__call__(func)
western_ai/usa/pytorch/torch/cpu/amp/grad_scaler.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing_extensions import deprecated
2
+
3
+ import torch
4
+
5
+
6
+ __all__ = ["GradScaler"]
7
+
8
+
9
+ class GradScaler(torch.amp.GradScaler):
10
+ r"""
11
+ See :class:`torch.amp.GradScaler`.
12
+ ``torch.cpu.amp.GradScaler(args...)`` is deprecated. Please use ``torch.amp.GradScaler("cpu", args...)`` instead.
13
+ """
14
+
15
+ @deprecated(
16
+ "`torch.cpu.amp.GradScaler(args...)` is deprecated. "
17
+ "Please use `torch.amp.GradScaler('cpu', args...)` instead.",
18
+ category=FutureWarning,
19
+ )
20
+ def __init__(
21
+ self,
22
+ init_scale: float = 2.0**16,
23
+ growth_factor: float = 2.0,
24
+ backoff_factor: float = 0.5,
25
+ growth_interval: int = 2000,
26
+ enabled: bool = True,
27
+ ) -> None:
28
+ super().__init__(
29
+ "cpu",
30
+ init_scale=init_scale,
31
+ growth_factor=growth_factor,
32
+ backoff_factor=backoff_factor,
33
+ growth_interval=growth_interval,
34
+ enabled=enabled,
35
+ )
western_ai/usa/pytorch/torch/csrc/CudaIPCTypes.cpp ADDED
@@ -0,0 +1,273 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include <ATen/MapAllocator.h>
2
+ #include <c10/cuda/CUDAGuard.h>
3
+ #include <torch/csrc/CudaIPCTypes.h>
4
+ #include <atomic>
5
+ #include <map>
6
+ #include <mutex>
7
+ #include <string>
8
+
9
+ namespace torch {
10
+
11
+ namespace {
12
+
13
+ void warnProducerTerminatedBeforeSharedTensorsReleased() {
14
+ static bool warned = false;
15
+ if (!warned) {
16
+ LOG(WARNING)
17
+ << "Producer process has been terminated before all shared CUDA tensors released. See Note [Sharing CUDA tensors]";
18
+ warned = true;
19
+ }
20
+ }
21
+
22
+ struct CudaIPCGlobalEntities {
23
+ // This class is used as a singleton (see cuda_ipc_global_entities)
24
+ // This variable is used to track its lifetime to avoid accessing it
25
+ // after it was destroyed which would lead to segmentation faults
26
+ // Note that a trivial type is used which doesn't suffer from construction
27
+ // and destruction order issues
28
+ static bool alive;
29
+
30
+ std::mutex ref_counters_mutex_;
31
+ std::atomic<int64_t> sync_events_used_{0};
32
+ std::map<std::string, std::shared_ptr<CudaIPCRefCountersFile>>
33
+ ref_counters_files_;
34
+ std::shared_ptr<CudaIPCRefCountersFile> next_available_ref_counters_file_;
35
+ CudaIPCSentDataLimbo CudaIPCSentDataLimbo_;
36
+ CudaIPCGlobalEntities() {
37
+ alive = true;
38
+ }
39
+ CudaIPCGlobalEntities(const CudaIPCGlobalEntities&) = delete;
40
+ CudaIPCGlobalEntities(CudaIPCGlobalEntities&&) = delete;
41
+ CudaIPCGlobalEntities& operator=(const CudaIPCGlobalEntities&) = delete;
42
+ CudaIPCGlobalEntities& operator=(CudaIPCGlobalEntities&&) = delete;
43
+ ~CudaIPCGlobalEntities() {
44
+ alive = false;
45
+ CudaIPCSentDataLimbo_.collect();
46
+ safe_clean_current_file();
47
+ if (next_available_ref_counters_file_) {
48
+ warnProducerTerminatedBeforeSharedTensorsReleased();
49
+ }
50
+ }
51
+ void safe_clean_current_file() {
52
+ std::lock_guard<std::mutex> lock(ref_counters_mutex_);
53
+ if (next_available_ref_counters_file_ &&
54
+ next_available_ref_counters_file_->offsets_in_use() == 0) {
55
+ ref_counters_files_.erase(next_available_ref_counters_file_->handle());
56
+ next_available_ref_counters_file_.reset();
57
+ }
58
+ }
59
+ };
60
+
61
+ bool CudaIPCGlobalEntities::alive = false;
62
+ CudaIPCGlobalEntities cuda_ipc_global_entities;
63
+
64
+ CudaIPCSentDataLimbo::~CudaIPCSentDataLimbo() {
65
+ collect();
66
+ if (size() > 0) {
67
+ warnProducerTerminatedBeforeSharedTensorsReleased();
68
+ }
69
+ }
70
+
71
+ bool CudaIPCSentDataLimbo::collect() {
72
+ bool freed_memory = false;
73
+ std::vector<std::unique_ptr<CudaIPCSentData>> reset_blocks;
74
+ { // Begin critical section to modify shared blocks
75
+ std::lock_guard<std::mutex> lock(limbo_mutex_);
76
+ std::vector<std::unique_ptr<CudaIPCSentData>> kept_blocks;
77
+ for (auto& sd : shared_blocks_) {
78
+ if (sd->counter_value() > 0) {
79
+ kept_blocks.push_back(std::move(sd));
80
+ } else {
81
+ freed_memory = true;
82
+ reset_blocks.push_back(std::move(sd));
83
+ }
84
+ }
85
+ shared_blocks_ = std::move(kept_blocks);
86
+ }
87
+ // Need to reset blocks out of the critical section here, otherwise it
88
+ // deadlocks.
89
+ for (auto& sd : reset_blocks) {
90
+ sd.reset();
91
+ }
92
+ return freed_memory;
93
+ }
94
+
95
+ void CudaIPCSentDataLimbo::add(std::unique_ptr<CudaIPCSentData> shared_block) {
96
+ std::lock_guard<std::mutex> lock(limbo_mutex_);
97
+ static bool warned = false;
98
+ if (shared_blocks_.size() > CUDA_IPC_WARN_AFTER_X_BLOCKS_IN_LIMBO &&
99
+ !warned) {
100
+ LOG(WARNING)
101
+ << "Producer process tried to deallocate over "
102
+ << CUDA_IPC_WARN_AFTER_X_BLOCKS_IN_LIMBO
103
+ << " memory blocks referred by consumer processes. Deallocation might be significantly slowed down. "
104
+ << "We assume it will never going to be the case, but if it is, please file but to https://github.com/pytorch/pytorch";
105
+ warned = true;
106
+ }
107
+ shared_blocks_.push_back(std::move(shared_block));
108
+ }
109
+
110
+ uint64_t CudaIPCSentDataLimbo::size() {
111
+ std::lock_guard<std::mutex> lock(limbo_mutex_);
112
+ return shared_blocks_.size();
113
+ }
114
+
115
+ void CudaIPCSentDataDelete(void* ptr) {
116
+ std::unique_ptr<CudaIPCSentData> sent_data(
117
+ static_cast<CudaIPCSentData*>(ptr));
118
+ if (!CudaIPCGlobalEntities::alive) {
119
+ return;
120
+ }
121
+ if (sent_data->counter_value() > 0) {
122
+ cuda_ipc_global_entities.CudaIPCSentDataLimbo_.add(std::move(sent_data));
123
+ }
124
+ cuda_ipc_global_entities.CudaIPCSentDataLimbo_.collect();
125
+ }
126
+
127
+ void ReturnRefCounter(const std::string& handle, uint64_t offset /* unused */) {
128
+ if (!CudaIPCGlobalEntities::alive) {
129
+ return;
130
+ }
131
+ std::lock_guard<std::mutex> lock(
132
+ cuda_ipc_global_entities.ref_counters_mutex_);
133
+ auto& map = cuda_ipc_global_entities.ref_counters_files_;
134
+ auto it = map.find(handle);
135
+ if (it != map.end()) {
136
+ it->second->return_offset(offset);
137
+ if (it->second->offsets_in_use() == 0 && !it->second->have_offsets()) {
138
+ map.erase(handle);
139
+ }
140
+ }
141
+ }
142
+
143
+ } // namespace
144
+
145
+ CudaIPCSentData::CudaIPCSentData(
146
+ std::string handle,
147
+ uint64_t offset,
148
+ uint64_t* counter_ptr,
149
+ at::Device device)
150
+ : handle_(std::move(handle)),
151
+ offset_(offset),
152
+ counter_ptr_(counter_ptr),
153
+ device_(device) {
154
+ #if !defined(USE_ROCM)
155
+ // CUDA have the unofficial limit on the number of recorded blocking
156
+ // interprocess events, to prevent using of all events, we are switching to
157
+ // StreamSync before limit reached.
158
+ //
159
+ // ```python
160
+ // import torch
161
+ // a = [ torch.cuda.Event(
162
+ // enable_timing=False, blocking=True, interprocess=True) for i in
163
+ // range(30000) ]
164
+ // [i.record() for i in a]
165
+ // ```
166
+ //
167
+ if (cuda_ipc_global_entities.sync_events_used_.load() <
168
+ CUDA_IPC_MAXIMUM_EVENTS_TO_USE) {
169
+ // TODO: More efficient would be to create event inside of main thread (at
170
+ // the moment of the queue.put). The reason this is more efficient is
171
+ // because the main thread may have queued extra work on the stream, which
172
+ // this event will consequently wait for (uselessly).
173
+ cuda_ipc_global_entities.sync_events_used_++;
174
+ C10_CUDA_CHECK(cudaEventCreateWithFlags(
175
+ &event_,
176
+ cudaEventDisableTiming | cudaEventInterprocess |
177
+ cudaEventBlockingSync));
178
+ C10_CUDA_CHECK(cudaEventRecord(
179
+ event_, c10::cuda::getCurrentCUDAStream(device.index())));
180
+ event_sync_required_ = true;
181
+ } else {
182
+ auto stream = c10::cuda::getCurrentCUDAStream(device.index());
183
+ at::cuda::stream_synchronize(stream);
184
+ event_ = nullptr;
185
+ event_sync_required_ = false;
186
+ }
187
+ #else
188
+ // cuIpcGetEventHandle with HIP is not supported, so we have to sync
189
+ // stream instead of passing event
190
+ auto stream = c10::cuda::getCurrentCUDAStream(device.index());
191
+ at::cuda::stream_synchronize(stream);
192
+ event_sync_required_ = false;
193
+ #endif
194
+ }
195
+
196
+ CudaIPCSentData::~CudaIPCSentData() {
197
+ if (!CudaIPCGlobalEntities::alive) {
198
+ original_ptr_.release_context();
199
+ }
200
+ ReturnRefCounter(handle_, offset_);
201
+ #if !defined(USE_ROCM)
202
+ try {
203
+ if (event_sync_required_) {
204
+ at::cuda::CUDAGuard device_guard(device_.index());
205
+ C10_CUDA_CHECK(cudaEventDestroy(event_));
206
+ if (!CudaIPCGlobalEntities::alive) {
207
+ return;
208
+ }
209
+ cuda_ipc_global_entities.sync_events_used_--;
210
+ }
211
+ // NOLINTNEXTLINE(bugprone-empty-catch)
212
+ } catch (...) { /* No throw */
213
+ }
214
+ #endif
215
+ }
216
+
217
+ uint64_t CudaIPCSentData::counter_value() {
218
+ return *counter_ptr_;
219
+ }
220
+
221
+ at::DataPtr GetNewRefCountedSentData(void* data, at::Device device) {
222
+ {
223
+ std::lock_guard<std::mutex> lock(
224
+ cuda_ipc_global_entities.ref_counters_mutex_);
225
+ if (!cuda_ipc_global_entities.next_available_ref_counters_file_) {
226
+ std::string ref_counter_handle = at::NewProcessWideShmHandle();
227
+
228
+ int flags =
229
+ at::ALLOCATOR_MAPPED_SHAREDMEM | at::ALLOCATOR_MAPPED_EXCLUSIVE;
230
+ at::DataPtr sptr = at::RefcountedMapAllocator::makeDataPtr(
231
+ ref_counter_handle.c_str(),
232
+ flags,
233
+ sizeof(int64_t) * CUDA_IPC_REF_COUNTER_FILE_SIZE,
234
+ nullptr);
235
+ auto rc = std::make_shared<CudaIPCRefCountersFile>(
236
+ ref_counter_handle, CUDA_IPC_REF_COUNTER_FILE_SIZE, std::move(sptr));
237
+ cuda_ipc_global_entities.ref_counters_files_[ref_counter_handle] = rc;
238
+ cuda_ipc_global_entities.next_available_ref_counters_file_ = rc;
239
+ }
240
+ }
241
+ cuda_ipc_global_entities.next_available_ref_counters_file_->set_counter(1);
242
+ auto sent_data = new CudaIPCSentData(
243
+ cuda_ipc_global_entities.next_available_ref_counters_file_->handle(),
244
+ cuda_ipc_global_entities.next_available_ref_counters_file_->get_offset(),
245
+ cuda_ipc_global_entities.next_available_ref_counters_file_->counter_ptr(),
246
+ device);
247
+
248
+ cuda_ipc_global_entities.next_available_ref_counters_file_->rotate_offset();
249
+ if (!cuda_ipc_global_entities.next_available_ref_counters_file_
250
+ ->have_offsets()) {
251
+ cuda_ipc_global_entities.next_available_ref_counters_file_.reset();
252
+ }
253
+ return at::DataPtr(data, sent_data, CudaIPCSentDataDelete, device);
254
+ }
255
+
256
+ bool CudaIPCCollect() {
257
+ if (!CudaIPCGlobalEntities::alive) {
258
+ return true;
259
+ }
260
+ bool freed_memory = cuda_ipc_global_entities.CudaIPCSentDataLimbo_.collect();
261
+ if (cuda_ipc_global_entities.CudaIPCSentDataLimbo_.size() == 0) {
262
+ cuda_ipc_global_entities.safe_clean_current_file();
263
+ }
264
+ return freed_memory;
265
+ }
266
+
267
+ } // namespace torch
268
+
269
+ namespace c10 {
270
+ namespace {
271
+ REGISTER_FREE_MEMORY_CALLBACK("cuda_ipc_collect", CudaIPCCollectCallback)
272
+ }
273
+ } // namespace c10