giancap61 commited on
Commit
834e7fc
·
verified ·
1 Parent(s): b44c17e

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/BoxedKernel.h +218 -0
  2. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/BoxedKernel_impl.h +111 -0
  3. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/KernelFunction.h +346 -0
  4. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/KernelFunction_impl.h +395 -0
  5. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/OperatorKernel.h +32 -0
  6. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/impl/WrapFunctionIntoFunctor.h +43 -0
  7. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/impl/WrapFunctionIntoRuntimeFunctor.h +46 -0
  8. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/impl/boxing.h +415 -0
  9. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/impl/make_boxed_from_unboxed_functor.h +790 -0
  10. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/impl/test_helpers.h +145 -0
  11. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/CppSignature.h +72 -0
  12. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/DispatchKeyExtractor.h +285 -0
  13. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/Dispatcher.h +955 -0
  14. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/ObservedOperators.h +22 -0
  15. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/OperatorEntry.h +342 -0
  16. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/OperatorOptions.h +35 -0
  17. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/RegistrationHandleRAII.h +41 -0
  18. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/op_registration/adaption.h +86 -0
  19. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/op_registration/infer_schema.h +162 -0
  20. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/op_registration/op_allowlist.h +186 -0
  21. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/op_registration/op_registration.h +599 -0
  22. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/functional.h +9 -0
  23. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/functional_base.h +480 -0
  24. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/functional_bfloat16.h +652 -0
  25. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/intrinsics.h +6 -0
  26. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/vec.h +62 -0
  27. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/vec_base.h +1537 -0
  28. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/vec_convert.h +84 -0
  29. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_log_softmax_ops.h +45 -0
  30. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lstm_mps_compositeexplicitautograd_dispatch.h +29 -0
  31. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lstm_mps_native.h +26 -0
  32. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lstm_mps_ops.h +45 -0
  33. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lu_with_info.h +36 -0
  34. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lu_with_info_compositeimplicitautograd_dispatch.h +28 -0
  35. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lu_with_info_native.h +26 -0
  36. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lu_with_info_ops.h +34 -0
  37. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dep_token.h +40 -0
  38. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dep_token_cpu_dispatch.h +29 -0
  39. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dep_token_native.h +26 -0
  40. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dep_token_ops.h +34 -0
  41. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual.h +36 -0
  42. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_compositeexplicitautograd_dispatch.h +28 -0
  43. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_copy.h +45 -0
  44. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_copy_compositeexplicitautograd_dispatch.h +29 -0
  45. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_copy_compositeexplicitautogradnonfunctional_dispatch.h +28 -0
  46. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_copy_native.h +27 -0
  47. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_copy_ops.h +45 -0
  48. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_native.h +26 -0
  49. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_ops.h +34 -0
  50. TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_per_channel_quantized_tensor.h +45 -0
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/BoxedKernel.h ADDED
@@ -0,0 +1,218 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <ATen/core/boxing/OperatorKernel.h>
5
+ #include <c10/core/DispatchKeySet.h>
6
+ #include <c10/util/intrusive_ptr.h>
7
+
8
+ namespace c10 {
9
+
10
+ struct IValue;
11
+ using Stack = std::vector<IValue>;
12
+
13
+ class OperatorHandle;
14
+ class KernelFunction;
15
+
16
+ // This kernel implements the behavior of falling through to the next available
17
+ // registered dispatch key. The implementation of this function is FAST; it is
18
+ // no overhead to fallthrough to the next key. See cpp file for some more
19
+ // implementation notes; notably, this does NOT actually go through the
20
+ // boxing/unboxing codepath.
21
+ TORCH_API void fallthrough_kernel(
22
+ OperatorKernel* /*unused*/,
23
+ const OperatorHandle& /*unused*/,
24
+ DispatchKeySet /*unused*/,
25
+ Stack* /*unused*/);
26
+
27
+ // Note [Ambiguity in AutogradOther kernel]
28
+ // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
29
+ // This error-reporting kernel is registered to the AutogradOther entry in the
30
+ // dispatch table when there is both a CompositeImplicitAutograd kernel and a
31
+ // backend kernel for ANY backend that maps to AutogradOther. To see why
32
+ // this is necessary in the AutogradOther case, it's helpful to first see
33
+ // why everything works out fine for a backend that has a reserved Autograd
34
+ // entry (see rule 2.2 in [Note] DispatchTable computation):
35
+ //
36
+ // CPU AutogradCPU
37
+ // reg? registers with...
38
+ // -------------------------------------------------
39
+ // y Autograd registration takes precedence
40
+ // over CompositeImplicitAutograd.
41
+ // This is good, because the CPU specific backend
42
+ // implementation is more specialized and typically better;
43
+ // if we used the composite, we would bypass it.
44
+ // (NB: the Autograd key is guaranteed to exist because
45
+ // the autograd codegen requires it!)
46
+ //
47
+ // n CompositeImplicitAutograd takes precedence.
48
+ // This is also good, because the Autograd
49
+ // registration (if it exists) would try to redispatch
50
+ // to the (non-existent) CPU implementation; by
51
+ // using the composite, we ensure the operator
52
+ // actually works.
53
+ //
54
+ // As you can see, when we have a specific Autograd key (AutogradCPU), we can
55
+ // decide whether or not to use the CompositeImplicitAutograd kernel or the
56
+ // Autograd kernel based on whether or not the backend kernel exists.
57
+ //
58
+ // However, for AutogradOther (which is the catchall autograd kernel for
59
+ // everything that doesn't have a specific Autograd key), we can't do this
60
+ // trick because there isn't any unique backend to peek at to disambiguate;
61
+ // if there are some backends that have implementations they prefer Autograd,
62
+ // but unimplemented backends would prefer CompositeImplicitAutograd. Rather
63
+ // than arbitrarily pick one or the other, we just register a kernel that raises
64
+ // an error and let the user decide how to proceed.
65
+ TORCH_API void ambiguous_autogradother_kernel(
66
+ OperatorKernel* /*unused*/,
67
+ const OperatorHandle& /*op*/,
68
+ DispatchKeySet /*unused*/,
69
+ Stack* /*unused*/);
70
+
71
+ // Note [named_not_supported_kernel]
72
+ // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
73
+ // This kernel implements reporting an error message saying that named tensor is
74
+ // not supported. This kernel doesn't rely on the Stack, and so it is special
75
+ // cased in the dispatcher to be triggered before we attempt boxing (so we can
76
+ // give a good error message in cases when boxing is not supported). When
77
+ // boxing is universally supported this can be removed.
78
+ [[noreturn]] TORCH_API void named_not_supported_kernel(
79
+ OperatorKernel* /*unused*/,
80
+ const OperatorHandle& /*op*/,
81
+ DispatchKeySet /*unused*/,
82
+ Stack* /*unused*/);
83
+
84
+ /**
85
+ * BoxedKernel is similar to a std::function storing a boxed kernel.
86
+ */
87
+ class TORCH_API BoxedKernel final {
88
+ public:
89
+ // This is how boxed kernels are actually stored
90
+ //
91
+ // Note [Plumbing Keys Through The Dispatcher]
92
+ // Benchmarks have shown that it is expensive for the dispatcher to read from
93
+ // thread-local storage (TLS) upon every dispatch call into order to compute
94
+ // which kernel to dispatch to.
95
+ //
96
+ // To mitigate this, we've updated the calling convention inside the
97
+ // dispatcher to expect every kernel that it stores to have a first argument
98
+ // of type DispatchKeySet.
99
+ //
100
+ // What are the invariants of the DispatchKeySet when it gets passed to a
101
+ // kernel?
102
+ // - All keys to the left of the current dispatch key have been masked out.
103
+ // (e.g. a Tracing kernel that takes in the DispatchKeySet will expect the
104
+ // highest bit to be DispatchKey::Tracer)
105
+ // - All other keys that dispatcher normally would have computed through TLS +
106
+ // global state + op arguments
107
+ // are still in the set.
108
+ //
109
+ // Kernels can then opt into using this keyset to save the dispatcher from
110
+ // doing repeated work during redispatches: recalculating the highest-priority
111
+ // dispatch key, which involves reading from TLS. Instead, the kernels that
112
+ // opt in will calculate an updated DispatchKeySet directly from the old one,
113
+ // and pass the updated set directly into the dispatcher upon redispatching.
114
+ //
115
+ // This is an opt-in mechanism: Kernels can automatically opt in by setting
116
+ // the first argument in their signature to be of type DispatchKeySet. See the
117
+ // kernels in VariableTypeEverything.cpp and TraceTypeEverything.cpp for
118
+ // examples.
119
+ //
120
+ // The mechanism for optionally passing that DispatchKeySet into the kernel
121
+ // lives in make_boxed_from_unboxed_functor.h. See Note [Plumbing Keys Through
122
+ // The Dispatcher 2] for details.
123
+ using InternalBoxedKernelFunction =
124
+ void(OperatorKernel*, const OperatorHandle&, DispatchKeySet, Stack*);
125
+ // This is the public API for how boxed kernels are defined
126
+ using BoxedKernelFunction = void(const OperatorHandle&, Stack*);
127
+ using BoxedKernelFunction_withDispatchKeys =
128
+ void(const OperatorHandle&, DispatchKeySet, Stack*);
129
+
130
+ BoxedKernel();
131
+
132
+ // Fast path for dispatch to allow not touching the boxed kernel in
133
+ // the common case where unboxed is available.
134
+ bool isValid() const;
135
+ bool isFallthrough() const;
136
+
137
+ /**
138
+ * Call the function with boxed arguments.
139
+ */
140
+ void callBoxed(
141
+ const OperatorHandle& opHandle,
142
+ DispatchKeySet dispatchKeySet,
143
+ Stack* stack) const;
144
+
145
+ /**
146
+ * Create a KernelFunction from a boxed function.
147
+ *
148
+ * Example:
149
+ *
150
+ * > void boxed_func(OperatorKernel*, Stack* stack) {...}
151
+ * > BoxedFunction func = BoxedKernel::makeFromFunction<&boxed_func>();
152
+ */
153
+ template <BoxedKernelFunction* func>
154
+ static BoxedKernel makeFromFunction();
155
+
156
+ /**
157
+ * TODO: This will only be useful if we write a backend fallback that plumbs
158
+ * dispatch keys (currently there are none) See Note [Plumbing Keys Through
159
+ * The Dispatcher] for details.
160
+ */
161
+ template <BoxedKernelFunction_withDispatchKeys* func>
162
+ static BoxedKernel makeFromFunction();
163
+
164
+ /**
165
+ * Create a KernelFunction from a boxed functor.
166
+ *
167
+ * Example:
168
+ *
169
+ * > class MyFunctor final : public c10::OperatorKernel {
170
+ * > public:
171
+ * > void operator()(const OperatorHandle&, DispatchKeySet, Stack*) {...}
172
+ * > };
173
+ * > BoxedKernel func =
174
+ * BoxedKernel::makeFromFunctor(std::make_unique<MyFunctor>());
175
+ */
176
+ template <class KernelFunctor>
177
+ static BoxedKernel makeFromFunctor(
178
+ std::unique_ptr<KernelFunctor> kernelFunctor);
179
+
180
+ static BoxedKernel makeFallthrough();
181
+ static BoxedKernel makeAmbiguousAutogradOther();
182
+ static BoxedKernel makeNamedNotSupported();
183
+
184
+ private:
185
+ friend class KernelFunction;
186
+
187
+ template <BoxedKernelFunction* func>
188
+ static void make_boxed_function(
189
+ OperatorKernel* /*unused*/,
190
+ const OperatorHandle& opHandle,
191
+ DispatchKeySet /*unused*/,
192
+ Stack* stack);
193
+
194
+ template <BoxedKernelFunction_withDispatchKeys* func>
195
+ static void make_boxed_function(
196
+ OperatorKernel* /*unused*/,
197
+ const OperatorHandle& opHandle,
198
+ DispatchKeySet /*ks*/,
199
+ Stack* stack);
200
+
201
+ explicit BoxedKernel(
202
+ std::unique_ptr<OperatorKernel> functor,
203
+ InternalBoxedKernelFunction* boxed_kernel_func);
204
+
205
+ OperatorKernel* getFunctor() const;
206
+ InternalBoxedKernelFunction* getFnPtr() const;
207
+
208
+ c10::intrusive_ptr<OperatorKernel> functor_;
209
+ InternalBoxedKernelFunction* boxed_kernel_func_;
210
+ };
211
+
212
+ } // namespace c10
213
+
214
+ #include <ATen/core/boxing/BoxedKernel_impl.h>
215
+
216
+ #else
217
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
218
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/BoxedKernel_impl.h ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ namespace c10 {
5
+
6
+ inline BoxedKernel::BoxedKernel() : boxed_kernel_func_(nullptr) {}
7
+
8
+ inline BoxedKernel::BoxedKernel(
9
+ std::unique_ptr<OperatorKernel> functor,
10
+ InternalBoxedKernelFunction* boxed_kernel_func)
11
+ : functor_(std::move(functor)), boxed_kernel_func_(boxed_kernel_func) {}
12
+
13
+ template <BoxedKernel::BoxedKernelFunction* func>
14
+ inline void BoxedKernel::make_boxed_function(
15
+ OperatorKernel* /*unused*/,
16
+ const OperatorHandle& opHandle,
17
+ DispatchKeySet /*unused*/,
18
+ Stack* stack) {
19
+ // Note that we're dropping the DispatchKeySet argument.
20
+ // See Note [Plumbing Keys Through The Dispatcher 2] for details.
21
+ func(opHandle, stack);
22
+ }
23
+
24
+ template <BoxedKernel::BoxedKernelFunction_withDispatchKeys* func>
25
+ inline void BoxedKernel::make_boxed_function(
26
+ OperatorKernel* /*unused*/,
27
+ const OperatorHandle& opHandle,
28
+ DispatchKeySet ks,
29
+ Stack* stack) {
30
+ // See Note [Plumbing Keys Through The Dispatcher 2] for details.
31
+ func(opHandle, ks, stack);
32
+ }
33
+
34
+ inline bool BoxedKernel::isValid() const {
35
+ return boxed_kernel_func_ != nullptr;
36
+ }
37
+
38
+ inline bool BoxedKernel::isFallthrough() const {
39
+ return boxed_kernel_func_ == &fallthrough_kernel;
40
+ }
41
+
42
+ inline void BoxedKernel::callBoxed(
43
+ const OperatorHandle& opHandle,
44
+ DispatchKeySet dispatchKeySet,
45
+ Stack* stack) const {
46
+ TORCH_INTERNAL_ASSERT_DEBUG_ONLY(
47
+ boxed_kernel_func_ != nullptr,
48
+ "Tried to call BoxedKernel::callBoxed() on an uninitialized BoxedKernel.");
49
+ (*boxed_kernel_func_)(functor_.get(), opHandle, dispatchKeySet, stack);
50
+ }
51
+
52
+ template <BoxedKernel::BoxedKernelFunction* func>
53
+ inline BoxedKernel BoxedKernel::makeFromFunction() {
54
+ return BoxedKernel(
55
+ nullptr, // no functor_ object
56
+ &make_boxed_function<func>);
57
+ }
58
+
59
+ template <BoxedKernel::BoxedKernelFunction_withDispatchKeys* func>
60
+ inline BoxedKernel BoxedKernel::makeFromFunction() {
61
+ return BoxedKernel(
62
+ nullptr, // no functor_ object
63
+ &make_boxed_function<func>);
64
+ }
65
+
66
+ inline BoxedKernel BoxedKernel::makeFallthrough() {
67
+ return BoxedKernel(
68
+ nullptr, // no functor_ object
69
+ &fallthrough_kernel);
70
+ }
71
+
72
+ inline BoxedKernel BoxedKernel::makeAmbiguousAutogradOther() {
73
+ return BoxedKernel(
74
+ nullptr, // no functor_ object
75
+ &ambiguous_autogradother_kernel);
76
+ }
77
+
78
+ inline BoxedKernel BoxedKernel::makeNamedNotSupported() {
79
+ return BoxedKernel(
80
+ nullptr, // no functor_ object
81
+ &named_not_supported_kernel);
82
+ }
83
+
84
+ template <class KernelFunctor>
85
+ inline BoxedKernel BoxedKernel::makeFromFunctor(
86
+ std::unique_ptr<KernelFunctor> kernelFunctor) {
87
+ static_assert(
88
+ std::is_base_of_v<OperatorKernel, KernelFunctor>,
89
+ "Tried to call BoxedKernel::makeFromFunctor<KernelFunctor>, but the functor doesn't inherit from c10::OperatorKernel. Please have the functor inherit from it.");
90
+ return BoxedKernel(
91
+ std::move(kernelFunctor),
92
+ [](OperatorKernel* kernel,
93
+ const OperatorHandle& op,
94
+ DispatchKeySet ks,
95
+ Stack* stack) {
96
+ (*static_cast<KernelFunctor*>(kernel))(op, ks, stack);
97
+ });
98
+ }
99
+
100
+ inline OperatorKernel* BoxedKernel::getFunctor() const {
101
+ return functor_.get();
102
+ }
103
+ inline BoxedKernel::InternalBoxedKernelFunction* BoxedKernel::getFnPtr() const {
104
+ return boxed_kernel_func_;
105
+ }
106
+
107
+ } // namespace c10
108
+
109
+ #else
110
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
111
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/KernelFunction.h ADDED
@@ -0,0 +1,346 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <ATen/core/ATen_fwd.h>
5
+ #include <ATen/core/boxing/BoxedKernel.h>
6
+ #include <ATen/core/stack.h>
7
+ #include <c10/core/DispatchKeySet.h>
8
+ #include <c10/util/TypeList.h>
9
+ #include <c10/util/intrusive_ptr.h>
10
+ #include <atomic>
11
+ #include <memory>
12
+ #include <type_traits>
13
+
14
+ namespace c10 {
15
+
16
+ using Stack = torch::jit::Stack; // TODO Instead of this, move torch::jit::Stack
17
+ // to the c10 namespace.
18
+
19
+ class OperatorHandle;
20
+ struct OperatorKernel;
21
+ class KernelFunction;
22
+
23
+ class KernelToken;
24
+ class SafeKernelFunction;
25
+
26
+ template <typename T>
27
+ using has_symint = std::disjunction<
28
+ std::is_same<c10::SymInt, T>,
29
+ std::is_same<c10::SymIntArrayRef, T>,
30
+ std::is_same<at::OptionalSymIntArrayRef, T>,
31
+ std::is_same<std::optional<c10::SymInt>, T>>;
32
+
33
+ template <typename T>
34
+ struct remove_symint {
35
+ using type = T;
36
+ };
37
+
38
+ template <>
39
+ struct remove_symint<c10::SymInt> {
40
+ using type = int64_t;
41
+ };
42
+
43
+ template <>
44
+ struct remove_symint<at::OptionalSymIntArrayRef> {
45
+ using type = OptionalIntArrayRef;
46
+ };
47
+
48
+ template <>
49
+ struct remove_symint<c10::SymIntArrayRef> {
50
+ using type = c10::IntArrayRef;
51
+ };
52
+
53
+ template <>
54
+ struct remove_symint<std::optional<c10::SymInt>> {
55
+ using type = std::optional<int64_t>;
56
+ };
57
+
58
+ template <bool symint, typename T>
59
+ struct maybe_keep_symint final {};
60
+
61
+ template <typename T>
62
+ struct maybe_keep_symint<true, T> {
63
+ using type = T;
64
+ };
65
+
66
+ template <typename T>
67
+ struct maybe_keep_symint<false, T> {
68
+ using type = typename remove_symint<T>::type;
69
+ };
70
+
71
+ template <typename T>
72
+ using fn_has_symint = typename guts::typelist::true_for_any_type<
73
+ has_symint,
74
+ typename guts::infer_function_traits<T>::type::parameter_types>;
75
+
76
+ template <typename T>
77
+ struct fn_remove_symint;
78
+
79
+ template <typename Ret, typename... Args>
80
+ struct fn_remove_symint<Ret(Args...)> {
81
+ using type = Ret(typename remove_symint<Args>::type...);
82
+ };
83
+
84
+ /**
85
+ * KernelFunction is similar to std::function but stores a kernel function.
86
+ * You can create a KernelFunction from a boxed or unboxed
87
+ * function/functor/lambda and call it in a boxed or unboxed way. If the way it
88
+ * was created doesn't match the way it was called, it will do boxing or
89
+ * unboxing as necessary.
90
+ */
91
+ class TORCH_API KernelFunction final {
92
+ public:
93
+ using InternalBoxedKernelFunction = BoxedKernel::InternalBoxedKernelFunction;
94
+ using BoxedKernelFunction = BoxedKernel::BoxedKernelFunction;
95
+ using BoxedKernelFunction_withDispatchKeys =
96
+ BoxedKernel::BoxedKernelFunction_withDispatchKeys;
97
+
98
+ KernelFunction();
99
+ ~KernelFunction();
100
+
101
+ KernelFunction(const KernelFunction& other);
102
+ KernelFunction& operator=(const KernelFunction& other);
103
+
104
+ KernelFunction(KernelFunction&&) noexcept = default;
105
+
106
+ // Fast path for dispatch to allow not touching the boxed kernel in
107
+ // the common case where unboxed is available.
108
+ bool isValidUnboxed() const;
109
+ bool isValidSymUnboxed() const;
110
+ bool isValid() const;
111
+ bool isFallthrough() const;
112
+
113
+ /**
114
+ * Call the function in a boxed way.
115
+ * If the kernel function was created with an unboxed function,
116
+ * this will call an unboxing wrapper which then calls into that
117
+ * unboxed function.
118
+ *
119
+ * Example:
120
+ *
121
+ * > void boxed_func(OperatorKernel*, Stack* stack) {...}
122
+ * > KernelFunction func = KernelFunction::makeFromBoxedFunction(&boxed_func);
123
+ * > Tensor result = func.callBoxed(stack);
124
+ *
125
+ * Or, with an unboxed implementation:
126
+ *
127
+ * > KernelFunction func = KernelFunction::makeFromUnboxedLambda(
128
+ * > [] (Tensor a, bool b) -> Tensor {...});
129
+ * > Tensor result = func.callBoxed(stack);
130
+ */
131
+ void callBoxed(
132
+ const OperatorHandle& opHandle,
133
+ DispatchKeySet dispatchKeySet,
134
+ Stack* stack) const;
135
+
136
+ /**
137
+ * Call the function in an unboxed way.
138
+ * If the kernel function was created with a boxed function,
139
+ * this will box all inputs and then call into that boxed function.
140
+ *
141
+ * Note that this doesn't work for all types yet.
142
+ *
143
+ * Example:
144
+ *
145
+ * > KernelFunction func = KernelFunction::makeFromUnboxedLambda(
146
+ * > [] (Tensor a, bool b) -> Tensor {...});
147
+ * > Tensor result = func.call<Tensor, Tensor, bool>(tensor1, true);
148
+ *
149
+ * Or, with a boxed implementation:
150
+ *
151
+ * > void boxed_func(OperatorKernel*, Stack* stack) {...}
152
+ * > KernelFunction func = KernelFunction::makeFromBoxedFunction(&boxed_func);
153
+ * > Tensor result = func.call<Tensor, Tensor, bool>(tensor1, true);
154
+ */
155
+ template <class Return, class... Args>
156
+ Return call(
157
+ const OperatorHandle& opHandle,
158
+ DispatchKeySet dispatchKeySet,
159
+ Args... args) const;
160
+
161
+ /**
162
+ * Create a KernelFunction from a BoxedKernel.
163
+ */
164
+ static KernelFunction makeFromBoxedKernel(BoxedKernel boxed_fn);
165
+
166
+ /**
167
+ * Create a KernelFunction from a boxed function.
168
+ *
169
+ * Example:
170
+ *
171
+ * > void boxed_func(OperatorKernel*, Stack* stack) {...}
172
+ * > KernelFunction func =
173
+ * KernelFunction::makeFromBoxedFunction<&boxed_func>();
174
+ */
175
+ template <BoxedKernelFunction* func>
176
+ static KernelFunction makeFromBoxedFunction();
177
+
178
+ /**
179
+ * TODO: This will only be useful if we write a backend fallback that plumbs
180
+ * dispatch keys (currently there are none) See Note [Plumbing Keys Through
181
+ * The Dispatcher] for details.
182
+ */
183
+ template <BoxedKernelFunction_withDispatchKeys* func>
184
+ static KernelFunction makeFromBoxedFunction();
185
+
186
+ /**
187
+ * Create a KernelFunction from an unboxed functor.
188
+ *
189
+ * Example:
190
+ *
191
+ * > class MyFunctor final : public c10::OperatorKernel {
192
+ * > public:
193
+ * > Tensor operator()(Tensor a, Tensor b) {...}
194
+ * > };
195
+ * > KernelFunction func =
196
+ * KernelFunction::makeFromUnboxedFunctor<MyFunctor>(std::make_unique<MyFunctor>());
197
+ */
198
+ template <bool AllowLegacyTypes = false, class KernelFunctor>
199
+ static KernelFunction makeFromUnboxedFunctor(
200
+ std::unique_ptr<OperatorKernel> kernelFunctor);
201
+
202
+ /**
203
+ * Create a KernelFunction from a boxed functor.
204
+ *
205
+ * Example:
206
+ *
207
+ * > class MyFunctor final : public c10::OperatorKernel {
208
+ * > public:
209
+ * > void operator()(const OperatorHandle&, DispatchKeySet, Stack*) {...}
210
+ * > };
211
+ * > KernelFunction func =
212
+ * KernelFunction::makeFromBoxedFunctor(std::make_unique<MyFunctor>());
213
+ */
214
+ template <class KernelFunctor>
215
+ static KernelFunction makeFromBoxedFunctor(
216
+ std::unique_ptr<KernelFunctor> kernelFunctor);
217
+
218
+ /**
219
+ * Create a KernelFunction from an unboxed function.
220
+ * This is usually better than KernelFunction::makeFromUnboxedRuntimeFunction
221
+ * because knowing the function pointer as a template argument (i.e. at
222
+ * compile time) allows the compiler to inline the function into its
223
+ * unboxing wrapper and yields better performance when calling the function.
224
+ *
225
+ * Example:
226
+ *
227
+ * > Tensor unboxed_func(Tensor a, Tensor b) {...}
228
+ * > KernelFunction func =
229
+ * KernelFunction::makeFromUnboxedFunction<decltype(unboxed_func),
230
+ * &unboxed_func>();
231
+ */
232
+ template <class FuncPtr, bool AllowLegacyTypes = false>
233
+ static KernelFunction makeFromUnboxedFunction(FuncPtr /*func_ptr*/);
234
+
235
+ /**
236
+ * Create a KernelFunction from an unboxed function.
237
+ * KernelFunction::makeFromUnboxedFunction is usually a better choice than
238
+ * this if you know the function pointer at compile time, see doc comment
239
+ * there for an explanation.
240
+ *
241
+ * Example:
242
+ *
243
+ * > Tensor unboxed_func(Tensor a, Tensor b) {...}
244
+ * > KernelFunction func =
245
+ * KernelFunction::makeFromUnboxedRuntimeFunction(&unboxed_func);
246
+ */
247
+ template <bool AllowLegacyTypes = false, class FuncType>
248
+ static KernelFunction makeFromUnboxedRuntimeFunction(FuncType* func);
249
+
250
+ static KernelFunction makeFallthrough();
251
+ static KernelFunction makeAmbiguousAutogradOther();
252
+ static KernelFunction makeNamedNotSupported();
253
+
254
+ /**
255
+ * Create a KernelFunction from an unboxed lambda.
256
+ *
257
+ * Example:
258
+ *
259
+ * > KernelFunction func = KernelFunction::makeFromUnboxedLambda(
260
+ * > [] (Tensor a, bool b) -> Tensor {...});
261
+ */
262
+ template <bool AllowLegacyTypes = false, class Lambda>
263
+ static std::enable_if_t<
264
+ guts::is_stateless_lambda<std::decay_t<Lambda>>::value,
265
+ KernelFunction>
266
+ makeFromUnboxedLambda(Lambda&& lambda);
267
+ template <bool AllowLegacyTypes = false, class Lambda>
268
+ static std::enable_if_t<
269
+ !guts::is_stateless_lambda<std::decay_t<Lambda>>::value,
270
+ KernelFunction>
271
+ makeFromUnboxedLambda(Lambda&& lambda);
272
+
273
+ std::string dumpState() const;
274
+ // For testing internal invariants only
275
+ bool _equalsBoxedAndUnboxed(const KernelFunction& /*other*/) const;
276
+
277
+ // Register a token to be invalidated when this KernelFunction is destroyed
278
+ void registerToken(std::weak_ptr<KernelToken> token) const;
279
+
280
+ private:
281
+ explicit KernelFunction(
282
+ std::unique_ptr<OperatorKernel> functor,
283
+ InternalBoxedKernelFunction* boxed_kernel_func,
284
+ void* unboxed_kernel_func,
285
+ void* sym_unboxed_kernel_func);
286
+ explicit KernelFunction(
287
+ BoxedKernel boxed_fn,
288
+ void* unboxed_kernel_func,
289
+ void* sym_unboxed_kernel_func);
290
+
291
+ BoxedKernel boxed_kernel_func_;
292
+ void* unboxed_kernel_func_;
293
+ void* sym_unboxed_kernel_func_;
294
+ // List of tokens that need to be invalidated when this KernelFunction is
295
+ // destroyed (lazy allocation to save memory when empty)
296
+ mutable std::unique_ptr<std::vector<std::weak_ptr<KernelToken>>> tokens_;
297
+ };
298
+
299
+ // Token held by SafeKernelFunction that gets invalidated when KernelFunction is
300
+ // destroyed
301
+ class KernelToken {
302
+ public:
303
+ bool isValid() const;
304
+ void invalidate();
305
+
306
+ private:
307
+ std::atomic<bool> invalid_{false};
308
+ };
309
+
310
+ class SafeKernelFunction {
311
+ public:
312
+ SafeKernelFunction(
313
+ const KernelFunction* kernel,
314
+ std::string debug,
315
+ std::shared_ptr<OperatorHandle> opHandle);
316
+
317
+ // Safe callBoxed - checks token validity first
318
+ void callBoxed(
319
+ const OperatorHandle& opHandle,
320
+ DispatchKeySet dispatchKeySet,
321
+ Stack* stack) const;
322
+
323
+ // Get debug information
324
+ const std::string& debug() const {
325
+ return debug_;
326
+ }
327
+
328
+ // Get the OpHandle that lives on this SafeKernelFunction
329
+ const OperatorHandle& opHandle() const {
330
+ return *opHandle_;
331
+ }
332
+
333
+ private:
334
+ KernelFunction kernel_;
335
+ std::shared_ptr<KernelToken> token_;
336
+ std::string debug_;
337
+ std::shared_ptr<OperatorHandle> opHandle_;
338
+ };
339
+
340
+ } // namespace c10
341
+
342
+ #include <ATen/core/boxing/KernelFunction_impl.h>
343
+
344
+ #else
345
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
346
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/KernelFunction_impl.h ADDED
@@ -0,0 +1,395 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #include <ATen/core/boxing/impl/WrapFunctionIntoFunctor.h>
3
+ #include <ATen/core/boxing/impl/WrapFunctionIntoRuntimeFunctor.h>
4
+ #include <ATen/core/boxing/impl/boxing.h>
5
+ #include <ATen/core/boxing/impl/make_boxed_from_unboxed_functor.h>
6
+
7
+ #include <c10/util/C++17.h>
8
+ #include <type_traits>
9
+
10
+ namespace c10 {
11
+
12
+ namespace detail {
13
+ template <typename Base, typename Child, typename... Args>
14
+ std::enable_if_t<
15
+ !std::is_array_v<Base> && !std::is_array_v<Child> &&
16
+ std::is_base_of_v<Base, Child>,
17
+ std::unique_ptr<Base>>
18
+ make_unique_base(Args&&... args) {
19
+ return std::make_unique<Child>(std::forward<Args>(args)...);
20
+ }
21
+ } // namespace detail
22
+
23
+ inline KernelFunction::KernelFunction()
24
+ : unboxed_kernel_func_(nullptr), sym_unboxed_kernel_func_(nullptr) {}
25
+
26
+ inline KernelFunction::~KernelFunction() {
27
+ if (tokens_) {
28
+ for (auto& weak_token : *tokens_) {
29
+ if (auto token = weak_token.lock()) {
30
+ token->invalidate();
31
+ }
32
+ }
33
+ }
34
+ }
35
+
36
+ inline KernelFunction::KernelFunction(const KernelFunction& other)
37
+ : boxed_kernel_func_(other.boxed_kernel_func_),
38
+ unboxed_kernel_func_(other.unboxed_kernel_func_),
39
+ sym_unboxed_kernel_func_(other.sym_unboxed_kernel_func_) {
40
+ // tokens_ is intentionally not copied as we only care about invalidating
41
+ // tokens if the original KernelFunction is destroyed
42
+ }
43
+
44
+ inline KernelFunction& KernelFunction::operator=(const KernelFunction& other) {
45
+ if (this != &other) {
46
+ boxed_kernel_func_ = other.boxed_kernel_func_;
47
+ unboxed_kernel_func_ = other.unboxed_kernel_func_;
48
+ sym_unboxed_kernel_func_ = other.sym_unboxed_kernel_func_;
49
+
50
+ // tokens_ is intentionally not copied as we only care about invalidating
51
+ // tokens if the original KernelFunction is destroyed
52
+ }
53
+ return *this;
54
+ }
55
+
56
+ inline KernelFunction::KernelFunction(
57
+ std::unique_ptr<OperatorKernel> functor,
58
+ InternalBoxedKernelFunction* boxed_kernel_func,
59
+ void* unboxed_kernel_func,
60
+ void* sym_unboxed_kernel_func = nullptr)
61
+ : boxed_kernel_func_(std::move(functor), boxed_kernel_func),
62
+ unboxed_kernel_func_(unboxed_kernel_func),
63
+ sym_unboxed_kernel_func_(sym_unboxed_kernel_func) {}
64
+
65
+ inline KernelFunction::KernelFunction(
66
+ BoxedKernel boxed_fn,
67
+ void* unboxed_kernel_func,
68
+ void* sym_unboxed_kernel_func = nullptr)
69
+ : boxed_kernel_func_(std::move(boxed_fn)),
70
+ unboxed_kernel_func_(unboxed_kernel_func),
71
+ sym_unboxed_kernel_func_(sym_unboxed_kernel_func) {}
72
+
73
+ inline bool KernelFunction::isValidUnboxed() const {
74
+ return unboxed_kernel_func_ != nullptr;
75
+ }
76
+
77
+ inline bool KernelFunction::isValidSymUnboxed() const {
78
+ return sym_unboxed_kernel_func_ != nullptr;
79
+ }
80
+
81
+ inline bool KernelFunction::isValid() const {
82
+ return boxed_kernel_func_.isValid();
83
+ }
84
+
85
+ inline bool KernelFunction::isFallthrough() const {
86
+ return boxed_kernel_func_.isFallthrough();
87
+ }
88
+
89
+ inline void KernelFunction::callBoxed(
90
+ const OperatorHandle& opHandle,
91
+ DispatchKeySet dispatchKeySet,
92
+ Stack* stack) const {
93
+ boxed_kernel_func_.callBoxed(opHandle, dispatchKeySet, stack);
94
+ }
95
+
96
+ template <class Return, class... Args>
97
+ inline Return callUnboxedKernelFunction(
98
+ void* unboxed_kernel_func,
99
+ OperatorKernel* functor,
100
+ DispatchKeySet dispatchKeySet,
101
+ Args&&... args) {
102
+ using ActualSignature = Return(OperatorKernel*, DispatchKeySet, Args...);
103
+ ActualSignature* func =
104
+ reinterpret_cast<ActualSignature*>(unboxed_kernel_func);
105
+ return (*func)(functor, dispatchKeySet, std::forward<Args>(args)...);
106
+ }
107
+
108
+ // This template requires you to explicitly specify the argument you want to
109
+ // forward; it doesn't work if you try to deduce it
110
+ // NB: keep this in sync with cloneWithRealTypes in function_schema.cpp
111
+
112
+ template <typename T>
113
+ inline typename remove_symint<T>::type unpackSymInt(T x) {
114
+ return x;
115
+ }
116
+
117
+ template <>
118
+ inline remove_symint<c10::SymInt>::type unpackSymInt(c10::SymInt x) {
119
+ return x.guard_int(__FILE__, __LINE__);
120
+ }
121
+
122
+ template <>
123
+ inline remove_symint<c10::SymIntArrayRef>::type unpackSymInt(
124
+ c10::SymIntArrayRef x) {
125
+ return C10_AS_INTARRAYREF_SLOW(x);
126
+ }
127
+
128
+ template <>
129
+ inline remove_symint<std::optional<c10::SymInt>>::type unpackSymInt(
130
+ std::optional<c10::SymInt> x) {
131
+ return x.has_value() ? std::make_optional(x->guard_int(__FILE__, __LINE__))
132
+ : std::nullopt;
133
+ }
134
+
135
+ template <>
136
+ inline remove_symint<at::OptionalSymIntArrayRef>::type unpackSymInt(
137
+ at::OptionalSymIntArrayRef x) {
138
+ return x.has_value() ? std::make_optional(C10_AS_INTARRAYREF_SLOW(*x))
139
+ : std::nullopt;
140
+ }
141
+
142
+ template <class Return, class... Args>
143
+ C10_ALWAYS_INLINE Return KernelFunction::call(
144
+ const OperatorHandle& opHandle,
145
+ DispatchKeySet dispatchKeySet,
146
+ Args... args) const {
147
+ // note: Args above is intentionally not Args&&. We don't want perfect
148
+ // forwarding, which would require Args to be deduced, but instead we
149
+ // want callers to explicitly specify the Args.
150
+
151
+ if constexpr (std::disjunction_v<has_symint<Args>...>) {
152
+ if (sym_unboxed_kernel_func_ != nullptr) {
153
+ auto* functor = boxed_kernel_func_.getFunctor();
154
+ return callUnboxedKernelFunction<Return, Args...>(
155
+ sym_unboxed_kernel_func_,
156
+ functor,
157
+ dispatchKeySet,
158
+ std::forward<Args>(args)...);
159
+ }
160
+
161
+ if (unboxed_kernel_func_ != nullptr) {
162
+ auto* functor = boxed_kernel_func_.getFunctor();
163
+ return callUnboxedKernelFunction<
164
+ Return,
165
+ typename remove_symint<Args>::type...>(
166
+ unboxed_kernel_func_,
167
+ functor,
168
+ dispatchKeySet,
169
+ unpackSymInt<Args>(args)...);
170
+ }
171
+ } else {
172
+ if (C10_LIKELY(unboxed_kernel_func_ != nullptr)) {
173
+ auto* functor = boxed_kernel_func_.getFunctor();
174
+ return callUnboxedKernelFunction<Return, Args...>(
175
+ unboxed_kernel_func_,
176
+ functor,
177
+ dispatchKeySet,
178
+ std::forward<Args>(args)...);
179
+ }
180
+ }
181
+
182
+ return impl::BoxedKernelWrapper<Return(Args...)>::call(
183
+ boxed_kernel_func_,
184
+ opHandle,
185
+ dispatchKeySet,
186
+ std::forward<Args>(args)...);
187
+ }
188
+
189
+ inline void KernelFunction::registerToken(
190
+ std::weak_ptr<KernelToken> token) const {
191
+ if (!tokens_) {
192
+ tokens_ = std::make_unique<std::vector<std::weak_ptr<KernelToken>>>();
193
+ }
194
+ tokens_->push_back(std::move(token));
195
+ }
196
+
197
+ inline KernelFunction KernelFunction::makeFromBoxedKernel(
198
+ BoxedKernel boxed_fn) {
199
+ return KernelFunction(
200
+ std::move(boxed_fn), nullptr); // no unboxed function pointer
201
+ }
202
+
203
+ template <KernelFunction::BoxedKernelFunction* func>
204
+ inline KernelFunction KernelFunction::makeFromBoxedFunction() {
205
+ return KernelFunction::makeFromBoxedKernel(
206
+ BoxedKernel::makeFromFunction<func>());
207
+ }
208
+
209
+ template <KernelFunction::BoxedKernelFunction_withDispatchKeys* func>
210
+ inline KernelFunction KernelFunction::makeFromBoxedFunction() {
211
+ return KernelFunction::makeFromBoxedKernel(
212
+ BoxedKernel::makeFromFunction<func>());
213
+ }
214
+
215
+ inline KernelFunction KernelFunction::makeFallthrough() {
216
+ return KernelFunction::makeFromBoxedKernel(BoxedKernel::makeFallthrough());
217
+ }
218
+
219
+ inline KernelFunction KernelFunction::makeAmbiguousAutogradOther() {
220
+ return KernelFunction::makeFromBoxedKernel(
221
+ BoxedKernel::makeAmbiguousAutogradOther());
222
+ }
223
+
224
+ inline KernelFunction KernelFunction::makeNamedNotSupported() {
225
+ return KernelFunction::makeFromBoxedKernel(
226
+ BoxedKernel::makeNamedNotSupported());
227
+ }
228
+
229
+ template <bool AllowLegacyTypes, class KernelFunctor>
230
+ inline KernelFunction KernelFunction::makeFromUnboxedFunctor(
231
+ std::unique_ptr<OperatorKernel> kernelFunctor) {
232
+ #ifndef NDEBUG
233
+ // This assertion is costly for build time so it's debug-gated.
234
+ static_assert(
235
+ guts::is_functor<KernelFunctor>::value,
236
+ "Tried to call KernelFunction::makeFromUnboxedFunctor<KernelFunctor> but the argument is not a functor.");
237
+ #endif
238
+ static_assert(
239
+ std::is_base_of_v<OperatorKernel, KernelFunctor>,
240
+ "Tried to call KernelFunction::makeFromUnboxedFunctor<KernelFunctor>, but the functor doesn't inherit from c10::OperatorKernel. Please have the functor inherit from it.");
241
+
242
+ auto* unboxed_fn = &impl::wrap_kernel_functor_unboxed<KernelFunctor>::call;
243
+ void* void_unboxed_fn = reinterpret_cast<void*>(unboxed_fn);
244
+ bool is_symint = fn_has_symint<decltype(unboxed_fn)>::value;
245
+ return KernelFunction(
246
+ std::move(kernelFunctor),
247
+ &impl::make_boxed_from_unboxed_functor<KernelFunctor, AllowLegacyTypes>::
248
+ call,
249
+ is_symint ? nullptr : void_unboxed_fn,
250
+ is_symint ? void_unboxed_fn : nullptr);
251
+ }
252
+
253
+ template <class KernelFunctor>
254
+ inline KernelFunction KernelFunction::makeFromBoxedFunctor(
255
+ std::unique_ptr<KernelFunctor> kernelFunctor) {
256
+ return KernelFunction::makeFromBoxedKernel(
257
+ BoxedKernel::makeFromFunctor(std::move(kernelFunctor)));
258
+ }
259
+
260
+ template <class FuncPtr, bool AllowLegacyTypes>
261
+ inline KernelFunction KernelFunction::makeFromUnboxedFunction(
262
+ FuncPtr func_ptr) {
263
+ static_assert(
264
+ is_compile_time_function_pointer<FuncPtr>::value,
265
+ "Tried to call KernelFunction::makeFromUnboxedFunction with an invalid parameter. It must be a function pointer created with TORCH_FN.");
266
+ static_assert(
267
+ !std::is_same_v<typename FuncPtr::FuncType, BoxedKernelFunction>,
268
+ "Tried to call KernelFunction::makeFromUnboxedFunction with a boxed function pointer. Please use KernelFunction::makeFromBoxedFunction instead.");
269
+ #if defined(__GNUC__) && defined(__SANITIZE_ADDRESS__) && !defined(__CUDACC__)
270
+ TORCH_INTERNAL_ASSERT(
271
+ FuncPtr::func_ptr() != nullptr, "Kernel function cannot be nullptr");
272
+ #else
273
+ static_assert(
274
+ FuncPtr::func_ptr() != nullptr, "Kernel function cannot be nullptr");
275
+ #endif
276
+
277
+ #if !defined(C10_MOBILE)
278
+ (void)func_ptr; // Suppress unused variable warning
279
+ return makeFromUnboxedFunctor<
280
+ AllowLegacyTypes,
281
+ typename impl::WrapFunctionIntoFunctor<FuncPtr>::type>(
282
+ detail::make_unique_base<
283
+ OperatorKernel,
284
+ typename impl::WrapFunctionIntoFunctor<FuncPtr>::type>());
285
+ #else
286
+ // On mobile, we rather want to optimize for binary size than for performance,
287
+ // so let's not inline the kernel into the wrapper but use
288
+ // makeFromUnboxedRuntimeFunction instead.
289
+ return makeFromUnboxedRuntimeFunction(func_ptr.func_ptr());
290
+ #endif
291
+ }
292
+
293
+ template <bool AllowLegacyTypes, class FuncType>
294
+ inline KernelFunction KernelFunction::makeFromUnboxedRuntimeFunction(
295
+ FuncType* func) {
296
+ static_assert(
297
+ guts::is_function_type<FuncType>::value,
298
+ "Tried to call KernelFunction::makeFromUnboxedRuntimeFunction with a non-function type.");
299
+ static_assert(
300
+ !std::is_same_v<FuncType, BoxedKernelFunction>,
301
+ "Tried to call KernelFunction::makeFromUnboxedRuntimeFunction with a boxed function pointer. Please use KernelFunction::makeFromBoxedFunction instead.");
302
+ TORCH_INTERNAL_ASSERT(func != nullptr, "Kernel function cannot be nullptr");
303
+
304
+ return makeFromUnboxedFunctor<
305
+ AllowLegacyTypes,
306
+ impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<FuncType>>>(
307
+ detail::make_unique_base<
308
+ OperatorKernel,
309
+ impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<FuncType>>>(func));
310
+ }
311
+
312
+ template <bool AllowLegacyTypes, class Lambda>
313
+ inline std::enable_if_t<
314
+ guts::is_stateless_lambda<std::decay_t<Lambda>>::value,
315
+ KernelFunction>
316
+ KernelFunction::makeFromUnboxedLambda(Lambda&& lambda) {
317
+ static_assert(
318
+ guts::is_functor<std::decay_t<Lambda>>::value,
319
+ "Tried to call KernelFunction::makeFromUnboxedLambda with a non-lambda type.");
320
+
321
+ #if !defined(C10_MOBILE)
322
+ return makeFromUnboxedFunctor<
323
+ AllowLegacyTypes,
324
+ impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<Lambda>>>(
325
+ detail::make_unique_base<
326
+ OperatorKernel,
327
+ impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<Lambda>>>(
328
+ std::forward<Lambda>(lambda)));
329
+ #else
330
+ // On mobile, we rather want to optimize for binary size than for performance,
331
+ // so let's not inline the kernel into the wrapper but use
332
+ // makeFromUnboxedRuntimeFunction instead.
333
+ using FuncType =
334
+ typename guts::infer_function_traits_t<std::decay_t<Lambda>>::func_type;
335
+ return makeFromUnboxedRuntimeFunction<AllowLegacyTypes, FuncType>(lambda);
336
+ #endif
337
+ }
338
+
339
+ template <bool AllowLegacyTypes, class Lambda>
340
+ inline std::enable_if_t<
341
+ !guts::is_stateless_lambda<std::decay_t<Lambda>>::value,
342
+ KernelFunction>
343
+ KernelFunction::makeFromUnboxedLambda(Lambda&& lambda) {
344
+ static_assert(
345
+ guts::is_functor<std::decay_t<Lambda>>::value,
346
+ "Tried to call KernelFunction::makeFromUnboxedLambda with a non-lambda type.");
347
+
348
+ return makeFromUnboxedFunctor<
349
+ AllowLegacyTypes,
350
+ impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<Lambda>>>(
351
+ detail::make_unique_base<
352
+ OperatorKernel,
353
+ impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<Lambda>>>(
354
+ std::forward<Lambda>(lambda)));
355
+ }
356
+
357
+ inline bool KernelToken::isValid() const {
358
+ return !invalid_.load(std::memory_order_acquire);
359
+ }
360
+
361
+ inline void KernelToken::invalidate() {
362
+ invalid_.store(true, std::memory_order_release);
363
+ }
364
+
365
+ inline SafeKernelFunction::SafeKernelFunction(
366
+ const KernelFunction* kernel,
367
+ std::string debug,
368
+ std::shared_ptr<OperatorHandle> opHandle)
369
+ : kernel_(kernel ? *kernel : KernelFunction()),
370
+ token_(std::make_shared<KernelToken>()),
371
+ debug_(std::move(debug)),
372
+ opHandle_(std::move(opHandle)) {
373
+ // Register the token with the original kernel so it gets invalidated when the
374
+ // kernel is destroyed
375
+ if (kernel) {
376
+ kernel->registerToken(token_);
377
+ }
378
+ }
379
+
380
+ inline void SafeKernelFunction::callBoxed(
381
+ const OperatorHandle& opHandle,
382
+ DispatchKeySet dispatchKeySet,
383
+ Stack* stack) const {
384
+ TORCH_CHECK(
385
+ token_ && token_->isValid(),
386
+ "SafeKernelFunction has been invalidated ",
387
+ debug_);
388
+ kernel_.callBoxed(opHandle, dispatchKeySet, stack);
389
+ }
390
+
391
+ } // namespace c10
392
+
393
+ #else
394
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
395
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/OperatorKernel.h ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+ #include <c10/util/intrusive_ptr.h>
4
+
5
+ namespace c10 {
6
+
7
+ /**
8
+ * Inherit from OperatorKernel to implement a c10 kernel.
9
+ *
10
+ * Example:
11
+ * > namespace {
12
+ * > class my_kernel_cpu final : public c10::OperatorKernel {
13
+ * > public:
14
+ * > Tensor operator()(Tensor a, Tensor b) {...}
15
+ * > };
16
+ * > }
17
+ *
18
+ * The kernel class is allowed to have members but these are equivalent
19
+ * to global variables. The kernel implementation is responsible for
20
+ * preventing race conditions on them.
21
+ *
22
+ * See below for how to register this kernel with PyTorch.
23
+ */
24
+ struct TORCH_API OperatorKernel : public c10::intrusive_ptr_target {
25
+ ~OperatorKernel() override = default;
26
+ };
27
+
28
+ } // namespace c10
29
+
30
+ #else
31
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
32
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/impl/WrapFunctionIntoFunctor.h ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <c10/core/CompileTimeFunctionPointer.h>
5
+
6
+ namespace c10::impl {
7
+ namespace detail {
8
+ template <class FuncPtr, class ReturnType, class ParameterList>
9
+ class WrapFunctionIntoFunctor_ {};
10
+ template <class FuncPtr, class ReturnType, class... Parameters>
11
+ class WrapFunctionIntoFunctor_<
12
+ FuncPtr,
13
+ ReturnType,
14
+ guts::typelist::typelist<Parameters...>>
15
+ final : public c10::OperatorKernel {
16
+ public:
17
+ C10_ALWAYS_INLINE decltype(auto) operator()(Parameters... args) {
18
+ return (*FuncPtr::func_ptr())(std::forward<Parameters>(args)...);
19
+ }
20
+ };
21
+ } // namespace detail
22
+
23
+ // WrapFunctionIntoFunctor: Wraps a compile time function pointer into a kernel
24
+ // functor. Since it is a compile time function pointer, many compilers can
25
+ // inline it into the wrapper and you don't get any performance overhead for
26
+ // wrapping.
27
+ template <class FuncPtr>
28
+ struct WrapFunctionIntoFunctor final {
29
+ static_assert(
30
+ c10::is_compile_time_function_pointer<FuncPtr>::value,
31
+ "WrapFunctionIntoFunctor can only wrap functions created with TORCH_FN.");
32
+ using type = detail::WrapFunctionIntoFunctor_<
33
+ FuncPtr,
34
+ typename guts::function_traits<typename FuncPtr::FuncType>::return_type,
35
+ typename guts::function_traits<
36
+ typename FuncPtr::FuncType>::parameter_types>;
37
+ };
38
+
39
+ } // namespace c10::impl
40
+
41
+ #else
42
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
43
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/impl/WrapFunctionIntoRuntimeFunctor.h ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <c10/util/TypeTraits.h>
5
+
6
+ namespace c10::impl {
7
+
8
+ namespace detail {
9
+ template <class FuncType, class ReturnType, class ParameterList>
10
+ class WrapFunctionIntoRuntimeFunctor_ {};
11
+ template <class FuncType, class ReturnType, class... Parameters>
12
+ class WrapFunctionIntoRuntimeFunctor_<
13
+ FuncType,
14
+ ReturnType,
15
+ guts::typelist::typelist<Parameters...>>
16
+ final : public c10::OperatorKernel {
17
+ public:
18
+ template <class FuncType_>
19
+ explicit WrapFunctionIntoRuntimeFunctor_(FuncType_&& kernel_func)
20
+ : kernel_func_(std::forward<FuncType_>(kernel_func)) {}
21
+
22
+ decltype(auto) operator()(Parameters... args) {
23
+ return kernel_func_(std::forward<Parameters>(args)...);
24
+ }
25
+
26
+ private:
27
+ FuncType kernel_func_;
28
+ };
29
+ } // namespace detail
30
+
31
+ // WrapFunctionIntoRuntimeFunctor: Wraps any runtime functor into a functor that
32
+ // inherits from c10::OperatorKernel, so it can be used as a c10 kernel.
33
+ // This can, for example, be used for lambdas, functors or even function
34
+ // pointers. In the case of function pointers, since it is a runtime function
35
+ // pointer, there is an overhead for calling it whenever the kernel is invoked.
36
+ template <class FuncType>
37
+ using WrapFunctionIntoRuntimeFunctor = detail::WrapFunctionIntoRuntimeFunctor_<
38
+ FuncType,
39
+ typename guts::infer_function_traits_t<FuncType>::return_type,
40
+ typename guts::infer_function_traits_t<FuncType>::parameter_types>;
41
+
42
+ } // namespace c10::impl
43
+
44
+ #else
45
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
46
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/impl/boxing.h ADDED
@@ -0,0 +1,415 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // This file contains boxing (not unboxing) logic,
5
+ // i.e. how to make a vector<IValue> from a set of concrete arguments.
6
+
7
+ #include <ATen/core/ivalue.h>
8
+ #include <ATen/core/stack.h>
9
+ #include <c10/core/TensorOptions.h>
10
+
11
+ #include <ATen/core/boxing/BoxedKernel.h>
12
+
13
+ #include <c10/util/Metaprogramming.h>
14
+ #include <type_traits>
15
+
16
+ namespace c10::impl {
17
+
18
+ //
19
+ // utils
20
+ //
21
+
22
+ // is_mutable_tensor_ref
23
+ template <class T>
24
+ struct is_mutable_tensor_ref : std::false_type {};
25
+ template <>
26
+ struct is_mutable_tensor_ref<at::Tensor&> : std::true_type {};
27
+
28
+ // is_tuple_of_mutable_tensor_refs
29
+ //
30
+ template <class T, class Enable = void>
31
+ struct is_tuple_of_mutable_tensor_refs : std::false_type {};
32
+
33
+ template <class T>
34
+ struct is_tuple_of_mutable_tensor_refs<
35
+ T,
36
+ std::enable_if_t<guts::is_instantiation_of<std::tuple, T>::value, void>>
37
+ : guts::typelist::
38
+ all<is_mutable_tensor_ref, guts::typelist::from_tuple_t<T>> {};
39
+
40
+ // has_ivalue_to<T> tests the presence/absence of instance method
41
+ // IValue::to<T>()
42
+ //
43
+ template <class T, class Enable = void>
44
+ struct has_ivalue_to : std::false_type {};
45
+
46
+ template <class T>
47
+ struct ivalue_to_helper {
48
+ using type = decltype(std::declval<IValue>().template to<T>());
49
+ };
50
+ template <class T>
51
+ using ivalue_to_helper_t = typename ivalue_to_helper<T>::type;
52
+
53
+ template <class T>
54
+ struct has_ivalue_to<T, std::void_t<ivalue_to_helper_t<T>>> : std::true_type {};
55
+
56
+ //
57
+ // boxing predicates
58
+ //
59
+
60
+ // A boxable arg type is one that IValue has a constructor for.
61
+ template <typename T>
62
+ using can_box = std::disjunction<
63
+ std::is_constructible<IValue, std::decay_t<T>>,
64
+ // TensorOptions are not directly constructible into IValue,
65
+ // but torch::jit::push knows how to handle them
66
+ std::is_same<TensorOptions, std::decay_t<T>>>;
67
+
68
+ template <typename... Ts>
69
+ using can_box_all = std::conjunction<can_box<Ts>...>;
70
+
71
+ // an unboxable result is one that can be extracted from an IValue
72
+ template <typename T>
73
+ using can_unbox = std::conjunction<
74
+ std::disjunction<
75
+ has_ivalue_to<T>,
76
+ // void returns are ok
77
+ std::is_same<void, T>>,
78
+ std::negation<std::is_lvalue_reference<T>>>;
79
+
80
+ //
81
+ // boxArgs - utility for pushing unboxed args onto IValue stack
82
+ //
83
+ template <class... Args>
84
+ torch::jit::Stack boxArgs(Args... args) {
85
+ // TODO Reuse stack vector instead of allocating?
86
+ torch::jit::Stack stack;
87
+ stack.reserve(sizeof...(Args));
88
+ torch::jit::push(stack, std::forward<Args>(args)...);
89
+ return stack;
90
+ }
91
+
92
+ template <class T>
93
+ inline constexpr size_t boxed_size_one() {
94
+ static_assert(
95
+ !std::is_same_v<std::decay_t<T>, c10::TensorOptions>,
96
+ "need to patch this path to support TensorOptions passed by reference");
97
+ return 1;
98
+ }
99
+
100
+ // torch::jit::push pushes 4 values for a TensorOptions; this needs to
101
+ // be kept in sync.
102
+ template <>
103
+ inline constexpr size_t boxed_size_one<c10::TensorOptions>() {
104
+ return 4;
105
+ }
106
+
107
+ // NOTE: this could probably be simplified with C++17 fold expressions.
108
+ template <typename...>
109
+ struct BoxedSize : std::integral_constant<size_t, 0> {};
110
+ template <class T, class... Args>
111
+ struct BoxedSize<T, Args...>
112
+ : std::integral_constant<
113
+ size_t,
114
+ boxed_size_one<T>() + BoxedSize<Args...>::value> {};
115
+
116
+ template <class... Args>
117
+ static inline constexpr size_t boxed_size() {
118
+ return BoxedSize<Args...>::value;
119
+ }
120
+
121
+ template <typename T>
122
+ C10_ALWAYS_INLINE_UNLESS_MOBILE void boxToStack(IValue*& dest, T& arg) {
123
+ new (dest++) IValue(arg);
124
+ }
125
+
126
+ C10_ALWAYS_INLINE_UNLESS_MOBILE void boxToStack(
127
+ IValue*& dest,
128
+ c10::TensorOptions options) {
129
+ new (dest++) IValue(c10::typeMetaToScalarType(options.dtype()));
130
+ new (dest++) IValue(options.layout());
131
+ new (dest++) IValue(options.device());
132
+ new (dest++) IValue(options.pinned_memory());
133
+ }
134
+
135
+ inline void boxArgsToStack(IValue*& /*unused*/) {}
136
+
137
+ template <typename T, typename... Args>
138
+ C10_ALWAYS_INLINE_UNLESS_MOBILE void boxArgsToStack(
139
+ IValue*& dest,
140
+ T& arg,
141
+ Args&... args) {
142
+ boxToStack(dest, arg);
143
+ boxArgsToStack(dest, args...);
144
+ }
145
+
146
+ //
147
+ // PopResult is a helper class whose specializations handle popping single and
148
+ // multiple return values, respectively.
149
+ //
150
+ template <class Result>
151
+ struct PopResult final {
152
+ static Result call(Stack& stack) {
153
+ TORCH_INTERNAL_ASSERT_DEBUG_ONLY(
154
+ stack.size() == 1,
155
+ "Boxed kernel was expected to return one value on the stack, ",
156
+ "but instead pushed ",
157
+ stack.size(),
158
+ " values.");
159
+ return std::move(stack[0]).to<Result>();
160
+ }
161
+ };
162
+
163
+ template <class... Types>
164
+ struct PopResult<std::tuple<Types...>> final {
165
+ using Result = std::tuple<Types...>;
166
+
167
+ static Result call(Stack& stack) {
168
+ // for tuple return types, boxed kernel has pushed multiple values onto the
169
+ // stack
170
+ constexpr int RetCount = sizeof...(Types);
171
+ TORCH_INTERNAL_ASSERT_DEBUG_ONLY(
172
+ stack.size() == RetCount,
173
+ "Boxed kernel was expected to return ",
174
+ RetCount,
175
+ " values on the stack, ",
176
+ "but instead pushed ",
177
+ stack.size(),
178
+ " values.");
179
+ return pop_to_tuple_impl(stack, std::make_index_sequence<RetCount>());
180
+ }
181
+
182
+ private:
183
+ // note: this has been moved into its own helper only to avoid a parse error
184
+ // on `indices` otherwise. I'm sure there's an incantation that slips it past
185
+ // the parser but eh
186
+ template <size_t... indices>
187
+ static Result pop_to_tuple_impl(
188
+ Stack& stack,
189
+ std::index_sequence<indices...> /*unused*/) {
190
+ return std::make_tuple((std::move(stack[indices]).template to<Types>())...);
191
+ }
192
+ };
193
+
194
+ //
195
+ // BoxedKernelWrapper
196
+ //
197
+ // For a given function type FT, BoxedKernelWrapper<FT> implements
198
+ // a `call` method that
199
+ // - takes a boxed kernel and unboxed arguments as specified by FT,
200
+ // - calls `boxArgs` to box the arguments
201
+ // - calls the boxed kernel
202
+ // - unboxes and returns the result
203
+ //
204
+ // The partial specializations below handle various cases: in
205
+ // particular, not all types appearing in op signatures are supported,
206
+ // and ops returning references have nonstandard wrapper implementations.
207
+ //
208
+
209
+ // 1. The base specialization of BoxedKernelWrapper should never be
210
+ // instantiated. A "no call method defined on BoxedKernelWrapper" compile error
211
+ // means that an op signature has failed to trigger any of the partial
212
+ // specializations that follow this one.
213
+ //
214
+ template <class FuncType, class Enable = void>
215
+ struct BoxedKernelWrapper {
216
+ // The reason we're not just doing straight up static_assert(false, ...) here:
217
+ // Basically, the way to make sure a static_assert only fires if a template
218
+ // is actually instantiated (rather than every time the file is parsed) is to
219
+ // use template parameters in the expression, e.g. FuncType here. However,
220
+ // since `sizeof(FuncType) != sizeof(FuncType)` is always false, this has the
221
+ // same effect.
222
+ static_assert(
223
+ sizeof(FuncType) != sizeof(FuncType),
224
+ "Function signature contains one or more unsupported parameter and/or return types. "
225
+ "Look for a nearby error like "
226
+ "\"'call' is not a member of 'c10::impl::BoxedKernelWrapper<(your function type), void>'\" "
227
+ "- (your function type) is the unsupported signature.");
228
+ };
229
+
230
+ //
231
+ // 2. Supported signatures, other than those involving non-const Tensor refs -
232
+ // i.e., "functional" ops.
233
+ //
234
+
235
+ template <class Result, class... Args>
236
+ struct BoxedKernelWrapper<
237
+ Result(Args...),
238
+ std::enable_if_t<
239
+ can_box_all<Args...>::value && can_unbox<Result>::value &&
240
+ !is_tuple_of_mutable_tensor_refs<Result>::value,
241
+ void>> {
242
+ static Result call(
243
+ const BoxedKernel& boxed_kernel_func,
244
+ const OperatorHandle& opHandle,
245
+ DispatchKeySet dispatchKeySet,
246
+ Args... args) {
247
+ torch::jit::Stack stack = boxArgs<Args...>(std::forward<Args>(args)...);
248
+ boxed_kernel_func.callBoxed(opHandle, dispatchKeySet, &stack);
249
+
250
+ if constexpr (!std::is_same_v<void, Result>) {
251
+ // op has pushed one or more values onto the stack.
252
+ return PopResult<Result>::call(stack);
253
+ } else {
254
+ // op returns void, boxed kernel has pushed nothing onto stack.
255
+ TORCH_INTERNAL_ASSERT_DEBUG_ONLY(
256
+ stack.empty(),
257
+ "Boxed kernel was expected to return no values on the stack, ",
258
+ "but instead returned ",
259
+ stack.size(),
260
+ " values.");
261
+ }
262
+ }
263
+ };
264
+
265
+ //
266
+ // 3. in-place ops take a single non-const Tensor reference
267
+ // as their first argument, and return it.
268
+ //
269
+ // Note: all signatures matching this pattern are assumed to be for such ops.
270
+ // Because of this, the generated BoxedKernelWrapper specializations simply
271
+ // return the in-place argument.
272
+ //
273
+
274
+ template <class... OtherArgs>
275
+ struct BoxedKernelWrapper<
276
+ at::Tensor&(at::Tensor&, OtherArgs...),
277
+ std::enable_if_t<can_box_all<OtherArgs...>::value, void>> {
278
+ static at::Tensor& call(
279
+ const BoxedKernel& boxed_kernel_func,
280
+ const OperatorHandle& opHandle,
281
+ DispatchKeySet dispatchKeySet,
282
+ at::Tensor& outArg,
283
+ OtherArgs... otherArgs) {
284
+ torch::jit::Stack stack = boxArgs<at::Tensor&, OtherArgs...>(
285
+ outArg, std::forward<OtherArgs>(otherArgs)...);
286
+ boxed_kernel_func.callBoxed(opHandle, dispatchKeySet, &stack);
287
+ TORCH_INTERNAL_ASSERT_DEBUG_ONLY(
288
+ stack.size() == 1,
289
+ "Boxed kernel was expected to return a single value on the stack, ",
290
+ "but instead returned ",
291
+ stack.size(),
292
+ " values.");
293
+
294
+ return outArg;
295
+ }
296
+ };
297
+
298
+ //
299
+ // 3.5. In-process migration to make in-place ops take and return
300
+ // const references instead.
301
+ template <class... OtherArgs>
302
+ struct BoxedKernelWrapper<
303
+ const at::Tensor&(const at::Tensor&, OtherArgs...),
304
+ std::enable_if_t<can_box_all<OtherArgs...>::value, void>> {
305
+ static const at::Tensor& call(
306
+ const BoxedKernel& boxed_kernel_func,
307
+ const OperatorHandle& opHandle,
308
+ DispatchKeySet dispatchKeySet,
309
+ const at::Tensor& outArg,
310
+ OtherArgs... otherArgs) {
311
+ torch::jit::Stack stack = boxArgs(outArg, otherArgs...);
312
+ boxed_kernel_func.callBoxed(opHandle, dispatchKeySet, &stack);
313
+ TORCH_INTERNAL_ASSERT_DEBUG_ONLY(
314
+ stack.size() == 1,
315
+ "Boxed kernel was expected to return a single value on the stack, ",
316
+ "but instead returned ",
317
+ stack.size(),
318
+ " values.");
319
+
320
+ return outArg;
321
+ }
322
+ };
323
+
324
+ //
325
+ // 4. out of place ops that take a single non-const Tensor reference as their
326
+ // final argument, and also return it.
327
+ //
328
+ // Note: all signatures matching this pattern are assumed to be for such ops.
329
+ // This assumption permits the generated BoxedKernelWrapper specializations to
330
+ // simply return out arguments.
331
+ //
332
+ template <class FirstArg, class... RestArgs>
333
+ struct BoxedKernelWrapper<
334
+ at::Tensor&(FirstArg, RestArgs...),
335
+ std::enable_if_t<
336
+ can_box_all<FirstArg, RestArgs...>::value
337
+ // this skips over in-place kernels with a non-const Tensor
338
+ // arg at the front, so those can unambiguously trigger the
339
+ // preceding specialization.
340
+ && !is_mutable_tensor_ref<FirstArg>::value,
341
+ void>> {
342
+ static at::Tensor& call(
343
+ const BoxedKernel& boxed_kernel_func,
344
+ const OperatorHandle& opHandle,
345
+ DispatchKeySet dispatchKeySet,
346
+ FirstArg firstArg,
347
+ RestArgs... restArgs) {
348
+ torch::jit::Stack stack = boxArgs<FirstArg, RestArgs...>(
349
+ std::forward<FirstArg>(firstArg), std::forward<RestArgs>(restArgs)...);
350
+ boxed_kernel_func.callBoxed(opHandle, dispatchKeySet, &stack);
351
+ TORCH_INTERNAL_ASSERT_DEBUG_ONLY(
352
+ stack.size() == 1,
353
+ "Boxed kernel was expected to return a single value on the stack, ",
354
+ "but instead returned ",
355
+ stack.size(),
356
+ " values.");
357
+
358
+ // reusing restArgs after it has been forwarded here is ok because we know
359
+ // that the last element is of type `Tensor&`.
360
+ return std::get<sizeof...(RestArgs) - 1>(
361
+ std::tuple<RestArgs...>{restArgs...});
362
+ }
363
+ };
364
+
365
+ //
366
+ // 5. out of place ops that take multiple non-const Tensor references as their
367
+ // final arguments, and return them in a std::tuple.
368
+ //
369
+ // Note: all signatures matching this pattern are assumed to be for such ops.
370
+ // This assumption permits the generated BoxedKernelWrapper specializations to
371
+ // simply return the out arguments.
372
+ //
373
+ template <class Result, class... Args>
374
+ struct BoxedKernelWrapper<
375
+ Result(Args...),
376
+ std::enable_if_t<
377
+ can_box_all<Args...>::value &&
378
+ is_tuple_of_mutable_tensor_refs<Result>::value,
379
+ void>> {
380
+ static Result call(
381
+ const BoxedKernel& boxed_kernel_func,
382
+ const OperatorHandle& opHandle,
383
+ DispatchKeySet dispatchKeySet,
384
+ Args... args) {
385
+ using ArgTuple = std::tuple<Args...>;
386
+ constexpr int RetCount = std::tuple_size<Result>();
387
+
388
+ torch::jit::Stack stack = boxArgs<Args...>(std::forward<Args>(args)...);
389
+ boxed_kernel_func.callBoxed(opHandle, dispatchKeySet, &stack);
390
+ TORCH_INTERNAL_ASSERT_DEBUG_ONLY(
391
+ stack.size() == RetCount,
392
+ "Boxed kernel was expected to return ",
393
+ RetCount,
394
+ " values on the stack, ",
395
+ "but instead returned ",
396
+ stack.size(),
397
+ " values.");
398
+
399
+ // reusing args after it has been forwarded here is ok because we know
400
+ // that the last RetCount elements are of type `Tensor&`.
401
+ auto result = guts::tuple_take<ArgTuple, -RetCount>(
402
+ ArgTuple{std::forward<Args>(args)...});
403
+ static_assert(
404
+ std::is_same_v<Result, decltype(result)>,
405
+ "The parameter list of an op returning a tuple of Tensor references "
406
+ "must end with an equal number of Tensor reference parameters.");
407
+ return result;
408
+ }
409
+ };
410
+
411
+ } // namespace c10::impl
412
+
413
+ #else
414
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
415
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/impl/make_boxed_from_unboxed_functor.h ADDED
@@ -0,0 +1,790 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <ATen/core/IListRef.h>
5
+ #include <ATen/core/boxing/OperatorKernel.h>
6
+ #include <ATen/core/ivalue.h>
7
+ #include <ATen/core/stack.h>
8
+ #include <c10/util/Metaprogramming.h>
9
+ #include <c10/util/TypeList.h>
10
+ #include <c10/util/intrusive_ptr.h>
11
+
12
+ #include <utility>
13
+
14
+ namespace c10 {
15
+
16
+ using Stack = torch::jit::Stack; // TODO Instead of this, move torch::jit::Stack
17
+ // to the c10 namespace.
18
+ class OperatorHandle;
19
+
20
+ /*
21
+ * [Note: Argument forwarding in the dispatcher]
22
+ *
23
+ * The dispatcher uses a somewhat unusual way to forward arguments through
24
+ * several layers of wrapper functions. This can be confusing because an
25
+ * experienced C++ programmer would look at this and think "oh this is supposed
26
+ * to be forwarding a universal reference but the && is missing. This is a
27
+ * bug.". It is not a bug. The common way in C++ to forward arguments is to use
28
+ * universal references:
29
+ *
30
+ * > template<class T> void func(T&& arg) { func2(std::forward<T>(arg)); }
31
+ *
32
+ * but that relies on inferring the correct reference type (i.e. value vs & vs
33
+ * &&) from the argument. In our case, we cannot rely on the argument as
34
+ * supplied by the caller, because that could infer a different reference type
35
+ * than was used in the kernel function. The correct reference type is dictated
36
+ * by the kernel signature and must be identical since we cast function pointers
37
+ * through void* pointers and mismatches would be UB. So we need a forwarding
38
+ * pattern that determines the reference type to use by looking at the
39
+ * explicitly supplied operator signature, not by looking at the argument we're
40
+ * calling it with.
41
+ *
42
+ * What does std::forward do, exactly?
43
+ * ------------------------------------
44
+ * std::forward<T>(t) is a way to cast t to the reference type supplied in T.
45
+ * Let's assume decay_t<T> == U and T is either U or some reference of U.
46
+ * - std::forward<T&>(t) will return U&, no matter what kind of reference t is.
47
+ * - std::forward<T&&>(t) will return U&&, no matter what kind of reference t
48
+ * is.
49
+ * - std::forward<T>(t) will return U&& (not U!), no matter what kind of
50
+ * reference t is.
51
+ *
52
+ * For universal references, that means that in the following function
53
+ * > template<class T> void func(T&& arg) { func2(std::forward<T>(arg)); }
54
+ *
55
+ * - when called with arg being a rvalue reference or non-reference value, T
56
+ * gets inferred to be a non-reference U, and std::forward<T>(t) will return
57
+ * U&&, correctly moving the argument.
58
+ * - when called with arg behind a lvalue reference, T gets inferred to be U&
59
+ * because that's the only way to match the signature (in C++, a type that is
60
+ * (T&)&& will collapse to T&). That means std::forward<T>(t) will return U& and
61
+ * the value will not be moved but passed on as a lvalue reference.
62
+ *
63
+ * How do we use that?
64
+ * ------------------------------------
65
+ * But std::forward can also be used outside of the common "universal
66
+ * forwarding" pattern to change reference types. So instead of following the
67
+ * common C++ pattern, we notice what std::forward<T>() actually does, and that
68
+ * is it takes a value and changes its reference to the type of reference passed
69
+ * in as T. If we don't infer T but explicitly specify it, we can use this to
70
+ * forward based on an explicitly specified reference type instead of the
71
+ * inferred argument type.
72
+ *
73
+ * This is why many of the dispatcher functions look like
74
+ * > template<class T> func(T t) { func2<T>(std::forward<T>(t)); }
75
+ * instead of the common
76
+ * > template<class T> func(T&& t) { func2(std::forward<T>(t)); }
77
+ *
78
+ * and are expected to be called by explicitly specifying the template
79
+ * parameters in a way that matches the expected operator signature at each call
80
+ * site.
81
+ */
82
+
83
+ namespace impl {
84
+ // supported_primitive_arg_types defines which primitive types we allow in
85
+ // kernel functions as arguments or returns.
86
+ // Additionally, we support lists, dicts and optionals containing these types.
87
+ using supported_primitive_arg_types = guts::typelist::typelist<
88
+ int64_t,
89
+ double,
90
+ bool,
91
+ std::string_view,
92
+ at::Tensor,
93
+ at::Scalar,
94
+ c10::QScheme,
95
+ c10::ScalarType,
96
+ c10::Device,
97
+ c10::DeviceIndex,
98
+ c10::Layout,
99
+ c10::MemoryFormat,
100
+ at::Dimname>;
101
+
102
+ // We have an unboxed functor in hand that takes C++ arguments, and
103
+ // we're building a boxed functor wrapper for it that takes IValues.
104
+ // So "outside" is boxed and "inside" is unboxed.
105
+ //
106
+ // So a valid input type is one that our boxed functor wrapper can
107
+ // unbox from an IValue into a C++ value.
108
+ //
109
+ // Whereas a valid output type is one that our wrapper can receive
110
+ // as a C++ value from the unboxed functor, and box into an IValue.
111
+
112
+ //
113
+ // assert_is_valid_input_type
114
+ // checks that T can be unboxed from an IValue into a C++ value.
115
+ //
116
+
117
+ template <class T, bool AllowDeprecatedTypes, class Enable = void>
118
+ struct assert_is_valid_input_type {
119
+ assert_is_valid_input_type() {
120
+ if constexpr (guts::typelist::contains<supported_primitive_arg_types, T>::
121
+ value) {
122
+ /* everything is ok, this is a primitive type */
123
+ } else {
124
+ /* otherwise this must be an instance of a valid custom class, since it
125
+ can only have been created via IValue(x), which ensures this. */
126
+ }
127
+ }
128
+ };
129
+
130
+ template <class T, bool AllowDeprecatedTypes>
131
+ struct assert_is_valid_input_type<std::optional<T>, AllowDeprecatedTypes>
132
+ : assert_is_valid_input_type<T, AllowDeprecatedTypes> {};
133
+
134
+ template <bool AllowDeprecatedTypes, class... Args>
135
+ struct TypeCheckHelper;
136
+
137
+ template <bool AllowDeprecatedTypes>
138
+ struct TypeCheckHelper<AllowDeprecatedTypes> {};
139
+
140
+ template <bool AllowDeprecatedTypes, class Head, class... Rest>
141
+ struct TypeCheckHelper<AllowDeprecatedTypes, Head, Rest...>
142
+ : TypeCheckHelper<AllowDeprecatedTypes, Rest...> {
143
+ assert_is_valid_input_type<Head, AllowDeprecatedTypes> check;
144
+ };
145
+
146
+ template <class... Contained, bool AllowDeprecatedTypes>
147
+ struct assert_is_valid_input_type<
148
+ std::tuple<Contained...>,
149
+ AllowDeprecatedTypes>
150
+ : TypeCheckHelper<AllowDeprecatedTypes, Contained...> {};
151
+
152
+ template <class Key, class Value, bool AllowDeprecatedTypes>
153
+ struct assert_is_valid_input_type<Dict<Key, Value>, AllowDeprecatedTypes>
154
+ : assert_is_valid_input_type<Value, AllowDeprecatedTypes> {
155
+ static_assert(
156
+ guts::typelist::contains<impl::valid_dict_key_types, Key>::value,
157
+ "You tried to register a kernel with an unsupported input type: Dict<Key, Value> where Key is invalid. We only support int64_t, double, bool, and string.");
158
+ };
159
+
160
+ template <class Key, class Value, bool AllowDeprecatedTypes>
161
+ struct assert_is_valid_input_type<
162
+ std::unordered_map<Key, Value>,
163
+ AllowDeprecatedTypes>
164
+ : assert_is_valid_input_type<Value, AllowDeprecatedTypes> {
165
+ static_assert(
166
+ AllowDeprecatedTypes,
167
+ "You tried to register a kernel with an unsupported input type: std::unordered_map<Key, Value>. Please use Dict<Key, Value> instead.");
168
+ static_assert(
169
+ guts::typelist::contains<impl::valid_dict_key_types, Key>::value,
170
+ "You tried to register a kernel with an unsupported input type: std::unordered_map<Key, Value> where Key is invalid. We only support int64_t, double, bool, and string.");
171
+ };
172
+
173
+ template <class T, bool AllowDeprecatedTypes>
174
+ struct assert_is_valid_input_type<List<T>, AllowDeprecatedTypes>
175
+ : assert_is_valid_input_type<T, AllowDeprecatedTypes> {
176
+ static_assert(
177
+ !std::is_same_v<T, at::Scalar>,
178
+ "You tried to register a kernel with an unsupported input type: List<Scalar>. Please use List<int64_t>, List<double> or Tensor instead.");
179
+ };
180
+
181
+ template <class T, bool AllowDeprecatedTypes>
182
+ struct assert_is_valid_input_type<c10::ArrayRef<T>, AllowDeprecatedTypes>
183
+ : assert_is_valid_input_type<T, AllowDeprecatedTypes> {
184
+ static_assert(
185
+ !std::is_same_v<T, at::Scalar>,
186
+ "You tried to register a kernel with an unsupported input type: ArrayRef<Scalar>. Please use List<int64_t>, List<double> or Tensor instead.");
187
+ };
188
+
189
+ template <class T, bool AllowDeprecatedTypes>
190
+ struct assert_is_valid_input_type<
191
+ c10::OptionalArrayRef<T>,
192
+ AllowDeprecatedTypes>
193
+ : assert_is_valid_input_type<T, AllowDeprecatedTypes> {
194
+ static_assert(
195
+ !std::is_same_v<T, at::Scalar>,
196
+ "You tried to register a kernel with an unsupported input type: OptionalArrayRef<Scalar>. Please use List<int64_t>, List<double> or Tensor instead.");
197
+ };
198
+
199
+ template <class T, size_t N, bool AllowDeprecatedTypes>
200
+ struct assert_is_valid_input_type<std::array<T, N>, AllowDeprecatedTypes>
201
+ : assert_is_valid_input_type<T, AllowDeprecatedTypes> {
202
+ static_assert(
203
+ !std::is_same_v<T, at::Scalar>,
204
+ "You tried to register a kernel with an unsupported input type: std::array<Scalar, N>. Please use std::array<int64_t, N> instead.");
205
+ };
206
+
207
+ template <class T, bool AllowDeprecatedTypes>
208
+ struct assert_is_valid_input_type<
209
+ T,
210
+ AllowDeprecatedTypes,
211
+ std::enable_if_t<std::is_same_v<float, T>>> {
212
+ // There is no reason to support float when we have double. Keep the API lean.
213
+ static_assert(
214
+ guts::false_t<T>::value,
215
+ "You tried to register a kernel with an unsupported input type: float. Please use double instead; you should use `double` in the C++ function signature and `float` in the schema string.");
216
+ };
217
+ template <class T, bool AllowDeprecatedTypes>
218
+ struct assert_is_valid_input_type<
219
+ T,
220
+ AllowDeprecatedTypes,
221
+ std::enable_if_t<std::is_same_v<const char*, T>>> {
222
+ static_assert(
223
+ guts::false_t<T>::value,
224
+ "You tried to register a kernel with an unsupported input type: const char*. Please use std::string_view instead.");
225
+ };
226
+ template <class T, bool AllowDeprecatedTypes>
227
+ struct assert_is_valid_input_type<
228
+ T,
229
+ AllowDeprecatedTypes,
230
+ std::enable_if_t<std::is_same_v<std::vector<bool>, T>>> {
231
+ static_assert(
232
+ guts::false_t<T>::value,
233
+ "You tried to register a kernel with an unsupported input type: vector<bool>. Please use List<bool> instead.");
234
+ };
235
+ template <class T, bool AllowDeprecatedTypes>
236
+ struct assert_is_valid_input_type<
237
+ T,
238
+ AllowDeprecatedTypes,
239
+ std::enable_if_t<
240
+ std::is_integral_v<T> &&
241
+ !guts::typelist::contains<supported_primitive_arg_types, T>::value>> {
242
+ static_assert(
243
+ guts::false_t<T>::value,
244
+ "You tried to register a kernel with an unsupported integral input type. Please use int64_t instead; you should use `int64_t` in the C++ function signature and `int` in the schema string.");
245
+ };
246
+ template <class T, bool AllowDeprecatedTypes>
247
+ struct assert_is_valid_input_type<
248
+ T,
249
+ AllowDeprecatedTypes,
250
+ std::enable_if_t<std::is_same_v<const c10::SymInt&, T>>> {
251
+ static_assert(
252
+ guts::false_t<T>::value,
253
+ "You tried to register a kernel taking c10::SymInt by reference. Please accept it by value instead.");
254
+ };
255
+
256
+ // TODO: it probably would be good to tighten this up quite a bit more with
257
+ // an explicit list for everything
258
+
259
+ //
260
+ // assert_is_valid_output_type
261
+ //
262
+
263
+ template <class T, bool AllowDeprecatedTypes, class Enable = void>
264
+ struct assert_is_valid_output_type {
265
+ assert_is_valid_output_type() {
266
+ if constexpr (guts::typelist::contains<supported_primitive_arg_types, T>::
267
+ value) {
268
+ /* everything is ok, this is a primitive type */
269
+ } else {
270
+ /* otherwise T is verified to be a registered custom class in the IValue
271
+ constructor, so no benefit in double-checking here */
272
+ }
273
+ }
274
+ };
275
+
276
+ template <class T, bool AllowDeprecatedTypes>
277
+ struct assert_is_valid_output_type<std::optional<T>, AllowDeprecatedTypes>
278
+ : assert_is_valid_output_type<T, AllowDeprecatedTypes> {};
279
+
280
+ template <class T, bool AllowDeprecatedTypes>
281
+ struct assert_is_valid_output_type<
282
+ c10::OptionalArrayRef<T>,
283
+ AllowDeprecatedTypes>
284
+ : assert_is_valid_output_type<T, AllowDeprecatedTypes> {};
285
+
286
+ template <class Key, class Value, bool AllowDeprecatedTypes>
287
+ struct assert_is_valid_output_type<Dict<Key, Value>, AllowDeprecatedTypes>
288
+ : assert_is_valid_output_type<Value, AllowDeprecatedTypes> {
289
+ static_assert(
290
+ guts::typelist::contains<impl::valid_dict_key_types, Key>::value,
291
+ "You tried to register a kernel with an unsupported output type: Dict<Key, Value> where Key is invalid. We only support int64_t, double, bool, and string.");
292
+ static_assert(
293
+ !std::is_same_v<Value, at::Scalar>,
294
+ "You tried to register a kernel with an unsupported output type: Dict<Key, Scalar>. Please use Dict<Key, int64_t> or Dict<Key, double>.");
295
+ };
296
+
297
+ template <class Key, class Value, bool AllowDeprecatedTypes>
298
+ struct assert_is_valid_output_type<
299
+ std::unordered_map<Key, Value>,
300
+ AllowDeprecatedTypes>
301
+ : assert_is_valid_output_type<Value, AllowDeprecatedTypes> {
302
+ static_assert(
303
+ AllowDeprecatedTypes,
304
+ "You tried to register a kernel with an unsupported output type: std::unordered_map<Key, Value>. Please use Dict<Key, Value> instead.");
305
+ static_assert(
306
+ guts::typelist::contains<impl::valid_dict_key_types, Key>::value,
307
+ "You tried to register a kernel with an unsupported output type: std::unordered_map<Key, Value> where Key is invalid. We only support int64_t, double, bool, and string.");
308
+ static_assert(
309
+ !std::is_same_v<Value, at::Scalar>,
310
+ "You tried to register a kernel with an unsupported output type: std::unordered_map<Key, Scalar>. Please use Dict<Key, int64_t> or Dict<Key, double>.");
311
+ };
312
+
313
+ template <class T, bool AllowDeprecatedTypes>
314
+ struct assert_is_valid_output_type<List<T>, AllowDeprecatedTypes>
315
+ : assert_is_valid_output_type<T, AllowDeprecatedTypes> {
316
+ static_assert(
317
+ !std::is_same_v<T, at::Scalar>,
318
+ "You tried to register a kernel with an unsupported output type: List<Scalar>. Please use List<int64_t>, List<double> or Tensor instead.");
319
+ };
320
+
321
+ template <class T, bool AllowDeprecatedTypes>
322
+ struct assert_is_valid_output_type<std::vector<T>, AllowDeprecatedTypes>
323
+ : assert_is_valid_output_type<T, AllowDeprecatedTypes> {
324
+ static_assert(
325
+ !std::is_same_v<T, at::Scalar>,
326
+ "You tried to register a kernel with an unsupported output type: std::vector<Scalar>. Please use List<int64_t>, List<double> or Tensor instead.");
327
+ // TODO static_assert(AllowDeprecatedTypes, "You tried to register a kernel
328
+ // with an unsupported output type: std::vector<T>. Please use List<T>
329
+ // instead.");
330
+ };
331
+
332
+ template <class T, size_t N, bool AllowDeprecatedTypes>
333
+ struct assert_is_valid_output_type<std::array<T, N>, AllowDeprecatedTypes>
334
+ : assert_is_valid_output_type<T, AllowDeprecatedTypes> {
335
+ static_assert(
336
+ !std::is_same_v<T, at::Scalar>,
337
+ "You tried to register a kernel with an unsupported output type: std::array<Scalar, N>. Please use std::array<int64_t, N> instead.");
338
+ };
339
+
340
+ // The following specialisations of assert_is_valid_output_type are technically
341
+ // not necessary since we would hit the base case and show an error message
342
+ // there if they didn't exist, but we can show a better error message
343
+ // in some common error scenarios.
344
+ template <class T, bool AllowDeprecatedTypes>
345
+ struct assert_is_valid_output_type<
346
+ T,
347
+ AllowDeprecatedTypes,
348
+ std::enable_if_t<std::is_same_v<float, T>>> {
349
+ // There is no reason to support float when we have double. Keep the API lean.
350
+ static_assert(
351
+ guts::false_t<T>::value,
352
+ "You tried to register a kernel with an unsupported output type: float. Please use double instead; you should use `double` in the C++ function signature and `float` in the schema string.");
353
+ };
354
+ template <class T, bool AllowDeprecatedTypes>
355
+ struct assert_is_valid_output_type<
356
+ T,
357
+ AllowDeprecatedTypes,
358
+ std::enable_if_t<std::is_same_v<const char*, T>>> {
359
+ static_assert(
360
+ guts::false_t<T>::value,
361
+ "You tried to register a kernel with an unsupported output type: const char*. Please use std::string_view instead.");
362
+ };
363
+ template <class T, bool AllowDeprecatedTypes>
364
+ struct assert_is_valid_output_type<
365
+ T,
366
+ AllowDeprecatedTypes,
367
+ std::enable_if_t<std::is_same_v<std::vector<bool>, T>>> {
368
+ static_assert(
369
+ guts::false_t<T>::value,
370
+ "You tried to register a kernel with an unsupported output type: vector<bool>. Please use List<bool> instead.");
371
+ };
372
+ template <class T, bool AllowDeprecatedTypes>
373
+ struct assert_is_valid_output_type<
374
+ T,
375
+ AllowDeprecatedTypes,
376
+ std::enable_if_t<
377
+ std::is_integral_v<T> &&
378
+ !guts::typelist::contains<supported_primitive_arg_types, T>::value>> {
379
+ static_assert(
380
+ guts::false_t<T>::value,
381
+ "You tried to register a kernel with an unsupported integral output type. Please use int64_t instead; you should use `int64_t` in the C++ function signature and `int` in the schema string.");
382
+ };
383
+
384
+ // ivalue_to_arg
385
+
386
+ template <class T>
387
+ struct decay_if_not_tensor final {
388
+ using type = std::decay_t<T>;
389
+ };
390
+
391
+ template <>
392
+ struct decay_if_not_tensor<at::Tensor&> final {
393
+ using type = at::Tensor&;
394
+ };
395
+
396
+ template <>
397
+ struct decay_if_not_tensor<const at::Tensor&> final {
398
+ using type = const at::Tensor&;
399
+ };
400
+
401
+ template <class T, bool AllowDeprecatedTypes>
402
+ struct ivalue_to_arg final {
403
+ static decltype(auto) call(IValue& v) {
404
+ assert_is_valid_input_type<T, AllowDeprecatedTypes>();
405
+ return std::move(v).to<T>();
406
+ }
407
+ };
408
+
409
+ // The following two specializations take advantage of specialized
410
+ // `toTensor()` overloads on IValue to avoid copying.
411
+ template <bool AllowDeprecatedTypes>
412
+ struct ivalue_to_arg<at::Tensor&, AllowDeprecatedTypes> final {
413
+ // We cannot use the default implementation if they asked for a
414
+ // `at::Tensor&` because it moves from the IValue, so it can't get
415
+ // an lvalue reference.
416
+ static at::Tensor& call(IValue& v) {
417
+ // Tensor& is valid, don't bother asserting
418
+ return v.toTensor();
419
+ }
420
+ };
421
+
422
+ template <bool AllowDeprecatedTypes>
423
+ struct ivalue_to_arg<const at::Tensor&, AllowDeprecatedTypes> final {
424
+ // We should not use the default implementation if they asked for
425
+ // a `const at::Tensor&` because it moves from the IValue and they
426
+ // didn't ask for that.
427
+ static const at::Tensor& call(IValue& v) {
428
+ // const Tensor& is valid, don't bother asserting
429
+ return v.toTensor();
430
+ }
431
+ };
432
+
433
+ template <bool AllowDeprecatedTypes>
434
+ struct ivalue_to_arg<at::ITensorListRef, AllowDeprecatedTypes> final {
435
+ static List<at::Tensor> call(IValue& v) {
436
+ return v.toTensorList();
437
+ }
438
+ };
439
+
440
+ template <class T, bool AllowDeprecatedTypes>
441
+ struct ivalue_to_arg<ArrayRef<T>, AllowDeprecatedTypes> final {
442
+ // If an argument is ArrayRef<T>, convert the IValue to a std::vector<T> and
443
+ // pass that to the operator. std::vector<T> is implicitly convertible to
444
+ // ArrayRef<T>.
445
+ static std::vector<T> call(IValue& v) {
446
+ return ivalue_to_arg<std::vector<T>, AllowDeprecatedTypes>::call(v);
447
+ }
448
+ };
449
+ template <bool AllowDeprecatedTypes>
450
+ struct ivalue_to_arg<c10::SymIntArrayRef, AllowDeprecatedTypes> final {
451
+ static std::vector<c10::SymInt> call(IValue& v) {
452
+ if (v.isIntList()) {
453
+ std::vector<c10::SymInt> r;
454
+ auto src = v.toIntList();
455
+ std::transform(
456
+ src.begin(), src.end(), std::back_inserter(r), [](int64_t i) {
457
+ return c10::SymInt(i);
458
+ });
459
+ return r;
460
+ } else {
461
+ return ivalue_to_arg<std::vector<c10::SymInt>, AllowDeprecatedTypes>::
462
+ call(v);
463
+ }
464
+ }
465
+ };
466
+ template <bool AllowDeprecatedTypes>
467
+ struct ivalue_to_arg<c10::OptionalArray<c10::SymInt>, AllowDeprecatedTypes>
468
+ final {
469
+ static OptionalArray<c10::SymInt> call(IValue& v) {
470
+ if (v.isIntList()) {
471
+ std::vector<c10::SymInt> r;
472
+ auto src = v.toIntList();
473
+ std::transform(
474
+ src.begin(), src.end(), std::back_inserter(r), [](int64_t i) {
475
+ return c10::SymInt(i);
476
+ });
477
+ return OptionalArray<c10::SymInt>(std::move(r));
478
+ } else {
479
+ return std::move(v).to<OptionalArray<c10::SymInt>>();
480
+ }
481
+ }
482
+ };
483
+ template <class T, bool AllowDeprecatedTypes>
484
+ struct ivalue_to_arg<std::optional<ArrayRef<T>>, AllowDeprecatedTypes> final {
485
+ // If an argument is std::optional<ArrayRef<T>>, convert the IValue to an
486
+ // std::optional<std::vector<T>> and pass that to the operator.
487
+ // OptionalArray<T> is basically a std::optional<std::vector<T>> but
488
+ // implicitly convertible to std::optional<ArrayRef<T>>.
489
+ static OptionalArray<T> call(IValue& v) {
490
+ return ivalue_to_arg<OptionalArray<T>, AllowDeprecatedTypes>::call(v);
491
+ }
492
+ };
493
+
494
+ template <class T, bool AllowDeprecatedTypes>
495
+ struct ivalue_to_arg<OptionalArrayRef<T>, AllowDeprecatedTypes> final {
496
+ // If an argument is OptionalArrayRef<T>, convert the IValue to an
497
+ // std::optional<std::vector<T>> and pass that to the operator.
498
+ // OptionalArray<T> is basically a std::optional<std::vector<T>> but
499
+ // implicitly convertible to OptionalArrayRef<T>
500
+ static OptionalArray<T> call(IValue& v) {
501
+ return ivalue_to_arg<OptionalArray<T>, AllowDeprecatedTypes>::call(v);
502
+ }
503
+ };
504
+
505
+ // return_to_ivalue
506
+ template <class T, bool AllowDeprecatedTypes, class Enable = void>
507
+ struct return_to_ivalue final {};
508
+
509
+ template <class T, bool AllowDeprecatedTypes>
510
+ struct return_to_ivalue<
511
+ T,
512
+ AllowDeprecatedTypes,
513
+ std::enable_if_t<!std::is_same_v<at::Tensor&, T>>>
514
+ final {
515
+ static IValue call(T&& v) {
516
+ assert_is_valid_output_type<T, AllowDeprecatedTypes>();
517
+ return c10::ivalue::from(std::move(v));
518
+ }
519
+ static IValue copy(const T& v) {
520
+ assert_is_valid_output_type<T, AllowDeprecatedTypes>();
521
+ return IValue(v);
522
+ }
523
+ };
524
+
525
+ // Special case to allow kernels to return `Tensor&`.
526
+ // TODO Delete this once kernels don't do that anymore
527
+ template <bool AllowDeprecatedTypes>
528
+ struct return_to_ivalue<at::Tensor&, AllowDeprecatedTypes, void> final {
529
+ static IValue call(at::Tensor& v) {
530
+ return c10::ivalue::from(v);
531
+ }
532
+ static IValue copy(at::Tensor& v) {
533
+ return IValue(v);
534
+ }
535
+ };
536
+
537
+ // wrap_kernel_functor_unboxed_
538
+
539
+ template <class KernelFunctor, class OpSignature>
540
+ struct wrap_kernel_functor_unboxed_ final {};
541
+
542
+ // This specialization is for kernels with a first argument that is NOT of type
543
+ // DispatchKeySet This includes kernels with 0 arguments.
544
+ template <class KernelFunctor, class ReturnType, class... ParameterTypes>
545
+ struct wrap_kernel_functor_unboxed_<
546
+ KernelFunctor,
547
+ ReturnType(ParameterTypes...)>
548
+ final {
549
+ static_assert(
550
+ std::is_same_v<
551
+ ReturnType,
552
+ typename guts::infer_function_traits_t<KernelFunctor>::return_type>,
553
+ "Return type mismatch");
554
+ static_assert(
555
+ std::is_same_v<
556
+ guts::typelist::typelist<ParameterTypes...>,
557
+ typename guts::infer_function_traits_t<
558
+ KernelFunctor>::parameter_types>,
559
+ "Parameter types mismatch");
560
+
561
+ // See [Note: Argument forwarding in the dispatcher] for why ParameterTypes
562
+ // doesn't use &&
563
+ static ReturnType call(
564
+ OperatorKernel* functor,
565
+ DispatchKeySet /*unused*/,
566
+ ParameterTypes... args) {
567
+ KernelFunctor* functor_ = static_cast<KernelFunctor*>(functor);
568
+ // Note [Plumbing Keys Through The Dispatcher 2]
569
+ // See Note [Plumbing Keys Through The Dispatcher] for the background.
570
+ // This functor explicitly takes in a dispatchKeySet and drops it on the
571
+ // floor- it does not forward it to the registered kernel.
572
+ //
573
+ // This is due to the calling convention within the dispatcher, which
574
+ // expects all registered kernels to have a first argument of type
575
+ // DispatchKeySet.
576
+ // This is not the case for pretty much all manually written kernels,
577
+ // however- this functor serves to separate the calling convention of the
578
+ // dispatcher from the calling convention of manually written kernels.
579
+ return (*functor_)(std::forward<ParameterTypes>(args)...);
580
+ }
581
+ };
582
+
583
+ // This specialization is for kernels with a first argument of type
584
+ // DispatchKeySet
585
+ template <class KernelFunctor, class ReturnType, class... ParameterTypes>
586
+ struct wrap_kernel_functor_unboxed_<
587
+ KernelFunctor,
588
+ ReturnType(DispatchKeySet, ParameterTypes...)>
589
+ final {
590
+ static_assert(
591
+ std::is_same_v<
592
+ ReturnType,
593
+ typename guts::infer_function_traits_t<KernelFunctor>::return_type>,
594
+ "Return type mismatch");
595
+ static_assert(
596
+ std::is_same_v<
597
+ guts::typelist::typelist<DispatchKeySet, ParameterTypes...>,
598
+ typename guts::infer_function_traits_t<
599
+ KernelFunctor>::parameter_types>,
600
+ "Parameter types mismatch");
601
+
602
+ // See [Note: Argument forwarding in the dispatcher] for why ParameterTypes
603
+ // doesn't use &&
604
+ static ReturnType call(
605
+ OperatorKernel* functor,
606
+ DispatchKeySet dispatchKeySet,
607
+ ParameterTypes... args) {
608
+ KernelFunctor* functor_ = static_cast<KernelFunctor*>(functor);
609
+ // We're explicitly taking in a dispatchKeySet and forwarding it to the
610
+ // registered kernel. See Note [Plumbing Keys Through The Dispatcher 2] for
611
+ // details.
612
+ return (*functor_)(dispatchKeySet, std::forward<ParameterTypes>(args)...);
613
+ }
614
+ };
615
+
616
+ template <class KernelFunctor>
617
+ using wrap_kernel_functor_unboxed = wrap_kernel_functor_unboxed_<
618
+ KernelFunctor,
619
+ typename guts::infer_function_traits_t<KernelFunctor>::func_type>;
620
+
621
+ // call_functor_with_args_from_stack
622
+
623
+ template <
624
+ class Functor,
625
+ bool AllowDeprecatedTypes,
626
+ size_t... ivalue_arg_indices,
627
+ typename... ArgTypes>
628
+ std::decay_t<typename guts::infer_function_traits_t<Functor>::return_type>
629
+ call_functor_with_args_from_stack_(
630
+ OperatorKernel* functor,
631
+ DispatchKeySet dispatchKeySet,
632
+ Stack* stack,
633
+ std::index_sequence<ivalue_arg_indices...> /*unused*/,
634
+ guts::typelist::typelist<ArgTypes...>* /*unused*/) {
635
+ (void)stack; // when sizeof...(ivalue_arg_indices) == 0, this argument would
636
+ // be unused and we have to silence the compiler warning.
637
+
638
+ // We're explicitly filtering out DispatchKeySet from the argument list.
639
+ // Some kernels take a DispatchKeySet as their first argument in order to
640
+ // plumb keys through the dispatcher. We don't want to expose the
641
+ // DispatchKeySet type to jit, so we don't include this argument on the stack.
642
+ // See Note [Plumbing Keys Through The Dispatcher] for the background.
643
+ return wrap_kernel_functor_unboxed<Functor>::call(
644
+ functor,
645
+ dispatchKeySet,
646
+ ivalue_to_arg<
647
+ typename decay_if_not_tensor<ArgTypes>::type,
648
+ AllowDeprecatedTypes>::
649
+ call(torch::jit::peek(
650
+ *stack, ivalue_arg_indices, sizeof...(ivalue_arg_indices)))...);
651
+ }
652
+
653
+ template <class Functor, bool AllowDeprecatedTypes>
654
+ std::decay_t<typename guts::infer_function_traits_t<Functor>::return_type>
655
+ call_functor_with_args_from_stack(
656
+ OperatorKernel* functor,
657
+ DispatchKeySet dispatchKeySet,
658
+ Stack* stack) {
659
+ // We're explicitly filtering out DispatchKeySet from the argument list.
660
+ // Some kernels take a DispatchKeySet as their first argument in order to
661
+ // plumb keys through the dispatcher. We don't want to expose the
662
+ // DispatchKeySet type to jit, so we don't include this argument on the stack.
663
+ // See Note [Plumbing Keys Through The Dispatcher] for the background.
664
+ using ArgTypes = typename c10::remove_DispatchKeySet_arg_from_func<
665
+ Functor>::parameter_types;
666
+ constexpr size_t num_ivalue_args = guts::typelist::size<ArgTypes>::value;
667
+ return call_functor_with_args_from_stack_<Functor, AllowDeprecatedTypes>(
668
+ functor,
669
+ dispatchKeySet,
670
+ stack,
671
+ std::make_index_sequence<num_ivalue_args>(),
672
+ static_cast<ArgTypes*>(nullptr));
673
+ }
674
+
675
+ // push_outputs
676
+
677
+ template <class OutputType, bool AllowDeprecatedTypes>
678
+ struct push_outputs final {
679
+ // Contrary to [Note: Argument forwarding in the dispatcher], we use
680
+ // OutputType&& here to avoid one extra call to the move constructor in this
681
+ // case. This is still not a universal reference though because OutputType is
682
+ // an explicitly specified class template parameter.
683
+ static void call(OutputType&& output, Stack* stack) {
684
+ torch::jit::push(
685
+ *stack,
686
+ return_to_ivalue<OutputType, AllowDeprecatedTypes>::call(
687
+ std::forward<OutputType>(output)));
688
+ }
689
+ static void copy(const OutputType& output, Stack* stack) {
690
+ torch::jit::push(
691
+ *stack,
692
+ return_to_ivalue<OutputType, AllowDeprecatedTypes>::copy(output));
693
+ }
694
+ };
695
+ template <class... OutputTypes, bool AllowDeprecatedTypes>
696
+ struct push_outputs<std::tuple<OutputTypes...>, AllowDeprecatedTypes> final {
697
+ static void call(std::tuple<OutputTypes...>&& output, Stack* stack) {
698
+ call_(
699
+ std::move(output),
700
+ stack,
701
+ std::make_index_sequence<sizeof...(OutputTypes)>());
702
+ }
703
+ static void copy(const std::tuple<OutputTypes...>& output, Stack* stack) {
704
+ copy_(output, stack, std::make_index_sequence<sizeof...(OutputTypes)>());
705
+ }
706
+
707
+ private:
708
+ template <size_t... indices>
709
+ static void call_(
710
+ std::tuple<OutputTypes...>&& output,
711
+ Stack* stack,
712
+ std::index_sequence<indices...> /*unused*/) {
713
+ torch::jit::push(
714
+ *stack,
715
+ return_to_ivalue<OutputTypes, AllowDeprecatedTypes>::call(
716
+ std::forward<OutputTypes>(std::get<indices>(output)))...);
717
+ }
718
+ template <size_t... indices>
719
+ static void copy_(
720
+ const std::tuple<OutputTypes...>& output,
721
+ Stack* stack,
722
+ std::index_sequence<indices...> /*unused*/) {
723
+ torch::jit::push(
724
+ *stack,
725
+ return_to_ivalue<OutputTypes, AllowDeprecatedTypes>::copy(
726
+ std::get<indices>(output))...);
727
+ }
728
+ };
729
+ template <bool AllowDeprecatedTypes>
730
+ struct push_outputs<void, AllowDeprecatedTypes> final {
731
+ static void call(int /*dummy*/, Stack* /*stack*/) {}
732
+ static void copy(int /*dummy*/, Stack* /*stack*/) {}
733
+ };
734
+
735
+ // make_boxed_from_unboxed_functor
736
+
737
+ template <class KernelFunctor, bool AllowDeprecatedTypes>
738
+ struct make_boxed_from_unboxed_functor final {
739
+ static_assert(
740
+ std::is_base_of_v<OperatorKernel, KernelFunctor>,
741
+ "Tried to register a kernel functor using the kernel<Functor>() API, but it doesn't inherit from c10::OperatorKernel. Please have the functor inherit from it.");
742
+
743
+ static void call(
744
+ OperatorKernel* functor,
745
+ const OperatorHandle& /*unused*/,
746
+ DispatchKeySet dispatchKeySet,
747
+ Stack* stack) {
748
+ using ReturnType =
749
+ typename guts::infer_function_traits_t<KernelFunctor>::return_type;
750
+ // We're explicitly filtering out DispatchKeySet from the argument list.
751
+ // Some kernels take a DispatchKeySet as their first argument in order to
752
+ // plumb keys through the dispatcher. We don't want to expose the
753
+ // DispatchKeySet type to jit, so we don't include this argument on the
754
+ // stack. See Note [Plumbing Keys Through The Dispatcher] for the
755
+ // background.
756
+ using ArgTypes = typename c10::remove_DispatchKeySet_arg_from_func<
757
+ KernelFunctor>::parameter_types;
758
+ constexpr bool has_outputs = !std::is_same_v<void, ReturnType>;
759
+ constexpr size_t num_inputs = guts::typelist::size<ArgTypes>::value;
760
+ if constexpr (has_outputs) {
761
+ // Decay ReturnType to ReturnType_ so that if a reference gets returned,
762
+ // we actually store it by value and don't get a dangling reference. This
763
+ // is only required because some kernels still return `Tensor&`. [Note:
764
+ // VC++ and 'std': ambiguous symbol]
765
+ using ReturnType_ = ::std::decay_t<ReturnType>;
766
+ ReturnType_ output = call_functor_with_args_from_stack<
767
+ KernelFunctor,
768
+ AllowDeprecatedTypes>(functor, dispatchKeySet, stack);
769
+ torch::jit::drop(*stack, num_inputs);
770
+ // See note [ VC++ and 'std': ambiguous symbol]
771
+ push_outputs<ReturnType_, AllowDeprecatedTypes>::call(
772
+ ::std::move(output), stack);
773
+ } else {
774
+ call_functor_with_args_from_stack<KernelFunctor, AllowDeprecatedTypes>(
775
+ functor, dispatchKeySet, stack);
776
+ torch::jit::drop(*stack, num_inputs);
777
+ }
778
+ }
779
+ };
780
+ } // namespace impl
781
+
782
+ } // namespace c10
783
+
784
+ namespace torch {
785
+ using OperatorKernel = c10::OperatorKernel;
786
+ }
787
+
788
+ #else
789
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
790
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/boxing/impl/test_helpers.h ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <gmock/gmock.h>
5
+ #include <gtest/gtest.h>
6
+
7
+ #include <ATen/core/Tensor.h>
8
+ #include <ATen/core/dispatch/Dispatcher.h>
9
+ #include <ATen/core/ivalue.h>
10
+ #include <c10/core/CPUAllocator.h>
11
+ #include <c10/util/irange.h>
12
+
13
+ template <class... Inputs>
14
+ inline std::vector<c10::IValue> makeStack(Inputs&&... inputs) {
15
+ return {std::forward<Inputs>(inputs)...};
16
+ }
17
+
18
+ inline at::Tensor dummyTensor(
19
+ c10::DispatchKeySet ks,
20
+ bool requires_grad = false) {
21
+ auto* allocator = c10::GetCPUAllocator();
22
+ int64_t nelements = 1;
23
+ auto dtype = caffe2::TypeMeta::Make<float>();
24
+ int64_t size_bytes = nelements * dtype.itemsize();
25
+ auto storage_impl = c10::make_intrusive<c10::StorageImpl>(
26
+ c10::StorageImpl::use_byte_size_t(),
27
+ size_bytes,
28
+ allocator->allocate(size_bytes),
29
+ allocator,
30
+ /*resizable=*/true);
31
+ at::Tensor t =
32
+ at::detail::make_tensor<c10::TensorImpl>(storage_impl, ks, dtype);
33
+ // TODO: We add this to simulate the ideal case where we only have Autograd
34
+ // backend keys
35
+ // on Tensor when it requires grad. But currently Autograd keys are
36
+ // added in TensorImpl constructor by default.
37
+ if (!requires_grad) {
38
+ t.unsafeGetTensorImpl()->remove_autograd_key();
39
+ }
40
+ return t;
41
+ }
42
+
43
+ inline at::Tensor dummyTensor(
44
+ c10::DispatchKey dispatch_key,
45
+ bool requires_grad = false) {
46
+ return dummyTensor(c10::DispatchKeySet(dispatch_key), requires_grad);
47
+ }
48
+
49
+ template <class... Args>
50
+ inline std::vector<c10::IValue> callOp(
51
+ const c10::OperatorHandle& op,
52
+ Args... args) {
53
+ auto stack = makeStack(std::forward<Args>(args)...);
54
+ op.callBoxed(&stack);
55
+ return stack;
56
+ }
57
+
58
+ template <class Result, class... Args>
59
+ inline Result callOpUnboxed(const c10::OperatorHandle& op, Args... args) {
60
+ return op.typed<Result(Args...)>().call(std::forward<Args>(args)...);
61
+ }
62
+
63
+ template <class Result, class... Args>
64
+ inline Result callOpUnboxedWithDispatchKey(
65
+ const c10::OperatorHandle& op,
66
+ c10::DispatchKey dispatchKey,
67
+ Args... args) {
68
+ return op.typed<Result(Args...)>().callWithDispatchKey(
69
+ dispatchKey, std::forward<Args>(args)...);
70
+ }
71
+
72
+ template <class Result, class... Args>
73
+ inline Result callOpUnboxedWithPrecomputedDispatchKeySet(
74
+ const c10::OperatorHandle& op,
75
+ c10::DispatchKeySet ks,
76
+ Args... args) {
77
+ return op.typed<Result(Args...)>().redispatch(
78
+ ks, std::forward<Args>(args)...);
79
+ }
80
+
81
+ inline void expectDoesntFindKernel(
82
+ const char* op_name,
83
+ c10::DispatchKey dispatch_key) {
84
+ auto op = c10::Dispatcher::singleton().findSchema({op_name, ""});
85
+ EXPECT_ANY_THROW(callOp(*op, dummyTensor(dispatch_key), 5););
86
+ }
87
+
88
+ inline void expectDoesntFindOperator(const char* op_name) {
89
+ auto op = c10::Dispatcher::singleton().findSchema({op_name, ""});
90
+ EXPECT_FALSE(op.has_value());
91
+ }
92
+
93
+ template <class Exception, class Functor>
94
+ inline void expectThrows(Functor&& functor, const char* expectMessageContains) {
95
+ try {
96
+ std::forward<Functor>(functor)();
97
+ } catch (const Exception& e) {
98
+ EXPECT_THAT(e.what(), testing::HasSubstr(expectMessageContains));
99
+ return;
100
+ }
101
+ ADD_FAILURE() << "Expected to throw exception containing \""
102
+ << expectMessageContains << "\" but didn't throw";
103
+ }
104
+
105
+ template <class T, size_t N>
106
+ void expectListEquals(c10::ArrayRef<T> expected, std::array<T, N> actual) {
107
+ EXPECT_EQ(expected.size(), actual.size());
108
+ for (const auto i : c10::irange(expected.size())) {
109
+ EXPECT_EQ(expected[i], actual[i]);
110
+ }
111
+ }
112
+
113
+ template <class T>
114
+ void expectListEquals(c10::ArrayRef<T> expected, c10::ArrayRef<T> actual) {
115
+ EXPECT_EQ(expected.size(), actual.size());
116
+ for (const auto i : c10::irange(expected.size())) {
117
+ EXPECT_EQ(expected[i], actual[i]);
118
+ }
119
+ }
120
+
121
+ template <class T>
122
+ void expectListEquals(c10::ArrayRef<T> expected, c10::List<T> actual) {
123
+ EXPECT_EQ(expected.size(), actual.size());
124
+ for (const auto i : c10::irange(expected.size())) {
125
+ EXPECT_EQ(expected[i], actual.get(i));
126
+ }
127
+ }
128
+
129
+ template <class T>
130
+ void expectListEquals(c10::ArrayRef<T> expected, std::vector<T> actual) {
131
+ EXPECT_EQ(expected.size(), actual.size());
132
+ for (const auto i : c10::irange(expected.size())) {
133
+ EXPECT_EQ(expected[i], actual[i]);
134
+ }
135
+ }
136
+
137
+ // NB: This is not really sound, but all of the type sets constructed here
138
+ // are singletons so it's fine
139
+ static inline c10::DispatchKey extractDispatchKey(const at::Tensor& t) {
140
+ return legacyExtractDispatchKey(t.key_set());
141
+ }
142
+
143
+ #else
144
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
145
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/CppSignature.h ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <c10/core/DispatchKeySet.h>
5
+ #include <c10/macros/Macros.h>
6
+ #include <c10/util/Metaprogramming.h>
7
+ #include <c10/util/Type.h>
8
+ #include <typeindex>
9
+
10
+ namespace c10::impl {
11
+
12
+ // A CppSignature object holds RTTI information about a C++ function signature
13
+ // at runtime and can compare them or get a debug-printable name.
14
+ class TORCH_API CppSignature final {
15
+ public:
16
+ CppSignature(const CppSignature&) = default;
17
+ CppSignature(CppSignature&&) noexcept = default;
18
+ CppSignature& operator=(const CppSignature&) = default;
19
+ CppSignature& operator=(CppSignature&&) noexcept = default;
20
+
21
+ template <class FuncType>
22
+ static CppSignature make() {
23
+ // Normalize functors, lambdas, function pointers, etc. into the plain
24
+ // function type The first argument of the schema might be of type
25
+ // DispatchKeySet, in which case we remove it. We do this to guarantee that
26
+ // all CppSignature's for an operator will match, even if they're registered
27
+ // with different calling conventions.
28
+ // See Note [Plumbing Keys Through The Dispatcher]
29
+ using decayed_function_type =
30
+ typename c10::remove_DispatchKeySet_arg_from_func<
31
+ std::decay_t<FuncType>>::func_type;
32
+
33
+ return CppSignature(std::type_index(typeid(decayed_function_type)));
34
+ }
35
+
36
+ std::string name() const {
37
+ return c10::demangle(signature_.name());
38
+ }
39
+
40
+ friend bool operator==(const CppSignature& lhs, const CppSignature& rhs) {
41
+ if (lhs.signature_ == rhs.signature_) {
42
+ return true;
43
+ }
44
+ // Without RTLD_GLOBAL, the type_index comparison could yield false because
45
+ // they point to different instances of the RTTI data, but the types would
46
+ // still be the same. Let's check for that case too.
47
+ // Note that there still is a case where this might not work, i.e. when
48
+ // linking libraries of different compilers together, they might have
49
+ // different ways to serialize a type name. That, together with a missing
50
+ // RTLD_GLOBAL, would still fail this.
51
+ if (0 == strcmp(lhs.signature_.name(), rhs.signature_.name())) {
52
+ return true;
53
+ }
54
+
55
+ return false;
56
+ }
57
+
58
+ private:
59
+ explicit CppSignature(std::type_index signature)
60
+ : signature_(std::move(signature)) {}
61
+ std::type_index signature_;
62
+ };
63
+
64
+ inline bool operator!=(const CppSignature& lhs, const CppSignature& rhs) {
65
+ return !(lhs == rhs);
66
+ }
67
+
68
+ } // namespace c10::impl
69
+
70
+ #else
71
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
72
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/DispatchKeyExtractor.h ADDED
@@ -0,0 +1,285 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <ATen/core/Variadic.h>
5
+ #include <ATen/core/function_schema.h>
6
+ #include <ATen/core/jit_type.h>
7
+ #include <ATen/core/stack.h>
8
+ #include <c10/core/DispatchKeySet.h>
9
+ #include <c10/util/Bitset.h>
10
+ #include <c10/util/irange.h>
11
+ #include <cstdint>
12
+
13
+ namespace c10 {
14
+
15
+ namespace impl {
16
+
17
+ // Take a DispatchKeySet for a Tensor and determine what the actual dispatch
18
+ // DispatchKey should be, taking into account TLS, and skipping backends which
19
+ // fall through.
20
+ //
21
+ // Unlike Tensor::key_set(), the value of this on a tensor can change depending
22
+ // on TLS.
23
+ //
24
+ // NB: If there is no valid dispatch key, this will return Undefined
25
+ inline DispatchKeySet computeDispatchKeySet(
26
+ DispatchKeySet ks,
27
+ // The key mask lets us eliminate (by zero entries) keys which should not
28
+ // be considered for dispatch. There are two cases when we use this:
29
+ //
30
+ // - If an operator's dispatch table contains a fallthrough entry, we
31
+ // should bypass it entirely when finding the key
32
+ // - If a user invokes with redispatch, the mask lets us
33
+ // zero out the key the user asked us to stop.
34
+ //
35
+ // These excluded backends are NOT tracked in the TLS, but must be applied
36
+ // AFTER TLS (since the backend may have been introduced for consideration
37
+ // by the included TLS), which is why you have to pass them in to this
38
+ // function (as opposed to just applying it to the input 'ks').
39
+ DispatchKeySet key_mask) {
40
+ c10::impl::LocalDispatchKeySet local =
41
+ c10::impl::tls_local_dispatch_key_set();
42
+ // TODO: It's a bit irritating that we have to do logical ORs here, it would
43
+ // be nice to only do one. Can always_included be folded into the TLS? Well,
44
+ // it's a bit troublesome, because fastpath TLS access requires the type of
45
+ // the TLS in question to be zero-initialized, so you don't actually win
46
+ // anything in that case.
47
+ return (((ks | local.included_) - local.excluded_) & key_mask);
48
+ }
49
+
50
+ } // namespace impl
51
+
52
+ namespace detail {
53
+ // A small gadget to extract the DispatchKeySet from types which are known
54
+ // to have it. Used to extract dispatch keys from unboxed calls.
55
+ struct MultiDispatchKeySet : at::IterArgs<MultiDispatchKeySet> {
56
+ DispatchKeySet ts;
57
+ void operator()(const at::Tensor& x) {
58
+ ts = ts | x.key_set();
59
+ }
60
+ void operator()(const std::optional<at::Tensor>& x) {
61
+ if (x.has_value()) {
62
+ ts = ts | x->key_set();
63
+ }
64
+ }
65
+ void operator()(at::ArrayRef<at::Tensor> xs) {
66
+ for (const auto& x : xs) {
67
+ ts = ts | x.key_set();
68
+ }
69
+ }
70
+ // Tensor?[] translates to this case.
71
+ void operator()(const c10::List<std::optional<at::Tensor>>& xs) {
72
+ for (std::optional<at::Tensor> x : xs) {
73
+ if (x.has_value()) {
74
+ ts = ts | x.value().key_set();
75
+ }
76
+ }
77
+ }
78
+ // Structured Tensor[] translates to this case
79
+ void operator()(const at::ITensorListRef& xs) {
80
+ for (const auto& x : xs) {
81
+ ts = ts | x.key_set();
82
+ }
83
+ }
84
+ [[noreturn]] void operator()(
85
+ at::ArrayRef<std::optional<at::Tensor>> /*unused*/) {
86
+ // Just checking that the handling of Tensor?[] didn't change.
87
+ TORCH_INTERNAL_ASSERT(false);
88
+ }
89
+ void operator()(const at::Generator& gen) {
90
+ if (gen.defined()) {
91
+ ts = ts | gen.key_set();
92
+ }
93
+ }
94
+ void operator()(const std::optional<at::Generator>& gen) {
95
+ if (gen.has_value() && gen->defined()) {
96
+ ts = ts | gen->key_set();
97
+ }
98
+ }
99
+ template <typename T>
100
+ void operator()(const T& /*unused*/) {
101
+ // do nothing
102
+ }
103
+ };
104
+
105
+ // NB: take by const reference (Don't do universal forwarding here! You
106
+ // don't want to move into this function!)
107
+ template <typename... Args>
108
+ DispatchKeySet multi_dispatch_key_set(const Args&... args) {
109
+ return MultiDispatchKeySet().apply(args...).ts;
110
+ }
111
+ } // namespace detail
112
+
113
+ /**
114
+ * An instance of DispatchKeyExtractor knows how to get a dispatch key given
115
+ * a list of arguments for an operator call.
116
+ *
117
+ * The instance is specific for a certain operator as:
118
+ * - In boxed dispatch, different operators have different ways to extract
119
+ * the dispatch key (e.g. different numbers of arguments), and we precompute
120
+ * the stack locations we should look at; and
121
+ * - In all dispatch, some backends should be excluded from dispatch because
122
+ * they have been registered as fallthrough. The set of excluded backends
123
+ * varies from operator, as some operators may have overridden the
124
+ * fallthrough with custom behavior.
125
+ *
126
+ * Note - this should maintain identical impl to the py dispatcher key
127
+ * extraction logic at pytorch/torch/dispatcher.py
128
+ */
129
+ struct TORCH_API DispatchKeyExtractor final {
130
+ public:
131
+ static DispatchKeyExtractor make(const FunctionSchema& schema) {
132
+ return DispatchKeyExtractor(makeBitsetForDispatchArgs(schema));
133
+ }
134
+
135
+ static DispatchKeyExtractor makeUninitialized() {
136
+ return DispatchKeyExtractor(c10::utils::bitset());
137
+ }
138
+
139
+ void registerSchema(const FunctionSchema& schema) {
140
+ TORCH_INTERNAL_ASSERT(dispatch_arg_indices_reverse_.is_entirely_unset());
141
+ dispatch_arg_indices_reverse_ = makeBitsetForDispatchArgs(schema);
142
+ }
143
+ void deregisterSchema() {
144
+ dispatch_arg_indices_reverse_ = c10::utils::bitset();
145
+ }
146
+
147
+ DispatchKeySet getDispatchKeySetBoxed(const torch::jit::Stack* stack) const {
148
+ DispatchKeySet ks;
149
+ dispatch_arg_indices_reverse_.for_each_set_bit([&](size_t
150
+ reverse_arg_index) {
151
+ const auto& ivalue = torch::jit::peek(*stack, 0, reverse_arg_index + 1);
152
+ if (C10_LIKELY(ivalue.isTensor())) {
153
+ // NB: Take care not to introduce a refcount bump (there's
154
+ // no safe toTensorRef method, alas)
155
+ ks = ks | ivalue.unsafeToTensorImpl()->key_set();
156
+ } else if (C10_UNLIKELY(ivalue.isTensorList())) {
157
+ // NB: use toListRef as it doesn't induce refcount bumps
158
+ // (toTensorListRef is not a thing)
159
+ for (const auto& nv : ivalue.toListRef()) {
160
+ auto* tensor = nv.unsafeToTensorImpl();
161
+ ks = ks | tensor->key_set();
162
+ }
163
+ }
164
+ // Tensor?[] translates to a c10::List<IValue> so we need to peek inside
165
+ else if (C10_UNLIKELY(ivalue.isList())) {
166
+ for (const auto& elt : ivalue.toListRef()) {
167
+ if (elt.isTensor()) {
168
+ ks = ks | elt.toTensor().key_set();
169
+ }
170
+ }
171
+ }
172
+ });
173
+ // Keys that are fallthrough should be skipped
174
+ if (requiresBitsetPerBackend_) {
175
+ c10::impl::LocalDispatchKeySet tls =
176
+ c10::impl::tls_local_dispatch_key_set();
177
+ auto backend_idx =
178
+ ((ks | tls.included_) - tls.excluded_).getBackendIndex();
179
+ return impl::computeDispatchKeySet(
180
+ ks, nonFallthroughKeysPerBackend_[backend_idx]);
181
+ } else {
182
+ return impl::computeDispatchKeySet(ks, nonFallthroughKeys_);
183
+ }
184
+ }
185
+
186
+ template <class... Args>
187
+ DispatchKeySet getDispatchKeySetUnboxed(const Args&... args) const {
188
+ auto ks = detail::multi_dispatch_key_set(args...);
189
+ // Keys that are fallthrough should be skipped
190
+ if (requiresBitsetPerBackend_) {
191
+ c10::impl::LocalDispatchKeySet tls =
192
+ c10::impl::tls_local_dispatch_key_set();
193
+ auto backend_idx =
194
+ ((ks | tls.included_) - tls.excluded_).getBackendIndex();
195
+ return impl::computeDispatchKeySet(
196
+ ks, nonFallthroughKeysPerBackend_[backend_idx]);
197
+ } else {
198
+ return impl::computeDispatchKeySet(ks, nonFallthroughKeys_);
199
+ }
200
+ }
201
+
202
+ void setOperatorHasFallthroughForKey(DispatchKey k, bool has_fallthrough);
203
+
204
+ std::string dumpState() const;
205
+ void checkInvariants(const FunctionSchema& schema) const;
206
+
207
+ private:
208
+ static bool isDispatchType(const Type& type) {
209
+ // Checking isSubtypeOf on a DynamicType heap-allocates a
210
+ // DynamicType version of the argument if it's not a DynamicType
211
+ // already, and this has measurable overhead during startup.
212
+ #ifdef C10_MOBILE
213
+ struct CachedTypes {
214
+ DynamicTypePtr listOfTensors;
215
+ DynamicTypePtr listOfOptionalTensors;
216
+ DynamicTypePtr optionalOfTensor;
217
+ };
218
+ static const CachedTypes ct = {
219
+ DynamicType::create(*ListType::ofTensors()),
220
+ DynamicType::create(*ListType::ofOptionalTensors()),
221
+ DynamicType::create(*OptionalType::ofTensor())};
222
+ return type.isSubtypeOf(c10::TypeFactory::get<TensorType>()) ||
223
+ type.isSubtypeOf(ct.listOfTensors) ||
224
+ type.isSubtypeOf(ct.listOfOptionalTensors) ||
225
+ type.isSubtypeOf(ct.optionalOfTensor);
226
+ #else // C10_MOBILE
227
+ return type.isSubtypeOf(*TensorType::get()) ||
228
+ type.isSubtypeOf(*ListType::ofTensors()) ||
229
+ type.isSubtypeOf(*ListType::ofOptionalTensors()) ||
230
+ type.isSubtypeOf(*OptionalType::ofTensor());
231
+ #endif // C10_MOBILE
232
+ }
233
+ static c10::utils::bitset makeBitsetForDispatchArgs(
234
+ const FunctionSchema& schema) {
235
+ TORCH_CHECK(
236
+ schema.arguments().size() <= c10::utils::bitset::NUM_BITS(),
237
+ "The function schema has ",
238
+ schema.arguments().size(),
239
+ " arguments but this PyTorch build only supports ",
240
+ c10::utils::bitset::NUM_BITS());
241
+ c10::utils::bitset dispatch_arg_indices_reverse;
242
+ for (const auto index : c10::irange(schema.arguments().size())) {
243
+ if (isDispatchType(*schema.arguments()[index].type())) {
244
+ dispatch_arg_indices_reverse.set(schema.arguments().size() - 1 - index);
245
+ }
246
+ }
247
+ return dispatch_arg_indices_reverse;
248
+ }
249
+
250
+ explicit DispatchKeyExtractor(c10::utils::bitset dispatch_arg_indices_reverse)
251
+ : dispatch_arg_indices_reverse_(dispatch_arg_indices_reverse),
252
+ nonFallthroughKeys_(DispatchKeySet::FULL) {
253
+ for (const auto i : c10::irange(nonFallthroughKeysPerBackend_.size())) {
254
+ nonFallthroughKeysPerBackend_[i] = DispatchKeySet::FULL;
255
+ }
256
+ }
257
+
258
+ // this is a bitset that has ones for each argument index which has to be
259
+ // considered for dispatch. This avoids having to iterate over the stack
260
+ // to find all the tensors. The bits are stored in reverse order, i.e.
261
+ // dispatch_arg_indices_reverse_[i] == true, then the i-th argument from
262
+ // the top of the stack (i.e. the i-th last argument of the function)
263
+ // is relevant for dispatch.
264
+ // dispatch_arg_indices_reverse_ is allowed to have zero bits set; that just
265
+ // means you must do the fallthrough
266
+ c10::utils::bitset dispatch_arg_indices_reverse_;
267
+
268
+ // Set of functionality keys for which the operator does NOT have fallthrough
269
+ // kernel.
270
+ DispatchKeySet nonFallthroughKeys_;
271
+ // Set of functionality keys for which the operator does NOT have fallthrough
272
+ // kernel, defined PER BACKEND. This is only needed if we know that the
273
+ // operator has a different set of fallthroughs defined for some backends.
274
+ std::array<DispatchKeySet, num_backends> nonFallthroughKeysPerBackend_;
275
+ // Flag to tell us if we can use the single set of nonFallthroughKeys_ (fast
276
+ // path), or if we need to fall back to the slower path and check
277
+ // nonFallthroughKeysPerBackend_
278
+ bool requiresBitsetPerBackend_{false};
279
+ };
280
+
281
+ } // namespace c10
282
+
283
+ #else
284
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
285
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/Dispatcher.h ADDED
@@ -0,0 +1,955 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <ATen/SequenceNumber.h>
5
+ #include <ATen/core/boxing/KernelFunction.h>
6
+ #include <ATen/core/boxing/impl/boxing.h>
7
+ #include <ATen/core/dispatch/CppSignature.h>
8
+ #include <ATen/core/dispatch/OperatorEntry.h>
9
+ #include <ATen/core/dispatch/RegistrationHandleRAII.h>
10
+ #include <ATen/record_function.h>
11
+ #include <c10/core/SafePyObject.h>
12
+ #include <c10/util/Exception.h>
13
+ #include <c10/util/LeftRight.h>
14
+ #include <condition_variable>
15
+ #include <list>
16
+ #include <mutex>
17
+ #include <type_traits>
18
+
19
+ #include <ATen/core/enum_tag.h>
20
+ #include <ATen/core/grad_mode.h>
21
+
22
+ #ifndef NDEBUG
23
+ #include <iostream>
24
+ #endif
25
+
26
+ namespace c10 {
27
+
28
+ TORCH_API bool show_dispatch_trace();
29
+ TORCH_API void dispatch_trace_nesting_incr();
30
+ TORCH_API void dispatch_trace_nesting_decr();
31
+ TORCH_API int64_t dispatch_trace_nesting_value();
32
+
33
+ struct DispatchTraceNestingGuard {
34
+ DispatchTraceNestingGuard() {
35
+ dispatch_trace_nesting_incr();
36
+ }
37
+ ~DispatchTraceNestingGuard() {
38
+ dispatch_trace_nesting_decr();
39
+ }
40
+ };
41
+
42
+ class TORCH_API OperatorHandle;
43
+ template <class FuncType>
44
+ class TypedOperatorHandle;
45
+
46
+ /**
47
+ * Implement this interface and register your instance with the dispatcher
48
+ * to get notified when operators are registered or deregistered with
49
+ * the dispatcher.
50
+ *
51
+ * NB: registration events only occur when a 'def' occurs; we don't trigger
52
+ * on 'impl' or 'fallback' calls.
53
+ */
54
+ class TORCH_API OpRegistrationListener {
55
+ public:
56
+ virtual ~OpRegistrationListener();
57
+
58
+ virtual void onOperatorRegistered(const OperatorHandle& op) = 0;
59
+ virtual void onOperatorDeregistered(const OperatorHandle& op) = 0;
60
+ };
61
+
62
+ namespace detail {
63
+ class RegistrationListenerList;
64
+ }
65
+ class SchemaRegistrationHandleRAII;
66
+
67
+ /**
68
+ * Top-level dispatch interface for dispatching via the dynamic dispatcher.
69
+ * Most end users shouldn't use this directly; if you're trying to register
70
+ * ops look in op_registration
71
+ */
72
+ class TORCH_API Dispatcher final {
73
+ private:
74
+ // For direct access to backend fallback information
75
+ friend class impl::OperatorEntry;
76
+
77
+ struct OperatorDef final {
78
+ explicit OperatorDef(OperatorName&& op_name) : op(std::move(op_name)) {}
79
+
80
+ impl::OperatorEntry op;
81
+
82
+ // These refer to the number of outstanding RegistrationHandleRAII
83
+ // for this operator. def_count reflects only def() registrations
84
+ // (in the new world, this should only ever be 1, but old style
85
+ // registrations may register the schema multiple times, which
86
+ // will increase this count). def_and_impl_count reflects the number
87
+ // of combined def() and impl() registrations. When the last def() gets
88
+ // unregistered, we must immediately call the Deregistered listeners, but we
89
+ // must not actually delete the handle as there are other outstanding RAII
90
+ // destructors which will try to destruct and they had better still have a
91
+ // working operator handle in this case
92
+ size_t def_count = 0;
93
+ size_t def_and_impl_count = 0;
94
+ };
95
+ friend class OperatorHandle;
96
+ template <class>
97
+ friend class TypedOperatorHandle;
98
+
99
+ struct Guard final {
100
+ Guard() : alive(true) {}
101
+ std::atomic<bool> alive;
102
+ std::mutex mutex;
103
+ };
104
+
105
+ public:
106
+ ~Dispatcher();
107
+
108
+ // Implementation note: this class abstracts over the fact that we have
109
+ // per-operator dispatch tables. This could be easily adjusted to have a
110
+ // single global hash table.
111
+ static Dispatcher& realSingleton();
112
+
113
+ C10_ALWAYS_INLINE static Dispatcher& singleton() {
114
+ #if !defined C10_MOBILE
115
+ // Implemented inline so that steady-state code needn't incur
116
+ // function-call overhead. We can't just inline `realSingleton`
117
+ // because the function-local static would get duplicated across
118
+ // all DSOs that include & use this header, leading to multiple
119
+ // singleton instances.
120
+ static Dispatcher& s = realSingleton();
121
+ return s;
122
+ #else
123
+ // For C10_MOBILE, we should never inline a static function that
124
+ // has a static member, since the generated code calls
125
+ // __cxa_guard_acquire and __cxa_guard_release which help
126
+ // implement exactly once semantics for the initialization of the
127
+ // static Dispatcher& s above (for the non-mobile case). That
128
+ // additional code when duplicated across all operator stubs
129
+ // for every backend results in a lot of additional code
130
+ // being generated by the compiler.
131
+ return realSingleton();
132
+ #endif
133
+ }
134
+
135
+ // ------------------------------------------------------------------------
136
+ //
137
+ // Accessing operators by schema
138
+ //
139
+ // ------------------------------------------------------------------------
140
+
141
+ /**
142
+ * Looks for an operator schema with the given name and overload name
143
+ * and returns it if it is registered WITH A SCHEMA.
144
+ * Returns nullopt otherwise.
145
+ */
146
+ std::optional<OperatorHandle> findSchema(const OperatorName& operator_name);
147
+
148
+ /**
149
+ * Variant of findSchema that results in less code generated at the call site.
150
+ * It (1) takes const char* pointer rather than OperatorName (so we skip
151
+ * generating std::string constructor calls at the call site), and (2)
152
+ * it raises an exception if the operator is not found (so we skip
153
+ * generating exception raising code at the call site)
154
+ *
155
+ * Irritatingly, we still have to generate the handful of instructions
156
+ * for dealing with an exception being thrown during static initialization
157
+ * (e.g. __cxa_guard_abort). If we could annotate this method noexcept we
158
+ * could avoid this code too, but as the name of the function suggests,
159
+ * it does throw exceptions.
160
+ */
161
+ OperatorHandle findSchemaOrThrow(const char* name, const char* overload_name);
162
+
163
+ // Like findSchema, but also returns OperatorHandle even if there is no schema
164
+ std::optional<OperatorHandle> findOp(const OperatorName& operator_name);
165
+
166
+ // Returns a list of all operator names present in the operatorLookupTable_
167
+ const std::vector<OperatorName> getAllOpNames();
168
+
169
+ // Returns a list of all operator names present in the operatorLookupTable_
170
+ // for a given dispatch key
171
+ const std::vector<OperatorName> getAllOpNamesForDispatchKey(DispatchKey k);
172
+
173
+ // ------------------------------------------------------------------------
174
+ //
175
+ // Invoking operators
176
+ //
177
+ // ------------------------------------------------------------------------
178
+
179
+ template <class Return, class... Args>
180
+ Return call(const TypedOperatorHandle<Return(Args...)>& op, Args... args)
181
+ const;
182
+
183
+ template <class Return, class... Args>
184
+ static Return callWithDispatchKeySlowPath(
185
+ const TypedOperatorHandle<Return(Args...)>& op,
186
+ at::StepCallbacks& stepCallbacks,
187
+ DispatchKeySet dispatchKeySet,
188
+ const KernelFunction& kernel,
189
+ Args... args);
190
+
191
+ // Like call, but intended for use in a redispatch in kernels that have
192
+ // explicitly performed the DispatchKey update calculatulation. This will take
193
+ // the DispatchKeySet completely as is and dispatch to the kernel of the
194
+ // corresponding highest priority key in the set. Note that this version of
195
+ // redispatch treats the inputted DispatchKeySet *as is*, and does NOT mask
196
+ // out the highest priority key. See Note [Plumbing Keys Through The
197
+ // Dispatcher]
198
+ template <class Return, class... Args>
199
+ Return redispatch(
200
+ const TypedOperatorHandle<Return(Args...)>& op,
201
+ DispatchKeySet currentDispatchKeySet,
202
+ Args... args) const;
203
+
204
+ // Invoke an operator via the boxed calling convention using an IValue stack
205
+ void callBoxed(const OperatorHandle& op, Stack* stack) const;
206
+ void callBoxedForDispatchKey(
207
+ const OperatorHandle& op,
208
+ DispatchKey dk,
209
+ Stack* stack) const;
210
+
211
+ // TODO: This will only be useful if we write a backend fallback that plumbs
212
+ // dispatch keys (currently there are none) See Note [Plumbing Keys Through
213
+ // The Dispatcher]
214
+ void redispatchBoxed(
215
+ const OperatorHandle& op,
216
+ DispatchKeySet dispatchKeySet,
217
+ Stack* stack) const;
218
+
219
+ bool hasBackendFallbackForDispatchKey(DispatchKey dk) {
220
+ auto dispatch_ix = getDispatchTableIndexForDispatchKey(dk);
221
+ if (dispatch_ix < 0)
222
+ return false;
223
+ return backendFallbackKernels_[dispatch_ix].kernel.isValid();
224
+ }
225
+
226
+ // Used by torchdeploy/multipy for multiple // codespell:ignore: multipy
227
+ // interpreters racing.
228
+ void waitForDef(const FunctionSchema& schema);
229
+ void waitForImpl(
230
+ const OperatorName& op_name,
231
+ std::optional<DispatchKey> dispatch_key);
232
+
233
+ // ------------------------------------------------------------------------
234
+ //
235
+ // Performing registrations (NON user public; use op_registration)
236
+ //
237
+ // ------------------------------------------------------------------------
238
+
239
+ /**
240
+ * Register a new operator schema.
241
+ *
242
+ * If a schema with the same operator name and overload name already exists,
243
+ * this function will check that both schemas are exactly identical.
244
+ */
245
+ RegistrationHandleRAII registerDef(
246
+ FunctionSchema schema,
247
+ std::string debug,
248
+ std::vector<at::Tag> tags = {});
249
+
250
+ /**
251
+ * Register a kernel to the dispatch table for an operator.
252
+ * If dispatch_key is nullopt, then this registers a fallback kernel.
253
+ *
254
+ * @return A RAII object that manages the lifetime of the registration.
255
+ * Once that object is destructed, the kernel will be deregistered.
256
+ */
257
+ // NB: steals the inferred function schema, as we may need to hold on to
258
+ // it for a bit until the real schema turns up
259
+ RegistrationHandleRAII registerImpl(
260
+ OperatorName op_name,
261
+ std::optional<DispatchKey> dispatch_key,
262
+ KernelFunction kernel,
263
+ std::optional<impl::CppSignature> cpp_signature,
264
+ std::unique_ptr<FunctionSchema> inferred_function_schema,
265
+ std::string debug);
266
+
267
+ /**
268
+ * Given an operator, tells the Dispatcher that we have implemented a fake
269
+ * impl for this op in the given Python module. Call this a "pystub".
270
+ */
271
+ RegistrationHandleRAII registerPythonModule(
272
+ const OperatorName& op_name,
273
+ const char* pymodule,
274
+ const char* context);
275
+
276
+ /**
277
+ * Given an operator, throws if we have a pystub.
278
+ */
279
+ void throwIfHasPythonModule(OperatorName op_name);
280
+
281
+ std::optional<std::pair<const char*, const char*>> getPyStub(
282
+ OperatorName op_name);
283
+
284
+ /**
285
+ * Register a new operator by name.
286
+ */
287
+ RegistrationHandleRAII registerName(OperatorName op_name);
288
+
289
+ /**
290
+ * Register a fallback kernel for a backend.
291
+ * If an operator is called but there is no concrete kernel for the dispatch
292
+ * key of the given operator arguments, it will check if there is such a
293
+ * fallback kernel for the given dispatch key and, if yes, call that one.
294
+ */
295
+ RegistrationHandleRAII registerFallback(
296
+ DispatchKey dispatch_key,
297
+ KernelFunction kernel,
298
+ std::string debug);
299
+
300
+ /**
301
+ * Use to register whenever we had a TORCH_LIBRARY declaration in the frontend
302
+ * API. These invocations are only permitted once per program, so we raise
303
+ * an error if this is called again for the same namespace.
304
+ */
305
+ RegistrationHandleRAII registerLibrary(std::string ns, std::string debug);
306
+
307
+ // ------------------------------------------------------------------------
308
+ //
309
+ // Listeners on registrations
310
+ //
311
+ // ------------------------------------------------------------------------
312
+
313
+ /**
314
+ * Add a listener that gets called whenever a new op is registered or an
315
+ * existing op is deregistered. Immediately after registering, this listener
316
+ * gets called for all previously registered ops, so it can be used to keep
317
+ * track of ops registered with this dispatcher.
318
+ */
319
+ RegistrationHandleRAII addRegistrationListener(
320
+ std::unique_ptr<OpRegistrationListener> listener);
321
+
322
+ void checkInvariants() const;
323
+
324
+ //
325
+ // ------------------------------------------------------------------------
326
+ //
327
+ // Assertions
328
+ //
329
+ // ------------------------------------------------------------------------
330
+
331
+ /**
332
+ * For testing purposes.
333
+ * Returns a list of all operators that were created through calls to
334
+ * registerImpl(), without any corresponding calls to registerDef(). After
335
+ * static initialization is done this is almost certainly a bug, as the
336
+ * created OperatorHandle won't have any schema associated with it and users
337
+ * calling the op through the dispatcher won't be able to access it
338
+ *
339
+ * Note that we cannot enforce this invariant "as we go" during static
340
+ * initialization, due to undefined static initialization order- we have no
341
+ * guarantees over the order in which .def() and .impl() calls are registered
342
+ * in the dispatcher at static initialization time. So this function should
343
+ * only be called after static initialization.
344
+ */
345
+ std::vector<OperatorHandle> findDanglingImpls() const;
346
+
347
+ /**
348
+ * Useful for inspecting global Dispatcher registration state.
349
+ * Returns the names of all operators with a kernel registered for the
350
+ * specified DispatchKey. If no DispatchKey is specified, it returns all
351
+ * registered operators.
352
+ */
353
+ std::vector<OperatorName> getRegistrationsForDispatchKey(
354
+ std::optional<DispatchKey> k) const;
355
+
356
+ private:
357
+ Dispatcher();
358
+
359
+ static int64_t sequenceNumberForRunningRecordFunction(
360
+ DispatchKey dispatchKey,
361
+ DispatchKeySet dispatchKeySet);
362
+ static void runRecordFunction(
363
+ at::RecordFunction& guard,
364
+ at::RecordFunction::schema_ref_t schema_ref,
365
+ DispatchKey dispatchKey,
366
+ DispatchKeySet dispatchKeySet);
367
+ static void runRecordFunction(
368
+ at::RecordFunction& guard,
369
+ at::RecordFunction::schema_ref_t schema_ref,
370
+ DispatchKey dispatchKey,
371
+ DispatchKeySet dispatchKeySet,
372
+ c10::ArrayRef<const c10::IValue> args);
373
+
374
+ #ifdef FBCODE_CAFFE2
375
+ static bool profilingOperatorEvents();
376
+ static void fireOpStartUSDT(
377
+ at::RecordFunction::schema_ref_t schema_ref,
378
+ std::vector<void*>& argsAddresses,
379
+ std::vector<const char*>& argsTypes);
380
+ static void fireOpEndUSDT(at::RecordFunction::schema_ref_t schema_ref);
381
+ #endif // FBCODE_CAFFE2
382
+
383
+ OperatorHandle findOrRegisterSchema_(FunctionSchema&& schema);
384
+ OperatorHandle findOrRegisterName_(const OperatorName& op_name);
385
+
386
+ void deregisterDef_(const OperatorHandle& op, const OperatorName& op_name);
387
+ void deregisterImpl_(
388
+ const OperatorHandle& op,
389
+ const OperatorName& op_name,
390
+ std::optional<DispatchKey> dispatch_key,
391
+ impl::OperatorEntry::AnnotatedKernelContainerIterator kernel_handle);
392
+ void deregisterName_(const OperatorHandle& op, const OperatorName& op_name);
393
+ void deregisterFallback_(DispatchKey dispatchKey);
394
+ void deregisterLibrary_(const std::string& ns);
395
+ void cleanup(const OperatorHandle& op, const OperatorName& op_name);
396
+ void checkSchemaCompatibility(
397
+ const OperatorHandle& op,
398
+ const FunctionSchema& schema,
399
+ const std::string& debug);
400
+
401
+ std::list<OperatorDef> operators_;
402
+ #if !defined(C10_MOBILE)
403
+ LeftRight<ska::flat_hash_map<OperatorName, OperatorHandle>>
404
+ operatorLookupTable_;
405
+ #else
406
+ RWSafeLeftRightWrapper<ska::flat_hash_map<OperatorName, OperatorHandle>>
407
+ operatorLookupTable_;
408
+ #endif
409
+ // Map from namespace to debug string (saying, e.g., where the library was
410
+ // defined)
411
+ ska::flat_hash_map<std::string, std::string> libraries_;
412
+
413
+ std::array<impl::AnnotatedKernel, num_runtime_entries>
414
+ backendFallbackKernels_;
415
+
416
+ std::unique_ptr<detail::RegistrationListenerList> listeners_;
417
+
418
+ // This condition variable gets notified whenever we add a new def/impl to the
419
+ // dispatch table. This is primarily used by multiply/torchdeploy, when
420
+ // we have multiple interpreters trying to register to the dispatch table.
421
+ // In this situation, whenever the non-primary interpreter would have tried
422
+ // to register to the dispatch table, instead it will check to see if the
423
+ // expected registration has already been made, and if it hasn't, wait on
424
+ // this condition variable to see if it was just racing with the primary
425
+ // interpreter.
426
+ //
427
+ // We expect it to be rare for there to be any waiters on this condition
428
+ // variable. This is mostly just to help give better diagnostics if
429
+ // something goes horribly wrong
430
+ std::condition_variable cond_var_;
431
+
432
+ // Protect concurrent access to the dispatcher. We store this in a
433
+ // `shared_ptr` as we return callbacks that call back into dispatcher methods,
434
+ // and we need to be able to handle and guard against the event when the
435
+ // `Dispatcher` has been destroyed before the callbacks fire.
436
+ std::shared_ptr<Guard> guard_;
437
+ };
438
+
439
+ /**
440
+ * This is a handle to an operator schema registered with the dispatcher.
441
+ * This handle can be used to register kernels with the dispatcher or
442
+ * to lookup a kernel for a certain set of arguments.
443
+ */
444
+ class TORCH_API OperatorHandle {
445
+ template <typename T>
446
+ friend struct std::hash;
447
+
448
+ public:
449
+ OperatorHandle(OperatorHandle&&) noexcept = default;
450
+ OperatorHandle& operator=(OperatorHandle&&) noexcept = default;
451
+ OperatorHandle(const OperatorHandle&) = default;
452
+ OperatorHandle& operator=(const OperatorHandle&) = default;
453
+ // NOLINTNEXTLINE(performance-trivially-destructible)
454
+ ~OperatorHandle();
455
+
456
+ const OperatorName& operator_name() const {
457
+ return operatorDef_->op.operator_name();
458
+ }
459
+
460
+ bool hasSchema() const {
461
+ return operatorDef_->op.hasSchema();
462
+ }
463
+
464
+ const FunctionSchema& schema() const {
465
+ return operatorDef_->op.schema();
466
+ }
467
+
468
+ const std::string& debug() const {
469
+ return operatorDef_->op.debug();
470
+ }
471
+
472
+ std::string dumpState() const {
473
+ return operatorDef_->op.dumpState();
474
+ }
475
+
476
+ bool hasKernelForDispatchKey(DispatchKey k) const {
477
+ return operatorDef_->op.hasKernelForDispatchKey(k);
478
+ }
479
+
480
+ bool isKernelFallthroughKernel(DispatchKey k) const {
481
+ return operatorDef_->op.kernelForDispatchKey(k).isFallthrough();
482
+ }
483
+
484
+ bool hasKernelForAnyDispatchKey(DispatchKeySet k) const {
485
+ return operatorDef_->op.hasKernelForAnyDispatchKey(k);
486
+ }
487
+
488
+ bool hasComputedKernelForDispatchKey(DispatchKey k) const {
489
+ return operatorDef_->op.hasComputedKernelForDispatchKey(k);
490
+ }
491
+
492
+ SafeKernelFunction getComputedKernelForDispatchKey(DispatchKey k) const {
493
+ return operatorDef_->op.getComputedKernelForDispatchKey(k);
494
+ }
495
+
496
+ std::string dumpComputedTable() const {
497
+ return operatorDef_->op.dumpComputedTable();
498
+ }
499
+
500
+ void checkInvariants() const {
501
+ operatorDef_->op.checkInvariants();
502
+ }
503
+
504
+ c10::ArrayRef<at::Tag> getTags() const {
505
+ return operatorDef_->op.getTags();
506
+ }
507
+
508
+ void setReportErrorCallback_(std::unique_ptr<c10::SafePyObject> callback) {
509
+ operatorDef_->op.setReportErrorCallback_(std::move(callback));
510
+ }
511
+
512
+ bool hasTag(const at::Tag& tag) const {
513
+ for (const auto& tag_ : getTags()) {
514
+ if (tag == tag_) {
515
+ return true;
516
+ }
517
+ }
518
+ return false;
519
+ }
520
+
521
+ template <class FuncType>
522
+ TypedOperatorHandle<FuncType> typed() const {
523
+ // NB: This assert is not 100% sound: you can retrieve a typed() operator
524
+ // handle prior to ANY C++ signature being registered on the operator
525
+ // and the check will say everything is OK (at which point you can then
526
+ // smuggle in a kernel that is typed incorrectly). For everything
527
+ // in core library this won't happen, because all the static registrations
528
+ // will be done by the time a typed() handle is acquired.
529
+ #if !defined C10_MOBILE
530
+ operatorDef_->op.assertSignatureIsCorrect<FuncType>();
531
+ if (fn_has_symint<FuncType>::value) {
532
+ operatorDef_->op.assertSignatureIsCorrect<
533
+ typename fn_remove_symint<FuncType>::type>();
534
+ }
535
+ #endif
536
+ return TypedOperatorHandle<FuncType>(operatorIterator_);
537
+ }
538
+
539
+ void callBoxed(Stack* stack) const {
540
+ c10::Dispatcher::singleton().callBoxed(*this, stack);
541
+ }
542
+
543
+ void callBoxed(Stack& stack) const {
544
+ callBoxed(&stack);
545
+ }
546
+
547
+ void callBoxedForDispatchKey(DispatchKey dk, Stack& stack) const {
548
+ c10::Dispatcher::singleton().callBoxedForDispatchKey(*this, dk, &stack);
549
+ }
550
+
551
+ void redispatchBoxed(DispatchKeySet ks, Stack* stack) const {
552
+ c10::Dispatcher::singleton().redispatchBoxed(*this, ks, stack);
553
+ }
554
+
555
+ template <typename F>
556
+ PyObject* getPythonOp(
557
+ c10::impl::PyInterpreter* self_interpreter,
558
+ F slow_accessor) const {
559
+ return operatorDef_->op.getPythonOp(self_interpreter, slow_accessor);
560
+ }
561
+
562
+ bool operator==(const OperatorHandle& other) const {
563
+ return operatorDef_ == other.operatorDef_;
564
+ }
565
+
566
+ bool operator!=(const OperatorHandle& other) const {
567
+ return operatorDef_ != other.operatorDef_;
568
+ }
569
+
570
+ private:
571
+ explicit OperatorHandle(
572
+ std::list<Dispatcher::OperatorDef>::iterator operatorIterator)
573
+ : operatorDef_(&*operatorIterator), operatorIterator_(operatorIterator) {}
574
+ friend class Dispatcher;
575
+ template <class>
576
+ friend class TypedOperatorHandle;
577
+
578
+ // Storing a direct pointer to the OperatorDef even though we
579
+ // already have the iterator saves an instruction in the critical
580
+ // dispatch path. The iterator is effectively a
581
+ // pointer-to-std::list-node, and (at least in libstdc++'s
582
+ // implementation) the element is at an offset 16 bytes from that,
583
+ // because the prev/next pointers come first in the list node
584
+ // struct. So, an add instruction would be necessary to convert from the
585
+ // iterator to an OperatorDef*.
586
+ Dispatcher::OperatorDef* operatorDef_;
587
+
588
+ // We need to store this iterator in order to make
589
+ // Dispatcher::cleanup() fast -- it runs a lot on program
590
+ // termination (and presumably library unloading).
591
+ std::list<Dispatcher::OperatorDef>::iterator operatorIterator_;
592
+ };
593
+
594
+ /**
595
+ * This is a handle to an operator schema registered with the dispatcher.
596
+ * It holds the same information as an OperatorHandle, but it is templated
597
+ * on the operator arguments and allows calling the operator in an
598
+ * unboxed way.
599
+ */
600
+ template <class FuncType>
601
+ class TypedOperatorHandle final {
602
+ static_assert(
603
+ guts::false_t<FuncType>(),
604
+ "FuncType in OperatorHandle::typed<FuncType> was not a valid function type");
605
+ };
606
+ template <class Return, class... Args>
607
+ class TypedOperatorHandle<Return(Args...)> final : public OperatorHandle {
608
+ public:
609
+ TypedOperatorHandle(TypedOperatorHandle&&) noexcept = default;
610
+ TypedOperatorHandle& operator=(TypedOperatorHandle&&) noexcept = default;
611
+ TypedOperatorHandle(const TypedOperatorHandle&) = default;
612
+ TypedOperatorHandle& operator=(const TypedOperatorHandle&) = default;
613
+
614
+ // See [Note: Argument forwarding in the dispatcher] for why Args doesn't use
615
+ // &&
616
+ C10_ALWAYS_INLINE Return call(Args... args) const {
617
+ return c10::Dispatcher::singleton().call<Return, Args...>(
618
+ *this, std::forward<Args>(args)...);
619
+ }
620
+
621
+ // See [Note: Argument forwarding in the dispatcher] for why Args doesn't use
622
+ // &&
623
+ C10_ALWAYS_INLINE Return
624
+ redispatch(DispatchKeySet currentDispatchKeySet, Args... args) const {
625
+ return c10::Dispatcher::singleton().redispatch<Return, Args...>(
626
+ *this, currentDispatchKeySet, std::forward<Args>(args)...);
627
+ }
628
+
629
+ private:
630
+ explicit TypedOperatorHandle(
631
+ std::list<Dispatcher::OperatorDef>::iterator operatorIterator)
632
+ : OperatorHandle(operatorIterator) {}
633
+ friend class OperatorHandle;
634
+ };
635
+
636
+ namespace detail {
637
+ template <class... Args>
638
+ inline void unused_arg_(const Args&... /*unused*/) {}
639
+
640
+ // CaptureKernelCall is intended to capture return values from Dispatcher
641
+ // unboxed kernel calls. A record function may request to get outputs from the
642
+ // kernel calls. For boxed kernels, it's straightforward, the returned values
643
+ // are in the stack object. The stack can be passed to record functions. For
644
+ // unboxed kernels, we need to handle different kinds of return values, cache
645
+ // them temporarily, then release the values for the actual function call
646
+ // return.
647
+ template <typename ReturnType>
648
+ struct CaptureKernelCall {
649
+ template <typename F, typename... Args>
650
+ CaptureKernelCall(
651
+ const F& kernel,
652
+ const TypedOperatorHandle<ReturnType(Args...)>& op,
653
+ const DispatchKeySet& dispatchKeySet,
654
+ Args&&... args)
655
+ // Calls the kernel and capture the result in output_.
656
+ : output_{kernel.template call<ReturnType, Args...>(
657
+ op,
658
+ dispatchKeySet,
659
+ std::forward<Args>(args)...)} {}
660
+ // Wraps the return values in a Stack.
661
+ Stack getOutputs() {
662
+ Stack stack;
663
+ impl::push_outputs<ReturnType, false>::copy(output_, &stack);
664
+ return stack;
665
+ }
666
+ // Since we are returning the output_, we don't expect the output_ to be used
667
+ // afterward. Copy elision and RVO do not apply to class data members. Using
668
+ // move semantic to avoid copies when possible.
669
+ ReturnType release() && {
670
+ return std::move(output_);
671
+ }
672
+
673
+ private:
674
+ ReturnType output_;
675
+ };
676
+
677
+ // Handle the lvalue reference differently since it should not be moved.
678
+ template <>
679
+ inline at::Tensor& CaptureKernelCall<at::Tensor&>::release() && {
680
+ return output_;
681
+ }
682
+
683
+ // Handle case where the kernel returns void.
684
+ template <>
685
+ struct CaptureKernelCall<void> {
686
+ template <typename F, typename... Args>
687
+ CaptureKernelCall(
688
+ const F& kernel,
689
+ const TypedOperatorHandle<void(Args...)>& op,
690
+ const DispatchKeySet& dispatchKeySet,
691
+ Args&&... args) {
692
+ // Calling the kernel and no need to capture void.
693
+ kernel.template call<void, Args...>(
694
+ op, dispatchKeySet, std::forward<Args>(args)...);
695
+ }
696
+ Stack getOutputs() {
697
+ return Stack();
698
+ }
699
+ void release() && {}
700
+ };
701
+
702
+ TORCH_API void _print_dispatch_trace(
703
+ const std::string& label,
704
+ const std::string& op_name,
705
+ const DispatchKeySet& dispatchKeySet);
706
+
707
+ } // namespace detail
708
+
709
+ // See [Note: Argument forwarding in the dispatcher] for why Args doesn't use &&
710
+ template <class Return, class... Args>
711
+ inline Return Dispatcher::callWithDispatchKeySlowPath(
712
+ const TypedOperatorHandle<Return(Args...)>& op,
713
+ at::StepCallbacks& stepCallbacks,
714
+ DispatchKeySet dispatchKeySet,
715
+ const KernelFunction& kernel,
716
+ Args... args) {
717
+ // If callbacks need inputs, we box the arguments and pass them to the guard.
718
+ // Note: For perf reasons we wouldn't want to prematurely box the arguments.
719
+ at::RecordFunction guard(std::move(stepCallbacks));
720
+ TORCH_INTERNAL_ASSERT_DEBUG_ONLY(op.operatorDef_->op.isObserved());
721
+ auto dispatchKey = dispatchKeySet.highestPriorityTypeId();
722
+ auto& schema = op.schema();
723
+ auto schema_ref = std::reference_wrapper<const FunctionSchema>(schema);
724
+ constexpr auto num_boxed_args = impl::boxed_size<Args...>();
725
+ if constexpr (num_boxed_args != 0) {
726
+ if (guard.needsInputs()) {
727
+ // If we used std::array<IValue, num_boxed_args> here, we would
728
+ // have to spend time default constructing the IValues in
729
+ // boxedArgs. aligned_storage has no such requirement.
730
+ // NOLINTNEXTLINE(*array*)
731
+ alignas(IValue) std::byte boxedArgs[num_boxed_args * sizeof(IValue)];
732
+ // For debugging only; could be removed (but the compiler will do
733
+ // that for us and it's nice to have the extra assurance of
734
+ // correctness from our debug builds).
735
+ IValue* boxedArgsPtr = reinterpret_cast<IValue*>(boxedArgs);
736
+ impl::boxArgsToStack(boxedArgsPtr, args...);
737
+ TORCH_INTERNAL_ASSERT_DEBUG_ONLY(
738
+ reinterpret_cast<std::byte*>(boxedArgsPtr) ==
739
+ boxedArgs + num_boxed_args * sizeof(IValue));
740
+ // I don't *think* we need std::launder here, because IValue has
741
+ // no subclasses and no const or reference fields.
742
+ runRecordFunction(
743
+ guard,
744
+ schema_ref,
745
+ dispatchKey,
746
+ dispatchKeySet,
747
+ c10::ArrayRef<const c10::IValue>(
748
+ reinterpret_cast<IValue*>(boxedArgs), num_boxed_args));
749
+ boxedArgsPtr = reinterpret_cast<IValue*>(boxedArgs);
750
+ for (size_t ii = 0; ii < num_boxed_args; ++ii) {
751
+ (boxedArgsPtr + ii)->~IValue();
752
+ }
753
+ } else {
754
+ runRecordFunction(guard, schema_ref, dispatchKey, dispatchKeySet);
755
+ }
756
+ } else {
757
+ runRecordFunction(guard, schema_ref, dispatchKey, dispatchKeySet);
758
+ }
759
+
760
+ if (C10_UNLIKELY(guard.needsOutputs())) {
761
+ // Calls the kernel and capture the output temporarily to pass to
762
+ // RecordFunction.
763
+ detail::CaptureKernelCall<Return> captureKernelCall(
764
+ kernel, op, dispatchKeySet, std::forward<Args>(args)...);
765
+ guard.setOutputs(captureKernelCall.getOutputs());
766
+ // Releases the captured output to return to caller.
767
+ return std::move(captureKernelCall).release();
768
+ }
769
+
770
+ // keeping the guard alive while executing the kernel
771
+ return kernel.template call<Return, Args...>(
772
+ op, dispatchKeySet, std::forward<Args>(args)...);
773
+ }
774
+
775
+ // See [Note: Argument forwarding in the dispatcher] for why Args doesn't use &&
776
+ template <class Return, class... Args>
777
+ C10_ALWAYS_INLINE_UNLESS_MOBILE Return Dispatcher::call(
778
+ const TypedOperatorHandle<Return(Args...)>& op,
779
+ Args... args) const {
780
+ auto dispatchKeySet =
781
+ op.operatorDef_->op.dispatchKeyExtractor()
782
+ .template getDispatchKeySetUnboxed<Args...>(args...);
783
+ #if defined(HAS_TORCH_SHOW_DISPATCH_TRACE) || !defined(NDEBUG)
784
+ DispatchTraceNestingGuard debug_guard;
785
+ if (show_dispatch_trace()) {
786
+ detail::_print_dispatch_trace(
787
+ "[call]", toString(op.operator_name()), dispatchKeySet);
788
+ }
789
+ #endif
790
+ const KernelFunction& kernel = op.operatorDef_->op.lookup(dispatchKeySet);
791
+ #ifndef PYTORCH_DISABLE_PER_OP_PROFILING
792
+ auto step_callbacks =
793
+ at::getStepCallbacksUnlessEmpty(at::RecordScope::FUNCTION);
794
+ if (C10_UNLIKELY(
795
+ step_callbacks.has_value() && op.operatorDef_->op.isObserved())) {
796
+ return callWithDispatchKeySlowPath<Return, Args...>(
797
+ op,
798
+ *step_callbacks,
799
+ dispatchKeySet,
800
+ kernel,
801
+ std::forward<Args>(args)...);
802
+ }
803
+ #endif // PYTORCH_DISABLE_PER_OP_PROFILING
804
+
805
+ #ifdef FBCODE_CAFFE2
806
+ if (profilingOperatorEvents()) {
807
+ std::vector<void*> argsAddresses = {(void*)(&args)...};
808
+ std::vector<const char*> argsTypes = {(typeid(args).name())...};
809
+ struct FireOpRAII {
810
+ FireOpRAII(
811
+ at::RecordFunction::schema_ref_t schema_ref,
812
+ std::vector<void*>& argsAddresses,
813
+ std::vector<const char*>& argsTypes)
814
+ : schema_ref_(schema_ref) {
815
+ fireOpStartUSDT(schema_ref, argsAddresses, argsTypes);
816
+ }
817
+ ~FireOpRAII() {
818
+ fireOpEndUSDT(schema_ref_);
819
+ }
820
+ at::RecordFunction::schema_ref_t schema_ref_;
821
+ } event(op.schema(), argsAddresses, argsTypes);
822
+ return kernel.template call<Return, Args...>(
823
+ op, dispatchKeySet, std::forward<Args>(args)...);
824
+ } else {
825
+ return kernel.template call<Return, Args...>(
826
+ op, dispatchKeySet, std::forward<Args>(args)...);
827
+ }
828
+ #else
829
+ return kernel.template call<Return, Args...>(
830
+ op, dispatchKeySet, std::forward<Args>(args)...);
831
+ #endif // FBCODE_CAFFE2
832
+ }
833
+
834
+ // See [Note: Argument forwarding in the dispatcher] for why Args doesn't use &&
835
+ template <class Return, class... Args>
836
+ inline Return Dispatcher::redispatch(
837
+ const TypedOperatorHandle<Return(Args...)>& op,
838
+ DispatchKeySet currentDispatchKeySet,
839
+ Args... args) const {
840
+ // do not use RecordFunction on redispatch
841
+ #if defined(HAS_TORCH_SHOW_DISPATCH_TRACE) || !defined(NDEBUG)
842
+ DispatchTraceNestingGuard debug_guard;
843
+ if (show_dispatch_trace()) {
844
+ detail::_print_dispatch_trace(
845
+ "[redispatch]", toString(op.operator_name()), currentDispatchKeySet);
846
+ }
847
+ #endif
848
+ const KernelFunction& kernel =
849
+ op.operatorDef_->op.lookup(currentDispatchKeySet);
850
+ return kernel.template call<Return, Args...>(
851
+ op, currentDispatchKeySet, std::forward<Args>(args)...);
852
+ }
853
+
854
+ inline void Dispatcher::callBoxed(const OperatorHandle& op, Stack* stack)
855
+ const {
856
+ // note: this doesn't need the mutex because write operations on the list keep
857
+ // iterators intact.
858
+ const auto& entry = op.operatorDef_->op;
859
+ auto dispatchKeySet =
860
+ entry.dispatchKeyExtractor().getDispatchKeySetBoxed(stack);
861
+ #if defined(HAS_TORCH_SHOW_DISPATCH_TRACE) || !defined(NDEBUG)
862
+ DispatchTraceNestingGuard debug_guard;
863
+ if (show_dispatch_trace()) {
864
+ detail::_print_dispatch_trace(
865
+ "[callBoxed]", toString(op.operator_name()), dispatchKeySet);
866
+ }
867
+ #endif
868
+ const auto& kernel = entry.lookup(dispatchKeySet);
869
+ #ifndef PYTORCH_DISABLE_PER_OP_PROFILING
870
+ auto step_callbacks =
871
+ at::getStepCallbacksUnlessEmpty(at::RecordScope::FUNCTION);
872
+ if (C10_UNLIKELY(step_callbacks.has_value() && entry.isObserved())) {
873
+ at::RecordFunction guard(std::move(*step_callbacks));
874
+ auto dispatchKey = dispatchKeySet.highestPriorityTypeId();
875
+ auto& schema = op.schema();
876
+ auto schema_ref = std::reference_wrapper<const FunctionSchema>(schema);
877
+ guard.needsInputs()
878
+ ? runRecordFunction(
879
+ guard,
880
+ schema_ref,
881
+ dispatchKey,
882
+ dispatchKeySet,
883
+ c10::ArrayRef<const c10::IValue>(stack->data(), stack->size()))
884
+ : runRecordFunction(guard, schema_ref, dispatchKey, dispatchKeySet);
885
+
886
+ // keeping the guard alive while executing the kernel
887
+ kernel.callBoxed(op, dispatchKeySet, stack);
888
+
889
+ if (C10_UNLIKELY(guard.needsOutputs())) {
890
+ guard.setOutputs(*stack);
891
+ }
892
+ return;
893
+ }
894
+ #endif // PYTORCH_DISABLE_PER_OP_PROFILING
895
+ kernel.callBoxed(op, dispatchKeySet, stack);
896
+ }
897
+
898
+ // NB: this doesn't count as a "true" dispatcher jump, so no instrumentation
899
+ inline void Dispatcher::callBoxedForDispatchKey(
900
+ const OperatorHandle& op,
901
+ DispatchKey dk,
902
+ Stack* stack) const {
903
+ // note: this doesn't need the mutex because write operations on the list keep
904
+ // iterators intact.
905
+ const auto& entry = op.operatorDef_->op;
906
+ // We still compute this as we're obligated to pass it on to the internal
907
+ // kernel, if it is a boxed fallback
908
+ auto dispatchKeySet =
909
+ entry.dispatchKeyExtractor().getDispatchKeySetBoxed(stack);
910
+ const auto& kernel = ([&]() {
911
+ if (op.hasKernelForDispatchKey(dk)) {
912
+ return entry.kernelForDispatchKey(dk);
913
+ } else {
914
+ auto idx = getDispatchTableIndexForDispatchKey(dk);
915
+ TORCH_INTERNAL_ASSERT(idx >= 0);
916
+ return backendFallbackKernels_[idx].kernel;
917
+ }
918
+ })();
919
+ kernel.callBoxed(op, dispatchKeySet, stack);
920
+ }
921
+
922
+ inline void Dispatcher::redispatchBoxed(
923
+ const OperatorHandle& op,
924
+ DispatchKeySet dispatchKeySet,
925
+ Stack* stack) const {
926
+ // note: this doesn't need the mutex because write operations on the list keep
927
+ // iterators intact.
928
+ const auto& entry = op.operatorDef_->op;
929
+ #if defined(HAS_TORCH_SHOW_DISPATCH_TRACE) || !defined(NDEBUG)
930
+ DispatchTraceNestingGuard debug_guard;
931
+ if (show_dispatch_trace()) {
932
+ detail::_print_dispatch_trace(
933
+ "[redispatchBoxed]", toString(op.operator_name()), dispatchKeySet);
934
+ }
935
+ #endif
936
+ const auto& kernel = entry.lookup(dispatchKeySet);
937
+ kernel.callBoxed(op, dispatchKeySet, stack);
938
+ }
939
+
940
+ } // namespace c10
941
+
942
+ namespace std {
943
+
944
+ template <>
945
+ struct hash<c10::OperatorHandle> {
946
+ size_t operator()(const c10::OperatorHandle& op) const noexcept {
947
+ return std::hash<void*>{}(static_cast<void*>(op.operatorDef_));
948
+ }
949
+ };
950
+
951
+ } // namespace std
952
+
953
+ #else
954
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
955
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/ObservedOperators.h ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <ATen/core/operator_name.h>
5
+ #include <string>
6
+ #include <unordered_set>
7
+
8
+ namespace c10 {
9
+
10
+ struct TORCH_API ObservedOperators {
11
+ ObservedOperators() = delete;
12
+
13
+ static bool isObserved(const OperatorName& name);
14
+
15
+ static std::unordered_set<std::string>& getUnobservedOperatorList();
16
+ };
17
+
18
+ } // namespace c10
19
+
20
+ #else
21
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
22
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/OperatorEntry.h ADDED
@@ -0,0 +1,342 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <ATen/core/boxing/KernelFunction.h>
5
+ #include <ATen/core/dispatch/DispatchKeyExtractor.h>
6
+ #include <ATen/core/function_schema.h>
7
+ #include <ATen/core/ivalue.h>
8
+ #include <c10/core/DispatchKey.h>
9
+ #include <c10/core/PyHandleCache.h>
10
+ #include <c10/core/SafePyObject.h>
11
+ #include <c10/util/Metaprogramming.h>
12
+ #include <c10/util/flat_hash_map.h>
13
+
14
+ #include <ATen/core/dispatch/CppSignature.h>
15
+ #include <ATen/core/dispatch/OperatorOptions.h>
16
+ #include <ATen/core/dispatch/RegistrationHandleRAII.h>
17
+ #include <ATen/core/enum_tag.h>
18
+
19
+ #include <array>
20
+ #include <list>
21
+ #include <optional>
22
+
23
+ #ifdef C10_MOBILE
24
+ #define C10_DISPATCHER_ONE_KERNEL_PER_DISPATCH_KEY
25
+ #endif
26
+
27
+ namespace c10 {
28
+
29
+ class Dispatcher;
30
+
31
+ namespace impl {
32
+
33
+ // This data structure represents a kernel that was registered to us from a
34
+ // user. Unlike KernelFunction, AnnotatedKernel contains some extra metadata
35
+ // about the kernel that isn't necessary for actual dispatching (this is why
36
+ // we don't put AnnotatedKernel in the actual DispatchTable), but is useful for
37
+ // giving good error messages.
38
+ struct AnnotatedKernel final {
39
+ AnnotatedKernel(
40
+ KernelFunction k,
41
+ std::unique_ptr<FunctionSchema> s,
42
+ std::string d)
43
+ : kernel(std::move(k)),
44
+ inferred_function_schema(std::move(s)),
45
+ debug(std::move(d)) {}
46
+ AnnotatedKernel() = default;
47
+ KernelFunction kernel;
48
+ std::unique_ptr<FunctionSchema> inferred_function_schema;
49
+ // A little debug string to help us identify the kernel in question.
50
+ // Most importantly it records the TORCH_LIBRARY block that did the
51
+ // registration.
52
+ std::string debug;
53
+ };
54
+
55
+ // This data structure represents operator schema, with metadata specifying
56
+ // where the registration of this schema occurred
57
+ struct AnnotatedSchema final {
58
+ AnnotatedSchema(FunctionSchema s, std::string d)
59
+ : schema(std::move(s)), debug(std::move(d)) {}
60
+ FunctionSchema schema;
61
+ std::string debug;
62
+ };
63
+
64
+ // Internal data structure that records information about a specific operator.
65
+ // It's not part of the public API; typically, users will interact with
66
+ // OperatorHandle instead.
67
+ //
68
+ // Concurrent writes to OperatorEntry are protected by the GLOBAL Dispatcher
69
+ // lock (this is important because some methods in OperatorEntry access
70
+ // dispatcher state)
71
+ class TORCH_API OperatorEntry final {
72
+ public:
73
+ explicit OperatorEntry(OperatorName&& operator_name);
74
+
75
+ OperatorEntry(const OperatorEntry&) = delete;
76
+ OperatorEntry(OperatorEntry&&) noexcept = delete;
77
+ OperatorEntry& operator=(const OperatorEntry&) = delete;
78
+ OperatorEntry& operator=(OperatorEntry&&) noexcept = delete;
79
+
80
+ const FunctionSchema& schema() const {
81
+ TORCH_INTERNAL_ASSERT(
82
+ schema_.has_value(),
83
+ "Tried to access the schema for ",
84
+ name_,
85
+ " which doesn't have a schema registered yet");
86
+ return schema_->schema;
87
+ }
88
+ const std::string& debug() const {
89
+ TORCH_INTERNAL_ASSERT(schema_.has_value());
90
+ return schema_->debug;
91
+ }
92
+ bool hasSchema() const {
93
+ return schema_.has_value();
94
+ }
95
+
96
+ bool isObserved() const {
97
+ return is_observed_;
98
+ }
99
+
100
+ // We may allocate an OperatorEntry for an operator even when we don't
101
+ // have a schema. When we receive the schema registration, we post
102
+ // facto register a schema.
103
+ //
104
+ // NB: registerSchema/deregisterSchema are not idempotent; if you
105
+ // attempt to register a schema when one is already present or vice
106
+ // versa that is an error. (Refcounting for the registrations is
107
+ // handled in the OperatorHandle in Dispatcher)
108
+ void registerSchema(
109
+ FunctionSchema&& /*schema*/,
110
+ std::string&& debug,
111
+ std::vector<at::Tag> tags = {});
112
+ void deregisterSchema();
113
+
114
+ const OperatorName& operator_name() const {
115
+ return name_;
116
+ }
117
+
118
+ #ifdef C10_DISPATCHER_ONE_KERNEL_PER_DISPATCH_KEY
119
+ using AnnotatedKernelContainer = std::array<AnnotatedKernel, 1>;
120
+ #else
121
+ using AnnotatedKernelContainer = std::list<AnnotatedKernel>;
122
+ #endif
123
+ using AnnotatedKernelContainerIterator = AnnotatedKernelContainer::iterator;
124
+
125
+ // Why are kernels and fallback asymmetric? It has to do with ownership.
126
+ // Kernels and the computed dispatch tables for them are canonically
127
+ // owned by OperatorEntry, but backend fallbacks are specified once
128
+ // and apply for all operators, so they should be owned by Dispatcher.
129
+ // However, the registration of a backend fallback affects the
130
+ // state of the computed dispatch table, so when a backend fallback
131
+ // is updated, we need to update the operator tables too. Thus,
132
+ // registerKernel is the mechanism by which we give kernels to
133
+ // operator entry to own (and update dispatch table), but we only
134
+ // need a non-owning mechanism to update fallback.
135
+
136
+ // Precondition: Dispatcher::mutex_ is held
137
+ // Postcondition: caller is responsible for disposing of the kernel
138
+ AnnotatedKernelContainerIterator registerKernel(
139
+ const Dispatcher& dispatcher,
140
+ std::optional<DispatchKey> dispatch_key,
141
+ KernelFunction kernel,
142
+ std::optional<CppSignature> cpp_signature,
143
+ std::unique_ptr<FunctionSchema> inferred_function_schema,
144
+ std::string debug);
145
+
146
+ // Precondition: Dispatcher::mutex_ is held
147
+ void deregisterKernel_(
148
+ const Dispatcher& dispatcher,
149
+ std::optional<DispatchKey> dispatch_key,
150
+ AnnotatedKernelContainerIterator kernel);
151
+
152
+ // Precondition: Dispatcher::mutex_ is held
153
+ void updateFallback(const Dispatcher& dispatcher, DispatchKey dispatch_key);
154
+
155
+ // Precondition: Dispatcher::mutex_ is held
156
+ void updateSchemaAliasAnalysis(AliasAnalysisKind a) {
157
+ TORCH_INTERNAL_ASSERT(schema_.has_value());
158
+ schema_->schema.setAliasAnalysis(a);
159
+ }
160
+
161
+ std::string dumpComputedTable() const;
162
+ std::string dumpState() const;
163
+ void checkInvariants() const;
164
+
165
+ const DispatchKeyExtractor& dispatchKeyExtractor() const {
166
+ return dispatchKeyExtractor_;
167
+ }
168
+
169
+ // Asserts that the given FuncType is correct for calling this operator in an
170
+ // unboxed way.
171
+ template <class FuncType>
172
+ inline void assertSignatureIsCorrect() {
173
+ assertSignatureIsCorrect(
174
+ CppSignature::make<FuncType>(), fn_has_symint<FuncType>::value);
175
+ }
176
+
177
+ void assertSignatureIsCorrect(
178
+ const CppSignature& call_signature,
179
+ bool has_symint) const;
180
+
181
+ [[noreturn]] void reportError(DispatchKey dispatchKey) const;
182
+
183
+ const KernelFunction& lookup(DispatchKeySet ks) const {
184
+ const auto idx = ks.getDispatchTableIndexForDispatchKeySet();
185
+ if (C10_UNLIKELY(idx == -1)) {
186
+ reportError(ks.highestPriorityTypeId());
187
+ }
188
+ const auto& kernel = dispatchTable_[idx];
189
+ // A valid kernel *always* has a boxed kernel and *may* have an
190
+ // unboxed kernel. However, we typically do unboxed calls in at::
191
+ // APIs, where the kernel 1) will very likely be valid and 2)
192
+ // should have an unboxed kernel. Checking the unboxed kernel
193
+ // first will allow us to avoid touching the boxed kernel at all
194
+ // in the common case.
195
+ if (C10_UNLIKELY(!kernel.isValidUnboxed())) {
196
+ if (!kernel.isValid()) {
197
+ reportError(ks.highestPriorityTypeId());
198
+ }
199
+ }
200
+ return kernel;
201
+ }
202
+
203
+ std::string listAllDispatchKeys() const;
204
+
205
+ // Returns true if kernel_ has entry for any key in ks.
206
+ //
207
+ // Invariant: There are no alias keys in the passed-in dispatch key set.
208
+ // Note [No Alias Keys in DispatchKeySet]
209
+ // Alias keys should be checked using `hasKernelForDispatchKey`
210
+ // Alias keys shouldn't go inside of a DispatchKeySet, since they can
211
+ // technically have a value > 63 (causing overflow).
212
+ bool hasKernelForAnyDispatchKey(DispatchKeySet ks) const;
213
+ // Returns true if kernel_ has entry for a particular key.
214
+ bool hasKernelForDispatchKey(DispatchKey k) const;
215
+ // Retrieves the kernel entry at a particular key. Symmetric with
216
+ // hasKernelForDispatchKey. To get the AnnotatedKernel, see
217
+ // getKernelForDispatchKey (private)
218
+ const KernelFunction& kernelForDispatchKey(DispatchKey k) const;
219
+ // Returns true if the "computed table" has an entry for a particular key.
220
+ bool hasComputedKernelForDispatchKey(DispatchKey k) const;
221
+ // Returns a KernelFunction corresponding to the kernel in dispatchTable
222
+ SafeKernelFunction getComputedKernelForDispatchKey(DispatchKey k) const;
223
+ // Returns all the operator tags added at the time of registration
224
+ const std::vector<at::Tag>& getTags() const;
225
+ void setReportErrorCallback_(std::unique_ptr<c10::SafePyObject> callback);
226
+
227
+ template <typename F>
228
+ PyObject* getPythonOp(PyInterpreter* self_interpreter, F slow_accessor)
229
+ const {
230
+ return py_cache_.ptr_or(self_interpreter, slow_accessor);
231
+ }
232
+
233
+ private:
234
+ OperatorName name_;
235
+ std::optional<AnnotatedSchema> schema_;
236
+ #ifndef C10_MOBILE
237
+ std::vector<at::Tag> tags_;
238
+ #endif
239
+ std::array<KernelFunction, c10::num_runtime_entries> dispatchTable_;
240
+ DispatchKeyExtractor dispatchKeyExtractor_;
241
+ // Pointer to the torch.ops.ns.op.overload object for speed
242
+ c10::PyHandleCache py_cache_;
243
+
244
+ // kernels_ stores all registered kernels for the corresponding dispatch key
245
+ // and catchAllKernels_ stores the catch-all kernels.
246
+ // If an operator library gets loaded that overwrites an already existing
247
+ // kernel, both kernels will be in that list but only the newer one will be in
248
+ // dispatchTable. If any of the kernels go away (say the library gets
249
+ // unloaded), we remove the kernel from this list and update the
250
+ // dispatchTable if necessary.
251
+ // Kernels in the list are ordered by registration time descendingly,
252
+ // newer registrations are before older registrations.
253
+ // We do not combine dispatchTable and kernels into one hash map because
254
+ // kernels is a larger data structure and accessed quite infrequently
255
+ // while dispatchTable is accessed often and should be kept small to fit
256
+ // into CPU caches.
257
+ // Invariants:
258
+ // - dispatchTable[dispatch_key] == kernels_[dispatch_key].front()
259
+ // - dispatchTable[dispatch_key] does not exist if and only if
260
+ // kernels_[dispatch_key] does not exist
261
+ // - If kernels_[dispatch_key] exists, then it has elements.
262
+ // It is never an empty list.
263
+ //
264
+ // Why do we do that?
265
+ // -----
266
+ // We mostly do this to enable Jupyter notebooks where a cell registering
267
+ // a kernel could be executed multiple times and the later execution
268
+ // should overwrite the earlier one. Note that this still fails when the
269
+ // function schema changed between the executions, but it works as long
270
+ // as the function schema didn't change. A better solution would be to
271
+ // unload the old extension library from the Jupyter cell when the cell is
272
+ // re-executed and then only allow one kernel here, i.e. error if a kernel
273
+ // is already registered, but that's a lot of effort to implement and
274
+ // currently not high-pri.
275
+ ska::flat_hash_map<
276
+ DispatchKey,
277
+ #ifdef C10_DISPATCHER_ONE_KERNEL_PER_DISPATCH_KEY
278
+ // On mobile, we needn't worry about Jupyter notebooks.
279
+ std::array<AnnotatedKernel, 1>
280
+ #else
281
+ std::list<AnnotatedKernel>
282
+ #endif
283
+ >
284
+ kernels_;
285
+
286
+ const AnnotatedKernel& missingKernel() const;
287
+ const AnnotatedKernel& ambiguousAutogradOtherKernel() const;
288
+
289
+ // cpp_signature_ stores function signature if any of
290
+ // the kernels was created in a way that allowed us to know the function
291
+ // signature (i.e. by supplying an unboxed C++ kernel function).
292
+ // If this is set, it will be used to check that future kernel
293
+ // registrations match and it will be used in unboxed function calls
294
+ // to verify their arguments against the known function signature.
295
+ struct CppSignatureWithDebug {
296
+ CppSignature signature;
297
+ std::string debug;
298
+ std::optional<DispatchKey> dispatch_key;
299
+ };
300
+ std::optional<CppSignatureWithDebug> cpp_signature_;
301
+ std::optional<CppSignatureWithDebug> sym_cpp_signature_;
302
+
303
+ // A Python custom error handler for OperatorEntry::reportError
304
+ std::unique_ptr<c10::SafePyObject> report_error_callback_;
305
+
306
+ // Whether this operator needs to be observed with RecordFunction
307
+ const bool is_observed_;
308
+
309
+ [[noreturn]] void reportSignatureError(
310
+ const CppSignature& call_signature,
311
+ const CppSignatureWithDebug& saved_signature) const;
312
+ const KernelFunction& computeDispatchTableEntry(
313
+ const c10::Dispatcher& dispatcher,
314
+ DispatchKey dispatch_key) const;
315
+ std::pair<const AnnotatedKernel&, const char*>
316
+ computeDispatchTableEntryWithDebug(
317
+ const c10::Dispatcher& dispatcher,
318
+ DispatchKey dispatch_key) const;
319
+ // This function re-establishes the invariant that dispatchTable
320
+ // contains the front element from the kernels list for a given runtime
321
+ // dispatch key.
322
+ void updateDispatchTableEntry_(
323
+ const c10::Dispatcher& dispatcher,
324
+ DispatchKey dispatch_key);
325
+ // Like above, but also handles alias dispatch keys.
326
+ void updateDispatchTable_(
327
+ const c10::Dispatcher& dispatcher,
328
+ DispatchKey dispatch_key);
329
+ // Like above, but for ALL entries in the dispatch table.
330
+ void updateDispatchTableFull_(const c10::Dispatcher& dispatcher);
331
+ // Retrieves a pointer to AnnotatedKernel at
332
+ // kernels_.at(dispatch_key).front().
333
+ const AnnotatedKernel* getKernelForDispatchKey(
334
+ DispatchKey dispatch_key) const;
335
+ };
336
+
337
+ } // namespace impl
338
+ } // namespace c10
339
+
340
+ #else
341
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
342
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/OperatorOptions.h ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <cstdint>
5
+
6
+ namespace c10 {
7
+
8
+ enum class AliasAnalysisKind : uint8_t {
9
+ INTERNAL_SPECIAL_CASE,
10
+ CONSERVATIVE, // The most conservative alias analysis type, assumes
11
+ // side-effects. This is the default analysis.
12
+ FROM_SCHEMA,
13
+ PURE_FUNCTION
14
+ };
15
+
16
+ #if !defined(_MSC_VER)
17
+ constexpr // Our current MSVC version has a bug that doesn't allow this to be
18
+ // constexpr.
19
+ #endif
20
+ inline const char*
21
+ toString(AliasAnalysisKind aliasAnalysisKind) {
22
+ return (aliasAnalysisKind == AliasAnalysisKind::CONSERVATIVE) ? "CONSERVATIVE"
23
+ : (aliasAnalysisKind == AliasAnalysisKind::FROM_SCHEMA) ? "FROM_SCHEMA"
24
+ : (aliasAnalysisKind == AliasAnalysisKind::PURE_FUNCTION)
25
+ ? "PURE_FUNCTION"
26
+ : (aliasAnalysisKind == AliasAnalysisKind::INTERNAL_SPECIAL_CASE)
27
+ ? "INTERNAL_SPECIAL_CASE"
28
+ : "UNKNOWN";
29
+ }
30
+
31
+ } // namespace c10
32
+
33
+ #else
34
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
35
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/dispatch/RegistrationHandleRAII.h ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <functional>
5
+
6
+ namespace c10 {
7
+
8
+ class RegistrationHandleRAII final {
9
+ public:
10
+ explicit RegistrationHandleRAII(std::function<void()> onDestruction)
11
+ : onDestruction_(std::move(onDestruction)) {}
12
+
13
+ ~RegistrationHandleRAII() {
14
+ if (onDestruction_) {
15
+ onDestruction_();
16
+ }
17
+ }
18
+
19
+ RegistrationHandleRAII(const RegistrationHandleRAII&) = delete;
20
+ RegistrationHandleRAII& operator=(const RegistrationHandleRAII&) = delete;
21
+
22
+ RegistrationHandleRAII(RegistrationHandleRAII&& rhs) noexcept
23
+ : onDestruction_(std::move(rhs.onDestruction_)) {
24
+ rhs.onDestruction_ = nullptr;
25
+ }
26
+
27
+ RegistrationHandleRAII& operator=(RegistrationHandleRAII&& rhs) noexcept {
28
+ onDestruction_ = std::move(rhs.onDestruction_);
29
+ rhs.onDestruction_ = nullptr;
30
+ return *this;
31
+ }
32
+
33
+ private:
34
+ std::function<void()> onDestruction_;
35
+ };
36
+
37
+ } // namespace c10
38
+
39
+ #else
40
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
41
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/op_registration/adaption.h ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <ATen/Tensor.h>
5
+ #include <ATen/TensorUtils.h>
6
+ #include <ATen/core/List.h>
7
+ #include <c10/core/TensorOptions.h>
8
+
9
+ /*
10
+ * [Note: hacky wrapper removal for optional tensor]
11
+ *
12
+ * The kernel implementation takes an optional tensor marked in the schema as
13
+ * Tensor? but the C++ function takes Tensor instead of the std::optional<Tensor>
14
+ * expected by the dispatcher.
15
+ *
16
+ * To remove the hacky wrapper, the C++ function is changed to take
17
+ * std::optional<Tensor> and unwrap the Tensor value at the beginning of
18
+ * the function, e.g.:
19
+ * > c10::MaybeOwned<Tensor> weight_maybe_owned =
20
+ * > at::borrow_from_optional_tensor(weight_opt);
21
+ * > const Tensor& weight = *weight_maybe_owned;
22
+ *
23
+ * We may want to make the kernel handle optional directly without
24
+ * going through the creation of a default-constructed Tensor in
25
+ * at::borrow_from_optional_tensor.
26
+ */
27
+
28
+ /*
29
+ * [Note: hacky wrapper removal for TensorOptions]
30
+ *
31
+ * The kernel implementation takes a TensorOptions argument but the dispatcher
32
+ * expects separate arguments for dtype, layout, device, pin_memory.
33
+ *
34
+ * To remove the hacky wrapper, the kernel implementation is changed to take
35
+ * the 4 arguments (dtype, layout, device, pin_memory), and assemble the
36
+ * TensorOptions value at the beginning of the function, e.g.:
37
+ * > TensorOptions options = TensorOptions().dtype(dtype).layout(layout)
38
+ * > .device(device).pinned_memory(pin_memory);
39
+ *
40
+ * We may want make the kernel handle these parameters directly without going
41
+ * through the creation of a TensorOptions value.
42
+ */
43
+
44
+ namespace c10::impl {
45
+
46
+ TORCH_API void common_device_check_failure(Device common_device, const at::Tensor& tensor, at::CheckedFrom methodName, at::CheckedFrom argName);
47
+
48
+ inline void check_and_update_common_device(std::optional<Device>& common_device, const at::Tensor& tensor, at::CheckedFrom methodName, at::CheckedFrom argName) {
49
+ // TODO: Remove this once the following issue is addressed:
50
+ // https://github.com/pytorch/pytorch/issues/57380
51
+ if (!tensor.defined()) {
52
+ return;
53
+ }
54
+
55
+ if (!common_device.has_value()) {
56
+ common_device = tensor.device();
57
+ return;
58
+ }
59
+
60
+ if (C10_UNLIKELY(common_device != tensor.device())) {
61
+ common_device_check_failure(*common_device, tensor, methodName, argName);
62
+ }
63
+ }
64
+
65
+ inline void check_and_update_common_device(std::optional<Device>& common_device, const std::optional<at::Tensor>& tensor, at::CheckedFrom methodName, at::CheckedFrom argName) {
66
+ if (tensor.has_value()) {
67
+ check_and_update_common_device(common_device, tensor.value(), methodName, argName);
68
+ }
69
+ }
70
+
71
+ inline void check_and_update_common_device(std::optional<Device>& common_device, at::ITensorListRef tensors, at::CheckedFrom methodName, at::CheckedFrom argName) {
72
+ for (const auto& tensor : tensors) {
73
+ check_and_update_common_device(common_device, tensor, methodName, argName);
74
+ }
75
+ }
76
+
77
+ inline void check_and_update_common_device(std::optional<Device>& common_device, const List<std::optional<at::Tensor>>& tensors, at::CheckedFrom methodName, at::CheckedFrom argName) {
78
+ for (const auto& tensor : tensors) {
79
+ check_and_update_common_device(common_device, tensor, methodName, argName);
80
+ }
81
+ }
82
+ } // namespace c10::impl
83
+
84
+ #else
85
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
86
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/op_registration/infer_schema.h ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ /**
5
+ * This file contains functionality to take a C++ function and infer its
6
+ * c10::FunctionSchema.
7
+ */
8
+
9
+ #include <ATen/core/function_schema.h>
10
+ #include <c10/util/Metaprogramming.h>
11
+
12
+ namespace c10 {
13
+ namespace detail::infer_schema {
14
+
15
+ /// The templated inference code creates `ArgumentDef` instead of `Argument`,
16
+ /// because that can be constructed at compile time and has a much smaller
17
+ /// binary size than having calls to `Argument` constructors in the template.
18
+ /// Creating `Argument` objects from `ArgumentDef` can then be done at
19
+ /// runtime in a non-templated way.
20
+ struct ArgumentDef final {
21
+ using GetTypeFn = TypePtr();
22
+ GetTypeFn* getTypeFn;
23
+ GetTypeFn* getFakeTypeFn;
24
+ constexpr ArgumentDef(): getTypeFn(nullptr), getFakeTypeFn(nullptr) {}
25
+ explicit constexpr ArgumentDef(GetTypeFn *getTypeFn, GetTypeFn *getFakeTypeFn): getTypeFn(getTypeFn), getFakeTypeFn(getFakeTypeFn) {}
26
+ };
27
+
28
+ template<bool V>
29
+ struct bool_t {};
30
+ template<> struct bool_t<true> : std::true_type {};
31
+ template<> struct bool_t<false> : std::false_type {};
32
+
33
+ /// Checks the static C++ types `Types` for correctness to catch common error cases.
34
+ template <class... Types>
35
+ constexpr int checkStaticTypes() {
36
+ // Give nice error messages for some of the common error cases.
37
+ // Use a LOUD ERROR MESSAGE SO USERS SEE THE STATIC_ASSERT
38
+ static_assert(std::conjunction_v<
39
+ bool_t<!std::is_integral_v<Types> || std::is_same_v<Types, int8_t> || std::is_same_v<Types, int64_t> || std::is_same_v<Types, bool>>...
40
+ >, "INVALID TYPE: Only int8_t, int64_t and bool are supported as an integral argument type");
41
+ static_assert(std::conjunction_v<
42
+ bool_t<!std::is_same_v<Types, float>>...
43
+ >, "INVALID TYPE: float is not supported as an argument type, use double instead");
44
+ return 0;
45
+ }
46
+
47
+ template <typename... Ts, size_t... Is>
48
+ constexpr std::array<ArgumentDef, sizeof...(Ts)> createArgumentVectorFromTypes(std::index_sequence<Is...> /*unused*/) {
49
+ return (
50
+ // Check types for common errors
51
+ checkStaticTypes<Ts...>(),
52
+
53
+ // Create the return value
54
+ std::array<ArgumentDef, sizeof...(Ts)>{
55
+ ArgumentDef(&getTypePtrCopy<std::decay_t<Ts>>, &getFakeTypePtrCopy<std::decay_t<Ts>>)...}
56
+ );
57
+ }
58
+
59
+ /// Creates a vector of `ArgumentDef` from a list of C++ types that are specified
60
+ /// as template arguments.
61
+ template<class ParameterTypes> struct createArguments final {};
62
+ template<class... ParameterTypes>
63
+ struct createArguments<guts::typelist::typelist<ParameterTypes...>> final {
64
+ static constexpr std::array<ArgumentDef, sizeof...(ParameterTypes)> call() {
65
+ return createArgumentVectorFromTypes<ParameterTypes...>(
66
+ std::make_index_sequence<sizeof...(ParameterTypes)>()
67
+ );
68
+ }
69
+ };
70
+
71
+ /// Creates a vector of `ArgumentDef` from a list of C++ types that are specified
72
+ /// as a tuple (i.e. in the way c10 kernels return values).
73
+ /// It can be a tuple<A, B, C> if there's three output arguments with types A, B, C.
74
+ /// It can be an empty tuple<>, or void for kernels that don't return anything.
75
+ /// It can be a single type A (i.e. no tuple) for the case where a kernel just
76
+ /// returns one value.
77
+ template<class ReturnTypeTuple, class Enable = void> struct createReturns final {};
78
+
79
+ template<class... ReturnTypes>
80
+ struct createReturns<std::tuple<ReturnTypes...>, void> final {
81
+ static constexpr std::array<ArgumentDef, sizeof...(ReturnTypes)> call() {
82
+ return createArgumentVectorFromTypes<ReturnTypes...>(
83
+ std::make_index_sequence<sizeof...(ReturnTypes)>()
84
+ );
85
+ }
86
+ };
87
+
88
+ template<class ReturnType>
89
+ struct createReturns<ReturnType, std::enable_if_t<!std::is_same_v<void, ReturnType> && !guts::is_instantiation_of<std::tuple, ReturnType>::value>> final {
90
+ static constexpr std::array<ArgumentDef, 1> call() {
91
+ return createReturns<std::tuple<ReturnType>>::call();
92
+ }
93
+ };
94
+
95
+ template<>
96
+ struct createReturns<void, void> final {
97
+ static constexpr std::array<ArgumentDef, 0> call() {
98
+ return createReturns<std::tuple<>>::call();
99
+ }
100
+ };
101
+
102
+ template <typename ReturnType>
103
+ struct createSingleReturn {
104
+ static constexpr std::array<ArgumentDef, 1> call() {
105
+ return createArgumentVectorFromTypes<ReturnType>(std::make_index_sequence<1>());
106
+ }
107
+ };
108
+
109
+ TORCH_API FunctionSchema make_function_schema(std::string&& name, std::string&& overload_name, c10::ArrayRef<ArgumentDef> arguments, c10::ArrayRef<ArgumentDef> returns);
110
+ TORCH_API FunctionSchema make_function_schema(c10::ArrayRef<ArgumentDef> arguments, c10::ArrayRef<ArgumentDef> returns);
111
+
112
+ /// Creates a `FunctionSchema` object from a `FunctionTraits` type for a
113
+ /// function. Flattens std::tuple returns into multiple return types
114
+ template <typename FunctionTraits>
115
+ FunctionSchema createFunctionSchemaFromTraitsFlattenedReturns() {
116
+ using ReturnType = typename FunctionTraits::return_type;
117
+ using ParameterTypes = typename FunctionTraits::parameter_types;
118
+
119
+ // arguments and returns are computed into a std::array at compile time and embedded into the binary.
120
+ // The only code executed at runtime here is the one that creates a std::vector
121
+ // of the arguments/returns from the std::array.
122
+ constexpr auto arguments = createArguments<ParameterTypes>::call();
123
+ constexpr auto returns = createReturns<ReturnType>::call();
124
+
125
+ return make_function_schema(arguments, returns);
126
+ }
127
+
128
+ /// Creates a `FunctionSchema` object from a `FunctionTraits` type for a
129
+ /// function. Preserves std::tuple returns as a Tuple return type
130
+ template <typename FunctionTraits>
131
+ FunctionSchema createFunctionSchemaFromTraitsSingleReturn(std::string&& name, std::string&& overload_name) {
132
+ using ReturnType = typename FunctionTraits::return_type;
133
+ using ParameterTypes = typename FunctionTraits::parameter_types;
134
+
135
+ // arguments and returns are computed into a std::array at compile time and embedded into the binary.
136
+ // The only code executed at runtime here is the one that creates a std::vector
137
+ // of the arguments/returns from the std::array.
138
+ constexpr auto arguments = createArguments<ParameterTypes>::call();
139
+ constexpr auto returns = createSingleReturn<ReturnType>::call();
140
+
141
+ return make_function_schema(std::move(name), std::move(overload_name), arguments, returns);
142
+ }
143
+
144
+ }
145
+
146
+ template<class FuncType>
147
+ FunctionSchema inferFunctionSchemaFlattenedReturns() {
148
+ return detail::infer_schema::createFunctionSchemaFromTraitsFlattenedReturns<guts::infer_function_traits_t<FuncType>>();
149
+ }
150
+
151
+ template<class FuncType>
152
+ FunctionSchema inferFunctionSchemaSingleReturn(std::string&& name, std::string&& overload_name) {
153
+ return detail::infer_schema::createFunctionSchemaFromTraitsSingleReturn<guts::infer_function_traits_t<FuncType>>(std::move(name), std::move(overload_name));
154
+ }
155
+
156
+ TORCH_API std::optional<std::string> findSchemaDifferences(const FunctionSchema& inferred, const FunctionSchema& specified);
157
+
158
+ }
159
+
160
+ #else
161
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
162
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/op_registration/op_allowlist.h ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // TODO: unify to C10_MOBILE. In theory this header could be used in OSS.
5
+ #ifdef TEMPLATE_SELECTIVE_BUILD
6
+ #include <ATen/selected_mobile_ops.h>
7
+ #endif
8
+
9
+ /**
10
+ * This header implements functionality to build PyTorch with only a certain
11
+ * set of operators (+ dependencies) included.
12
+ *
13
+ * - Build with -DTORCH_OPERATOR_WHITELIST="aten::add;aten::sub" and only these
14
+ * two ops will be included in your build. The allowlist records operators
15
+ * only, no overloads; if you include aten::add, all overloads of aten::add
16
+ * will be included.
17
+ *
18
+ * Internally, this is done by removing the operator registration calls
19
+ * using compile time programming, and the linker will then prune all
20
+ * operator functions that weren't registered.
21
+ * See Note [Selective build] for more details
22
+ *
23
+ * WARNING: The allowlist mechanism doesn't work for all ways you could go about
24
+ * registering an operator. If the dispatch key / operator name is not
25
+ * sufficiently obvious at compile time, then the allowlisting mechanism
26
+ * will fail (and the operator will be included in the binary anyway).
27
+ */
28
+
29
+ #include <string_view>
30
+ #include <c10/core/DispatchKey.h>
31
+ #include <c10/macros/Macros.h>
32
+
33
+
34
+ #if defined(ENABLE_RECORD_KERNEL_FUNCTION_DTYPE)
35
+ #include <ATen/record_function.h>
36
+ #endif
37
+
38
+ namespace c10::impl {
39
+
40
+ constexpr bool allowlist_contains(std::string_view allowlist, std::string_view item); // Forward Declare
41
+
42
+ /**
43
+ * In selective build mode returns true/false depending on whether a build
44
+ * feature is available or not.
45
+ *
46
+ * In instrumenting mode (tracing mode), always returns true, and doesn't
47
+ * trigger any side effects.
48
+ */
49
+ constexpr bool is_build_feature_available(const char* name) {
50
+ #if !defined(ENABLE_RECORD_KERNEL_FUNCTION_DTYPE)
51
+ // Selective Build mode.
52
+ #if !defined(TORCH_BUILD_FEATURE_ALLOWLIST)
53
+ (void)name;
54
+ return true;
55
+ #else
56
+ return allowlist_contains(
57
+ C10_STRINGIZE(TORCH_BUILD_FEATURE_ALLOWLIST),
58
+ name);
59
+ #endif
60
+
61
+ #else
62
+ // Instrumenting mode.
63
+ (void)name;
64
+ return true;
65
+ #endif
66
+ }
67
+
68
+ [[noreturn]] void build_feature_required_feature_not_available(const char* feature);
69
+
70
+ /**
71
+ * Use BUILD_FEATURE_REQUIRED macro in user-code.
72
+ *
73
+ * In selective build mode becomes a no-op if the build feature passed
74
+ * in is available. If not available, throws an exception (c10::Error).
75
+ * The compiler is able to perform dead code elimination for code
76
+ * following this method if the build feature is not available.
77
+ *
78
+ * In instrumenting mode (tracing mode), registers (as a side effect)
79
+ * the presence of this specific build feature being triggered.
80
+ */
81
+ #if !defined(ENABLE_RECORD_KERNEL_FUNCTION_DTYPE) // selective build mode
82
+
83
+ #if defined(TORCH_BUILD_FEATURE_ALLOWLIST)
84
+ #define BUILD_FEATURE_REQUIRED(NAME) \
85
+ if (!c10::impl::is_build_feature_available(NAME)) { \
86
+ ::c10::impl::build_feature_required_feature_not_available(NAME); \
87
+ }
88
+ #else // Everything trivially selected
89
+ #define BUILD_FEATURE_REQUIRED(NAME)
90
+
91
+ #endif
92
+
93
+ #else // trace mode
94
+ #define BUILD_FEATURE_REQUIRED(NAME) \
95
+ RECORD_FUNCTION_WITH_SCOPE( \
96
+ at::RecordScope::BUILD_FEATURE, \
97
+ std::string(NAME), \
98
+ {});
99
+ #endif
100
+
101
+ // Use this macro, and not is_build_feature_available
102
+ #define BUILD_FEATURE_AVAILABLE(NAME) ::c10::impl::is_build_feature_available(NAME)
103
+
104
+ // returns true iff allowlist contains item
105
+ // allowlist_contains("a;bc;d", "bc") == true
106
+ constexpr bool allowlist_contains(std::string_view allowlist, std::string_view item) {
107
+ //Choose a really big value for next so that if something goes wrong
108
+ //this code will blow up in a hopefully detectable way.
109
+ size_t next = std::numeric_limits<size_t>::max();
110
+ for (size_t cur = 0; cur <= allowlist.size(); cur = next) {
111
+ next = allowlist.find(';', cur);
112
+ if (next != std::string_view::npos) {
113
+ if (allowlist.substr(cur, next - cur) == item) {
114
+ return true;
115
+ }
116
+ next++;
117
+ } else {
118
+ if (allowlist.substr(cur) == item) {
119
+ return true;
120
+ }
121
+ break;
122
+ }
123
+ }
124
+ return false;
125
+ }
126
+
127
+ // Returns true iff the given op name is on the allowlist
128
+ // and should be registered
129
+ constexpr bool op_allowlist_check(std::string_view op_name [[maybe_unused]]) {
130
+ assert(op_name.find("::") != std::string_view::npos);
131
+ // Use assert() instead of throw() due to a gcc bug. See:
132
+ // https://stackoverflow.com/questions/34280729/throw-in-constexpr-function
133
+ // https://github.com/fmtlib/fmt/issues/682
134
+ assert(op_name.find('(') == std::string_view::npos);
135
+ #if !defined(TORCH_OPERATOR_WHITELIST)
136
+ // If the TORCH_OPERATOR_WHITELIST parameter is not defined,
137
+ // all ops are to be registered
138
+ return true;
139
+ #else
140
+ return allowlist_contains(
141
+ C10_STRINGIZE(TORCH_OPERATOR_WHITELIST),
142
+ // This function is majorly used for mobile selective build with
143
+ // root operators, where the overload is included in the allowlist.
144
+ op_name);
145
+ // // Strip overload name (as allowlist doesn't contain overloads)
146
+ // // Another function based on this may be added when there's usage
147
+ // // on op names without overload.
148
+ // OperatorNameView::parse(op_name).name);
149
+ #endif
150
+ }
151
+
152
+ // Returns true iff the given schema string is on the allowlist
153
+ // and should be registered
154
+ constexpr bool schema_allowlist_check(std::string_view schema) {
155
+ #if defined(TORCH_FORCE_SCHEMA_REGISTRATION)
156
+ return true;
157
+ #else
158
+ return op_allowlist_check(schema.substr(0, schema.find('(')));
159
+ #endif
160
+ }
161
+
162
+ // Returns true iff the given custom class name is on the allowlist
163
+ // and should be registered
164
+ constexpr bool custom_class_allowlist_check(std::string_view custom_class_name [[maybe_unused]]) {
165
+ #if !defined(TORCH_CUSTOM_CLASS_ALLOWLIST)
166
+ // If the TORCH_CUSTOM_CLASS_ALLOWLIST parameter is not defined,
167
+ // all custom classes are to be registered
168
+ return true;
169
+ #else
170
+ return allowlist_contains(
171
+ C10_STRINGIZE(TORCH_CUSTOM_CLASS_ALLOWLIST),
172
+ custom_class_name);
173
+ #endif
174
+ }
175
+
176
+ // schema_allowlist_check() implicitly depends on a macro, TORCH_OPERATOR_WHITELIST.
177
+ // Add this API to pass arbitrary allowlist.
178
+ constexpr bool op_allowlist_contains_name_in_schema(std::string_view allowlist, std::string_view schema) {
179
+ return allowlist_contains(allowlist, schema.substr(0, schema.find('(')));
180
+ }
181
+
182
+ } // namespace c10::impl
183
+
184
+ #else
185
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
186
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/core/op_registration/op_registration.h ADDED
@@ -0,0 +1,599 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ /**
5
+ * Include this file if you want to register operators. It includes all
6
+ * functionality needed to do so for you.
7
+ */
8
+
9
+ #include <c10/core/DispatchKey.h>
10
+ #include <c10/core/DispatchKeySet.h>
11
+ #include <c10/core/CompileTimeFunctionPointer.h>
12
+ #include <ATen/core/boxing/KernelFunction.h>
13
+ #include <ATen/core/dispatch/CppSignature.h>
14
+ #include <ATen/core/dispatch/RegistrationHandleRAII.h>
15
+ #include <ATen/core/op_registration/infer_schema.h>
16
+ #if defined(EXPOSE_C2_OPS) || !defined(CAFFE2_IS_XPLAT_BUILD)
17
+ #include <torch/csrc/jit/frontend/function_schema_parser.h>
18
+ #endif
19
+ #include <ATen/core/ATenOpList.h>
20
+
21
+ namespace c10 {
22
+
23
+ namespace detail {
24
+ // The first argument of the schema might be of type DispatchKeySet, in which case we remove it.
25
+ // We do this because every argument in a function schema is expected to be convertible
26
+ // to an ivalue, but DispatchKeySet is not a type we want the jit to be aware of.
27
+ // See Note [Plumbing Keys Through The Dispatcher]
28
+ template<class KernelFunctor>
29
+ std::unique_ptr<FunctionSchema> inferFunctionSchemaFromFunctor() {
30
+ using func_type = typename c10::remove_DispatchKeySet_arg_from_func<KernelFunctor>::func_type;
31
+ return std::make_unique<FunctionSchema>(inferFunctionSchemaFlattenedReturns<func_type>());
32
+ }
33
+ }
34
+
35
+ /**
36
+ * An instance of this class handles the registration for one or more operators.
37
+ * Make sure you keep the RegisterOperators instance around since it will
38
+ * deregister the operator it's responsible for in its destructor.
39
+ *
40
+ * Example:
41
+ *
42
+ * > namespace {
43
+ * > class my_kernel_cpu final : public c10::OperatorKernel {
44
+ * > public:
45
+ * > Tensor operator()(Tensor a, Tensor b) {...}
46
+ * > };
47
+ * > }
48
+ * >
49
+ * > static auto registry = c10::RegisterOperators()
50
+ * > .op(c10::RegisterOperators::options()
51
+ * > .schema("my_op")
52
+ * > .kernel<my_kernel_cpu>(DispatchKey::CPU));
53
+ */
54
+ class TORCH_API RegisterOperators final {
55
+ public:
56
+ RegisterOperators() = default;
57
+ ~RegisterOperators() = default;
58
+
59
+ RegisterOperators(const RegisterOperators&) = delete;
60
+ RegisterOperators& operator=(const RegisterOperators&) = delete;
61
+ RegisterOperators(RegisterOperators&&) noexcept = default;
62
+ RegisterOperators& operator=(RegisterOperators&&) noexcept = default;
63
+
64
+ class TORCH_API Options final {
65
+ public:
66
+ Options(const Options&) = delete;
67
+ Options(Options&&) noexcept = delete;
68
+ Options& operator=(const Options&) = delete;
69
+ Options& operator=(Options&&) noexcept = delete;
70
+
71
+ // internal-only for registering stack based kernels
72
+ template<KernelFunction::BoxedKernelFunction* kernel_func>
73
+ Options&& kernel(DispatchKey dispatch_key) && {
74
+ return std::move(*this).kernel(dispatch_key, KernelFunction::makeFromBoxedFunction<kernel_func>(), std::nullopt, nullptr);
75
+ }
76
+
77
+ // internal-only for registering stack based catch-all kernels
78
+ template<KernelFunction::BoxedKernelFunction* kernel_func>
79
+ Options&& catchAllKernel() && {
80
+ return std::move(*this).kernel(std::nullopt, KernelFunction::makeFromBoxedFunction<kernel_func>(), std::nullopt, nullptr);
81
+ }
82
+
83
+ // internal only for registering caffe2 ops
84
+ Options&& schema(FunctionSchema&& schema) {
85
+ TORCH_CHECK(!schemaOrName_.has_value(), "You can only specify the schema once per operator registration.");
86
+ schemaOrName_ = FunctionSchema(std::move(schema));
87
+ return std::move(*this);
88
+ }
89
+
90
+ /**
91
+ * Use this to specify the schema for an operator. You can also specify
92
+ * the operator name only to have the function signature part of the
93
+ * schema be inferred from the kernel function.
94
+ *
95
+ * Example:
96
+ *
97
+ * > // Infer function signature from my_kernel_cpu
98
+ * > static auto registry = c10::RegisterOperators()
99
+ * > .op(c10::RegisterOperators::options()
100
+ * > .schema("my_op")
101
+ * > .kernel<my_kernel_cpu>(DispatchKey::CPU));
102
+ * >
103
+ * >
104
+ * > // Explicitly specify full schema
105
+ * > static auto registry = c10::RegisterOperators()
106
+ * > .op(c10::RegisterOperators::options()
107
+ * > .schema("my_op(Tensor a) -> Tensor")
108
+ * > .kernel<my_kernel_cpu>(DispatchKey::CPU));
109
+ */
110
+ Options&& schema(const std::string& schemaOrName) {
111
+ TORCH_CHECK(!schemaOrName_.has_value(), "Tried to register operator ", schemaOrName," but specified schema multiple times. You can only specify the schema once per operator registration.");
112
+
113
+ #if !defined(EXPOSE_C2_OPS) && defined(CAFFE2_IS_XPLAT_BUILD)
114
+ throw std::logic_error("Tried to register operator " + schemaOrName + ". We don't support registering c10 ops on mobile yet because the function schema parser isn't present in the mobile build.");
115
+ #else
116
+ schemaOrName_ = torch::jit::parseSchemaOrName(schemaOrName);
117
+ #endif
118
+
119
+ return std::move(*this);
120
+ }
121
+
122
+ /**
123
+ * Use this to register an operator whose kernel is implemented as a functor.
124
+ * The kernel is only called for inputs matching the given dispatch key.
125
+ * You can register multiple kernels for different dispatch keys.
126
+ *
127
+ * Example:
128
+ *
129
+ * > namespace {
130
+ * > class my_kernel_cpu final : public c10::OperatorKernel {
131
+ * > public:
132
+ * > Tensor operator()(Tensor a, Tensor b) {...}
133
+ * > };
134
+ * > }
135
+ * >
136
+ * > static auto registry = c10::RegisterOperators()
137
+ * > .op(c10::RegisterOperators::options()
138
+ * > .schema("my_op")
139
+ * > .kernel<my_kernel_cpu>(DispatchKey::CPU));
140
+ *
141
+ * The functor constructor can take arguments to configure the kernel.
142
+ * The arguments are defined in the kernel registration.
143
+ * Example:
144
+ *
145
+ * > namespace {
146
+ * > class my_kernel_cpu final : public c10::OperatorKernel {
147
+ * > public:
148
+ * > explicit my_kernel_cpu(std::string some_configuration, int a, bool b)
149
+ * > : ... {...}
150
+ * >
151
+ * > Tensor operator()(Tensor a, Tensor b) {...}
152
+ * > };
153
+ * > }
154
+ * >
155
+ * > static auto registry = c10::RegisterOperators()
156
+ * > .op(c10::RegisterOperators::options()
157
+ * > .schema("my_op")
158
+ * > .kernel<my_kernel_cpu>(DispatchKey::CPU, "some_configuration", 3, true));
159
+ */
160
+ template<class KernelFunctor, class... ConstructorParameters>
161
+ // enable_if: only enable it if KernelFunctor is actually a functor
162
+ std::enable_if_t<guts::is_functor<KernelFunctor>::value, Options&&> kernel(DispatchKey dispatch_key, ConstructorParameters&&... constructorParameters) && {
163
+ static_assert(std::is_base_of_v<OperatorKernel, KernelFunctor>, "Tried to register a kernel functor using the kernel<Functor>() API, but it doesn't inherit from c10::OperatorKernel. Please have the functor inherit from it.");
164
+ static_assert(std::is_constructible_v<KernelFunctor, ConstructorParameters...>, "Wrong argument list for constructor of kernel functor. The arguments to kernel<Functor>(arguments...) must match one of the constructors of Functor.");
165
+
166
+ return std::move(*this).kernel(
167
+ dispatch_key,
168
+ KernelFunction::makeFromUnboxedFunctor<false, KernelFunctor>(std::make_unique<KernelFunctor>(std::forward<ConstructorParameters>(constructorParameters)...)),
169
+ impl::CppSignature::make<KernelFunctor>(),
170
+ detail::inferFunctionSchemaFromFunctor<KernelFunctor>()
171
+ );
172
+ }
173
+
174
+ /**
175
+ * Use this to register an operator whose kernel is implemented as a functor.
176
+ * The kernel is a catch-all kernel, meaning it's called independent from
177
+ * the input. Dispatch is disabled for this operator.
178
+ *
179
+ * Example:
180
+ *
181
+ * > namespace {
182
+ * > class my_kernel_cpu final : public c10::OperatorKernel {
183
+ * > public:
184
+ * > Tensor operator()(Tensor a, Tensor b) {...}
185
+ * > };
186
+ * > }
187
+ * >
188
+ * > static auto registry = c10::RegisterOperators()
189
+ * > .op(c10::RegisterOperators::options()
190
+ * > .schema("my_op")
191
+ * > .catchAllKernel<my_kernel_cpu>());
192
+ *
193
+ * The functor constructor can take arguments to configure the kernel.
194
+ * The arguments are defined in the kernel registration.
195
+ * Example:
196
+ *
197
+ * > namespace {
198
+ * > class my_kernel_cpu final : public c10::OperatorKernel {
199
+ * > public:
200
+ * > explicit my_kernel_cpu(std::string some_configuration, int a, bool b)
201
+ * > : ... {...}
202
+ * >
203
+ * > Tensor operator()(Tensor a, Tensor b) {...}
204
+ * > };
205
+ * > }
206
+ * >
207
+ * > static auto registry = c10::RegisterOperators()
208
+ * > .op(c10::RegisterOperators::options()
209
+ * > .schema("my_op")
210
+ * > .catchAllKernel<my_kernel_cpu>("some_configuration", 3, true));
211
+ */
212
+ template<class KernelFunctor, class... ConstructorParameters>
213
+ // enable_if: only enable it if KernelFunctor is actually a functor
214
+ std::enable_if_t<guts::is_functor<KernelFunctor>::value, Options&&> catchAllKernel(ConstructorParameters&&... constructorParameters) && {
215
+ static_assert(std::is_base_of_v<OperatorKernel, KernelFunctor>, "Tried to register a kernel functor using the kernel<Functor>() API, but it doesn't inherit from c10::OperatorKernel. Please have the functor inherit from it.");
216
+ static_assert(std::is_constructible_v<KernelFunctor, ConstructorParameters...>, "Wrong argument list for constructor of kernel functor. The arguments to kernel<Functor>(arguments...) must match one of the constructors of Functor.");
217
+
218
+ return std::move(*this).kernel(
219
+ std::nullopt,
220
+ KernelFunction::makeFromUnboxedFunctor<false, KernelFunctor>(std::make_unique<KernelFunctor>(std::forward<ConstructorParameters>(constructorParameters)...)),
221
+ impl::CppSignature::make<KernelFunctor>(),
222
+ detail::inferFunctionSchemaFromFunctor<KernelFunctor>()
223
+ );
224
+ }
225
+
226
+ /**
227
+ * Use this to register an operator whose kernel is implemented by a function.
228
+ * The kernel is only called for inputs matching the given dispatch key.
229
+ * You can register multiple kernels for different dispatch keys.
230
+ *
231
+ * Example:
232
+ *
233
+ * > namespace { Tensor my_kernel_cpu(Tensor a, Tensor b) {...} }
234
+ * >
235
+ * > static auto registry = c10::RegisterOperators()
236
+ * > .op(c10::RegisterOperators::options()
237
+ * > .schema("my_op")
238
+ * > .kernel<decltype(my_kernel_cpu), &my_kernel_cpu>(DispatchKey::CPU));
239
+ */
240
+ template<class FuncType, FuncType* kernel_func>
241
+ // enable_if: only enable it if FuncType is actually a function
242
+ std::enable_if_t<guts::is_function_type<FuncType>::value, Options&&> kernel(DispatchKey dispatch_key) && {
243
+ static_assert(!std::is_same_v<FuncType, KernelFunction::BoxedKernelFunction>, "Tried to register a stackbased (i.e. internal) kernel function using the public kernel<...>() API. Please either use the internal kernel(...) API or also implement the kernel function as defined by the public API.");
244
+ static_assert(kernel_func != nullptr, "Kernel function cannot be nullptr");
245
+
246
+ return std::move(*this).kernel(
247
+ dispatch_key,
248
+ KernelFunction::makeFromUnboxedFunction(TORCH_FN(kernel_func)),
249
+ impl::CppSignature::make<FuncType>(),
250
+ // TODO Do schema inference without relying on WrapFunctionIntoFunctor
251
+ detail::inferFunctionSchemaFromFunctor<typename impl::WrapFunctionIntoFunctor<CompileTimeFunctionPointer<FuncType, kernel_func>>::type>()
252
+ );
253
+ }
254
+
255
+ /**
256
+ * Use this to register an operator whose kernel is implemented by a function.
257
+ * The kernel is a catch-all kernel, meaning it's called independent from
258
+ * the input. Dispatch is disabled for this operator.
259
+ *
260
+ * Example:
261
+ *
262
+ * > namespace { Tensor my_kernel_cpu(Tensor a, Tensor b) {...} }
263
+ * >
264
+ * > static auto registry = c10::RegisterOperators()
265
+ * > .op(c10::RegisterOperators::options()
266
+ * > .schema("my_op")
267
+ * > .catchAllKernel<decltype(my_kernel_cpu), &my_kernel_cpu>());
268
+ */
269
+ template<class FuncType, FuncType* kernel_func>
270
+ // enable_if: only enable it if FuncType is actually a function
271
+ std::enable_if_t<guts::is_function_type<FuncType>::value, Options&&> catchAllKernel() && {
272
+ static_assert(!std::is_same_v<FuncType, KernelFunction::BoxedKernelFunction>, "Tried to register a stackbased (i.e. internal) kernel function using the public kernel<...>() API. Please either use the internal kernel(...) API or also implement the kernel function as defined by the public API.");
273
+ static_assert(kernel_func != nullptr, "Kernel function cannot be nullptr");
274
+
275
+ return std::move(*this).kernel(
276
+ std::nullopt,
277
+ KernelFunction::makeFromUnboxedFunction(TORCH_FN(kernel_func)),
278
+ impl::CppSignature::make<FuncType>(),
279
+ // TODO Do schema inference without relying on WrapFunctionIntoFunctor
280
+ detail::inferFunctionSchemaFromFunctor<typename impl::WrapFunctionIntoFunctor<CompileTimeFunctionPointer<FuncType, kernel_func>>::type>()
281
+ );
282
+ }
283
+
284
+ template<class FuncType>
285
+ // enable_if: only enable it if FuncType is actually a function
286
+ std::enable_if_t<guts::is_function_type<FuncType>::value, Options&&> kernel(DispatchKey dispatch_key, FuncType* kernel_func) && {
287
+ static_assert(!std::is_same_v<FuncType, KernelFunction::BoxedKernelFunction>, "Tried to register a stackbased (i.e. internal) kernel function using the public kernel<...>() API. Please either use the internal kernel(...) API or also implement the kernel function as defined by the public API.");
288
+ TORCH_INTERNAL_ASSERT(kernel_func != nullptr, "Kernel function cannot be nullptr");
289
+
290
+ return std::move(*this).kernel(
291
+ dispatch_key,
292
+ KernelFunction::makeFromUnboxedRuntimeFunction(kernel_func),
293
+ impl::CppSignature::make<FuncType>(),
294
+ // TODO Do schema inference without relying on WrapFunctionIntoFunctor
295
+ detail::inferFunctionSchemaFromFunctor<impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<FuncType>>>()
296
+ );
297
+ }
298
+
299
+ template<class FuncType>
300
+ // enable_if: only enable it if FuncType is actually a function
301
+ std::enable_if_t<guts::is_function_type<FuncType>::value, Options&&> catchAllKernel(FuncType* kernel_func) && {
302
+ static_assert(!std::is_same_v<FuncType, KernelFunction::BoxedKernelFunction>, "Tried to register a stackbased (i.e. internal) kernel function using the public kernel<...>() API. Please either use the internal kernel(...) API or also implement the kernel function as defined by the public API.");
303
+ TORCH_INTERNAL_ASSERT(kernel_func != nullptr, "Kernel function cannot be nullptr");
304
+
305
+ return std::move(*this).kernel(
306
+ std::nullopt,
307
+ KernelFunction::makeFromUnboxedRuntimeFunction(kernel_func),
308
+ impl::CppSignature::make<FuncType>(),
309
+ // TODO Do schema inference without relying on WrapFunctionIntoFunctor
310
+ detail::inferFunctionSchemaFromFunctor<impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<FuncType>>>()
311
+ );
312
+ }
313
+
314
+ /**
315
+ * Use this to register an operator whose kernel is implemented as a lambda.
316
+ * The kernel is only called for inputs matching the given dispatch key.
317
+ * You can register multiple kernels for different dispatch keys.
318
+ *
319
+ * The lambda must be stateless, i.e. not have a capture. If your kernel
320
+ * needs to store some configuration parameters, write the kernel as a
321
+ * functor instead.
322
+ *
323
+ * Example:
324
+ *
325
+ * > static auto registry = c10::RegisterOperators()
326
+ * > .op(c10::RegisterOperators::options()
327
+ * > .schema("my_op")
328
+ * > .kernel(DispatchKey::CPU, [] (Tensor a) -> Tensor {...}));
329
+ */
330
+ template<class Lambda>
331
+ // enable_if: only enable it if Lambda is a functor (note: lambdas are functors)
332
+ std::enable_if_t<
333
+ guts::is_functor<std::decay_t<Lambda>>::value
334
+ && !std::is_same_v<typename guts::infer_function_traits_t<std::decay_t<Lambda>>::func_type, KernelFunction::BoxedKernelFunction>,
335
+ Options&&> kernel(DispatchKey dispatch_key, Lambda&& functor) && {
336
+ static_assert(!std::is_base_of_v<OperatorKernel, std::decay_t<Lambda>>, "The kernel(x) API for registering a kernel is only meant to be used with lambdas. Your kernel is a functor. Please use the kernel<Functor>() API instead.");
337
+
338
+ // We don't support stateful lambdas (i.e. lambdas with a capture), because their
339
+ // behavior would be nonobvious. A functor kernel with cache gets a new instance of
340
+ // its cache each time the kernel is looked up from the dispatch table.
341
+ // A lambda with a capture would be global and share its capture between all kernel lookups.
342
+ // So, instead of making users having to think about it (including the thread-safety
343
+ // issues this causes), let's just forbid stateful lambdas altogether.
344
+ static_assert(guts::is_stateless_lambda<std::decay_t<Lambda>>::value, "The kernel(x) API for registering a kernel only works for stateless lambdas (i.e. lambdas without captures). If you need a cache, please use the functor based API kernel<Functor>() instead.");
345
+
346
+ return std::move(*this).kernel(
347
+ dispatch_key,
348
+ KernelFunction::makeFromUnboxedLambda(std::forward<Lambda>(functor)),
349
+ impl::CppSignature::make<Lambda>(),
350
+ // TODO Do schema inference without relying on WrapFunctionIntoRuntimeFunctor
351
+ detail::inferFunctionSchemaFromFunctor<impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<Lambda>>>()
352
+ );
353
+ }
354
+
355
+ /**
356
+ * Use this to register an operator whose kernel is implemented as a lambda.
357
+ * The kernel is a catch-all kernel, meaning it's called independent from
358
+ * the input. Dispatch is disabled for this operator.
359
+ *
360
+ * The lambda must be stateless, i.e. not have a capture. If your kernel
361
+ * needs to store some configuration parameters, write the kernel as a
362
+ * functor instead.
363
+ *
364
+ * Example:
365
+ *
366
+ * > static auto registry = c10::RegisterOperators()
367
+ * > .op(c10::RegisterOperators::options()
368
+ * > .schema("my_op")
369
+ * > .catchAllKernel([] (Tensor a) -> Tensor {...}));
370
+ */
371
+ template<class Lambda>
372
+ // enable_if: only enable it if Lambda is a functor (note: lambdas are functors)
373
+ std::enable_if_t<
374
+ guts::is_functor<std::decay_t<Lambda>>::value
375
+ && !std::is_same_v<typename guts::infer_function_traits_t<std::decay_t<Lambda>>::func_type, KernelFunction::BoxedKernelFunction>,
376
+ Options&&> catchAllKernel(Lambda&& lambda) && {
377
+ static_assert(!std::is_base_of_v<OperatorKernel, std::decay_t<Lambda>>, "The kernel(x) API for registering a kernel is only meant to be used with lambdas. Your kernel is a functor. Please use the kernel<Functor>() API instead.");
378
+
379
+ // We don't support stateful lambdas (i.e. lambdas with a capture), because their
380
+ // behavior would be nonobvious.
381
+ // A lambda with a capture would be global and share its capture between all kernel lookups.
382
+ // This would be a likely source for unexpected race conditions, so we forbid it.
383
+ // If a kernel really needs global state, they can just have regular global state
384
+ // in their .cpp file next to the kernel lambda.
385
+ static_assert(guts::is_stateless_lambda<std::decay_t<Lambda>>::value, "The kernel(x) API for registering a kernel only works for stateless lambdas (i.e. lambdas without captures). If you need a cache, please use the functor based API kernel<Functor>() instead.");
386
+
387
+ return std::move(*this).kernel(
388
+ std::nullopt,
389
+ KernelFunction::makeFromUnboxedLambda(std::forward<Lambda>(lambda)),
390
+ impl::CppSignature::make<Lambda>(),
391
+ // TODO Do schema inference without relying on WrapFunctionIntoRuntimeFunctor
392
+ detail::inferFunctionSchemaFromFunctor<impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<Lambda>>>()
393
+ );
394
+ }
395
+
396
+ Options&& aliasAnalysis(AliasAnalysisKind aliasAnalysisKind) && {
397
+ TORCH_CHECK(!aliasAnalysisKind_.has_value(), "You can only call aliasAnalysis() once per operator registration.");
398
+ aliasAnalysisKind_ = aliasAnalysisKind;
399
+ return std::move(*this);
400
+ }
401
+
402
+ private:
403
+ Options&& kernel(std::optional<DispatchKey> dispatch_key, KernelFunction&& func, std::optional<impl::CppSignature> cpp_signature, std::unique_ptr<FunctionSchema>&& inferred_function_schema) && {
404
+ KernelRegistrationConfig config;
405
+ config.dispatch_key = dispatch_key;
406
+ config.func = std::move(func);
407
+ config.cpp_signature = cpp_signature;
408
+ config.inferred_function_schema = std::move(inferred_function_schema);
409
+ kernels.push_back(std::move(config));
410
+ return std::move(*this);
411
+ }
412
+
413
+ Options()
414
+ : schemaOrName_(std::nullopt)
415
+ , aliasAnalysisKind_(std::nullopt)
416
+ {}
417
+
418
+ // KernelRegistrationConfig accumulates all information from the config
419
+ // parameters passed to a RegisterOperators::op() call into one object.
420
+ struct KernelRegistrationConfig final {
421
+ KernelRegistrationConfig()
422
+ : dispatch_key(std::nullopt)
423
+ , cpp_signature(std::nullopt)
424
+ , inferred_function_schema(nullptr)
425
+ {}
426
+
427
+ std::optional<DispatchKey> dispatch_key;
428
+ KernelFunction func;
429
+ std::optional<impl::CppSignature> cpp_signature;
430
+ std::unique_ptr<FunctionSchema> inferred_function_schema;
431
+ };
432
+
433
+ std::optional<std::variant<OperatorName, FunctionSchema>> schemaOrName_;
434
+
435
+ std::vector<KernelRegistrationConfig> kernels;
436
+ std::optional<AliasAnalysisKind> aliasAnalysisKind_;
437
+ friend class RegisterOperators;
438
+ friend class Library;
439
+ };
440
+
441
+ /**
442
+ * Call this to get an instance of registration options, which
443
+ * can be passed to a call to RegisterOperators::op() to specify
444
+ * these options for the operator registration.
445
+ * See class doc comment for examples.
446
+ */
447
+ static Options options() {
448
+ return {};
449
+ }
450
+
451
+ /**
452
+ * Call this to register an operator. See class doc comment for examples.
453
+ */
454
+ RegisterOperators&& op(Options&& options) && {
455
+ checkSchemaAndRegisterOp_(std::move(options));
456
+ return std::move(*this);
457
+ }
458
+
459
+ // Regular mutator version of the && version above
460
+ RegisterOperators& op(Options&& options) & {
461
+ checkSchemaAndRegisterOp_(std::move(options));
462
+ return *this;
463
+ }
464
+
465
+ /**
466
+ * This is a shorthand for RegisterOperators::op(Options) where you can
467
+ * specify the operator schema outside of the options parameter.
468
+ * See class doc comment for examples.
469
+ */
470
+ RegisterOperators&& op(const std::string& schemaOrName, Options&& options = RegisterOperators::options()) && {
471
+ return std::move(*this).op(std::move(options).schema(schemaOrName));
472
+ }
473
+
474
+ // internal only for registering caffe2 ops
475
+ RegisterOperators&& op(FunctionSchema schema, Options&& options) && {
476
+ return std::move(*this).op(std::move(options).schema(std::move(schema)));
477
+ }
478
+
479
+ template<class FuncType>
480
+ explicit RegisterOperators(const std::string& schemaOrName, FuncType&& func, Options&& options = RegisterOperators::options())
481
+ : RegisterOperators() {
482
+ std::move(*this).op(schemaOrName, std::forward<FuncType>(func), std::move(options));
483
+ }
484
+
485
+ /**
486
+ * This API registers an operator based on a kernel function pointer.
487
+ *
488
+ * Given a kernel
489
+ *
490
+ * > namespace { Tensor my_kernel_cpu(Tensor a, Tensor b) {...} }
491
+ *
492
+ * This API looks like:
493
+ *
494
+ * > static auto registry = c10::RegisterOperators()
495
+ * > .op("my_op", &my_kernel_cpu);
496
+ *
497
+ * If your kernel is small and the overhead of calling it matters,
498
+ * then this API might be the wrong choice since the following API
499
+ * has a slightly lower overhead for calling into the kernel:
500
+ *
501
+ * > static auto registry = c10::RegisterOperators()
502
+ * > .op("my_op", c10::RegisterOperators::options()
503
+ * > .kernel<decltype(my_kernel_cpu), &my_kernel_cpu>());
504
+ *
505
+ * Or, alternatively, write your kernel as a functor:
506
+ *
507
+ * > namespace {
508
+ * > class my_kernel_cpu final : public c10::OperatorKernel {
509
+ * > public:
510
+ * > Tensor operator()(Tensor a, Tensor b) {...}
511
+ * > };
512
+ * > }
513
+ * >
514
+ * > static auto registry = c10::RegisterOperators()
515
+ * > .op("my_op", c10::RegisterOperators::options()
516
+ * > .kernel<my_kernel_cpu>());
517
+ */
518
+ template<class FuncType>
519
+ // enable_if: only enable it if FuncType is actually a function, but not a stack based BoxedKernelFunction.
520
+ std::enable_if_t<guts::is_function_type<FuncType>::value && !std::is_same_v<FuncType, KernelFunction::BoxedKernelFunction>, RegisterOperators&&>
521
+ op(const std::string& schemaOrName, FuncType* func, Options&& options = RegisterOperators::options()) && {
522
+ constexpr bool AllowLegacyTypes = true;
523
+ return std::move(*this).op(std::move(options).schema(schemaOrName).kernel(
524
+ std::nullopt,
525
+ KernelFunction::makeFromUnboxedRuntimeFunction<AllowLegacyTypes>(func),
526
+ impl::CppSignature::make<FuncType>(),
527
+ // TODO Do schema inference without relying on WrapFunctionIntoRuntimeFunctor
528
+ detail::inferFunctionSchemaFromFunctor<impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<FuncType>>>()
529
+ ));
530
+ }
531
+
532
+ /**
533
+ * This API registers an operator based on a kernel lambda.
534
+ *
535
+ * This API looks like:
536
+ *
537
+ * > static auto registry = c10::RegisterOperators()
538
+ * > .op("my_op", [] (Tensor a, Tensor b) {...});
539
+ *
540
+ * This is equivalent to:
541
+ *
542
+ * > static auto registry = c10::RegisterOperators()
543
+ * > .op("my_op", c10::RegisterOperators::options()
544
+ * > .catchAllKernel([] (Tensor a, Tensor b) {...}));
545
+ *
546
+ */
547
+ template<class Lambda>
548
+ // enable_if: only enable it if Lambda is actually a stateless lambda
549
+ std::enable_if_t<guts::is_functor<Lambda>::value && guts::is_stateless_lambda<std::decay_t<Lambda>>::value, RegisterOperators&&>
550
+ op(const std::string& schemaOrName, Lambda&& lambda, Options&& options = RegisterOperators::options()) && {
551
+ static_assert(!std::is_base_of_v<OperatorKernel, Lambda>, "c10::OperatorKernel is part of the new kernel registration API and shouldn't be used together with the deprecated registration API. Please use the new RegisterOperators::options().kernel() based API instead.");
552
+
553
+ constexpr bool AllowLegacyTypes = true;
554
+ return std::move(*this).op(std::move(options).schema(schemaOrName).kernel(
555
+ std::nullopt,
556
+ KernelFunction::makeFromUnboxedLambda<AllowLegacyTypes>(std::forward<Lambda>(lambda)),
557
+ impl::CppSignature::make<Lambda>(),
558
+ // TODO Do schema inference without relying on WrapFunctionIntoRuntimeFunctor
559
+ detail::inferFunctionSchemaFromFunctor<impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<Lambda>>>()
560
+ ));
561
+ }
562
+
563
+ template<class Lambda>
564
+ C10_DEPRECATED_MESSAGE("Registering operator kernels with stateful lambdas (i.e. lambdas with a capture) has non-obvious behavior. This is deprecated. Please use a lambda without a capture or a functor class instead.")
565
+ // enable_if: only enable it if Lambda is actually a functor but not a stateless lambda
566
+ std::enable_if_t<guts::is_functor<Lambda>::value && !guts::is_stateless_lambda<std::decay_t<Lambda>>::value, RegisterOperators&&>
567
+ op(const std::string& schemaOrName, Lambda&& lambda, Options&& options = RegisterOperators::options()) && {
568
+ static_assert(!std::is_base_of_v<OperatorKernel, Lambda>, "c10::OperatorKernel is part of the new kernel registration API and shouldn't be used together with the deprecated registration API. Please use the new RegisterOperators::options().kernel() based API instead.");
569
+
570
+ constexpr bool AllowLegacyTypes = true;
571
+ return std::move(*this).op(std::move(options).schema(schemaOrName).kernel(
572
+ std::nullopt,
573
+ KernelFunction::makeFromUnboxedLambda<AllowLegacyTypes>(std::forward<Lambda>(lambda)),
574
+ impl::CppSignature::make<Lambda>(),
575
+ // TODO Do schema inference without relying on WrapFunctionIntoRuntimeFunctor
576
+ detail::inferFunctionSchemaFromFunctor<impl::WrapFunctionIntoRuntimeFunctor<std::decay_t<Lambda>>>()
577
+ ));
578
+ }
579
+
580
+ private:
581
+ void checkSchemaAndRegisterOp_(Options&& config);
582
+
583
+ static c10::FunctionSchema inferSchemaFromKernels_(const OperatorName& opNameStr, const Options& options);
584
+ void checkNoDuplicateKernels_(const Options& options);
585
+ void registerOp_(Options&& options);
586
+
587
+ std::vector<RegistrationHandleRAII> registrars_;
588
+ };
589
+
590
+ } // namespace c10
591
+
592
+ namespace torch {
593
+ // Old-style API
594
+ using RegisterOperators = c10::RegisterOperators;
595
+ }
596
+
597
+ #else
598
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
599
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/functional.h ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <ATen/cpu/vec/functional_base.h>
5
+ #include <ATen/cpu/vec/functional_bfloat16.h>
6
+
7
+ #else
8
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
9
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/functional_base.h ADDED
@@ -0,0 +1,480 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // DO NOT DEFINE STATIC DATA IN THIS HEADER!
5
+ // See Note [Do not compile initializers with AVX]
6
+
7
+ #include <ATen/cpu/vec/vec.h>
8
+ #include <c10/util/irange.h>
9
+
10
+ namespace at {
11
+ namespace detail {
12
+ // We prefer to convert through float for reduced-precision floating
13
+ // point types if we have a Vectorized specialization for float and we
14
+ // don't have one for the actual type in question.
15
+ template <typename T>
16
+ struct should_prefer_converting_through_float
17
+ : std::bool_constant<
18
+ is_reduced_floating_point_v<T> &&
19
+ vec::is_vec_specialized_for_v<float> &&
20
+ !vec::is_vec_specialized_for_v<T>> {};
21
+
22
+ template <typename T>
23
+ constexpr auto should_prefer_converting_through_float_v =
24
+ should_prefer_converting_through_float<T>::value;
25
+ } // namespace detail
26
+
27
+ namespace vec {
28
+ // slow path
29
+ template <typename scalar_t, typename Op>
30
+ inline scalar_t vec_reduce_all(
31
+ const Op& vec_fun,
32
+ vec::Vectorized<scalar_t> acc_vec,
33
+ int64_t size) {
34
+ using Vec = vec::Vectorized<scalar_t>;
35
+ scalar_t acc_arr[Vec::size()];
36
+ acc_vec.store(acc_arr);
37
+ for (const auto i : c10::irange(1, size)) {
38
+ std::array<scalar_t, Vec::size()> acc_arr_next = {0};
39
+ acc_arr_next[0] = acc_arr[i];
40
+ Vec acc_vec_next = Vec::loadu(acc_arr_next.data());
41
+ acc_vec = vec_fun(acc_vec, acc_vec_next);
42
+ }
43
+ acc_vec.store(acc_arr);
44
+ return acc_arr[0];
45
+ }
46
+
47
+ template <typename scalar_t, typename Op>
48
+ struct VecReduceAllSIMD {
49
+ static inline scalar_t apply(
50
+ const Op& vec_fun,
51
+ const Vectorized<scalar_t>& acc_vec) {
52
+ return vec_reduce_all(vec_fun, acc_vec, Vectorized<scalar_t>::size());
53
+ }
54
+ };
55
+
56
+ #if defined(__GNUC__) && (__GNUC__ > 5) && !defined(_MSC_VER) && \
57
+ !defined(C10_MOBILE)
58
+ #if defined(CPU_CAPABILITY_AVX2)
59
+ template <typename Op>
60
+ struct VecReduceAllSIMD<float, Op> {
61
+ static inline float apply(
62
+ const Op& vec_fun,
63
+ const Vectorized<float>& acc_vec) {
64
+ using Vec = Vectorized<float>;
65
+ Vec v = acc_vec;
66
+ // 128-bit shuffle
67
+ Vec v1 = _mm256_permute2f128_ps(v, v, 0x1);
68
+ v = vec_fun(v, v1);
69
+ // 64-bit shuffle
70
+ v1 = _mm256_shuffle_ps(v, v, 0x4E);
71
+ v = vec_fun(v, v1);
72
+ // 32-bit shuffle
73
+ v1 = _mm256_shuffle_ps(v, v, 0xB1);
74
+ v = vec_fun(v, v1);
75
+ return _mm256_cvtss_f32(v);
76
+ }
77
+ };
78
+ #endif // defined(CPU_CAPABILITY_AVX2)
79
+ #if defined(CPU_CAPABILITY_AVX512)
80
+ template <typename Op>
81
+ struct VecReduceAllSIMD<float, Op> {
82
+ static inline float apply(
83
+ const Op& vec_fun,
84
+ const Vectorized<float>& acc_vec) {
85
+ using Vec = Vectorized<float>;
86
+ Vec v = acc_vec;
87
+ // 256-bit shuffle
88
+ Vec v1 = _mm512_shuffle_f32x4(v, v, 0x4E);
89
+ v = vec_fun(v, v1);
90
+ // 128-bit shuffle
91
+ v1 = _mm512_shuffle_f32x4(v, v, 0xB1);
92
+ v = vec_fun(v, v1);
93
+ // 64-bit shuffle
94
+ v1 = _mm512_shuffle_ps(v, v, 0x4E);
95
+ v = vec_fun(v, v1);
96
+ // 32-bit shuffle
97
+ v1 = _mm512_shuffle_ps(v, v, 0xB1);
98
+ v = vec_fun(v, v1);
99
+ return _mm512_cvtss_f32(v);
100
+ }
101
+ };
102
+ #endif // defined(CPU_CAPABILITY_AVX512)
103
+ #endif // defined(__GNUC__) && (__GNUC__ > 5) && !defined(_MSC_VER) &&
104
+ // !defined(C10_MOBILE)
105
+
106
+ #if defined(__aarch64__) && !defined(C10_MOBILE) && !defined(__CUDACC__) && \
107
+ !defined(CPU_CAPABILITY_SVE)
108
+ template <typename Op>
109
+ struct VecReduceAllSIMD<float, Op> {
110
+ static inline float apply(
111
+ const Op& vec_fun,
112
+ const Vectorized<float>& acc_vec) {
113
+ using Vec = Vectorized<float>;
114
+ Vec v = acc_vec;
115
+
116
+ // 64-bit shuffle: [a1+a5, a2+a6, a3+a7, a4+a8, -, -, -, -] -> [a3+a7,
117
+ // a4+a8, a1+a5, a2+a6, -, -, -, -]
118
+ float32x4_t v1_1 = vextq_f32(v, v, 2);
119
+ Vec v1 = v1_1;
120
+ // [a1+a3+a5+a7, a2+a4+a6+a8, a1+a3+a5+a7, a2+a4+a6+a8, -, -, -, -]
121
+ v = vec_fun(v, v1);
122
+
123
+ // 32-bit shuffle: [a1+a3+a5+a7, a2+a4+a6+a8, a1+a3+a5+a7, a2+a4+a6+a8, -,
124
+ // -, -, -] -> [a2+a4+a6+a8, a1+a3+a5+a7, a2+a4+a6+a8, a1+a3+a5+a7, -, -, -,
125
+ // -]
126
+ v1_1 = vrev64q_f32(v);
127
+ v1 = v1_1;
128
+ // [a1+a2+a3+a4+a5+a6+a7+a8, a1+a2+a3+a4+a5+a6+a7+a8,
129
+ // a1+a2+a3+a4+a5+a6+a7+a8, a1+a2+a3+a4+a5+a6+a7+a8, -, -, -, -]
130
+ v = vec_fun(v, v1);
131
+
132
+ return v[0];
133
+ }
134
+ };
135
+
136
+ template <>
137
+ struct VecReduceAllSIMD<float, std::plus<Vectorized<float>>> {
138
+ static inline float apply(
139
+ const std::plus<Vectorized<float>>& vec_fun,
140
+ const Vectorized<float>& acc_vec) {
141
+ return vaddvq_f32(acc_vec);
142
+ }
143
+ };
144
+ #endif // defined(__aarch64__) && !defined(C10_MOBILE) && !defined(__CUDACC__)
145
+ // && !defined(CPU_CAPABILITY_SVE)
146
+
147
+ #if defined(__aarch64__) && !defined(C10_MOBILE) && !defined(__CUDACC__) && \
148
+ defined(CPU_CAPABILITY_SVE256)
149
+ template <typename Op>
150
+ struct VecReduceAllSIMD<float, Op> {
151
+ static inline float apply(
152
+ const Op& vec_fun,
153
+ const Vectorized<float>& acc_vec) {
154
+ using Vec = Vectorized<float>;
155
+ Vec v = acc_vec;
156
+ // 128-bit shuffle
157
+ svuint32_t ind = svdupq_n_u32(4, 5, 6, 7);
158
+ Vec v1 = svtbl_f32(v, ind);
159
+ v = vec_fun(v, v1);
160
+ // 64-bit shuffle
161
+ ind = svdupq_n_u32(2, 3, 0, 1);
162
+ v1 = svtbl_f32(v, ind);
163
+ v = vec_fun(v, v1);
164
+ // 32-bit shuffle
165
+ ind = svdupq_n_u32(1, 0, 2, 3);
166
+ v1 = svtbl_f32(v, ind);
167
+ v = vec_fun(v, v1);
168
+ return svlasta(svpfalse(), v);
169
+ }
170
+ };
171
+ #endif // defined(__aarch64__) && !defined(C10_MOBILE) && !defined(__CUDACC__)
172
+ // && defined(CPU_CAPABILITY_SVE256)
173
+
174
+ template <typename scalar_t, typename Op>
175
+ inline scalar_t vec_reduce_all(
176
+ const Op& vec_fun,
177
+ const Vectorized<scalar_t>& acc_vec) {
178
+ return VecReduceAllSIMD<scalar_t, Op>::apply(vec_fun, acc_vec);
179
+ }
180
+
181
+ template <
182
+ typename scalar_t,
183
+ typename Op,
184
+ typename std::enable_if_t<!is_reduced_floating_point_v<scalar_t>, int> = 0>
185
+ inline scalar_t reduce_all(
186
+ const Op& vec_fun,
187
+ const scalar_t* data,
188
+ int64_t size) {
189
+ using Vec = vec::Vectorized<scalar_t>;
190
+ if (size < Vec::size())
191
+ return vec_reduce_all(vec_fun, Vec::loadu(data, size), size);
192
+ int64_t d = Vec::size();
193
+ Vec acc_vec = Vec::loadu(data);
194
+ for (; d < size - (size % Vec::size()); d += Vec::size()) {
195
+ Vec data_vec = Vec::loadu(data + d);
196
+ acc_vec = vec_fun(acc_vec, data_vec);
197
+ }
198
+ if (size - d > 0) {
199
+ Vec data_vec = Vec::loadu(data + d, size - d);
200
+ acc_vec = Vec::set(acc_vec, vec_fun(acc_vec, data_vec), size - d);
201
+ }
202
+ return vec_reduce_all(vec_fun, acc_vec);
203
+ }
204
+
205
+ // similar to reduce_all, but reduces into two outputs
206
+ template <
207
+ typename scalar_t,
208
+ typename Op1,
209
+ typename Op2,
210
+ typename std::enable_if_t<!is_reduced_floating_point_v<scalar_t>, int> = 0>
211
+ inline std::pair<scalar_t, scalar_t> reduce2_all(
212
+ const Op1& vec_fun1,
213
+ const Op2& vec_fun2,
214
+ const scalar_t* data,
215
+ int64_t size) {
216
+ using Vec = vec::Vectorized<scalar_t>;
217
+ if (size < Vec::size()) {
218
+ auto loaded_data = Vec::loadu(data, size);
219
+ return std::pair<scalar_t, scalar_t>(
220
+ vec_reduce_all(vec_fun1, loaded_data, size),
221
+ vec_reduce_all(vec_fun2, loaded_data, size));
222
+ }
223
+ int64_t d = Vec::size();
224
+ Vec acc_vec1 = Vec::loadu(data);
225
+ Vec acc_vec2 = Vec::loadu(data);
226
+ for (; d < size - (size % Vec::size()); d += Vec::size()) {
227
+ Vec data_vec = Vec::loadu(data + d);
228
+ acc_vec1 = vec_fun1(acc_vec1, data_vec);
229
+ acc_vec2 = vec_fun2(acc_vec2, data_vec);
230
+ }
231
+ if (size - d > 0) {
232
+ Vec data_vec = Vec::loadu(data + d, size - d);
233
+ acc_vec1 = Vec::set(acc_vec1, vec_fun1(acc_vec1, data_vec), size - d);
234
+ acc_vec2 = Vec::set(acc_vec2, vec_fun2(acc_vec2, data_vec), size - d);
235
+ }
236
+ return std::pair<scalar_t, scalar_t>(
237
+ vec_reduce_all(vec_fun1, acc_vec1), vec_reduce_all(vec_fun2, acc_vec2));
238
+ }
239
+
240
+ template <
241
+ typename scalar_t,
242
+ typename MapOp,
243
+ typename ReduceOp,
244
+ typename std::enable_if_t<!is_reduced_floating_point_v<scalar_t>, int> = 0>
245
+ inline scalar_t map_reduce_all(
246
+ const MapOp& map_fun,
247
+ const ReduceOp& red_fun,
248
+ const scalar_t* data,
249
+ int64_t size) {
250
+ using Vec = vec::Vectorized<scalar_t>;
251
+ if (size < Vec::size())
252
+ return vec_reduce_all(red_fun, map_fun(Vec::loadu(data, size)), size);
253
+ int64_t d = Vec::size();
254
+ Vec acc_vec = map_fun(Vec::loadu(data));
255
+ for (; d < size - (size % Vec::size()); d += Vec::size()) {
256
+ Vec data_vec = Vec::loadu(data + d);
257
+ data_vec = map_fun(data_vec);
258
+ acc_vec = red_fun(acc_vec, data_vec);
259
+ }
260
+ if (size - d > 0) {
261
+ Vec data_vec = Vec::loadu(data + d, size - d);
262
+ data_vec = map_fun(data_vec);
263
+ acc_vec = Vec::set(acc_vec, red_fun(acc_vec, data_vec), size - d);
264
+ }
265
+ return vec_reduce_all(red_fun, acc_vec);
266
+ }
267
+
268
+ template <
269
+ typename scalar_t,
270
+ typename MapOp,
271
+ typename ReduceOp,
272
+ typename std::enable_if_t<!is_reduced_floating_point_v<scalar_t>, int> = 0>
273
+ inline scalar_t map2_reduce_all(
274
+ const MapOp& map_fun,
275
+ const ReduceOp& red_fun,
276
+ const scalar_t* data,
277
+ const scalar_t* data2,
278
+ int64_t size) {
279
+ using Vec = vec::Vectorized<scalar_t>;
280
+ if (size < Vec::size()) {
281
+ Vec data_vec = Vec::loadu(data, size);
282
+ Vec data2_vec = Vec::loadu(data2, size);
283
+ data_vec = map_fun(data_vec, data2_vec);
284
+ return vec_reduce_all(red_fun, data_vec, size);
285
+ }
286
+ int64_t d = Vec::size();
287
+ Vec acc_vec = map_fun(Vec::loadu(data), Vec::loadu(data2));
288
+ for (; d < size - (size % Vec::size()); d += Vec::size()) {
289
+ Vec data_vec = Vec::loadu(data + d);
290
+ Vec data2_vec = Vec::loadu(data2 + d);
291
+ data_vec = map_fun(data_vec, data2_vec);
292
+ acc_vec = red_fun(acc_vec, data_vec);
293
+ }
294
+ if (size - d > 0) {
295
+ Vec data_vec = Vec::loadu(data + d, size - d);
296
+ Vec data2_vec = Vec::loadu(data2 + d, size - d);
297
+ data_vec = map_fun(data_vec, data2_vec);
298
+ acc_vec = Vec::set(acc_vec, red_fun(acc_vec, data_vec), size - d);
299
+ }
300
+ return vec_reduce_all(red_fun, acc_vec);
301
+ }
302
+
303
+ template <
304
+ typename scalar_t,
305
+ typename MapOp,
306
+ typename ReduceOp,
307
+ typename std::enable_if_t<!is_reduced_floating_point_v<scalar_t>, int> = 0>
308
+ inline scalar_t map3_reduce_all(
309
+ const MapOp& map_fun,
310
+ const ReduceOp& red_fun,
311
+ const scalar_t* data,
312
+ const scalar_t* data2,
313
+ const scalar_t* data3,
314
+ int64_t size) {
315
+ using Vec = vec::Vectorized<scalar_t>;
316
+ if (size < Vec::size()) {
317
+ Vec data_vec = Vec::loadu(data, size);
318
+ Vec data2_vec = Vec::loadu(data2, size);
319
+ Vec data3_vec = Vec::loadu(data3, size);
320
+ data_vec = map_fun(data_vec, data2_vec, data3_vec);
321
+ return vec_reduce_all(red_fun, data_vec, size);
322
+ }
323
+
324
+ int64_t d = Vec::size();
325
+ Vec acc_vec = map_fun(Vec::loadu(data), Vec::loadu(data2), Vec::loadu(data3));
326
+ for (; d < size - (size % Vec::size()); d += Vec::size()) {
327
+ Vec data_vec = Vec::loadu(data + d);
328
+ Vec data2_vec = Vec::loadu(data2 + d);
329
+ Vec data3_vec = Vec::loadu(data3 + d);
330
+ data_vec = map_fun(data_vec, data2_vec, data3_vec);
331
+ acc_vec = red_fun(acc_vec, data_vec);
332
+ }
333
+ if (size - d > 0) {
334
+ Vec data_vec = Vec::loadu(data + d, size - d);
335
+ Vec data2_vec = Vec::loadu(data2 + d, size - d);
336
+ Vec data3_vec = Vec::loadu(data3 + d, size - d);
337
+ data_vec = map_fun(data_vec, data2_vec, data3_vec);
338
+ acc_vec = Vec::set(acc_vec, red_fun(acc_vec, data_vec), size - d);
339
+ }
340
+ return vec_reduce_all(red_fun, acc_vec);
341
+ }
342
+
343
+ template <
344
+ typename scalar_t,
345
+ typename Op,
346
+ typename std::enable_if_t<
347
+ !detail::should_prefer_converting_through_float_v<scalar_t> &&
348
+ std::is_invocable_v<Op, vec::Vectorized<scalar_t>>,
349
+ int> = 0>
350
+ inline void map(
351
+ const Op& vec_fun,
352
+ scalar_t* output_data,
353
+ const scalar_t* input_data,
354
+ int64_t size) {
355
+ using Vec = vec::Vectorized<scalar_t>;
356
+ int64_t d = 0;
357
+ for (; d < size - (size % Vec::size()); d += Vec::size()) {
358
+ Vec output_vec = vec_fun(Vec::loadu(input_data + d));
359
+ output_vec.store(output_data + d);
360
+ }
361
+ if (size - d > 0) {
362
+ Vec output_vec = vec_fun(Vec::loadu(input_data + d, size - d));
363
+ output_vec.store(output_data + d, size - d);
364
+ }
365
+ }
366
+
367
+ template <
368
+ typename scalar_t,
369
+ typename Op,
370
+ typename std::enable_if_t<
371
+ !detail::should_prefer_converting_through_float_v<scalar_t> &&
372
+ std::is_invocable_v<
373
+ Op,
374
+ vec::Vectorized<scalar_t>,
375
+ vec::Vectorized<scalar_t>>,
376
+ int> = 0>
377
+ inline void map2(
378
+ const Op& vec_fun,
379
+ scalar_t* output_data,
380
+ const scalar_t* input_data,
381
+ const scalar_t* input_data2,
382
+ int64_t size) {
383
+ using Vec = vec::Vectorized<scalar_t>;
384
+ int64_t d = 0;
385
+ for (; d < size - (size % Vec::size()); d += Vec::size()) {
386
+ Vec data_vec = Vec::loadu(input_data + d);
387
+ Vec data_vec2 = Vec::loadu(input_data2 + d);
388
+ Vec output_vec = vec_fun(data_vec, data_vec2);
389
+ output_vec.store(output_data + d);
390
+ }
391
+ if (size - d > 0) {
392
+ Vec data_vec = Vec::loadu(input_data + d, size - d);
393
+ Vec data_vec2 = Vec::loadu(input_data2 + d, size - d);
394
+ Vec output_vec = vec_fun(data_vec, data_vec2);
395
+ output_vec.store(output_data + d, size - d);
396
+ }
397
+ }
398
+
399
+ template <
400
+ typename scalar_t,
401
+ typename Op,
402
+ typename std::enable_if_t<
403
+ !detail::should_prefer_converting_through_float_v<scalar_t> &&
404
+ std::is_invocable_v<
405
+ Op,
406
+ vec::Vectorized<scalar_t>,
407
+ vec::Vectorized<scalar_t>,
408
+ vec::Vectorized<scalar_t>>,
409
+ int> = 0>
410
+ inline void map3(
411
+ const Op& vec_fun,
412
+ scalar_t* output_data,
413
+ const scalar_t* input_data1,
414
+ const scalar_t* input_data2,
415
+ const scalar_t* input_data3,
416
+ int64_t size) {
417
+ using Vec = vec::Vectorized<scalar_t>;
418
+ int64_t d = 0;
419
+ for (; d < size - (size % Vec::size()); d += Vec::size()) {
420
+ Vec data_vec1 = Vec::loadu(input_data1 + d);
421
+ Vec data_vec2 = Vec::loadu(input_data2 + d);
422
+ Vec data_vec3 = Vec::loadu(input_data3 + d);
423
+ Vec output_vec = vec_fun(data_vec1, data_vec2, data_vec3);
424
+ output_vec.store(output_data + d);
425
+ }
426
+ if (size - d > 0) {
427
+ Vec data_vec1 = Vec::loadu(input_data1 + d, size - d);
428
+ Vec data_vec2 = Vec::loadu(input_data2 + d, size - d);
429
+ Vec data_vec3 = Vec::loadu(input_data3 + d, size - d);
430
+ Vec output_vec = vec_fun(data_vec1, data_vec2, data_vec3);
431
+ output_vec.store(output_data + d, size - d);
432
+ }
433
+ }
434
+
435
+ template <
436
+ typename scalar_t,
437
+ typename Op,
438
+ typename std::enable_if_t<
439
+ !detail::should_prefer_converting_through_float_v<scalar_t> &&
440
+ std::is_invocable_v<
441
+ Op,
442
+ vec::Vectorized<scalar_t>,
443
+ vec::Vectorized<scalar_t>,
444
+ vec::Vectorized<scalar_t>,
445
+ vec::Vectorized<scalar_t>>,
446
+ int> = 0>
447
+ inline void map4(
448
+ const Op& vec_fun,
449
+ scalar_t* output_data,
450
+ const scalar_t* input_data1,
451
+ const scalar_t* input_data2,
452
+ const scalar_t* input_data3,
453
+ const scalar_t* input_data4,
454
+ int64_t size) {
455
+ using Vec = vec::Vectorized<scalar_t>;
456
+ int64_t d = 0;
457
+ for (; d < size - (size % Vec::size()); d += Vec::size()) {
458
+ Vec data_vec1 = Vec::loadu(input_data1 + d);
459
+ Vec data_vec2 = Vec::loadu(input_data2 + d);
460
+ Vec data_vec3 = Vec::loadu(input_data3 + d);
461
+ Vec data_vec4 = Vec::loadu(input_data4 + d);
462
+ Vec output_vec = vec_fun(data_vec1, data_vec2, data_vec3, data_vec4);
463
+ output_vec.store(output_data + d);
464
+ }
465
+ if (size - d > 0) {
466
+ Vec data_vec1 = Vec::loadu(input_data1 + d, size - d);
467
+ Vec data_vec2 = Vec::loadu(input_data2 + d, size - d);
468
+ Vec data_vec3 = Vec::loadu(input_data3 + d, size - d);
469
+ Vec data_vec4 = Vec::loadu(input_data4 + d, size - d);
470
+ Vec output_vec = vec_fun(data_vec1, data_vec2, data_vec3, data_vec4);
471
+ output_vec.store(output_data + d, size - d);
472
+ }
473
+ }
474
+
475
+ } // namespace vec
476
+ } // namespace at
477
+
478
+ #else
479
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
480
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/functional_bfloat16.h ADDED
@@ -0,0 +1,652 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // DO NOT DEFINE STATIC DATA IN THIS HEADER!
5
+ // See Note [Do not compile initializers with AVX]
6
+
7
+ #include <ATen/cpu/vec/vec.h>
8
+
9
+ namespace at::vec {
10
+ // BFloat16 specification
11
+ template <typename scalar_t>
12
+ struct VecScalarType {
13
+ using type = scalar_t;
14
+ };
15
+ template <>
16
+ struct VecScalarType<BFloat16> {
17
+ using type = float;
18
+ };
19
+ template <>
20
+ struct VecScalarType<Half> {
21
+ using type = float;
22
+ };
23
+
24
+ // This is different from at::acc_type since we only need to specialize BFloat16
25
+ template <typename scalar_t>
26
+ using vec_scalar_t = typename VecScalarType<scalar_t>::type;
27
+
28
+ // Vector conversion between float and bfloat16/half
29
+ template <>
30
+ inline std::tuple<Vectorized<float>, Vectorized<float>> convert_to_float<
31
+ BFloat16>(const Vectorized<BFloat16>& a) {
32
+ return convert_bfloat16_float(a);
33
+ }
34
+
35
+ template <>
36
+ inline std::tuple<Vectorized<float>, Vectorized<float>> convert_to_float<Half>(
37
+ const Vectorized<Half>& a) {
38
+ return convert_half_float(a);
39
+ }
40
+
41
+ template <>
42
+ inline Vectorized<BFloat16> convert_from_float<BFloat16>(
43
+ const Vectorized<float>& a,
44
+ const Vectorized<float>& b) {
45
+ return convert_float_bfloat16(a, b);
46
+ }
47
+
48
+ template <>
49
+ inline Vectorized<Half> convert_from_float<Half>(
50
+ const Vectorized<float>& a,
51
+ const Vectorized<float>& b) {
52
+ return convert_float_half(a, b);
53
+ }
54
+
55
+ template <
56
+ typename scalar_t,
57
+ typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
58
+ inline void load_to_float(
59
+ const scalar_t* data,
60
+ Vectorized<float>& out1,
61
+ Vectorized<float>& out2);
62
+
63
+ template <>
64
+ inline void load_to_float<BFloat16>(
65
+ const BFloat16* data,
66
+ Vectorized<float>& out1,
67
+ Vectorized<float>& out2) {
68
+ load_fp32_from_bf16(data, out1, out2);
69
+ }
70
+
71
+ template <>
72
+ inline void load_to_float<Half>(
73
+ const Half* data,
74
+ Vectorized<float>& out1,
75
+ Vectorized<float>& out2) {
76
+ load_fp32_from_fp16(data, out1, out2);
77
+ }
78
+
79
+ template <
80
+ typename scalar_t,
81
+ typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
82
+ inline void load_to_float(const scalar_t* data, Vectorized<float>& out);
83
+
84
+ template <>
85
+ inline void load_to_float<BFloat16>(
86
+ const BFloat16* data,
87
+ Vectorized<float>& out) {
88
+ load_fp32_from_bf16(data, out);
89
+ }
90
+
91
+ template <>
92
+ inline void load_to_float<Half>(const Half* data, Vectorized<float>& out) {
93
+ load_fp32_from_fp16(data, out);
94
+ }
95
+
96
+ // Note that we already have specialized member of Vectorized<scalar_t> for
97
+ // BFloat16 so the following functions would run smoothly:
98
+ // using Vec = Vectorized<BFloat16>;
99
+ // Vec one = Vec(BFloat16(1));
100
+ // vec::map([](Vec x) { return one / (one + x.exp()); }, y_ptr, x_ptr, N);
101
+ //
102
+ // Then why we still need to specialize "functional"?
103
+ // If we do specialization at Vectorized<> level, the above example would need
104
+ // 3 pairs of conversion of bf16->fp32/fp32->bf16, each for ".exp()", "+" and
105
+ // "/". If we do specialization at vec::map<>() level, we have only 1 pair of
106
+ // conversion of bf16->fp32/fp32->bf16, for the input and output BFloat16
107
+ // vector only.
108
+ //
109
+ // The following BFloat16 functionality will only do data type conversion for
110
+ // input and output vector (reduce functionality will only convert the final
111
+ // scalar back to bf16). Compared to Vectorized<> specialization,
112
+ // 1. better performance since we have less data type conversion;
113
+ // 2. less rounding error since immediate results are kept in fp32;
114
+ // 3. accumulation done on data type of fp32.
115
+ //
116
+ // If you plan to extend this file, please ensure adding unit tests at
117
+ // aten/src/ATen/test/vec_test_all_types.cpp
118
+ //
119
+ template <
120
+ typename scalar_t,
121
+ typename Op,
122
+ typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
123
+ inline float reduce_all(const Op& vec_fun, const scalar_t* data, int64_t size) {
124
+ using bVec = vec::Vectorized<scalar_t>;
125
+ using fVec = vec::Vectorized<float>;
126
+ if (size < bVec::size()) {
127
+ bVec data_bvec = bVec::loadu(data, size);
128
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
129
+ if (size > fVec::size()) {
130
+ data_fvec0 = fVec::set(
131
+ data_fvec0, vec_fun(data_fvec0, data_fvec1), size - fVec::size());
132
+ return vec_reduce_all<float>(vec_fun, data_fvec0, fVec::size());
133
+ } else {
134
+ return vec_reduce_all<float>(vec_fun, data_fvec0, size);
135
+ }
136
+ }
137
+ int64_t d = bVec::size();
138
+ bVec acc_bvec = bVec::loadu(data);
139
+ auto [acc_fvec0, acc_fvec1] = convert_to_float<scalar_t>(acc_bvec);
140
+ for (; d < size - (size % bVec::size()); d += bVec::size()) {
141
+ bVec data_bvec = bVec::loadu(data + d);
142
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
143
+ acc_fvec0 = vec_fun(acc_fvec0, data_fvec0);
144
+ acc_fvec1 = vec_fun(acc_fvec1, data_fvec1);
145
+ }
146
+ if (size - d > 0) {
147
+ bVec data_bvec = bVec::loadu(data + d, size - d);
148
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
149
+ if (size - d > fVec::size()) {
150
+ acc_fvec0 = vec_fun(acc_fvec0, data_fvec0);
151
+ acc_fvec1 = fVec::set(
152
+ acc_fvec1, vec_fun(acc_fvec1, data_fvec1), size - d - fVec::size());
153
+ } else {
154
+ acc_fvec0 =
155
+ fVec::set(acc_fvec0, vec_fun(acc_fvec0, data_fvec0), size - d);
156
+ }
157
+ }
158
+ acc_fvec0 = vec_fun(acc_fvec0, acc_fvec1);
159
+ return vec_reduce_all<float>(vec_fun, acc_fvec0);
160
+ }
161
+
162
+ template <
163
+ typename scalar_t,
164
+ typename Op1,
165
+ typename Op2,
166
+ typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
167
+ inline std::pair<float, float> reduce2_all(
168
+ const Op1& vec_fun1,
169
+ const Op2& vec_fun2,
170
+ const scalar_t* data,
171
+ int64_t size) {
172
+ using bVec = vec::Vectorized<scalar_t>;
173
+ using fVec = vec::Vectorized<float>;
174
+ if (size < bVec::size()) {
175
+ bVec data_bvec = bVec::loadu(data, size);
176
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
177
+ if (size > fVec::size()) {
178
+ fVec acc1_fvec = fVec::set(
179
+ data_fvec0, vec_fun1(data_fvec0, data_fvec1), size - fVec::size());
180
+ fVec acc2_fvec = fVec::set(
181
+ data_fvec0, vec_fun2(data_fvec0, data_fvec1), size - fVec::size());
182
+ return std::pair<scalar_t, scalar_t>(
183
+ vec_reduce_all<float>(vec_fun1, acc1_fvec, fVec::size()),
184
+ vec_reduce_all<float>(vec_fun2, acc2_fvec, fVec::size()));
185
+ } else {
186
+ return std::pair<scalar_t, scalar_t>(
187
+ vec_reduce_all<float>(vec_fun1, data_fvec0, size),
188
+ vec_reduce_all<float>(vec_fun2, data_fvec0, size));
189
+ }
190
+ }
191
+ int64_t d = bVec::size();
192
+ bVec acc_bvec = bVec::loadu(data);
193
+ auto [acc1_fvec0, acc1_fvec1] = convert_to_float<scalar_t>(acc_bvec);
194
+ auto [acc2_fvec0, acc2_fvec1] = convert_to_float<scalar_t>(acc_bvec);
195
+ for (; d < size - (size % bVec::size()); d += bVec::size()) {
196
+ bVec data_bvec = bVec::loadu(data + d);
197
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
198
+ acc1_fvec0 = vec_fun1(acc1_fvec0, data_fvec0);
199
+ acc1_fvec1 = vec_fun1(acc1_fvec1, data_fvec1);
200
+ acc2_fvec0 = vec_fun2(acc2_fvec0, data_fvec0);
201
+ acc2_fvec1 = vec_fun2(acc2_fvec1, data_fvec1);
202
+ }
203
+ if (size - d > 0) {
204
+ bVec data_bvec = bVec::loadu(data + d, size - d);
205
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
206
+ if (size - d > fVec::size()) {
207
+ acc1_fvec0 = vec_fun1(acc1_fvec0, data_fvec0);
208
+ acc1_fvec1 = fVec::set(
209
+ acc1_fvec1,
210
+ vec_fun1(acc1_fvec1, data_fvec1),
211
+ size - d - fVec::size());
212
+ acc2_fvec0 = vec_fun2(acc2_fvec0, data_fvec0);
213
+ acc2_fvec1 = fVec::set(
214
+ acc2_fvec1,
215
+ vec_fun2(acc2_fvec1, data_fvec1),
216
+ size - d - fVec::size());
217
+ } else {
218
+ acc1_fvec0 =
219
+ fVec::set(acc1_fvec0, vec_fun1(acc1_fvec0, data_fvec0), size - d);
220
+ acc2_fvec0 =
221
+ fVec::set(acc2_fvec0, vec_fun2(acc2_fvec0, data_fvec0), size - d);
222
+ }
223
+ }
224
+ acc1_fvec0 = vec_fun1(acc1_fvec0, acc1_fvec1);
225
+ acc2_fvec0 = vec_fun2(acc2_fvec0, acc2_fvec1);
226
+ return std::pair<scalar_t, scalar_t>(
227
+ vec_reduce_all<float>(vec_fun1, acc1_fvec0),
228
+ vec_reduce_all<float>(vec_fun2, acc2_fvec0));
229
+ }
230
+
231
+ template <
232
+ typename scalar_t,
233
+ typename MapOp,
234
+ typename ReduceOp,
235
+ typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
236
+ inline float map_reduce_all(
237
+ const MapOp& map_fun,
238
+ const ReduceOp& red_fun,
239
+ const scalar_t* data,
240
+ int64_t size) {
241
+ using bVec = vec::Vectorized<scalar_t>;
242
+ using fVec = vec::Vectorized<float>;
243
+ if (size < bVec::size()) {
244
+ bVec data_bvec = bVec::loadu(data, size);
245
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
246
+ if (size > fVec::size()) {
247
+ data_fvec0 = map_fun(data_fvec0);
248
+ data_fvec1 = map_fun(data_fvec1);
249
+ data_fvec0 = fVec::set(
250
+ data_fvec0, red_fun(data_fvec0, data_fvec1), size - fVec::size());
251
+ return vec_reduce_all<float>(red_fun, data_fvec0, fVec::size());
252
+ } else {
253
+ data_fvec0 = map_fun(data_fvec0);
254
+ return vec_reduce_all<float>(red_fun, data_fvec0, size);
255
+ }
256
+ }
257
+ int64_t d = bVec::size();
258
+ bVec acc_bvec = bVec::loadu(data);
259
+ auto [acc_fvec0, acc_fvec1] = convert_to_float<scalar_t>(acc_bvec);
260
+ acc_fvec0 = map_fun(acc_fvec0);
261
+ acc_fvec1 = map_fun(acc_fvec1);
262
+ for (; d < size - (size % bVec::size()); d += bVec::size()) {
263
+ bVec data_bvec = bVec::loadu(data + d);
264
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
265
+ data_fvec0 = map_fun(data_fvec0);
266
+ data_fvec1 = map_fun(data_fvec1);
267
+ acc_fvec0 = red_fun(acc_fvec0, data_fvec0);
268
+ acc_fvec1 = red_fun(acc_fvec1, data_fvec1);
269
+ }
270
+ if (size - d > 0) {
271
+ bVec data_bvec = bVec::loadu(data + d, size - d);
272
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
273
+ if (size - d > fVec::size()) {
274
+ data_fvec0 = map_fun(data_fvec0);
275
+ data_fvec1 = map_fun(data_fvec1);
276
+ acc_fvec0 = red_fun(acc_fvec0, data_fvec0);
277
+ acc_fvec1 = fVec::set(
278
+ acc_fvec1, red_fun(acc_fvec1, data_fvec1), size - d - fVec::size());
279
+ } else {
280
+ data_fvec0 = map_fun(data_fvec0);
281
+ acc_fvec0 =
282
+ fVec::set(acc_fvec0, red_fun(acc_fvec0, data_fvec0), size - d);
283
+ }
284
+ }
285
+ acc_fvec0 = red_fun(acc_fvec0, acc_fvec1);
286
+ return vec_reduce_all<float>(red_fun, acc_fvec0);
287
+ }
288
+
289
+ template <
290
+ typename scalar_t,
291
+ typename MapOp,
292
+ typename ReduceOp,
293
+ typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
294
+ inline float map2_reduce_all(
295
+ const MapOp& map_fun,
296
+ const ReduceOp& red_fun,
297
+ const scalar_t* data,
298
+ const scalar_t* data2,
299
+ int64_t size) {
300
+ using bVec = vec::Vectorized<scalar_t>;
301
+ using fVec = vec::Vectorized<float>;
302
+ if (size < bVec::size()) {
303
+ bVec data_bvec = bVec::loadu(data, size);
304
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
305
+ bVec data2_bvec = bVec::loadu(data2, size);
306
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
307
+ if (size > fVec::size()) {
308
+ data_fvec0 = map_fun(data_fvec0, data2_fvec0);
309
+ data_fvec1 = map_fun(data_fvec1, data2_fvec1);
310
+ data_fvec0 = fVec::set(
311
+ data_fvec0, red_fun(data_fvec0, data_fvec1), size - fVec::size());
312
+ return vec_reduce_all<float>(red_fun, data_fvec0, fVec::size());
313
+ } else {
314
+ data_fvec0 = map_fun(data_fvec0, data2_fvec0);
315
+ return vec_reduce_all<float>(red_fun, data_fvec0, size);
316
+ }
317
+ }
318
+ int64_t d = bVec::size();
319
+ bVec acc_bvec = bVec::loadu(data);
320
+ auto [acc_fvec0, acc_fvec1] = convert_to_float<scalar_t>(acc_bvec);
321
+ bVec acc2_bvec = bVec::loadu(data2);
322
+ auto [acc2_fvec0, acc2_fvec1] = convert_to_float<scalar_t>(acc2_bvec);
323
+ acc_fvec0 = map_fun(acc_fvec0, acc2_fvec0);
324
+ acc_fvec1 = map_fun(acc_fvec1, acc2_fvec1);
325
+ for (; d < size - (size % bVec::size()); d += bVec::size()) {
326
+ bVec data_bvec = bVec::loadu(data + d);
327
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
328
+ bVec data2_bvec = bVec::loadu(data2 + d);
329
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
330
+ data_fvec0 = map_fun(data_fvec0, data2_fvec0);
331
+ data_fvec1 = map_fun(data_fvec1, data2_fvec1);
332
+ acc_fvec0 = red_fun(acc_fvec0, data_fvec0);
333
+ acc_fvec1 = red_fun(acc_fvec1, data_fvec1);
334
+ }
335
+ if (size - d > 0) {
336
+ bVec data_bvec = bVec::loadu(data + d, size - d);
337
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
338
+ bVec data2_bvec = bVec::loadu(data2 + d, size - d);
339
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
340
+ if (size - d > fVec::size()) {
341
+ data_fvec0 = map_fun(data_fvec0, data2_fvec0);
342
+ data_fvec1 = map_fun(data_fvec1, data2_fvec1);
343
+ acc_fvec0 = red_fun(acc_fvec0, data_fvec0);
344
+ acc_fvec1 = fVec::set(
345
+ acc_fvec1, red_fun(acc_fvec1, data_fvec1), size - d - fVec::size());
346
+ } else {
347
+ data_fvec0 = map_fun(data_fvec0, data2_fvec0);
348
+ acc_fvec0 =
349
+ fVec::set(acc_fvec0, red_fun(acc_fvec0, data_fvec0), size - d);
350
+ }
351
+ }
352
+ acc_fvec0 = red_fun(acc_fvec0, acc_fvec1);
353
+ return vec_reduce_all<float>(red_fun, acc_fvec0);
354
+ }
355
+
356
+ template <
357
+ typename scalar_t,
358
+ typename MapOp,
359
+ typename ReduceOp,
360
+ typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
361
+ inline float map3_reduce_all(
362
+ const MapOp& map_fun,
363
+ const ReduceOp& red_fun,
364
+ const scalar_t* data,
365
+ const scalar_t* data2,
366
+ const scalar_t* data3,
367
+ int64_t size) {
368
+ using bVec = vec::Vectorized<scalar_t>;
369
+ using fVec = vec::Vectorized<float>;
370
+ if (size < bVec::size()) {
371
+ bVec data_bvec = bVec::loadu(data, size);
372
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
373
+ bVec data2_bvec = bVec::loadu(data2, size);
374
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
375
+ bVec data3_bvec = bVec::loadu(data3, size);
376
+ auto [data3_fvec0, data3_fvec1] = convert_to_float<scalar_t>(data3_bvec);
377
+ if (size > fVec::size()) {
378
+ data_fvec0 = map_fun(data_fvec0, data2_fvec0, data3_fvec0);
379
+ data_fvec1 = map_fun(data_fvec1, data2_fvec1, data3_fvec1);
380
+ data_fvec0 = fVec::set(
381
+ data_fvec0, red_fun(data_fvec0, data_fvec1), size - fVec::size());
382
+ return vec_reduce_all<float>(red_fun, data_fvec0, fVec::size());
383
+ } else {
384
+ data_fvec0 = map_fun(data_fvec0, data2_fvec0, data3_fvec0);
385
+ return vec_reduce_all<float>(red_fun, data_fvec0, size);
386
+ }
387
+ }
388
+ int64_t d = bVec::size();
389
+ bVec acc_bvec = bVec::loadu(data);
390
+ auto [acc_fvec0, acc_fvec1] = convert_to_float<scalar_t>(acc_bvec);
391
+ bVec acc2_bvec = bVec::loadu(data2);
392
+ auto [acc2_fvec0, acc2_fvec1] = convert_to_float<scalar_t>(acc2_bvec);
393
+ bVec acc3_bvec = bVec::loadu(data3);
394
+ auto [acc3_fvec0, acc3_fvec1] = convert_to_float<scalar_t>(acc3_bvec);
395
+ acc_fvec0 = map_fun(acc_fvec0, acc2_fvec0, acc3_fvec0);
396
+ acc_fvec1 = map_fun(acc_fvec1, acc2_fvec1, acc3_fvec1);
397
+ for (; d < size - (size % bVec::size()); d += bVec::size()) {
398
+ bVec data_bvec = bVec::loadu(data + d);
399
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
400
+ bVec data2_bvec = bVec::loadu(data2 + d);
401
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
402
+ bVec data3_bvec = bVec::loadu(data3 + d);
403
+ auto [data3_fvec0, data3_fvec1] = convert_to_float<scalar_t>(data3_bvec);
404
+ data_fvec0 = map_fun(data_fvec0, data2_fvec0, data3_fvec0);
405
+ data_fvec1 = map_fun(data_fvec1, data2_fvec1, data3_fvec1);
406
+ acc_fvec0 = red_fun(acc_fvec0, data_fvec0);
407
+ acc_fvec1 = red_fun(acc_fvec1, data_fvec1);
408
+ }
409
+ if (size - d > 0) {
410
+ bVec data_bvec = bVec::loadu(data + d, size - d);
411
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
412
+ bVec data2_bvec = bVec::loadu(data2 + d, size - d);
413
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
414
+ bVec data3_bvec = bVec::loadu(data3 + d, size - d);
415
+ auto [data3_fvec0, data3_fvec1] = convert_to_float<scalar_t>(data3_bvec);
416
+ if (size - d > fVec::size()) {
417
+ data_fvec0 = map_fun(data_fvec0, data2_fvec0, data3_fvec0);
418
+ data_fvec1 = map_fun(data_fvec1, data2_fvec1, data3_fvec1);
419
+ acc_fvec0 = red_fun(acc_fvec0, data_fvec0);
420
+ acc_fvec1 = fVec::set(
421
+ acc_fvec1, red_fun(acc_fvec1, data_fvec1), size - d - fVec::size());
422
+ } else {
423
+ data_fvec0 = map_fun(data_fvec0, data2_fvec0, data3_fvec0);
424
+ acc_fvec0 =
425
+ fVec::set(acc_fvec0, red_fun(acc_fvec0, data_fvec0), size - d);
426
+ }
427
+ }
428
+ acc_fvec0 = red_fun(acc_fvec0, acc_fvec1);
429
+ return vec_reduce_all<float>(red_fun, acc_fvec0);
430
+ }
431
+
432
+ template <
433
+ typename scalar_t,
434
+ typename Op,
435
+ typename std::enable_if_t<
436
+ !(!detail::should_prefer_converting_through_float_v<scalar_t> &&
437
+ std::is_invocable_v<Op, vec::Vectorized<scalar_t>>),
438
+ int> = 0>
439
+ inline void map(
440
+ const Op& vec_fun,
441
+ scalar_t* output_data,
442
+ const scalar_t* input_data,
443
+ int64_t size) {
444
+ using bVec = vec::Vectorized<scalar_t>;
445
+ using fVec = vec::Vectorized<float>;
446
+ int64_t d = 0;
447
+ for (; d < size - (size % bVec::size()); d += bVec::size()) {
448
+ bVec data_bvec = bVec::loadu(input_data + d);
449
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
450
+ fVec output_fvec0 = vec_fun(data_fvec0);
451
+ fVec output_fvec1 = vec_fun(data_fvec1);
452
+ bVec output_bvec = convert_from_float<scalar_t>(output_fvec0, output_fvec1);
453
+ output_bvec.store(output_data + d);
454
+ }
455
+ if (size - d > 0) {
456
+ bVec data_bvec = bVec::loadu(input_data + d, size - d);
457
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
458
+ fVec output_fvec0 = vec_fun(data_fvec0);
459
+ fVec output_fvec1 = vec_fun(data_fvec1);
460
+ bVec output_bvec = convert_from_float<scalar_t>(output_fvec0, output_fvec1);
461
+ output_bvec.store(output_data + d, size - d);
462
+ }
463
+ }
464
+
465
+ template <
466
+ typename scalar_t,
467
+ typename Op,
468
+ typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
469
+ inline void map(
470
+ const Op& vec_fun,
471
+ scalar_t* output_data,
472
+ const float* input_data,
473
+ int64_t size) {
474
+ using bVec = vec::Vectorized<scalar_t>;
475
+ using fVec = vec::Vectorized<float>;
476
+ int64_t d = 0;
477
+ for (; d < size - (size % bVec::size()); d += bVec::size()) {
478
+ fVec data_fvec0 = fVec::loadu(input_data + d);
479
+ fVec data_fvec1 = fVec::loadu(input_data + d + fVec::size());
480
+ fVec output_fvec0 = vec_fun(data_fvec0);
481
+ fVec output_fvec1 = vec_fun(data_fvec1);
482
+ bVec output_bvec = convert_from_float<scalar_t>(output_fvec0, output_fvec1);
483
+ output_bvec.store(output_data + d);
484
+ }
485
+ if (size - d > 0) {
486
+ fVec data_fvec0, data_fvec1;
487
+ if (size - d > fVec::size()) {
488
+ data_fvec0 = fVec::loadu(input_data + d);
489
+ data_fvec1 =
490
+ fVec::loadu(input_data + d + fVec::size(), size - d - fVec::size());
491
+ } else {
492
+ // choose to align with behaviour of bVec::loadu(ptr, size),
493
+ // which leaves data_fvec1 uninitialized
494
+ data_fvec0 = fVec::loadu(input_data + d, size - d);
495
+ }
496
+ fVec output_fvec0 = vec_fun(data_fvec0);
497
+ fVec output_fvec1 = vec_fun(data_fvec1);
498
+ bVec output_bvec = convert_from_float<scalar_t>(output_fvec0, output_fvec1);
499
+ output_bvec.store(output_data + d, size - d);
500
+ }
501
+ }
502
+
503
+ template <
504
+ typename scalar_t,
505
+ typename Op,
506
+ typename std::enable_if_t<
507
+ !(!detail::should_prefer_converting_through_float_v<scalar_t> &&
508
+ std::is_invocable_v<
509
+ Op,
510
+ vec::Vectorized<scalar_t>,
511
+ vec::Vectorized<scalar_t>>),
512
+ int> = 0>
513
+ inline void map2(
514
+ const Op& vec_fun,
515
+ scalar_t* output_data,
516
+ const scalar_t* input_data,
517
+ const scalar_t* input_data2,
518
+ int64_t size) {
519
+ using bVec = vec::Vectorized<scalar_t>;
520
+ using fVec = vec::Vectorized<float>;
521
+ int64_t d = 0;
522
+ for (; d < size - (size % bVec::size()); d += bVec::size()) {
523
+ bVec data_bvec = bVec::loadu(input_data + d);
524
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
525
+ bVec data2_bvec = bVec::loadu(input_data2 + d);
526
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
527
+ fVec output_fvec0 = vec_fun(data_fvec0, data2_fvec0);
528
+ fVec output_fvec1 = vec_fun(data_fvec1, data2_fvec1);
529
+ bVec output_bvec = convert_from_float<scalar_t>(output_fvec0, output_fvec1);
530
+ output_bvec.store(output_data + d);
531
+ }
532
+ if (size - d > 0) {
533
+ bVec data_bvec = bVec::loadu(input_data + d, size - d);
534
+ auto [data_fvec0, data_fvec1] = convert_to_float<scalar_t>(data_bvec);
535
+ bVec data2_bvec = bVec::loadu(input_data2 + d, size - d);
536
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
537
+ fVec output_fvec0 = vec_fun(data_fvec0, data2_fvec0);
538
+ fVec output_fvec1 = vec_fun(data_fvec1, data2_fvec1);
539
+ bVec output_bvec = convert_from_float<scalar_t>(output_fvec0, output_fvec1);
540
+ output_bvec.store(output_data + d, size - d);
541
+ }
542
+ }
543
+
544
+ template <
545
+ typename scalar_t,
546
+ typename Op,
547
+ typename std::enable_if_t<
548
+ !(!detail::should_prefer_converting_through_float_v<scalar_t> &&
549
+ std::is_invocable_v<
550
+ Op,
551
+ vec::Vectorized<scalar_t>,
552
+ vec::Vectorized<scalar_t>,
553
+ vec::Vectorized<scalar_t>>),
554
+ int> = 0>
555
+ inline void map3(
556
+ const Op& vec_fun,
557
+ scalar_t* output_data,
558
+ const scalar_t* input_data1,
559
+ const scalar_t* input_data2,
560
+ const scalar_t* input_data3,
561
+ int64_t size) {
562
+ using bVec = vec::Vectorized<scalar_t>;
563
+ using fVec = vec::Vectorized<float>;
564
+ int64_t d = 0;
565
+ for (; d < size - (size % bVec::size()); d += bVec::size()) {
566
+ bVec data1_bvec = bVec::loadu(input_data1 + d);
567
+ auto [data1_fvec0, data1_fvec1] = convert_to_float<scalar_t>(data1_bvec);
568
+ bVec data2_bvec = bVec::loadu(input_data2 + d);
569
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
570
+ bVec data3_bvec = bVec::loadu(input_data3 + d);
571
+ auto [data3_fvec0, data3_fvec1] = convert_to_float<scalar_t>(data3_bvec);
572
+ fVec output_fvec0 = vec_fun(data1_fvec0, data2_fvec0, data3_fvec0);
573
+ fVec output_fvec1 = vec_fun(data1_fvec1, data2_fvec1, data3_fvec1);
574
+ bVec output_bvec = convert_from_float<scalar_t>(output_fvec0, output_fvec1);
575
+ output_bvec.store(output_data + d);
576
+ }
577
+ if (size - d > 0) {
578
+ bVec data1_bvec = bVec::loadu(input_data1 + d, size - d);
579
+ auto [data1_fvec0, data1_fvec1] = convert_to_float<scalar_t>(data1_bvec);
580
+ bVec data2_bvec = bVec::loadu(input_data2 + d, size - d);
581
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
582
+ bVec data3_bvec = bVec::loadu(input_data3 + d, size - d);
583
+ auto [data3_fvec0, data3_fvec1] = convert_to_float<scalar_t>(data3_bvec);
584
+ fVec output_fvec0 = vec_fun(data1_fvec0, data2_fvec0, data3_fvec0);
585
+ fVec output_fvec1 = vec_fun(data1_fvec1, data2_fvec1, data3_fvec1);
586
+ bVec output_bvec = convert_from_float<scalar_t>(output_fvec0, output_fvec1);
587
+ output_bvec.store(output_data + d, size - d);
588
+ }
589
+ }
590
+
591
+ template <
592
+ typename scalar_t,
593
+ typename Op,
594
+ typename std::enable_if_t<
595
+ !(!detail::should_prefer_converting_through_float_v<scalar_t> &&
596
+ std::is_invocable_v<
597
+ Op,
598
+ vec::Vectorized<scalar_t>,
599
+ vec::Vectorized<scalar_t>,
600
+ vec::Vectorized<scalar_t>,
601
+ vec::Vectorized<scalar_t>>),
602
+ int> = 0>
603
+ inline void map4(
604
+ const Op& vec_fun,
605
+ scalar_t* output_data,
606
+ const scalar_t* input_data1,
607
+ const scalar_t* input_data2,
608
+ const scalar_t* input_data3,
609
+ const scalar_t* input_data4,
610
+ int64_t size) {
611
+ using bVec = vec::Vectorized<scalar_t>;
612
+ using fVec = vec::Vectorized<float>;
613
+ int64_t d = 0;
614
+ for (; d < size - (size % bVec::size()); d += bVec::size()) {
615
+ bVec data1_bvec = bVec::loadu(input_data1 + d);
616
+ auto [data1_fvec0, data1_fvec1] = convert_to_float<scalar_t>(data1_bvec);
617
+ bVec data2_bvec = bVec::loadu(input_data2 + d);
618
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
619
+ bVec data3_bvec = bVec::loadu(input_data3 + d);
620
+ auto [data3_fvec0, data3_fvec1] = convert_to_float<scalar_t>(data3_bvec);
621
+ bVec data4_bvec = bVec::loadu(input_data4 + d);
622
+ auto [data4_fvec0, data4_fvec1] = convert_to_float<scalar_t>(data4_bvec);
623
+ fVec output_fvec0 =
624
+ vec_fun(data1_fvec0, data2_fvec0, data3_fvec0, data4_fvec0);
625
+ fVec output_fvec1 =
626
+ vec_fun(data1_fvec1, data2_fvec1, data3_fvec1, data4_fvec1);
627
+ bVec output_bvec = convert_from_float<scalar_t>(output_fvec0, output_fvec1);
628
+ output_bvec.store(output_data + d);
629
+ }
630
+ if (size - d > 0) {
631
+ bVec data1_bvec = bVec::loadu(input_data1 + d, size - d);
632
+ auto [data1_fvec0, data1_fvec1] = convert_to_float<scalar_t>(data1_bvec);
633
+ bVec data2_bvec = bVec::loadu(input_data2 + d, size - d);
634
+ auto [data2_fvec0, data2_fvec1] = convert_to_float<scalar_t>(data2_bvec);
635
+ bVec data3_bvec = bVec::loadu(input_data3 + d, size - d);
636
+ auto [data3_fvec0, data3_fvec1] = convert_to_float<scalar_t>(data3_bvec);
637
+ bVec data4_bvec = bVec::loadu(input_data4 + d, size - d);
638
+ auto [data4_fvec0, data4_fvec1] = convert_to_float<scalar_t>(data4_bvec);
639
+ fVec output_fvec0 =
640
+ vec_fun(data1_fvec0, data2_fvec0, data3_fvec0, data4_fvec0);
641
+ fVec output_fvec1 =
642
+ vec_fun(data1_fvec1, data2_fvec1, data3_fvec1, data4_fvec1);
643
+ bVec output_bvec = convert_from_float<scalar_t>(output_fvec0, output_fvec1);
644
+ output_bvec.store(output_data + d, size - d);
645
+ }
646
+ }
647
+
648
+ } // namespace at::vec
649
+
650
+ #else
651
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
652
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/intrinsics.h ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #include <torch/headeronly/cpu/vec/intrinsics.h>
3
+
4
+ #else
5
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
6
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/vec.h ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #if defined(CPU_CAPABILITY_AVX512)
5
+ #include <ATen/cpu/vec/vec512/vec512.h>
6
+ #else
7
+ #include <ATen/cpu/vec/vec128/vec128.h>
8
+ #include <ATen/cpu/vec/vec256/vec256.h>
9
+ #endif
10
+
11
+ namespace at::vec {
12
+ // See Note [CPU_CAPABILITY namespace]
13
+ inline namespace CPU_CAPABILITY {
14
+
15
+ inline Vectorized<bool> convert_to_bool(Vectorized<int8_t> x) {
16
+ __at_align__ bool buffer[x.size()];
17
+ x.ne(Vectorized<int8_t>(0)).store(buffer);
18
+
19
+ Vectorized<bool> ret;
20
+ static_assert(x.size() == ret.size());
21
+ std::memcpy(ret, buffer, ret.size() * sizeof(bool));
22
+ return ret;
23
+ }
24
+
25
+ template <>
26
+ inline Vectorized<bool> Vectorized<bool>::loadu(const void* ptr) {
27
+ // See NOTE [Loading boolean values]
28
+ return convert_to_bool(Vectorized<int8_t>::loadu(ptr));
29
+ }
30
+
31
+ template <>
32
+ inline Vectorized<bool> Vectorized<bool>::loadu(
33
+ const void* ptr,
34
+ int64_t count) {
35
+ // See NOTE [Loading boolean values]
36
+ return convert_to_bool(Vectorized<int8_t>::loadu(ptr, count));
37
+ }
38
+
39
+ template <typename VT>
40
+ struct VecHoldType {
41
+ using hold_type = typename VT::value_type;
42
+ };
43
+
44
+ template <>
45
+ struct VecHoldType<Vectorized<BFloat16>> {
46
+ using hold_type = BFloat16;
47
+ };
48
+
49
+ template <>
50
+ struct VecHoldType<Vectorized<Half>> {
51
+ using hold_type = Half;
52
+ };
53
+
54
+ template <typename VT>
55
+ using vechold_type = typename VecHoldType<VT>::hold_type;
56
+
57
+ } // namespace CPU_CAPABILITY
58
+ } // namespace at::vec
59
+
60
+ #else
61
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
62
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/vec_base.h ADDED
@@ -0,0 +1,1537 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+ #if defined(__GNUC__) && __GNUC__ == 10 && __GNUC_MINOR__ <= 2 && \
4
+ defined(__ARM_FEATURE_SVE)
5
+ // Workaround for https: //gcc.gnu.org/bugzilla/show_bug.cgi?id=117161
6
+ #pragma GCC optimize("no-tree-vectorize")
7
+ #endif
8
+
9
+ // DO NOT DEFINE STATIC DATA IN THIS HEADER!
10
+ // See Note [Do not compile initializers with AVX]
11
+ //
12
+ // Note [Do not compile initializers with AVX]
13
+ // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
14
+ // If you define a static initializer in this file, the initialization will use
15
+ // AVX instructions because these object files are compiled with AVX enabled.
16
+ // We need to avoid non-trivial global data in these architecture specific files
17
+ // because there's no way to guard the global initializers with CPU capability
18
+ // detection.
19
+ //
20
+ // See https://github.com/pytorch/pytorch/issues/37577 for an instance
21
+ // of this bug in the past.
22
+
23
+ #include <algorithm>
24
+ #include <array>
25
+ #include <cassert>
26
+ #include <climits>
27
+ #include <cmath>
28
+ #include <cstring>
29
+ #include <functional>
30
+ #include <type_traits>
31
+
32
+ #include <ATen/NumericUtils.h>
33
+ #include <ATen/cpu/vec/intrinsics.h>
34
+ #include <ATen/native/Math.h>
35
+ #include <ATen/native/cpu/zmath.h>
36
+ #include <c10/macros/Macros.h>
37
+ #include <c10/util/BFloat16-math.h>
38
+ #include <c10/util/BFloat16.h>
39
+ #include <c10/util/Half.h>
40
+ #include <c10/util/Load.h>
41
+ #include <c10/util/TypeCast.h>
42
+ #include <c10/util/copysign.h>
43
+ #include <c10/util/irange.h>
44
+
45
+ #if defined(__GNUC__)
46
+ #define __FORCE_INLINE __attribute__((always_inline)) inline
47
+ #elif defined(_MSC_VER)
48
+ #define __FORCE_INLINE __forceinline
49
+ #endif
50
+
51
+ #if defined(_MSC_FULL_VER)
52
+ /*
53
+ https://learn.microsoft.com/en-us/cpp/overview/compiler-versions?view=msvc-170
54
+ Use _MSC_FULL_VER to identify current compiler is msvc,
55
+ Windows llvm will not have this definition.
56
+ */
57
+ #define __msvc_cl__
58
+ #endif
59
+
60
+ // These macros helped us unify vec_base.h
61
+ #ifdef CPU_CAPABILITY_AVX512
62
+ #if defined(__GNUC__)
63
+ #define __at_align__ __attribute__((aligned(64)))
64
+ #elif defined(_WIN32)
65
+ #define __at_align__ __declspec(align(64))
66
+ #else
67
+ #define __at_align__
68
+ #endif
69
+ #define VECTOR_WIDTH 64
70
+ #define int_vector __m512i
71
+ #elif defined(__aarch64__) && \
72
+ !defined(CPU_CAPABILITY_SVE) // CPU_CAPABILITY_AVX512
73
+ // SVE code expects 256-vectors; leave that set for SVE?
74
+ #if defined(__GNUC__)
75
+ #define __at_align__ __attribute__((aligned(16)))
76
+ #elif defined(_WIN32)
77
+ #define __at_align__ __declspec(align(16))
78
+ #else
79
+ #define __at_align__
80
+ #endif
81
+ #define VECTOR_WIDTH 16
82
+ #else // CPU_CAPABILITY_AVX512
83
+ #if defined(__GNUC__)
84
+ #define __at_align__ __attribute__((aligned(32)))
85
+ #elif defined(_WIN32)
86
+ #define __at_align__ __declspec(align(32))
87
+ #else
88
+ #define __at_align__
89
+ #endif
90
+ #define VECTOR_WIDTH 32
91
+ #define int_vector __m256i
92
+ #endif // CPU_CAPABILITY_AVX512
93
+
94
+ namespace at::vec {
95
+ // See Note [CPU_CAPABILITY namespace]
96
+ inline namespace CPU_CAPABILITY {
97
+ // at::Half and at::BFloat16 should be treated as floating point
98
+ template <typename T>
99
+ struct is_floating_point
100
+ : std::integral_constant<
101
+ bool,
102
+ std::is_floating_point_v<T> || std::is_same_v<T, at::Half> ||
103
+ std::is_same_v<T, at::BFloat16>> {};
104
+
105
+ template <typename T>
106
+ constexpr bool is_floating_point_v = is_floating_point<T>::value;
107
+
108
+ template <typename T>
109
+ struct is_reduced_floating_point
110
+ : std::integral_constant<
111
+ bool,
112
+ std::is_same_v<T, at::Half> || std::is_same_v<T, at::BFloat16>> {};
113
+
114
+ template <typename T>
115
+ constexpr bool is_reduced_floating_point_v =
116
+ is_reduced_floating_point<T>::value;
117
+
118
+ template <typename T>
119
+ struct is_8bit_integer
120
+ : std::integral_constant<
121
+ bool,
122
+ std::is_same_v<T, unsigned char> || std::is_same_v<T, signed char>> {
123
+ };
124
+
125
+ template <typename T>
126
+ constexpr bool is_8bit_integer_v = is_8bit_integer<T>::value;
127
+
128
+ template <size_t n>
129
+ struct int_of_size;
130
+
131
+ #define DEFINE_INT_OF_SIZE(int_t) \
132
+ template <> \
133
+ struct int_of_size<sizeof(int_t)> { \
134
+ using type = int_t; \
135
+ }
136
+
137
+ DEFINE_INT_OF_SIZE(int64_t);
138
+ DEFINE_INT_OF_SIZE(int32_t);
139
+ DEFINE_INT_OF_SIZE(int16_t);
140
+ DEFINE_INT_OF_SIZE(int8_t);
141
+
142
+ #undef DEFINE_INT_OF_SIZE
143
+
144
+ template <typename T>
145
+ using int_same_size_t = typename int_of_size<sizeof(T)>::type;
146
+
147
+ /**
148
+ * Detect at compile time whether Vectorized has an explicit
149
+ * specialization for T. (You are required to specialize this type
150
+ * whenever you specialize Vectorized). Useful for generic algorithms
151
+ * to decide whether to rely on a specialization being fast. For
152
+ * example, they might choose to handle reduced-precision floating
153
+ * point types directly if they're supported, or convert through float
154
+ * if not.
155
+ */
156
+ #if defined(__s390x__)
157
+ template <class T, class TEMP = void>
158
+ #else
159
+ template <typename T>
160
+ #endif
161
+ struct is_vec_specialized_for : std::bool_constant<false> {
162
+ };
163
+
164
+ template <typename T>
165
+ constexpr bool is_vec_specialized_for_v = is_vec_specialized_for<T>::value;
166
+
167
+ // NOTE: If you specialize Vectorized on a type, you must define all
168
+ // operations! You must also specialize is_vec_specialized_for for
169
+ // that type.
170
+
171
+ // emulates Vectorized types
172
+ #if defined(__s390x__)
173
+ template <class T, class TEMP = void>
174
+ #else
175
+ template <class T>
176
+ #endif
177
+ struct Vectorized {
178
+ private:
179
+ __at_align__ T values[VECTOR_WIDTH / sizeof(T)];
180
+
181
+ public:
182
+ using value_type = T;
183
+ using size_type = int;
184
+
185
+ static constexpr size_type kSize = VECTOR_WIDTH / sizeof(T);
186
+ static constexpr size_type size() {
187
+ return kSize;
188
+ }
189
+ Vectorized() : values{static_cast<T>(0)} {}
190
+ Vectorized(T val) {
191
+ for (int i = 0; i != size(); i++) {
192
+ values[i] = val;
193
+ }
194
+ }
195
+ template <
196
+ typename... Args,
197
+ typename = std::enable_if_t<(sizeof...(Args) == size())>>
198
+ Vectorized(Args... vals) : values{vals...} {}
199
+ Vectorized(const T (&arr)[kSize]) {
200
+ std::memcpy(values, arr, sizeof(values));
201
+ }
202
+ // This also implies const T& operator[](int idx) const
203
+ inline operator const T*() const {
204
+ return values;
205
+ }
206
+ // This also implies T& operator[](int idx)
207
+ inline operator T*() {
208
+ return values;
209
+ }
210
+ // Return the values as char* for type punning
211
+ auto as_bytes() const -> const char* {
212
+ return reinterpret_cast<const char*>(values);
213
+ }
214
+ template <int64_t mask_>
215
+ static Vectorized<T> blend(const Vectorized<T>& a, const Vectorized<T>& b) {
216
+ int64_t mask = mask_;
217
+ Vectorized vector;
218
+ for (const auto i : c10::irange(size())) {
219
+ if (mask & 0x01) {
220
+ vector[i] = b[i];
221
+ } else {
222
+ vector[i] = a[i];
223
+ }
224
+ mask = mask >> 1;
225
+ }
226
+ return vector;
227
+ }
228
+ // Workaround for https: //gcc.gnu.org/bugzilla/show_bug.cgi?id=117001
229
+ #if __GNUC__ <= 12 && !defined(__clang__) && defined(__ARM_FEATURE_SVE)
230
+ static Vectorized<T> __attribute__((optimize("-fno-tree-loop-vectorize")))
231
+ blendv(
232
+ const Vectorized<T>& a,
233
+ #else
234
+ static Vectorized<T> blendv(
235
+ const Vectorized<T>& a,
236
+ #endif
237
+ const Vectorized<T>& b,
238
+ const Vectorized<T>& mask) {
239
+ Vectorized vector;
240
+ int_same_size_t<T> buffer[size()];
241
+ mask.store(buffer);
242
+ for (const auto i : c10::irange(size())) {
243
+ if (buffer[i] & 0x01) {
244
+ vector[i] = b[i];
245
+ } else {
246
+ vector[i] = a[i];
247
+ }
248
+ }
249
+ return vector;
250
+ }
251
+ template <typename step_t> // step sometimes requires a higher precision type
252
+ // (e.g., T=int, step_t=double)
253
+ static Vectorized<T> arange(
254
+ T base = static_cast<T>(0),
255
+ step_t step = static_cast<step_t>(1)) {
256
+ Vectorized vector;
257
+ for (const auto i : c10::irange(size())) {
258
+ vector.values[i] = base + i * step;
259
+ }
260
+ return vector;
261
+ }
262
+ static Vectorized<T> set(
263
+ const Vectorized<T>& a,
264
+ const Vectorized<T>& b,
265
+ int64_t count = size()) {
266
+ Vectorized vector;
267
+ for (const auto i : c10::irange(size())) {
268
+ if (i < count) {
269
+ vector[i] = b[i];
270
+ } else {
271
+ vector[i] = a[i];
272
+ }
273
+ }
274
+ return vector;
275
+ }
276
+ static Vectorized<T> loadu(const void* ptr) {
277
+ Vectorized vector;
278
+ std::memcpy(vector.values, ptr, VECTOR_WIDTH);
279
+ return vector;
280
+ }
281
+ static Vectorized<T> loadu(const void* ptr, int64_t count) {
282
+ Vectorized vector;
283
+ std::memcpy(vector.values, ptr, count * sizeof(T));
284
+ return vector;
285
+ }
286
+ static Vectorized<T> loadu_one_fourth(const void* ptr) {
287
+ static_assert(
288
+ std::is_same_v<T, signed char> || std::is_same_v<T, unsigned char>,
289
+ "For byte types only");
290
+ return Vectorized::loadu(ptr, 8);
291
+ }
292
+
293
+ void store(void* ptr, int count = size()) const {
294
+ std::memcpy(ptr, values, count * sizeof(T));
295
+ }
296
+ int zero_mask() const {
297
+ // returns an integer mask where all zero elements are translated to 1-bit
298
+ // and others are translated to 0-bit
299
+ int mask = 0;
300
+ for (int i = 0; i < size(); ++i) {
301
+ if (values[i] == static_cast<T>(0)) {
302
+ mask |= (1 << i);
303
+ }
304
+ }
305
+ return mask;
306
+ }
307
+ Vectorized<T> isnan() const {
308
+ Vectorized<T> vector;
309
+ for (int64_t i = 0; i != size(); i++) {
310
+ if (_isnan(values[i])) {
311
+ std::memset(static_cast<void*>(vector.values + i), 0xFF, sizeof(T));
312
+ } else {
313
+ std::memset(static_cast<void*>(vector.values + i), 0, sizeof(T));
314
+ }
315
+ }
316
+ return vector;
317
+ }
318
+ bool has_inf_nan() const {
319
+ for (int64_t i = 0; i != size(); i++) {
320
+ if (_isnan(values[i]) || _isinf(values[i])) {
321
+ return true;
322
+ }
323
+ }
324
+ return false;
325
+ }
326
+ // MSVC versions between 14.36 and 14.42 has a loop unrolling bug on Windows
327
+ // Arm64
328
+ // See
329
+ // https://developercommunity.visualstudio.com/t/MSVC-loop-unrolling-problem-194033813-/10720692
330
+ #if defined(_WIN32) && defined(__aarch64__) && \
331
+ ((_MSVC_VER >= 1936) && (_MSVC_VER <= 1942))
332
+ Vectorized<T> map(T (*const f)(T)) const {
333
+ Vectorized<T> ret;
334
+ for (int64_t i = 0; i < size(); i++) {
335
+ ret[i] = f(values[i]);
336
+ if (++i < size())
337
+ ret[i] = f(values[i]);
338
+ }
339
+ return ret;
340
+ }
341
+ T reduce(T (*const f)(T)) const {
342
+ T ret = 0;
343
+ for (int64_t i = 0; i < size(); i++) {
344
+ ret = f(ret, values[i]);
345
+ if (++i < size())
346
+ ret = f(ret, values[i]);
347
+ }
348
+ return ret;
349
+ }
350
+ #else
351
+ Vectorized<T> map(T (*const f)(T)) const {
352
+ Vectorized<T> ret;
353
+ for (int64_t i = 0; i != size(); i++) {
354
+ ret[i] = f(values[i]);
355
+ }
356
+ return ret;
357
+ }
358
+ T reduce(T (*const f)(T)) const {
359
+ T ret = 0;
360
+ for (int64_t i = 0; i != size(); i++) {
361
+ ret = f(ret, values[i]);
362
+ }
363
+ return ret;
364
+ }
365
+ #endif
366
+ Vectorized<T> map(T (*const f)(const T&)) const {
367
+ Vectorized<T> ret;
368
+ for (int64_t i = 0; i != size(); i++) {
369
+ ret[i] = f(values[i]);
370
+ }
371
+ return ret;
372
+ }
373
+ T reduce(T (*const f)(const T&)) const {
374
+ T ret = 0;
375
+ for (int64_t i = 0; i != size(); i++) {
376
+ ret = f(ret, values[i]);
377
+ }
378
+ return ret;
379
+ }
380
+ template <
381
+ typename other_t_abs = T,
382
+ typename std::enable_if_t<
383
+ !is_floating_point_v<other_t_abs> &&
384
+ !c10::is_complex<other_t_abs>::value,
385
+ int> = 0>
386
+ Vectorized<T> abs() const {
387
+ // other_t_abs is for SFINAE and clarity. Make sure it is not changed.
388
+ static_assert(std::is_same_v<other_t_abs, T>, "other_t_abs must be T");
389
+ return map([](T x) -> T { return x < static_cast<T>(0) ? -x : x; });
390
+ }
391
+ template <
392
+ typename float_t_abs = T,
393
+ typename std::enable_if_t<is_floating_point_v<float_t_abs>, int> = 0>
394
+ Vectorized<T> abs() const {
395
+ // float_t_abs is for SFINAE and clarity. Make sure it is not changed.
396
+ static_assert(std::is_same_v<float_t_abs, T>, "float_t_abs must be T");
397
+ // Specifically deal with floating-point because the generic code above
398
+ // won't handle -0.0 (which should result in 0.0) properly.
399
+ return map([](T x) -> T { return std::abs(x); });
400
+ }
401
+ template <
402
+ typename complex_t_abs = T,
403
+ typename std::enable_if_t<c10::is_complex<complex_t_abs>::value, int> = 0>
404
+ Vectorized<T> abs() const {
405
+ // complex_t_abs is for SFINAE and clarity. Make sure it is not changed.
406
+ static_assert(std::is_same_v<complex_t_abs, T>, "complex_t_abs must be T");
407
+ // Specifically map() does not perform the type conversion needed by abs.
408
+ return map([](T x) { return static_cast<T>(std::abs(x)); });
409
+ }
410
+
411
+ template <
412
+ typename other_t_sgn = T,
413
+ typename std::enable_if_t<c10::is_complex<other_t_sgn>::value, int> = 0>
414
+ Vectorized<T> sgn() const {
415
+ return map(at::native::sgn_impl);
416
+ }
417
+
418
+ template <
419
+ typename other_t_angle = T,
420
+ typename std::enable_if_t<!c10::is_complex<other_t_angle>::value, int> =
421
+ 0>
422
+ Vectorized<T> angle() const {
423
+ // other_t_angle is for SFINAE and clarity. Make sure it is not changed.
424
+ static_assert(std::is_same_v<other_t_angle, T>, "other_t_angle must be T");
425
+ return map(at::native::angle_impl<T>); // compiler is unable to resolve the
426
+ // overload without <T>
427
+ }
428
+ template <
429
+ typename complex_t_angle = T,
430
+ typename std::enable_if_t<c10::is_complex<complex_t_angle>::value, int> =
431
+ 0>
432
+ Vectorized<T> angle() const {
433
+ // complex_t_angle is for SFINAE and clarity. Make sure it is not changed.
434
+ static_assert(
435
+ std::is_same_v<complex_t_angle, T>, "complex_t_angle must be T");
436
+ return map([](T x) { return static_cast<T>(std::arg(x)); });
437
+ }
438
+ template <
439
+ typename other_t_real = T,
440
+ typename std::enable_if_t<!c10::is_complex<other_t_real>::value, int> = 0>
441
+ Vectorized<T> real() const {
442
+ // other_t_real is for SFINAE and clarity. Make sure it is not changed.
443
+ static_assert(std::is_same_v<other_t_real, T>, "other_t_real must be T");
444
+ return *this;
445
+ }
446
+ template <
447
+ typename complex_t_real = T,
448
+ typename std::enable_if_t<c10::is_complex<complex_t_real>::value, int> =
449
+ 0>
450
+ Vectorized<T> real() const {
451
+ // complex_t_real is for SFINAE and clarity. Make sure it is not changed.
452
+ static_assert(
453
+ std::is_same_v<complex_t_real, T>, "complex_t_real must be T");
454
+ return map([](T x) { return static_cast<T>(x.real()); });
455
+ }
456
+ template <
457
+ typename other_t_imag = T,
458
+ typename std::enable_if_t<!c10::is_complex<other_t_imag>::value, int> = 0>
459
+ Vectorized<T> imag() const {
460
+ // other_t_imag is for SFINAE and clarity. Make sure it is not changed.
461
+ static_assert(std::is_same_v<other_t_imag, T>, "other_t_imag must be T");
462
+ return Vectorized(0);
463
+ }
464
+ template <
465
+ typename complex_t_imag = T,
466
+ typename std::enable_if_t<c10::is_complex<complex_t_imag>::value, int> =
467
+ 0>
468
+ Vectorized<T> imag() const {
469
+ // complex_t_imag is for SFINAE and clarity. Make sure it is not changed.
470
+ static_assert(
471
+ std::is_same_v<complex_t_imag, T>, "complex_t_imag must be T");
472
+ return map([](T x) { return static_cast<T>(x.imag()); });
473
+ }
474
+ template <
475
+ typename other_t_conj = T,
476
+ typename std::enable_if_t<!c10::is_complex<other_t_conj>::value, int> = 0>
477
+ Vectorized<T> conj() const {
478
+ // other_t_conj is for SFINAE and clarity. Make sure it is not changed.
479
+ static_assert(std::is_same_v<other_t_conj, T>, "other_t_conj must be T");
480
+ return *this;
481
+ }
482
+ template <
483
+ typename complex_t_conj = T,
484
+ typename std::enable_if_t<c10::is_complex<complex_t_conj>::value, int> =
485
+ 0>
486
+ Vectorized<T> conj() const {
487
+ // complex_t_conj is for SFINAE and clarity. Make sure it is not changed.
488
+ static_assert(
489
+ std::is_same_v<complex_t_conj, T>, "complex_t_conj must be T");
490
+ return map([](T x) { return static_cast<T>(std::conj(x)); });
491
+ }
492
+ Vectorized<T> acos() const {
493
+ return map(std::acos);
494
+ }
495
+ Vectorized<T> acosh() const {
496
+ return map(std::acosh);
497
+ }
498
+ Vectorized<T> asin() const {
499
+ return map(std::asin);
500
+ }
501
+ Vectorized<T> asinh() const {
502
+ return map(std::asinh);
503
+ }
504
+ Vectorized<T> atan() const {
505
+ return map(std::atan);
506
+ }
507
+ Vectorized<T> atanh() const {
508
+ return map(std::atanh);
509
+ }
510
+ Vectorized<T> atan2(const Vectorized<T>& exp) const {
511
+ Vectorized<T> ret;
512
+ for (const auto i : c10::irange(size())) {
513
+ ret[i] = std::atan2(values[i], exp[i]);
514
+ }
515
+ return ret;
516
+ }
517
+ template <
518
+ typename U = T,
519
+ typename std::enable_if_t<is_floating_point_v<U>, int> = 0>
520
+ Vectorized<T> copysign(const Vectorized<T>& sign) const {
521
+ Vectorized<T> ret;
522
+ for (size_type i = 0; i < size(); i++) {
523
+ ret[i] = c10::copysign(values[i], sign[i]);
524
+ }
525
+ return ret;
526
+ }
527
+ Vectorized<T> erf() const {
528
+ return map(std::erf);
529
+ }
530
+ Vectorized<T> erfc() const {
531
+ return map(std::erfc);
532
+ }
533
+ Vectorized<T> erfinv() const {
534
+ return map(calc_erfinv);
535
+ }
536
+ Vectorized<T> exp() const {
537
+ return map(std::exp);
538
+ }
539
+ Vectorized<T> exp2() const {
540
+ return map(exp2_impl);
541
+ }
542
+ Vectorized<T> expm1() const {
543
+ return map(std::expm1);
544
+ }
545
+ Vectorized<T> exp_u20() const {
546
+ return map(std::exp);
547
+ }
548
+ Vectorized<T> fexp_u20() const {
549
+ return map(std::exp);
550
+ }
551
+ Vectorized<T> frac() const {
552
+ return *this - this->trunc();
553
+ }
554
+ template <
555
+ typename U = T,
556
+ typename std::enable_if_t<is_floating_point_v<U>, int> = 0>
557
+ Vectorized<T> fmod(const Vectorized<T>& q) const {
558
+ // U is for SFINAE purposes only. Make sure it is not changed.
559
+ static_assert(std::is_same_v<U, T>, "U must be T");
560
+ Vectorized<T> ret;
561
+ for (const auto i : c10::irange(size())) {
562
+ ret[i] = std::fmod(values[i], q[i]);
563
+ }
564
+ return ret;
565
+ }
566
+ Vectorized<T> log() const {
567
+ return map(std::log);
568
+ }
569
+ Vectorized<T> log10() const {
570
+ return map(std::log10);
571
+ }
572
+ Vectorized<T> log1p() const {
573
+ return map(std::log1p);
574
+ }
575
+ template <
576
+ typename other_t_log2 = T,
577
+ typename std::enable_if_t<!c10::is_complex<other_t_log2>::value, int> = 0>
578
+ Vectorized<T> log2() const {
579
+ // other_t_log2 is for SFINAE and clarity. Make sure it is not changed.
580
+ static_assert(std::is_same_v<other_t_log2, T>, "other_t_log2 must be T");
581
+ return map(std::log2);
582
+ }
583
+ template <
584
+ typename complex_t_log2 = T,
585
+ typename std::enable_if_t<c10::is_complex<complex_t_log2>::value, int> =
586
+ 0>
587
+ Vectorized<T> log2() const {
588
+ // complex_t_log2 is for SFINAE and clarity. Make sure it is not changed.
589
+ static_assert(
590
+ std::is_same_v<complex_t_log2, T>, "complex_t_log2 must be T");
591
+ const T log_2 = T(std::log(2.0));
592
+ return Vectorized(map(std::log)) / Vectorized(log_2);
593
+ }
594
+ Vectorized<T> ceil() const {
595
+ return map(at::native::ceil_impl);
596
+ }
597
+ Vectorized<T> cos() const {
598
+ return map(std::cos);
599
+ }
600
+ Vectorized<T> cosh() const {
601
+ return map(std::cosh);
602
+ }
603
+ Vectorized<T> floor() const {
604
+ return map(at::native::floor_impl);
605
+ }
606
+ Vectorized<T> hypot(const Vectorized<T>& b) const {
607
+ Vectorized<T> ret;
608
+ for (const auto i : c10::irange(size())) {
609
+ ret[i] = std::hypot(values[i], b[i]);
610
+ }
611
+ return ret;
612
+ }
613
+ Vectorized<T> i0() const {
614
+ return map(calc_i0);
615
+ }
616
+ Vectorized<T> i0e() const {
617
+ return map(calc_i0e);
618
+ }
619
+ Vectorized<T> digamma() const {
620
+ return map(calc_digamma);
621
+ }
622
+ Vectorized<T> igamma(const Vectorized<T>& x) const {
623
+ Vectorized<T> ret;
624
+ for (const auto i : c10::irange(size())) {
625
+ ret[i] = calc_igamma(values[i], x[i]);
626
+ }
627
+ return ret;
628
+ }
629
+ Vectorized<T> igammac(const Vectorized<T>& x) const {
630
+ Vectorized<T> ret;
631
+ for (const auto i : c10::irange(size())) {
632
+ ret[i] = calc_igammac(values[i], x[i]);
633
+ }
634
+ return ret;
635
+ }
636
+ Vectorized<T> neg() const {
637
+ // NB: the trailing return type is needed because we need to coerce the
638
+ // return value back to T in the case of unary operator- incurring a
639
+ // promotion
640
+ return map([](T x) -> T { return -x; });
641
+ }
642
+ Vectorized<T> nextafter(const Vectorized<T>& b) const {
643
+ Vectorized<T> ret;
644
+ for (const auto i : c10::irange(size())) {
645
+ ret[i] = std::nextafter(values[i], b[i]);
646
+ }
647
+ return ret;
648
+ }
649
+ Vectorized<T> round() const {
650
+ // We do not use std::round because we would like to round midway numbers to
651
+ // the nearest even integer.
652
+ return map(at::native::round_impl);
653
+ }
654
+ Vectorized<T> sin() const {
655
+ return map(std::sin);
656
+ }
657
+ Vectorized<T> sinh() const {
658
+ return map(std::sinh);
659
+ }
660
+ Vectorized<T> tan() const {
661
+ return map(std::tan);
662
+ }
663
+ Vectorized<T> tanh() const {
664
+ return map(std::tanh);
665
+ }
666
+ Vectorized<T> trunc() const {
667
+ return map(at::native::trunc_impl);
668
+ }
669
+ Vectorized<T> lgamma() const {
670
+ return map(std::lgamma);
671
+ }
672
+ Vectorized<T> sqrt() const {
673
+ return map(std::sqrt);
674
+ }
675
+ Vectorized<T> reciprocal() const {
676
+ return map([](T x) { return (T)1 / x; });
677
+ }
678
+ Vectorized<T> rsqrt() const {
679
+ return map([](T x) { return (T)1 / std::sqrt(x); });
680
+ }
681
+ Vectorized<T> pow(const Vectorized<T>& exp) const {
682
+ Vectorized<T> ret;
683
+ for (const auto i : c10::irange(size())) {
684
+ ret[i] = std::pow(values[i], exp[i]);
685
+ }
686
+ return ret;
687
+ }
688
+ T reduce_add() const {
689
+ return reduce([](T x, T y) -> T { return x + y; });
690
+ }
691
+ T reduce_max() const {
692
+ return reduce(std::max);
693
+ }
694
+
695
+ private:
696
+ template <typename Op>
697
+ inline Vectorized<T> binary_pred(const Vectorized<T>& other, Op op) const {
698
+ // All bits are set to 1 if the pred is true, otherwise 0.
699
+ Vectorized<T> vector;
700
+ for (int64_t i = 0; i != size(); i++) {
701
+ if (op(values[i], other.values[i])) {
702
+ std::memset(static_cast<void*>(vector.values + i), 0xFF, sizeof(T));
703
+ } else {
704
+ std::memset(static_cast<void*>(vector.values + i), 0, sizeof(T));
705
+ }
706
+ }
707
+ return vector;
708
+ }
709
+
710
+ public:
711
+ Vectorized<T> operator==(const Vectorized<T>& other) const {
712
+ return binary_pred(other, std::equal_to<T>());
713
+ }
714
+ Vectorized<T> operator!=(const Vectorized<T>& other) const {
715
+ return binary_pred(other, std::not_equal_to<T>());
716
+ }
717
+ Vectorized<T> operator>=(const Vectorized<T>& other) const {
718
+ return binary_pred(other, std::greater_equal<T>());
719
+ }
720
+ Vectorized<T> operator<=(const Vectorized<T>& other) const {
721
+ return binary_pred(other, std::less_equal<T>());
722
+ }
723
+ Vectorized<T> operator>(const Vectorized<T>& other) const {
724
+ return binary_pred(other, std::greater<T>());
725
+ }
726
+ Vectorized<T> operator<(const Vectorized<T>& other) const {
727
+ return binary_pred(other, std::less<T>());
728
+ }
729
+
730
+ private:
731
+ template <typename Op>
732
+ inline Vectorized<T> binary_pred_bool(const Vectorized<T>& other, Op op)
733
+ const {
734
+ // 1 if the pred is true, otherwise 0.
735
+ Vectorized<T> vector;
736
+ for (int i = 0; i != size(); ++i) {
737
+ vector[i] = static_cast<T>(op(values[i], other.values[i]));
738
+ }
739
+ return vector;
740
+ }
741
+
742
+ public:
743
+ Vectorized<T> eq(const Vectorized<T>& other) const {
744
+ return binary_pred_bool(other, std::equal_to<T>());
745
+ }
746
+ Vectorized<T> ne(const Vectorized<T>& other) const {
747
+ return binary_pred_bool(other, std::not_equal_to<T>());
748
+ }
749
+ Vectorized<T> gt(const Vectorized<T>& other) const {
750
+ return binary_pred_bool(other, std::greater<T>());
751
+ }
752
+ Vectorized<T> ge(const Vectorized<T>& other) const {
753
+ return binary_pred_bool(other, std::greater_equal<T>());
754
+ }
755
+ Vectorized<T> lt(const Vectorized<T>& other) const {
756
+ return binary_pred_bool(other, std::less<T>());
757
+ }
758
+ Vectorized<T> le(const Vectorized<T>& other) const {
759
+ return binary_pred_bool(other, std::less_equal<T>());
760
+ }
761
+ };
762
+
763
+ template <class T>
764
+ Vectorized<T> inline operator-(const Vectorized<T>& a) {
765
+ return a.neg();
766
+ }
767
+
768
+ // There is an implicit conversion that would make this work if
769
+ // these operators weren't template functions, but they are template
770
+ // functions (and can't be moved to be non-member friends defined in
771
+ // the class body as suggested in
772
+ // https://stackoverflow.com/questions/9787593/implicit-type-conversion-with-template/9788255#9788255
773
+ // because we have a lot of disparate specializations of
774
+ // Vectorized). So, just explicitly make scalars work.
775
+ #define VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_FUNC(name) \
776
+ template <class T> \
777
+ Vectorized<T> inline name(const Vectorized<T>& a, T b) { \
778
+ return name(a, Vectorized<T>(b)); \
779
+ } \
780
+ template <class T> \
781
+ Vectorized<T> inline name(T a, const Vectorized<T>& b) { \
782
+ return name(Vectorized<T>(a), b); \
783
+ }
784
+ #define VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP(op) \
785
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_FUNC(operator op)
786
+
787
+ template <class T>
788
+ Vectorized<T> inline operator+(const Vectorized<T>& a, const Vectorized<T>& b) {
789
+ Vectorized<T> c;
790
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
791
+ c[i] = a[i] + b[i];
792
+ }
793
+ return c;
794
+ }
795
+
796
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP(+)
797
+
798
+ template <class T>
799
+ Vectorized<T> inline operator-(const Vectorized<T>& a, const Vectorized<T>& b) {
800
+ Vectorized<T> c;
801
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
802
+ c[i] = a[i] - b[i];
803
+ }
804
+ return c;
805
+ }
806
+
807
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP(-)
808
+
809
+ template <class T>
810
+ Vectorized<T> inline operator*(const Vectorized<T>& a, const Vectorized<T>& b) {
811
+ Vectorized<T> c;
812
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
813
+ c[i] = a[i] * b[i];
814
+ }
815
+ return c;
816
+ }
817
+
818
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP(*)
819
+
820
+ template <class T>
821
+ Vectorized<T> inline operator/(const Vectorized<T>& a, const Vectorized<T>& b)
822
+ __ubsan_ignore_float_divide_by_zero__ {
823
+ Vectorized<T> c;
824
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
825
+ c[i] = a[i] / b[i];
826
+ }
827
+ return c;
828
+ }
829
+
830
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP(/)
831
+
832
+ template <class T, typename std::enable_if_t<!is_floating_point_v<T>, int> = 0>
833
+ Vectorized<T> inline operator%(const Vectorized<T>& a, const Vectorized<T>& b)
834
+ __ubsan_ignore_float_divide_by_zero__ {
835
+ return a - a / b * b;
836
+ }
837
+
838
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP(%)
839
+
840
+ template <class T>
841
+ Vectorized<T> inline operator||(
842
+ const Vectorized<T>& a,
843
+ const Vectorized<T>& b) {
844
+ Vectorized<T> c;
845
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
846
+ c[i] = a[i] || b[i];
847
+ }
848
+ return c;
849
+ }
850
+
851
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP(||)
852
+
853
+ // Implements the IEEE 754 201X `maximum` operation, which propagates NaN if
854
+ // either input is a NaN.
855
+ template <
856
+ class T,
857
+ typename std::enable_if_t<!c10::is_complex<T>::value, int> = 0>
858
+ Vectorized<T> inline maximum(const Vectorized<T>& a, const Vectorized<T>& b) {
859
+ Vectorized<T> c;
860
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
861
+ c[i] = (a[i] > b[i]) ? a[i] : b[i];
862
+ if (_isnan(a[i])) {
863
+ // If either input is NaN, propagate a NaN.
864
+ // NOTE: The case where b[i] was NaN is handled correctly by the naive
865
+ // ternary operator above.
866
+ c[i] = a[i];
867
+ }
868
+ }
869
+ return c;
870
+ }
871
+
872
+ template <
873
+ class T,
874
+ typename std::enable_if_t<c10::is_complex<T>::value, int> = 0>
875
+ Vectorized<T> inline maximum(const Vectorized<T>& a, const Vectorized<T>& b) {
876
+ Vectorized<T> c;
877
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
878
+ c[i] = (std::abs(a[i]) > std::abs(b[i])) ? a[i] : b[i];
879
+ if (_isnan(a[i])) {
880
+ // If either input is NaN, propagate a NaN.
881
+ // NOTE: The case where b[i] was NaN is handled correctly by the naive
882
+ // ternary operator above.
883
+ c[i] = a[i];
884
+ }
885
+ }
886
+ return c;
887
+ }
888
+
889
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_FUNC(maximum)
890
+
891
+ // Implements the IEEE 754 201X `minimum` operation, which propagates NaN if
892
+ // either input is a NaN.
893
+ template <
894
+ class T,
895
+ typename std::enable_if_t<!c10::is_complex<T>::value, int> = 0>
896
+ Vectorized<T> inline minimum(const Vectorized<T>& a, const Vectorized<T>& b) {
897
+ Vectorized<T> c;
898
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
899
+ c[i] = (a[i] < b[i]) ? a[i] : b[i];
900
+ if (_isnan(a[i])) {
901
+ // If either input is NaN, propagate a NaN.
902
+ // NOTE: The case where b[i] was NaN is handled correctly by the naive
903
+ // ternary operator above.
904
+ c[i] = a[i];
905
+ }
906
+ }
907
+ return c;
908
+ }
909
+
910
+ template <
911
+ class T,
912
+ typename std::enable_if_t<c10::is_complex<T>::value, int> = 0>
913
+ Vectorized<T> inline minimum(const Vectorized<T>& a, const Vectorized<T>& b) {
914
+ Vectorized<T> c;
915
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
916
+ c[i] = (std::abs(a[i]) < std::abs(b[i])) ? a[i] : b[i];
917
+ if (_isnan(a[i])) {
918
+ // If either input is NaN, propagate a NaN.
919
+ // NOTE: The case where b[i] was NaN is handled correctly by the naive
920
+ // ternary operator above.
921
+ c[i] = a[i];
922
+ }
923
+ }
924
+ return c;
925
+ }
926
+
927
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_FUNC(minimum)
928
+
929
+ template <
930
+ class T,
931
+ typename std::enable_if_t<!c10::is_complex<T>::value, int> = 0>
932
+ Vectorized<T> inline clamp(
933
+ const Vectorized<T>& a,
934
+ const Vectorized<T>& min_vec,
935
+ const Vectorized<T>& max_vec) {
936
+ Vectorized<T> c;
937
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
938
+ c[i] = std::min(std::max(a[i], min_vec[i]), max_vec[i]);
939
+ }
940
+ return c;
941
+ }
942
+
943
+ #define VECTORIZED_SUPPORT_SCALARS_FOR_TERNARY_FUNC(name) \
944
+ template <class T> \
945
+ Vectorized<T> inline name( \
946
+ const Vectorized<T>& a, const Vectorized<T>& b, T c) { \
947
+ return name(a, b, Vectorized<T>(c)); \
948
+ } \
949
+ \
950
+ template <class T> \
951
+ Vectorized<T> inline name( \
952
+ const Vectorized<T>& a, T b, const Vectorized<T>& c) { \
953
+ return name(a, Vectorized<T>(b), c); \
954
+ } \
955
+ \
956
+ template <class T> \
957
+ Vectorized<T> inline name(const Vectorized<T>& a, T b, T c) { \
958
+ return name(a, Vectorized<T>(b), Vectorized<T>(c)); \
959
+ } \
960
+ \
961
+ template <class T> \
962
+ Vectorized<T> inline name( \
963
+ T a, const Vectorized<T>& b, const Vectorized<T>& c) { \
964
+ return name(Vectorized<T>(a), b, c); \
965
+ } \
966
+ \
967
+ template <class T> \
968
+ Vectorized<T> inline name(T a, const Vectorized<T>& b, T c) { \
969
+ return name(Vectorized<T>(a), b, Vectorized<T>(c)); \
970
+ } \
971
+ \
972
+ template <class T> \
973
+ Vectorized<T> inline name(T a, T b, const Vectorized<T>& c) { \
974
+ return name(Vectorized<T>(a), Vectorized<T>(b), c); \
975
+ }
976
+
977
+ VECTORIZED_SUPPORT_SCALARS_FOR_TERNARY_FUNC(clamp)
978
+
979
+ template <
980
+ class T,
981
+ typename std::enable_if_t<!c10::is_complex<T>::value, int> = 0>
982
+ Vectorized<T> inline clamp_max(
983
+ const Vectorized<T>& a,
984
+ const Vectorized<T>& max_vec) {
985
+ Vectorized<T> c;
986
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
987
+ c[i] = a[i] > max_vec[i] ? max_vec[i] : a[i];
988
+ }
989
+ return c;
990
+ }
991
+
992
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_FUNC(clamp_max)
993
+
994
+ template <
995
+ class T,
996
+ typename std::enable_if_t<!c10::is_complex<T>::value, int> = 0>
997
+ Vectorized<T> inline clamp_min(
998
+ const Vectorized<T>& a,
999
+ const Vectorized<T>& min_vec) {
1000
+ Vectorized<T> c;
1001
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
1002
+ c[i] = a[i] < min_vec[i] ? min_vec[i] : a[i];
1003
+ }
1004
+ return c;
1005
+ }
1006
+
1007
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_FUNC(clamp_min)
1008
+
1009
+ struct Vectorizedi;
1010
+
1011
+ #if defined(CPU_CAPABILITY_AVX2) || defined(CPU_CAPABILITY_AVX512)
1012
+ template <class T, typename Op>
1013
+ static inline Vectorized<T> bitwise_binary_op(
1014
+ const Vectorized<T>& a,
1015
+ const Vectorized<T>& b,
1016
+ Op op) {
1017
+ int_vector buffer;
1018
+ #if defined(CPU_CAPABILITY_AVX2)
1019
+ int_vector a_buffer =
1020
+ _mm256_load_si256(reinterpret_cast<const int_vector*>((const T*)a));
1021
+ int_vector b_buffer =
1022
+ _mm256_load_si256(reinterpret_cast<const int_vector*>((const T*)b));
1023
+ #elif defined(CPU_CAPABILITY_AVX512)
1024
+ int_vector a_buffer =
1025
+ _mm512_load_si512(reinterpret_cast<const int_vector*>((const T*)a));
1026
+ int_vector b_buffer =
1027
+ _mm512_load_si512(reinterpret_cast<const int_vector*>((const T*)b));
1028
+ #endif
1029
+ buffer = op(a_buffer, b_buffer);
1030
+ __at_align__ T results[Vectorized<T>::size()];
1031
+
1032
+ #if defined(CPU_CAPABILITY_AVX2)
1033
+ _mm256_store_si256(reinterpret_cast<int_vector*>(results), buffer);
1034
+ #elif defined(CPU_CAPABILITY_AVX512)
1035
+ _mm512_store_si512(reinterpret_cast<int_vector*>(results), buffer);
1036
+ #endif
1037
+ return Vectorized<T>::loadu(results);
1038
+ }
1039
+
1040
+ template <
1041
+ class T,
1042
+ typename std::enable_if_t<
1043
+ !std::is_base_of<Vectorizedi, Vectorized<T>>::value,
1044
+ int> = 0>
1045
+ inline Vectorized<T> operator&(const Vectorized<T>& a, const Vectorized<T>& b) {
1046
+ // We enclose _mm512_and_si512 or _mm256_and_si256 with lambda because it is
1047
+ // always_inline
1048
+ #if defined(CPU_CAPABILITY_AVX2)
1049
+ return bitwise_binary_op(
1050
+ a, b, [](int_vector a, int_vector b) { return _mm256_and_si256(a, b); });
1051
+ #elif defined(CPU_CAPABILITY_AVX512)
1052
+ return bitwise_binary_op(
1053
+ a, b, [](int_vector a, int_vector b) { return _mm512_and_si512(a, b); });
1054
+ #endif
1055
+ }
1056
+ template <
1057
+ class T,
1058
+ typename std::enable_if_t<
1059
+ !std::is_base_of<Vectorizedi, Vectorized<T>>::value,
1060
+ int> = 0>
1061
+ inline Vectorized<T> operator|(const Vectorized<T>& a, const Vectorized<T>& b) {
1062
+ // We enclose _mm512_or_si512 or _mm256_or_si256 with lambda because it is
1063
+ // always_inline
1064
+ #if defined(CPU_CAPABILITY_AVX2)
1065
+ return bitwise_binary_op(
1066
+ a, b, [](int_vector a, int_vector b) { return _mm256_or_si256(a, b); });
1067
+ #elif defined(CPU_CAPABILITY_AVX512)
1068
+ return bitwise_binary_op(
1069
+ a, b, [](int_vector a, int_vector b) { return _mm512_or_si512(a, b); });
1070
+ #endif
1071
+ }
1072
+ template <
1073
+ class T,
1074
+ typename std::enable_if_t<
1075
+ !std::is_base_of<Vectorizedi, Vectorized<T>>::value,
1076
+ int> = 0>
1077
+ inline Vectorized<T> operator^(const Vectorized<T>& a, const Vectorized<T>& b) {
1078
+ // We enclose _mm512_xor_si512 or _mm256_xor_si256 with lambda because it is
1079
+ // always_inline
1080
+ #if defined(CPU_CAPABILITY_AVX2)
1081
+ return bitwise_binary_op(
1082
+ a, b, [](int_vector a, int_vector b) { return _mm256_xor_si256(a, b); });
1083
+ #elif defined(CPU_CAPABILITY_AVX512)
1084
+ return bitwise_binary_op(
1085
+ a, b, [](int_vector a, int_vector b) { return _mm512_xor_si512(a, b); });
1086
+ #endif
1087
+ }
1088
+
1089
+ #else
1090
+
1091
+ template <typename T>
1092
+ auto load(char const* data) -> T {
1093
+ T ret;
1094
+ std::memcpy(&ret, data, sizeof(ret));
1095
+ return ret;
1096
+ }
1097
+
1098
+ template <class T, typename Op>
1099
+ static inline Vectorized<T> bitwise_binary_op(
1100
+ const Vectorized<T>& a,
1101
+ const Vectorized<T>& b,
1102
+ Op op) {
1103
+ static constexpr uint32_t element_no = VECTOR_WIDTH / sizeof(intmax_t);
1104
+ __at_align__ intmax_t buffer[element_no];
1105
+ static_assert(
1106
+ VECTOR_WIDTH % sizeof(intmax_t) == 0,
1107
+ "VECTOR_WIDTH not a multiple of sizeof(intmax_t)");
1108
+ static_assert(
1109
+ sizeof(buffer) == sizeof(Vectorized<T>),
1110
+ "sizeof(buffer) must match sizeof(Vectorized<T>)");
1111
+ // We should be using memcpy in order to respect the strict aliasing rule
1112
+ // see: https://github.com/pytorch/pytorch/issues/66119
1113
+ // Using char* is defined in the C11 standard 6.5 Expression paragraph 7
1114
+ // (http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1570.pdf)
1115
+ const auto* a_data = a.as_bytes();
1116
+ const auto* b_data = b.as_bytes();
1117
+ // load each intmax_t chunk and process; increase pointers by sizeof(intmax_t)
1118
+ for (auto& out : buffer) {
1119
+ out = op(load<intmax_t>(a_data), load<intmax_t>(b_data));
1120
+ a_data += sizeof(intmax_t);
1121
+ b_data += sizeof(intmax_t);
1122
+ }
1123
+ assert(a_data == a.as_bytes() + sizeof(a));
1124
+ assert(b_data == b.as_bytes() + sizeof(b));
1125
+ return Vectorized<T>::loadu(buffer);
1126
+ }
1127
+
1128
+ template <
1129
+ class T,
1130
+ typename std::
1131
+ enable_if_t<!std::is_base_of_v<Vectorizedi, Vectorized<T>>, int> = 0>
1132
+ inline Vectorized<T> operator&(const Vectorized<T>& a, const Vectorized<T>& b) {
1133
+ return bitwise_binary_op(a, b, std::bit_and<intmax_t>());
1134
+ }
1135
+ template <
1136
+ class T,
1137
+ typename std::
1138
+ enable_if_t<!std::is_base_of_v<Vectorizedi, Vectorized<T>>, int> = 0>
1139
+ inline Vectorized<T> operator|(const Vectorized<T>& a, const Vectorized<T>& b) {
1140
+ return bitwise_binary_op(a, b, std::bit_or<intmax_t>());
1141
+ }
1142
+ template <
1143
+ class T,
1144
+ typename std::
1145
+ enable_if_t<!std::is_base_of_v<Vectorizedi, Vectorized<T>>, int> = 0>
1146
+ inline Vectorized<T> operator^(const Vectorized<T>& a, const Vectorized<T>& b) {
1147
+ return bitwise_binary_op(a, b, std::bit_xor<intmax_t>());
1148
+ }
1149
+
1150
+ #endif // defined(CPU_CAPABILITY_AVX2) || defined(CPU_CAPABILITY_AVX512)
1151
+
1152
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP(&)
1153
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP(|)
1154
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP(^)
1155
+
1156
+ template <
1157
+ class T,
1158
+ typename std::
1159
+ enable_if_t<!std::is_base_of_v<Vectorizedi, Vectorized<T>>, int> = 0>
1160
+ inline Vectorized<T> operator~(const Vectorized<T>& a) {
1161
+ using int_t = int_same_size_t<T>;
1162
+ Vectorized<T> ones(c10::bit_cast<T>((int_t)(~(int_t)0))); // All bits are 1
1163
+ return a ^ ones;
1164
+ }
1165
+
1166
+ template <class T>
1167
+ Vectorized<T> inline operator<<(
1168
+ const Vectorized<T>& a,
1169
+ const Vectorized<T>& b) {
1170
+ constexpr T max_shift = sizeof(T) * CHAR_BIT;
1171
+ Vectorized<T> c;
1172
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
1173
+ T shift = b[i];
1174
+ if ((static_cast<std::make_signed_t<T>>(shift) < 0) ||
1175
+ (shift >= max_shift)) {
1176
+ c[i] = 0;
1177
+ } else {
1178
+ c[i] = static_cast<std::make_unsigned_t<T>>(a[i]) << shift;
1179
+ }
1180
+ }
1181
+ return c;
1182
+ }
1183
+
1184
+ template <class T>
1185
+ Vectorized<T> inline operator>>(
1186
+ const Vectorized<T>& a,
1187
+ const Vectorized<T>& b) {
1188
+ // right shift value to retain sign bit for signed and no bits for unsigned
1189
+ constexpr T max_shift = sizeof(T) * CHAR_BIT - std::is_signed_v<T>;
1190
+ Vectorized<T> c;
1191
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
1192
+ T shift = b[i];
1193
+ if ((static_cast<std::make_signed_t<T>>(shift) < 0) ||
1194
+ (shift >= max_shift)) {
1195
+ c[i] = a[i] >> max_shift;
1196
+ } else {
1197
+ c[i] = a[i] >> shift;
1198
+ }
1199
+ }
1200
+ return c;
1201
+ }
1202
+
1203
+ template <typename T>
1204
+ inline Vectorized<T>& operator+=(Vectorized<T>& a, const Vectorized<T>& b) {
1205
+ a = a + b;
1206
+ return a;
1207
+ }
1208
+ template <typename T>
1209
+ inline Vectorized<T>& operator-=(Vectorized<T>& a, const Vectorized<T>& b) {
1210
+ a = a - b;
1211
+ return a;
1212
+ }
1213
+ template <typename T>
1214
+ inline Vectorized<T>& operator/=(Vectorized<T>& a, const Vectorized<T>& b) {
1215
+ a = a / b;
1216
+ return a;
1217
+ }
1218
+ template <typename T>
1219
+ inline Vectorized<T>& operator%=(Vectorized<T>& a, const Vectorized<T>& b) {
1220
+ a = a % b;
1221
+ return a;
1222
+ }
1223
+ template <typename T>
1224
+ inline Vectorized<T>& operator*=(Vectorized<T>& a, const Vectorized<T>& b) {
1225
+ a = a * b;
1226
+ return a;
1227
+ }
1228
+
1229
+ template <typename T>
1230
+ inline Vectorized<T>& operator<<=(Vectorized<T>& a, const Vectorized<T>& b) {
1231
+ a = a << b;
1232
+ return a;
1233
+ }
1234
+
1235
+ template <typename T>
1236
+ inline Vectorized<T>& operator>>=(Vectorized<T>& a, const Vectorized<T>& b) {
1237
+ a = a >> b;
1238
+ return a;
1239
+ }
1240
+
1241
+ template <typename T>
1242
+ inline Vectorized<T> fmadd(
1243
+ const Vectorized<T>& a,
1244
+ const Vectorized<T>& b,
1245
+ const Vectorized<T>& c) {
1246
+ return a * b + c;
1247
+ }
1248
+
1249
+ VECTORIZED_SUPPORT_SCALARS_FOR_TERNARY_FUNC(fmadd)
1250
+
1251
+ template <typename T>
1252
+ inline Vectorized<T> fnmadd(
1253
+ const Vectorized<T>& a,
1254
+ const Vectorized<T>& b,
1255
+ const Vectorized<T>& c) {
1256
+ return -(a * b) + c;
1257
+ }
1258
+
1259
+ VECTORIZED_SUPPORT_SCALARS_FOR_TERNARY_FUNC(fnmadd)
1260
+
1261
+ template <typename T>
1262
+ inline Vectorized<T> fmsub(
1263
+ const Vectorized<T>& a,
1264
+ const Vectorized<T>& b,
1265
+ const Vectorized<T>& c) {
1266
+ return a * b - c;
1267
+ }
1268
+
1269
+ VECTORIZED_SUPPORT_SCALARS_FOR_TERNARY_FUNC(fmsub)
1270
+
1271
+ template <typename T>
1272
+ inline Vectorized<T> fnmsub(
1273
+ const Vectorized<T>& a,
1274
+ const Vectorized<T>& b,
1275
+ const Vectorized<T>& c) {
1276
+ return -(a * b) - c;
1277
+ }
1278
+
1279
+ VECTORIZED_SUPPORT_SCALARS_FOR_TERNARY_FUNC(fnmsub)
1280
+
1281
+ template <typename T>
1282
+ Vectorized<T> inline operator&&(
1283
+ const Vectorized<T>& a,
1284
+ const Vectorized<T>& b) {
1285
+ Vectorized<T> ret;
1286
+ for (int i = 0; i != Vectorized<T>::size(); i++) {
1287
+ ret[i] = a[i] && b[i];
1288
+ }
1289
+ return ret;
1290
+ }
1291
+
1292
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP(&&)
1293
+
1294
+ template <int64_t scale = 1, typename T = void>
1295
+ std::enable_if_t<
1296
+ scale == 1 || scale == 2 || scale == 4 || scale == 8,
1297
+ Vectorized<
1298
+ T>> inline gather(T const* base_addr, const Vectorized<int_same_size_t<T>>& vindex) {
1299
+ static constexpr int size = Vectorized<T>::size();
1300
+ int_same_size_t<T> index_arr[size];
1301
+ vindex.store(static_cast<void*>(index_arr));
1302
+ T buffer[size];
1303
+ for (const auto i : c10::irange(size)) {
1304
+ buffer[i] = base_addr[index_arr[i] * scale / sizeof(T)];
1305
+ }
1306
+ return Vectorized<T>::loadu(static_cast<void*>(buffer));
1307
+ }
1308
+
1309
+ template <int64_t scale = 1, typename T = void>
1310
+ std::
1311
+ enable_if_t<scale == 1 || scale == 2 || scale == 4 || scale == 8, Vectorized<T>> inline mask_gather(
1312
+ const Vectorized<T>& src,
1313
+ T const* base_addr,
1314
+ const Vectorized<int_same_size_t<T>>& vindex,
1315
+ Vectorized<T>& mask) {
1316
+ static constexpr int size = Vectorized<T>::size();
1317
+ T src_arr[size];
1318
+ int_same_size_t<T> mask_arr[size]; // use int type so we can logical and
1319
+ int_same_size_t<T> index_arr[size];
1320
+ src.store(static_cast<void*>(src_arr));
1321
+ mask.store(static_cast<void*>(mask_arr));
1322
+ vindex.store(static_cast<void*>(index_arr));
1323
+ T buffer[size];
1324
+ for (const auto i : c10::irange(size)) {
1325
+ if (mask_arr[i] & 0x01) { // check highest bit
1326
+ buffer[i] = base_addr[index_arr[i] * scale / sizeof(T)];
1327
+ } else {
1328
+ buffer[i] = src_arr[i];
1329
+ }
1330
+ }
1331
+ mask = Vectorized<T>(static_cast<T>(0)); // "zero out" mask
1332
+ return Vectorized<T>::loadu(static_cast<void*>(buffer));
1333
+ }
1334
+
1335
+ // Cast a given vector to another type without changing the bits representation.
1336
+ // So a Vectorized<double> of 512 bits containing all ones can be cast to a
1337
+ // Vectorized<int64_t> of 512 bits containing all ones (i.e., eight negative
1338
+ // 1s). A Vec<double> of 256 bits containing all ones can be cast to a
1339
+ // Vec<int64_t> of 256 bits containing all ones (i.e., four negative 1s).
1340
+ // There is a struct here because we don't have static_if and I can't
1341
+ // partially specialize a templated function.
1342
+ template <typename dst_t, typename src_t>
1343
+ struct CastImpl {
1344
+ static inline Vectorized<dst_t> apply(const Vectorized<src_t>& src) {
1345
+ src_t src_arr[Vectorized<src_t>::size()];
1346
+ src.store(static_cast<void*>(src_arr));
1347
+ return Vectorized<dst_t>::loadu(static_cast<const void*>(src_arr));
1348
+ }
1349
+ };
1350
+
1351
+ template <typename scalar_t>
1352
+ struct CastImpl<scalar_t, scalar_t> {
1353
+ static inline Vectorized<scalar_t> apply(const Vectorized<scalar_t>& src) {
1354
+ return src;
1355
+ }
1356
+ };
1357
+
1358
+ template <typename dst_t, typename src_t>
1359
+ inline Vectorized<dst_t> cast(const Vectorized<src_t>& src) {
1360
+ return CastImpl<dst_t, src_t>::apply(src);
1361
+ }
1362
+
1363
+ template <typename T, typename IntType = int_same_size_t<T>>
1364
+ inline Vectorized<IntType> convert_to_int_of_same_size(
1365
+ const Vectorized<T>& src) {
1366
+ static_assert(sizeof(T) == sizeof(IntType));
1367
+ static constexpr int size = Vectorized<T>::size();
1368
+
1369
+ std::array<T, size> src_arr = {};
1370
+ src.store(static_cast<void*>(src_arr.data()));
1371
+ std::array<IntType, size> buffer;
1372
+ std::transform(
1373
+ src_arr.cbegin(), src_arr.cend(), buffer.begin(), [](const T& x) {
1374
+ return static_cast<IntType>(x);
1375
+ });
1376
+ return Vectorized<IntType>::loadu(static_cast<const void*>(buffer.data()));
1377
+ }
1378
+
1379
+ template <typename T, typename IntType = int_same_size_t<T>>
1380
+ inline Vectorized<T> convert_to_fp_of_same_size(
1381
+ const Vectorized<IntType>& src) {
1382
+ static_assert(sizeof(T) == sizeof(IntType));
1383
+ static constexpr int size = Vectorized<T>::size();
1384
+
1385
+ std::array<IntType, size> src_arr;
1386
+ src.store(static_cast<void*>(src_arr.data()));
1387
+ std::array<T, size> buffer;
1388
+ std::transform(
1389
+ src_arr.cbegin(), src_arr.cend(), buffer.begin(), [](const IntType& x) {
1390
+ return static_cast<T>(x);
1391
+ });
1392
+ return Vectorized<T>::loadu(static_cast<const void*>(buffer.data()));
1393
+ }
1394
+
1395
+ // clang-format off
1396
+ // Example inputs for AVX512:
1397
+ // a Vectorized<float> = {a0, b0, a1, b1, a2, b2, a3, b3, a4, b4, a5, b5, a6, b6, a7, b7}
1398
+ // b Vectorized<float> = {a8, b8, a9, b9, a10, b10, a11, b11, a12, b12, a13, b13, a14, b14, a15, b15}
1399
+ // returns:
1400
+ // Vectorized<float> = {a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15}
1401
+ // Vectorized<float> = {b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15}
1402
+ // Example inputs for AVX2: a Vectorized<float> = {a0, b0, a1, b1, a2, b2, a3, b3}
1403
+ // b Vectorized<float> = {a4, b4, a5, b5, a6, b6, a7, b7}
1404
+ // returns: Vectorized<float> = {a0, a1, a2, a3, a4, a5, a6, a7}
1405
+ // Vectorized<float> = {b0, b1, b2, b3, b4, b5, b6, b7}
1406
+ // clang-format on
1407
+ template <typename T>
1408
+ inline std::enable_if_t<
1409
+ Vectorized<T>::size() % 2 == 0,
1410
+ std::pair<Vectorized<T>, Vectorized<T>>>
1411
+ deinterleave2(const Vectorized<T>& a, const Vectorized<T>& b) {
1412
+ static constexpr int size = Vectorized<T>::size();
1413
+ static constexpr int half_size = size / 2;
1414
+ T a_arr[size];
1415
+ T b_arr[size];
1416
+ T buffer1[size];
1417
+ T buffer2[size];
1418
+ a.store(static_cast<void*>(a_arr));
1419
+ b.store(static_cast<void*>(b_arr));
1420
+ for (const auto i : c10::irange(half_size)) {
1421
+ buffer1[i] = a_arr[i * 2];
1422
+ buffer1[half_size + i] = b_arr[i * 2];
1423
+ buffer2[i] = a_arr[i * 2 + 1];
1424
+ buffer2[half_size + i] = b_arr[i * 2 + 1];
1425
+ }
1426
+ return std::make_pair(
1427
+ Vectorized<T>::loadu(static_cast<void*>(buffer1)),
1428
+ Vectorized<T>::loadu(static_cast<void*>(buffer2)));
1429
+ }
1430
+
1431
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_FUNC(deinterleave2)
1432
+
1433
+ // clang-format off
1434
+ // inverse operation of deinterleave2
1435
+ // Example inputs for AVX512:
1436
+ // a Vectorized<float> = {a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15}
1437
+ // b Vectorized<float> = {b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15}
1438
+ // returns, for AVX512:
1439
+ // Vectorized<float> = {a0, b0, a1, b1, a2, b2, a3, b3, a4, b4, a5, b5, a6, b6, a7, b7}
1440
+ // Vectorized<float> = {a8, b8, a9, b9, a10, b10, a11, b11, a12, b12, a13, b13, a14, b14, a15, b15}
1441
+ // Example inputs for AVX2 : a Vectorized<float> = {a0, a1, a2, a3, a4, a5, a6, a7}
1442
+ // b Vectorized<float> = {b0, b1, b2, b3, b4, b5, b6, b7}
1443
+ // returns: Vectorized<float> = {a0, b0, a1, b1, a2, b2, a3, b3}
1444
+ // Vectorized<float> = {a4, b4, a5, b5, a6, b6, a7, b7}
1445
+ // clang-format on
1446
+ template <typename T>
1447
+ inline std::enable_if_t<
1448
+ Vectorized<T>::size() % 2 == 0,
1449
+ std::pair<Vectorized<T>, Vectorized<T>>>
1450
+ interleave2(const Vectorized<T>& a, const Vectorized<T>& b) {
1451
+ static constexpr int size = Vectorized<T>::size();
1452
+ static constexpr int half_size = size / 2;
1453
+ T a_arr[size];
1454
+ T b_arr[size];
1455
+ T buffer1[size];
1456
+ T buffer2[size];
1457
+ a.store(static_cast<void*>(a_arr));
1458
+ b.store(static_cast<void*>(b_arr));
1459
+ for (const auto i : c10::irange(half_size)) {
1460
+ buffer1[i * 2] = a_arr[i];
1461
+ buffer1[i * 2 + 1] = b_arr[i];
1462
+ buffer2[i * 2] = a_arr[half_size + i];
1463
+ buffer2[i * 2 + 1] = b_arr[half_size + i];
1464
+ }
1465
+ return std::make_pair(
1466
+ Vectorized<T>::loadu(static_cast<void*>(buffer1)),
1467
+ Vectorized<T>::loadu(static_cast<void*>(buffer2)));
1468
+ }
1469
+
1470
+ VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_FUNC(interleave2)
1471
+
1472
+ #undef VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_FUNC
1473
+ #undef VECTORIZED_SUPPORT_SCALARS_FOR_BINARY_OP
1474
+ #undef VECTORIZED_SUPPORT_SCALARS_FOR_TERNARY_FUNC
1475
+
1476
+ template <typename src_T, typename dst_T>
1477
+ inline void convert(const src_T* src, dst_T* dst, int64_t n) {
1478
+ #ifndef _MSC_VER
1479
+ #pragma unroll
1480
+ #endif
1481
+ for ([[maybe_unused]] const auto i : c10::irange(n)) {
1482
+ *dst = c10::convert<dst_T>(c10::load(src));
1483
+ src++;
1484
+ dst++;
1485
+ }
1486
+ }
1487
+
1488
+ template <typename T>
1489
+ inline Vectorized<T> flip(const Vectorized<T>& data) {
1490
+ static constexpr int size = Vectorized<T>::size();
1491
+ T output[size];
1492
+ T buffer[size];
1493
+ data.store(static_cast<void*>(buffer));
1494
+ for (const auto i : c10::irange(size)) {
1495
+ output[i] = buffer[size - i - 1];
1496
+ }
1497
+ return Vectorized<T>::loadu(static_cast<void*>(output));
1498
+ }
1499
+
1500
+ // Transpose the `src` buffer of type `T` and size (M,N) into the `dst` buffer.
1501
+ // `ld_src` is the leading dimension of `src` and `ld_dst` is the leading
1502
+ // dimension of `dst`.
1503
+ template <typename T>
1504
+ inline void transpose_mxn(
1505
+ const T* src,
1506
+ int64_t ld_src,
1507
+ T* dst,
1508
+ int64_t ld_dst,
1509
+ int M,
1510
+ int N) {
1511
+ for (int i = 0; i < M; i++) {
1512
+ for (int j = 0; j < N; j++) {
1513
+ dst[j * ld_dst + i] = src[i * ld_src + j];
1514
+ }
1515
+ }
1516
+ }
1517
+
1518
+ template <typename T, int M, int N>
1519
+ inline void transpose_mxn(
1520
+ const T* src,
1521
+ int64_t ld_src,
1522
+ T* dst,
1523
+ int64_t ld_dst) {
1524
+ transpose_mxn<T>(src, ld_src, dst, ld_dst, M, N);
1525
+ }
1526
+
1527
+ } // namespace CPU_CAPABILITY
1528
+ } // namespace at::vec
1529
+
1530
+ // additional headers for more operations that depend on vec_base
1531
+ #include <ATen/cpu/vec/vec_convert.h>
1532
+ #include <ATen/cpu/vec/vec_mask.h>
1533
+ #include <ATen/cpu/vec/vec_n.h>
1534
+
1535
+ #else
1536
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
1537
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/cpu/vec/vec_convert.h ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ #include <ATen/cpu/vec/vec_base.h>
5
+ #include <ATen/cpu/vec/vec_n.h>
6
+
7
+ namespace at::vec {
8
+ inline namespace CPU_CAPABILITY {
9
+
10
+ template <
11
+ typename dst_t,
12
+ int dst_n,
13
+ typename src_t,
14
+ int src_n,
15
+ typename Enabled = void>
16
+ struct VecConvert {
17
+ static inline VectorizedN<dst_t, dst_n> apply(
18
+ const VectorizedN<src_t, src_n>& src) {
19
+ constexpr int count = std::min(
20
+ VectorizedN<src_t, src_n>::size(), VectorizedN<dst_t, dst_n>::size());
21
+ __at_align__ src_t src_buf[VectorizedN<src_t, src_n>::size()];
22
+ src.store(src_buf);
23
+ __at_align__ dst_t dst_buf[VectorizedN<dst_t, dst_n>::size()];
24
+ for (int i = 0; i < count; i++) {
25
+ dst_buf[i] = static_cast<dst_t>(src_buf[i]);
26
+ }
27
+ return VectorizedN<dst_t, dst_n>::loadu(dst_buf, count);
28
+ }
29
+ };
30
+
31
+ template <typename dst_t, typename src_t>
32
+ inline std::enable_if_t<std::is_same_v<dst_t, src_t>, Vectorized<src_t>> convert(
33
+ const Vectorized<src_t>& src) {
34
+ return src;
35
+ }
36
+
37
+ template <typename dst_t, typename src_t>
38
+ inline std::enable_if_t<!std::is_same_v<dst_t, src_t>, Vectorized<dst_t>>
39
+ convert(const Vectorized<src_t>& src) {
40
+ return VecConvert<dst_t, 1, src_t, 1>::apply(src);
41
+ }
42
+
43
+ template <
44
+ typename dst_t,
45
+ int dst_n,
46
+ typename src_t,
47
+ int src_n,
48
+ std::enable_if_t<dst_n != 1, int> = 0>
49
+ inline VectorizedN<dst_t, dst_n> convert(const VectorizedN<src_t, src_n>& src) {
50
+ return VecConvert<dst_t, dst_n, src_t, src_n>::apply(src);
51
+ }
52
+
53
+ template <
54
+ typename dst_t,
55
+ int dst_n,
56
+ typename src_t,
57
+ int src_n,
58
+ bool keep = false,
59
+ std::enable_if_t<dst_n == 1, int> = 0>
60
+ inline std::conditional_t<keep, VectorizedN<dst_t, 1>, Vectorized<dst_t>>
61
+ convert(const VectorizedN<src_t, src_n>& src) {
62
+ return VecConvert<dst_t, dst_n, src_t, src_n>::apply(src);
63
+ }
64
+
65
+ } // namespace CPU_CAPABILITY
66
+
67
+ template <
68
+ typename scalar_t,
69
+ typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
70
+ inline std::tuple<Vectorized<float>, Vectorized<float>> convert_to_float(
71
+ const Vectorized<scalar_t>&);
72
+
73
+ template <
74
+ typename scalar_t,
75
+ typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
76
+ inline Vectorized<scalar_t> convert_from_float(
77
+ const Vectorized<float>&,
78
+ const Vectorized<float>&);
79
+
80
+ } // namespace at::vec
81
+
82
+ #else
83
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
84
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_log_softmax_ops.h ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from Operator.h
5
+
6
+ #include <string_view>
7
+ #include <tuple>
8
+ #include <vector>
9
+
10
+ // Forward declarations of any types needed in the operator signatures.
11
+ // We can't directly include these classes because it will cause circular include dependencies.
12
+ // This file is included by TensorBody.h, which defines the Tensor class.
13
+ #include <ATen/core/ATen_fwd.h>
14
+
15
+ namespace at {
16
+ namespace _ops {
17
+
18
+
19
+ struct TORCH_API _log_softmax {
20
+ using schema = at::Tensor (const at::Tensor &, int64_t, bool);
21
+ using ptr_schema = schema*;
22
+ // See Note [static constexpr char* members for windows NVCC]
23
+ static constexpr const char* name = "aten::_log_softmax";
24
+ static constexpr const char* overload_name = "";
25
+ static constexpr const char* schema_str = "_log_softmax(Tensor self, int dim, bool half_to_float) -> Tensor";
26
+ static at::Tensor call(const at::Tensor & self, int64_t dim, bool half_to_float);
27
+ static at::Tensor redispatch(c10::DispatchKeySet dispatchKeySet, const at::Tensor & self, int64_t dim, bool half_to_float);
28
+ };
29
+
30
+ struct TORCH_API _log_softmax_out {
31
+ using schema = at::Tensor & (const at::Tensor &, int64_t, bool, at::Tensor &);
32
+ using ptr_schema = schema*;
33
+ // See Note [static constexpr char* members for windows NVCC]
34
+ static constexpr const char* name = "aten::_log_softmax";
35
+ static constexpr const char* overload_name = "out";
36
+ static constexpr const char* schema_str = "_log_softmax.out(Tensor self, int dim, bool half_to_float, *, Tensor(a!) out) -> Tensor(a!)";
37
+ static at::Tensor & call(const at::Tensor & self, int64_t dim, bool half_to_float, at::Tensor & out);
38
+ static at::Tensor & redispatch(c10::DispatchKeySet dispatchKeySet, const at::Tensor & self, int64_t dim, bool half_to_float, at::Tensor & out);
39
+ };
40
+
41
+ }} // namespace at::_ops
42
+
43
+ #else
44
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
45
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lstm_mps_compositeexplicitautograd_dispatch.h ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+ // @generated by torchgen/gen.py from DispatchKeyFunction.h
4
+
5
+ // NB: The implementing C++ file is RegisterDispatchKey.cpp
6
+
7
+ // The only #includes we need are for custom classes that have defaults in the C++ API
8
+ #include <c10/core/MemoryFormat.h>
9
+ #include <c10/core/Scalar.h>
10
+ #include <ATen/core/Reduction.h>
11
+
12
+ // Forward declarations of any types needed in the operator signatures.
13
+ // We can't directly include these classes because it will cause circular include dependencies.
14
+ // This file is included by TensorBody.h, which defines the Tensor class.
15
+ #include <ATen/core/ATen_fwd.h>
16
+
17
+ namespace at {
18
+
19
+ namespace compositeexplicitautograd {
20
+
21
+ TORCH_API ::std::tuple<at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &> _lstm_mps_out(at::Tensor & out0, at::Tensor & out1, at::Tensor & out2, at::Tensor & out3, at::Tensor & out4, at::Tensor & out5, const at::Tensor & input, at::TensorList hx, at::TensorList params, bool has_biases, int64_t num_layers, double dropout, bool train, bool bidirectional, bool batch_first);
22
+ TORCH_API ::std::tuple<at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &> _lstm_mps_outf(const at::Tensor & input, at::TensorList hx, at::TensorList params, bool has_biases, int64_t num_layers, double dropout, bool train, bool bidirectional, bool batch_first, at::Tensor & out0, at::Tensor & out1, at::Tensor & out2, at::Tensor & out3, at::Tensor & out4, at::Tensor & out5);
23
+
24
+ } // namespace compositeexplicitautograd
25
+ } // namespace at
26
+
27
+ #else
28
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
29
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lstm_mps_native.h ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from NativeFunction.h
5
+
6
+ #include <c10/core/Scalar.h>
7
+ #include <c10/core/Storage.h>
8
+ #include <c10/core/TensorOptions.h>
9
+ #include <c10/util/Deprecated.h>
10
+ #include <optional>
11
+ #include <c10/core/QScheme.h>
12
+ #include <ATen/core/Reduction.h>
13
+ #include <ATen/core/Tensor.h>
14
+ #include <tuple>
15
+ #include <vector>
16
+
17
+
18
+ namespace at {
19
+ namespace native {
20
+ TORCH_API ::std::tuple<at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &> _lstm_mps_out(const at::Tensor & input, at::TensorList hx, at::TensorList params, bool has_biases, int64_t num_layers, double dropout, bool train, bool bidirectional, bool batch_first, at::Tensor & out0, at::Tensor & out1, at::Tensor & out2, at::Tensor & out3, at::Tensor & out4, at::Tensor & out5);
21
+ } // namespace native
22
+ } // namespace at
23
+
24
+ #else
25
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
26
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lstm_mps_ops.h ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from Operator.h
5
+
6
+ #include <string_view>
7
+ #include <tuple>
8
+ #include <vector>
9
+
10
+ // Forward declarations of any types needed in the operator signatures.
11
+ // We can't directly include these classes because it will cause circular include dependencies.
12
+ // This file is included by TensorBody.h, which defines the Tensor class.
13
+ #include <ATen/core/ATen_fwd.h>
14
+
15
+ namespace at {
16
+ namespace _ops {
17
+
18
+
19
+ struct TORCH_API _lstm_mps {
20
+ using schema = ::std::tuple<at::Tensor,at::Tensor,at::Tensor,at::Tensor,at::Tensor,at::Tensor> (const at::Tensor &, at::TensorList, at::TensorList, bool, int64_t, double, bool, bool, bool);
21
+ using ptr_schema = schema*;
22
+ // See Note [static constexpr char* members for windows NVCC]
23
+ static constexpr const char* name = "aten::_lstm_mps";
24
+ static constexpr const char* overload_name = "";
25
+ static constexpr const char* schema_str = "_lstm_mps(Tensor input, Tensor[] hx, Tensor[] params, bool has_biases, int num_layers, float dropout, bool train, bool bidirectional, bool batch_first) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor)";
26
+ static ::std::tuple<at::Tensor,at::Tensor,at::Tensor,at::Tensor,at::Tensor,at::Tensor> call(const at::Tensor & input, at::TensorList hx, at::TensorList params, bool has_biases, int64_t num_layers, double dropout, bool train, bool bidirectional, bool batch_first);
27
+ static ::std::tuple<at::Tensor,at::Tensor,at::Tensor,at::Tensor,at::Tensor,at::Tensor> redispatch(c10::DispatchKeySet dispatchKeySet, const at::Tensor & input, at::TensorList hx, at::TensorList params, bool has_biases, int64_t num_layers, double dropout, bool train, bool bidirectional, bool batch_first);
28
+ };
29
+
30
+ struct TORCH_API _lstm_mps_out {
31
+ using schema = ::std::tuple<at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &> (const at::Tensor &, at::TensorList, at::TensorList, bool, int64_t, double, bool, bool, bool, at::Tensor &, at::Tensor &, at::Tensor &, at::Tensor &, at::Tensor &, at::Tensor &);
32
+ using ptr_schema = schema*;
33
+ // See Note [static constexpr char* members for windows NVCC]
34
+ static constexpr const char* name = "aten::_lstm_mps";
35
+ static constexpr const char* overload_name = "out";
36
+ static constexpr const char* schema_str = "_lstm_mps.out(Tensor input, Tensor[] hx, Tensor[] params, bool has_biases, int num_layers, float dropout, bool train, bool bidirectional, bool batch_first, *, Tensor(a!) out0, Tensor(b!) out1, Tensor(c!) out2, Tensor(d!) out3, Tensor(e!) out4, Tensor(f!) out5) -> (Tensor(a!), Tensor(b!), Tensor(c!), Tensor(d!), Tensor(e!), Tensor(f!))";
37
+ static ::std::tuple<at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &> call(const at::Tensor & input, at::TensorList hx, at::TensorList params, bool has_biases, int64_t num_layers, double dropout, bool train, bool bidirectional, bool batch_first, at::Tensor & out0, at::Tensor & out1, at::Tensor & out2, at::Tensor & out3, at::Tensor & out4, at::Tensor & out5);
38
+ static ::std::tuple<at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &,at::Tensor &> redispatch(c10::DispatchKeySet dispatchKeySet, const at::Tensor & input, at::TensorList hx, at::TensorList params, bool has_biases, int64_t num_layers, double dropout, bool train, bool bidirectional, bool batch_first, at::Tensor & out0, at::Tensor & out1, at::Tensor & out2, at::Tensor & out3, at::Tensor & out4, at::Tensor & out5);
39
+ };
40
+
41
+ }} // namespace at::_ops
42
+
43
+ #else
44
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
45
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lu_with_info.h ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from Function.h
5
+
6
+ #include <ATen/Context.h>
7
+ #include <ATen/DeviceGuard.h>
8
+ #include <ATen/TensorUtils.h>
9
+ #include <ATen/TracerMode.h>
10
+ #include <ATen/core/Generator.h>
11
+ #include <ATen/core/Reduction.h>
12
+ #include <ATen/core/Tensor.h>
13
+ #include <c10/core/Scalar.h>
14
+ #include <c10/core/Storage.h>
15
+ #include <c10/core/TensorOptions.h>
16
+ #include <c10/util/Deprecated.h>
17
+ #include <optional>
18
+ #include <string_view>
19
+
20
+
21
+
22
+ #include <ATen/ops/_lu_with_info_ops.h>
23
+
24
+ namespace at {
25
+
26
+
27
+ // aten::_lu_with_info(Tensor self, bool pivot=True, bool check_errors=True) -> (Tensor LU, Tensor pivots, Tensor info)
28
+ inline ::std::tuple<at::Tensor,at::Tensor,at::Tensor> _lu_with_info(const at::Tensor & self, bool pivot=true, bool check_errors=true) {
29
+ return at::_ops::_lu_with_info::call(self, pivot, check_errors);
30
+ }
31
+
32
+ }
33
+
34
+ #else
35
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
36
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lu_with_info_compositeimplicitautograd_dispatch.h ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+ // @generated by torchgen/gen.py from DispatchKeyFunction.h
4
+
5
+ // NB: The implementing C++ file is RegisterDispatchKey.cpp
6
+
7
+ // The only #includes we need are for custom classes that have defaults in the C++ API
8
+ #include <c10/core/MemoryFormat.h>
9
+ #include <c10/core/Scalar.h>
10
+ #include <ATen/core/Reduction.h>
11
+
12
+ // Forward declarations of any types needed in the operator signatures.
13
+ // We can't directly include these classes because it will cause circular include dependencies.
14
+ // This file is included by TensorBody.h, which defines the Tensor class.
15
+ #include <ATen/core/ATen_fwd.h>
16
+
17
+ namespace at {
18
+
19
+ namespace compositeimplicitautograd {
20
+
21
+ TORCH_API ::std::tuple<at::Tensor,at::Tensor,at::Tensor> _lu_with_info(const at::Tensor & self, bool pivot=true, bool check_errors=true);
22
+
23
+ } // namespace compositeimplicitautograd
24
+ } // namespace at
25
+
26
+ #else
27
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
28
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lu_with_info_native.h ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from NativeFunction.h
5
+
6
+ #include <c10/core/Scalar.h>
7
+ #include <c10/core/Storage.h>
8
+ #include <c10/core/TensorOptions.h>
9
+ #include <c10/util/Deprecated.h>
10
+ #include <optional>
11
+ #include <c10/core/QScheme.h>
12
+ #include <ATen/core/Reduction.h>
13
+ #include <ATen/core/Tensor.h>
14
+ #include <tuple>
15
+ #include <vector>
16
+
17
+
18
+ namespace at {
19
+ namespace native {
20
+ TORCH_API ::std::tuple<at::Tensor,at::Tensor,at::Tensor> _lu_with_info(const at::Tensor & self, bool pivot=true, bool check_errors=true);
21
+ } // namespace native
22
+ } // namespace at
23
+
24
+ #else
25
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
26
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_lu_with_info_ops.h ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from Operator.h
5
+
6
+ #include <string_view>
7
+ #include <tuple>
8
+ #include <vector>
9
+
10
+ // Forward declarations of any types needed in the operator signatures.
11
+ // We can't directly include these classes because it will cause circular include dependencies.
12
+ // This file is included by TensorBody.h, which defines the Tensor class.
13
+ #include <ATen/core/ATen_fwd.h>
14
+
15
+ namespace at {
16
+ namespace _ops {
17
+
18
+
19
+ struct TORCH_API _lu_with_info {
20
+ using schema = ::std::tuple<at::Tensor,at::Tensor,at::Tensor> (const at::Tensor &, bool, bool);
21
+ using ptr_schema = schema*;
22
+ // See Note [static constexpr char* members for windows NVCC]
23
+ static constexpr const char* name = "aten::_lu_with_info";
24
+ static constexpr const char* overload_name = "";
25
+ static constexpr const char* schema_str = "_lu_with_info(Tensor self, bool pivot=True, bool check_errors=True) -> (Tensor LU, Tensor pivots, Tensor info)";
26
+ static ::std::tuple<at::Tensor,at::Tensor,at::Tensor> call(const at::Tensor & self, bool pivot, bool check_errors);
27
+ static ::std::tuple<at::Tensor,at::Tensor,at::Tensor> redispatch(c10::DispatchKeySet dispatchKeySet, const at::Tensor & self, bool pivot, bool check_errors);
28
+ };
29
+
30
+ }} // namespace at::_ops
31
+
32
+ #else
33
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
34
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dep_token.h ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from Function.h
5
+
6
+ #include <ATen/Context.h>
7
+ #include <ATen/DeviceGuard.h>
8
+ #include <ATen/TensorUtils.h>
9
+ #include <ATen/TracerMode.h>
10
+ #include <ATen/core/Generator.h>
11
+ #include <ATen/core/Reduction.h>
12
+ #include <ATen/core/Tensor.h>
13
+ #include <c10/core/Scalar.h>
14
+ #include <c10/core/Storage.h>
15
+ #include <c10/core/TensorOptions.h>
16
+ #include <c10/util/Deprecated.h>
17
+ #include <optional>
18
+ #include <string_view>
19
+
20
+
21
+
22
+ #include <ATen/ops/_make_dep_token_ops.h>
23
+
24
+ namespace at {
25
+
26
+
27
+ // aten::_make_dep_token(*, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor
28
+ inline at::Tensor _make_dep_token(at::TensorOptions options={}, ::std::optional<at::MemoryFormat> memory_format=::std::nullopt) {
29
+ return at::_ops::_make_dep_token::call(c10::optTypeMetaToScalarType(options.dtype_opt()), options.layout_opt(), options.device_opt(), options.pinned_memory_opt(), c10::impl::check_tensor_options_and_extract_memory_format(options, memory_format));
30
+ }
31
+ // aten::_make_dep_token(*, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor
32
+ inline at::Tensor _make_dep_token(::std::optional<at::ScalarType> dtype, ::std::optional<at::Layout> layout, ::std::optional<at::Device> device, ::std::optional<bool> pin_memory, ::std::optional<at::MemoryFormat> memory_format) {
33
+ return at::_ops::_make_dep_token::call(dtype, layout, device, pin_memory, memory_format);
34
+ }
35
+
36
+ }
37
+
38
+ #else
39
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
40
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dep_token_cpu_dispatch.h ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+ // @generated by torchgen/gen.py from DispatchKeyFunction.h
4
+
5
+ // NB: The implementing C++ file is RegisterDispatchKey.cpp
6
+
7
+ // The only #includes we need are for custom classes that have defaults in the C++ API
8
+ #include <c10/core/MemoryFormat.h>
9
+ #include <c10/core/Scalar.h>
10
+ #include <ATen/core/Reduction.h>
11
+
12
+ // Forward declarations of any types needed in the operator signatures.
13
+ // We can't directly include these classes because it will cause circular include dependencies.
14
+ // This file is included by TensorBody.h, which defines the Tensor class.
15
+ #include <ATen/core/ATen_fwd.h>
16
+
17
+ namespace at {
18
+
19
+ namespace cpu {
20
+
21
+ TORCH_API at::Tensor _make_dep_token(at::TensorOptions options={}, ::std::optional<at::MemoryFormat> memory_format=::std::nullopt);
22
+ TORCH_API at::Tensor _make_dep_token(::std::optional<at::ScalarType> dtype, ::std::optional<at::Layout> layout, ::std::optional<at::Device> device, ::std::optional<bool> pin_memory, ::std::optional<at::MemoryFormat> memory_format);
23
+
24
+ } // namespace cpu
25
+ } // namespace at
26
+
27
+ #else
28
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
29
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dep_token_native.h ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from NativeFunction.h
5
+
6
+ #include <c10/core/Scalar.h>
7
+ #include <c10/core/Storage.h>
8
+ #include <c10/core/TensorOptions.h>
9
+ #include <c10/util/Deprecated.h>
10
+ #include <optional>
11
+ #include <c10/core/QScheme.h>
12
+ #include <ATen/core/Reduction.h>
13
+ #include <ATen/core/Tensor.h>
14
+ #include <tuple>
15
+ #include <vector>
16
+
17
+
18
+ namespace at {
19
+ namespace native {
20
+ TORCH_API at::Tensor _make_dep_token_cpu(::std::optional<at::ScalarType> dtype={}, ::std::optional<at::Layout> layout={}, ::std::optional<at::Device> device={}, ::std::optional<bool> pin_memory={}, ::std::optional<at::MemoryFormat> memory_format=::std::nullopt);
21
+ } // namespace native
22
+ } // namespace at
23
+
24
+ #else
25
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
26
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dep_token_ops.h ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from Operator.h
5
+
6
+ #include <string_view>
7
+ #include <tuple>
8
+ #include <vector>
9
+
10
+ // Forward declarations of any types needed in the operator signatures.
11
+ // We can't directly include these classes because it will cause circular include dependencies.
12
+ // This file is included by TensorBody.h, which defines the Tensor class.
13
+ #include <ATen/core/ATen_fwd.h>
14
+
15
+ namespace at {
16
+ namespace _ops {
17
+
18
+
19
+ struct TORCH_API _make_dep_token {
20
+ using schema = at::Tensor (::std::optional<at::ScalarType>, ::std::optional<at::Layout>, ::std::optional<at::Device>, ::std::optional<bool>, ::std::optional<at::MemoryFormat>);
21
+ using ptr_schema = schema*;
22
+ // See Note [static constexpr char* members for windows NVCC]
23
+ static constexpr const char* name = "aten::_make_dep_token";
24
+ static constexpr const char* overload_name = "";
25
+ static constexpr const char* schema_str = "_make_dep_token(*, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None, MemoryFormat? memory_format=None) -> Tensor";
26
+ static at::Tensor call(::std::optional<at::ScalarType> dtype, ::std::optional<at::Layout> layout, ::std::optional<at::Device> device, ::std::optional<bool> pin_memory, ::std::optional<at::MemoryFormat> memory_format);
27
+ static at::Tensor redispatch(c10::DispatchKeySet dispatchKeySet, ::std::optional<at::ScalarType> dtype, ::std::optional<at::Layout> layout, ::std::optional<at::Device> device, ::std::optional<bool> pin_memory, ::std::optional<at::MemoryFormat> memory_format);
28
+ };
29
+
30
+ }} // namespace at::_ops
31
+
32
+ #else
33
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
34
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual.h ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from Function.h
5
+
6
+ #include <ATen/Context.h>
7
+ #include <ATen/DeviceGuard.h>
8
+ #include <ATen/TensorUtils.h>
9
+ #include <ATen/TracerMode.h>
10
+ #include <ATen/core/Generator.h>
11
+ #include <ATen/core/Reduction.h>
12
+ #include <ATen/core/Tensor.h>
13
+ #include <c10/core/Scalar.h>
14
+ #include <c10/core/Storage.h>
15
+ #include <c10/core/TensorOptions.h>
16
+ #include <c10/util/Deprecated.h>
17
+ #include <optional>
18
+ #include <string_view>
19
+
20
+
21
+
22
+ #include <ATen/ops/_make_dual_ops.h>
23
+
24
+ namespace at {
25
+
26
+
27
+ // aten::_make_dual(Tensor(a) primal, Tensor tangent, int level) -> Tensor(a)
28
+ inline at::Tensor _make_dual(const at::Tensor & primal, const at::Tensor & tangent, int64_t level) {
29
+ return at::_ops::_make_dual::call(primal, tangent, level);
30
+ }
31
+
32
+ }
33
+
34
+ #else
35
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
36
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_compositeexplicitautograd_dispatch.h ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+ // @generated by torchgen/gen.py from DispatchKeyFunction.h
4
+
5
+ // NB: The implementing C++ file is RegisterDispatchKey.cpp
6
+
7
+ // The only #includes we need are for custom classes that have defaults in the C++ API
8
+ #include <c10/core/MemoryFormat.h>
9
+ #include <c10/core/Scalar.h>
10
+ #include <ATen/core/Reduction.h>
11
+
12
+ // Forward declarations of any types needed in the operator signatures.
13
+ // We can't directly include these classes because it will cause circular include dependencies.
14
+ // This file is included by TensorBody.h, which defines the Tensor class.
15
+ #include <ATen/core/ATen_fwd.h>
16
+
17
+ namespace at {
18
+
19
+ namespace compositeexplicitautograd {
20
+
21
+ TORCH_API at::Tensor _make_dual(const at::Tensor & primal, const at::Tensor & tangent, int64_t level);
22
+
23
+ } // namespace compositeexplicitautograd
24
+ } // namespace at
25
+
26
+ #else
27
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
28
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_copy.h ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from Function.h
5
+
6
+ #include <ATen/Context.h>
7
+ #include <ATen/DeviceGuard.h>
8
+ #include <ATen/TensorUtils.h>
9
+ #include <ATen/TracerMode.h>
10
+ #include <ATen/core/Generator.h>
11
+ #include <ATen/core/Reduction.h>
12
+ #include <ATen/core/Tensor.h>
13
+ #include <c10/core/Scalar.h>
14
+ #include <c10/core/Storage.h>
15
+ #include <c10/core/TensorOptions.h>
16
+ #include <c10/util/Deprecated.h>
17
+ #include <optional>
18
+ #include <string_view>
19
+
20
+
21
+
22
+ #include <ATen/ops/_make_dual_copy_ops.h>
23
+
24
+ namespace at {
25
+
26
+
27
+ // aten::_make_dual_copy(Tensor primal, Tensor tangent, int level) -> Tensor
28
+ inline at::Tensor _make_dual_copy(const at::Tensor & primal, const at::Tensor & tangent, int64_t level) {
29
+ return at::_ops::_make_dual_copy::call(primal, tangent, level);
30
+ }
31
+
32
+ // aten::_make_dual_copy.out(Tensor primal, Tensor tangent, int level, *, Tensor(a!) out) -> Tensor(a!)
33
+ inline at::Tensor & _make_dual_copy_out(at::Tensor & out, const at::Tensor & primal, const at::Tensor & tangent, int64_t level) {
34
+ return at::_ops::_make_dual_copy_out::call(primal, tangent, level, out);
35
+ }
36
+ // aten::_make_dual_copy.out(Tensor primal, Tensor tangent, int level, *, Tensor(a!) out) -> Tensor(a!)
37
+ inline at::Tensor & _make_dual_copy_outf(const at::Tensor & primal, const at::Tensor & tangent, int64_t level, at::Tensor & out) {
38
+ return at::_ops::_make_dual_copy_out::call(primal, tangent, level, out);
39
+ }
40
+
41
+ }
42
+
43
+ #else
44
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
45
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_copy_compositeexplicitautograd_dispatch.h ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+ // @generated by torchgen/gen.py from DispatchKeyFunction.h
4
+
5
+ // NB: The implementing C++ file is RegisterDispatchKey.cpp
6
+
7
+ // The only #includes we need are for custom classes that have defaults in the C++ API
8
+ #include <c10/core/MemoryFormat.h>
9
+ #include <c10/core/Scalar.h>
10
+ #include <ATen/core/Reduction.h>
11
+
12
+ // Forward declarations of any types needed in the operator signatures.
13
+ // We can't directly include these classes because it will cause circular include dependencies.
14
+ // This file is included by TensorBody.h, which defines the Tensor class.
15
+ #include <ATen/core/ATen_fwd.h>
16
+
17
+ namespace at {
18
+
19
+ namespace compositeexplicitautograd {
20
+
21
+ TORCH_API at::Tensor & _make_dual_copy_out(at::Tensor & out, const at::Tensor & primal, const at::Tensor & tangent, int64_t level);
22
+ TORCH_API at::Tensor & _make_dual_copy_outf(const at::Tensor & primal, const at::Tensor & tangent, int64_t level, at::Tensor & out);
23
+
24
+ } // namespace compositeexplicitautograd
25
+ } // namespace at
26
+
27
+ #else
28
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
29
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_copy_compositeexplicitautogradnonfunctional_dispatch.h ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+ // @generated by torchgen/gen.py from DispatchKeyFunction.h
4
+
5
+ // NB: The implementing C++ file is RegisterDispatchKey.cpp
6
+
7
+ // The only #includes we need are for custom classes that have defaults in the C++ API
8
+ #include <c10/core/MemoryFormat.h>
9
+ #include <c10/core/Scalar.h>
10
+ #include <ATen/core/Reduction.h>
11
+
12
+ // Forward declarations of any types needed in the operator signatures.
13
+ // We can't directly include these classes because it will cause circular include dependencies.
14
+ // This file is included by TensorBody.h, which defines the Tensor class.
15
+ #include <ATen/core/ATen_fwd.h>
16
+
17
+ namespace at {
18
+
19
+ namespace compositeexplicitautogradnonfunctional {
20
+
21
+ TORCH_API at::Tensor _make_dual_copy(const at::Tensor & primal, const at::Tensor & tangent, int64_t level);
22
+
23
+ } // namespace compositeexplicitautogradnonfunctional
24
+ } // namespace at
25
+
26
+ #else
27
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
28
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_copy_native.h ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from NativeFunction.h
5
+
6
+ #include <c10/core/Scalar.h>
7
+ #include <c10/core/Storage.h>
8
+ #include <c10/core/TensorOptions.h>
9
+ #include <c10/util/Deprecated.h>
10
+ #include <optional>
11
+ #include <c10/core/QScheme.h>
12
+ #include <ATen/core/Reduction.h>
13
+ #include <ATen/core/Tensor.h>
14
+ #include <tuple>
15
+ #include <vector>
16
+
17
+
18
+ namespace at {
19
+ namespace native {
20
+ TORCH_API at::Tensor & _make_dual_copy_out(const at::Tensor & primal, const at::Tensor & tangent, int64_t level, at::Tensor & out);
21
+ TORCH_API at::Tensor _make_dual_copy(const at::Tensor & primal, const at::Tensor & tangent, int64_t level);
22
+ } // namespace native
23
+ } // namespace at
24
+
25
+ #else
26
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
27
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_copy_ops.h ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from Operator.h
5
+
6
+ #include <string_view>
7
+ #include <tuple>
8
+ #include <vector>
9
+
10
+ // Forward declarations of any types needed in the operator signatures.
11
+ // We can't directly include these classes because it will cause circular include dependencies.
12
+ // This file is included by TensorBody.h, which defines the Tensor class.
13
+ #include <ATen/core/ATen_fwd.h>
14
+
15
+ namespace at {
16
+ namespace _ops {
17
+
18
+
19
+ struct TORCH_API _make_dual_copy {
20
+ using schema = at::Tensor (const at::Tensor &, const at::Tensor &, int64_t);
21
+ using ptr_schema = schema*;
22
+ // See Note [static constexpr char* members for windows NVCC]
23
+ static constexpr const char* name = "aten::_make_dual_copy";
24
+ static constexpr const char* overload_name = "";
25
+ static constexpr const char* schema_str = "_make_dual_copy(Tensor primal, Tensor tangent, int level) -> Tensor";
26
+ static at::Tensor call(const at::Tensor & primal, const at::Tensor & tangent, int64_t level);
27
+ static at::Tensor redispatch(c10::DispatchKeySet dispatchKeySet, const at::Tensor & primal, const at::Tensor & tangent, int64_t level);
28
+ };
29
+
30
+ struct TORCH_API _make_dual_copy_out {
31
+ using schema = at::Tensor & (const at::Tensor &, const at::Tensor &, int64_t, at::Tensor &);
32
+ using ptr_schema = schema*;
33
+ // See Note [static constexpr char* members for windows NVCC]
34
+ static constexpr const char* name = "aten::_make_dual_copy";
35
+ static constexpr const char* overload_name = "out";
36
+ static constexpr const char* schema_str = "_make_dual_copy.out(Tensor primal, Tensor tangent, int level, *, Tensor(a!) out) -> Tensor(a!)";
37
+ static at::Tensor & call(const at::Tensor & primal, const at::Tensor & tangent, int64_t level, at::Tensor & out);
38
+ static at::Tensor & redispatch(c10::DispatchKeySet dispatchKeySet, const at::Tensor & primal, const at::Tensor & tangent, int64_t level, at::Tensor & out);
39
+ };
40
+
41
+ }} // namespace at::_ops
42
+
43
+ #else
44
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
45
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_native.h ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from NativeFunction.h
5
+
6
+ #include <c10/core/Scalar.h>
7
+ #include <c10/core/Storage.h>
8
+ #include <c10/core/TensorOptions.h>
9
+ #include <c10/util/Deprecated.h>
10
+ #include <optional>
11
+ #include <c10/core/QScheme.h>
12
+ #include <ATen/core/Reduction.h>
13
+ #include <ATen/core/Tensor.h>
14
+ #include <tuple>
15
+ #include <vector>
16
+
17
+
18
+ namespace at {
19
+ namespace native {
20
+ TORCH_API at::Tensor _make_dual(const at::Tensor & primal, const at::Tensor & tangent, int64_t level);
21
+ } // namespace native
22
+ } // namespace at
23
+
24
+ #else
25
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
26
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_dual_ops.h ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from Operator.h
5
+
6
+ #include <string_view>
7
+ #include <tuple>
8
+ #include <vector>
9
+
10
+ // Forward declarations of any types needed in the operator signatures.
11
+ // We can't directly include these classes because it will cause circular include dependencies.
12
+ // This file is included by TensorBody.h, which defines the Tensor class.
13
+ #include <ATen/core/ATen_fwd.h>
14
+
15
+ namespace at {
16
+ namespace _ops {
17
+
18
+
19
+ struct TORCH_API _make_dual {
20
+ using schema = at::Tensor (const at::Tensor &, const at::Tensor &, int64_t);
21
+ using ptr_schema = schema*;
22
+ // See Note [static constexpr char* members for windows NVCC]
23
+ static constexpr const char* name = "aten::_make_dual";
24
+ static constexpr const char* overload_name = "";
25
+ static constexpr const char* schema_str = "_make_dual(Tensor(a) primal, Tensor tangent, int level) -> Tensor(a)";
26
+ static at::Tensor call(const at::Tensor & primal, const at::Tensor & tangent, int64_t level);
27
+ static at::Tensor redispatch(c10::DispatchKeySet dispatchKeySet, const at::Tensor & primal, const at::Tensor & tangent, int64_t level);
28
+ };
29
+
30
+ }} // namespace at::_ops
31
+
32
+ #else
33
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
34
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
TTS/Kokoro/.venv/Lib/site-packages/torch/include/ATen/ops/_make_per_channel_quantized_tensor.h ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)
2
+ #pragma once
3
+
4
+ // @generated by torchgen/gen.py from Function.h
5
+
6
+ #include <ATen/Context.h>
7
+ #include <ATen/DeviceGuard.h>
8
+ #include <ATen/TensorUtils.h>
9
+ #include <ATen/TracerMode.h>
10
+ #include <ATen/core/Generator.h>
11
+ #include <ATen/core/Reduction.h>
12
+ #include <ATen/core/Tensor.h>
13
+ #include <c10/core/Scalar.h>
14
+ #include <c10/core/Storage.h>
15
+ #include <c10/core/TensorOptions.h>
16
+ #include <c10/util/Deprecated.h>
17
+ #include <optional>
18
+ #include <string_view>
19
+
20
+
21
+
22
+ #include <ATen/ops/_make_per_channel_quantized_tensor_ops.h>
23
+
24
+ namespace at {
25
+
26
+
27
+ // aten::_make_per_channel_quantized_tensor(Tensor self, Tensor scale, Tensor zero_point, int axis) -> Tensor
28
+ inline at::Tensor _make_per_channel_quantized_tensor(const at::Tensor & self, const at::Tensor & scale, const at::Tensor & zero_point, int64_t axis) {
29
+ return at::_ops::_make_per_channel_quantized_tensor::call(self, scale, zero_point, axis);
30
+ }
31
+
32
+ // aten::_make_per_channel_quantized_tensor.out(Tensor self, Tensor scale, Tensor zero_point, int axis, *, Tensor(a!) out) -> Tensor(a!)
33
+ inline at::Tensor & _make_per_channel_quantized_tensor_out(at::Tensor & out, const at::Tensor & self, const at::Tensor & scale, const at::Tensor & zero_point, int64_t axis) {
34
+ return at::_ops::_make_per_channel_quantized_tensor_out::call(self, scale, zero_point, axis, out);
35
+ }
36
+ // aten::_make_per_channel_quantized_tensor.out(Tensor self, Tensor scale, Tensor zero_point, int axis, *, Tensor(a!) out) -> Tensor(a!)
37
+ inline at::Tensor & _make_per_channel_quantized_tensor_outf(const at::Tensor & self, const at::Tensor & scale, const at::Tensor & zero_point, int64_t axis, at::Tensor & out) {
38
+ return at::_ops::_make_per_channel_quantized_tensor_out::call(self, scale, zero_point, axis, out);
39
+ }
40
+
41
+ }
42
+
43
+ #else
44
+ #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined."
45
+ #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)