Upload lib/python3.12/site-packages/torch/_utils.py with huggingface_hub
Browse files
lib/python3.12/site-packages/torch/_utils.py
ADDED
|
@@ -0,0 +1,1019 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# mypy: allow-untyped-defs
|
| 2 |
+
import copyreg
|
| 3 |
+
import functools
|
| 4 |
+
import logging
|
| 5 |
+
import sys
|
| 6 |
+
import traceback
|
| 7 |
+
import warnings
|
| 8 |
+
from collections import defaultdict
|
| 9 |
+
from typing import Any, Callable, DefaultDict, Generic, List, Optional
|
| 10 |
+
from typing_extensions import ParamSpec
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _type(self, dtype=None, non_blocking=False, **kwargs):
|
| 16 |
+
"""Returns the type if `dtype` is not provided, else casts this object to
|
| 17 |
+
the specified type.
|
| 18 |
+
|
| 19 |
+
If this is already of the correct type, no copy is performed and the
|
| 20 |
+
original object is returned.
|
| 21 |
+
|
| 22 |
+
Args:
|
| 23 |
+
dtype (type or string): The desired type
|
| 24 |
+
non_blocking (bool): If ``True``, and the source is in pinned memory
|
| 25 |
+
and destination is on the GPU or vice versa, the copy is performed
|
| 26 |
+
asynchronously with respect to the host. Otherwise, the argument
|
| 27 |
+
has no effect.
|
| 28 |
+
**kwargs: For compatibility, may contain the key ``async`` in place of
|
| 29 |
+
the ``non_blocking`` argument. The ``async`` arg is deprecated.
|
| 30 |
+
"""
|
| 31 |
+
non_blocking = _get_async_or_non_blocking("type", non_blocking, kwargs)
|
| 32 |
+
if dtype is None:
|
| 33 |
+
return self.__module__ + "." + self.__class__.__name__
|
| 34 |
+
|
| 35 |
+
if isinstance(dtype, str):
|
| 36 |
+
dtype = _import_dotted_name(dtype)
|
| 37 |
+
if dtype == type(self):
|
| 38 |
+
return self
|
| 39 |
+
if self.is_sparse:
|
| 40 |
+
if not dtype.is_sparse:
|
| 41 |
+
raise RuntimeError("Cannot cast sparse tensor to dense tensor")
|
| 42 |
+
new_module_name = dtype.__module__.replace(".sparse", "")
|
| 43 |
+
new_values_type_name = new_module_name + "." + dtype.__name__
|
| 44 |
+
new_values = torch.Tensor._values(self).type(new_values_type_name, non_blocking)
|
| 45 |
+
new_indices_type_name = new_module_name + ".LongTensor"
|
| 46 |
+
new_indices = torch.Tensor._indices(self).type(
|
| 47 |
+
new_indices_type_name, non_blocking
|
| 48 |
+
)
|
| 49 |
+
return dtype(new_indices, new_values, self.size())
|
| 50 |
+
if dtype.is_sparse:
|
| 51 |
+
raise RuntimeError("Cannot cast dense tensor to sparse tensor")
|
| 52 |
+
return dtype(self.size()).copy_(self, non_blocking)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _to(self, device, non_blocking=False):
|
| 56 |
+
"""Returns a copy of this object in device memory.
|
| 57 |
+
|
| 58 |
+
If this object is already on the correct device, then no copy is performed
|
| 59 |
+
and the original object is returned.
|
| 60 |
+
|
| 61 |
+
Args:
|
| 62 |
+
device (int): The destination device.
|
| 63 |
+
non_blocking (bool): If ``True`` and the source is in pinned memory,
|
| 64 |
+
the copy will be asynchronous with respect to the host. Otherwise,
|
| 65 |
+
the argument has no effect.
|
| 66 |
+
"""
|
| 67 |
+
if self.device == device:
|
| 68 |
+
return self
|
| 69 |
+
|
| 70 |
+
device_module = getattr(torch, device.type, None)
|
| 71 |
+
assert (
|
| 72 |
+
device_module is not None
|
| 73 |
+
), f"{device.type.upper()} device module is not loaded"
|
| 74 |
+
with device_module.device(device):
|
| 75 |
+
if self.is_sparse and hasattr(device_module, "sparse"):
|
| 76 |
+
new_type = getattr(device_module.sparse, self.__class__.__name__)
|
| 77 |
+
indices = getattr(torch.Tensor._indices(self), device.type)(
|
| 78 |
+
device, non_blocking
|
| 79 |
+
)
|
| 80 |
+
values = getattr(torch.Tensor._values(self), device.type)(
|
| 81 |
+
device, non_blocking
|
| 82 |
+
)
|
| 83 |
+
return new_type(indices, values, self.size())
|
| 84 |
+
else:
|
| 85 |
+
assert (
|
| 86 |
+
not self.is_sparse
|
| 87 |
+
), f"sparse storage is not supported for {device.type.upper()} tensors"
|
| 88 |
+
untyped_storage = torch.UntypedStorage(self.size(), device=device)
|
| 89 |
+
untyped_storage.copy_(self, non_blocking)
|
| 90 |
+
return untyped_storage
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _get_async_or_non_blocking(function_name, non_blocking, kwargs):
|
| 94 |
+
"""Return the non-blocking flag given the function name and kwargs.
|
| 95 |
+
|
| 96 |
+
Args:
|
| 97 |
+
function_name (str): the name of the function being used.
|
| 98 |
+
non_blocking (bool): the default value.
|
| 99 |
+
**kwargs (dict): the kwargs passed to the function.
|
| 100 |
+
"""
|
| 101 |
+
if not kwargs:
|
| 102 |
+
return non_blocking
|
| 103 |
+
if len(kwargs) != 1 or "async" not in kwargs:
|
| 104 |
+
message = "{}() got an unexpected keyword argument '{}'"
|
| 105 |
+
argument = list(kwargs.keys()).pop()
|
| 106 |
+
raise TypeError(message.format(function_name, argument))
|
| 107 |
+
warnings.warn("'async' is deprecated; use 'non_blocking'")
|
| 108 |
+
return kwargs["async"]
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _get_restore_location(device):
|
| 112 |
+
"""Return the map_location location.
|
| 113 |
+
|
| 114 |
+
Used for rebuild functions where the tensor device is distinct from the storage
|
| 115 |
+
"""
|
| 116 |
+
|
| 117 |
+
map_location = torch.serialization._serialization_tls.map_location
|
| 118 |
+
if map_location is None:
|
| 119 |
+
return device
|
| 120 |
+
else:
|
| 121 |
+
if isinstance(map_location, dict):
|
| 122 |
+
return map_location.get(device, device)
|
| 123 |
+
elif isinstance(map_location, (str, torch.device)):
|
| 124 |
+
return map_location
|
| 125 |
+
else:
|
| 126 |
+
assert callable(map_location)
|
| 127 |
+
raise RuntimeError(
|
| 128 |
+
"Callable map_location not supported with _rebuild_wrapper_subclass "
|
| 129 |
+
"or _rebuild_device_tensor_from_numpy"
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
# Note [Don't serialize hooks]
|
| 134 |
+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
| 135 |
+
# Since time immemorial, we have serialized the backward hooks associated with
|
| 136 |
+
# variables. This kind of half-worked--Python can pickle global functions
|
| 137 |
+
# (but not closures!)--but there were problems.
|
| 138 |
+
#
|
| 139 |
+
# - It's fragile. If you serialize a backward hook into a saved
|
| 140 |
+
# model, and then you rename the function associated with the hook,
|
| 141 |
+
# now your saved model is broken and you can't load it anymore.
|
| 142 |
+
#
|
| 143 |
+
# - It's not actually used. The standard recommendation is to
|
| 144 |
+
# serialize the *state_dict* of a model, not the model itself
|
| 145 |
+
# (since this is more stable to code changes affecting the model
|
| 146 |
+
# serialization), and the state dict saves "data" only, thus
|
| 147 |
+
# stripping the backward hooks. In some cases, hooks are
|
| 148 |
+
# essential to the well-functioning of a model (e.g., DDP),
|
| 149 |
+
# but DDP already manages readding the hooks!
|
| 150 |
+
#
|
| 151 |
+
# - We didn't serialize them in many cases. Prior to #10220, we
|
| 152 |
+
# were dropping backward hooks in ForkingPickler. We "fixed" this
|
| 153 |
+
# to be convenient with other serialization sites, but lack of
|
| 154 |
+
# serializing backward hooks wasn't actually the root cause of
|
| 155 |
+
# the bug.
|
| 156 |
+
#
|
| 157 |
+
# With these cases in mind, we have decided that a better strategy
|
| 158 |
+
# is to just NOT serialize hooks at all.
|
| 159 |
+
#
|
| 160 |
+
# Since this is a BC-breaking change, we should warn when we previously
|
| 161 |
+
# serialized a hook, but no longer do so. This will be done by adding a special
|
| 162 |
+
# sentinel property to hooks will be used to suppress this warning. If a hook
|
| 163 |
+
# has the property _torch_serialize_ignore, we will not emit a warning if we
|
| 164 |
+
# attempt to serialize a Tensor with this hook attached to it.
|
| 165 |
+
#
|
| 166 |
+
# By the way, when _backward_hooks is skipped, we must give an EMPTY
|
| 167 |
+
# OrderedDict(), if you pass a None you'll run afoul #12219.
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
# TODO: Once we decide to break serialization FC, `storage` no longer needs to
|
| 171 |
+
# be a TypedStorage
|
| 172 |
+
def _rebuild_tensor(storage, storage_offset, size, stride):
|
| 173 |
+
# first construct a tensor with the correct dtype/device
|
| 174 |
+
t = torch.empty((0,), dtype=storage.dtype, device=storage._untyped_storage.device)
|
| 175 |
+
return t.set_(storage._untyped_storage, storage_offset, size, stride)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def get_tensor_metadata(tensor):
|
| 179 |
+
# Tensor's Metadata for serializing.
|
| 180 |
+
# Currently, this only returns a dict[string, bool] specifing whether
|
| 181 |
+
# `conj` or `neg` bit is set.
|
| 182 |
+
assert isinstance(tensor, torch.Tensor)
|
| 183 |
+
return torch._C._get_tensor_metadata(tensor) # type: ignore[attr-defined]
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def set_tensor_metadata(tensor, metadata):
|
| 187 |
+
# See `get_tensor_metadata` above
|
| 188 |
+
assert isinstance(metadata, dict)
|
| 189 |
+
assert isinstance(tensor, torch.Tensor)
|
| 190 |
+
torch._C._set_tensor_metadata(tensor, metadata) # type: ignore[attr-defined]
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _rebuild_tensor_v2(
|
| 194 |
+
storage,
|
| 195 |
+
storage_offset,
|
| 196 |
+
size,
|
| 197 |
+
stride,
|
| 198 |
+
requires_grad,
|
| 199 |
+
backward_hooks,
|
| 200 |
+
metadata=None,
|
| 201 |
+
):
|
| 202 |
+
tensor = _rebuild_tensor(storage, storage_offset, size, stride)
|
| 203 |
+
tensor.requires_grad = requires_grad
|
| 204 |
+
if metadata:
|
| 205 |
+
set_tensor_metadata(tensor, metadata)
|
| 206 |
+
|
| 207 |
+
# NB: This line exists only for backwards compatibility; the
|
| 208 |
+
# general expectation is that backward_hooks is an empty
|
| 209 |
+
# OrderedDict. See Note [Don't serialize hooks]
|
| 210 |
+
tensor._backward_hooks = backward_hooks
|
| 211 |
+
return tensor
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def _rebuild_tensor_v3(
|
| 215 |
+
storage,
|
| 216 |
+
storage_offset,
|
| 217 |
+
size,
|
| 218 |
+
stride,
|
| 219 |
+
requires_grad,
|
| 220 |
+
backward_hooks,
|
| 221 |
+
dtype,
|
| 222 |
+
metadata=None,
|
| 223 |
+
):
|
| 224 |
+
t = torch.empty(
|
| 225 |
+
(0,),
|
| 226 |
+
dtype=dtype,
|
| 227 |
+
device=storage._untyped_storage.device,
|
| 228 |
+
requires_grad=requires_grad,
|
| 229 |
+
)
|
| 230 |
+
t.set_(storage._untyped_storage, storage_offset, size, stride)
|
| 231 |
+
if metadata:
|
| 232 |
+
set_tensor_metadata(t, metadata)
|
| 233 |
+
t._backward_hooks = backward_hooks
|
| 234 |
+
return t
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
_sparse_tensors_to_validate: List["torch.Tensor"] = []
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
# In _legacy_load() in serialization.py we unpickle storages after the sparse
|
| 241 |
+
# tensors have been already unpickled. Those storages contain data necessary for
|
| 242 |
+
# validating sparse tensors: indices and values. That's why sparse tensors are
|
| 243 |
+
# first unpickled without any validation, and then this function is called just
|
| 244 |
+
# before _legacy_load() returns, so that all the sparse tensors can be validated
|
| 245 |
+
# in bulk.
|
| 246 |
+
#
|
| 247 |
+
# The same procedure must be followed by _load() in serialization.py because due
|
| 248 |
+
# to Pickler semantics, we have to use the same (non-validating) function for
|
| 249 |
+
# unpickling sparse tensors, regardless of the caller.
|
| 250 |
+
def _validate_loaded_sparse_tensors():
|
| 251 |
+
try:
|
| 252 |
+
for t in _sparse_tensors_to_validate:
|
| 253 |
+
if t.layout is torch.sparse_coo:
|
| 254 |
+
torch._validate_sparse_coo_tensor_args(
|
| 255 |
+
t._indices(), t._values(), t.size(), t.is_coalesced()
|
| 256 |
+
)
|
| 257 |
+
elif t.layout in {
|
| 258 |
+
torch.sparse_csr,
|
| 259 |
+
torch.sparse_csc,
|
| 260 |
+
torch.sparse_bsr,
|
| 261 |
+
torch.sparse_bsc,
|
| 262 |
+
}:
|
| 263 |
+
# TODO: Validation currently involves an expensive traversal
|
| 264 |
+
# on CPU, which may include a device transfer.
|
| 265 |
+
if t.layout in {torch.sparse_csr, torch.sparse_bsr}:
|
| 266 |
+
compressed_indices, plain_indices = (
|
| 267 |
+
t.crow_indices(),
|
| 268 |
+
t.col_indices(),
|
| 269 |
+
)
|
| 270 |
+
else:
|
| 271 |
+
compressed_indices, plain_indices = (
|
| 272 |
+
t.ccol_indices(),
|
| 273 |
+
t.row_indices(),
|
| 274 |
+
)
|
| 275 |
+
torch._validate_sparse_compressed_tensor_args(
|
| 276 |
+
compressed_indices, plain_indices, t.values(), t.size(), t.layout
|
| 277 |
+
)
|
| 278 |
+
else:
|
| 279 |
+
raise NotImplementedError(
|
| 280 |
+
f"_validate_loaded_sparse_tensors for layout `{t.layout}`"
|
| 281 |
+
)
|
| 282 |
+
|
| 283 |
+
finally:
|
| 284 |
+
_sparse_tensors_to_validate.clear()
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def _rebuild_sparse_tensor(layout, data):
|
| 288 |
+
"""
|
| 289 |
+
Rebuilds a sparse tensor from its sparse storage representation.
|
| 290 |
+
|
| 291 |
+
Args:
|
| 292 |
+
layout (str): The sparse storage layout of the tensor.
|
| 293 |
+
data (tuple): The tensor's sparse storage representation.
|
| 294 |
+
"""
|
| 295 |
+
if layout == torch.sparse_coo:
|
| 296 |
+
if len(data) == 3:
|
| 297 |
+
# For BC:
|
| 298 |
+
indices, values, size = data
|
| 299 |
+
is_coalesced = None
|
| 300 |
+
else:
|
| 301 |
+
indices, values, size, is_coalesced = data
|
| 302 |
+
result = torch.sparse_coo_tensor(
|
| 303 |
+
indices, values, size, check_invariants=False, is_coalesced=is_coalesced
|
| 304 |
+
)
|
| 305 |
+
_sparse_tensors_to_validate.append(result)
|
| 306 |
+
return result
|
| 307 |
+
|
| 308 |
+
elif layout in {
|
| 309 |
+
torch.sparse_csr,
|
| 310 |
+
torch.sparse_csc,
|
| 311 |
+
torch.sparse_bsr,
|
| 312 |
+
torch.sparse_bsc,
|
| 313 |
+
}:
|
| 314 |
+
compressed_indices, plain_indices, values, size = data
|
| 315 |
+
result = torch.sparse_compressed_tensor(
|
| 316 |
+
compressed_indices,
|
| 317 |
+
plain_indices,
|
| 318 |
+
values,
|
| 319 |
+
size,
|
| 320 |
+
layout=layout,
|
| 321 |
+
check_invariants=False,
|
| 322 |
+
)
|
| 323 |
+
_sparse_tensors_to_validate.append(result)
|
| 324 |
+
return result
|
| 325 |
+
|
| 326 |
+
raise NotImplementedError(f"rebuilding sparse tensor for layout {layout}")
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def _rebuild_nested_tensor(buffer, sizes, strides, storage_offsets):
|
| 330 |
+
return torch._nested_view_from_buffer(buffer, sizes, strides, storage_offsets)
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def _rebuild_device_tensor_from_numpy(data, dtype, device, requires_grad):
|
| 334 |
+
device = _get_restore_location(device)
|
| 335 |
+
tensor = torch.from_numpy(data).to(dtype=dtype, device=device)
|
| 336 |
+
tensor.requires_grad = requires_grad
|
| 337 |
+
return tensor
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
# Should not be used, only here to be able to load Tensors serialized with older versions of pytorch
|
| 341 |
+
_rebuild_xla_tensor = _rebuild_device_tensor_from_numpy
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
def _rebuild_meta_tensor_no_storage(dtype, size, stride, requires_grad):
|
| 345 |
+
return torch.empty_strided(
|
| 346 |
+
size, stride, dtype=dtype, device="meta", requires_grad=requires_grad
|
| 347 |
+
)
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
def _rebuild_wrapper_subclass(
|
| 351 |
+
cls,
|
| 352 |
+
dtype,
|
| 353 |
+
size,
|
| 354 |
+
stride,
|
| 355 |
+
storage_offset,
|
| 356 |
+
layout,
|
| 357 |
+
device,
|
| 358 |
+
requires_grad,
|
| 359 |
+
):
|
| 360 |
+
device = _get_restore_location(device)
|
| 361 |
+
return torch.Tensor._make_wrapper_subclass( # type: ignore[attr-defined]
|
| 362 |
+
cls,
|
| 363 |
+
size,
|
| 364 |
+
strides=stride,
|
| 365 |
+
dtype=dtype,
|
| 366 |
+
storage_offset=storage_offset,
|
| 367 |
+
layout=layout,
|
| 368 |
+
device=device,
|
| 369 |
+
requires_grad=requires_grad,
|
| 370 |
+
)
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
# TODO: Once we decide to break serialization FC, `storage` no longer needs to
|
| 374 |
+
# be a TypedStorage
|
| 375 |
+
def _rebuild_qtensor(
|
| 376 |
+
storage,
|
| 377 |
+
storage_offset,
|
| 378 |
+
size,
|
| 379 |
+
stride,
|
| 380 |
+
quantizer_params,
|
| 381 |
+
requires_grad,
|
| 382 |
+
backward_hooks,
|
| 383 |
+
):
|
| 384 |
+
qscheme = quantizer_params[0]
|
| 385 |
+
if qscheme == torch.per_tensor_affine:
|
| 386 |
+
_, scale, zero_point = quantizer_params
|
| 387 |
+
tensor = torch._empty_affine_quantized(
|
| 388 |
+
size,
|
| 389 |
+
scale=scale,
|
| 390 |
+
zero_point=zero_point,
|
| 391 |
+
dtype=storage.dtype,
|
| 392 |
+
device=storage.device,
|
| 393 |
+
)
|
| 394 |
+
elif qscheme in (torch.per_channel_affine, torch.per_channel_affine_float_qparams):
|
| 395 |
+
_, scales, zero_points, axis = quantizer_params
|
| 396 |
+
if type(scales) is list and type(zero_points) is list:
|
| 397 |
+
if qscheme == torch.per_channel_affine:
|
| 398 |
+
scales = torch.tensor(scales, dtype=torch.double, device=storage.device)
|
| 399 |
+
zero_points = torch.tensor(
|
| 400 |
+
zero_points, dtype=torch.long, device=storage.device
|
| 401 |
+
)
|
| 402 |
+
else:
|
| 403 |
+
scales = torch.tensor(scales, dtype=torch.float, device=storage.device)
|
| 404 |
+
zero_points = torch.tensor(
|
| 405 |
+
zero_points, dtype=torch.float, device=storage.device
|
| 406 |
+
)
|
| 407 |
+
tensor = torch._empty_per_channel_affine_quantized(
|
| 408 |
+
size,
|
| 409 |
+
scales=scales,
|
| 410 |
+
zero_points=zero_points,
|
| 411 |
+
axis=axis,
|
| 412 |
+
dtype=storage.dtype,
|
| 413 |
+
device=storage.device,
|
| 414 |
+
)
|
| 415 |
+
else:
|
| 416 |
+
raise RuntimeError(f"Can't deserialize quantized tensor with qscheme {qscheme}")
|
| 417 |
+
tensor.set_(storage, storage_offset, size, stride)
|
| 418 |
+
tensor.requires_grad = requires_grad
|
| 419 |
+
# NB: This line exists only for backwards compatibility; the
|
| 420 |
+
# general expectation is that backward_hooks is an empty
|
| 421 |
+
# OrderedDict. See Note [Don't serialize hooks]
|
| 422 |
+
tensor._backward_hooks = backward_hooks
|
| 423 |
+
return tensor
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
def _rebuild_parameter(data, requires_grad, backward_hooks):
|
| 427 |
+
param = torch.nn.Parameter(data, requires_grad)
|
| 428 |
+
# NB: This line exists only for backwards compatibility; the
|
| 429 |
+
# general expectation is that backward_hooks is an empty
|
| 430 |
+
# OrderedDict. See Note [Don't serialize hooks]
|
| 431 |
+
param._backward_hooks = backward_hooks
|
| 432 |
+
|
| 433 |
+
return param
|
| 434 |
+
|
| 435 |
+
|
| 436 |
+
def _rebuild_parameter_with_state(data, requires_grad, backward_hooks, state):
|
| 437 |
+
param = torch.nn.Parameter(data, requires_grad)
|
| 438 |
+
# NB: This line exists only for backwards compatibility; the
|
| 439 |
+
# general expectation is that backward_hooks is an empty
|
| 440 |
+
# OrderedDict. See Note [Don't serialize hooks]
|
| 441 |
+
param._backward_hooks = backward_hooks
|
| 442 |
+
|
| 443 |
+
# Restore state on Parameter like python attr.
|
| 444 |
+
param = _set_obj_state(param, state)
|
| 445 |
+
return param
|
| 446 |
+
|
| 447 |
+
|
| 448 |
+
def _get_obj_state(obj):
|
| 449 |
+
# Get the state of the python subclass
|
| 450 |
+
# This loosely mimicks the function on the object class but since Tensor do not inherit
|
| 451 |
+
# from it, we cannot call that function directly
|
| 452 |
+
# https://github.com/python/cpython/blob/c83919bd635f4433f1c6ae8504996a9fe3c215e5/Objects/typeobject.c#L4891
|
| 453 |
+
# Note that starting with Python 3.11, this `__getstate__` is always defined and thus
|
| 454 |
+
# the else branch will never be taken.
|
| 455 |
+
getstate_fn = getattr(obj, "__getstate__", None)
|
| 456 |
+
if getstate_fn:
|
| 457 |
+
state = getstate_fn()
|
| 458 |
+
else:
|
| 459 |
+
slots_to_save = copyreg._slotnames(obj.__class__) # type: ignore[attr-defined]
|
| 460 |
+
if slots_to_save:
|
| 461 |
+
state = (
|
| 462 |
+
obj.__dict__,
|
| 463 |
+
{
|
| 464 |
+
name: getattr(obj, name)
|
| 465 |
+
for name in slots_to_save
|
| 466 |
+
if hasattr(obj, name)
|
| 467 |
+
},
|
| 468 |
+
)
|
| 469 |
+
else:
|
| 470 |
+
state = obj.__dict__
|
| 471 |
+
|
| 472 |
+
return state
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def _set_obj_state(obj, state):
|
| 476 |
+
if isinstance(state, tuple):
|
| 477 |
+
if not len(state) == 2:
|
| 478 |
+
raise RuntimeError(f"Invalid serialized state: {state}")
|
| 479 |
+
dict_state = state[0]
|
| 480 |
+
slots_state = state[1]
|
| 481 |
+
else:
|
| 482 |
+
dict_state = state
|
| 483 |
+
slots_state = None
|
| 484 |
+
|
| 485 |
+
# Starting with Python 3.11, the __dict__ attribute is lazily created
|
| 486 |
+
# and is serialized as None when not needed.
|
| 487 |
+
if dict_state:
|
| 488 |
+
for k, v in dict_state.items():
|
| 489 |
+
setattr(obj, k, v)
|
| 490 |
+
|
| 491 |
+
if slots_state:
|
| 492 |
+
for k, v in slots_state.items():
|
| 493 |
+
setattr(obj, k, v)
|
| 494 |
+
return obj
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def _import_dotted_name(name):
|
| 498 |
+
components = name.split(".")
|
| 499 |
+
obj = __import__(components[0])
|
| 500 |
+
for component in components[1:]:
|
| 501 |
+
obj = getattr(obj, component)
|
| 502 |
+
return obj
|
| 503 |
+
|
| 504 |
+
|
| 505 |
+
def _flatten_dense_tensors(tensors):
|
| 506 |
+
"""Flatten dense tensors into a contiguous 1D buffer. Assume tensors are of
|
| 507 |
+
same dense type.
|
| 508 |
+
|
| 509 |
+
Since inputs are dense, the resulting tensor will be a concatenated 1D
|
| 510 |
+
buffer. Element-wise operation on this buffer will be equivalent to
|
| 511 |
+
operating individually.
|
| 512 |
+
|
| 513 |
+
Args:
|
| 514 |
+
tensors (Iterable[Tensor]): dense tensors to flatten.
|
| 515 |
+
|
| 516 |
+
Returns:
|
| 517 |
+
A contiguous 1D buffer containing input tensors.
|
| 518 |
+
"""
|
| 519 |
+
return torch._C._nn.flatten_dense_tensors(tensors)
|
| 520 |
+
|
| 521 |
+
|
| 522 |
+
def _flatten_sparse_tensors(tensors):
|
| 523 |
+
"""Flatten sparse tensors into two contiguous 1D buffers, one of indices and
|
| 524 |
+
one of values. Assume tensors are of same sparse type.
|
| 525 |
+
|
| 526 |
+
Args:
|
| 527 |
+
tensors (Iterable[Tensor]): sparse tensors to flatten.
|
| 528 |
+
|
| 529 |
+
Returns:
|
| 530 |
+
A tuple of two contiguous 1D buffers, one containing input tensors'
|
| 531 |
+
indices and the other containing the values.
|
| 532 |
+
"""
|
| 533 |
+
flat_indices = torch._C._nn.flatten_dense_tensors(
|
| 534 |
+
[torch.Tensor._indices(t) for t in tensors]
|
| 535 |
+
)
|
| 536 |
+
flat_values = torch._C._nn.flatten_dense_tensors(
|
| 537 |
+
[torch.Tensor._values(t) for t in tensors]
|
| 538 |
+
)
|
| 539 |
+
return flat_indices, flat_values
|
| 540 |
+
|
| 541 |
+
|
| 542 |
+
def _unflatten_dense_tensors(flat, tensors):
|
| 543 |
+
"""View a flat buffer using the sizes of tensors. Assume that tensors are of
|
| 544 |
+
same dense type, and that flat is given by _flatten_dense_tensors.
|
| 545 |
+
|
| 546 |
+
Args:
|
| 547 |
+
flat (Tensor): flattened dense tensors to unflatten.
|
| 548 |
+
tensors (Iterable[Tensor]): dense tensors whose sizes will be used to
|
| 549 |
+
unflatten flat.
|
| 550 |
+
|
| 551 |
+
Returns:
|
| 552 |
+
Unflattened dense tensors with sizes same as tensors and values from
|
| 553 |
+
flat.
|
| 554 |
+
"""
|
| 555 |
+
return torch._C._nn.unflatten_dense_tensors(flat, tensors)
|
| 556 |
+
|
| 557 |
+
|
| 558 |
+
def _unflatten_sparse_tensors(flat, tensors):
|
| 559 |
+
"""View flat buffer (containing indices and values) using the sizes of
|
| 560 |
+
tensors. Assume that tensors are of same sparse type, and that flat is given
|
| 561 |
+
by _flatten_sparse_tensors.
|
| 562 |
+
|
| 563 |
+
Args:
|
| 564 |
+
flat (tuple(Tensor, Tensor)): flattened indices and values of sparse
|
| 565 |
+
tensors to unflatten.
|
| 566 |
+
tensors (Iterable[Tensor]): sparse tensors whose sizes will be used to
|
| 567 |
+
unflatten flat.
|
| 568 |
+
|
| 569 |
+
Returns:
|
| 570 |
+
Unflattened sparse tensors with sizes same as tensors and values from
|
| 571 |
+
flat.
|
| 572 |
+
"""
|
| 573 |
+
flat_indices, flat_values = flat
|
| 574 |
+
indices = torch._C._nn.unflatten_dense_tensors(
|
| 575 |
+
flat_indices, [torch.Tensor._indices(t) for t in tensors]
|
| 576 |
+
)
|
| 577 |
+
values = torch._C._nn.unflatten_dense_tensors(
|
| 578 |
+
flat_values, [torch.Tensor._values(t) for t in tensors]
|
| 579 |
+
)
|
| 580 |
+
outputs = []
|
| 581 |
+
for t, i, v in zip(tensors, indices, values):
|
| 582 |
+
outputs.append(t.new(i, v, t.size()))
|
| 583 |
+
return tuple(outputs)
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def _reorder_tensors_as(tensors, ordered_tensors):
|
| 587 |
+
"""Assume that tensors are of same order as ordered_tensors within their
|
| 588 |
+
types, e.g., from _take_tensors. Reorder them to be of same order as
|
| 589 |
+
ordered_tensors.
|
| 590 |
+
|
| 591 |
+
Args:
|
| 592 |
+
tensors (Iterable[Tensor]): tensors to be reordered. They should be of
|
| 593 |
+
the same order as ordered_tensors within their own types.
|
| 594 |
+
ordered_tensors (Iterable[Tensor]): tensors whose order will be the
|
| 595 |
+
reference.
|
| 596 |
+
|
| 597 |
+
Returns:
|
| 598 |
+
Ordered tuple of tensors with contents from tensors and order of
|
| 599 |
+
ordered_tensors.
|
| 600 |
+
"""
|
| 601 |
+
type_dict = defaultdict(list)
|
| 602 |
+
for tensor in tensors:
|
| 603 |
+
type_dict[tensor.type()].append(tensor)
|
| 604 |
+
type_dict_ = {t: iter(coll) for t, coll in type_dict.items()}
|
| 605 |
+
return tuple(next(type_dict_[tensor.type()]) for tensor in ordered_tensors)
|
| 606 |
+
|
| 607 |
+
|
| 608 |
+
def _take_tensors(tensors, size_limit):
|
| 609 |
+
"""Group tensors into chunks. This generator yields a chunk at each time,
|
| 610 |
+
each containing tensors of same type up to certain byte limit in total size.
|
| 611 |
+
|
| 612 |
+
Args:
|
| 613 |
+
tensors (Sequence): A sequence of tensors to be separated into chunks.
|
| 614 |
+
size_limit (int): The limit of each chunk in bytes.
|
| 615 |
+
|
| 616 |
+
Yields:
|
| 617 |
+
Blocks of tensors of same type and within size_limit. The yielded
|
| 618 |
+
tensors are only ordered as the original sequence within its types.
|
| 619 |
+
"""
|
| 620 |
+
buf_dict: DefaultDict[str, List] = defaultdict(lambda: [[], 0])
|
| 621 |
+
for tensor in tensors:
|
| 622 |
+
t = tensor.type()
|
| 623 |
+
if tensor.is_sparse:
|
| 624 |
+
indices = torch.Tensor._indices(tensor)
|
| 625 |
+
values = torch.Tensor._values(tensor)
|
| 626 |
+
size = (
|
| 627 |
+
indices.numel() * indices.element_size()
|
| 628 |
+
+ values.numel() * values.element_size()
|
| 629 |
+
)
|
| 630 |
+
else:
|
| 631 |
+
size = tensor.numel() * tensor.element_size()
|
| 632 |
+
buf_and_size = buf_dict[t]
|
| 633 |
+
if buf_and_size[1] + size > size_limit and buf_and_size[1] > 0:
|
| 634 |
+
yield buf_and_size[0]
|
| 635 |
+
buf_and_size = buf_dict[t] = [[], 0]
|
| 636 |
+
buf_and_size[0].append(tensor)
|
| 637 |
+
buf_and_size[1] += size
|
| 638 |
+
for buf, _ in buf_dict.values():
|
| 639 |
+
if len(buf) > 0:
|
| 640 |
+
yield buf
|
| 641 |
+
|
| 642 |
+
|
| 643 |
+
# annotation decorator to get annotations in a way that is compatible
|
| 644 |
+
# with both Python 2 and 3
|
| 645 |
+
def annotate(ret, **kwargs):
|
| 646 |
+
def dec(fun):
|
| 647 |
+
fun.__annotations__ = dict(kwargs)
|
| 648 |
+
fun.__annotations__["return"] = ret
|
| 649 |
+
return fun
|
| 650 |
+
|
| 651 |
+
return dec
|
| 652 |
+
|
| 653 |
+
|
| 654 |
+
def render_call(fn, args, kwargs):
|
| 655 |
+
str_fn = torch.overrides.resolve_name(fn)
|
| 656 |
+
if str_fn is None:
|
| 657 |
+
str_fn = str(fn)
|
| 658 |
+
|
| 659 |
+
str_args: List[str] = []
|
| 660 |
+
with torch._tensor_str.printoptions(threshold=0, edgeitems=0):
|
| 661 |
+
str_args.extend(repr(a) for a in args)
|
| 662 |
+
str_args.extend(f"{k}={repr(v)}" for k, v in kwargs.items())
|
| 663 |
+
r = f"{str_fn}({', '.join(str_args)})"
|
| 664 |
+
return r
|
| 665 |
+
|
| 666 |
+
|
| 667 |
+
# NOTE [ Python Traceback Reference Cycle Problem ]
|
| 668 |
+
#
|
| 669 |
+
# When using sys.exc_info(), it is important to **not** store the exc_info[2],
|
| 670 |
+
# which is the traceback, because otherwise you will run into the traceback
|
| 671 |
+
# reference cycle problem, i.e., the traceback holding reference to the frame,
|
| 672 |
+
# and the frame (which holds reference to all the object in its temporary scope)
|
| 673 |
+
# holding reference the traceback.
|
| 674 |
+
|
| 675 |
+
|
| 676 |
+
class KeyErrorMessage(str):
|
| 677 |
+
r"""str subclass that returns itself in repr"""
|
| 678 |
+
|
| 679 |
+
def __repr__(self):
|
| 680 |
+
return self
|
| 681 |
+
|
| 682 |
+
|
| 683 |
+
class ExceptionWrapper:
|
| 684 |
+
r"""Wraps an exception plus traceback to communicate across threads"""
|
| 685 |
+
|
| 686 |
+
def __init__(self, exc_info=None, where="in background"):
|
| 687 |
+
# It is important that we don't store exc_info, see
|
| 688 |
+
# NOTE [ Python Traceback Reference Cycle Problem ]
|
| 689 |
+
if exc_info is None:
|
| 690 |
+
exc_info = sys.exc_info()
|
| 691 |
+
self.exc_type = exc_info[0]
|
| 692 |
+
self.exc_msg = "".join(traceback.format_exception(*exc_info))
|
| 693 |
+
self.where = where
|
| 694 |
+
|
| 695 |
+
def reraise(self):
|
| 696 |
+
r"""Reraises the wrapped exception in the current thread"""
|
| 697 |
+
# Format a message such as: "Caught ValueError in DataLoader worker
|
| 698 |
+
# process 2. Original Traceback:", followed by the traceback.
|
| 699 |
+
msg = f"Caught {self.exc_type.__name__} {self.where}.\nOriginal {self.exc_msg}"
|
| 700 |
+
if self.exc_type == KeyError:
|
| 701 |
+
# KeyError calls repr() on its argument (usually a dict key). This
|
| 702 |
+
# makes stack traces unreadable. It will not be changed in Python
|
| 703 |
+
# (https://bugs.python.org/issue2651), so we work around it.
|
| 704 |
+
msg = KeyErrorMessage(msg)
|
| 705 |
+
elif getattr(self.exc_type, "message", None):
|
| 706 |
+
# Some exceptions have first argument as non-str but explicitly
|
| 707 |
+
# have message field
|
| 708 |
+
raise self.exc_type(message=msg)
|
| 709 |
+
try:
|
| 710 |
+
exception = self.exc_type(msg)
|
| 711 |
+
except TypeError:
|
| 712 |
+
# If the exception takes multiple arguments, don't try to
|
| 713 |
+
# instantiate since we don't know how to
|
| 714 |
+
raise RuntimeError(msg) from None
|
| 715 |
+
raise exception
|
| 716 |
+
|
| 717 |
+
|
| 718 |
+
def _get_available_device_type():
|
| 719 |
+
if torch.cuda.is_available():
|
| 720 |
+
return "cuda"
|
| 721 |
+
if hasattr(torch, "xpu") and torch.xpu.is_available(): # type: ignore[attr-defined]
|
| 722 |
+
return "xpu"
|
| 723 |
+
if hasattr(torch, "mtia") and torch.mtia.is_available():
|
| 724 |
+
return "mtia"
|
| 725 |
+
custom_backend_name = torch._C._get_privateuse1_backend_name()
|
| 726 |
+
custom_device_mod = getattr(torch, custom_backend_name, None)
|
| 727 |
+
if custom_device_mod and custom_device_mod.is_available():
|
| 728 |
+
return custom_backend_name
|
| 729 |
+
# add more available device types here
|
| 730 |
+
return None
|
| 731 |
+
|
| 732 |
+
|
| 733 |
+
def _get_device_attr(get_member):
|
| 734 |
+
device_type = _get_available_device_type()
|
| 735 |
+
if device_type and device_type.lower() == "cuda":
|
| 736 |
+
return get_member(torch.cuda)
|
| 737 |
+
if device_type and device_type.lower() == "xpu":
|
| 738 |
+
return get_member(torch.xpu) # type: ignore[attr-defined]
|
| 739 |
+
if device_type and device_type.lower() == "mtia":
|
| 740 |
+
return get_member(torch.mtia)
|
| 741 |
+
if device_type == torch._C._get_privateuse1_backend_name():
|
| 742 |
+
return get_member(getattr(torch, device_type))
|
| 743 |
+
# add more available device types here
|
| 744 |
+
return None
|
| 745 |
+
|
| 746 |
+
|
| 747 |
+
def _get_current_device_index():
|
| 748 |
+
# current device index
|
| 749 |
+
return _get_device_attr(lambda m: m.current_device())
|
| 750 |
+
|
| 751 |
+
|
| 752 |
+
def _get_all_device_indices():
|
| 753 |
+
# all device index
|
| 754 |
+
return _get_device_attr(lambda m: list(range(m.device_count())))
|
| 755 |
+
|
| 756 |
+
|
| 757 |
+
def _get_devices_properties(device_ids):
|
| 758 |
+
# all device properties
|
| 759 |
+
return [_get_device_attr(lambda m: m.get_device_properties(i)) for i in device_ids]
|
| 760 |
+
|
| 761 |
+
|
| 762 |
+
def get_current_device_index() -> int:
|
| 763 |
+
r"""Checks if there are CUDA devices available and
|
| 764 |
+
returns the device index of the current default CUDA device.
|
| 765 |
+
Returns -1 in case there are no CUDA devices available.
|
| 766 |
+
Arguments: ``None``
|
| 767 |
+
"""
|
| 768 |
+
if torch.cuda.device_count() > 0:
|
| 769 |
+
return torch.cuda.current_device()
|
| 770 |
+
return -1
|
| 771 |
+
|
| 772 |
+
|
| 773 |
+
def _get_device_index(
|
| 774 |
+
device: Any,
|
| 775 |
+
optional: bool = False,
|
| 776 |
+
allow_cpu: bool = False,
|
| 777 |
+
) -> int:
|
| 778 |
+
r"""Gets the device index from :attr:`device`, which can be a torch.device
|
| 779 |
+
object, a Python integer, or ``None``.
|
| 780 |
+
|
| 781 |
+
If :attr:`device` is a torch.device object, returns the device index if it
|
| 782 |
+
has index. Note that for a device without a specified index,
|
| 783 |
+
i.e., ``torch.device('xxx')``, this will return the current default
|
| 784 |
+
device of that type if :attr:`optional` is ``True``. If :attr:`allow_cpu` is ``True``,
|
| 785 |
+
CPU devices will be accepted and ``-1`` will be returned in this case.
|
| 786 |
+
|
| 787 |
+
If :attr:`device` is a Python integer, it is returned as is.
|
| 788 |
+
|
| 789 |
+
If :attr:`device` is ``None``, this will return the current default
|
| 790 |
+
device of the supported runtime platform if :attr:`optional` is ``True``.
|
| 791 |
+
i.e., the current default CUDA device will be returned if CUDA runtime is supported.
|
| 792 |
+
"""
|
| 793 |
+
if isinstance(device, str):
|
| 794 |
+
device = torch.device(device)
|
| 795 |
+
device_idx: Optional[int] = None
|
| 796 |
+
if isinstance(device, torch.device):
|
| 797 |
+
if not allow_cpu and device.type == "cpu":
|
| 798 |
+
raise ValueError(f"Expected a non cpu device, but got: {device}")
|
| 799 |
+
device_idx = -1 if device.type == "cpu" else device.index
|
| 800 |
+
if isinstance(device, int):
|
| 801 |
+
device_idx = device
|
| 802 |
+
if device_idx is None:
|
| 803 |
+
if optional:
|
| 804 |
+
# The eager API _get_current_device_index uses `lambda` functions which are
|
| 805 |
+
# not supported in JIT and hence not scriptable. The JIT equivalent API to get
|
| 806 |
+
# the current device index is `get_current_device_index()` which can
|
| 807 |
+
# be scripted. We use is_scripting to check the mode we are in and call the
|
| 808 |
+
# appropriate API.
|
| 809 |
+
if torch.jit.is_scripting():
|
| 810 |
+
device_idx = get_current_device_index()
|
| 811 |
+
else:
|
| 812 |
+
device_idx = _get_current_device_index()
|
| 813 |
+
else:
|
| 814 |
+
raise ValueError(
|
| 815 |
+
f"Expected a torch.device with a specified index or an integer, but got:{device}"
|
| 816 |
+
)
|
| 817 |
+
return device_idx
|
| 818 |
+
|
| 819 |
+
|
| 820 |
+
def _handle_complex(tensor):
|
| 821 |
+
"""
|
| 822 |
+
Returns a real view of a tensor if complex dtype else just the tensor
|
| 823 |
+
need to check if a UninitializedParameter because otherwise checking is_complex is an error for a LazyModule
|
| 824 |
+
"""
|
| 825 |
+
return (
|
| 826 |
+
torch.view_as_real(tensor)
|
| 827 |
+
if not isinstance(tensor, torch.nn.UninitializedParameter)
|
| 828 |
+
and tensor.is_complex()
|
| 829 |
+
else tensor
|
| 830 |
+
)
|
| 831 |
+
|
| 832 |
+
|
| 833 |
+
def _element_size(dtype):
|
| 834 |
+
"""
|
| 835 |
+
Returns the element size for a dtype, in bytes
|
| 836 |
+
"""
|
| 837 |
+
if not isinstance(dtype, torch.dtype):
|
| 838 |
+
raise RuntimeError(f"expected torch.dtype, but got {type(dtype)}")
|
| 839 |
+
|
| 840 |
+
if dtype.is_complex:
|
| 841 |
+
return torch.finfo(dtype).bits >> 2
|
| 842 |
+
elif dtype.is_floating_point:
|
| 843 |
+
return torch.finfo(dtype).bits >> 3
|
| 844 |
+
elif dtype == torch.bool:
|
| 845 |
+
# NOTE: torch.bool is not supported in torch.iinfo()
|
| 846 |
+
return 1
|
| 847 |
+
else:
|
| 848 |
+
return torch.iinfo(dtype).bits >> 3
|
| 849 |
+
|
| 850 |
+
|
| 851 |
+
class _ClassPropertyDescriptor:
|
| 852 |
+
def __init__(self, fget, fset=None):
|
| 853 |
+
self.fget = fget
|
| 854 |
+
|
| 855 |
+
def __get__(self, instance, owner=None):
|
| 856 |
+
if owner is None:
|
| 857 |
+
owner = type(instance)
|
| 858 |
+
return self.fget.__get__(instance, owner)()
|
| 859 |
+
|
| 860 |
+
|
| 861 |
+
def classproperty(func):
|
| 862 |
+
if not isinstance(func, (classmethod, staticmethod)):
|
| 863 |
+
func = classmethod(func)
|
| 864 |
+
return _ClassPropertyDescriptor(func)
|
| 865 |
+
|
| 866 |
+
|
| 867 |
+
def is_compiling() -> bool:
|
| 868 |
+
"""
|
| 869 |
+
Indicates whether we are tracing/compiling with torch.compile() or torch.export().
|
| 870 |
+
|
| 871 |
+
TODO(khabinov): we should deprecate this function and use torch.compiler.is_compiling().
|
| 872 |
+
"""
|
| 873 |
+
return torch.compiler.is_compiling()
|
| 874 |
+
|
| 875 |
+
|
| 876 |
+
def _functionalize_sync(t):
|
| 877 |
+
# This code lives in python instead of C++ since conditioning on a certain python subclass
|
| 878 |
+
# is much more of a pain in C++.
|
| 879 |
+
from torch._subclasses.functional_tensor import FunctionalTensor
|
| 880 |
+
|
| 881 |
+
if isinstance(t, FunctionalTensor):
|
| 882 |
+
# If a FunctionalTensorMode is active while syncing, we don't want it to intercept any ops that get called
|
| 883 |
+
# when we sync our inner tensor.
|
| 884 |
+
# Why?
|
| 885 |
+
# (1) If there are input mutations in the graph, then they will be re-applied during
|
| 886 |
+
# AOTAutograd when we call _sync() from inside of our functionalization kernels.
|
| 887 |
+
# (2) _sync() causes us to regenerate our updated the tensor from the updated base,
|
| 888 |
+
# which dispatches to a bunch of view ops
|
| 889 |
+
# (3) The input to these view ops is our inner FunctionalTensorWrapper
|
| 890 |
+
# (since the sync was called from C++), not the python FunctionalTensor
|
| 891 |
+
# (4) if a python FunctionalTensorMode is active, it will complain when it intercepts
|
| 892 |
+
# the view op, since it will see an input that is a C++ FunctionalTensorWrapper
|
| 893 |
+
# (aka a normal torch.Tensor) instead of a python `FunctionalTensor).
|
| 894 |
+
maybe_functional_mode = torch._C._unset_dispatch_mode(
|
| 895 |
+
torch._C._TorchDispatchModeKey.FUNCTIONAL
|
| 896 |
+
)
|
| 897 |
+
try:
|
| 898 |
+
torch._functionalize_sync(t.elem) # type: ignore[attr-defined]
|
| 899 |
+
finally:
|
| 900 |
+
if maybe_functional_mode is not None:
|
| 901 |
+
torch._C._set_dispatch_mode(maybe_functional_mode)
|
| 902 |
+
else:
|
| 903 |
+
torch._functionalize_sync(t) # type: ignore[attr-defined]
|
| 904 |
+
|
| 905 |
+
|
| 906 |
+
@functools.lru_cache(2)
|
| 907 |
+
def _get_device_module(device_type: str):
|
| 908 |
+
device_module = getattr(torch, device_type, None)
|
| 909 |
+
if device_module is None:
|
| 910 |
+
raise RuntimeError(
|
| 911 |
+
f"Device '{device_type}' does not have a corresponding module registered as 'torch.{device_type}'."
|
| 912 |
+
)
|
| 913 |
+
return device_module
|
| 914 |
+
|
| 915 |
+
|
| 916 |
+
def _dummy_type(name: str) -> type:
|
| 917 |
+
def get_err_fn(is_init: bool):
|
| 918 |
+
def err_fn(obj, *args, **kwargs):
|
| 919 |
+
if is_init:
|
| 920 |
+
class_name = obj.__class__.__name__
|
| 921 |
+
else:
|
| 922 |
+
class_name = obj.__name__
|
| 923 |
+
raise RuntimeError(f"Tried to instantiate dummy base class {class_name}")
|
| 924 |
+
|
| 925 |
+
return err_fn
|
| 926 |
+
|
| 927 |
+
return type(
|
| 928 |
+
name, (object,), {"__init__": get_err_fn(True), "__new__": get_err_fn(False)}
|
| 929 |
+
)
|
| 930 |
+
|
| 931 |
+
|
| 932 |
+
class _LazySeedTracker:
|
| 933 |
+
# Since seeding is memory-less, only track the latest seed.
|
| 934 |
+
# Note: `manual_seed_all` followed by `manual_seed` overwrites
|
| 935 |
+
# the seed on current device. We track the order of **latest**
|
| 936 |
+
# calls between these two API.
|
| 937 |
+
def __init__(self):
|
| 938 |
+
self.manual_seed_all_cb = None
|
| 939 |
+
self.manual_seed_cb = None
|
| 940 |
+
self.call_order = []
|
| 941 |
+
|
| 942 |
+
def queue_seed_all(self, cb, traceback):
|
| 943 |
+
self.manual_seed_all_cb = (cb, traceback)
|
| 944 |
+
# update seed_all to be latest
|
| 945 |
+
self.call_order = [self.manual_seed_cb, self.manual_seed_all_cb]
|
| 946 |
+
|
| 947 |
+
def queue_seed(self, cb, traceback):
|
| 948 |
+
self.manual_seed_cb = (cb, traceback)
|
| 949 |
+
# update seed to be latest
|
| 950 |
+
self.call_order = [self.manual_seed_all_cb, self.manual_seed_cb]
|
| 951 |
+
|
| 952 |
+
def get_calls(self) -> List:
|
| 953 |
+
return self.call_order
|
| 954 |
+
|
| 955 |
+
|
| 956 |
+
logger = logging.getLogger(__name__)
|
| 957 |
+
P = ParamSpec("P")
|
| 958 |
+
|
| 959 |
+
|
| 960 |
+
class CallbackRegistry(Generic[P]):
|
| 961 |
+
def __init__(self, name: str):
|
| 962 |
+
self.name = name
|
| 963 |
+
self.callback_list: List[Callable[P, None]] = []
|
| 964 |
+
|
| 965 |
+
def add_callback(self, cb: Callable[P, None]) -> None:
|
| 966 |
+
self.callback_list.append(cb)
|
| 967 |
+
|
| 968 |
+
def fire_callbacks(self, *args: P.args, **kwargs: P.kwargs) -> None:
|
| 969 |
+
for cb in self.callback_list:
|
| 970 |
+
try:
|
| 971 |
+
cb(*args, **kwargs)
|
| 972 |
+
except Exception as e:
|
| 973 |
+
logger.exception(
|
| 974 |
+
"Exception in callback for %s registered with gpu trace", self.name
|
| 975 |
+
)
|
| 976 |
+
|
| 977 |
+
|
| 978 |
+
# IMPORT_MAPPING and NAME_MAPPING are adapted from https://github.com/python/cpython/blob/main/Lib/_compat_pickle.py
|
| 979 |
+
# for use in the weights_only Unpickler.
|
| 980 |
+
|
| 981 |
+
IMPORT_MAPPING = {
|
| 982 |
+
"__builtin__": "builtins",
|
| 983 |
+
"copy_reg": "copyreg",
|
| 984 |
+
"Queue": "queue",
|
| 985 |
+
"repr": "reprlib",
|
| 986 |
+
"_abcoll": "collections.abc",
|
| 987 |
+
# Non-mutual mappings.
|
| 988 |
+
"UserDict": "collections",
|
| 989 |
+
"UserList": "collections",
|
| 990 |
+
"UserString": "collections",
|
| 991 |
+
"whichdb": "dbm",
|
| 992 |
+
"StringIO": "io",
|
| 993 |
+
"cStringIO": "io",
|
| 994 |
+
}
|
| 995 |
+
|
| 996 |
+
|
| 997 |
+
# This contains rename rules that are easy to handle. We ignore the more
|
| 998 |
+
# complex stuff (e.g. mapping the names in the urllib and types modules).
|
| 999 |
+
# These rules should be run before import names are fixed.
|
| 1000 |
+
NAME_MAPPING = {
|
| 1001 |
+
("__builtin__", "xrange"): ("builtins", "range"),
|
| 1002 |
+
("__builtin__", "reduce"): ("functools", "reduce"),
|
| 1003 |
+
("__builtin__", "intern"): ("sys", "intern"),
|
| 1004 |
+
("__builtin__", "unichr"): ("builtins", "chr"),
|
| 1005 |
+
("__builtin__", "unicode"): ("builtins", "str"),
|
| 1006 |
+
("__builtin__", "long"): ("builtins", "int"),
|
| 1007 |
+
("itertools", "izip"): ("builtins", "zip"),
|
| 1008 |
+
("itertools", "imap"): ("builtins", "map"),
|
| 1009 |
+
("itertools", "ifilter"): ("builtins", "filter"),
|
| 1010 |
+
("itertools", "ifilterfalse"): ("itertools", "filterfalse"),
|
| 1011 |
+
("itertools", "izip_longest"): ("itertools", "zip_longest"),
|
| 1012 |
+
("UserDict", "IterableUserDict"): ("collections", "UserDict"),
|
| 1013 |
+
("UserList", "UserList"): ("collections", "UserList"),
|
| 1014 |
+
("UserString", "UserString"): ("collections", "UserString"),
|
| 1015 |
+
# Non-mutual mappings.
|
| 1016 |
+
("__builtin__", "basestring"): ("builtins", "str"),
|
| 1017 |
+
("exceptions", "StandardError"): ("builtins", "Exception"),
|
| 1018 |
+
("UserDict", "UserDict"): ("collections", "UserDict"),
|
| 1019 |
+
}
|