Add files using upload-large-folder tool

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/__init__.py

@@ -0,0 +1,51 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+__version__ = "1.12.0"
+
+from .accelerator import Accelerator
+from .big_modeling import (
+    cpu_offload,
+    cpu_offload_with_hook,
+    disk_offload,
+    dispatch_model,
+    init_empty_weights,
+    init_on_device,
+    load_checkpoint_and_dispatch,
+)
+from .data_loader import skip_first_batches
+from .inference import prepare_pippy
+from .launchers import debug_launcher, notebook_launcher
+from .parallelism_config import ParallelismConfig
+from .state import PartialState
+from .utils import (
+    AutocastKwargs,
+    DataLoaderConfiguration,
+    DDPCommunicationHookType,
+    DeepSpeedPlugin,
+    DistributedDataParallelKwargs,
+    DistributedType,
+    FullyShardedDataParallelPlugin,
+    GradScalerKwargs,
+    InitProcessGroupKwargs,
+    ProfileKwargs,
+    find_executable_batch_size,
+    infer_auto_device_map,
+    is_rich_available,
+    load_checkpoint_in_model,
+    synchronize_rng_states,
+)
+
+
+if is_rich_available():
+    from .utils import rich

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/big_modeling.py

@@ -0,0 +1,789 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import re
+from contextlib import contextmanager
+from functools import wraps
+from typing import Optional, Union
+
+import torch
+import torch.nn as nn
+
+from .hooks import (
+    AlignDevicesHook,
+    CpuOffload,
+    LayerwiseCastingHook,
+    UserCpuOffloadHook,
+    add_hook_to_module,
+    attach_align_device_hook,
+    attach_align_device_hook_on_blocks,
+)
+from .utils import (
+    OffloadedWeightsLoader,
+    check_cuda_p2p_ib_support,
+    check_device_map,
+    extract_submodules_state_dict,
+    find_tied_parameters,
+    get_balanced_memory,
+    infer_auto_device_map,
+    is_bnb_available,
+    is_mlu_available,
+    is_musa_available,
+    is_npu_available,
+    is_sdaa_available,
+    is_xpu_available,
+    load_checkpoint_in_model,
+    offload_state_dict,
+    parse_flag_from_env,
+    retie_parameters,
+)
+from .utils.constants import SUPPORTED_PYTORCH_LAYERS_FOR_UPCASTING
+from .utils.other import recursive_getattr
+
+
+logger = logging.getLogger(__name__)
+
+
+@contextmanager
+def init_empty_weights(include_buffers: Optional[bool] = None):
+    """
+    A context manager under which models are initialized with all parameters on the meta device, therefore creating an
+    empty model. Useful when just initializing the model would blow the available RAM.
+
+    Args:
+        include_buffers (`bool`, *optional*):
+            Whether or not to also put all buffers on the meta device while initializing.
+
+    Example:
+
+    ```python
+    import torch.nn as nn
+    from accelerate import init_empty_weights
+
+    # Initialize a model with 100 billions parameters in no time and without using any RAM.
+    with init_empty_weights():
+        tst = nn.Sequential(*[nn.Linear(10000, 10000) for _ in range(1000)])
+    ```
+
+    <Tip warning={true}>
+
+    Any model created under this context manager has no weights. As such you can't do something like
+    `model.to(some_device)` with it. To load weights inside your empty model, see [`load_checkpoint_and_dispatch`].
+    Make sure to overwrite the default device_map param for [`load_checkpoint_and_dispatch`], otherwise dispatch is not
+    called.
+
+    </Tip>
+    """
+    if include_buffers is None:
+        include_buffers = parse_flag_from_env("ACCELERATE_INIT_INCLUDE_BUFFERS", False)
+    with init_on_device(torch.device("meta"), include_buffers=include_buffers) as f:
+        yield f
+
+
+@contextmanager
+def init_on_device(device: torch.device, include_buffers: Optional[bool] = None):
+    """
+    A context manager under which models are initialized with all parameters on the specified device.
+
+    Args:
+        device (`torch.device`):
+            Device to initialize all parameters on.
+        include_buffers (`bool`, *optional*):
+            Whether or not to also put all buffers on the meta device while initializing.
+
+    Example:
+
+    ```python
+    import torch.nn as nn
+    from accelerate import init_on_device
+
+    with init_on_device(device=torch.device("cuda")):
+        tst = nn.Linear(100, 100)  # on `cuda` device
+    ```
+    """
+    if include_buffers is None:
+        include_buffers = parse_flag_from_env("ACCELERATE_INIT_INCLUDE_BUFFERS", False)
+
+    if include_buffers:
+        with device:
+            yield
+        return
+
+    old_register_parameter = nn.Module.register_parameter
+    if include_buffers:
+        old_register_buffer = nn.Module.register_buffer
+
+    def register_empty_parameter(module, name, param):
+        old_register_parameter(module, name, param)
+        if param is not None:
+            param_cls = type(module._parameters[name])
+            kwargs = module._parameters[name].__dict__
+            kwargs["requires_grad"] = param.requires_grad
+            module._parameters[name] = param_cls(module._parameters[name].to(device), **kwargs)
+
+    def register_empty_buffer(module, name, buffer, persistent=True):
+        old_register_buffer(module, name, buffer, persistent=persistent)
+        if buffer is not None:
+            module._buffers[name] = module._buffers[name].to(device)
+
+    # Patch tensor creation
+    if include_buffers:
+        tensor_constructors_to_patch = {
+            torch_function_name: getattr(torch, torch_function_name)
+            for torch_function_name in ["empty", "zeros", "ones", "full"]
+        }
+    else:
+        tensor_constructors_to_patch = {}
+
+    def patch_tensor_constructor(fn):
+        def wrapper(*args, **kwargs):
+            kwargs["device"] = device
+            return fn(*args, **kwargs)
+
+        return wrapper
+
+    try:
+        nn.Module.register_parameter = register_empty_parameter
+        if include_buffers:
+            nn.Module.register_buffer = register_empty_buffer
+        for torch_function_name in tensor_constructors_to_patch.keys():
+            setattr(torch, torch_function_name, patch_tensor_constructor(getattr(torch, torch_function_name)))
+        yield
+    finally:
+        nn.Module.register_parameter = old_register_parameter
+        if include_buffers:
+            nn.Module.register_buffer = old_register_buffer
+        for torch_function_name, old_torch_function in tensor_constructors_to_patch.items():
+            setattr(torch, torch_function_name, old_torch_function)
+
+
+def cpu_offload(
+    model: nn.Module,
+    execution_device: Optional[torch.device] = None,
+    offload_buffers: bool = False,
+    state_dict: Optional[dict[str, torch.Tensor]] = None,
+    preload_module_classes: Optional[list[str]] = None,
+):
+    """
+    Activates full CPU offload for a model. As a result, all parameters of the model will be offloaded and only one
+    copy of the state dict of the model will be kept. During the forward pass, parameters will be extracted from that
+    state dict and put on the execution device passed as they are needed, then offloaded again.
+
+    Args:
+        model (`torch.nn.Module`):
+            The model to offload.
+        execution_device (`torch.device`, *optional*):
+            The device on which the forward pass of the model will be executed (should be a GPU). Will default to the
+            model first parameter device.
+        offload_buffers (`bool`, *optional*, defaults to `False`):
+            Whether or not to offload the buffers with the model parameters.
+        state_dict (`Dict[str, torch.Tensor]`, *optional*):
+            The state dict of the model that will be kept on CPU.
+        preload_module_classes (`List[str]`, *optional*):
+            A list of classes whose instances should load all their weights (even in the submodules) at the beginning
+            of the forward. This should only be used for classes that have submodules which are registered but not
+            called directly during the forward, for instance if a `dense` linear layer is registered, but at forward,
+            `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly.
+    """
+    if execution_device is None:
+        execution_device = next(iter(model.parameters())).device
+    if state_dict is None:
+        state_dict = {n: p.to("cpu") for n, p in model.state_dict().items()}
+
+    add_hook_to_module(model, AlignDevicesHook(io_same_device=True), append=True)
+    attach_align_device_hook(
+        model,
+        execution_device=execution_device,
+        offload=True,
+        offload_buffers=offload_buffers,
+        weights_map=state_dict,
+        preload_module_classes=preload_module_classes,
+    )
+
+    return model
+
+
+def cpu_offload_with_hook(
+    model: torch.nn.Module,
+    execution_device: Optional[Union[int, str, torch.device]] = None,
+    prev_module_hook: Optional[UserCpuOffloadHook] = None,
+):
+    """
+    Offloads a model on the CPU and puts it back to an execution device when executed. The difference with
+    [`cpu_offload`] is that the model stays on the execution device after the forward and is only offloaded again when
+    the `offload` method of the returned `hook` is called. Useful for pipelines running a model in a loop.
+
+    Args:
+        model (`torch.nn.Module`):
+            The model to offload.
+        execution_device(`str`, `int` or `torch.device`, *optional*):
+            The device on which the model should be executed. Will default to the MPS device if it's available, then
+            GPU 0 if there is a GPU, and finally to the CPU.
+        prev_module_hook (`UserCpuOffloadHook`, *optional*):
+            The hook sent back by this function for a previous model in the pipeline you are running. If passed, its
+            offload method will be called just before the forward of the model to which this hook is attached.
+
+    Example:
+
+    ```py
+    model_1, hook_1 = cpu_offload_with_hook(model_1, cuda_device)
+    model_2, hook_2 = cpu_offload_with_hook(model_2, cuda_device, prev_module_hook=hook_1)
+    model_3, hook_3 = cpu_offload_with_hook(model_3, cuda_device, prev_module_hook=hook_2)
+
+    hid_1 = model_1(input)
+    for i in range(50):
+        # model1 is offloaded on the CPU at the first iteration, model 2 stays on the GPU for this whole loop.
+        hid_2 = model_2(hid_1)
+    # model2 is offloaded to the CPU just before this forward.
+    hid_3 = model_3(hid_3)
+
+    # For model3, you need to manually call the hook offload method.
+    hook_3.offload()
+    ```
+    """
+    hook = CpuOffload(execution_device=execution_device, prev_module_hook=prev_module_hook)
+    add_hook_to_module(model, hook, append=True)
+    user_hook = UserCpuOffloadHook(model, hook)
+    return model, user_hook
+
+
+def disk_offload(
+    model: nn.Module,
+    offload_dir: Union[str, os.PathLike],
+    execution_device: Optional[torch.device] = None,
+    offload_buffers: bool = False,
+    preload_module_classes: Optional[list[str]] = None,
+):
+    """
+    Activates full disk offload for a model. As a result, all parameters of the model will be offloaded as
+    memory-mapped array in a given folder. During the forward pass, parameters will be accessed from that folder and
+    put on the execution device passed as they are needed, then offloaded again.
+
+    Args:
+        model (`torch.nn.Module`): The model to offload.
+        offload_dir (`str` or `os.PathLike`):
+            The folder in which to offload the model weights (or where the model weights are already offloaded).
+        execution_device (`torch.device`, *optional*):
+            The device on which the forward pass of the model will be executed (should be a GPU). Will default to the
+            model's first parameter device.
+        offload_buffers (`bool`, *optional*, defaults to `False`):
+            Whether or not to offload the buffers with the model parameters.
+        preload_module_classes (`List[str]`, *optional*):
+            A list of classes whose instances should load all their weights (even in the submodules) at the beginning
+            of the forward. This should only be used for classes that have submodules which are registered but not
+            called directly during the forward, for instance if a `dense` linear layer is registered, but at forward,
+            `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly.
+    """
+    if not os.path.isdir(offload_dir) or not os.path.isfile(os.path.join(offload_dir, "index.json")):
+        offload_state_dict(offload_dir, model.state_dict())
+    if execution_device is None:
+        execution_device = next(iter(model.parameters())).device
+    weights_map = OffloadedWeightsLoader(save_folder=offload_dir)
+
+    add_hook_to_module(model, AlignDevicesHook(io_same_device=True), append=True)
+    attach_align_device_hook(
+        model,
+        execution_device=execution_device,
+        offload=True,
+        offload_buffers=offload_buffers,
+        weights_map=weights_map,
+        preload_module_classes=preload_module_classes,
+    )
+
+    return model
+
+
+def dispatch_model(
+    model: nn.Module,
+    device_map: dict[str, Union[str, int, torch.device]],
+    main_device: Optional[torch.device] = None,
+    state_dict: Optional[dict[str, torch.Tensor]] = None,
+    offload_dir: Optional[Union[str, os.PathLike]] = None,
+    offload_index: Optional[dict[str, str]] = None,
+    offload_buffers: bool = False,
+    skip_keys: Optional[Union[str, list[str]]] = None,
+    preload_module_classes: Optional[list[str]] = None,
+    force_hooks: bool = False,
+):
+    """
+    Dispatches a model according to a given device map. Layers of the model might be spread across GPUs, offloaded on
+    the CPU or even the disk.
+
+    Args:
+        model (`torch.nn.Module`):
+            The model to dispatch.
+        device_map (`Dict[str, Union[str, int, torch.device]]`):
+            A dictionary mapping module names in the models `state_dict` to the device they should go to. Note that
+            `"disk"` is accepted even if it's not a proper value for `torch.device`.
+        main_device (`str`, `int` or `torch.device`, *optional*):
+            The main execution device. Will default to the first device in the `device_map` different from `"cpu"` or
+            `"disk"`.
+        state_dict (`Dict[str, torch.Tensor]`, *optional*):
+            The state dict of the part of the model that will be kept on CPU.
+        offload_dir (`str` or `os.PathLike`):
+            The folder in which to offload the model weights (or where the model weights are already offloaded).
+        offload_index (`Dict`, *optional*):
+            A dictionary from weight name to their information (`dtype`/ `shape` or safetensors filename). Will default
+            to the index saved in `save_folder`.
+        offload_buffers (`bool`, *optional*, defaults to `False`):
+            Whether or not to offload the buffers with the model parameters.
+        skip_keys (`str` or `List[str]`, *optional*):
+            A list of keys to ignore when moving inputs or outputs between devices.
+        preload_module_classes (`List[str]`, *optional*):
+            A list of classes whose instances should load all their weights (even in the submodules) at the beginning
+            of the forward. This should only be used for classes that have submodules which are registered but not
+            called directly during the forward, for instance if a `dense` linear layer is registered, but at forward,
+            `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly.
+        force_hooks (`bool`, *optional*, defaults to `False`):
+            Whether or not to force device hooks to be attached to the model even if all layers are dispatched to a
+            single device.
+    """
+    # Error early if the device map is incomplete.
+    check_device_map(model, device_map)
+
+    # We need to force hook for quantized model that can't be moved with to()
+    if getattr(model, "quantization_method", "bitsandbytes") == "bitsandbytes":
+        # since bnb 0.43.2, we can move 4-bit model
+        if getattr(model, "is_loaded_in_8bit", False) or (
+            getattr(model, "is_loaded_in_4bit", False) and not is_bnb_available(min_version="0.43.2")
+        ):
+            force_hooks = True
+
+    # We attach hooks if the device_map has at least 2 different devices or if
+    # force_hooks is set to `True`. Otherwise, the model in already loaded
+    # in the unique device and the user can decide where to dispatch the model.
+    # If the model is quantized, we always force-dispatch the model
+    if (len(set(device_map.values())) > 1) or force_hooks:
+        if main_device is None:
+            if set(device_map.values()) == {"cpu"} or set(device_map.values()) == {"cpu", "disk"}:
+                main_device = "cpu"
+            else:
+                main_device = [d for d in device_map.values() if d not in ["cpu", "disk"]][0]
+
+        if main_device != "cpu":
+            cpu_modules = [name for name, device in device_map.items() if device == "cpu"]
+            if state_dict is None and len(cpu_modules) > 0:
+                state_dict = extract_submodules_state_dict(model.state_dict(), cpu_modules)
+
+        disk_modules = [name for name, device in device_map.items() if device == "disk"]
+        if offload_dir is None and offload_index is None and len(disk_modules) > 0:
+            raise ValueError(
+                "We need an `offload_dir` to dispatch this model according to this `device_map`, the following submodules "
+                f"need to be offloaded: {', '.join(disk_modules)}."
+            )
+        if (
+            len(disk_modules) > 0
+            and offload_index is None
+            and (not os.path.isdir(offload_dir) or not os.path.isfile(os.path.join(offload_dir, "index.json")))
+        ):
+            disk_state_dict = extract_submodules_state_dict(model.state_dict(), disk_modules)
+            offload_state_dict(offload_dir, disk_state_dict)
+
+        execution_device = {
+            name: main_device if device in ["cpu", "disk"] else device for name, device in device_map.items()
+        }
+        execution_device[""] = main_device
+        offloaded_devices = ["disk"] if main_device == "cpu" or main_device == "mps" else ["cpu", "disk"]
+        offload = {name: device in offloaded_devices for name, device in device_map.items()}
+        save_folder = offload_dir if len(disk_modules) > 0 else None
+        if state_dict is not None or save_folder is not None or offload_index is not None:
+            device = main_device if offload_index is not None else None
+            weights_map = OffloadedWeightsLoader(
+                state_dict=state_dict, save_folder=save_folder, index=offload_index, device=device
+            )
+        else:
+            weights_map = None
+
+        # When dispatching the model's parameters to the devices specified in device_map, we want to avoid allocating memory several times for the
+        # tied parameters. The dictionary tied_params_map keeps track of the already allocated data for a given tied parameter (represented by its
+        # original pointer) on each devices.
+        tied_params = find_tied_parameters(model)
+
+        tied_params_map = {}
+        for group in tied_params:
+            for param_name in group:
+                # data_ptr() is enough here, as `find_tied_parameters` finds tied params simply by comparing `param1 is param2`, so we don't need
+                # to care about views of tensors through storage_offset.
+                data_ptr = recursive_getattr(model, param_name).data_ptr()
+                tied_params_map[data_ptr] = {}
+
+                # Note: To handle the disk offloading case, we can not simply use weights_map[param_name].data_ptr() as the reference pointer,
+                # as we have no guarantee that safetensors' `file.get_tensor()` will always give the same pointer.
+
+        attach_align_device_hook_on_blocks(
+            model,
+            execution_device=execution_device,
+            offload=offload,
+            offload_buffers=offload_buffers,
+            weights_map=weights_map,
+            skip_keys=skip_keys,
+            preload_module_classes=preload_module_classes,
+            tied_params_map=tied_params_map,
+        )
+
+        # warn if there is any params on the meta device
+        offloaded_devices_str = " and ".join(
+            [device for device in set(device_map.values()) if device in ("cpu", "disk")]
+        )
+        if len(offloaded_devices_str) > 0:
+            logger.warning(
+                f"Some parameters are on the meta device because they were offloaded to the {offloaded_devices_str}."
+            )
+
+        # Attaching the hook may break tied weights, so we retie them
+        retie_parameters(model, tied_params)
+
+        # add warning to cuda and to method
+        def add_warning(fn, model):
+            @wraps(fn)
+            def wrapper(*args, **kwargs):
+                warning_msg = "You shouldn't move a model that is dispatched using accelerate hooks."
+                if str(fn.__name__) == "to":
+                    to_device = torch._C._nn._parse_to(*args, **kwargs)[0]
+                    if to_device is not None:
+                        logger.warning(warning_msg)
+                else:
+                    logger.warning(warning_msg)
+                for param in model.parameters():
+                    if param.device == torch.device("meta"):
+                        raise RuntimeError("You can't move a model that has some modules offloaded to cpu or disk.")
+                return fn(*args, **kwargs)
+
+            return wrapper
+
+        # Make sure to update _accelerate_added_attributes in hooks.py if you add any hook
+        model.to = add_warning(model.to, model)
+        if is_npu_available():
+            model.npu = add_warning(model.npu, model)
+        elif is_mlu_available():
+            model.mlu = add_warning(model.mlu, model)
+        elif is_sdaa_available():
+            model.sdaa = add_warning(model.sdaa, model)
+        elif is_musa_available():
+            model.musa = add_warning(model.musa, model)
+        elif is_xpu_available():
+            model.xpu = add_warning(model.xpu, model)
+        else:
+            model.cuda = add_warning(model.cuda, model)
+
+        # Check if we are using multi-gpus with RTX 4000 series
+        use_multi_gpu = len([device for device in set(device_map.values()) if device not in ("cpu", "disk")]) > 1
+        if use_multi_gpu and not check_cuda_p2p_ib_support():
+            logger.warning(
+                "We've detected an older driver with an RTX 4000 series GPU. These drivers have issues with P2P. "
+                "This can affect the multi-gpu inference when using accelerate device_map."
+                "Please make sure to update your driver to the latest version which resolves this."
+            )
+    else:
+        device = list(device_map.values())[0]
+        # `torch.Tensor.to(<int num>)` is not supported by `torch_npu` (see this [issue](https://github.com/Ascend/pytorch/issues/16)).
+        if is_npu_available() and isinstance(device, int):
+            device = f"npu:{device}"
+        elif is_mlu_available() and isinstance(device, int):
+            device = f"mlu:{device}"
+        elif is_sdaa_available() and isinstance(device, int):
+            device = f"sdaa:{device}"
+        elif is_musa_available() and isinstance(device, int):
+            device = f"musa:{device}"
+        if device != "disk":
+            model.to(device)
+        else:
+            raise ValueError(
+                "You are trying to offload the whole model to the disk. Please use the `disk_offload` function instead."
+            )
+    # Convert OrderedDict back to dict for easier usage
+    model.hf_device_map = dict(device_map)
+    return model
+
+
+def load_checkpoint_and_dispatch(
+    model: nn.Module,
+    checkpoint: Union[str, os.PathLike],
+    device_map: Optional[Union[str, dict[str, Union[int, str, torch.device]]]] = None,
+    max_memory: Optional[dict[Union[int, str], Union[int, str]]] = None,
+    no_split_module_classes: Optional[list[str]] = None,
+    offload_folder: Optional[Union[str, os.PathLike]] = None,
+    offload_buffers: bool = False,
+    dtype: Optional[Union[str, torch.dtype]] = None,
+    offload_state_dict: Optional[bool] = None,
+    skip_keys: Optional[Union[str, list[str]]] = None,
+    preload_module_classes: Optional[list[str]] = None,
+    force_hooks: bool = False,
+    strict: bool = False,
+    full_state_dict: bool = True,
+    broadcast_from_rank0: bool = False,
+):
+    """
+    Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are
+    loaded and adds the various hooks that will make this model run properly (even if split across devices).
+
+    Args:
+        model (`torch.nn.Module`): The model in which we want to load a checkpoint.
+        checkpoint (`str` or `os.PathLike`):
+            The folder checkpoint to load. It can be:
+            - a path to a file containing a whole model state dict
+            - a path to a `.json` file containing the index to a sharded checkpoint
+            - a path to a folder containing a unique `.index.json` file and the shards of a checkpoint.
+        device_map (`Dict[str, Union[int, str, torch.device]]`, *optional*):
+            A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer
+            name, once a given module name is inside, every submodule of it will be sent to the same device.
+
+            To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For more
+            information about each option see [here](../concept_guides/big_model_inference#designing-a-device-map).
+            Defaults to None, which means [`dispatch_model`] will not be called.
+        max_memory (`Dict`, *optional*):
+            A dictionary device identifier to maximum memory. Will default to the maximum memory available for each GPU
+            and the available CPU RAM if unset.
+        no_split_module_classes (`List[str]`, *optional*):
+            A list of layer class names that should never be split across device (for instance any layer that has a
+            residual connection).
+        offload_folder (`str` or `os.PathLike`, *optional*):
+            If the `device_map` contains any value `"disk"`, the folder where we will offload weights.
+        offload_buffers (`bool`, *optional*, defaults to `False`):
+            In the layers that are offloaded on the CPU or the hard drive, whether or not to offload the buffers as
+            well as the parameters.
+        dtype (`str` or `torch.dtype`, *optional*):
+            If provided, the weights will be converted to that type when loaded.
+        offload_state_dict (`bool`, *optional*):
+            If `True`, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if
+            the weight of the CPU state dict + the biggest shard does not fit. Will default to `True` if the device map
+            picked contains `"disk"` values.
+        skip_keys (`str` or `List[str]`, *optional*):
+            A list of keys to ignore when moving inputs or outputs between devices.
+        preload_module_classes (`List[str]`, *optional*):
+            A list of classes whose instances should load all their weights (even in the submodules) at the beginning
+            of the forward. This should only be used for classes that have submodules which are registered but not
+            called directly during the forward, for instance if a `dense` linear layer is registered, but at forward,
+            `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly.
+        force_hooks (`bool`, *optional*, defaults to `False`):
+            Whether or not to force device hooks to be attached to the model even if all layers are dispatched to a
+            single device.
+        strict (`bool`, *optional*, defaults to `False`):
+            Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model's
+            state_dict.
+        full_state_dict (`bool`, *optional*, defaults to `True`): if this is set to `True`, all the tensors in the
+            loaded state_dict will be gathered. No ShardedTensor and DTensor will be in the loaded state_dict.
+        broadcast_from_rank0 (`False`, *optional*, defaults to `False`): when the option is `True`, a distributed
+            `ProcessGroup` must be initialized. rank0 should receive a full state_dict and will broadcast the tensors
+            in the state_dict one by one to other ranks. Other ranks will receive the tensors and shard (if applicable)
+            according to the local shards in the model.
+
+    Example:
+
+    ```python
+    >>> from accelerate import init_empty_weights, load_checkpoint_and_dispatch
+    >>> from huggingface_hub import hf_hub_download
+    >>> from transformers import AutoConfig, AutoModelForCausalLM
+
+    >>> # Download the Weights
+    >>> checkpoint = "EleutherAI/gpt-j-6B"
+    >>> weights_location = hf_hub_download(checkpoint, "pytorch_model.bin")
+
+    >>> # Create a model and initialize it with empty weights
+    >>> config = AutoConfig.from_pretrained(checkpoint)
+    >>> with init_empty_weights():
+    ...     model = AutoModelForCausalLM.from_config(config)
+
+    >>> # Load the checkpoint and dispatch it to the right devices
+    >>> model = load_checkpoint_and_dispatch(
+    ...     model, weights_location, device_map="auto", no_split_module_classes=["GPTJBlock"]
+    ... )
+    ```
+    """
+    if isinstance(device_map, str) and device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]:
+        raise ValueError(
+            "If passing a string for `device_map`, please choose 'auto', 'balanced', 'balanced_low_0' or 'sequential'."
+        )
+    if isinstance(device_map, str):
+        if device_map != "sequential":
+            max_memory = get_balanced_memory(
+                model,
+                max_memory=max_memory,
+                no_split_module_classes=no_split_module_classes,
+                dtype=dtype,
+                low_zero=(device_map == "balanced_low_0"),
+            )
+        device_map = infer_auto_device_map(
+            model,
+            max_memory=max_memory,
+            no_split_module_classes=no_split_module_classes,
+            dtype=dtype,
+            offload_buffers=offload_buffers,
+        )
+    if offload_state_dict is None and device_map is not None and "disk" in device_map.values():
+        offload_state_dict = True
+    load_checkpoint_in_model(
+        model,
+        checkpoint,
+        device_map=device_map,
+        offload_folder=offload_folder,
+        dtype=dtype,
+        offload_state_dict=offload_state_dict,
+        offload_buffers=offload_buffers,
+        strict=strict,
+        full_state_dict=full_state_dict,
+        broadcast_from_rank0=broadcast_from_rank0,
+    )
+    if device_map is None:
+        return model
+    return dispatch_model(
+        model,
+        device_map=device_map,
+        offload_dir=offload_folder,
+        offload_buffers=offload_buffers,
+        skip_keys=skip_keys,
+        preload_module_classes=preload_module_classes,
+        force_hooks=force_hooks,
+    )
+
+
+def attach_layerwise_casting_hooks(
+    module: torch.nn.Module,
+    storage_dtype: torch.dtype,
+    compute_dtype: torch.dtype,
+    skip_modules_pattern: Optional[Union[str, tuple[str, ...]]] = None,
+    skip_modules_classes: Optional[tuple[type[torch.nn.Module], ...]] = None,
+    non_blocking: bool = False,
+) -> None:
+    r"""
+    Applies layerwise casting to a given module. The module expected here is a PyTorch `nn.Module`. This is helpful for
+    reducing memory requirements when one doesn't want to fully quantize a model. Model params can be kept in say,
+    `torch.float8_e4m3fn` and upcasted to a higher precision like `torch.bfloat16` during forward pass and downcasted
+    back to `torch.float8_e4m3fn` to realize memory savings.
+
+    Args:
+        module (`torch.nn.Module`):
+            The module whose leaf modules will be cast to a high precision dtype for computation, and to a low
+            precision dtype for storage.
+        storage_dtype (`torch.dtype`):
+            The dtype to cast the module to before/after the forward pass for storage.
+        compute_dtype (`torch.dtype`):
+            The dtype to cast the module to during the forward pass for computation.
+        skip_modules_pattern (`tuple[str, ...]`, defaults to `None`):
+            A list of patterns to match the names of the modules to skip during the layerwise casting process. If set
+            to `None` alongside `skip_modules_classes` being `None`, the layerwise casting is applied directly to the
+            module instead of its internal submodules.
+        skip_modules_classes (`tuple[type[torch.nn.Module], ...]`, defaults to `None`):
+            A list of module classes to skip during the layerwise casting process.
+        non_blocking (`bool`, defaults to `False`):
+            If `True`, the weight casting operations are non-blocking.
+
+    Example:
+
+    ```python
+    >>> from accelerate.hooks import attach_layerwise_casting_hooks
+    >>> from transformers import AutoModelForCausalLM
+    >>> import torch
+
+    >>> # Model
+    >>> checkpoint = "EleutherAI/gpt-j-6B"
+    >>> model = AutoModelForCausalLM.from_pretrained(checkpoint)
+
+    >>> # Attach hooks and perform inference
+    >>> attach_layerwise_casting_hooks(model, storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16)
+    >>> with torch.no_grad():
+    ...     model(...)
+    ```
+
+    Users can also pass modules they want to avoid from getting downcasted.
+
+    ```py
+    >>> attach_layerwise_casting_hooks(
+    ...     model, storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16, skip_modules_pattern=["norm"]
+    ... )
+    ```
+    """
+    _attach_layerwise_casting_hooks(
+        module, storage_dtype, compute_dtype, skip_modules_pattern, skip_modules_classes, non_blocking
+    )
+
+
+def _attach_layerwise_casting_hooks(
+    module: torch.nn.Module,
+    storage_dtype: torch.dtype,
+    compute_dtype: torch.dtype,
+    skip_modules_pattern: Optional[Union[str, tuple[str, ...]]] = None,
+    skip_modules_classes: Optional[tuple[type[torch.nn.Module], ...]] = None,
+    non_blocking: bool = False,
+    _prefix: str = "",
+):
+    should_skip = (skip_modules_classes is not None and isinstance(module, skip_modules_classes)) or (
+        skip_modules_pattern is not None and any(re.search(pattern, _prefix) for pattern in skip_modules_pattern)
+    )
+    if should_skip:
+        logger.debug(f'Skipping layerwise casting for layer "{_prefix}"')
+        return
+
+    if isinstance(module, SUPPORTED_PYTORCH_LAYERS_FOR_UPCASTING):
+        logger.debug(f'Applying layerwise casting to layer "{_prefix}"')
+        add_hook_to_module(
+            module,
+            LayerwiseCastingHook(storage_dtype=storage_dtype, compute_dtype=compute_dtype, non_blocking=non_blocking),
+            append=True,
+        )
+        return
+
+    for name, submodule in module.named_children():
+        layer_name = f"{_prefix}.{name}" if _prefix else name
+        _attach_layerwise_casting_hooks(
+            submodule,
+            storage_dtype,
+            compute_dtype,
+            skip_modules_pattern,
+            skip_modules_classes,
+            non_blocking,
+            _prefix=layer_name,
+        )
+
+
+def _attach_context_parallel_hooks(
+    model: nn.Module,
+):
+    """
+    Monkeypatch huggingface's `transformers` model to fix attention mask issues when using context parallelism.
+
+    This function attaches forward_pre_hooks to each self_attn module of the model, where each hook checks the
+    args/kwargs, if they contain an attention mask, if it does, it will remove this mask, check if it is a causal mask,
+    if yes, will add a kwarg `is_causal=True`, otherwise will raise an error. This is because context parallelism does
+    not support attention masks. This function modifies the model in place.
+
+    Args:
+        model (`nn.Module`):
+            The model to attach the hooks to.
+
+    """
+
+    def _self_attn_pre_forward_hook(_module, module_args, module_kwargs):
+        if "attention_mask" in module_kwargs:
+            module_kwargs["attention_mask"] = None
+            module_kwargs["is_causal"] = True
+
+        return module_args, module_kwargs
+
+    for name, module in model.named_modules():
+        # We hope (assume) that if user uses their own model (without this structure which transformers uses), they read the docs saying they can't pass in attention masks
+        # Then these cases can happen:
+        # 1) some modules end with a `self-attn` module, in which case we attach the hook, but the
+        #    there's no attention mask kwarg -> hook is a no-op
+        # 2) some modules end with a `self-attn` module, in which case we attach the hook, and the
+        #    attention mask kwarg is passed -> hook will remove the attention mask and add
+        #    `is_causal=True` kwarg, which either crashes the training or fixes it
+        #    (training would crash anyway as attention mask isn't supported)
+        # 3) no modules end with a `self-attn` module, in which case we don't attach the hook, this is
+        #    a no-op as well
+        if name.endswith("self_attn"):
+            # we want the hook to be executed first, to avoid any other hooks doing work on the attention mask
+            module.register_forward_pre_hook(_self_attn_pre_forward_hook, with_kwargs=True, prepend=True)

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/checkpointing.py

@@ -0,0 +1,331 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import torch
+from safetensors.torch import load_model
+
+from .utils import (
+    MODEL_NAME,
+    OPTIMIZER_NAME,
+    RNG_STATE_NAME,
+    SAFE_MODEL_NAME,
+    SAFE_WEIGHTS_NAME,
+    SAMPLER_NAME,
+    SCALER_NAME,
+    SCHEDULER_NAME,
+    WEIGHTS_NAME,
+    get_pretty_name,
+    is_cuda_available,
+    is_hpu_available,
+    is_mlu_available,
+    is_musa_available,
+    is_sdaa_available,
+    is_torch_version,
+    is_torch_xla_available,
+    is_xpu_available,
+    load,
+    save,
+)
+
+
+if is_torch_version(">=", "2.4.0"):
+    from torch.amp import GradScaler
+else:
+    from torch.cuda.amp import GradScaler
+
+if is_torch_xla_available():
+    import torch_xla.core.xla_model as xm
+
+from .logging import get_logger
+from .state import PartialState
+
+
+logger = get_logger(__name__)
+
+
+def save_accelerator_state(
+    output_dir: str,
+    model_states: list[dict],
+    optimizers: list,
+    schedulers: list,
+    dataloaders: list,
+    process_index: int,
+    step: int,
+    scaler: Optional[GradScaler] = None,
+    save_on_each_node: bool = False,
+    safe_serialization: bool = True,
+):
+    """
+    Saves the current states of the models, optimizers, scaler, and RNG generators to a given directory.
+
+    <Tip>
+
+    If `safe_serialization` is `True`, models will be saved with `safetensors` while the rest are saved using native
+    `pickle`.
+
+    </Tip>
+
+    Args:
+        output_dir (`str` or `os.PathLike`):
+            The name of the folder to save all relevant weights and states.
+        model_states (`List[torch.nn.Module]`):
+            A list of model states
+        optimizers (`List[torch.optim.Optimizer]`):
+            A list of optimizer instances
+        schedulers (`List[torch.optim.lr_scheduler._LRScheduler]`):
+            A list of learning rate schedulers
+        dataloaders (`List[torch.utils.data.DataLoader]`):
+            A list of dataloader instances to save their sampler states
+        process_index (`int`):
+            The current process index in the Accelerator state
+        step (`int`):
+            The current step in the internal step tracker
+        scaler (`torch.amp.GradScaler`, *optional*):
+            An optional gradient scaler instance to save;
+        save_on_each_node (`bool`, *optional*):
+            Whether to save on every node, or only the main node.
+        safe_serialization (`bool`, *optional*, defaults to `True`):
+            Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+    """
+    output_dir = Path(output_dir)
+    # Model states
+    for i, state in enumerate(model_states):
+        weights_name = WEIGHTS_NAME if not safe_serialization else SAFE_WEIGHTS_NAME
+        if i > 0:
+            weights_name = weights_name.replace(".", f"_{i}.")
+        output_model_file = output_dir.joinpath(weights_name)
+        save(state, output_model_file, save_on_each_node=save_on_each_node, safe_serialization=safe_serialization)
+        logger.info(f"Model weights saved in {output_model_file}")
+    # Optimizer states
+    for i, opt in enumerate(optimizers):
+        state = opt.state_dict()
+        optimizer_name = f"{OPTIMIZER_NAME}.bin" if i == 0 else f"{OPTIMIZER_NAME}_{i}.bin"
+        output_optimizer_file = output_dir.joinpath(optimizer_name)
+        save(state, output_optimizer_file, save_on_each_node=save_on_each_node, safe_serialization=False)
+        logger.info(f"Optimizer state saved in {output_optimizer_file}")
+    # Scheduler states
+    for i, scheduler in enumerate(schedulers):
+        state = scheduler.state_dict()
+        scheduler_name = f"{SCHEDULER_NAME}.bin" if i == 0 else f"{SCHEDULER_NAME}_{i}.bin"
+        output_scheduler_file = output_dir.joinpath(scheduler_name)
+        save(state, output_scheduler_file, save_on_each_node=save_on_each_node, safe_serialization=False)
+        logger.info(f"Scheduler state saved in {output_scheduler_file}")
+    # DataLoader states
+    for i, dataloader in enumerate(dataloaders):
+        sampler_name = f"{SAMPLER_NAME}.bin" if i == 0 else f"{SAMPLER_NAME}_{i}.bin"
+        output_sampler_file = output_dir.joinpath(sampler_name)
+        # Only save if we have our custom sampler
+        from .data_loader import IterableDatasetShard, SeedableRandomSampler
+
+        if isinstance(dataloader.dataset, IterableDatasetShard):
+            sampler = dataloader.get_sampler()
+            if isinstance(sampler, SeedableRandomSampler):
+                save(sampler, output_sampler_file, save_on_each_node=save_on_each_node, safe_serialization=False)
+        if getattr(dataloader, "use_stateful_dataloader", False):
+            dataloader_state_dict_name = "dl_state_dict.bin" if i == 0 else f"dl_state_dict_{i}.bin"
+            output_dataloader_state_dict_file = output_dir.joinpath(dataloader_state_dict_name)
+            state_dict = dataloader.state_dict()
+            torch.save(state_dict, output_dataloader_state_dict_file)
+        logger.info(f"Sampler state for dataloader {i} saved in {output_sampler_file}")
+
+    # GradScaler state
+    if scaler is not None:
+        state = scaler.state_dict()
+        output_scaler_file = output_dir.joinpath(SCALER_NAME)
+        torch.save(state, output_scaler_file)
+        logger.info(f"Gradient scaler state saved in {output_scaler_file}")
+    # Random number generator states
+    states = {}
+    states_name = f"{RNG_STATE_NAME}_{process_index}.pkl"
+    states["step"] = step
+    states["random_state"] = random.getstate()
+    states["numpy_random_seed"] = np.random.get_state()
+    states["torch_manual_seed"] = torch.get_rng_state()
+    if is_xpu_available():
+        states["torch_xpu_manual_seed"] = torch.xpu.get_rng_state_all()
+    if is_mlu_available():
+        states["torch_mlu_manual_seed"] = torch.mlu.get_rng_state_all()
+    elif is_sdaa_available():
+        states["torch_sdaa_manual_seed"] = torch.sdaa.get_rng_state_all()
+    elif is_musa_available():
+        states["torch_musa_manual_seed"] = torch.musa.get_rng_state_all()
+    if is_hpu_available():
+        states["torch_hpu_manual_seed"] = torch.hpu.get_rng_state_all()
+    if is_cuda_available():
+        states["torch_cuda_manual_seed"] = torch.cuda.get_rng_state_all()
+    if is_torch_xla_available():
+        states["xm_seed"] = xm.get_rng_state()
+    output_states_file = output_dir.joinpath(states_name)
+    torch.save(states, output_states_file)
+    logger.info(f"Random states saved in {output_states_file}")
+    return output_dir
+
+
+def load_accelerator_state(
+    input_dir,
+    models,
+    optimizers,
+    schedulers,
+    dataloaders,
+    process_index,
+    scaler=None,
+    map_location=None,
+    load_kwargs=None,
+    **load_model_func_kwargs,
+):
+    """
+    Loads states of the models, optimizers, scaler, and RNG generators from a given directory.
+
+    Args:
+        input_dir (`str` or `os.PathLike`):
+            The name of the folder to load all relevant weights and states.
+        models (`List[torch.nn.Module]`):
+            A list of model instances
+        optimizers (`List[torch.optim.Optimizer]`):
+            A list of optimizer instances
+        schedulers (`List[torch.optim.lr_scheduler._LRScheduler]`):
+            A list of learning rate schedulers
+        process_index (`int`):
+            The current process index in the Accelerator state
+        scaler (`torch.amp.GradScaler`, *optional*):
+            An optional *GradScaler* instance to load
+        map_location (`str`, *optional*):
+            What device to load the optimizer state onto. Should be one of either "cpu" or "on_device".
+        load_kwargs (`dict`, *optional*):
+            Additional arguments that can be passed to the `load` function.
+        load_model_func_kwargs (`dict`, *optional*):
+            Additional arguments that can be passed to the model's `load_state_dict` method.
+
+    Returns:
+        `dict`: Contains the `Accelerator` attributes to override while loading the state.
+    """
+    # stores the `Accelerator` attributes to override
+    override_attributes = dict()
+    if map_location not in [None, "cpu", "on_device"]:
+        raise TypeError(
+            "Unsupported optimizer map location passed, please choose one of `None`, `'cpu'`, or `'on_device'`"
+        )
+    if map_location is None:
+        map_location = "cpu"
+    elif map_location == "on_device":
+        map_location = PartialState().device
+
+    if load_kwargs is None:
+        load_kwargs = {}
+
+    input_dir = Path(input_dir)
+    # Model states
+    for i, model in enumerate(models):
+        ending = f"_{i}" if i > 0 else ""
+        input_model_file = input_dir.joinpath(f"{SAFE_MODEL_NAME}{ending}.safetensors")
+        if input_model_file.exists():
+            load_model(model, input_model_file, device=str(map_location), **load_model_func_kwargs)
+        else:
+            # Load with torch
+            input_model_file = input_dir.joinpath(f"{MODEL_NAME}{ending}.bin")
+            state_dict = load(input_model_file, map_location=map_location)
+            model.load_state_dict(state_dict, **load_model_func_kwargs)
+    logger.info("All model weights loaded successfully")
+
+    # Optimizer states
+    for i, opt in enumerate(optimizers):
+        optimizer_name = f"{OPTIMIZER_NAME}.bin" if i == 0 else f"{OPTIMIZER_NAME}_{i}.bin"
+        input_optimizer_file = input_dir.joinpath(optimizer_name)
+        optimizer_state = load(input_optimizer_file, map_location=map_location, **load_kwargs)
+        optimizers[i].load_state_dict(optimizer_state)
+    logger.info("All optimizer states loaded successfully")
+
+    # Scheduler states
+    for i, scheduler in enumerate(schedulers):
+        scheduler_name = f"{SCHEDULER_NAME}.bin" if i == 0 else f"{SCHEDULER_NAME}_{i}.bin"
+        input_scheduler_file = input_dir.joinpath(scheduler_name)
+        scheduler_state = load(input_scheduler_file, **load_kwargs)
+        scheduler.load_state_dict(scheduler_state)
+    logger.info("All scheduler states loaded successfully")
+
+    for i, dataloader in enumerate(dataloaders):
+        sampler_name = f"{SAMPLER_NAME}.bin" if i == 0 else f"{SAMPLER_NAME}_{i}.bin"
+        input_sampler_file = input_dir.joinpath(sampler_name)
+        # Only load if we have our custom sampler
+        from .data_loader import IterableDatasetShard, SeedableRandomSampler
+
+        if isinstance(dataloader.dataset, IterableDatasetShard):
+            sampler = dataloader.get_sampler()
+            if isinstance(sampler, SeedableRandomSampler):
+                sampler = dataloader.set_sampler(load(input_sampler_file))
+        if getattr(dataloader, "use_stateful_dataloader", False):
+            dataloader_state_dict_name = "dl_state_dict.bin" if i == 0 else f"dl_state_dict_{i}.bin"
+            input_dataloader_state_dict_file = input_dir.joinpath(dataloader_state_dict_name)
+            if input_dataloader_state_dict_file.exists():
+                state_dict = load(input_dataloader_state_dict_file, **load_kwargs)
+                dataloader.load_state_dict(state_dict)
+    logger.info("All dataloader sampler states loaded successfully")
+
+    # GradScaler state
+    if scaler is not None:
+        input_scaler_file = input_dir.joinpath(SCALER_NAME)
+        scaler_state = load(input_scaler_file)
+        scaler.load_state_dict(scaler_state)
+        logger.info("GradScaler state loaded successfully")
+
+    # Random states
+    try:
+        states = load(input_dir.joinpath(f"{RNG_STATE_NAME}_{process_index}.pkl"))
+        if "step" in states:
+            override_attributes["step"] = states["step"]
+        random.setstate(states["random_state"])
+        np.random.set_state(states["numpy_random_seed"])
+        torch.set_rng_state(states["torch_manual_seed"])
+        if is_xpu_available():
+            torch.xpu.set_rng_state_all(states["torch_xpu_manual_seed"])
+        if is_mlu_available():
+            torch.mlu.set_rng_state_all(states["torch_mlu_manual_seed"])
+        elif is_sdaa_available():
+            torch.sdaa.set_rng_state_all(states["torch_sdaa_manual_seed"])
+        elif is_musa_available():
+            torch.musa.set_rng_state_all(states["torch_musa_manual_seed"])
+        else:
+            torch.cuda.set_rng_state_all(states["torch_cuda_manual_seed"])
+        if is_torch_xla_available():
+            xm.set_rng_state(states["xm_seed"])
+        logger.info("All random states loaded successfully")
+    except Exception:
+        logger.info("Could not load random states")
+
+    return override_attributes
+
+
+def save_custom_state(obj, path, index: int = 0, save_on_each_node: bool = False):
+    """
+    Saves the state of `obj` to `{path}/custom_checkpoint_{index}.pkl`
+    """
+    # Should this be the right way to get a qual_name type value from `obj`?
+    save_location = Path(path) / f"custom_checkpoint_{index}.pkl"
+    logger.info(f"Saving the state of {get_pretty_name(obj)} to {save_location}")
+    save(obj.state_dict(), save_location, save_on_each_node=save_on_each_node)
+
+
+def load_custom_state(obj, path, index: int = 0):
+    """
+    Loads the state of `obj` at `{path}/custom_checkpoint_{index}.pkl`. Will always set `weights_only=False` when
+    loading the state.
+    """
+    load_location = f"{path}/custom_checkpoint_{index}.pkl"
+    logger.info(f"Loading the state of {get_pretty_name(obj)} from {load_location}")
+    obj.load_state_dict(load(load_location, map_location="cpu", weights_only=False))

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/data_loader.py

@@ -0,0 +1,1451 @@
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import math
+from contextlib import suppress
+from typing import Callable, Optional, Union
+
+import torch
+from packaging import version
+from torch.utils.data import BatchSampler, DataLoader, IterableDataset, RandomSampler
+
+from .logging import get_logger
+from .state import DistributedType, GradientState, PartialState, is_torch_xla_available
+from .utils import (
+    RNGType,
+    broadcast,
+    broadcast_object_list,
+    compare_versions,
+    concatenate,
+    find_batch_size,
+    get_data_structure,
+    initialize_tensors,
+    is_datasets_available,
+    is_torch_version,
+    is_torchdata_stateful_dataloader_available,
+    send_to_device,
+    slice_tensors,
+    synchronize_rng_states,
+)
+
+
+logger = get_logger(__name__)
+
+# kwargs of the DataLoader in min version 2.0
+_PYTORCH_DATALOADER_KWARGS = {
+    "batch_size": 1,
+    "shuffle": False,
+    "sampler": None,
+    "batch_sampler": None,
+    "num_workers": 0,
+    "collate_fn": None,
+    "pin_memory": False,
+    "drop_last": False,
+    "timeout": 0,
+    "worker_init_fn": None,
+    "multiprocessing_context": None,
+    "generator": None,
+    "prefetch_factor": 2,
+    "persistent_workers": False,
+    "pin_memory_device": "",
+}
+
+# kwargs added after by version
+_PYTORCH_DATALOADER_ADDITIONAL_KWARGS = {"2.6.0": {"in_order": True}}
+
+for v, additional_kwargs in _PYTORCH_DATALOADER_ADDITIONAL_KWARGS.items():
+    if is_torch_version(">=", v):
+        _PYTORCH_DATALOADER_KWARGS.update(additional_kwargs)
+
+
+class SeedableRandomSampler(RandomSampler):
+    """
+    Same as a random sampler, except that in `__iter__` a seed can be used.
+
+    Needed specifically in distributed cases, when the random generator for each GPU needs to start from the same seed
+    and be fully reproducible on multiple iterations.
+
+    If a custom `generator` is passed, it will rely on its initial seed as well as the current iteration it is on
+    (stored in `self.epoch`).
+    """
+
+    def __init__(self, *args, **kwargs):
+        data_seed = kwargs.pop("data_seed", None)
+        super().__init__(*args, **kwargs)
+
+        self.initial_seed = data_seed if data_seed is not None else torch.random.initial_seed()
+        self.epoch = 0
+
+    def __iter__(self):
+        if self.generator is None:
+            self.generator = torch.Generator(
+                device=torch.get_default_device() if hasattr(torch, "get_default_device") else "cpu"
+            )
+            self.generator.manual_seed(self.initial_seed)
+
+        # Allow `self.epoch` to modify the seed of the generator
+        seed = self.epoch + self.initial_seed
+        # print("Setting seed at epoch", self.epoch, seed)
+        self.generator.manual_seed(seed)
+        yield from super().__iter__()
+        self.set_epoch(self.epoch + 1)
+
+    def set_epoch(self, epoch: int):
+        "Sets the current iteration of the sampler."
+        self.epoch = epoch
+
+
+class BatchSamplerShard(BatchSampler):
+    """
+    Wraps a PyTorch `BatchSampler` to generate batches for one of the processes only. Instances of this class will
+    always yield a number of batches that is a round multiple of `num_processes` and that all have the same size.
+    Depending on the value of the `drop_last` attribute of the batch sampler passed, it will either stop the iteration
+    at the first batch that would be too small / not present on all processes or loop with indices from the beginning.
+
+    Args:
+        batch_sampler (`torch.utils.data.sampler.BatchSampler`):
+            The batch sampler to split in several shards.
+        num_processes (`int`, *optional*, defaults to 1):
+            The number of processes running concurrently.
+        process_index (`int`, *optional*, defaults to 0):
+            The index of the current process.
+        split_batches (`bool`, *optional*, defaults to `False`):
+            Whether the shards should be created by splitting a batch to give a piece of it on each process, or by
+            yielding different full batches on each process.
+
+            On two processes with a sampler of `[[0, 1, 2, 3], [4, 5, 6, 7]]`, this will result in:
+
+            - the sampler on process 0 to yield `[0, 1, 2, 3]` and the sampler on process 1 to yield `[4, 5, 6, 7]` if
+              this argument is set to `False`.
+            - the sampler on process 0 to yield `[0, 1]` then `[4, 5]` and the sampler on process 1 to yield `[2, 3]`
+              then `[6, 7]` if this argument is set to `True`.
+        even_batches (`bool`, *optional*, defaults to `True`):
+            Whether or not to loop back at the beginning of the sampler when the number of samples is not a round
+            multiple of (original batch size / number of processes).
+
+    <Tip warning={true}>
+
+    `BatchSampler`s with varying batch sizes are not enabled by default. To enable this behaviour, set `even_batches`
+    equal to `False`
+
+    </Tip>"""
+
+    def __init__(
+        self,
+        batch_sampler: BatchSampler,
+        num_processes: int = 1,
+        process_index: int = 0,
+        split_batches: bool = False,
+        even_batches: bool = True,
+    ):
+        if split_batches and batch_sampler.batch_size % num_processes != 0:
+            raise ValueError(
+                f"To use `BatchSamplerShard` in `split_batches` mode, the batch size ({batch_sampler.batch_size}) "
+                f"needs to be a round multiple of the number of processes ({num_processes})."
+            )
+        self.batch_sampler = batch_sampler
+        self.num_processes = num_processes
+        self.process_index = process_index
+        self.split_batches = split_batches
+        self.even_batches = even_batches
+        self.batch_size = getattr(batch_sampler, "batch_size", None)
+        self.drop_last = getattr(batch_sampler, "drop_last", False)
+        if self.batch_size is None and self.even_batches:
+            raise ValueError(
+                "You need to use `even_batches=False` when the batch sampler has no batch size. If you "
+                "are not calling this method directly, set `accelerator.even_batches=False` instead."
+            )
+
+    @property
+    def total_length(self):
+        return len(self.batch_sampler)
+
+    def __len__(self):
+        if self.split_batches:
+            # Split batches does not change the length of the batch sampler
+            return len(self.batch_sampler)
+        if len(self.batch_sampler) % self.num_processes == 0:
+            # If the length is a round multiple of the number of processes, it's easy.
+            return len(self.batch_sampler) // self.num_processes
+        length = len(self.batch_sampler) // self.num_processes
+        if self.drop_last:
+            # Same if we drop the remainder.
+            return length
+        elif self.even_batches:
+            # When we even batches we always get +1
+            return length + 1
+        else:
+            # Otherwise it depends on the process index.
+            return length + 1 if self.process_index < len(self.batch_sampler) % self.num_processes else length
+
+    def __iter__(self):
+        return self._iter_with_split() if self.split_batches else self._iter_with_no_split()
+
+    def _iter_with_split(self):
+        initial_data = []
+        batch_length = self.batch_sampler.batch_size // self.num_processes
+        for idx, batch in enumerate(self.batch_sampler):
+            if idx == 0:
+                initial_data = batch
+            if len(batch) == self.batch_size:
+                # If the batch is full, we yield the part of it this process is responsible of.
+                yield batch[batch_length * self.process_index : batch_length * (self.process_index + 1)]
+
+        # If drop_last is True of the last batch was full, iteration is over, otherwise...
+        if not self.drop_last and len(initial_data) > 0 and len(batch) < self.batch_size:
+            if not self.even_batches:
+                if len(batch) > batch_length * self.process_index:
+                    yield batch[batch_length * self.process_index : batch_length * (self.process_index + 1)]
+            else:
+                # For degenerate cases where the dataset has less than num_process * batch_size samples
+                while len(initial_data) < self.batch_size:
+                    initial_data += initial_data
+                batch = batch + initial_data
+                yield batch[batch_length * self.process_index : batch_length * (self.process_index + 1)]
+
+    def _iter_with_no_split(self):
+        initial_data = []
+        batch_to_yield = []
+        for idx, batch in enumerate(self.batch_sampler):
+            # We gather the initial indices in case we need to circle back at the end.
+            if not self.drop_last and idx < self.num_processes:
+                initial_data += batch
+            # We identify the batch to yield but wait until we ar sure every process gets a full batch before actually
+            # yielding it.
+            if idx % self.num_processes == self.process_index:
+                batch_to_yield = batch
+            if idx % self.num_processes == self.num_processes - 1 and (
+                self.batch_size is None or len(batch) == self.batch_size
+            ):
+                yield batch_to_yield
+                batch_to_yield = []
+
+        # If drop_last is True, iteration is over, otherwise...
+        if not self.drop_last and len(initial_data) > 0:
+            if not self.even_batches:
+                if len(batch_to_yield) > 0:
+                    yield batch_to_yield
+            else:
+                # ... we yield the complete batch we had saved before if it has the proper length
+                if len(batch_to_yield) == self.batch_size:
+                    yield batch_to_yield
+
+                # For degenerate cases where the dataset has less than num_process * batch_size samples
+                while len(initial_data) < self.num_processes * self.batch_size:
+                    initial_data += initial_data
+
+                # If the last batch seen was of the proper size, it has been yielded by its process so we move to the next
+                if len(batch) == self.batch_size:
+                    batch = []
+                    idx += 1
+
+                # Make sure we yield a multiple of self.num_processes batches
+                cycle_index = 0
+                while idx % self.num_processes != 0 or len(batch) > 0:
+                    end_index = cycle_index + self.batch_size - len(batch)
+                    batch += initial_data[cycle_index:end_index]
+                    if idx % self.num_processes == self.process_index:
+                        yield batch
+                    cycle_index = end_index
+                    batch = []
+                    idx += 1
+
+
+class IterableDatasetShard(IterableDataset):
+    """
+    Wraps a PyTorch `IterableDataset` to generate samples for one of the processes only. Instances of this class will
+    always yield a number of samples that is a round multiple of the actual batch size (depending of the value of
+    `split_batches`, this is either `batch_size` or `batch_size x num_processes`). Depending on the value of the
+    `drop_last` attribute of the batch sampler passed, it will either stop the iteration at the first batch that would
+    be too small or loop with indices from the beginning.
+
+    Args:
+        dataset (`torch.utils.data.dataset.IterableDataset`):
+            The batch sampler to split in several shards.
+        batch_size (`int`, *optional*, defaults to 1):
+            The size of the batches per shard (if `split_batches=False`) or the size of the batches (if
+            `split_batches=True`).
+        drop_last (`bool`, *optional*, defaults to `False`):
+            Whether or not to drop the last incomplete batch or complete the last batches by using the samples from the
+            beginning.
+        num_processes (`int`, *optional*, defaults to 1):
+            The number of processes running concurrently.
+        process_index (`int`, *optional*, defaults to 0):
+            The index of the current process.
+        split_batches (`bool`, *optional*, defaults to `False`):
+            Whether the shards should be created by splitting a batch to give a piece of it on each process, or by
+            yielding different full batches on each process.
+
+            On two processes with an iterable dataset yielding of `[0, 1, 2, 3, 4, 5, 6, 7]`, this will result in:
+
+            - the shard on process 0 to yield `[0, 1, 2, 3]` and the shard on process 1 to yield `[4, 5, 6, 7]` if this
+              argument is set to `False`.
+            - the shard on process 0 to yield `[0, 1, 4, 5]` and the sampler on process 1 to yield `[2, 3, 6, 7]` if
+              this argument is set to `True`.
+    """
+
+    def __init__(
+        self,
+        dataset: IterableDataset,
+        batch_size: int = 1,
+        drop_last: bool = False,
+        num_processes: int = 1,
+        process_index: int = 0,
+        split_batches: bool = False,
+    ):
+        if split_batches and batch_size > 1 and batch_size % num_processes != 0:
+            raise ValueError(
+                f"To use `IterableDatasetShard` in `split_batches` mode, the batch size ({batch_size}) "
+                f"needs to be a round multiple of the number of processes ({num_processes})."
+            )
+        self.dataset: IterableDataset = dataset
+        self.batch_size = batch_size
+        self.drop_last = drop_last
+        self.num_processes = num_processes
+        self.process_index = process_index
+        self.split_batches = split_batches
+
+    def set_epoch(self, epoch):
+        self.epoch = epoch
+        if hasattr(self.dataset, "set_epoch"):
+            self.dataset.set_epoch(epoch)
+
+    def __len__(self):
+        # We will just raise the downstream error if the underlying dataset is not sized
+        if self.drop_last:
+            return (len(self.dataset) // (self.batch_size * self.num_processes)) * self.batch_size
+        else:
+            return math.ceil(len(self.dataset) / (self.batch_size * self.num_processes)) * self.batch_size
+
+    def __iter__(self):
+        if (
+            not hasattr(self.dataset, "set_epoch")
+            and hasattr(self.dataset, "generator")
+            and isinstance(self.dataset.generator, torch.Generator)
+        ):
+            self.dataset.generator.manual_seed(self.epoch)
+        real_batch_size = self.batch_size if self.split_batches else (self.batch_size * self.num_processes)
+        process_batch_size = (self.batch_size // self.num_processes) if self.split_batches else self.batch_size
+        process_slice = range(self.process_index * process_batch_size, (self.process_index + 1) * process_batch_size)
+
+        first_batch = None
+        current_batch = []
+        for element in self.dataset:
+            current_batch.append(element)
+            # Wait to have a full batch before yielding elements.
+            if len(current_batch) == real_batch_size:
+                for i in process_slice:
+                    yield current_batch[i]
+                if first_batch is None:
+                    first_batch = current_batch.copy()
+                current_batch = []
+
+        # Finished if drop_last is True, otherwise complete the last batch with elements from the beginning.
+        if not self.drop_last and len(current_batch) > 0:
+            if first_batch is None:
+                first_batch = current_batch.copy()
+            while len(current_batch) < real_batch_size:
+                current_batch += first_batch
+            for i in process_slice:
+                yield current_batch[i]
+
+
+class DataLoaderStateMixin:
+    """
+    Mixin class that adds a state to a `DataLoader` to keep track of the status inside the dataloader such as at the
+    end of the iteration, the number of items in the dataset in the last batch relative to the batch size, and other
+    useful information that might be needed.
+
+    **Available attributes:**
+
+        - **end_of_dataloader** (`bool`) -- Whether at the last iteration or batch
+        - **remainder** (`int`) -- The number of items that are remaining in the last batch, relative to the total
+          batch size
+
+    <Tip warning={true}>
+
+        Inheriters of this class should ensure that the class creates a `GradientState()` instance, stored in
+        `self.gradient_state`.
+
+    </Tip>
+
+    """
+
+    def __init_subclass__(cls, **kwargs):
+        cls.end_of_dataloader = False
+        cls.remainder = -1
+
+    def reset(self):
+        self.end_of_dataloader = False
+        self.remainder = -1
+
+    def begin(self):
+        "Prepares the gradient state for the current dataloader"
+        self.reset()
+        with suppress(Exception):
+            if not self._drop_last:
+                length = getattr(self.dataset, "total_dataset_length", len(self.dataset))
+                self.remainder = length % self.total_batch_size
+        self.gradient_state._add_dataloader(self)
+
+    def end(self):
+        "Cleans up the gradient state after exiting the dataloader"
+        self.gradient_state._remove_dataloader(self)
+
+
+class DataLoaderAdapter:
+    """
+    A class which wraps around a PyTorch `DataLoader` (or variants of it) to be used with the `Accelerator`. For
+    compatibility reasons, this class inherits from the class it wraps around, so it can be used as a drop-in.
+    """
+
+    def __init__(self, dataset, use_stateful_dataloader=False, batch_sampler=None, **kwargs):
+        self.use_stateful_dataloader = use_stateful_dataloader
+        if is_torchdata_stateful_dataloader_available():
+            from torchdata.stateful_dataloader import StatefulDataLoader
+
+        if use_stateful_dataloader and not is_torchdata_stateful_dataloader_available():
+            raise ImportError(
+                "StatefulDataLoader is not available. Please install torchdata version 0.8.0 or higher to use it."
+            )
+        if use_stateful_dataloader:
+            torchdata_version = version.parse(importlib.metadata.version("torchdata"))
+            if (
+                "in_order" in kwargs
+                and compare_versions(torchdata_version, "<", "0.11")
+                and is_torch_version(">=", "2.6.0")
+            ):
+                kwargs.pop("in_order")
+            self.base_dataloader = StatefulDataLoader(dataset, batch_sampler=batch_sampler, **kwargs)
+        else:
+            self.base_dataloader = DataLoader(dataset, batch_sampler=batch_sampler, **kwargs)
+
+        if hasattr(self.base_dataloader, "state_dict"):
+            self.dl_state_dict = self.base_dataloader.state_dict()
+
+    def __getattr__(self, name):
+        # Avoid infinite recursion if we try to access a nonexistent base_dataloader attribute.
+        if name == "base_dataloader":
+            raise AttributeError()
+        # Delegate attribute access to the internal dataloader
+        return getattr(self.base_dataloader, name)
+
+    def state_dict(self):
+        return self.dl_state_dict
+
+    def load_state_dict(self, state_dict):
+        self.base_dataloader.load_state_dict(state_dict)
+
+    @property
+    def __class__(self):
+        """
+        In order to maintain backwards compatibility with other code, we need to ensure `isinstance(obj, DataLoader)`
+        returns true. This is because some downstream code assumes that the `DataLoader` is the base class of the
+        object.
+        """
+        return self.base_dataloader.__class__
+
+    def __len__(self):
+        return len(self.base_dataloader)
+
+    def adjust_state_dict_for_prefetch(self):
+        """
+        Adjusts the state dict for prefetching. Natively, this will adjust all of the iters yielded keys in
+        `self.dl_state_dict` by a factor of `num_processes - 1`, however if a custom correction is needed, this can be
+        overridden.
+
+        This should modify `self.dl_state_dict` directly
+        """
+        # The state dict will be off by a factor of `n-1` batch too many during DDP,
+        # so we need to adjust it here
+        if PartialState().distributed_type != DistributedType.NO:
+            factor = PartialState().num_processes - 1
+            if self.dl_state_dict["_sampler_iter_yielded"] > 0:
+                self.dl_state_dict["_sampler_iter_yielded"] -= factor
+            if self.dl_state_dict["_num_yielded"] > 0:
+                self.dl_state_dict["_num_yielded"] -= factor
+            if self.dl_state_dict["_index_sampler_state"] is not None:
+                if (
+                    "samples_yielded" in self.dl_state_dict["_index_sampler_state"]
+                    and self.dl_state_dict["_index_sampler_state"]["samples_yielded"] > 0
+                ):
+                    self.dl_state_dict["_index_sampler_state"]["samples_yielded"] -= self.batch_size * factor
+
+    def _update_state_dict(self):
+        # The state_dict of the underlying base_dataloader may be ahead of what is currently being yielded.
+        # E.g. the implementation of DataLoaderShard involves having an underlying iterator 1 element ahead of
+        # what it wants to yield.
+        #
+        # _update_state_dict is called to snapshot the state_dict that would properly recover the DataLoaderAdapter.
+        if hasattr(self.base_dataloader, "state_dict"):
+            self.dl_state_dict = self.base_dataloader.state_dict()
+            # Potentially modify the state_dict to adjust for prefetching
+            self.adjust_state_dict_for_prefetch()
+            # Then tag if we are at the end of the dataloader
+            self.dl_state_dict["_iterator_finished"] = self.end_of_dataloader
+
+
+class DataLoaderShard(DataLoaderAdapter, DataLoaderStateMixin):
+    """
+    Subclass of `DataLoaderAdapter` that will deal with device placement and current distributed setup.
+
+    Args:
+        dataset (`torch.utils.data.dataset.Dataset`):
+            The dataset to use to build this dataloader.
+        device (`torch.device`, *optional*):
+            If passed, the device to put all batches on.
+        rng_types (list of `str` or [`~utils.RNGType`]):
+            The list of random number generators to synchronize at the beginning of each iteration. Should be one or
+            several of:
+
+            - `"torch"`: the base torch random number generator
+            - `"cuda"`: the CUDA random number generator (GPU only)
+            - `"xla"`: the XLA random number generator (TPU only)
+            - `"generator"`: an optional `torch.Generator`
+        synchronized_generator (`torch.Generator`, *optional*):
+            A random number generator to keep synchronized across processes.
+        skip_batches (`int`, *optional*, defaults to 0):
+            The number of batches to skip at the beginning.
+        use_stateful_dataloader (`bool`, *optional*, defaults to `False`):
+            Whether to have this class adapt `StatefulDataLoader` from `torchdata` instead of the regular `DataLoader`.
+        **kwargs (additional keyword arguments, *optional*):
+            All other keyword arguments to pass to the regular `DataLoader` initialization.
+
+    **Available attributes:**
+
+        - **total_batch_size** (`int`) -- Total batch size of the dataloader across all processes.
+            Equal to the original batch size when `split_batches=True`; otherwise the original batch size * the total
+            number of processes
+
+        - **total_dataset_length** (`int`) -- Total length of the inner dataset across all processes.
+    """
+
+    def __init__(
+        self,
+        dataset,
+        device=None,
+        rng_types=None,
+        synchronized_generator=None,
+        skip_batches=0,
+        use_stateful_dataloader=False,
+        _drop_last: bool = False,
+        _non_blocking: bool = False,
+        torch_device_mesh=None,
+        **kwargs,
+    ):
+        super().__init__(dataset, use_stateful_dataloader=use_stateful_dataloader, **kwargs)
+        self.device = device
+        self.rng_types = rng_types
+        self.synchronized_generator = synchronized_generator
+        self.skip_batches = skip_batches
+        self.gradient_state = GradientState()
+        self._drop_last = _drop_last
+        self._non_blocking = _non_blocking
+        self.iteration = 0
+
+    def __iter__(self):
+        if self.rng_types is not None:
+            synchronize_rng_states(self.rng_types, self.synchronized_generator)
+        self.begin()
+
+        self.set_epoch(self.iteration)
+        dataloader_iter = self.base_dataloader.__iter__()
+        # We iterate one batch ahead to check when we are at the end
+        try:
+            current_batch = next(dataloader_iter)
+        except StopIteration:
+            self.end()
+            return
+
+        batch_index = 0
+        while True:
+            try:
+                # But we still move it to the device so it is done before `StopIteration` is reached
+                if self.device is not None:
+                    current_batch = send_to_device(current_batch, self.device, non_blocking=self._non_blocking)
+                self._update_state_dict()
+                next_batch = next(dataloader_iter)
+                if batch_index >= self.skip_batches:
+                    yield current_batch
+                batch_index += 1
+                current_batch = next_batch
+            except StopIteration:
+                self.end_of_dataloader = True
+                self._update_state_dict()
+                if batch_index >= self.skip_batches:
+                    yield current_batch
+                break
+
+        self.iteration += 1
+        self.end()
+
+    def __reduce__(self):
+        """
+        Define the `__reduce__` method to ensure a `DataLoaderShard` can be pickled and unpickled. This needs to be
+        explicitly defined since default pickling behavior is broken by `DataLoaderAdapter` messing with its
+        `__class__` member.
+        """
+        args = super().__reduce__()
+        return (DataLoaderShard, *args[1:])
+
+    def set_epoch(self, epoch: int):
+        # In case it is manually passed in, the user can set it to what they like
+        if self.iteration != epoch:
+            self.iteration = epoch
+        if hasattr(self.batch_sampler, "set_epoch"):
+            self.batch_sampler.set_epoch(epoch)
+        if hasattr(self.batch_sampler, "sampler") and hasattr(self.batch_sampler.sampler, "set_epoch"):
+            self.batch_sampler.sampler.set_epoch(epoch)
+        if (
+            hasattr(self.batch_sampler, "batch_sampler")
+            and hasattr(self.batch_sampler.batch_sampler, "sampler")
+            and hasattr(self.batch_sampler.batch_sampler.sampler, "set_epoch")
+        ):
+            self.batch_sampler.batch_sampler.sampler.set_epoch(epoch)
+        # We support if a custom `Dataset` implementation has `set_epoch`
+        # or in general HF datasets `Datasets`
+        elif hasattr(self.dataset, "set_epoch"):
+            self.dataset.set_epoch(epoch)
+
+    @property
+    def total_batch_size(self):
+        batch_sampler = self.sampler if isinstance(self.sampler, BatchSampler) else self.batch_sampler
+        return (
+            batch_sampler.batch_size
+            if getattr(batch_sampler, "split_batches", False)
+            else (batch_sampler.batch_size * getattr(batch_sampler, "num_processes", 1))
+        )
+
+    @property
+    def total_dataset_length(self):
+        if hasattr(self.dataset, "total_length"):
+            return self.dataset.total_length
+        else:
+            return len(self.dataset)
+
+    def get_sampler(self):
+        return get_sampler(self)
+
+    def set_sampler(self, sampler):
+        sampler_is_batch_sampler = isinstance(self.sampler, BatchSampler)
+        if sampler_is_batch_sampler:
+            self.sampler.sampler = sampler
+        else:
+            self.batch_sampler.sampler = sampler
+            if hasattr(self.batch_sampler, "batch_sampler"):
+                self.batch_sampler.batch_sampler.sampler = sampler
+
+
+if is_torch_xla_available():
+    import torch_xla.distributed.parallel_loader as xpl
+
+    class MpDeviceLoaderWrapper(xpl.MpDeviceLoader):
+        """
+        Wrapper for the xpl.MpDeviceLoader class that knows the total batch size.
+
+        XLA preloading threads will all call DataLoaderShard's __iter__(). Remove rng_types from DataLoaderShard to
+        prevent it from using the XLA device in the preloading threads, and synchronize the RNG once from the main
+        thread only.
+
+        **Available attributes:**
+
+        - **total_batch_size** (`int`) -- Total batch size of the dataloader across all processes.
+            Equal to the original batch size when `split_batches=True`; otherwise the original batch size * the total
+            number of processes
+
+        - **total_dataset_length** (`int`) -- Total length of the inner dataset across all processes.
+        """
+
+        def __init__(self, dataloader: DataLoaderShard, device: torch.device):
+            super().__init__(dataloader, device)
+            self._rng_types = self._loader.rng_types
+            self._loader.rng_types = None
+            self.device = device
+
+        def __iter__(self):
+            if self._rng_types is not None:
+                synchronize_rng_states(self._rng_types, self._loader.synchronized_generator)
+
+            return super().__iter__()
+
+        def set_epoch(self, epoch: int):
+            if hasattr(self.dataloader, "set_epoch"):
+                self.dataloader.set_epoch(epoch)
+
+        @property
+        def total_batch_size(self):
+            return self._loader.total_batch_size
+
+        @property
+        def total_dataset_length(self):
+            return self._loader.total_dataset_length
+
+        @property
+        def batch_sampler(self):
+            return self._loader.batch_sampler
+
+        @property
+        def dataloader(self):
+            return self._loader
+
+
+class DataLoaderDispatcher(DataLoaderAdapter, DataLoaderStateMixin):
+    """
+    Subclass of `DataLoaderAdapter` that will iterate and preprocess on process 0 only, then dispatch on each process
+    their part of the batch.
+
+    Args:
+        split_batches (`bool`, *optional*, defaults to `False`):
+            Whether the resulting `DataLoader` should split the batches of the original data loader across devices or
+            yield full batches (in which case it will yield batches starting at the `process_index`-th and advancing of
+            `num_processes` batches at each iteration). Another way to see this is that the observed batch size will be
+            the same as the initial `dataloader` if this option is set to `True`, the batch size of the initial
+            `dataloader` multiplied by `num_processes` otherwise. Setting this option to `True` requires that the batch
+            size of the `dataloader` is a round multiple of `batch_size`.
+        skip_batches (`int`, *optional*, defaults to 0):
+            The number of batches to skip at the beginning of an iteration.
+        use_stateful_dataloader (`bool`, *optional*, defaults to `False`):
+            Whether to have this class adapt `StatefulDataLoader` from `torchdata` instead of the regular `DataLoader`.
+
+    **Available attributes:**
+
+        - **total_batch_size** (`int`) -- Total batch size of the dataloader across all processes.
+            Equal to the original batch size when `split_batches=True`; otherwise the original batch size * the total
+            number of processes
+
+        - **total_dataset_length** (`int`) -- Total length of the inner dataset across all processes.
+    """
+
+    def __init__(
+        self,
+        dataset,
+        split_batches: bool = False,
+        skip_batches=0,
+        use_stateful_dataloader=False,
+        _drop_last: bool = False,
+        _non_blocking: bool = False,
+        slice_fn=None,
+        torch_device_mesh=None,
+        **kwargs,
+    ):
+        shuffle = False
+        from torch.utils.data.datapipes.iter.combinatorics import ShufflerIterDataPipe
+
+        # We need to save the shuffling state of the DataPipe
+        if isinstance(dataset, ShufflerIterDataPipe):
+            shuffle = dataset._shuffle_enabled
+        super().__init__(dataset, use_stateful_dataloader=use_stateful_dataloader, **kwargs)
+        self.split_batches = split_batches
+        if shuffle:
+            torch.utils.data.graph_settings.apply_shuffle_settings(dataset, shuffle=shuffle)
+
+        self.gradient_state = GradientState()
+        self.state = PartialState()
+        self._drop_last = _drop_last
+        self._non_blocking = _non_blocking
+        self.skip_batches = skip_batches
+        self.torch_device_mesh = torch_device_mesh
+
+        self.slice_fn = slice_tensors if slice_fn is None else slice_fn
+        self.iteration = 0
+
+        # if a device mesh is provided extract each dimension (dp, fsdp, tp)
+        # device mesh may hold any number of dimensions, however,
+        # below code is for targeted support for dp, fsdp and tp
+
+        # device mesh will be used only if there is tp involved
+        # or any multi-dimensional parallelism involving tp
+        # (dp, tp) (fsdp, tp) (dp, fsdp, tp)
+        # otherwise the default behaviour not using device mesh should be sufficient
+        # since multi dimensional parallelism devoid of tp would anyway need
+        # different batches for each process irrespective of dp or fsdp
+        self.submesh_tp = None
+        self.submesh_dp = None
+        self.submesh_fsdp = None
+        if self.torch_device_mesh and "tp" in self.torch_device_mesh.mesh_dim_names:
+            self.submesh_tp = self.torch_device_mesh["tp"]
+            if "dp" in self.torch_device_mesh.mesh_dim_names:
+                self.submesh_dp = self.torch_device_mesh["dp"]
+            if "fsdp" in self.torch_device_mesh.mesh_dim_names:
+                self.submesh_fsdp = self.torch_device_mesh["fsdp"]
+        if self.submesh_tp and (self.submesh_dp or self.submesh_fsdp):
+            raise ValueError("TP + (DP/FSDP) is not yet supported in dispatch mode")
+
+    def _fetch_batches(self, iterator):
+        batches, batch = None, None
+        # On process 0, we gather the batch to dispatch.
+        if self.state.process_index == 0:
+            # Procedure to support TP only is simpler
+            # since we want to dispatch the same batch of samples across all ranks
+            # this removes complexity of handling multiple tp rank groups when TP + DP
+            # combination is involved.
+
+            try:
+                # for TP case avoid using split_batches
+                # since it would mean that the dataloader should be spilling out
+                # duplicates of batches.
+                if self.split_batches:
+                    # One batch of the main iterator is dispatched and split.
+                    if self.submesh_tp:
+                        logger.warning(
+                            "Use of split_batches for TP would need the dataloader to produce duplicate batches,"
+                            "otherwise, use dispatch_batches=True instead."
+                        )
+                    self._update_state_dict()
+                    batch = next(iterator)
+                else:
+                    # num_processes batches of the main iterator are concatenated then dispatched and split.
+                    # We add the batches one by one so we have the remainder available when drop_last=False.
+                    batches = []
+                    if self.submesh_tp:
+                        # when tp, extract single batch and then replicate
+                        self._update_state_dict()
+                        batch = next(iterator)
+                        batches = [batch] * self.state.num_processes
+                    else:
+                        for _ in range(self.state.num_processes):
+                            self._update_state_dict()
+                            batches.append(next(iterator))
+                    try:
+                        batch = concatenate(batches, dim=0)
+                    except RuntimeError as e:
+                        raise RuntimeError(
+                            "You can't use batches of different size with `dispatch_batches=True` or when using an `IterableDataset`."
+                            "either pass `dispatch_batches=False` and have each process fetch its own batch "
+                            " or pass `split_batches=True`. By doing so, the main process will fetch a full batch and "
+                            "slice it into `num_processes` batches for each process."
+                        ) from e
+                # In both cases, we need to get the structure of the batch that we will broadcast on other
+                # processes to initialize the tensors with the right shape.
+                # data_structure, stop_iteration
+                batch_info = [get_data_structure(batch), False]
+            except StopIteration:
+                batch_info = [None, True]
+        else:
+            batch_info = [None, self._stop_iteration]
+        # This is inplace, so after this instruction, every process has the same `batch_info` as process 0.
+        broadcast_object_list(batch_info)
+        self._stop_iteration = batch_info[1]
+        if self._stop_iteration:
+            # If drop_last is False and split_batches is False, we may have a remainder to take care of.
+            if not self.split_batches and not self._drop_last:
+                if self.state.process_index == 0 and len(batches) > 0:
+                    batch = concatenate(batches, dim=0)
+                    batch_info = [get_data_structure(batch), False]
+                else:
+                    batch_info = [None, True]
+                broadcast_object_list(batch_info)
+        return batch, batch_info
+
+    def __iter__(self):
+        self.begin()
+        self.set_epoch(self.iteration)
+        main_iterator = None
+        if is_torch_version(">=", "2.0.1"):
+            # NOTE PyTorch DataLoader adds forward compatibilities for DataPipes, which broadcasts
+            # shared seed to all dist processes. Thus, we need to create iterator for all dist processes.
+            # But, we only iterate through the DataLoader on process 0.
+            main_iterator = self.base_dataloader.__iter__()
+        elif self.state.process_index == 0:
+            main_iterator = self.base_dataloader.__iter__()
+        stop_iteration = False
+        self._stop_iteration = False
+        first_batch = None
+        next_batch, next_batch_info = self._fetch_batches(main_iterator)
+        batch_index = 0
+        while not stop_iteration:
+            batch, batch_info = next_batch, next_batch_info
+
+            if self.state.process_index != 0:
+                # Initialize tensors on other processes than process 0.
+                batch = initialize_tensors(batch_info[0])
+            batch = send_to_device(batch, self.state.device, non_blocking=self._non_blocking)
+            # Broadcast the batch before splitting it.
+            batch = broadcast(batch, from_process=0)
+
+            if not self._drop_last and first_batch is None:
+                # We keep at least num processes elements of the first batch to be able to complete the last batch
+                first_batch = self.slice_fn(
+                    batch,
+                    slice(0, self.state.num_processes),
+                    process_index=self.state.process_index,
+                    num_processes=self.state.num_processes,
+                )
+
+            if batch is None:
+                raise ValueError(
+                    f"Batch does not contain any data (`{batch}`). At the end of all iterable data available before expected stop iteration."
+                )
+
+            observed_batch_size = find_batch_size(batch)
+            batch_size = observed_batch_size // self.state.num_processes
+
+            stop_iteration = self._stop_iteration
+            if not stop_iteration:
+                # We may still be at the end of the dataloader without knowing it yet: if there is nothing left in
+                # the dataloader since the number of batches is a round multiple of the number of processes.
+                next_batch, next_batch_info = self._fetch_batches(main_iterator)
+                # next_batch_info[0] is None when there are no more batches, otherwise we still need to process them.
+                if self._stop_iteration and next_batch_info[0] is None:
+                    stop_iteration = True
+
+            if not self._drop_last and stop_iteration and observed_batch_size % self.state.num_processes != 0:
+                # If the last batch is not complete, let's add the first batch to it.
+                batch = concatenate([batch, first_batch], dim=0)
+                # Batch size computation above is wrong, it's off by 1 so we fix it.
+                batch_size += 1
+
+            data_slice = slice(self.state.process_index * batch_size, (self.state.process_index + 1) * batch_size)
+            batch = self.slice_fn(
+                batch,
+                data_slice,
+                process_index=self.state.process_index,
+                num_processes=self.state.num_processes,
+            )
+
+            if stop_iteration:
+                self.end_of_dataloader = True
+                self._update_state_dict()
+                self.remainder = observed_batch_size
+            if batch_index >= self.skip_batches:
+                yield batch
+            batch_index += 1
+        self.iteration += 1
+        self.end()
+
+    def set_epoch(self, epoch: int):
+        # In case it is manually passed in, the user can set it to what they like
+        if self.iteration != epoch:
+            self.iteration = epoch
+        if hasattr(self.batch_sampler, "sampler") and hasattr(self.batch_sampler.sampler, "set_epoch"):
+            self.batch_sampler.sampler.set_epoch(epoch)
+        elif hasattr(self.dataset, "set_epoch"):
+            self.dataset.set_epoch(epoch)
+
+    def __len__(self):
+        whole_length = len(self.base_dataloader)
+        if self.split_batches:
+            return whole_length
+        elif self._drop_last:
+            return whole_length // self.state.num_processes
+        else:
+            return math.ceil(whole_length / self.state.num_processes)
+
+    def __reduce__(self):
+        """
+        Define the `__reduce__` method to ensure a `DataLoaderDispatcher` can be pickled and unpickled. This needs to
+        be explicitly defined since default pickling behavior is broken by `DataLoaderAdapter` messing with its
+        `__class__` member.
+        """
+        args = super().__reduce__()
+        return (DataLoaderDispatcher, *args[1:])
+
+    @property
+    def total_batch_size(self):
+        return (
+            self.dataset.batch_size if self.split_batches else (self.dataset.batch_size * self.dataset.num_processes)
+        )
+
+    @property
+    def total_dataset_length(self):
+        return len(self.dataset)
+
+    def get_sampler(self):
+        return get_sampler(self)
+
+    def set_sampler(self, sampler):
+        sampler_is_batch_sampler = isinstance(self.sampler, BatchSampler)
+        if sampler_is_batch_sampler:
+            self.sampler.sampler = sampler
+        else:
+            self.batch_sampler.sampler = sampler
+            if hasattr(self.batch_sampler, "batch_sampler"):
+                self.batch_sampler.batch_sampler.sampler = sampler
+
+
+def get_sampler(dataloader):
+    """
+    Get the sampler associated to the dataloader
+
+    Args:
+        dataloader (`torch.utils.data.dataloader.DataLoader`):
+            The data loader to split across several devices.
+    Returns:
+        `torch.utils.data.Sampler`: The sampler associated to the dataloader
+    """
+    sampler_is_batch_sampler = isinstance(dataloader.sampler, BatchSampler)
+    if sampler_is_batch_sampler:
+        sampler = getattr(dataloader.sampler, "sampler", None)
+    else:
+        sampler = getattr(dataloader.batch_sampler, "sampler", None)
+    return sampler
+
+
+def prepare_data_loader(
+    dataloader: DataLoader,
+    device: Optional[torch.device] = None,
+    num_processes: Optional[int] = None,
+    process_index: Optional[int] = None,
+    split_batches: bool = False,
+    put_on_device: bool = False,
+    rng_types: Optional[list[Union[str, RNGType]]] = None,
+    dispatch_batches: Optional[bool] = None,
+    even_batches: bool = True,
+    slice_fn_for_dispatch: Optional[Callable] = None,
+    use_seedable_sampler: bool = False,
+    data_seed: Optional[int] = None,
+    non_blocking: bool = False,
+    use_stateful_dataloader: bool = False,
+    torch_device_mesh=None,
+) -> DataLoader:
+    """
+    Wraps a PyTorch `DataLoader` to generate batches for one of the processes only.
+
+    Depending on the value of the `drop_last` attribute of the `dataloader` passed, it will either stop the iteration
+    at the first batch that would be too small / not present on all processes or loop with indices from the beginning.
+
+    Args:
+        dataloader (`torch.utils.data.dataloader.DataLoader`):
+            The data loader to split across several devices.
+        device (`torch.device`):
+            The target device for the returned `DataLoader`.
+        num_processes (`int`, *optional*):
+            The number of processes running concurrently. Will default to the value given by [`~state.PartialState`].
+        process_index (`int`, *optional*):
+            The index of the current process. Will default to the value given by [`~state.PartialState`].
+        split_batches (`bool`, *optional*, defaults to `False`):
+            Whether the resulting `DataLoader` should split the batches of the original data loader across devices or
+            yield full batches (in which case it will yield batches starting at the `process_index`-th and advancing of
+            `num_processes` batches at each iteration).
+
+            Another way to see this is that the observed batch size will be the same as the initial `dataloader` if
+            this option is set to `True`, the batch size of the initial `dataloader` multiplied by `num_processes`
+            otherwise.
+
+            Setting this option to `True` requires that the batch size of the `dataloader` is a round multiple of
+            `batch_size`.
+        put_on_device (`bool`, *optional*, defaults to `False`):
+            Whether or not to put the batches on `device` (only works if the batches are nested list, tuples or
+            dictionaries of tensors).
+        rng_types (list of `str` or [`~utils.RNGType`]):
+            The list of random number generators to synchronize at the beginning of each iteration. Should be one or
+            several of:
+
+            - `"torch"`: the base torch random number generator
+            - `"cuda"`: the CUDA random number generator (GPU only)
+            - `"xla"`: the XLA random number generator (TPU only)
+            - `"generator"`: the `torch.Generator` of the sampler (or batch sampler if there is no sampler in your
+              dataloader) or of the iterable dataset (if it exists) if the underlying dataset is of that type.
+
+        dispatch_batches (`bool`, *optional*):
+            If set to `True`, the dataloader prepared is only iterated through on the main process and then the batches
+            are split and broadcast to each process. Will default to `True` when the underlying dataset is an
+            `IterableDataset`, `False` otherwise.
+        even_batches (`bool`, *optional*, defaults to `True`):
+            If set to `True`, in cases where the total batch size across all processes does not exactly divide the
+            dataset, samples at the start of the dataset will be duplicated so the batch can be divided equally among
+            all workers.
+        slice_fn_for_dispatch (`Callable`, *optional*`):
+            If passed, this function will be used to slice tensors across `num_processes`. Will default to
+            [`~utils.slice_tensors`]. This argument is used only when `dispatch_batches` is set to `True` and will be
+            ignored otherwise.
+        use_seedable_sampler (`bool`, *optional*, defaults to `False`):
+            Whether to use the [`~data_loader.SeedableRandomSampler`] instead of a `RandomSampler` for better
+            reproducibility. Comes at a cost of potentially different performances due to different shuffling
+            algorithms but ensures results will be the *exact* same. Should be paired with `set_seed()` at every
+            `self.set_epoch`
+        data_seed (`int`, *optional*, defaults to `None`):
+            The seed to use for the underlying generator when using `use_seedable_sampler`. If `None`, the generator
+            will use the current default seed from torch.
+        non_blocking (`bool`, *optional*, defaults to `False`):
+            If set to `True`, dataloader will utilize non-blocking host-to-device transfers. If the dataloader has
+            `pin_memory` set to `True`, this will help to increase overlap between data transfer and computations.
+        use_stateful_dataloader (`bool`, *optional*, defaults to `False`):
+            "If set to true, the dataloader prepared by the Accelerator will be backed by "
+            "[torchdata.StatefulDataLoader](https://github.com/pytorch/data/tree/main/torchdata/stateful_dataloader).
+            This requires `torchdata` version 0.8.0 or higher that supports StatefulDataLoader to be installed."
+        torch_device_mesh (`torch.distributed.DeviceMesh`, *optional*, defaults to `None`):
+            PyTorch device mesh.
+
+
+    Returns:
+        `torch.utils.data.dataloader.DataLoader`: A new data loader that will yield the portion of the batches
+
+    <Tip warning={true}>
+
+    `BatchSampler`s with varying batch sizes are not enabled by default. To enable this behaviour, set `even_batches`
+    equal to `False`
+
+    </Tip>
+    """
+    if dispatch_batches is None:
+        if not put_on_device:
+            dispatch_batches = False
+        else:
+            dispatch_batches = isinstance(dataloader.dataset, IterableDataset)
+
+    if dispatch_batches and not put_on_device:
+        raise ValueError("Using `dispatch_batches=True` requires `put_on_device=True`.")
+    # Grab defaults from PartialState
+    state = PartialState()
+    if num_processes is None:
+        num_processes = state.num_processes
+
+    if process_index is None:
+        process_index = state.process_index
+
+    if torch_device_mesh:
+        if state.distributed_type == DistributedType.DEEPSPEED:
+            # In DeepSpeed, the optimizer sharing level in DP is determined by the config file.
+            # Only considers "dp" and "tp".
+            # Given a device mesh (dp, tp) = (2, 3):
+            # - From the data parallel perspective, ranks should be structured as: 0 0 0 1 1 1
+            # - Processes with the same DP rank will receive the same batch.
+            submesh_tp_size = 1
+            if "tp" in torch_device_mesh.mesh_dim_names:
+                submesh_tp_size = torch_device_mesh["tp"].size()
+            process_index = process_index // submesh_tp_size
+            num_processes = num_processes // submesh_tp_size
+        else:
+            # when device mesh is used, specifically with TP
+            # then there is need to update process_index and num_processes
+            # to bring in the effect of generating same batch across TP ranks
+            # and different batch across FSDP and DP ranks.
+            # Example:
+            # if device mesh is (dp,fsdp,tp) = (2, 2, 3)
+            # ranks would range from 0...11
+            # from data angle ranks should look like 0 0 0 1 1 1 2 2 2 3 3 3
+            # processes with same ranks/ids would receive the same batch
+            # for CP the same as TP applies
+            submesh_fsdp_size = 1
+            submesh_dp_size = 1
+            submesh_tp_size = 1
+            submesh_cp_size = 1
+            if "tp" in torch_device_mesh.mesh_dim_names:
+                submesh_tp_size = torch_device_mesh["tp"].size()
+            if "cp" in torch_device_mesh.mesh_dim_names:
+                submesh_cp_size = torch_device_mesh["cp"].size()
+            if "dp_replicate" in torch_device_mesh.mesh_dim_names:
+                submesh_dp_size = torch_device_mesh["dp_replicate"].size()
+            if "dp_shard" in torch_device_mesh.mesh_dim_names:
+                submesh_fsdp_size = torch_device_mesh["dp_shard"].size()
+            process_index = process_index // (submesh_tp_size * submesh_cp_size)
+            num_processes = submesh_fsdp_size * submesh_dp_size
+
+    # Sanity check
+    if split_batches:
+        if dataloader.batch_size is not None:
+            batch_size_for_check = dataloader.batch_size
+        else:
+            # For custom batch_sampler
+            if hasattr(dataloader.batch_sampler, "batch_size"):
+                batch_size_for_check = dataloader.batch_sampler.batch_size
+            else:
+                raise ValueError(
+                    "In order to use `split_batches==True` you must have a `batch_size` attribute either in the passed "
+                    "`dataloader` or `dataloader.batch_sampler` objects, and it has to return a natural number. "
+                    "Your `dataloader.batch_size` is None and `dataloader.batch_sampler` "
+                    f"(`{type(dataloader.batch_sampler)}`) does not have the `batch_size` attribute set."
+                )
+
+        if batch_size_for_check > 1 and batch_size_for_check % num_processes != 0:
+            raise ValueError(
+                f"To use a `DataLoader` in `split_batches` mode, the batch size ({dataloader.batch_size}) "
+                f"needs to be a round multiple of the number of processes ({num_processes})."
+            )
+
+    new_dataset = dataloader.dataset
+    # Iterable dataset doesn't like batch_sampler, but data_loader creates a default one for it
+    new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None
+    sampler_is_batch_sampler = isinstance(dataloader.sampler, BatchSampler)
+    synchronized_generator = None
+
+    sampler = get_sampler(dataloader)
+    if isinstance(sampler, RandomSampler) and use_seedable_sampler:
+        # When iterating through the dataloader during distributed processes
+        # we want to ensure that on each process we are iterating through the same
+        # samples in the same order if a seed is set. This requires a tweak
+        # to the `torch.utils.data.RandomSampler` class (if used).
+        sampler = SeedableRandomSampler(
+            data_source=sampler.data_source,
+            replacement=sampler.replacement,
+            num_samples=sampler._num_samples,
+            generator=getattr(
+                sampler,
+                "generator",
+                torch.Generator(device=torch.get_default_device() if hasattr(torch, "get_default_device") else "cpu"),
+            ),
+            data_seed=data_seed,
+        )
+
+    if isinstance(dataloader.sampler, RandomSampler) and state.distributed_type == DistributedType.XLA:
+        # isinstance(dataloader.sampler, RandomSampler) indicates the original dataloader has `shuffle` enabled.
+        generator = torch.Generator(
+            device=torch.get_default_device() if hasattr(torch, "get_default_device") else "cpu"
+        )
+        seed = int(torch.empty((), dtype=torch.int64).random_().item())
+        generator.manual_seed(seed)
+        dataloader.generator = generator
+        dataloader.sampler.generator = generator
+    # No change if no multiprocess
+    if (num_processes != 1 or state.distributed_type == DistributedType.MEGATRON_LM) and not dispatch_batches:
+        if is_datasets_available():
+            from datasets import IterableDataset as DatasetsIterableDataset
+        if (
+            is_datasets_available()
+            and isinstance(new_dataset, DatasetsIterableDataset)
+            and not split_batches
+            and new_dataset.n_shards > num_processes
+        ):
+            new_dataset = new_dataset.shard(num_shards=num_processes, index=process_index)
+        elif isinstance(new_dataset, IterableDataset):
+            if getattr(dataloader.dataset, "generator", None) is not None:
+                synchronized_generator = dataloader.dataset.generator
+            new_dataset = IterableDatasetShard(
+                new_dataset,
+                batch_size=dataloader.batch_size,
+                drop_last=dataloader.drop_last,
+                num_processes=num_processes,
+                process_index=process_index,
+                split_batches=split_batches,
+            )
+        else:
+            if not use_seedable_sampler and hasattr(sampler, "generator"):
+                if sampler.generator is None:
+                    sampler.generator = torch.Generator(
+                        device=torch.get_default_device() if hasattr(torch, "get_default_device") else "cpu"
+                    )
+                    seed = int(torch.empty((), dtype=torch.int64).random_().item())
+                    sampler.generator.manual_seed(seed)
+                synchronized_generator = sampler.generator
+            batch_sampler = dataloader.sampler if sampler_is_batch_sampler else dataloader.batch_sampler
+            new_batch_sampler = BatchSamplerShard(
+                batch_sampler,
+                num_processes=num_processes,
+                process_index=process_index,
+                split_batches=split_batches,
+                even_batches=even_batches,
+            )
+
+    # We ignore all of those since they are all dealt with by our new_batch_sampler
+    ignore_kwargs = [
+        "batch_size",
+        "shuffle",
+        "sampler",
+        "batch_sampler",
+        "drop_last",
+    ]
+
+    if rng_types is not None and synchronized_generator is None and "generator" in rng_types:
+        rng_types.remove("generator")
+
+    kwargs = {
+        k: getattr(dataloader, k, _PYTORCH_DATALOADER_KWARGS[k])
+        for k in _PYTORCH_DATALOADER_KWARGS
+        if k not in ignore_kwargs
+    }
+
+    # Need to provide batch_size as batch_sampler is None for Iterable dataset
+    if new_batch_sampler is None:
+        kwargs["drop_last"] = dataloader.drop_last
+        kwargs["batch_size"] = (
+            dataloader.batch_size // num_processes if split_batches and not dispatch_batches else dataloader.batch_size
+        )
+    if dispatch_batches:
+        kwargs.pop("generator")
+        dataloader = DataLoaderDispatcher(
+            new_dataset,
+            split_batches=split_batches,
+            batch_sampler=new_batch_sampler,
+            _drop_last=dataloader.drop_last,
+            _non_blocking=non_blocking,
+            slice_fn=slice_fn_for_dispatch,
+            use_stateful_dataloader=use_stateful_dataloader,
+            torch_device_mesh=torch_device_mesh,
+            **kwargs,
+        )
+    elif sampler_is_batch_sampler:
+        dataloader = DataLoaderShard(
+            new_dataset,
+            device=device if put_on_device and state.distributed_type != DistributedType.XLA else None,
+            sampler=new_batch_sampler,
+            batch_size=dataloader.batch_size,
+            rng_types=rng_types,
+            _drop_last=dataloader.drop_last,
+            _non_blocking=non_blocking,
+            synchronized_generator=synchronized_generator,
+            use_stateful_dataloader=use_stateful_dataloader,
+            **kwargs,
+        )
+    else:
+        dataloader = DataLoaderShard(
+            new_dataset,
+            device=device if put_on_device and state.distributed_type != DistributedType.XLA else None,
+            batch_sampler=new_batch_sampler,
+            rng_types=rng_types,
+            synchronized_generator=synchronized_generator,
+            _drop_last=dataloader.drop_last,
+            _non_blocking=non_blocking,
+            use_stateful_dataloader=use_stateful_dataloader,
+            **kwargs,
+        )
+
+    if isinstance(sampler, SeedableRandomSampler) and use_seedable_sampler:
+        dataloader.set_sampler(sampler)
+    if state.distributed_type == DistributedType.XLA:
+        return MpDeviceLoaderWrapper(dataloader, device)
+    return dataloader
+
+
+class SkipBatchSampler(BatchSampler):
+    """
+    A `torch.utils.data.BatchSampler` that skips the first `n` batches of another `torch.utils.data.BatchSampler`.
+    Should not be used if the original dataloader is a `StatefulDataLoader`.
+    """
+
+    def __init__(self, batch_sampler, skip_batches=0):
+        self.batch_sampler = batch_sampler
+        self.skip_batches = skip_batches
+
+    def __iter__(self):
+        for index, samples in enumerate(self.batch_sampler):
+            if index >= self.skip_batches:
+                yield samples
+
+    @property
+    def total_length(self):
+        return len(self.batch_sampler)
+
+    def __len__(self):
+        return len(self.batch_sampler) - self.skip_batches
+
+
+class SkipDataLoader(DataLoaderAdapter, DataLoaderStateMixin):
+    """
+    Subclass of a PyTorch `DataLoader` that will skip the first batches. Generally it's preferable to use
+    `skip_first_batches`/`torchdata.StatefulDataLoader` instead of this class.
+
+    Args:
+        dataset (`torch.utils.data.dataset.Dataset`):
+            The dataset to use to build this dataloader.
+        skip_batches (`int`, *optional*, defaults to 0):
+            The number of batches to skip at the beginning.
+        kwargs:
+            All other keyword arguments to pass to the regular `DataLoader` initialization.
+    """
+
+    def __init__(self, dataset, skip_batches=0, use_stateful_dataloader=False, **kwargs):
+        super().__init__(dataset, use_stateful_dataloader=use_stateful_dataloader, **kwargs)
+        self.skip_batches = skip_batches
+        self.gradient_state = GradientState()
+
+    def __iter__(self):
+        self.begin()
+        for index, batch in enumerate(self.base_dataloader.__iter__()):
+            if index >= self.skip_batches:
+                self._update_state_dict()
+                yield batch
+        self.end()
+
+    def __len__(self):
+        return len(self.base_dataloader) - self.skip_batches
+
+    def __reduce__(self):
+        """
+        Define the `__reduce__` method to ensure a `SkipDataLoader` can be pickled and unpickled. This needs to be
+        explicitly defined since default pickling behavior is broken by `DataLoaderAdapter` messing with its
+        `__class__` member.
+        """
+        args = super().__reduce__()
+        return (SkipDataLoader, *args[1:])
+
+
+def skip_first_batches(dataloader, num_batches=0):
+    """
+    Creates a `torch.utils.data.DataLoader` that will efficiently skip the first `num_batches`. Should not be used if
+    the original dataloader is a `StatefulDataLoader`.
+    """
+    state = PartialState()
+    if state.distributed_type == DistributedType.XLA:
+        device = dataloader.device
+        dataloader = dataloader.dataloader
+
+    dataset = dataloader.dataset
+    sampler_is_batch_sampler = False
+    if isinstance(dataset, IterableDataset):
+        new_batch_sampler = None
+    else:
+        sampler_is_batch_sampler = isinstance(dataloader.sampler, BatchSampler)
+        batch_sampler = dataloader.sampler if sampler_is_batch_sampler else dataloader.batch_sampler
+        new_batch_sampler = SkipBatchSampler(batch_sampler, skip_batches=num_batches)
+
+    # We ignore all of those since they are all dealt with by our new_batch_sampler
+    ignore_kwargs = [
+        "batch_size",
+        "shuffle",
+        "sampler",
+        "batch_sampler",
+        "drop_last",
+    ]
+
+    kwargs = {
+        k: getattr(dataloader, k, _PYTORCH_DATALOADER_KWARGS[k])
+        for k in _PYTORCH_DATALOADER_KWARGS
+        if k not in ignore_kwargs
+    }
+
+    # Need to provide batch_size as batch_sampler is None for Iterable dataset
+    if new_batch_sampler is None:
+        kwargs["drop_last"] = dataloader.drop_last
+        kwargs["batch_size"] = dataloader.batch_size
+
+    if isinstance(dataloader, DataLoaderDispatcher):
+        if new_batch_sampler is None:
+            # Need to manually skip batches in the dataloader
+            kwargs["skip_batches"] = num_batches
+        dataloader = DataLoaderDispatcher(
+            dataset,
+            split_batches=dataloader.split_batches,
+            batch_sampler=new_batch_sampler,
+            _drop_last=dataloader._drop_last,
+            **kwargs,
+        )
+    elif isinstance(dataloader, DataLoaderShard):
+        if new_batch_sampler is None:
+            # Need to manually skip batches in the dataloader
+            kwargs["skip_batches"] = num_batches
+        elif sampler_is_batch_sampler:
+            kwargs["sampler"] = new_batch_sampler
+            kwargs["batch_size"] = dataloader.batch_size
+        else:
+            kwargs["batch_sampler"] = new_batch_sampler
+        dataloader = DataLoaderShard(
+            dataset,
+            device=dataloader.device,
+            rng_types=dataloader.rng_types,
+            synchronized_generator=dataloader.synchronized_generator,
+            **kwargs,
+        )
+    else:
+        if new_batch_sampler is None:
+            # Need to manually skip batches in the dataloader
+            dataloader = SkipDataLoader(dataset, skip_batches=num_batches, **kwargs)
+        else:
+            dataloader = DataLoader(dataset, batch_sampler=new_batch_sampler, **kwargs)
+
+    if state.distributed_type == DistributedType.XLA:
+        dataloader = MpDeviceLoaderWrapper(dataloader, device)
+
+    return dataloader

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/hooks.py

@@ -0,0 +1,776 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+from collections.abc import Mapping
+from typing import Optional, Union
+
+import torch
+import torch.nn as nn
+
+from .state import PartialState
+from .utils import (
+    PrefixedDataset,
+    find_device,
+    named_module_tensors,
+    send_to_device,
+    set_module_tensor_to_device,
+)
+from .utils.imports import (
+    is_mlu_available,
+    is_musa_available,
+    is_npu_available,
+)
+from .utils.memory import clear_device_cache
+from .utils.modeling import get_non_persistent_buffers
+from .utils.other import recursive_getattr
+
+
+_accelerate_added_attributes = ["to", "cuda", "npu", "xpu", "mlu", "sdaa", "musa"]
+
+
+class ModelHook:
+    """
+    A hook that contains callbacks to be executed just before and after the forward method of a model. The difference
+    with PyTorch existing hooks is that they get passed along the kwargs.
+
+    Class attribute:
+    - **no_grad** (`bool`, *optional*, defaults to `False`) -- Whether or not to execute the actual forward pass under
+      the `torch.no_grad()` context manager.
+    """
+
+    no_grad = False
+
+    def init_hook(self, module):
+        """
+        To be executed when the hook is attached to the module.
+
+        Args:
+            module (`torch.nn.Module`): The module attached to this hook.
+        """
+        return module
+
+    def pre_forward(self, module, *args, **kwargs):
+        """
+        To be executed just before the forward method of the model.
+
+        Args:
+            module (`torch.nn.Module`): The module whose forward pass will be executed just after this event.
+            args (`Tuple[Any]`): The positional arguments passed to the module.
+            kwargs (`Dict[Str, Any]`): The keyword arguments passed to the module.
+
+        Returns:
+            `Tuple[Tuple[Any], Dict[Str, Any]]`: A tuple with the treated `args` and `kwargs`.
+        """
+        return args, kwargs
+
+    def post_forward(self, module, output):
+        """
+        To be executed just after the forward method of the model.
+
+        Args:
+            module (`torch.nn.Module`): The module whose forward pass been executed just before this event.
+            output (`Any`): The output of the module.
+
+        Returns:
+            `Any`: The processed `output`.
+        """
+        return output
+
+    def detach_hook(self, module):
+        """
+        To be executed when the hook is detached from a module.
+
+        Args:
+            module (`torch.nn.Module`): The module detached from this hook.
+        """
+        return module
+
+
+class SequentialHook(ModelHook):
+    """
+    A hook that can contain several hooks and iterates through them at each event.
+    """
+
+    def __init__(self, *hooks):
+        self.hooks = hooks
+
+    def init_hook(self, module):
+        for hook in self.hooks:
+            module = hook.init_hook(module)
+        return module
+
+    def pre_forward(self, module, *args, **kwargs):
+        for hook in self.hooks:
+            args, kwargs = hook.pre_forward(module, *args, **kwargs)
+        return args, kwargs
+
+    def post_forward(self, module, output):
+        for hook in self.hooks:
+            output = hook.post_forward(module, output)
+        return output
+
+    def detach_hook(self, module):
+        for hook in self.hooks:
+            module = hook.detach_hook(module)
+        return module
+
+
+def add_hook_to_module(module: nn.Module, hook: ModelHook, append: bool = False):
+    """
+    Adds a hook to a given module. This will rewrite the `forward` method of the module to include the hook, to remove
+    this behavior and restore the original `forward` method, use `remove_hook_from_module`.
+
+    <Tip warning={true}>
+
+    If the module already contains a hook, this will replace it with the new hook passed by default. To chain two hooks
+    together, pass `append=True`, so it chains the current and new hook into an instance of the `SequentialHook` class.
+
+    </Tip>
+
+    Args:
+        module (`torch.nn.Module`):
+            The module to attach a hook to.
+        hook (`ModelHook`):
+            The hook to attach.
+        append (`bool`, *optional*, defaults to `False`):
+            Whether the hook should be chained with an existing one (if module already contains a hook) or not.
+
+    Returns:
+        `torch.nn.Module`: The same module, with the hook attached (the module is modified in place, so the result can
+        be discarded).
+    """
+    if append and (getattr(module, "_hf_hook", None) is not None):
+        old_hook = module._hf_hook
+        remove_hook_from_module(module)
+        hook = SequentialHook(old_hook, hook)
+
+    if hasattr(module, "_hf_hook") and hasattr(module, "_old_forward"):
+        # If we already put some hook on this module, we replace it with the new one.
+        old_forward = module._old_forward
+    else:
+        old_forward = module.forward
+        module._old_forward = old_forward
+
+    module = hook.init_hook(module)
+    module._hf_hook = hook
+
+    def new_forward(module, *args, **kwargs):
+        args, kwargs = module._hf_hook.pre_forward(module, *args, **kwargs)
+        if module._hf_hook.no_grad:
+            with torch.no_grad():
+                output = module._old_forward(*args, **kwargs)
+        else:
+            output = module._old_forward(*args, **kwargs)
+        return module._hf_hook.post_forward(module, output)
+
+    # Overriding a GraphModuleImpl forward freezes the forward call and later modifications on the graph will fail.
+    # Reference: https://pytorch.slack.com/archives/C3PDTEV8E/p1705929610405409
+    if "GraphModuleImpl" in str(type(module)):
+        module.__class__.forward = functools.update_wrapper(functools.partial(new_forward, module), old_forward)
+    else:
+        module.forward = functools.update_wrapper(functools.partial(new_forward, module), old_forward)
+
+    return module
+
+
+def remove_hook_from_module(module: nn.Module, recurse=False):
+    """
+    Removes any hook attached to a module via `add_hook_to_module`.
+
+    Args:
+        module (`torch.nn.Module`): The module to attach a hook to.
+        recurse (`bool`, **optional**): Whether to remove the hooks recursively
+
+    Returns:
+        `torch.nn.Module`: The same module, with the hook detached (the module is modified in place, so the result can
+        be discarded).
+    """
+
+    if hasattr(module, "_hf_hook"):
+        module._hf_hook.detach_hook(module)
+        delattr(module, "_hf_hook")
+
+    if hasattr(module, "_old_forward"):
+        # Overriding a GraphModuleImpl forward freezes the forward call and later modifications on the graph will fail.
+        # Reference: https://pytorch.slack.com/archives/C3PDTEV8E/p1705929610405409
+        if "GraphModuleImpl" in str(type(module)):
+            module.__class__.forward = module._old_forward
+        else:
+            module.forward = module._old_forward
+        delattr(module, "_old_forward")
+
+    # Remove accelerate added warning hooks from dispatch_model
+    for attr in _accelerate_added_attributes:
+        module.__dict__.pop(attr, None)
+
+    if recurse:
+        for child in module.children():
+            remove_hook_from_module(child, recurse)
+
+    return module
+
+
+class AlignDevicesHook(ModelHook):
+    """
+    A generic `ModelHook` that ensures inputs and model weights are on the same device for the forward pass of the
+    associated module, potentially offloading the weights after the forward pass.
+
+    Args:
+        execution_device (`torch.device`, *optional*):
+            The device on which inputs and model weights should be placed before the forward pass.
+        offload (`bool`, *optional*, defaults to `False`):
+            Whether or not the weights should be offloaded after the forward pass.
+        io_same_device (`bool`, *optional*, defaults to `False`):
+            Whether or not the output should be placed on the same device as the input was.
+        weights_map (`Mapping[str, torch.Tensor]`, *optional*):
+            When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.
+        offload_buffers (`bool`, *optional*, defaults to `False`):
+            Whether or not to include the associated module's buffers when offloading.
+        place_submodules (`bool`, *optional*, defaults to `False`):
+            Whether to place the submodules on `execution_device` during the `init_hook` event.
+    """
+
+    def __init__(
+        self,
+        execution_device: Optional[Union[int, str, torch.device]] = None,
+        offload: bool = False,
+        io_same_device: bool = False,
+        weights_map: Optional[Mapping] = None,
+        offload_buffers: bool = False,
+        place_submodules: bool = False,
+        skip_keys: Optional[Union[str, list[str]]] = None,
+        tied_params_map: Optional[dict[int, dict[torch.device, torch.Tensor]]] = None,
+    ):
+        self.execution_device = execution_device
+        self.offload = offload
+        self.io_same_device = io_same_device
+        self.weights_map = weights_map
+        self.offload_buffers = offload_buffers
+        self.place_submodules = place_submodules
+        self.skip_keys = skip_keys
+
+        # Will contain the input device when `io_same_device=True`.
+        self.input_device = None
+        self.param_original_devices = {}
+        self.buffer_original_devices = {}
+        self.tied_params_names = set()
+
+        # The hook pre_forward/post_forward need to have knowledge of this dictionary, as with offloading we want to avoid duplicating memory
+        # for tied weights already loaded on the target execution device.
+        self.tied_params_map = tied_params_map
+
+    def __repr__(self):
+        return (
+            f"AlignDevicesHook(execution_device={self.execution_device}, offload={self.offload}, "
+            f"io_same_device={self.io_same_device}, offload_buffers={self.offload_buffers}, "
+            f"place_submodules={self.place_submodules}, skip_keys={repr(self.skip_keys)})"
+        )
+
+    def init_hook(self, module):
+        # In case the AlignDevicesHook is on meta device, ignore tied weights as data_ptr() is then always zero.
+        if self.execution_device == "meta" or self.execution_device == torch.device("meta"):
+            self.tied_params_map = None
+
+        if not self.offload and self.execution_device is not None:
+            for name, _ in named_module_tensors(module, recurse=self.place_submodules):
+                set_module_tensor_to_device(module, name, self.execution_device, tied_params_map=self.tied_params_map)
+        elif self.offload:
+            self.original_devices = {
+                name: param.device for name, param in named_module_tensors(module, recurse=self.place_submodules)
+            }
+            if self.weights_map is None:
+                self.weights_map = {
+                    name: param.to("cpu")
+                    for name, param in named_module_tensors(
+                        module, include_buffers=self.offload_buffers, recurse=self.place_submodules
+                    )
+                }
+            for name, _ in named_module_tensors(
+                module, include_buffers=self.offload_buffers, recurse=self.place_submodules, remove_non_persistent=True
+            ):
+                # When using disk offloading, we can not rely on `weights_map[name].data_ptr()` as the reference pointer,
+                # as we have no guarantee that safetensors' `file.get_tensor()` will always give the same pointer.
+                # As we have no reliable way to track the shared data pointer of tied weights in this case, we use tied_params_names: List[str]
+                # to add on the fly pointers to `tied_params_map` in the pre_forward call.
+                if (
+                    self.tied_params_map is not None
+                    and recursive_getattr(module, name).data_ptr() in self.tied_params_map
+                ):
+                    self.tied_params_names.add(name)
+
+                set_module_tensor_to_device(module, name, "meta")
+
+            if not self.offload_buffers and self.execution_device is not None:
+                for name, _ in module.named_buffers(recurse=self.place_submodules):
+                    set_module_tensor_to_device(
+                        module, name, self.execution_device, tied_params_map=self.tied_params_map
+                    )
+            elif self.offload_buffers and self.execution_device is not None:
+                for name in get_non_persistent_buffers(module, recurse=self.place_submodules):
+                    set_module_tensor_to_device(
+                        module, name, self.execution_device, tied_params_map=self.tied_params_map
+                    )
+
+        return module
+
+    def pre_forward(self, module, *args, **kwargs):
+        if self.io_same_device:
+            self.input_device = find_device([args, kwargs])
+        if self.offload:
+            self.tied_pointers_to_remove = set()
+
+            for name, _ in named_module_tensors(
+                module,
+                include_buffers=self.offload_buffers,
+                recurse=self.place_submodules,
+                remove_non_persistent=True,
+            ):
+                fp16_statistics = None
+                value = self.weights_map[name]
+                if "weight" in name and name.replace("weight", "SCB") in self.weights_map.keys():
+                    if value.dtype == torch.int8:
+                        fp16_statistics = self.weights_map[name.replace("weight", "SCB")]
+
+                # In case we are using offloading with tied weights, we need to keep track of the offloaded weights
+                # that are loaded on device at this point, as we will need to remove them as well from the dictionary
+                # self.tied_params_map in order to allow to free memory.
+                if name in self.tied_params_names and value.data_ptr() not in self.tied_params_map:
+                    self.tied_params_map[value.data_ptr()] = {}
+
+                if (
+                    value is not None
+                    and self.tied_params_map is not None
+                    and value.data_ptr() in self.tied_params_map
+                    and self.execution_device not in self.tied_params_map[value.data_ptr()]
+                ):
+                    self.tied_pointers_to_remove.add((value.data_ptr(), self.execution_device))
+
+                set_module_tensor_to_device(
+                    module,
+                    name,
+                    self.execution_device,
+                    value=value,
+                    fp16_statistics=fp16_statistics,
+                    tied_params_map=self.tied_params_map,
+                )
+
+        return send_to_device(args, self.execution_device), send_to_device(
+            kwargs, self.execution_device, skip_keys=self.skip_keys
+        )
+
+    def post_forward(self, module, output):
+        if self.offload:
+            for name, _ in named_module_tensors(
+                module,
+                include_buffers=self.offload_buffers,
+                recurse=self.place_submodules,
+                remove_non_persistent=True,
+            ):
+                set_module_tensor_to_device(module, name, "meta")
+                if type(module).__name__ == "Linear8bitLt":
+                    module.state.SCB = None
+                    module.state.CxB = None
+
+            # We may have loaded tied weights into self.tied_params_map (avoiding to load them several times in e.g. submodules): remove them from
+            # this dictionary to allow the garbage collector to do its job.
+            for value_pointer, device in self.tied_pointers_to_remove:
+                if isinstance(device, int):
+                    if is_npu_available():
+                        device = f"npu:{device}"
+                    elif is_mlu_available():
+                        device = f"mlu:{device}"
+                    elif is_musa_available():
+                        device = f"musa:{device}"
+                if device in self.tied_params_map[value_pointer]:
+                    del self.tied_params_map[value_pointer][device]
+            self.tied_pointers_to_remove = set()
+        if self.io_same_device and self.input_device is not None:
+            output = send_to_device(output, self.input_device, skip_keys=self.skip_keys)
+
+        return output
+
+    def detach_hook(self, module):
+        if self.offload:
+            for name, device in self.original_devices.items():
+                if device != torch.device("meta"):
+                    set_module_tensor_to_device(module, name, device, value=self.weights_map.get(name, None))
+        return module
+
+
+def attach_execution_device_hook(
+    module: torch.nn.Module,
+    execution_device: Union[int, str, torch.device],
+    skip_keys: Optional[Union[str, list[str]]] = None,
+    preload_module_classes: Optional[list[str]] = None,
+    tied_params_map: Optional[dict[int, dict[torch.device, torch.Tensor]]] = None,
+):
+    """
+    Recursively attaches `AlignDevicesHook` to all submodules of a given model to make sure they have the right
+    execution device
+
+    Args:
+        module (`torch.nn.Module`):
+            The module where we want to attach the hooks.
+        execution_device (`int`, `str` or `torch.device`):
+            The device on which inputs and model weights should be placed before the forward pass.
+        skip_keys (`str` or `List[str]`, *optional*):
+            A list of keys to ignore when moving inputs or outputs between devices.
+        preload_module_classes (`List[str]`, *optional*):
+            A list of classes whose instances should load all their weights (even in the submodules) at the beginning
+            of the forward. This should only be used for classes that have submodules which are registered but not
+            called directly during the forward, for instance if a `dense` linear layer is registered, but at forward,
+            `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly.
+        tied_params_map (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], *optional*, defaults to `None`):
+            A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution
+            device, this parameter is useful to reuse the first available pointer of a shared weight for all others,
+            instead of duplicating memory.
+    """
+    if not hasattr(module, "_hf_hook") and len(module.state_dict()) > 0:
+        add_hook_to_module(
+            module,
+            AlignDevicesHook(execution_device, skip_keys=skip_keys, tied_params_map=tied_params_map),
+        )
+
+    # Break the recursion if we get to a preload module.
+    if preload_module_classes is not None and module.__class__.__name__ in preload_module_classes:
+        return
+
+    for child in module.children():
+        attach_execution_device_hook(
+            child,
+            execution_device,
+            skip_keys=skip_keys,
+            preload_module_classes=preload_module_classes,
+            tied_params_map=tied_params_map,
+        )
+
+
+def attach_align_device_hook(
+    module: torch.nn.Module,
+    execution_device: Optional[torch.device] = None,
+    offload: bool = False,
+    weights_map: Optional[Mapping] = None,
+    offload_buffers: bool = False,
+    module_name: str = "",
+    skip_keys: Optional[Union[str, list[str]]] = None,
+    preload_module_classes: Optional[list[str]] = None,
+    tied_params_map: Optional[dict[int, dict[torch.device, torch.Tensor]]] = None,
+):
+    """
+    Recursively attaches `AlignDevicesHook` to all submodules of a given model that have direct parameters and/or
+    buffers.
+
+    Args:
+        module (`torch.nn.Module`):
+            The module where we want to attach the hooks.
+        execution_device (`torch.device`, *optional*):
+            The device on which inputs and model weights should be placed before the forward pass.
+        offload (`bool`, *optional*, defaults to `False`):
+            Whether or not the weights should be offloaded after the forward pass.
+        weights_map (`Mapping[str, torch.Tensor]`, *optional*):
+            When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.
+        offload_buffers (`bool`, *optional*, defaults to `False`):
+            Whether or not to include the associated module's buffers when offloading.
+        module_name (`str`, *optional*, defaults to `""`):
+            The name of the module.
+        skip_keys (`str` or `List[str]`, *optional*):
+            A list of keys to ignore when moving inputs or outputs between devices.
+        preload_module_classes (`List[str]`, *optional*):
+            A list of classes whose instances should load all their weights (even in the submodules) at the beginning
+            of the forward. This should only be used for classes that have submodules which are registered but not
+            called directly during the forward, for instance if a `dense` linear layer is registered, but at forward,
+            `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly.
+        tied_params_map (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], *optional*, defaults to `None`):
+            A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution
+            device, this parameter is useful to reuse the first available pointer of a shared weight for all others,
+            instead of duplicating memory.
+    """
+    # Attach the hook on this module if it has any direct tensor.
+    directs = named_module_tensors(module)
+    full_offload = (
+        offload and preload_module_classes is not None and module.__class__.__name__ in preload_module_classes
+    )
+
+    if len(list(directs)) > 0 or full_offload:
+        if weights_map is not None:
+            prefix = f"{module_name}." if len(module_name) > 0 else ""
+            prefixed_weights_map = PrefixedDataset(weights_map, prefix)
+        else:
+            prefixed_weights_map = None
+        hook = AlignDevicesHook(
+            execution_device=execution_device,
+            offload=offload,
+            weights_map=prefixed_weights_map,
+            offload_buffers=offload_buffers,
+            place_submodules=full_offload,
+            skip_keys=skip_keys,
+            tied_params_map=tied_params_map,
+        )
+        add_hook_to_module(module, hook, append=True)
+
+    # We stop the recursion in case we hit the full offload.
+    if full_offload:
+        return
+
+    # Recurse on all children of the module.
+    for child_name, child in module.named_children():
+        child_name = f"{module_name}.{child_name}" if len(module_name) > 0 else child_name
+        attach_align_device_hook(
+            child,
+            execution_device=execution_device,
+            offload=offload,
+            weights_map=weights_map,
+            offload_buffers=offload_buffers,
+            module_name=child_name,
+            preload_module_classes=preload_module_classes,
+            skip_keys=skip_keys,
+            tied_params_map=tied_params_map,
+        )
+
+
+def remove_hook_from_submodules(module: nn.Module):
+    """
+    Recursively removes all hooks attached on the submodules of a given model.
+
+    Args:
+        module (`torch.nn.Module`): The module on which to remove all hooks.
+    """
+    remove_hook_from_module(module)
+    for child in module.children():
+        remove_hook_from_submodules(child)
+
+
+def attach_align_device_hook_on_blocks(
+    module: nn.Module,
+    execution_device: Optional[Union[torch.device, dict[str, torch.device]]] = None,
+    offload: Union[bool, dict[str, bool]] = False,
+    weights_map: Optional[Mapping] = None,
+    offload_buffers: bool = False,
+    module_name: str = "",
+    skip_keys: Optional[Union[str, list[str]]] = None,
+    preload_module_classes: Optional[list[str]] = None,
+    tied_params_map: Optional[dict[int, dict[torch.device, torch.Tensor]]] = None,
+):
+    """
+    Attaches `AlignDevicesHook` to all blocks of a given model as needed.
+
+    Args:
+        module (`torch.nn.Module`):
+            The module where we want to attach the hooks.
+        execution_device (`torch.device` or `Dict[str, torch.device]`, *optional*):
+            The device on which inputs and model weights should be placed before the forward pass. It can be one device
+            for the whole module, or a dictionary mapping module name to device.
+        offload (`bool`, *optional*, defaults to `False`):
+            Whether or not the weights should be offloaded after the forward pass. It can be one boolean for the whole
+            module, or a dictionary mapping module name to boolean.
+        weights_map (`Mapping[str, torch.Tensor]`, *optional*):
+            When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.
+        offload_buffers (`bool`, *optional*, defaults to `False`):
+            Whether or not to include the associated module's buffers when offloading.
+        module_name (`str`, *optional*, defaults to `""`):
+            The name of the module.
+        skip_keys (`str` or `List[str]`, *optional*):
+            A list of keys to ignore when moving inputs or outputs between devices.
+        preload_module_classes (`List[str]`, *optional*):
+            A list of classes whose instances should load all their weights (even in the submodules) at the beginning
+            of the forward. This should only be used for classes that have submodules which are registered but not
+            called directly during the forward, for instance if a `dense` linear layer is registered, but at forward,
+            `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly.
+        tied_params_map (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], *optional*, defaults to `None`):
+            A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution
+            device, this parameter is useful to reuse the first available pointer of a shared weight for all others,
+            instead of duplicating memory.
+    """
+    # If one device and one offload, we've got one hook.
+    if not isinstance(execution_device, Mapping) and not isinstance(offload, dict):
+        if not offload:
+            hook = AlignDevicesHook(
+                execution_device=execution_device,
+                io_same_device=True,
+                skip_keys=skip_keys,
+                place_submodules=True,
+                tied_params_map=tied_params_map,
+            )
+            add_hook_to_module(module, hook)
+        else:
+            attach_align_device_hook(
+                module,
+                execution_device=execution_device,
+                offload=True,
+                weights_map=weights_map,
+                offload_buffers=offload_buffers,
+                module_name=module_name,
+                skip_keys=skip_keys,
+                tied_params_map=tied_params_map,
+            )
+        return
+
+    if not isinstance(execution_device, Mapping):
+        execution_device = {key: execution_device for key in offload.keys()}
+    if not isinstance(offload, Mapping):
+        offload = {key: offload for key in execution_device.keys()}
+
+    if module_name in execution_device and module_name in offload and not offload[module_name]:
+        hook = AlignDevicesHook(
+            execution_device=execution_device[module_name],
+            offload_buffers=offload_buffers,
+            io_same_device=(module_name == ""),
+            place_submodules=True,
+            skip_keys=skip_keys,
+            tied_params_map=tied_params_map,
+        )
+        add_hook_to_module(module, hook)
+        attach_execution_device_hook(
+            module, execution_device[module_name], skip_keys=skip_keys, tied_params_map=tied_params_map
+        )
+    elif module_name in execution_device and module_name in offload:
+        attach_align_device_hook(
+            module,
+            execution_device=execution_device[module_name],
+            offload=True,
+            weights_map=weights_map,
+            offload_buffers=offload_buffers,
+            module_name=module_name,
+            skip_keys=skip_keys,
+            preload_module_classes=preload_module_classes,
+            tied_params_map=tied_params_map,
+        )
+        if not hasattr(module, "_hf_hook"):
+            hook = AlignDevicesHook(
+                execution_device=execution_device[module_name],
+                io_same_device=(module_name == ""),
+                skip_keys=skip_keys,
+                tied_params_map=tied_params_map,
+            )
+            add_hook_to_module(module, hook)
+        attach_execution_device_hook(
+            module,
+            execution_device[module_name],
+            preload_module_classes=preload_module_classes,
+            skip_keys=skip_keys,
+            tied_params_map=tied_params_map,
+        )
+    elif module_name == "":
+        hook = AlignDevicesHook(
+            execution_device=execution_device.get(""),
+            io_same_device=True,
+            skip_keys=skip_keys,
+            tied_params_map=tied_params_map,
+        )
+        add_hook_to_module(module, hook)
+
+    for child_name, child in module.named_children():
+        child_name = f"{module_name}.{child_name}" if len(module_name) > 0 else child_name
+        attach_align_device_hook_on_blocks(
+            child,
+            execution_device=execution_device,
+            offload=offload,
+            weights_map=weights_map,
+            offload_buffers=offload_buffers,
+            module_name=child_name,
+            preload_module_classes=preload_module_classes,
+            skip_keys=skip_keys,
+            tied_params_map=tied_params_map,
+        )
+
+
+class CpuOffload(ModelHook):
+    """
+    Offloads a model on the CPU until its forward pass is called. The model will not be offloaded back to the CPU after
+    the forward, the user needs to call the `init_hook` method again for this.
+
+    Args:
+        execution_device(`str`, `int` or `torch.device`, *optional*):
+            The device on which the model should be executed. Will default to the MPS device if it's available, then
+            GPU 0 if there is a GPU, and finally to the CPU.
+        prev_module_hook (`UserCpuOffloadHook`, *optional*):
+            The hook sent back by [`cpu_offload_with_hook`] for a previous model in the pipeline you are running. If
+            passed, its offload method will be called just before the forward of the model to which this hook is
+            attached.
+    """
+
+    def __init__(
+        self,
+        execution_device: Optional[Union[str, int, torch.device]] = None,
+        prev_module_hook: Optional["UserCpuOffloadHook"] = None,
+    ):
+        self.prev_module_hook = prev_module_hook
+
+        self.execution_device = execution_device if execution_device is not None else PartialState().default_device
+
+    def init_hook(self, module):
+        return module.to("cpu")
+
+    def pre_forward(self, module, *args, **kwargs):
+        if self.prev_module_hook is not None and isinstance(self.prev_module_hook, UserCpuOffloadHook):
+            prev_module = self.prev_module_hook.model
+            prev_device = next(prev_module.parameters()).device
+
+            # Only offload the previous module if it is not already on CPU.
+            if prev_device != torch.device("cpu"):
+                self.prev_module_hook.offload()
+                clear_device_cache()
+
+        # If the current device is already the self.execution_device, we can skip the transfer.
+        current_device = next(module.parameters()).device
+        if current_device == self.execution_device:
+            return args, kwargs
+
+        module.to(self.execution_device)
+        return send_to_device(args, self.execution_device), send_to_device(kwargs, self.execution_device)
+
+
+class UserCpuOffloadHook:
+    """
+    A simple hook grouping a model and a `ModelHook`, which provides easy APIs for to call the init method of the hook
+    or remove it entirely.
+    """
+
+    def __init__(self, model, hook):
+        self.model = model
+        self.hook = hook
+
+    def offload(self):
+        self.hook.init_hook(self.model)
+
+    def remove(self):
+        remove_hook_from_module(self.model)
+
+
+class LayerwiseCastingHook(ModelHook):
+    r"""
+    A hook that casts the weights of a module to a high precision dtype for computation, and to a low precision dtype
+    for storage. This process may lead to quality loss in the output, but can significantly reduce the memory
+    footprint.
+    """
+
+    _is_stateful = False
+
+    def __init__(self, storage_dtype: torch.dtype, compute_dtype: torch.dtype, non_blocking: bool) -> None:
+        self.storage_dtype = storage_dtype
+        self.compute_dtype = compute_dtype
+        self.non_blocking = non_blocking
+
+    def init_hook(self, module: torch.nn.Module):
+        module.to(dtype=self.storage_dtype, non_blocking=self.non_blocking)
+        return module
+
+    def pre_forward(self, module: torch.nn.Module, *args, **kwargs):
+        module.to(dtype=self.compute_dtype, non_blocking=self.non_blocking)
+        return args, kwargs
+
+    def post_forward(self, module: torch.nn.Module, output):
+        module.to(dtype=self.storage_dtype, non_blocking=self.non_blocking)
+        return output

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/inference.py

@@ -0,0 +1,186 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+from types import MethodType
+from typing import Any, Optional, Union
+
+from .state import PartialState
+from .utils import (
+    calculate_maximum_sizes,
+    convert_bytes,
+    copy_tensor_to_devices,
+    ignorant_find_batch_size,
+    infer_auto_device_map,
+    is_pippy_available,
+    pad_input_tensors,
+    send_to_device,
+)
+
+
+def generate_device_map(
+    model, num_processes: int = 1, no_split_module_classes=None, max_memory: Optional[dict] = None
+):
+    """
+    Calculates the device map for `model` with an offset for PiPPy
+    """
+    if num_processes == 1:
+        return infer_auto_device_map(model, no_split_module_classes=no_split_module_classes, clean_result=False)
+    if max_memory is None:
+        model_size, shared = calculate_maximum_sizes(model)
+
+        # Split into `n` chunks for each GPU
+        memory = (model_size + shared[0]) / num_processes
+        memory = convert_bytes(memory)
+        value, ending = memory.split(" ")
+
+        # Add a chunk to deal with potential extra shared memory instances
+        memory = math.ceil(float(value)) * 1.1
+        memory = f"{memory} {ending}"
+        max_memory = {i: memory for i in range(num_processes)}
+    device_map = infer_auto_device_map(
+        model,
+        max_memory=max_memory,
+        no_split_module_classes=no_split_module_classes,
+        clean_result=False,
+    )
+    return device_map
+
+
+def find_pippy_batch_size(args, kwargs):
+    found_batch_size = None
+    if args is not None:
+        for arg in args:
+            found_batch_size = ignorant_find_batch_size(arg)
+            if found_batch_size is not None:
+                break
+    if kwargs is not None and found_batch_size is None:
+        for kwarg in kwargs.values():
+            found_batch_size = ignorant_find_batch_size(kwarg)
+            if found_batch_size is not None:
+                break
+    return found_batch_size
+
+
+def build_pipeline(model, split_points, args, kwargs, num_chunks):
+    """
+    Attaches the split points to the model based on `self.device_map` and generates a `PipelineStage`. Requires passing
+    in needed `args` and `kwargs` as the model needs on the CPU.
+
+    Users can pass in custom `num_chunks` as an optional hyper-parameter. By default will use
+    `AcceleratorState.num_processes`
+    """
+    # Note: We import here to reduce import time from general modules, and isolate outside dependencies
+    from torch.distributed.pipelining import ScheduleGPipe, SplitPoint, pipeline
+
+    # We need to annotate the split points in the model for PiPPy
+    state = PartialState()
+    split_spec = {split_point: SplitPoint.BEGINNING for split_point in split_points}
+    pipe = pipeline(
+        model,
+        mb_args=args,
+        mb_kwargs=kwargs,
+        split_spec=split_spec,
+    )
+    stage = pipe.build_stage(state.local_process_index, device=state.device)
+    schedule = ScheduleGPipe(stage, num_chunks)
+
+    return schedule
+
+
+def pippy_forward(forward, num_chunks, gather_output, *args, **kwargs):
+    state = PartialState()
+    output = None
+
+    if state.num_processes == 1:
+        output = forward(*args, **kwargs)
+    elif state.is_local_main_process:
+        found_batch_size = find_pippy_batch_size(args, kwargs)
+        if found_batch_size is None:
+            raise ValueError("Could not find batch size from args or kwargs")
+        else:
+            if found_batch_size != num_chunks:
+                args = pad_input_tensors(args, found_batch_size, num_chunks)
+                kwargs = pad_input_tensors(kwargs, found_batch_size, num_chunks)
+        forward(*args, **kwargs)
+    elif state.is_last_process:
+        output = forward()
+    else:
+        forward()
+    if gather_output:
+        # Each node will get a copy of the full output which is only on the last GPU
+        output = copy_tensor_to_devices(output)
+    return output
+
+
+def prepare_pippy(
+    model,
+    split_points: Optional[Union[str, list[str]]] = "auto",
+    no_split_module_classes: Optional[list[str]] = None,
+    example_args: Optional[tuple[Any]] = (),
+    example_kwargs: Optional[dict[str, Any]] = None,
+    num_chunks: Optional[int] = None,
+    gather_output: Optional[bool] = False,
+):
+    """
+    Wraps `model` for pipeline parallel inference.
+
+    Args:
+        model (`torch.nn.Module`):
+            A model we want to split for pipeline-parallel inference
+        split_points (`str` or `List[str]`, defaults to 'auto'):
+            How to generate the split points and chunk the model across each GPU. 'auto' will find the best balanced
+            split given any model. Should be a list of layer names in the model to split by otherwise.
+        no_split_module_classes (`List[str]`):
+            A list of class names for layers we don't want to be split.
+        example_args (tuple of model inputs):
+            The expected inputs for the model that uses order-based inputs for a *single process*. Recommended to use
+            this method if possible.
+        example_kwargs (dict of model inputs)
+            The expected inputs for the model that uses dictionary-based inputs for a *single process*. This is a
+            *highly* limiting structure that requires the same keys be present at *all* inference calls. Not
+            recommended unless the prior condition is true for all cases.
+        num_chunks (`int`, defaults to the number of available GPUs):
+            The number of different stages the Pipeline will have. By default it will assign one chunk per GPU, but
+            this can be tuned and played with. In general one should have num_chunks >= num_gpus.
+        gather_output (`bool`, defaults to `False`):
+            If `True`, the output from the last GPU (which holds the true outputs) is sent across to all GPUs.
+    """
+    if not is_pippy_available():
+        raise ImportError("Using `torch.distributed.pipelining` requires PyTorch 2.4.0 or later.")
+    state = PartialState()
+    example_args = send_to_device(example_args, "cpu")
+    example_kwargs = send_to_device(example_kwargs, "cpu")
+    if num_chunks is None:
+        num_chunks = state.num_processes
+    if split_points == "auto":
+        device_map = generate_device_map(model, num_chunks, no_split_module_classes=no_split_module_classes)
+        split_points = []
+        for i in range(1, num_chunks):
+            split_points.append(next(k for k, v in device_map.items() if v == i))
+    model.hf_split_points = split_points
+    stage = build_pipeline(model, split_points, example_args, example_kwargs, num_chunks)
+    model._original_forward = model.forward
+    model._original_call = model.__call__
+    model.pippy_stage = stage
+    model.hf_split_points = split_points
+
+    def forward(*args, **kwargs):
+        return pippy_forward(stage.step, num_chunks, gather_output, *args, **kwargs)
+
+    # To act like a decorator so that it can be popped when doing `extract_model_from_parallel`
+    # Note: creates an infinite recursion loop with `generate`
+    model_forward = MethodType(forward, model)
+    forward.__wrapped__ = model_forward
+    model.forward = forward
+    return model

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/launchers.py

@@ -0,0 +1,307 @@
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import tempfile
+
+import torch
+
+from .state import AcceleratorState, PartialState
+from .utils import (
+    PrecisionType,
+    PrepareForLaunch,
+    are_libraries_initialized,
+    check_cuda_p2p_ib_support,
+    get_current_device_type,
+    get_gpu_info,
+    is_mps_available,
+    is_torch_version,
+    patch_environment,
+)
+from .utils.constants import ELASTIC_LOG_LINE_PREFIX_TEMPLATE_PYTORCH_VERSION
+
+
+def test_launch():
+    "Verify a `PartialState` can be initialized."
+    _ = PartialState()
+
+
+def notebook_launcher(
+    function,
+    args=(),
+    num_processes=None,
+    mixed_precision="no",
+    use_port="29500",
+    master_addr="127.0.0.1",
+    node_rank=0,
+    num_nodes=1,
+    rdzv_backend="static",
+    rdzv_endpoint="",
+    rdzv_conf=None,
+    rdzv_id="none",
+    max_restarts=0,
+    monitor_interval=0.1,
+    log_line_prefix_template=None,
+):
+    """
+    Launches a training function, using several processes or multiple nodes if it's possible in the current environment
+    (TPU with multiple cores for instance).
+
+    <Tip warning={true}>
+
+    To use this function absolutely zero calls to a device must be made in the notebook session before calling. If any
+    have been made, you will need to restart the notebook and make sure no cells use any device capability.
+
+    Setting `ACCELERATE_DEBUG_MODE="1"` in your environment will run a test before truly launching to ensure that none
+    of those calls have been made.
+
+    </Tip>
+
+    Args:
+        function (`Callable`):
+            The training function to execute. If it accepts arguments, the first argument should be the index of the
+            process run.
+        args (`Tuple`):
+            Tuple of arguments to pass to the function (it will receive `*args`).
+        num_processes (`int`, *optional*):
+            The number of processes to use for training. Will default to 8 in Colab/Kaggle if a TPU is available, to
+            the number of devices available otherwise.
+        mixed_precision (`str`, *optional*, defaults to `"no"`):
+            If `fp16` or `bf16`, will use mixed precision training on multi-device.
+        use_port (`str`, *optional*, defaults to `"29500"`):
+            The port to use to communicate between processes when launching a multi-device training.
+        master_addr (`str`, *optional*, defaults to `"127.0.0.1"`):
+            The address to use for communication between processes.
+        node_rank (`int`, *optional*, defaults to 0):
+            The rank of the current node.
+        num_nodes (`int`, *optional*, defaults to 1):
+            The number of nodes to use for training.
+        rdzv_backend (`str`, *optional*, defaults to `"static"`):
+            The rendezvous method to use, such as 'static' (the default) or 'c10d'
+        rdzv_endpoint (`str`, *optional*, defaults to `""`):
+            The endpoint of the rdzv sync. storage.
+        rdzv_conf (`Dict`, *optional*, defaults to `None`):
+            Additional rendezvous configuration.
+        rdzv_id (`str`, *optional*, defaults to `"none"`):
+            The unique run id of the job.
+        max_restarts (`int`, *optional*, defaults to 0):
+            The maximum amount of restarts that elastic agent will conduct on workers before failure.
+        monitor_interval (`float`, *optional*, defaults to 0.1):
+            The interval in seconds that is used by the elastic_agent as a period of monitoring workers.
+        log_line_prefix_template (`str`, *optional*, defaults to `None`):
+            The prefix template for elastic launch logging. Available from PyTorch 2.2.0.
+
+    Example:
+
+    ```python
+    # Assume this is defined in a Jupyter Notebook on an instance with two devices
+    from accelerate import notebook_launcher
+
+
+    def train(*args):
+        # Your training function here
+        ...
+
+
+    notebook_launcher(train, args=(arg1, arg2), num_processes=2, mixed_precision="fp16")
+    ```
+    """
+    # Are we in a google colab or a Kaggle Kernel?
+    in_colab = False
+    in_kaggle = False
+    if any(key.startswith("KAGGLE") for key in os.environ.keys()):
+        in_kaggle = True
+    elif "IPython" in sys.modules:
+        in_colab = "google.colab" in str(sys.modules["IPython"].get_ipython())
+
+    try:
+        mixed_precision = PrecisionType(mixed_precision.lower())
+    except ValueError:
+        raise ValueError(
+            f"Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}."
+        )
+
+    if (in_colab or in_kaggle) and (
+        (os.environ.get("TPU_NAME", None) is not None) or (os.environ.get("PJRT_DEVICE", "") == "TPU")
+    ):
+        # TPU launch
+        import torch_xla.distributed.xla_multiprocessing as xmp
+
+        if len(AcceleratorState._shared_state) > 0:
+            raise ValueError(
+                "To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside "
+                "your training function. Restart your notebook and make sure no cells initializes an "
+                "`Accelerator`."
+            )
+
+        launcher = PrepareForLaunch(function, distributed_type="XLA")
+        print("Launching a training on TPU cores.")
+        xmp.spawn(launcher, args=args, start_method="fork")
+    elif in_colab and get_gpu_info()[1] < 2:
+        # No need for a distributed launch otherwise as it's either CPU or one GPU.
+        if torch.cuda.is_available():
+            print("Launching training on one GPU.")
+        else:
+            print("Launching training on one CPU.")
+        function(*args)
+    else:
+        if num_processes is None:
+            raise ValueError(
+                "You have to specify the number of devices you would like to use, add `num_processes=...` to your call."
+            )
+        if node_rank >= num_nodes:
+            raise ValueError("The node_rank must be less than the number of nodes.")
+        if num_processes > 1:
+            # Multi-device launch
+            from torch.distributed.launcher.api import LaunchConfig, elastic_launch
+            from torch.multiprocessing import start_processes
+            from torch.multiprocessing.spawn import ProcessRaisedException
+
+            if len(AcceleratorState._shared_state) > 0:
+                raise ValueError(
+                    "To launch a multi-device training from your notebook, the `Accelerator` should only be initialized "
+                    "inside your training function. Restart your notebook and make sure no cells initializes an "
+                    "`Accelerator`."
+                )
+            # Check for specific libraries known to initialize device that users constantly use
+            problematic_imports = are_libraries_initialized("bitsandbytes")
+            if len(problematic_imports) > 0:
+                err = (
+                    "Could not start distributed process. Libraries known to initialize device upon import have been "
+                    "imported already. Please keep these imports inside your training function to try and help with this:"
+                )
+                for lib_name in problematic_imports:
+                    err += f"\n\t* `{lib_name}`"
+                raise RuntimeError(err)
+
+            patched_env = dict(
+                nproc=num_processes,
+                node_rank=node_rank,
+                world_size=num_nodes * num_processes,
+                master_addr=master_addr,
+                master_port=use_port,
+                mixed_precision=mixed_precision,
+            )
+
+            # Check for CUDA P2P and IB issues
+            if not check_cuda_p2p_ib_support():
+                patched_env["nccl_p2p_disable"] = "1"
+                patched_env["nccl_ib_disable"] = "1"
+
+            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
+            # process here (the other ones will be set be the launcher).
+            with patch_environment(**patched_env):
+                # First dummy launch
+                # Determine device type without initializing any device (which would break fork)
+                device_type, distributed_type = get_current_device_type()
+                if os.environ.get("ACCELERATE_DEBUG_MODE", "false").lower() == "true":
+                    launcher = PrepareForLaunch(test_launch, distributed_type=distributed_type)
+                    try:
+                        start_processes(launcher, args=(), nprocs=num_processes, start_method="fork")
+                    except ProcessRaisedException as e:
+                        err = "An issue was found when verifying a stable environment for the notebook launcher."
+                        if f"Cannot re-initialize {device_type.upper()} in forked subprocess" in e.args[0]:
+                            raise RuntimeError(
+                                f"{err}"
+                                "This likely stems from an outside import causing issues once the `notebook_launcher()` is called. "
+                                "Please review your imports and test them when running the `notebook_launcher()` to identify "
+                                f"which one is problematic and causing {device_type.upper()} to be initialized."
+                            ) from e
+                        else:
+                            raise RuntimeError(f"{err} The following error was raised: {e}") from e
+                # Now the actual launch
+                launcher = PrepareForLaunch(function, distributed_type=distributed_type)
+                print(f"Launching training on {num_processes} {device_type.upper()}s.")
+                try:
+                    if rdzv_conf is None:
+                        rdzv_conf = {}
+                    if rdzv_backend == "static":
+                        rdzv_conf["rank"] = node_rank
+                        if not rdzv_endpoint:
+                            rdzv_endpoint = f"{master_addr}:{use_port}"
+                    launch_config_kwargs = dict(
+                        min_nodes=num_nodes,
+                        max_nodes=num_nodes,
+                        nproc_per_node=num_processes,
+                        run_id=rdzv_id,
+                        rdzv_endpoint=rdzv_endpoint,
+                        rdzv_backend=rdzv_backend,
+                        rdzv_configs=rdzv_conf,
+                        max_restarts=max_restarts,
+                        monitor_interval=monitor_interval,
+                        start_method="fork",
+                    )
+                    if is_torch_version(">=", ELASTIC_LOG_LINE_PREFIX_TEMPLATE_PYTORCH_VERSION):
+                        launch_config_kwargs["log_line_prefix_template"] = log_line_prefix_template
+                    elastic_launch(config=LaunchConfig(**launch_config_kwargs), entrypoint=function)(*args)
+                except ProcessRaisedException as e:
+                    if f"Cannot re-initialize {device_type.upper()} in forked subprocess" in e.args[0]:
+                        raise RuntimeError(
+                            f"{device_type.upper()} has been initialized before the `notebook_launcher` could create a forked subprocess. "
+                            "This likely stems from an outside import causing issues once the `notebook_launcher()` is called. "
+                            "Please review your imports and test them when running the `notebook_launcher()` to identify "
+                            f"which one is problematic and causing {device_type.upper()} to be initialized."
+                        ) from e
+                    else:
+                        raise RuntimeError(f"An issue was found when launching the training: {e}") from e
+
+        else:
+            # No need for a distributed launch otherwise as it's either CPU, GPU, XPU or MPS.
+            if is_mps_available():
+                os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+                print("Launching training on MPS.")
+            elif torch.cuda.is_available():
+                print("Launching training on one GPU.")
+            elif torch.xpu.is_available():
+                print("Launching training on one XPU.")
+            else:
+                print("Launching training on CPU.")
+            function(*args)
+
+
+def debug_launcher(function, args=(), num_processes=2):
+    """
+    Launches a training function using several processes on CPU for debugging purposes.
+
+    <Tip warning={true}>
+
+    This function is provided for internal testing and debugging, but it's not intended for real trainings. It will
+    only use the CPU.
+
+    </Tip>
+
+    Args:
+        function (`Callable`):
+            The training function to execute.
+        args (`Tuple`):
+            Tuple of arguments to pass to the function (it will receive `*args`).
+        num_processes (`int`, *optional*, defaults to 2):
+            The number of processes to use for training.
+    """
+    from torch.multiprocessing import start_processes
+
+    with tempfile.NamedTemporaryFile() as tmp_file:
+        # torch.distributed will expect a few environment variable to be here. We set the ones common to each
+        # process here (the other ones will be set be the launcher).
+        with patch_environment(
+            world_size=num_processes,
+            master_addr="127.0.0.1",
+            master_port="29500",
+            accelerate_mixed_precision="no",
+            accelerate_debug_rdv_file=tmp_file.name,
+            accelerate_use_cpu="yes",
+        ):
+            launcher = PrepareForLaunch(function, debug=True)
+            start_processes(launcher, args=args, nprocs=num_processes, start_method="fork")

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/local_sgd.py

@@ -0,0 +1,106 @@
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import torch
+
+from accelerate import Accelerator, DistributedType
+
+
+class LocalSGD:
+    """
+    A helper class to support local SGD on top of Accelerator. It simply runs a given number of updates independently
+    on each device, and averages model weights every K synchronization step.
+
+    It should be used only in the multi-GPU (or multi-CPU) setup without extensions such as DeepSpeed. In particular,
+    this is a simple implementation that cannot support scenarios such as model parallelism.
+
+
+    Although we are not aware of the true origins of this simple approach, the idea of local SGD is quite old and goes
+    back to at least:
+
+    Zhang, J., De Sa, C., Mitliagkas, I., & Ré, C. (2016). [Parallel SGD: When does averaging help?. arXiv preprint
+    arXiv:1606.07365.](https://huggingface.co/papers/1606.07365)
+
+    We credit the term Local SGD to the following paper (but there might be earlier references we are not aware of).
+
+    Stich, Sebastian Urban. ["Local SGD Converges Fast and Communicates Little." ICLR 2019-International Conference on
+    Learning Representations. No. CONF. 2019.](https://huggingface.co/papers/1805.09767)
+
+    """
+
+    def __enter__(self):
+        if self.enabled:
+            self.model_sync_obj = self.model.no_sync()
+            self.model_sync_obj.__enter__()
+
+        return self
+
+    def __exit__(self, type, value, tb):
+        if self.enabled:
+            # Average all models on exit
+            self._sync_and_avg_model_params()
+            self.model_sync_obj.__exit__(type, value, tb)
+
+    def __init__(self, accelerator: Accelerator, model: torch.nn.Module, local_sgd_steps: int, enabled: bool = True):
+        """
+        Constructor.
+
+        Args:
+            model (`torch.nn.Module):
+                The model whose parameters we need to average.
+            accelerator (`Accelerator`):
+                Accelerator object.
+            local_sgd_steps (`int`):
+                A number of local SGD steps (before model parameters are synchronized).
+            enabled (`bool):
+                Local SGD is disabled if this parameter set to `False`.
+        """
+        if accelerator.distributed_type not in [
+            DistributedType.NO,
+            DistributedType.MULTI_CPU,
+            DistributedType.MULTI_GPU,
+            DistributedType.MULTI_XPU,
+            DistributedType.MULTI_MLU,
+            DistributedType.MULTI_HPU,
+            DistributedType.MULTI_SDAA,
+            DistributedType.MULTI_MUSA,
+            DistributedType.MULTI_NPU,
+        ]:
+            raise NotImplementedError("LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)")
+        self.enabled = enabled and accelerator.distributed_type != DistributedType.NO
+        self.num_steps = 0
+        if self.enabled:
+            self.accelerator = accelerator
+            self.model = model
+            self.local_sgd_steps = local_sgd_steps
+
+    def step(self):
+        """
+        This function makes a "step" and synchronizes model parameters if necessary.
+        """
+        self.num_steps += 1
+        if not self.enabled:
+            return
+
+        if self.num_steps % self.local_sgd_steps == 0:
+            self._sync_and_avg_model_params()
+
+    def _sync_and_avg_model_params(self):
+        """
+        Synchronize + Average model parameters across all GPUs
+        """
+
+        self.accelerator.wait_for_everyone()
+        with self.accelerator.autocast():
+            for param in self.model.parameters():
+                param.data = self.accelerator.reduce(param.data, reduction="mean")

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/logging.py

@@ -0,0 +1,126 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+import logging
+import os
+from typing import Optional
+
+from .state import PartialState
+
+
+class MultiProcessAdapter(logging.LoggerAdapter):
+    """
+    An adapter to assist with logging in multiprocess.
+
+    `log` takes in an additional `main_process_only` kwarg, which dictates whether it should be called on all processes
+    or only the main executed one. Default is `main_process_only=True`.
+
+    Does not require an `Accelerator` object to be created first.
+    """
+
+    @staticmethod
+    def _should_log(main_process_only):
+        "Check if log should be performed"
+        state = PartialState()
+        return not main_process_only or (main_process_only and state.is_main_process)
+
+    def log(self, level, msg, *args, **kwargs):
+        """
+        Delegates logger call after checking if we should log.
+
+        Accepts a new kwarg of `main_process_only`, which will dictate whether it will be logged across all processes
+        or only the main executed one. Default is `True` if not passed
+
+        Also accepts "in_order", which if `True` makes the processes log one by one, in order. This is much easier to
+        read, but comes at the cost of sometimes needing to wait for the other processes. Default is `False` to not
+        break with the previous behavior.
+
+        `in_order` is ignored if `main_process_only` is passed.
+        """
+        if PartialState._shared_state == {}:
+            raise RuntimeError(
+                "You must initialize the accelerate state by calling either `PartialState()` or `Accelerator()` before using the logging utility."
+            )
+        main_process_only = kwargs.pop("main_process_only", True)
+        in_order = kwargs.pop("in_order", False)
+        # set `stacklevel` to exclude ourself in `Logger.findCaller()` while respecting user's choice
+        kwargs.setdefault("stacklevel", 2)
+
+        if self.isEnabledFor(level):
+            if self._should_log(main_process_only):
+                msg, kwargs = self.process(msg, kwargs)
+                self.logger.log(level, msg, *args, **kwargs)
+
+            elif in_order:
+                state = PartialState()
+                for i in range(state.num_processes):
+                    if i == state.process_index:
+                        msg, kwargs = self.process(msg, kwargs)
+                        self.logger.log(level, msg, *args, **kwargs)
+                    state.wait_for_everyone()
+
+    @functools.lru_cache(None)
+    def warning_once(self, *args, **kwargs):
+        """
+        This method is identical to `logger.warning()`, but will emit the warning with the same message only once
+
+        Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the
+        cache. The assumption here is that all warning messages are unique across the code. If they aren't then need to
+        switch to another type of cache that includes the caller frame information in the hashing function.
+        """
+        self.warning(*args, **kwargs)
+
+
+def get_logger(name: str, log_level: Optional[str] = None):
+    """
+    Returns a `logging.Logger` for `name` that can handle multiprocessing.
+
+    If a log should be called on all processes, pass `main_process_only=False` If a log should be called on all
+    processes and in order, also pass `in_order=True`
+
+    Args:
+        name (`str`):
+            The name for the logger, such as `__file__`
+        log_level (`str`, *optional*):
+            The log level to use. If not passed, will default to the `LOG_LEVEL` environment variable, or `INFO` if not
+
+    Example:
+
+    ```python
+    >>> from accelerate.logging import get_logger
+    >>> from accelerate import Accelerator
+
+    >>> logger = get_logger(__name__)
+
+    >>> accelerator = Accelerator()
+    >>> logger.info("My log", main_process_only=False)
+    >>> logger.debug("My log", main_process_only=True)
+
+    >>> logger = get_logger(__name__, log_level="DEBUG")
+    >>> logger.info("My log")
+    >>> logger.debug("My second log")
+
+    >>> array = ["a", "b", "c", "d"]
+    >>> letter_at_rank = array[accelerator.process_index]
+    >>> logger.info(letter_at_rank, in_order=True)
+    ```
+    """
+    if log_level is None:
+        log_level = os.environ.get("ACCELERATE_LOG_LEVEL", None)
+    logger = logging.getLogger(name)
+    if log_level is not None:
+        logger.setLevel(log_level.upper())
+        logger.root.setLevel(log_level.upper())
+    return MultiProcessAdapter(logger, {})

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/memory_utils.py

@@ -0,0 +1,22 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+
+
+warnings.warn(
+    "memory_utils has been reorganized to utils.memory. Import `find_executable_batchsize` from the main `__init__`: "
+    "`from accelerate import find_executable_batch_size` to avoid this warning.",
+    FutureWarning,
+)

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/optimizer.py

@@ -0,0 +1,213 @@
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+
+import torch
+
+from .state import AcceleratorState, GradientState
+from .utils import DistributedType, honor_type, is_lomo_available, is_torch_xla_available
+
+
+if is_torch_xla_available():
+    import torch_xla.core.xla_model as xm
+    import torch_xla.runtime as xr
+
+
+def move_to_device(state, device):
+    if isinstance(state, (list, tuple)):
+        return honor_type(state, (move_to_device(t, device) for t in state))
+    elif isinstance(state, dict):
+        return type(state)({k: move_to_device(v, device) for k, v in state.items()})
+    elif isinstance(state, torch.Tensor):
+        return state.to(device)
+    return state
+
+
+class AcceleratedOptimizer(torch.optim.Optimizer):
+    """
+    Internal wrapper around a torch optimizer.
+
+    Conditionally will perform `step` and `zero_grad` if gradients should be synchronized when performing gradient
+    accumulation.
+
+    Args:
+        optimizer (`torch.optim.optimizer.Optimizer`):
+            The optimizer to wrap.
+        device_placement (`bool`, *optional*, defaults to `True`):
+            Whether or not the optimizer should handle device placement. If so, it will place the state dictionary of
+            `optimizer` on the right device.
+        scaler (`torch.amp.GradScaler` or `torch.cuda.amp.GradScaler`, *optional*):
+            The scaler to use in the step function if training with mixed precision.
+    """
+
+    def __init__(self, optimizer, device_placement=True, scaler=None):
+        self.optimizer = optimizer
+        self.scaler = scaler
+        self.accelerator_state = AcceleratorState()
+        self.gradient_state = GradientState()
+        self.device_placement = device_placement
+        self._is_overflow = False
+
+        if self.scaler is not None:
+            self._accelerate_step_called = False
+            self._optimizer_original_step_method = self.optimizer.step
+            self._optimizer_patched_step_method = patch_optimizer_step(self, self.optimizer.step)
+
+        # Handle device placement
+        if device_placement:
+            state_dict = self.optimizer.state_dict()
+            if self.accelerator_state.distributed_type == DistributedType.XLA:
+                xm.send_cpu_data_to_device(state_dict, self.accelerator_state.device)
+            else:
+                state_dict = move_to_device(state_dict, self.accelerator_state.device)
+            self.optimizer.load_state_dict(state_dict)
+
+    @property
+    def state(self):
+        return self.optimizer.state
+
+    @state.setter
+    def state(self, state):
+        self.optimizer.state = state
+
+    @property
+    def param_groups(self):
+        return self.optimizer.param_groups
+
+    @param_groups.setter
+    def param_groups(self, param_groups):
+        self.optimizer.param_groups = param_groups
+
+    @property
+    def defaults(self):
+        return self.optimizer.defaults
+
+    @defaults.setter
+    def defaults(self, defaults):
+        self.optimizer.defaults = defaults
+
+    def add_param_group(self, param_group):
+        self.optimizer.add_param_group(param_group)
+
+    def load_state_dict(self, state_dict):
+        if self.accelerator_state.distributed_type == DistributedType.XLA and self.device_placement:
+            xm.send_cpu_data_to_device(state_dict, self.accelerator_state.device)
+        self.optimizer.load_state_dict(state_dict)
+
+    def state_dict(self):
+        return self.optimizer.state_dict()
+
+    def zero_grad(self, set_to_none=None):
+        if self.gradient_state.sync_gradients:
+            accept_arg = "set_to_none" in inspect.signature(self.optimizer.zero_grad).parameters
+            if accept_arg:
+                if set_to_none is None:
+                    set_to_none = True
+                self.optimizer.zero_grad(set_to_none=set_to_none)
+            else:
+                if set_to_none is not None:
+                    raise ValueError("`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.")
+                self.optimizer.zero_grad()
+
+    def train(self):
+        """
+        Sets the optimizer to "train" mode. Useful for optimizers like `schedule_free`
+        """
+        if hasattr(self.optimizer, "train") and callable(self.optimizer.train):
+            self.optimizer.train()
+        elif (
+            hasattr(self.optimizer, "optimizer")
+            and hasattr(self.optimizer.optimizer, "train")
+            and callable(self.optimizer.optimizer.train)
+        ):
+            # the deepspeed optimizer further wraps the optimizer
+            self.optimizer.optimizer.train()
+
+    def eval(self):
+        """
+        Sets the optimizer to "eval" mode. Useful for optimizers like `schedule_free`
+        """
+        if hasattr(self.optimizer, "eval") and callable(self.optimizer.eval):
+            self.optimizer.eval()
+
+    def step(self, closure=None):
+        if is_lomo_available():
+            from lomo_optim import AdaLomo, Lomo
+
+        if (
+            not self.gradient_state.is_xla_gradients_synced
+            and self.accelerator_state.distributed_type == DistributedType.XLA
+        ):
+            gradients = xm._fetch_gradients(self.optimizer)
+            xm.all_reduce("sum", gradients, scale=1.0 / xr.world_size())
+            self.gradient_state.is_xla_gradients_synced = True
+
+        if is_lomo_available():
+            #  `step` should be a no-op for LOMO optimizers.
+            if isinstance(self.optimizer, (Lomo, AdaLomo)):
+                return
+
+        if self.gradient_state.sync_gradients:
+            if self.scaler is not None:
+                self.optimizer.step = self._optimizer_patched_step_method
+
+                self.scaler.step(self.optimizer, closure)
+                self.scaler.update()
+
+                if not self._accelerate_step_called:
+                    # If the optimizer step was skipped, gradient overflow was detected.
+                    self._is_overflow = True
+                else:
+                    self._is_overflow = False
+                # Reset the step method to the original one
+                self.optimizer.step = self._optimizer_original_step_method
+                # Reset the indicator
+                self._accelerate_step_called = False
+            else:
+                self.optimizer.step(closure)
+        if self.accelerator_state.distributed_type == DistributedType.XLA:
+            self.gradient_state.is_xla_gradients_synced = False
+
+    def _switch_parameters(self, parameters_map):
+        for param_group in self.optimizer.param_groups:
+            param_group["params"] = [parameters_map.get(p, p) for p in param_group["params"]]
+
+    @property
+    def step_was_skipped(self):
+        """Whether or not the optimizer step was skipped."""
+        return self._is_overflow
+
+    def __getstate__(self):
+        _ignored_keys = [
+            "_accelerate_step_called",
+            "_optimizer_original_step_method",
+            "_optimizer_patched_step_method",
+        ]
+        return {k: v for k, v in self.__dict__.items() if k not in _ignored_keys}
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+        if self.scaler is not None:
+            self._accelerate_step_called = False
+            self._optimizer_original_step_method = self.optimizer.step
+            self._optimizer_patched_step_method = patch_optimizer_step(self, self.optimizer.step)
+
+
+def patch_optimizer_step(accelerated_optimizer: AcceleratedOptimizer, method):
+    def patched_step(*args, **kwargs):
+        accelerated_optimizer._accelerate_step_called = True
+        return method(*args, **kwargs)
+
+    return patched_step

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/parallelism_config.py

@@ -0,0 +1,378 @@
+#
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import warnings
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Literal, Optional, Union
+
+from accelerate.utils.dataclasses import (
+    DeepSpeedSequenceParallelConfig,
+    DistributedType,
+    TorchContextParallelConfig,
+    TorchTensorParallelConfig,
+)
+from accelerate.utils.versions import is_torch_version
+
+
+if TYPE_CHECKING:
+    from accelerate import Accelerator
+
+
+@dataclass
+class ParallelismConfig:
+    """
+    A dataclass to configure parallelisms applied to the model. Inspired by torchtitan's `ParallelDims`
+    https://github.com/pytorch/torchtitan/blob/main/torchtitan/distributed/parallel_dims.py
+
+    Args:
+        dp_replicate_size (`int`, defaults to `1`):
+            The size of the data parallel group. If `dp_replicate_size` is set to 1, the data parallel replication
+            group will not be used.
+        dp_shard_size (`int`, defaults to `1`):
+            The size of the model shard group. If `dp_replicate_size > 1` and `tp_size > 1`, `dp_shard_size` must also
+            be greater than 1, as composing DDP + TP is currently not supported.
+        tp_size (`int`, defaults to `1`):
+            The size of the tensor parallel group. If `tp_size` is set to `1`, the tensor parallel group will not be
+            used.
+        tp_handler (`~utils.TorchTensorParallelConfig`, defaults to `None`):
+            The handler for the tensor parallel group.
+        cp_size (`int`, defaults to `1`):
+            The size of the context parallel group. Currently not supported, but reserved for future use and enabled
+            for downstream libraries.
+        cp_backend (`str`, defaults to `torch`):
+            Which CP backend to use: `torch` (FSDP2)
+        sp_size (`int`, defaults to `1`):
+            The size of the sequence parallel group.
+        sp_backend (`str`, defaults to `deepspeed`):
+            Which SP backend to use:`deepspeed` (ALST/Ulysses)
+
+    You may obtain different distributed data parallel paradigms by configuring `dp_replicate_size` and `dp_shard_size`
+    together:
+        - `dp_replicate_size == 1` and `dp_shard_size > 1`, we obtain Fully Sharded Data Parallel (FSDP).
+        - `dp_replicate_size > 1` and `dp_shard_size > 1`, we obtain Hybrid Sharded Data Parallel (HSDP).
+        - `dp_replicate_size > 1` and `dp_shard_size == 1` is an invalid configuration, to use pure DP, use
+          `DistributedDataParallelKwargs` instead.
+
+    """
+
+    dp_replicate_size: Optional[int] = None
+    dp_shard_size: Optional[int] = None
+    tp_size: Optional[int] = None
+    cp_size: Optional[int] = None
+    cp_backend: Literal["torch"] = None
+    sp_size: Optional[int] = None
+    sp_backend: Literal["deepspeed"] = None
+
+    # we use Union because we might support other x parallel plugins (i.e. deepspeed, etc)
+    tp_handler: Union[None, TorchTensorParallelConfig] = None
+    cp_handler: Union[None, TorchContextParallelConfig] = None
+    sp_handler: Union[None, DeepSpeedSequenceParallelConfig] = None
+
+    device_mesh = None
+
+    def __repr__(self):
+        return (
+            "ParallelismConfig(\n "
+            f"\tdp_replicate_size={self.dp_replicate_size},\n"
+            f"\tdp_shard_size={self.dp_shard_size},\n"
+            f"\ttp_size={self.tp_size},\n"
+            f"\tcp_size={self.cp_size},\n"
+            f"\tcp_backend={self.cp_backend},\n"
+            f"\tsp_size={self.sp_size},\n"
+            f"\tsp_backend={self.sp_backend},\n"
+            f"\ttotal_size={self.total_size}\n"
+            f"\ttp_handler={self.tp_handler},\n"
+            f"\tcp_handler={self.cp_handler})\n"
+        )
+
+    def to_json(self):
+        import copy
+
+        _non_serializable_fields = ["device_mesh"]
+
+        copy.deepcopy(
+            {
+                k: copy.deepcopy(v.__dict__) if hasattr(v, "__dict__") else v
+                for k, v in self.__dict__.items()
+                if k not in _non_serializable_fields
+            }
+        )
+
+    @property
+    def dp_dim_names(self):
+        """Names of enabled dimensions across which data parallelism is applied."""
+        dims = []
+        if self.dp_replicate_enabled:
+            dims += ["dp_replicate"]
+        if self.dp_shard_enabled:
+            dims += ["dp_shard"]
+        return dims
+
+    @property
+    def non_dp_dim_names(self):
+        """Names of enabled dimensions which will receive the same batch (non-data parallel dimensions)."""
+        dims = []
+        if self.tp_enabled:
+            dims += ["tp"]
+        if self.cp_enabled:
+            dims += ["cp"]
+        if self.sp_enabled:
+            dims += ["sp"]
+        return dims
+
+    @property
+    def dp_shard_cp_dim_names(self):
+        """Names of enabled dimensions which will be flattened into a joint mesh across which is model sharded in FSDP."""
+        dims = []
+        if self.dp_shard_enabled:
+            dims += ["dp_shard"]
+        if self.cp_enabled:
+            dims += ["cp"]
+        return dims
+
+    @property
+    def dp_cp_dim_names(self):
+        """Names of enabled dimensions across which loss should be averaged"""
+        dims = []
+        if self.dp_replicate_enabled:
+            dims += ["dp_replicate"]
+        if self.dp_shard_enabled:
+            dims += ["dp_shard"]
+        if self.cp_enabled:
+            dims += ["cp"]
+        return dims
+
+    @property
+    def fsdp_dim_names(self):
+        """Names of enabled dimensions across which FSDP is applied, including data parallel replication."""
+        dims = []
+        if self.dp_replicate_enabled:
+            dims += ["dp_replicate"]
+        dims += ["dp_shard_cp"]
+        return dims
+
+    @property
+    def total_size(self):
+        """The total size of the parallelism configuration, which is the product of all sizes."""
+        return self.dp_replicate_size * self.dp_shard_size * self.tp_size * self.cp_size * self.sp_size
+
+    @property
+    def non_data_parallel_size(self):
+        """The size of the non-data parallel dimensions, which is the product of tensor and context parallel sizes."""
+        return self.tp_size * self.cp_size * self.sp_size
+
+    @property
+    def data_parallel_size(self):
+        """The size of the data parallel dimensions, which is the product of data parallel replication and"""
+        return self.dp_replicate_size * self.dp_shard_size
+
+    @property
+    def dp_replicate_enabled(self):
+        """True if data parallel replication is enabled, i.e. `dp_replicate_size > 1`."""
+        return self.dp_replicate_size > 1
+
+    @property
+    def dp_shard_enabled(self):
+        """True if data parallel sharding is enabled, i.e. `dp_shard_size > 1`."""
+        return self.dp_shard_size > 1
+
+    @property
+    def tp_enabled(self):
+        """True if tensor parallelism is enabled, i.e. `tp_size > 1`."""
+        return self.tp_size > 1
+
+    @property
+    def cp_enabled(self):
+        """True if context parallelism is enabled, i.e. `cp_size > 1`."""
+        return self.cp_size > 1
+
+    @property
+    def sp_enabled(self):
+        """True if context parallelism is enabled, i.e. `sp_size > 1`."""
+        return self.sp_size > 1
+
+    @property
+    def active_mesh_dims(self):
+        """Names of all active mesh dimensions."""
+        return self.dp_dim_names + self.non_dp_dim_names
+
+    def build_device_mesh(self, device_type: str):
+        """Builds a device mesh for the given device type based on the parallelism configuration.
+        This method will also create required joint meshes (e.g. `dp_shard_cp`, `dp_cp`, `dp`).
+
+        Args:
+            device_type (`str`): The type of device for which to build the mesh, e
+        """
+        if is_torch_version(">=", "2.2.0"):
+            from torch.distributed.device_mesh import init_device_mesh
+        else:
+            raise RuntimeError("Building a device_mesh requires to have torch>=2.2.0")
+
+        mesh = self._get_mesh()
+        if len(mesh) == 0:
+            return None
+        mesh_dim_names, mesh_shape = mesh
+        device_mesh = init_device_mesh(
+            device_type,
+            mesh_shape,
+            mesh_dim_names=mesh_dim_names,
+        )
+        if self.dp_dim_names:
+            device_mesh[self.dp_dim_names]._flatten("dp")
+        if self.dp_shard_cp_dim_names:
+            device_mesh[self.dp_shard_cp_dim_names]._flatten("dp_shard_cp")
+        if self.dp_cp_dim_names:
+            device_mesh[self.dp_cp_dim_names]._flatten("dp_cp")
+
+        return device_mesh
+
+    def get_device_mesh(self, device_type: Optional[str] = None):
+        if self.device_mesh is None:
+            if device_type is not None:
+                self.device_mesh = self.build_device_mesh(device_type)
+            else:
+                raise ("You need to pass a device_type e.g cuda to build the device mesh")
+        else:
+            if device_type is not None:
+                if self.device_mesh.device_type != device_type:
+                    raise ValueError(
+                        f"The device_mesh is already created with device type {self.device_mesh.device_type}. However, you are trying to get a device mesh with device_type {device_type}. Please check if you correctly initialized your device_mesh"
+                    )
+        return self.device_mesh
+
+    def _get_mesh(self) -> tuple[tuple[int, ...], tuple[str, ...]]:
+        """Generate mesh shape and dimension names for torch.distributed.init_device_mesh()."""
+
+        # Build mesh dimensions dictionary
+        mesh_dims = {parallelism: self._sizes[parallelism] for parallelism in self.active_mesh_dims}
+
+        # Apply canonical ordering
+        mesh_order = ["dp_replicate", "dp_shard", "cp", "sp", "tp"]
+        sorted_items = sorted(
+            mesh_dims.items(),
+            key=lambda x: (mesh_order.index(x[0])),
+        )
+        return tuple(zip(*sorted_items))
+
+    def __post_init__(self):
+        # Basic size validation
+        if self.dp_replicate_size is None:
+            self.dp_replicate_size = int(os.environ.get("PARALLELISM_CONFIG_DP_REPLICATE_SIZE", "1"))
+        if self.dp_shard_size is None:
+            self.dp_shard_size = int(os.environ.get("PARALLELISM_CONFIG_DP_SHARD_SIZE", "1"))
+        if self.tp_size is None:
+            self.tp_size = int(os.environ.get("PARALLELISM_CONFIG_TP_SIZE", "1"))
+        if self.cp_size is None:
+            self.cp_size = int(os.environ.get("PARALLELISM_CONFIG_CP_SIZE", "1"))
+        if self.cp_backend is None:
+            self.cp_backend = os.environ.get("PARALLELISM_CONFIG_CP_BACKEND", "torch")
+        if self.sp_size is None:
+            self.sp_size = int(os.environ.get("PARALLELISM_CONFIG_SP_SIZE", "1"))
+        if self.sp_backend is None:
+            self.sp_backend = os.environ.get("PARALLELISM_CONFIG_SP_BACKEND", "deepspeed")
+
+        if self.tp_size > 1:
+            if self.tp_handler is None:
+                self.tp_handler = TorchTensorParallelConfig()
+
+        if self.cp_size > 1:
+            if self.cp_handler is None:
+                self.cp_handler = TorchContextParallelConfig()
+            else:
+                cp_backends_config_map = dict(
+                    torch=TorchContextParallelConfig,
+                )
+                if not isinstance(self.cp_handler, cp_backends_config_map[self.cp_backend]):
+                    raise ValueError(
+                        f"ParallelismConfig's cp_backend={self.cp_backend} requires {cp_backends_config_map[self.cp_backend]}, but cp_handler was set to {type(self.cp_handler)}"
+                    )
+
+        if self.sp_size > 1:
+            if self.sp_handler is None:
+                self.sp_handler = DeepSpeedSequenceParallelConfig()
+        if self.dp_replicate_size < 1:
+            raise ValueError(f"dp_replicate_size must be at least 1, but got {self.dp_replicate_size}")
+        if self.dp_shard_size < 1:
+            raise ValueError(f"dp_shard_size must be at least 1, but got {self.dp_shard_size}")
+        if self.tp_size < 1:
+            raise ValueError(f"tp_size must be at least 1, but got {self.tp_size}")
+        if self.cp_size < 1:
+            raise ValueError(f"cp_size must be at least 1, but got {self.cp_size}")
+        valid_cp_backends = ["torch"]
+        if self.cp_backend not in valid_cp_backends:
+            raise ValueError(f"cp_backend must be one of {valid_cp_backends}, but got {self.cp_backend}")
+
+        if self.sp_size < 1:
+            raise ValueError(f"sp_size must be at least 1, but got {self.sp_size}")
+        valid_sp_backends = ["deepspeed"]
+        if self.sp_backend not in valid_sp_backends:
+            raise ValueError(f"sp_backend must be one of {valid_sp_backends}, but got {self.sp_backend}")
+
+        if (self.tp_size > 1 or self.cp_size > 1) and self.dp_replicate_size > 1 and self.dp_shard_size == 1:
+            raise ValueError(
+                "Tensor/Context parallelism (tp/cp_size > 1) cannot be used with pure data parallelism (dp_replicate_size > 1 and dp_shard_size == 1). "
+                "Please set dp_shard_size > 1 and dp_replicate_size == 1 to compose FSDP + TP/CP for 2D parallel, "
+                "or set dp_replicate_size == 1 and dp_shard_size > 1 to compose HSDP + TP/CP for 3D parallel."
+            )
+        self._sizes = {
+            "dp_replicate": self.dp_replicate_size,
+            "dp_shard": self.dp_shard_size,
+            "tp": self.tp_size,
+            "cp": self.cp_size,
+            "sp": self.sp_size,
+        }
+
+    def _set_size(self, parallelism: str, size: int):
+        assert parallelism in self._sizes.keys(), f"Parallelism must be one of {self._sizes.keys()}"
+        self._sizes[parallelism] = size
+        setattr(self, f"{parallelism}_size", size)
+
+    def _validate_accelerator(self, accelerator: "Accelerator"):
+        _warnings = set()
+        if not accelerator.multi_device and self.total_size == 1:
+            # No distributed setup, valid parallelism config
+            return
+
+        # We need this to ensure DDP works
+        if self.total_size == 1:
+            self._set_size("dp_replicate", accelerator.num_processes)
+
+        if self.total_size != accelerator.num_processes:
+            raise ValueError(
+                f"ParallelismConfig total_size ({self.total_size}) does not match "
+                f"num_processes ({accelerator.num_processes}). Please adjust dp_replicate_size/ "
+                f"dp_shard_size/tp_size/cp_size/sp_size."
+            )
+
+        if self.total_size > 1 and not (
+            accelerator.is_fsdp2
+            or accelerator.multi_device
+            or accelerator.distributed_type == DistributedType.DEEPSPEED
+        ):
+            raise ValueError(
+                f"ParallelismConfig is only compatible DistributedType.FSDP (version 2) or DistributedType.Multi{{Device}} or DistributedType.DEEPSPEED, but got {accelerator.distributed_type}."
+            )
+
+        for parallelism, size in self._sizes.items():
+            if size == 1 and getattr(self, f"{parallelism}_handler", None) is not None:
+                _warnings.add(
+                    f"ParallelismConfig.{parallelism}_handler is set, but {parallelism}_size is set to 1. This handler will be ignored."
+                )
+
+        if _warnings and accelerator.is_main_process:
+            warnings.warn(
+                "ParallelismConfig has the following warnings:\n" + "\n".join(_warnings),
+                UserWarning,
+            )

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/scheduler.py

@@ -0,0 +1,98 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# We ignore warnings about stepping the scheduler since we step it ourselves during gradient accumulation
+
+import warnings
+
+from .state import AcceleratorState, GradientState
+
+
+warnings.filterwarnings("ignore", category=UserWarning, module="torch.optim.lr_scheduler")
+
+
+class AcceleratedScheduler:
+    """
+    A wrapper around a learning rate scheduler that will only step when the optimizer(s) have a training step. Useful
+    to avoid making a scheduler step too fast when gradients went overflow and there was no training step (in mixed
+    precision training)
+
+    When performing gradient accumulation scheduler lengths should not be changed accordingly, Accelerate will always
+    step the scheduler to account for it.
+
+    Args:
+        scheduler (`torch.optim.lr_scheduler._LRScheduler`):
+            The scheduler to wrap.
+        optimizers (one or a list of `torch.optim.Optimizer`):
+            The optimizers used.
+        step_with_optimizer (`bool`, *optional*, defaults to `True`):
+            Whether or not the scheduler should be stepped at each optimizer step.
+        split_batches (`bool`, *optional*, defaults to `False`):
+            Whether or not the dataloaders split one batch across the different processes (so batch size is the same
+            regardless of the number of processes) or create batches on each process (so batch size is the original
+            batch size multiplied by the number of processes).
+    """
+
+    def __init__(self, scheduler, optimizers, step_with_optimizer: bool = True, split_batches: bool = False):
+        self.scheduler = scheduler
+        self.optimizers = optimizers if isinstance(optimizers, (list, tuple)) else [optimizers]
+        self.split_batches = split_batches
+        self.step_with_optimizer = step_with_optimizer
+        self.gradient_state = GradientState()
+
+    def step(self, *args, **kwargs):
+        if not self.step_with_optimizer:
+            # No link between scheduler and optimizer -> just step
+            self.scheduler.step(*args, **kwargs)
+            return
+
+        # Otherwise, first make sure the optimizer was stepped.
+        if not self.gradient_state.sync_gradients:
+            if self.gradient_state.adjust_scheduler:
+                self.scheduler._step_count += 1
+            return
+
+        for opt in self.optimizers:
+            if opt.step_was_skipped:
+                return
+        if self.split_batches:
+            # Split batches -> the training dataloader batch size is not changed so one step per training step
+            self.scheduler.step(*args, **kwargs)
+        else:
+            # Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
+            # num_processes steps per training step
+            num_processes = AcceleratorState().num_processes
+            for _ in range(num_processes):
+                # Special case when using OneCycle and `drop_last` was not used
+                if hasattr(self.scheduler, "total_steps"):
+                    if self.scheduler._step_count <= self.scheduler.total_steps:
+                        self.scheduler.step(*args, **kwargs)
+                else:
+                    self.scheduler.step(*args, **kwargs)
+
+    # Passthroughs
+    def get_last_lr(self):
+        return self.scheduler.get_last_lr()
+
+    def state_dict(self):
+        return self.scheduler.state_dict()
+
+    def load_state_dict(self, state_dict):
+        self.scheduler.load_state_dict(state_dict)
+
+    def get_lr(self):
+        return self.scheduler.get_lr()
+
+    def print_lr(self, *args, **kwargs):
+        return self.scheduler.print_lr(*args, **kwargs)

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/state.py

@@ -0,0 +1,1373 @@
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import logging
+import os
+import threading
+import warnings
+import weakref
+from contextlib import contextmanager
+from functools import partial
+from typing import Any, Callable
+
+import torch
+
+from .utils import (
+    DistributedType,
+    DynamoBackend,
+    GradientAccumulationPlugin,
+    check_cuda_fp8_capability,
+    check_cuda_p2p_ib_support,
+    deepspeed_required,
+    get_cpu_distributed_information,
+    get_int_from_env,
+    is_ccl_available,
+    is_datasets_available,
+    is_deepspeed_available,
+    is_fp8_available,
+    is_habana_gaudi1,
+    is_hpu_available,
+    is_ipex_available,
+    is_mlu_available,
+    is_mps_available,
+    is_musa_available,
+    is_npu_available,
+    is_sdaa_available,
+    is_torch_xla_available,
+    is_xccl_available,
+    is_xpu_available,
+    parse_choice_from_env,
+    parse_flag_from_env,
+    set_numa_affinity,
+)
+from .utils.dataclasses import SageMakerDistributedType
+
+
+if is_torch_xla_available():
+    import torch_xla.core.xla_model as xm
+    import torch_xla.runtime as xr
+
+if is_mlu_available(check_device=False):
+    import torch_mlu  # noqa: F401
+
+if is_sdaa_available(check_device=False):
+    import torch_sdaa  # noqa: F401
+
+if is_musa_available(check_device=False):
+    import torch_musa  # noqa: F401
+
+if is_npu_available(check_device=False):
+    import torch_npu  # noqa: F401
+
+
+logger = logging.getLogger(__name__)
+
+
+def is_initialized() -> bool:
+    """
+    Checks if the `AcceleratorState` has been initialized from `Accelerator`. Same as `AcceleratorState.initialized`,
+    but works as a module method.
+    """
+    return AcceleratorState._shared_state != {}
+
+
+# Lambda function that does nothing
+def do_nothing(*args, **kwargs):
+    return None
+
+
+class ThreadLocalSharedDict(threading.local):
+    """
+    Descriptor that holds a dict shared between instances of a class in the same thread.
+
+    Note: Descriptors have slightly different semantics than just a dict field on its own.
+    `PartialState(...)._shared_state` and `PartialState._shared_state` (instance vs class) give the same value: the
+    underlying _storage dict. Likewise, `PartialState(...)._shared_state = {...}` overrides the _storage dict inside
+    the descriptor as you would expect. However, `PartialState._shared_state = {}` actually replaces the descriptor
+    object with a dict instead Thus, you should modify the _storage dict in-place (e.g. `_shared_state.clear()`).
+
+    See Python documentation for an explanation of descriptors: https://docs.python.org/3/howto/descriptor.html
+
+    This is required for using PyTorch/XLA with PJRT in multithreaded mode (required for TPU v2 and v3).
+
+    See https://github.com/pytorch/xla/blob/r2.0/docs/pjrt.md#multithreading-on-tpu-v2v3
+    """
+
+    def __init__(self, thread_local: bool = False):
+        self._storage = {}
+
+    def __get__(self, obj, objtype=None):
+        return self._storage
+
+    def __set__(self, obj, value):
+        self._storage = value
+
+
+# Prefer global shared dictionary, except when using TPU.
+SharedDict = dict if not is_torch_xla_available() else ThreadLocalSharedDict
+
+
+# Inspired by Alex Martelli's 'Borg'.
+class PartialState:
+    """
+    Singleton class that has information about the current training environment and functions to help with process
+    control. Designed to be used when only process control and device execution states are needed. Does *not* need to
+    be initialized from `Accelerator`.
+
+    Args:
+        cpu (`bool`, *optional*):
+            Whether or not to force the script to execute on CPU. Will ignore any accelerators available if set to
+            `True` and force the execution on the CPU.
+        kwargs (additional keyword arguments, *optional*):
+            Additional keyword arguments to pass to the relevant `init_process_group` function. Valid `kwargs` can be
+            found in [`utils.InitProcessGroupKwargs`]. See the example section for detailed usage.
+
+    **Available attributes:**
+
+        - **device** (`torch.device`) -- The device to use.
+        - **distributed_type** ([`~accelerate.state.DistributedType`]) -- The type of distributed environment currently
+          in use.
+        - **local_process_index** (`int`) -- The index of the current process on the current server.
+        - **mixed_precision** (`str`) -- Whether or not the current script will use mixed precision, and if so the type
+          of mixed precision being performed. (Choose from 'no','fp16','bf16 or 'fp8').
+        - **num_processes** (`int`) -- The number of processes currently launched in parallel.
+        - **process_index** (`int`) -- The index of the current process.
+        - **is_last_process** (`bool`) -- Whether or not the current process is the last one.
+        - **is_main_process** (`bool`) -- Whether or not the current process is the main one.
+        - **is_local_main_process** (`bool`) -- Whether or not the current process is the main one on the local node.
+        - **debug** (`bool`) -- Whether or not the current script is being run in debug mode.
+
+    Example:
+    ```python
+    from accelerate.utils import InitProcessGroupKwargs
+
+    # To include `InitProcessGroupKwargs`, init then call `.to_kwargs()`
+    kwargs = InitProcessGroupKwargs(...).to_kwargs()
+    state = PartialState(**kwargs)
+    ```
+    """
+
+    _shared_state = SharedDict()
+    _known_attrs = [
+        "_cpu",
+        "_mixed_precision",
+        "_shared_state",
+        "backend",
+        "debug",
+        "device",
+        "distributed_type",
+        "fork_launched",
+        "local_process_index",
+        "num_processes",
+        "process_index",
+    ]
+
+    def __init__(self, cpu: bool = False, **kwargs):
+        self.__dict__ = self._shared_state
+        if not self.initialized:
+            self._cpu = cpu
+            self.backend = None
+            env_device = os.environ.get("ACCELERATE_TORCH_DEVICE", None)
+            self.device = torch.device(env_device) if env_device is not None else None
+            self.debug = parse_flag_from_env("ACCELERATE_DEBUG_MODE")
+            use_sagemaker_dp = kwargs.pop("_use_sagemaker_dp", None)
+            dist_information = None
+            if use_sagemaker_dp is None:
+                use_sagemaker_dp = (
+                    os.environ.get("ACCELERATE_USE_SAGEMAKER", "false").lower() == "true"
+                    and os.environ.get("ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPE") != SageMakerDistributedType.NO
+                )
+
+            # Sets up self.backend + imports
+            original_backend = kwargs.pop("backend", None)
+            backend, distributed_type = self._prepare_backend(cpu, use_sagemaker_dp, original_backend)
+            if original_backend is not None and backend != original_backend:
+                raise ValueError(f"Your assigned backend {original_backend} is not available, please use {backend}")
+            self.backend = backend
+            self.distributed_type = distributed_type
+            use_deepspeed = False
+            if not cpu and self.backend != "xla":
+                if int(os.environ.get("LOCAL_RANK", -1)) != -1:
+                    # Deal with spawning deepspeed
+                    if os.environ.get("ACCELERATE_USE_DEEPSPEED", "false").lower() == "true":
+                        if not is_deepspeed_available():
+                            raise ImportError(
+                                "DeepSpeed is not available => install it using `pip3 install deepspeed` or build it from source"
+                            )
+                        from deepspeed import comm as dist
+
+                        if not dist.is_initialized():
+                            if self.backend == "tccl":
+                                local_rank = os.environ.get("LOCAL_RANK", -1)
+                                torch.sdaa.set_device(f"sdaa:{local_rank}")
+                            dist.init_distributed(dist_backend=self.backend, auto_mpi_discovery=False, **kwargs)
+                        # We need to flag to `use_deepspeed` to be True to override `distributed_type` later
+                        use_deepspeed = True
+                    # Deal with all other backends but XPU and CPU, that gets handled special later
+                    elif (
+                        self.distributed_type not in (DistributedType.MULTI_XPU, DistributedType.MULTI_CPU)
+                        and not torch.distributed.is_initialized()
+                    ):
+                        if self.backend == "tccl":
+                            local_rank = os.environ.get("LOCAL_RANK", -1)
+                            torch.sdaa.set_device(f"sdaa:{local_rank}")
+                        if (
+                            self.backend == "nccl"
+                            and os.environ.get("ACCELERATE_USE_FSDP", "false").lower() == "true"
+                            and (
+                                os.environ.get("FSDP_OFFLOAD_PARAMS", "false").lower() == "true"
+                                or os.environ.get("FSDP_STATE_DICT_TYPE", "SHARDED_STATE_DICT") == "FULL_STATE_DICT"
+                            )
+                        ):
+                            self.backend = "cuda:nccl,cpu:gloo"
+                        torch.distributed.init_process_group(backend=self.backend, **kwargs)
+
+            # XPU and CPU require special env configs to be set
+            if self.distributed_type in (DistributedType.MULTI_XPU, DistributedType.MULTI_CPU):
+                dist_information = get_cpu_distributed_information()
+                os.environ["RANK"] = str(dist_information.rank)
+                os.environ["WORLD_SIZE"] = str(dist_information.world_size)
+                os.environ["LOCAL_RANK"] = str(dist_information.local_rank)
+                os.environ["LOCAL_WORLD_SIZE"] = str(dist_information.local_world_size)
+                if not os.environ.get("MASTER_PORT", None):
+                    os.environ["MASTER_PORT"] = "29500"
+                if (
+                    not os.environ.get("MASTER_ADDR", None)
+                    and dist_information.local_world_size != dist_information.world_size
+                    and self.backend != "mpi"
+                ):
+                    raise ValueError(
+                        "Tried to launch on distributed with multinode, but `MASTER_ADDR` env was not set, "
+                        "please try exporting rank 0's hostname as `MASTER_ADDR`"
+                    )
+                kwargs["rank"] = dist_information.rank
+                kwargs["world_size"] = dist_information.world_size
+
+                if (
+                    self.distributed_type == DistributedType.MULTI_CPU
+                    and get_int_from_env(["OMP_NUM_THREADS"], 0) == 0
+                ):
+                    import psutil
+
+                    num_cpu_threads_per_process = int(
+                        psutil.cpu_count(logical=False) / dist_information.local_world_size
+                    )
+                    if num_cpu_threads_per_process == 0:
+                        num_cpu_threads_per_process = 1
+                    torch.set_num_threads(num_cpu_threads_per_process)
+                    warnings.warn(
+                        f"OMP_NUM_THREADS/MKL_NUM_THREADS unset, we set it at {num_cpu_threads_per_process} to improve oob"
+                        " performance."
+                    )
+
+                if not torch.distributed.is_initialized():
+                    torch.distributed.init_process_group(backend=self.backend, **kwargs)
+
+            # No backend == no distributed training
+            if self.backend is None:
+                self.distributed_type = DistributedType.NO
+                self.num_processes = 1
+                self.process_index = 0
+                self.local_process_index = 0
+            elif self.backend == "xla":
+                # XLA needs device setting first for `set_replication`
+                self.set_device()
+                xm.set_replication(self.device, xm.get_xla_supported_devices())
+                self.num_processes = xr.world_size()
+                self.process_index = xr.global_ordinal()
+                if is_torch_xla_available(check_is_tpu=True):
+                    self.local_process_index = xm.get_local_ordinal()
+                else:
+                    self.local_process_index = int(os.environ.get("LOCAL_RANK", -1))
+            else:
+                self.num_processes = torch.distributed.get_world_size()
+                self.process_index = torch.distributed.get_rank()
+                self.local_process_index = (
+                    int(os.environ.get("LOCAL_RANK", -1)) if dist_information is None else dist_information.local_rank
+                )
+            self.set_device()
+            # Now we can change to deepseed
+            if use_deepspeed:
+                self.distributed_type = DistributedType.DEEPSPEED
+
+            # Set CPU affinity if enabled
+            if parse_flag_from_env("ACCELERATE_CPU_AFFINITY", False):
+                set_numa_affinity(self.local_process_index)
+
+            # Check for old RTX 4000's that can't use P2P or IB and are on old drivers
+            if self.device.type == "cuda" and not check_cuda_p2p_ib_support():
+                if "NCCL_P2P_DISABLE" not in os.environ or "NCCL_IB_DISABLE" not in os.environ:
+                    raise NotImplementedError(
+                        "Using RTX 4000 series doesn't support faster communication broadband via P2P or IB. "
+                        'Please set `NCCL_P2P_DISABLE="1"` and `NCCL_IB_DISABLE="1" or use `accelerate launch` which '
+                        "will do this automatically."
+                    )
+
+        # Important: This should be the *only* code outside of `self.initialized!`
+        self.fork_launched = parse_flag_from_env("FORK_LAUNCHED", 0)
+
+    def __repr__(self) -> str:
+        return (
+            f"Distributed environment: {self.distributed_type}{('  Backend: ' + self.backend) if self.backend else ''}\n"
+            f"Num processes: {self.num_processes}\n"
+            f"Process index: {self.process_index}\n"
+            f"Local process index: {self.local_process_index}\n"
+            f"Device: {self.device}\n"
+        )
+
+    @staticmethod
+    def _reset_state():
+        "Resets `_shared_state`, is used internally and should not be called"
+        PartialState._shared_state.clear()
+
+    @property
+    def initialized(self) -> bool:
+        "Returns whether the `PartialState` has been initialized"
+        return self._shared_state != {}
+
+    @property
+    def use_distributed(self):
+        """
+        Whether the Accelerator is configured for distributed training
+        """
+        return self.distributed_type != DistributedType.NO and self.num_processes > 1
+
+    @property
+    def is_last_process(self) -> bool:
+        "Returns whether the current process is the last one"
+        return self.process_index == self.num_processes - 1
+
+    @property
+    def is_main_process(self) -> bool:
+        "Returns whether the current process is the main process"
+        return (
+            self.process_index == 0 if self.distributed_type != DistributedType.MEGATRON_LM else self.is_last_process
+        )
+
+    @property
+    def is_local_main_process(self) -> bool:
+        "Returns whether the current process is the main process on the local node"
+        return (
+            self.local_process_index == 0
+            if self.distributed_type != DistributedType.MEGATRON_LM
+            else self.is_last_process
+        )
+
+    def wait_for_everyone(self):
+        """
+        Will stop the execution of the current process until every other process has reached that point (so this does
+        nothing when the script is only run in one process). Useful to do before saving a model.
+
+        Example:
+
+        ```python
+        >>> # Assuming two GPU processes
+        >>> import time
+        >>> from accelerate.state import PartialState
+
+        >>> state = PartialState()
+        >>> if state.is_main_process:
+        ...     time.sleep(2)
+        >>> else:
+        ...     print("I'm waiting for the main process to finish its sleep...")
+        >>> state.wait_for_everyone()
+        >>> # Should print on every process at the same time
+        >>> print("Everyone is here")
+        ```
+        """
+        if self.distributed_type in (
+            DistributedType.MULTI_GPU,
+            DistributedType.MULTI_MLU,
+            DistributedType.MULTI_SDAA,
+            DistributedType.MULTI_MUSA,
+            DistributedType.MULTI_NPU,
+            DistributedType.MULTI_XPU,
+            DistributedType.MULTI_CPU,
+            DistributedType.MULTI_HPU,
+            DistributedType.DEEPSPEED,
+            DistributedType.FSDP,
+        ):
+            torch.distributed.barrier(device_ids=[self.local_process_index])
+        elif self.distributed_type == DistributedType.XLA:
+            xm.rendezvous("accelerate.utils.wait_for_everyone")
+
+    def _goes_first(self, is_main: bool):
+        if not is_main:
+            self.wait_for_everyone()
+
+        yield
+
+        if is_main:
+            self.wait_for_everyone()
+
+    @contextmanager
+    def split_between_processes(self, inputs: list | tuple | dict | torch.Tensor, apply_padding: bool = False):
+        """
+        Splits `input` between `self.num_processes` quickly and can be then used on that process. Useful when doing
+        distributed inference, such as with different prompts.
+
+        Note that when using a `dict`, all keys need to have the same number of elements.
+
+        Args:
+            inputs (`list`, `tuple`, `torch.Tensor`, `dict` of `list`/`tuple`/`torch.Tensor`, or `datasets.Dataset`):
+                The input to split between processes.
+            apply_padding (`bool`, `optional`, defaults to `False`):
+                Whether to apply padding by repeating the last element of the input so that all processes have the same
+                number of elements. Useful when trying to perform actions such as `gather()` on the outputs or passing
+                in less inputs than there are processes. If so, just remember to drop the padded elements afterwards.
+
+
+        Example:
+
+        ```python
+        # Assume there are two processes
+        from accelerate import PartialState
+
+        state = PartialState()
+        with state.split_between_processes(["A", "B", "C"]) as inputs:
+            print(inputs)
+        # Process 0
+        ["A", "B"]
+        # Process 1
+        ["C"]
+
+        with state.split_between_processes(["A", "B", "C"], apply_padding=True) as inputs:
+            print(inputs)
+        # Process 0
+        ["A", "B"]
+        # Process 1
+        ["C", "C"]
+        ```
+        """
+        if self.num_processes == 1:
+            yield inputs
+            return
+        length = len(inputs)
+        # Nested dictionary of any types
+        if isinstance(inputs, dict):
+            length = len(inputs[list(inputs.keys())[0]])
+            if not all(len(v) == length for v in inputs.values()):
+                raise ValueError("All values in the dictionary must have the same length")
+        num_samples_per_process, num_extras = divmod(length, self.num_processes)
+        start_index = self.process_index * num_samples_per_process + min(self.process_index, num_extras)
+        end_index = start_index + num_samples_per_process + (1 if self.process_index < num_extras else 0)
+
+        def _split_values(inputs, start_index, end_index):
+            if isinstance(inputs, (list, tuple, torch.Tensor)):
+                if start_index >= len(inputs):
+                    result = inputs[-1:]
+                else:
+                    result = inputs[start_index:end_index]
+                if apply_padding:
+                    if isinstance(result, torch.Tensor):
+                        from accelerate.utils import pad_across_processes, send_to_device
+
+                        # The tensor needs to be on the device before we can pad it
+                        tensorized_result = send_to_device(result, self.device)
+                        result = pad_across_processes(tensorized_result, pad_index=inputs[-1])
+                    else:
+                        result += [result[-1]] * (num_samples_per_process + (1 if num_extras > 0 else 0) - len(result))
+                return result
+            elif isinstance(inputs, dict):
+                for key in inputs.keys():
+                    inputs[key] = _split_values(inputs[key], start_index, end_index)
+                return inputs
+            else:
+                if is_datasets_available():
+                    from datasets import Dataset
+
+                    if isinstance(inputs, Dataset):
+                        if start_index >= len(inputs):
+                            start_index = len(inputs) - 1
+                        if end_index > len(inputs):
+                            end_index = len(inputs)
+                        result_idcs = list(range(start_index, end_index))
+                        if apply_padding:
+                            result_idcs += [end_index - 1] * (
+                                num_samples_per_process + (1 if num_extras > 0 else 0) - len(result_idcs)
+                            )
+                        return inputs.select(result_idcs)
+                return inputs
+
+        yield _split_values(inputs, start_index, end_index)
+
+    @contextmanager
+    def main_process_first(self):
+        """
+        Lets the main process go first inside a with block.
+
+        The other processes will enter the with block after the main process exits.
+
+        Example:
+
+        ```python
+        >>> from accelerate import Accelerator
+
+        >>> accelerator = Accelerator()
+        >>> with accelerator.main_process_first():
+        ...     # This will be printed first by process 0 then in a seemingly
+        ...     # random order by the other processes.
+        ...     print(f"This will be printed by process {accelerator.process_index}")
+        ```
+        """
+        yield from self._goes_first(self.is_main_process)
+
+    @contextmanager
+    def local_main_process_first(self):
+        """
+        Lets the local main process go inside a with block.
+
+        The other processes will enter the with block after the main process exits.
+
+        Example:
+
+        ```python
+        >>> from accelerate.state import PartialState
+
+        >>> state = PartialState()
+        >>> with state.local_main_process_first():
+        ...     # This will be printed first by local process 0 then in a seemingly
+        ...     # random order by the other processes.
+        ...     print(f"This will be printed by process {state.local_process_index}")
+        ```
+        """
+        yield from self._goes_first(self.is_local_main_process)
+
+    def on_main_process(self, function: Callable[..., Any] | None = None):
+        """
+        Decorator that only runs the decorated function on the main process.
+
+        Args:
+            function (`Callable`): The function to decorate.
+
+        Example:
+
+        ```python
+        >>> from accelerate.state import PartialState
+
+        >>> state = PartialState()
+
+
+        >>> @state.on_main_process
+        ... def print_something():
+        ...     print("This will be printed by process 0 only.")
+
+
+        >>> print_something()
+        "This will be printed by process 0 only"
+        ```
+        """
+        if not self.initialized:
+            raise ValueError("The `PartialState` or `Accelerator` must be initialized before calling this function.")
+        if self.is_main_process or not self.use_distributed:
+            return function
+        return do_nothing
+
+    def on_local_main_process(self, function: Callable[..., Any] | None = None):
+        """
+        Decorator that only runs the decorated function on the local main process.
+
+        Args:
+            function (`Callable`): The function to decorate.
+
+        Example:
+        ```python
+        # Assume we have 2 servers with 4 processes each.
+        from accelerate.state import PartialState
+
+        state = PartialState()
+
+
+        @state.on_local_main_process
+        def print_something():
+            print("This will be printed by process 0 only on each server.")
+
+
+        print_something()
+        # On server 1:
+        "This will be printed by process 0 only"
+        # On server 2:
+        "This will be printed by process 0 only"
+        ```
+        """
+        if self.is_local_main_process or not self.use_distributed:
+            return function
+        return do_nothing
+
+    def on_last_process(self, function: Callable[..., Any]):
+        """
+        Decorator that only runs the decorated function on the last process.
+
+        Args:
+            function (`Callable`): The function to decorate.
+
+        Example:
+        ```python
+        # Assume we have 4 processes.
+        from accelerate.state import PartialState
+
+        state = PartialState()
+
+
+        @state.on_last_process
+        def print_something():
+            print(f"Printed on process {state.process_index}")
+
+
+        print_something()
+        "Printed on process 3"
+        ```
+        """
+        if self.is_last_process or not self.use_distributed:
+            return function
+        return do_nothing
+
+    def on_process(self, function: Callable[..., Any] | None = None, process_index: int | None = None):
+        """
+        Decorator that only runs the decorated function on the process with the given index.
+
+        Args:
+            function (`Callable`, `optional`):
+                The function to decorate.
+            process_index (`int`, `optional`):
+                The index of the process on which to run the function.
+
+        Example:
+        ```python
+        # Assume we have 4 processes.
+        from accelerate.state import PartialState
+
+        state = PartialState()
+
+
+        @state.on_process(process_index=2)
+        def print_something():
+            print(f"Printed on process {state.process_index}")
+
+
+        print_something()
+        "Printed on process 2"
+        ```
+        """
+        if function is None:
+            return partial(self.on_process, process_index=process_index)
+        if (self.process_index == process_index) or (not self.use_distributed):
+            return function
+        return do_nothing
+
+    def on_local_process(self, function: Callable[..., Any] | None = None, local_process_index: int | None = None):
+        """
+        Decorator that only runs the decorated function on the process with the given index on the current node.
+
+        Args:
+            function (`Callable`, *optional*):
+                The function to decorate.
+            local_process_index (`int`, *optional*):
+                The index of the local process on which to run the function.
+
+        Example:
+        ```python
+        # Assume we have 2 servers with 4 processes each.
+        from accelerate import Accelerator
+
+        accelerator = Accelerator()
+
+
+        @accelerator.on_local_process(local_process_index=2)
+        def print_something():
+            print(f"Printed on process {accelerator.local_process_index}")
+
+
+        print_something()
+        # On server 1:
+        "Printed on process 2"
+        # On server 2:
+        "Printed on process 2"
+        ```
+        """
+        if function is None:
+            return partial(self.on_local_process, local_process_index=local_process_index)
+        if (self.local_process_index == local_process_index) or (not self.use_distributed):
+            return function
+        return do_nothing
+
+    def print(self, *args, **kwargs):
+        if self.is_local_main_process:
+            print(*args, **kwargs)
+
+    @property
+    def default_device(self) -> torch.device:
+        """
+        Returns the default device which is:
+        - MPS if `torch.backends.mps.is_available()` and `torch.backends.mps.is_built()` both return True.
+        - CUDA if `torch.cuda.is_available()`
+        - MLU if `is_mlu_available()`
+        - SDAA if `is_sdaa_available()`
+        - MUSA if `is_musa_available()`
+        - NPU if `is_npu_available()`
+        - HPU if `is_hpu_available()`
+        - CPU otherwise
+        """
+        if is_mps_available():
+            os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+            return torch.device("mps")
+        elif is_mlu_available():
+            return torch.device("mlu")
+        elif is_sdaa_available():
+            return torch.device("sdaa")
+        elif is_musa_available():
+            return torch.device("musa")
+        # NPU should be checked before CUDA when using `transfer_to_npu`
+        # See issue #3020: https://github.com/huggingface/accelerate/issues/3020
+        elif is_npu_available():
+            return torch.device("npu")
+        elif is_hpu_available():
+            return torch.device("hpu")
+        elif torch.cuda.is_available():
+            return torch.device("cuda")
+        elif is_xpu_available():
+            return torch.device("xpu")
+        else:
+            return torch.device("cpu")
+
+    def _prepare_backend(
+        self, cpu: bool = False, sagemaker_dp=False, backend: str | None = None
+    ) -> tuple[str, DistributedType]:
+        "Prepares any imports needed before initializing the distributed backend and sets `self.backend` properly"
+        distributed_type = None
+        if sagemaker_dp:
+            import smdistributed.dataparallel.torch.torch_smddp  # noqa
+
+            backend = "smddp"
+            distributed_type = DistributedType.MULTI_GPU
+        elif is_torch_xla_available():
+            backend = "xla"
+            distributed_type = DistributedType.XLA
+
+        elif int(os.environ.get("LOCAL_RANK", -1)) != -1 and not cpu:
+            if is_mlu_available():
+                backend = "cncl"
+                distributed_type = DistributedType.MULTI_MLU
+            if is_sdaa_available():
+                backend = "tccl"
+                distributed_type = DistributedType.MULTI_SDAA
+            elif is_musa_available():
+                backend = "mccl"
+                distributed_type = DistributedType.MULTI_MUSA
+            # NPU should be checked before CUDA when using `transfer_to_npu`
+            # See issue #3020: https://github.com/huggingface/accelerate/issues/3020
+            elif is_npu_available():
+                backend = "hccl"
+                distributed_type = DistributedType.MULTI_NPU
+            elif is_hpu_available(init_hccl=True):
+                if backend is None:
+                    backend = "hccl"
+                distributed_type = DistributedType.MULTI_HPU
+            elif torch.cuda.is_available():
+                if backend is None:
+                    backend = "nccl"
+                distributed_type = DistributedType.MULTI_GPU
+            elif is_xpu_available() and is_xccl_available():
+                if backend is None:
+                    backend = "xccl"
+                distributed_type = DistributedType.MULTI_XPU
+
+        if distributed_type is None and (
+            int(os.environ.get("LOCAL_RANK", -1)) != -1
+            or get_int_from_env(["PMI_SIZE", "OMPI_COMM_WORLD_SIZE", "MV2_COMM_WORLD_SIZE", "WORLD_SIZE"], 1) > 1
+        ):
+            if not cpu and is_xpu_available():
+                distributed_type = DistributedType.MULTI_XPU
+            else:
+                distributed_type = DistributedType.MULTI_CPU
+
+            if (
+                backend in (None, "ccl")
+                and is_ccl_available()
+                and (get_int_from_env(["CCL_WORKER_COUNT"], 0) > 0 or distributed_type == DistributedType.MULTI_XPU)
+            ):
+                import oneccl_bindings_for_pytorch  # noqa: F401
+
+                backend = "ccl"
+            elif backend in (None, "mpi") and torch.distributed.is_mpi_available():
+                backend = "mpi"
+            else:
+                backend = "gloo"
+        if distributed_type is None:
+            distributed_type = DistributedType.NO
+
+        return backend, distributed_type
+
+    def set_device(self):
+        """
+        Sets the device in `self.device` to the current distributed environment.
+        """
+        if self.device is not None:
+            return
+        if self.distributed_type == DistributedType.NO:
+            self.device = torch.device("cpu") if self._cpu else self.default_device
+            return
+        device = str(self.distributed_type).split(".")[-1].replace("MULTI_", "").lower()
+        if device not in ("cpu", "gpu", "mlu", "musa", "npu", "xpu", "xla", "hpu", "sdaa"):
+            raise ValueError(
+                f"Can't set device for {self.distributed_type} ({device}), verify we should be calling `_set_device()` for it!"
+            )
+        if device == "xla":
+            self.device = xm.xla_device()
+        elif device == "hpu":
+            self.device = torch.device("hpu", torch.hpu.current_device())
+        else:
+            if device == "gpu":
+                device = "cuda"
+            device_module = getattr(torch, device)
+            device_index = self.local_process_index % device_module.device_count()
+            self.device = torch.device(device, device_index)
+            device_module.set_device(self.device)
+
+    def destroy_process_group(self, group=None):
+        """
+        Destroys the process group. If one is not specified, the default process group is destroyed.
+        """
+        if self.fork_launched and group is None:
+            return
+        # needed when using torch.distributed.init_process_group
+        if torch.distributed.is_initialized():
+            torch.distributed.destroy_process_group(group)
+
+    def __getattr__(self, name: str):
+        # By this point we know that no attributes of `self` contain `name`,
+        # so we just modify the error message
+        if name in self._known_attrs:
+            raise AttributeError(
+                f"`PartialState` object has no attribute `{name}`. "
+                "This happens if `PartialState._reset_state()` was called and "
+                "an `Accelerator` or `PartialState` was not reinitialized."
+            )
+        # Raise a typical AttributeError
+        raise AttributeError(f"'PartialState' object has no attribute '{name}'")
+
+
+class AcceleratorState:
+    """
+    Singleton class that has information about the current training environment.
+
+    **Available attributes:**
+
+        - **device** (`torch.device`) -- The device to use.
+        - **distributed_type** ([`~accelerate.state.DistributedType`]) -- The type of distributed environment currently
+          in use.
+        - **parallelism_config** ([`~accelerate.utils.ParallelismConfig`]) -- The parallelism configuration for the
+          current training environment. This is used to configure the distributed training environment.
+        - **initialized** (`bool`) -- Whether or not the `AcceleratorState` has been initialized from `Accelerator`.
+        - **local_process_index** (`int`) -- The index of the current process on the current server.
+        - **mixed_precision** (`str`) -- Whether or not the current script will use mixed precision, and if so the type
+          of mixed precision being performed. (Choose from 'no','fp16','bf16 or 'fp8').
+        - **num_processes** (`int`) -- The number of processes currently launched in parallel.
+        - **process_index** (`int`) -- The index of the current process.
+        - **is_last_process** (`bool`) -- Whether or not the current process is the last one.
+        - **is_main_process** (`bool`) -- Whether or not the current process is the main one.
+        - **is_local_main_process** (`bool`) -- Whether or not the current process is the main one on the local node.
+        - **debug** (`bool`) -- Whether or not the current script is being run in debug mode.
+    """
+
+    _shared_state = SharedDict()
+    _known_attrs = PartialState._known_attrs + [
+        "deepspeed_plugin",
+        "use_ipex",
+        "fsdp_plugin",
+        "megatron_lm_plugin",
+        "dynamo_plugin",
+    ]
+
+    def __init__(
+        self,
+        mixed_precision: str | None = None,
+        cpu: bool = False,
+        dynamo_plugin=None,
+        deepspeed_plugin=None,
+        fsdp_plugin=None,
+        torch_tp_plugin=None,
+        megatron_lm_plugin=None,
+        parallelism_config=None,
+        _from_accelerator: bool = False,
+        **kwargs,
+    ):
+        self.__dict__ = self._shared_state
+        if parse_flag_from_env("ACCELERATE_USE_CPU"):
+            cpu = True
+        if PartialState._shared_state == {}:
+            PartialState(cpu, **kwargs)
+        self.__dict__.update(PartialState._shared_state)
+        self._check_initialized(mixed_precision, cpu)
+        if not self.initialized:
+            self.deepspeed_plugins = None
+            self.use_ipex = None
+            self.torch_tp_plugin = torch_tp_plugin
+            self.parallelism_config = parallelism_config
+            self.device_mesh = None
+            mixed_precision = (
+                parse_choice_from_env("ACCELERATE_MIXED_PRECISION", "no")
+                if mixed_precision is None
+                else mixed_precision.lower()
+            )
+            if mixed_precision == "fp8":
+                # this is confusing, why is is_fp8_available only checks for library availability ?
+                if not is_fp8_available():
+                    raise ValueError(
+                        "Using `fp8` precision requires `transformer_engine` or `MS-AMP` to be installed."
+                    )
+                elif torch.cuda.is_available() and not check_cuda_fp8_capability():
+                    logger.warning(
+                        f"The current device has compute capability of {torch.cuda.get_device_capability()} which is "
+                        "insufficient for FP8 mixed precision training (requires a GPU Hopper/Ada Lovelace "
+                        "or higher, compute capability of 8.9 or higher). Will use FP16 instead."
+                    )
+                    mixed_precision = "fp16"
+                elif is_habana_gaudi1():
+                    logger.warning(
+                        "The current HPU device is Gaudi1 which does not support FP8 mixed precision training (requires "
+                        "Gaudi2 or higher). Will use BF16 instead."
+                    )
+                    mixed_precision = "bf16"
+
+            self.dynamo_plugin = dynamo_plugin
+            if not _from_accelerator:
+                raise ValueError(
+                    "Please make sure to properly initialize your accelerator via `accelerator = Accelerator()` "
+                    "before using any functionality from the `accelerate` library."
+                )
+            # deepspeed handles mixed_precision using deepspeed_config. But we need to set it to fp8
+            # if we're using fp8.
+            if self.distributed_type == DistributedType.DEEPSPEED and mixed_precision != "fp8":
+                self._mixed_precision = "no"
+            else:
+                self._mixed_precision = mixed_precision
+
+            if self.distributed_type == DistributedType.XLA and is_torch_xla_available(check_is_tpu=True):
+                if mixed_precision == "bf16":
+                    if os.environ.get("ACCELERATE_DOWNCAST_BF16"):
+                        os.environ["XLA_USE_BF16"] = str(0)
+                        os.environ["XLA_DOWNCAST_BF16"] = str(1)
+                        self.downcast_bfloat = True
+                    else:
+                        os.environ["XLA_USE_BF16"] = str(1)
+                        os.environ["XLA_DOWNCAST_BF16"] = str(0)
+                        self.downcast_bfloat = False
+            elif os.environ.get("ACCELERATE_USE_DEEPSPEED", "false").lower() == "true" and not cpu:
+                self.distributed_type = DistributedType.DEEPSPEED
+                if not isinstance(deepspeed_plugin, dict):
+                    deepspeed_plugin.set_mixed_precision(mixed_precision)
+                    deepspeed_plugin.select(_from_accelerator_state=True)
+                else:
+                    for plugin in deepspeed_plugin.values():
+                        plugin.set_mixed_precision(mixed_precision)
+                    # The first plugin passed in is always the active one
+                    first_plugin = next(iter(deepspeed_plugin.values()))
+                    first_plugin.select(_from_accelerator_state=True)
+                self.deepspeed_plugins = deepspeed_plugin
+            elif self.distributed_type in [
+                DistributedType.MULTI_GPU,
+                DistributedType.MULTI_MLU,
+                DistributedType.MULTI_SDAA,
+                DistributedType.MULTI_MUSA,
+                DistributedType.MULTI_NPU,
+                DistributedType.MULTI_XPU,
+                DistributedType.MULTI_HPU,
+            ]:
+                # TODO: Siro - remove when axolotl fixes their side
+                if not os.environ.get("ACCELERATE_ALLOW_CP_STANDALONE", "false").lower() == "true":
+                    if self.parallelism_config and self.parallelism_config.cp_enabled and fsdp_plugin is None:
+                        raise ValueError(
+                            "`cp_size > 1` specified in the `parallelism_config`, but no `fsdp_plugin` was provided. We need a `fsdp_plugin` to use context parallelism with `cp_backend=torch`, as we also shard the model across the device mesh to save more memory"
+                        )
+                    if (
+                        self.parallelism_config is not None
+                        and self.parallelism_config.cp_enabled
+                        and fsdp_plugin.fsdp_version == 1
+                    ):
+                        raise ValueError(
+                            "Using `cp_size>1` requires FSDP2, but the provided `fsdp_plugin` is using FSDP1. "
+                        )
+                if (os.environ.get("ACCELERATE_USE_FSDP", "false").lower() == "true" or fsdp_plugin is not None) or (
+                    self.parallelism_config is not None and self.parallelism_config.cp_enabled
+                ):
+                    self.distributed_type = DistributedType.FSDP
+                    if self._mixed_precision != "no" and fsdp_plugin is not None:
+                        fsdp_plugin.set_mixed_precision(self._mixed_precision)
+                    self.fsdp_plugin = fsdp_plugin
+                if os.environ.get(
+                    "ACCELERATE_USE_MEGATRON_LM", "false"
+                ).lower() == "true" and self.distributed_type not in [
+                    DistributedType.MULTI_XPU,
+                ]:
+                    self.distributed_type = DistributedType.MEGATRON_LM
+                    megatron_lm_plugin.set_mixed_precision(self._mixed_precision)
+                    self.megatron_lm_plugin = megatron_lm_plugin
+            elif self.distributed_type in [DistributedType.MULTI_CPU, DistributedType.MULTI_XPU, DistributedType.NO]:
+                if is_ipex_available():
+                    # check if user disables it explicitly
+                    self.use_ipex = parse_flag_from_env("ACCELERATE_USE_IPEX", default=True)
+                else:
+                    self.use_ipex = False
+            if (
+                self.dynamo_plugin.backend != DynamoBackend.NO
+                and self._mixed_precision == "no"
+                and self.device.type == "cuda"
+            ):
+                torch.backends.cuda.matmul.allow_tf32 = True
+            if (
+                self.dynamo_plugin.backend != DynamoBackend.NO
+                and self._mixed_precision == "no"
+                and self.device.type == "musa"
+            ):
+                torch.backends.musa.matmul.allow_tf32 = True
+            PartialState._shared_state["distributed_type"] = self.distributed_type
+
+    @property
+    def initialized(self) -> bool:
+        return self._shared_state != PartialState._shared_state
+
+    def __repr__(self):
+        repr = PartialState().__repr__() + f"\nMixed precision type: {self.mixed_precision}\n"
+        if self.distributed_type == DistributedType.DEEPSPEED:
+            repr += f"ds_config: {self.deepspeed_plugin.deepspeed_config}\n"
+        return repr
+
+    def _check_initialized(self, mixed_precision=None, cpu=None):
+        "Checks if a modification is trying to be made and the `AcceleratorState` has already been initialized"
+        if self.initialized:
+            err = "AcceleratorState has already been initialized and cannot be changed, restart your runtime completely and pass `{flag}` to `Accelerator()`."
+            if cpu and self.device.type != "cpu":
+                raise ValueError(err.format(flag="cpu=True"))
+            if (
+                mixed_precision is not None
+                and mixed_precision != self._mixed_precision
+                and self.distributed_type != DistributedType.DEEPSPEED
+            ):
+                raise ValueError(err.format(flag=f"mixed_precision='{mixed_precision}'"))
+
+    @property
+    def mixed_precision(self):
+        if self.distributed_type == DistributedType.DEEPSPEED and self._mixed_precision != "fp8":
+            config = self.deepspeed_plugin.deepspeed_config
+            if config.get("fp16", {}).get("enabled", False):
+                mixed_precision = "fp16"
+            elif config.get("bf16", {}).get("enabled", False):
+                mixed_precision = "bf16"
+            else:
+                mixed_precision = "no"
+        else:
+            mixed_precision = self._mixed_precision
+        return mixed_precision
+
+    @staticmethod
+    def _reset_state(reset_partial_state: bool = False):
+        "Resets `_shared_state`, is used internally and should not be called"
+        AcceleratorState._shared_state.clear()
+        if reset_partial_state:
+            PartialState._reset_state()
+
+    def destroy_process_group(self, group=None):
+        """
+        Destroys the process group. If one is not specified, the default process group is destroyed.
+
+        If `self.fork_launched` is `True` and `group` is `None`, nothing happens.
+        """
+        PartialState().destroy_process_group(group)
+
+    @property
+    def fork_launched(self):
+        return PartialState().fork_launched
+
+    @property
+    def use_distributed(self):
+        """
+        Whether the Accelerator is configured for distributed training
+        """
+        return PartialState().use_distributed
+
+    @property
+    def is_fsdp2(self) -> bool:
+        return self.distributed_type == DistributedType.FSDP and self.fsdp_plugin.fsdp_version == 2
+
+    @property
+    def is_last_process(self) -> bool:
+        "Returns whether the current process is the last one"
+        return PartialState().is_last_process
+
+    @property
+    def is_main_process(self) -> bool:
+        "Returns whether the current process is the main process"
+        return PartialState().is_main_process
+
+    @property
+    def is_local_main_process(self) -> bool:
+        "Returns whether the current process is the main process on the local node"
+        return PartialState().is_local_main_process
+
+    def wait_for_everyone(self):
+        PartialState().wait_for_everyone()
+
+    @contextmanager
+    def split_between_processes(self, inputs: list | tuple | dict | torch.Tensor, apply_padding: bool = False):
+        """
+        Splits `input` between `self.num_processes` quickly and can be then used on that process. Useful when doing
+        distributed inference, such as with different prompts.
+
+        Note that when using a `dict`, all keys need to have the same number of elements.
+
+        Args:
+            inputs (`list`, `tuple`, `torch.Tensor`, or `dict` of `list`/`tuple`/`torch.Tensor`):
+                The input to split between processes.
+            apply_padding (`bool`, `optional`, defaults to `False`):
+                Whether to apply padding by repeating the last element of the input so that all processes have the same
+                number of elements. Useful when trying to perform actions such as `gather()` on the outputs or passing
+                in less inputs than there are processes. If so, just remember to drop the padded elements afterwards.
+
+
+        Example:
+
+        ```python
+        # Assume there are two processes
+        from accelerate.state import AcceleratorState
+
+        state = AcceleratorState()
+        with state.split_between_processes(["A", "B", "C"]) as inputs:
+            print(inputs)
+        # Process 0
+        ["A", "B"]
+        # Process 1
+        ["C"]
+
+        with state.split_between_processes(["A", "B", "C"], apply_padding=True) as inputs:
+            print(inputs)
+        # Process 0
+        ["A", "B"]
+        # Process 1
+        ["C", "C"]
+        ```
+        """
+        with PartialState().split_between_processes(inputs, apply_padding=apply_padding) as inputs:
+            yield inputs
+
+    @contextmanager
+    def main_process_first(self):
+        """
+        Lets the main process go first inside a with block.
+
+        The other processes will enter the with block after the main process exits.
+        """
+        with PartialState().main_process_first():
+            yield
+
+    @contextmanager
+    def local_main_process_first(self):
+        """
+        Lets the local main process go inside a with block.
+
+        The other processes will enter the with block after the main process exits.
+        """
+        with PartialState().local_main_process_first():
+            yield
+
+    @property
+    def deepspeed_plugin(self):
+        """
+        Returns the currently active DeepSpeedPlugin.
+
+        If not using deepspeed, returns `None`.
+        """
+        # To maintain original behavior, return None if not using deepspeed.
+        if self.distributed_type != DistributedType.DEEPSPEED:
+            return None
+        from accelerate.utils.deepspeed import get_active_deepspeed_plugin
+
+        return get_active_deepspeed_plugin(self)
+
+    @deepspeed_required
+    def get_deepspeed_plugin(self, name: str):
+        """
+        Returns the DeepSpeedPlugin with the given plugin_key.
+        """
+        return self.deepspeed_plugins[name]
+
+    @deepspeed_required
+    def select_deepspeed_plugin(self, name: str | None = None):
+        """
+        Activates the DeepSpeedPlugin with the given `name`, and will disable all other plugins.
+        """
+        for key, plugin in self.deepspeed_plugins.items():
+            if key != name:
+                plugin._unselect()
+        self.deepspeed_plugins[name].select(_from_accelerator_state=True)
+
+    def print(self, *args, **kwargs):
+        PartialState().print(*args, **kwargs)
+
+    def __getattr__(self, name: str):
+        # By this point we know that no attributes of `self` contain `name`,
+        # so we just modify the error message
+        if name in self._known_attrs:
+            raise AttributeError(
+                f"`AcceleratorState` object has no attribute `{name}`. "
+                "This happens if `AcceleratorState._reset_state()` was called and "
+                "an `Accelerator` or `PartialState` was not reinitialized."
+            )
+        # Raise a typical AttributeError
+        raise AttributeError(f"'AcceleratorState' object has no attribute '{name}'")
+
+
+class GradientState:
+    """
+    Singleton class that has information related to gradient synchronization for gradient accumulation
+
+    **Available attributes:**
+
+        - **end_of_dataloader** (`bool`) -- Whether we have reached the end the current dataloader
+        - **remainder** (`int`) -- The number of extra samples that were added from padding the dataloader
+        - **sync_gradients** (`bool`) -- Whether the gradients should be synced across all devices
+        - **active_dataloader** (`Optional[DataLoader]`) -- The dataloader that is currently being iterated over
+        - **dataloader_references** (`List[Optional[DataLoader]]`) -- A list of references to the dataloaders that are
+            being iterated over
+        - **num_steps** (`int`) -- The number of steps to accumulate over
+        - **adjust_scheduler** (`bool`) -- Whether the scheduler should be adjusted to account for the gradient
+            accumulation
+        - **sync_with_dataloader** (`bool`) -- Whether the gradients should be synced at the end of the dataloader
+            iteration and the number of total steps reset
+        - **is_xla_gradients_synced** (`bool`) -- Whether the XLA gradients have been synchronized. It is initialized
+          as false. Once gradients have been reduced before the optimizer step, this flag is set to true. Subsequently,
+            after each step, the flag is reset to false. FSDP will always synchronize the gradients, hence
+            is_xla_gradients_synced is always true.
+    """
+
+    _shared_state = SharedDict()
+
+    def __init__(self, gradient_accumulation_plugin: GradientAccumulationPlugin | None = None):
+        self.__dict__ = self._shared_state
+        if not self.initialized:
+            self.sync_gradients = True
+            self._dataloader_references_ref = [None]
+            self.plugin_kwargs = (
+                gradient_accumulation_plugin.to_kwargs() if gradient_accumulation_plugin is not None else {}
+            )
+            self._is_xla_gradients_synced = False
+
+        # Plugin args are different and can be updated
+        if gradient_accumulation_plugin is not None and self.plugin_kwargs != gradient_accumulation_plugin.to_kwargs():
+            self.plugin_kwargs = gradient_accumulation_plugin.to_kwargs()
+
+    @property
+    def num_steps(self) -> int:
+        "Returns the number of steps to accumulate over"
+        return self.plugin_kwargs.get("num_steps", 1)
+
+    @property
+    def adjust_scheduler(self) -> bool:
+        "Returns whether the scheduler should be adjusted"
+        return self.plugin_kwargs.get("adjust_scheduler", False)
+
+    @property
+    def sync_with_dataloader(self) -> bool:
+        "Returns whether the gradients should be synced at the end of the dataloader iteration and the number of total steps reset"
+        return self.plugin_kwargs.get("sync_with_dataloader", True)
+
+    @property
+    def initialized(self) -> bool:
+        "Returns whether the `GradientState` has been initialized"
+        return GradientState._shared_state != {}
+
+    @property
+    def end_of_dataloader(self) -> bool:
+        "Returns whether we have reached the end of the current dataloader"
+        if not self.in_dataloader:
+            return False
+        return self.active_dataloader.end_of_dataloader
+
+    @property
+    def remainder(self) -> int:
+        "Returns the number of extra samples that were added from padding the dataloader"
+        if not self.in_dataloader:
+            return -1
+        return self.active_dataloader.remainder
+
+    def __repr__(self):
+        return (
+            f"Sync Gradients: {self.sync_gradients}\n"
+            f"At end of current dataloader: {self.end_of_dataloader}\n"
+            f"Extra samples added: {self.remainder}\n"
+            f"Gradient accumulation plugin: {self.plugin_kwargs}\n"
+        )
+
+    @property
+    def is_xla_gradients_synced(self):
+        "Returns the value of is_xla_gradients_synced. FSDP will always synchronize the gradients, hence is_xla_gradients_synced is always true."
+        if parse_flag_from_env("ACCELERATE_USE_FSDP", default=False):
+            return True
+        return self._is_xla_gradients_synced
+
+    @is_xla_gradients_synced.setter
+    def is_xla_gradients_synced(self, is_synced):
+        "Set the _is_xla_gradients_synced attribute."
+        self._is_xla_gradients_synced = is_synced
+
+    def _set_sync_gradients(self, sync_gradients):
+        "Private function that sets whether gradients should be synchronized. Users should not have to call this."
+        self.sync_gradients = sync_gradients
+        # Allow grad-sync to automatically work on TPUs
+        if (
+            self.sync_gradients
+            and is_torch_xla_available(check_is_tpu=True)
+            and PartialState().distributed_type == DistributedType.XLA
+        ):
+            xm.mark_step()
+
+    def _add_dataloader(self, dataloader):
+        "Private function that adds a dataloader to `self.dataloader_references` and sets `in_dataloader` to `True`. Users should not have to call this."
+        # We explicitly use assignment to ensure that the property setter is triggered, which is required for garbage collection.
+        # Avoid using self.dataloader_references.append as it will not trigger the setter.
+        self.dataloader_references += [dataloader]
+
+    def _remove_dataloader(self, dataloader):
+        "Private function that removes a dataloader from `self.dataloader_references` and sets `in_dataloader` to `False` if there are no more dataloaders. Users should not have to call this."
+        # We explicitly use assignment to ensure that the property setter is triggered.
+        self.dataloader_references = [
+            dataloader_ref for dataloader_ref in self.dataloader_references if dataloader_ref != dataloader
+        ]
+
+    @property
+    def active_dataloader(self):
+        return self.dataloader_references[-1]
+
+    @property
+    def dataloader_references(self):
+        # We use a property getter and setter with weakrefs to avoid circular references that prevent garbage collection
+        return [reference() if reference is not None else reference for reference in self._dataloader_references_ref]
+
+    @dataloader_references.setter
+    def dataloader_references(self, references):
+        self._dataloader_references_ref = [
+            weakref.ref(dataloader) if dataloader is not None else dataloader for dataloader in references
+        ]
+
+    @property
+    def in_dataloader(self) -> bool:
+        "Returns whether the current process is in a dataloader"
+        return self.active_dataloader is not None
+
+    @staticmethod
+    def _reset_state():
+        "Resets `_shared_state`, is used internally and should not be called"
+        GradientState._shared_state.clear()

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/accelerate/tracking.py

@@ -0,0 +1,1326 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Expectation:
+# Provide a project dir name, then each type of logger gets stored in project/{`logging_dir`}
+
+import json
+import os
+import time
+from functools import wraps
+from typing import Any, Optional, Union
+
+import yaml
+from packaging import version
+
+from .logging import get_logger
+from .state import PartialState
+from .utils import (
+    LoggerType,
+    compare_versions,
+    is_aim_available,
+    is_clearml_available,
+    is_comet_ml_available,
+    is_dvclive_available,
+    is_mlflow_available,
+    is_swanlab_available,
+    is_tensorboard_available,
+    is_trackio_available,
+    is_wandb_available,
+    listify,
+)
+
+
+_available_trackers = []
+
+if is_tensorboard_available():
+    _available_trackers.append(LoggerType.TENSORBOARD)
+
+if is_wandb_available():
+    _available_trackers.append(LoggerType.WANDB)
+
+if is_comet_ml_available():
+    _available_trackers.append(LoggerType.COMETML)
+
+if is_aim_available():
+    _available_trackers.append(LoggerType.AIM)
+
+if is_mlflow_available():
+    _available_trackers.append(LoggerType.MLFLOW)
+
+if is_clearml_available():
+    _available_trackers.append(LoggerType.CLEARML)
+
+if is_dvclive_available():
+    _available_trackers.append(LoggerType.DVCLIVE)
+
+if is_swanlab_available():
+    _available_trackers.append(LoggerType.SWANLAB)
+
+if is_trackio_available():
+    _available_trackers.append(LoggerType.TRACKIO)
+
+logger = get_logger(__name__)
+
+
+def on_main_process(function):
+    """
+    Decorator to selectively run the decorated function on the main process only based on the `main_process_only`
+    attribute in a class.
+
+    Checks at function execution rather than initialization time, not triggering the initialization of the
+    `PartialState`.
+    """
+
+    @wraps(function)
+    def execute_on_main_process(self, *args, **kwargs):
+        if getattr(self, "main_process_only", False):
+            return PartialState().on_main_process(function)(self, *args, **kwargs)
+        else:
+            return function(self, *args, **kwargs)
+
+    return execute_on_main_process
+
+
+def get_available_trackers():
+    "Returns a list of all supported available trackers in the system"
+    return _available_trackers
+
+
+class GeneralTracker:
+    """
+    A base Tracker class to be used for all logging integration implementations.
+
+    Each function should take in `**kwargs` that will automatically be passed in from a base dictionary provided to
+    [`Accelerator`].
+
+    Should implement `name`, `requires_logging_directory`, and `tracker` properties such that:
+
+    `name` (`str`): String representation of the tracker class name, such as "TensorBoard" `requires_logging_directory`
+    (`bool`): Whether the logger requires a directory to store their logs. `tracker` (`object`): Should return internal
+    tracking mechanism used by a tracker class (such as the `run` for wandb)
+
+    Implementations can also include a `main_process_only` (`bool`) attribute to toggle if relevant logging, init, and
+    other functions should occur on the main process or across all processes (by default will use `True`)
+    """
+
+    main_process_only = True
+
+    def __init__(self, _blank=False):
+        if not _blank:
+            err = ""
+            if not hasattr(self, "name"):
+                err += "`name`"
+            if not hasattr(self, "requires_logging_directory"):
+                if len(err) > 0:
+                    err += ", "
+                err += "`requires_logging_directory`"
+
+            # as tracker is a @property that relies on post-init
+            if "tracker" not in dir(self):
+                if len(err) > 0:
+                    err += ", "
+                err += "`tracker`"
+            if len(err) > 0:
+                raise NotImplementedError(
+                    f"The implementation for this tracker class is missing the following "
+                    f"required attributes. Please define them in the class definition: "
+                    f"{err}"
+                )
+
+    def start(self):
+        """
+        Lazy initialization of the tracker inside Accelerator to avoid initializing PartialState before
+        InitProcessGroupKwargs.
+        """
+        pass
+
+    def store_init_configuration(self, values: dict):
+        """
+        Logs `values` as hyperparameters for the run. Implementations should use the experiment configuration
+        functionality of a tracking API.
+
+        Args:
+            values (Dictionary `str` to `bool`, `str`, `float` or `int`):
+                Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`,
+                `str`, `float`, `int`, or `None`.
+        """
+        pass
+
+    def log(self, values: dict, step: Optional[int], **kwargs):
+        """
+        Logs `values` to the current run. Base `log` implementations of a tracking API should go in here, along with
+        special behavior for the `step parameter.
+
+        Args:
+            values (Dictionary `str` to `str`, `float`, or `int`):
+                Values to be logged as key-value pairs. The values need to have type `str`, `float`, or `int`.
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+        """
+        pass
+
+    def finish(self):
+        """
+        Should run any finalizing functions within the tracking API. If the API should not have one, just don't
+        overwrite that method.
+        """
+        pass
+
+
+class TensorBoardTracker(GeneralTracker):
+    """
+    A `Tracker` class that supports `tensorboard`. Should be initialized at the start of your script.
+
+    Args:
+        run_name (`str`):
+            The name of the experiment run
+        logging_dir (`str`, `os.PathLike`):
+            Location for TensorBoard logs to be stored.
+        **kwargs (additional keyword arguments, *optional*):
+            Additional key word arguments passed along to the `tensorboard.SummaryWriter.__init__` method.
+    """
+
+    name = "tensorboard"
+    requires_logging_directory = True
+
+    def __init__(self, run_name: str, logging_dir: Union[str, os.PathLike], **kwargs):
+        super().__init__()
+        self.run_name = run_name
+        self.logging_dir_param = logging_dir
+        self.init_kwargs = kwargs
+
+    @on_main_process
+    def start(self):
+        try:
+            from torch.utils import tensorboard
+        except ModuleNotFoundError:
+            import tensorboardX as tensorboard
+        self.logging_dir = os.path.join(self.logging_dir_param, self.run_name)
+        self.writer = tensorboard.SummaryWriter(self.logging_dir, **self.init_kwargs)
+        logger.debug(f"Initialized TensorBoard project {self.run_name} logging to {self.logging_dir}")
+        logger.debug(
+            "Make sure to log any initial configurations with `self.store_init_configuration` before training!"
+        )
+
+    @property
+    def tracker(self):
+        return self.writer
+
+    @on_main_process
+    def store_init_configuration(self, values: dict):
+        """
+        Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment. Stores the
+        hyperparameters in a yaml file for future use.
+
+        Args:
+            values (Dictionary `str` to `bool`, `str`, `float` or `int`):
+                Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`,
+                `str`, `float`, `int`, or `None`.
+        """
+        self.writer.add_hparams(values, metric_dict={})
+        self.writer.flush()
+        project_run_name = time.time()
+        dir_name = os.path.join(self.logging_dir, str(project_run_name))
+        os.makedirs(dir_name, exist_ok=True)
+        with open(os.path.join(dir_name, "hparams.yml"), "w") as outfile:
+            try:
+                yaml.dump(values, outfile)
+            except yaml.representer.RepresenterError:
+                logger.error("Serialization to store hyperparameters failed")
+                raise
+        logger.debug("Stored initial configuration hyperparameters to TensorBoard and hparams yaml file")
+
+    @on_main_process
+    def log(self, values: dict, step: Optional[int] = None, **kwargs):
+        """
+        Logs `values` to the current run.
+
+        Args:
+            values (Dictionary `str` to `str`, `float`, `int` or `dict` of `str` to `float`/`int`):
+                Values to be logged as key-value pairs. The values need to have type `str`, `float`, `int` or `dict` of
+                `str` to `float`/`int`.
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs:
+                Additional key word arguments passed along to either `SummaryWriter.add_scaler`,
+                `SummaryWriter.add_text`, or `SummaryWriter.add_scalers` method based on the contents of `values`.
+        """
+        values = listify(values)
+        for k, v in values.items():
+            if isinstance(v, (int, float)):
+                self.writer.add_scalar(k, v, global_step=step, **kwargs)
+            elif isinstance(v, str):
+                self.writer.add_text(k, v, global_step=step, **kwargs)
+            elif isinstance(v, dict):
+                self.writer.add_scalars(k, v, global_step=step, **kwargs)
+        self.writer.flush()
+        logger.debug("Successfully logged to TensorBoard")
+
+    @on_main_process
+    def log_images(self, values: dict, step: Optional[int], **kwargs):
+        """
+        Logs `images` to the current run.
+
+        Args:
+            values (Dictionary `str` to `List` of `np.ndarray` or `PIL.Image`):
+                Values to be logged as key-value pairs. The values need to have type `List` of `np.ndarray` or
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs:
+                Additional key word arguments passed along to the `SummaryWriter.add_image` method.
+        """
+        for k, v in values.items():
+            self.writer.add_images(k, v, global_step=step, **kwargs)
+        logger.debug("Successfully logged images to TensorBoard")
+
+    @on_main_process
+    def finish(self):
+        """
+        Closes `TensorBoard` writer
+        """
+        self.writer.close()
+        logger.debug("TensorBoard writer closed")
+
+
+class WandBTracker(GeneralTracker):
+    """
+    A `Tracker` class that supports `wandb`. Should be initialized at the start of your script.
+
+    Args:
+        run_name (`str`):
+            The name of the experiment run.
+        **kwargs (additional keyword arguments, *optional*):
+            Additional key word arguments passed along to the `wandb.init` method.
+    """
+
+    name = "wandb"
+    requires_logging_directory = False
+    main_process_only = False
+
+    def __init__(self, run_name: str, **kwargs):
+        super().__init__()
+        self.run_name = run_name
+        self.init_kwargs = kwargs
+
+    @on_main_process
+    def start(self):
+        import wandb
+
+        self.run = wandb.init(project=self.run_name, **self.init_kwargs)
+        logger.debug(f"Initialized WandB project {self.run_name}")
+        logger.debug(
+            "Make sure to log any initial configurations with `self.store_init_configuration` before training!"
+        )
+
+    @property
+    def tracker(self):
+        return self.run
+
+    @on_main_process
+    def store_init_configuration(self, values: dict):
+        """
+        Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment.
+
+        Args:
+            values (Dictionary `str` to `bool`, `str`, `float` or `int`):
+                Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`,
+                `str`, `float`, `int`, or `None`.
+        """
+        import wandb
+
+        if os.environ.get("WANDB_MODE") == "offline":
+            # In offline mode, restart wandb with config included
+            if hasattr(self, "run") and self.run:
+                self.run.finish()
+
+            init_kwargs = self.init_kwargs.copy()
+            init_kwargs["config"] = values
+            self.run = wandb.init(project=self.run_name, **init_kwargs)
+        else:
+            wandb.config.update(values, allow_val_change=True)
+        logger.debug("Stored initial configuration hyperparameters to WandB")
+
+    @on_main_process
+    def log(self, values: dict, step: Optional[int] = None, **kwargs):
+        """
+        Logs `values` to the current run.
+
+        Args:
+            values (Dictionary `str` to `str`, `float`, `int` or `dict` of `str` to `float`/`int`):
+                Values to be logged as key-value pairs. The values need to have type `str`, `float`, `int` or `dict` of
+                `str` to `float`/`int`.
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs:
+                Additional key word arguments passed along to the `wandb.log` method.
+        """
+        self.run.log(values, step=step, **kwargs)
+        logger.debug("Successfully logged to WandB")
+
+    @on_main_process
+    def log_images(self, values: dict, step: Optional[int] = None, **kwargs):
+        """
+        Logs `images` to the current run.
+
+        Args:
+            values (Dictionary `str` to `List` of `np.ndarray` or `PIL.Image`):
+                Values to be logged as key-value pairs. The values need to have type `List` of `np.ndarray` or
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs:
+                Additional key word arguments passed along to the `wandb.log` method.
+        """
+        import wandb
+
+        for k, v in values.items():
+            self.log({k: [wandb.Image(image) for image in v]}, step=step, **kwargs)
+        logger.debug("Successfully logged images to WandB")
+
+    @on_main_process
+    def log_table(
+        self,
+        table_name: str,
+        columns: Optional[list[str]] = None,
+        data: Optional[list[list[Any]]] = None,
+        dataframe: Any = None,
+        step: Optional[int] = None,
+        **kwargs,
+    ):
+        """
+        Log a Table containing any object type (text, image, audio, video, molecule, html, etc). Can be defined either
+        with `columns` and `data` or with `dataframe`.
+
+        Args:
+            table_name (`str`):
+                The name to give to the logged table on the wandb workspace
+            columns (list of `str`, *optional*):
+                The name of the columns on the table
+            data (List of List of Any data type, *optional*):
+                The data to be logged in the table
+            dataframe (Any data type, *optional*):
+                The data to be logged in the table
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+        """
+        import wandb
+
+        values = {table_name: wandb.Table(columns=columns, data=data, dataframe=dataframe)}
+        self.log(values, step=step, **kwargs)
+
+    @on_main_process
+    def finish(self):
+        """
+        Closes `wandb` writer
+        """
+        self.run.finish()
+        logger.debug("WandB run closed")
+
+
+class TrackioTracker(GeneralTracker):
+    """
+    A `Tracker` class that supports `trackio`. Should be initialized at the start of your script.
+
+    Args:
+        run_name (`str`):
+            The name of the experiment run. Will be used as the `project` name when instantiating trackio.
+        **kwargs (additional keyword arguments, *optional*):
+            Additional key word arguments passed along to the `trackio.init` method. Refer to this
+            [init](https://github.com/gradio-app/trackio/blob/814809552310468b13f84f33764f1369b4e5136c/trackio/__init__.py#L22)
+            to see all supported key word arguments.
+    """
+
+    name = "trackio"
+    requires_logging_directory = False
+    main_process_only = False
+
+    def __init__(self, run_name: str, **kwargs):
+        super().__init__()
+        self.run_name = run_name
+        self.init_kwargs = kwargs
+
+    @on_main_process
+    def start(self):
+        import trackio
+
+        self.run = trackio.init(project=self.run_name, **self.init_kwargs)
+        logger.debug(f"Initialized trackio project {self.run_name}")
+        logger.debug(
+            "Make sure to log any initial configurations with `self.store_init_configuration` before training!"
+        )
+
+    @property
+    def tracker(self):
+        return self.run
+
+    @on_main_process
+    def store_init_configuration(self, values: dict):
+        """
+        Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment.
+
+        Args:
+            values (Dictionary `str` to `bool`, `str`, `float` or `int`):
+                Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`,
+                `str`, `float`, `int`, or `None`.
+        """
+        import trackio
+
+        trackio.config.update(values, allow_val_change=True)
+        logger.debug("Stored initial configuration hyperparameters to trackio")
+
+    @on_main_process
+    def log(self, values: dict, step: Optional[int] = None, **kwargs):
+        """
+        Logs `values` to the current run.
+
+        Args:
+            values (Dictionary `str` to `str`, `float`, `int` or `dict` of `str` to `float`/`int`):
+                Values to be logged as key-value pairs. The values need to have type `str`, `float`, `int` or `dict` of
+                `str` to `float`/`int`.
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs:
+                Additional key word arguments passed along to the `trackio.log` method.
+        """
+        self.run.log(values, **kwargs)
+        logger.debug("Successfully logged to trackio")
+
+    @on_main_process
+    def finish(self):
+        """
+        Closes `trackio` run
+        """
+        self.run.finish()
+        logger.debug("trackio run closed")
+
+
+class CometMLTracker(GeneralTracker):
+    """
+    A `Tracker` class that supports `comet_ml`. Should be initialized at the start of your script.
+
+    API keys must be stored in a Comet config file.
+
+    Note:
+        For `comet_ml` versions < 3.41.0, additional keyword arguments are passed to `comet_ml.Experiment` instead:
+        https://www.comet.com/docs/v2/api-and-sdk/python-sdk/reference/Experiment/#comet_ml.Experiment.__init__
+
+    Args:
+        run_name (`str`):
+            The name of the experiment run.
+        **kwargs (additional keyword arguments, *optional*):
+            Additional key word arguments passed along to the `comet_ml.start` method:
+            https://www.comet.com/docs/v2/api-and-sdk/python-sdk/reference/start/
+    """
+
+    name = "comet_ml"
+    requires_logging_directory = False
+
+    def __init__(self, run_name: str, **kwargs):
+        super().__init__()
+        self.run_name = run_name
+        self.init_kwargs = kwargs
+
+    @on_main_process
+    def start(self):
+        import comet_ml
+
+        comet_version = version.parse(comet_ml.__version__)
+        if compare_versions(comet_version, ">=", "3.41.0"):
+            self.writer = comet_ml.start(project_name=self.run_name, **self.init_kwargs)
+        else:
+            logger.info("Update `comet_ml` (>=3.41.0) for experiment reuse and offline support.")
+            self.writer = comet_ml.Experiment(project_name=self.run_name, **self.init_kwargs)
+
+        logger.debug(f"Initialized CometML project {self.run_name}")
+        logger.debug(
+            "Make sure to log any initial configurations with `self.store_init_configuration` before training!"
+        )
+
+    @property
+    def tracker(self):
+        return self.writer
+
+    @on_main_process
+    def store_init_configuration(self, values: dict):
+        """
+        Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment.
+
+        Args:
+            values (Dictionary `str` to `bool`, `str`, `float` or `int`):
+                Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`,
+                `str`, `float`, `int`, or `None`.
+        """
+        self.writer.log_parameters(values)
+        logger.debug("Stored initial configuration hyperparameters to Comet")
+
+    @on_main_process
+    def log(self, values: dict, step: Optional[int] = None, **kwargs):
+        """
+        Logs `values` to the current run.
+
+        Args:
+            values (Dictionary `str` to `str`, `float`, `int` or `dict` of `str` to `float`/`int`):
+                Values to be logged as key-value pairs. The values need to have type `str`, `float`, `int` or `dict` of
+                `str` to `float`/`int`.
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs:
+                Additional key word arguments passed along to either `Experiment.log_metric`, `Experiment.log_other`,
+                or `Experiment.log_metrics` method based on the contents of `values`.
+        """
+        if step is not None:
+            self.writer.set_step(step)
+        for k, v in values.items():
+            if isinstance(v, (int, float)):
+                self.writer.log_metric(k, v, step=step, **kwargs)
+            elif isinstance(v, str):
+                self.writer.log_other(k, v, **kwargs)
+            elif isinstance(v, dict):
+                self.writer.log_metrics(v, step=step, **kwargs)
+        logger.debug("Successfully logged to Comet")
+
+    @on_main_process
+    def finish(self):
+        """
+        Flush `comet-ml` writer
+        """
+        self.writer.end()
+        logger.debug("Comet run flushed")
+
+
+class AimTracker(GeneralTracker):
+    """
+    A `Tracker` class that supports `aim`. Should be initialized at the start of your script.
+
+    Args:
+        run_name (`str`):
+            The name of the experiment run.
+        **kwargs (additional keyword arguments, *optional*):
+            Additional key word arguments passed along to the `Run.__init__` method.
+    """
+
+    name = "aim"
+    requires_logging_directory = True
+
+    def __init__(self, run_name: str, logging_dir: Optional[Union[str, os.PathLike]] = ".", **kwargs):
+        super().__init__()
+        self.run_name = run_name
+        self.aim_repo_path = logging_dir
+        self.init_kwargs = kwargs
+
+    @on_main_process
+    def start(self):
+        from aim import Run
+
+        self.writer = Run(repo=self.aim_repo_path, **self.init_kwargs)
+        self.writer.name = self.run_name
+        logger.debug(f"Initialized Aim project {self.run_name}")
+        logger.debug(
+            "Make sure to log any initial configurations with `self.store_init_configuration` before training!"
+        )
+
+    @property
+    def tracker(self):
+        return self.writer
+
+    @on_main_process
+    def store_init_configuration(self, values: dict):
+        """
+        Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment.
+
+        Args:
+            values (`dict`):
+                Values to be stored as initial hyperparameters as key-value pairs.
+        """
+        self.writer["hparams"] = values
+
+    @on_main_process
+    def log(self, values: dict, step: Optional[int], **kwargs):
+        """
+        Logs `values` to the current run.
+
+        Args:
+            values (`dict`):
+                Values to be logged as key-value pairs.
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs:
+                Additional key word arguments passed along to the `Run.track` method.
+        """
+        # Note: replace this with the dictionary support when merged
+        for key, value in values.items():
+            self.writer.track(value, name=key, step=step, **kwargs)
+
+    @on_main_process
+    def log_images(self, values: dict, step: Optional[int] = None, kwargs: Optional[dict[str, dict]] = None):
+        """
+        Logs `images` to the current run.
+
+        Args:
+            values (`Dict[str, Union[np.ndarray, PIL.Image, Tuple[np.ndarray, str], Tuple[PIL.Image, str]]]`):
+                Values to be logged as key-value pairs. The values need to have type `np.ndarray` or PIL.Image. If a
+                tuple is provided, the first element should be the image and the second element should be the caption.
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs (`Dict[str, dict]`):
+                Additional key word arguments passed along to the `Run.Image` and `Run.track` method specified by the
+                keys `aim_image` and `track`, respectively.
+        """
+        import aim
+
+        aim_image_kw = {}
+        track_kw = {}
+
+        if kwargs is not None:
+            aim_image_kw = kwargs.get("aim_image", {})
+            track_kw = kwargs.get("track", {})
+
+        for key, value in values.items():
+            if isinstance(value, tuple):
+                img, caption = value
+            else:
+                img, caption = value, ""
+            aim_image = aim.Image(img, caption=caption, **aim_image_kw)
+            self.writer.track(aim_image, name=key, step=step, **track_kw)
+
+    @on_main_process
+    def finish(self):
+        """
+        Closes `aim` writer
+        """
+        self.writer.close()
+
+
+class MLflowTracker(GeneralTracker):
+    """
+    A `Tracker` class that supports `mlflow`. Should be initialized at the start of your script.
+
+    Args:
+        experiment_name (`str`, *optional*):
+            Name of the experiment. Environment variable MLFLOW_EXPERIMENT_NAME has priority over this argument.
+        logging_dir (`str` or `os.PathLike`, defaults to `"."`):
+            Location for mlflow logs to be stored.
+        run_id (`str`, *optional*):
+            If specified, get the run with the specified UUID and log parameters and metrics under that run. The run’s
+            end time is unset and its status is set to running, but the run’s other attributes (source_version,
+            source_type, etc.) are not changed. Environment variable MLFLOW_RUN_ID has priority over this argument.
+        tags (`Dict[str, str]`, *optional*):
+            An optional `dict` of `str` keys and values, or a `str` dump from a `dict`, to set as tags on the run. If a
+            run is being resumed, these tags are set on the resumed run. If a new run is being created, these tags are
+            set on the new run. Environment variable MLFLOW_TAGS has priority over this argument.
+        nested_run (`bool`, *optional*, defaults to `False`):
+            Controls whether run is nested in parent run. True creates a nested run. Environment variable
+            MLFLOW_NESTED_RUN has priority over this argument.
+        run_name (`str`, *optional*):
+            Name of new run (stored as a mlflow.runName tag). Used only when `run_id` is unspecified.
+        description (`str`, *optional*):
+            An optional string that populates the description box of the run. If a run is being resumed, the
+            description is set on the resumed run. If a new run is being created, the description is set on the new
+            run.
+    """
+
+    name = "mlflow"
+    requires_logging_directory = False
+
+    def __init__(
+        self,
+        experiment_name: Optional[str] = None,
+        logging_dir: Optional[Union[str, os.PathLike]] = None,
+        run_id: Optional[str] = None,
+        tags: Optional[Union[dict[str, Any], str]] = None,
+        nested_run: Optional[bool] = False,
+        run_name: Optional[str] = None,
+        description: Optional[str] = None,
+    ):
+        experiment_name = os.environ.get("MLFLOW_EXPERIMENT_NAME", experiment_name)
+        run_id = os.environ.get("MLFLOW_RUN_ID", run_id)
+        tags = os.environ.get("MLFLOW_TAGS", tags)
+        if isinstance(tags, str):
+            tags = json.loads(tags)
+
+        nested_run = os.environ.get("MLFLOW_NESTED_RUN", nested_run)
+
+        self.experiment_name = experiment_name
+        self.logging_dir = logging_dir
+        self.run_id = run_id
+        self.tags = tags
+        self.nested_run = nested_run
+        self.run_name = run_name
+        self.description = description
+
+    @on_main_process
+    def start(self):
+        import mlflow
+
+        exps = mlflow.search_experiments(filter_string=f"name = '{self.experiment_name}'")
+        if len(exps) > 0:
+            if len(exps) > 1:
+                logger.warning("Multiple experiments with the same name found. Using first one.")
+            experiment_id = exps[0].experiment_id
+        else:
+            experiment_id = mlflow.create_experiment(
+                name=self.experiment_name,
+                artifact_location=self.logging_dir,
+                tags=self.tags,
+            )
+
+        self.active_run = mlflow.start_run(
+            run_id=self.run_id,
+            experiment_id=experiment_id,
+            run_name=self.run_name,
+            nested=self.nested_run,
+            tags=self.tags,
+            description=self.description,
+        )
+
+        logger.debug(f"Initialized mlflow experiment {self.experiment_name}")
+        logger.debug(
+            "Make sure to log any initial configurations with `self.store_init_configuration` before training!"
+        )
+
+    @property
+    def tracker(self):
+        return self.active_run
+
+    @on_main_process
+    def store_init_configuration(self, values: dict):
+        """
+        Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment.
+
+        Args:
+            values (`dict`):
+                Values to be stored as initial hyperparameters as key-value pairs.
+        """
+        import mlflow
+
+        for name, value in list(values.items()):
+            # internally, all values are converted to str in MLflow
+            if len(str(value)) > mlflow.utils.validation.MAX_PARAM_VAL_LENGTH:
+                logger.warning_once(
+                    f'Accelerate is attempting to log a value of "{value}" for key "{name}" as a parameter. MLflow\'s'
+                    f" log_param() only accepts values no longer than {mlflow.utils.validation.MAX_PARAM_VAL_LENGTH} characters so we dropped this attribute."
+                )
+                del values[name]
+
+        values_list = list(values.items())
+
+        # MLflow cannot log more than 100 values in one go, so we have to split it
+        for i in range(0, len(values_list), mlflow.utils.validation.MAX_PARAMS_TAGS_PER_BATCH):
+            mlflow.log_params(dict(values_list[i : i + mlflow.utils.validation.MAX_PARAMS_TAGS_PER_BATCH]))
+
+        logger.debug("Stored initial configuration hyperparameters to MLflow")
+
+    @on_main_process
+    def log(self, values: dict, step: Optional[int]):
+        """
+        Logs `values` to the current run.
+
+        Args:
+            values (`dict`):
+                Values to be logged as key-value pairs.
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+        """
+        metrics = {}
+        for k, v in values.items():
+            if isinstance(v, (int, float)):
+                metrics[k] = v
+            else:
+                logger.warning_once(
+                    f'MLflowTracker is attempting to log a value of "{v}" of type {type(v)} for key "{k}" as a metric. '
+                    "MLflow's log_metric() only accepts float and int types so we dropped this attribute."
+                )
+        import mlflow
+
+        mlflow.log_metrics(metrics, step=step)
+        logger.debug("Successfully logged to mlflow")
+
+    @on_main_process
+    def log_figure(self, figure: Any, artifact_file: str, **save_kwargs):
+        """
+        Logs an figure to the current run.
+
+        Args:
+            figure (Any):
+            The figure to be logged.
+            artifact_file (`str`, *optional*):
+            The run-relative artifact file path in posixpath format to which the image is saved.
+            If not provided, the image is saved to a default location.
+            **kwargs:
+            Additional keyword arguments passed to the underlying mlflow.log_image function.
+        """
+        import mlflow
+
+        mlflow.log_figure(figure=figure, artifact_file=artifact_file, **save_kwargs)
+        logger.debug("Successfully logged image to mlflow")
+
+    @on_main_process
+    def log_artifacts(self, local_dir: str, artifact_path: Optional[str] = None):
+        """
+        Logs an artifacts (all content of a dir) to the current run.
+
+            local_dir (`str`):
+                Path to the directory to be logged as an artifact.
+            artifact_path (`str`, *optional*):
+                Directory within the run's artifact directory where the artifact will be logged. If omitted, the
+                artifact will be logged to the root of the run's artifact directory. The run step. If included, the
+                artifact will be affiliated with this step.
+        """
+        import mlflow
+
+        mlflow.log_artifacts(local_dir=local_dir, artifact_path=artifact_path)
+        logger.debug("Successfully logged artofact to mlflow")
+
+    @on_main_process
+    def log_artifact(self, local_path: str, artifact_path: Optional[str] = None):
+        """
+        Logs an artifact (file) to the current run.
+
+            local_path (`str`):
+                Path to the file to be logged as an artifact.
+            artifact_path (`str`, *optional*):
+                Directory within the run's artifact directory where the artifact will be logged. If omitted, the
+                artifact will be logged to the root of the run's artifact directory. The run step. If included, the
+                artifact will be affiliated with this step.
+        """
+        import mlflow
+
+        mlflow.log_artifact(local_path=local_path, artifact_path=artifact_path)
+        logger.debug("Successfully logged artofact to mlflow")
+
+    @on_main_process
+    def finish(self):
+        """
+        End the active MLflow run.
+        """
+        import mlflow
+
+        mlflow.end_run()
+
+
+class ClearMLTracker(GeneralTracker):
+    """
+    A `Tracker` class that supports `clearml`. Should be initialized at the start of your script.
+
+    Args:
+        run_name (`str`, *optional*):
+            Name of the experiment. Environment variables `CLEARML_PROJECT` and `CLEARML_TASK` have priority over this
+            argument.
+        **kwargs (additional keyword arguments, *optional*):
+            Kwargs passed along to the `Task.__init__` method.
+    """
+
+    name = "clearml"
+    requires_logging_directory = False
+
+    def __init__(self, run_name: Optional[str] = None, **kwargs):
+        super().__init__()
+        self.user_provided_run_name = run_name
+        self._initialized_externally = False
+        self.init_kwargs = kwargs
+
+    @on_main_process
+    def start(self):
+        from clearml import Task
+
+        current_task = Task.current_task()
+        if current_task:
+            self._initialized_externally = True
+            self.task = current_task
+            return
+
+        task_init_args = {**self.init_kwargs}
+        task_init_args.setdefault("project_name", os.environ.get("CLEARML_PROJECT", self.user_provided_run_name))
+        task_init_args.setdefault("task_name", os.environ.get("CLEARML_TASK", self.user_provided_run_name))
+        self.task = Task.init(**task_init_args)
+
+    @property
+    def tracker(self):
+        return self.task
+
+    @on_main_process
+    def store_init_configuration(self, values: dict):
+        """
+        Connect configuration dictionary to the Task object. Should be run at the beginning of your experiment.
+
+        Args:
+            values (`dict`):
+                Values to be stored as initial hyperparameters as key-value pairs.
+        """
+        return self.task.connect_configuration(values)
+
+    @on_main_process
+    def log(self, values: dict[str, Union[int, float]], step: Optional[int] = None, **kwargs):
+        """
+        Logs `values` dictionary to the current run. The dictionary keys must be strings. The dictionary values must be
+        ints or floats
+
+        Args:
+            values (`Dict[str, Union[int, float]]`):
+                Values to be logged as key-value pairs. If the key starts with 'eval_'/'test_'/'train_', the value will
+                be reported under the 'eval'/'test'/'train' series and the respective prefix will be removed.
+                Otherwise, the value will be reported under the 'train' series, and no prefix will be removed.
+            step (`int`, *optional*):
+                If specified, the values will be reported as scalars, with the iteration number equal to `step`.
+                Otherwise they will be reported as single values.
+            kwargs:
+                Additional key word arguments passed along to the `clearml.Logger.report_single_value` or
+                `clearml.Logger.report_scalar` methods.
+        """
+        clearml_logger = self.task.get_logger()
+        for k, v in values.items():
+            if not isinstance(v, (int, float)):
+                logger.warning_once(
+                    "Accelerator is attempting to log a value of "
+                    f'"{v}" of type {type(v)} for key "{k}" as a scalar. '
+                    "This invocation of ClearML logger's  report_scalar() "
+                    "is incorrect so we dropped this attribute."
+                )
+                continue
+            if step is None:
+                clearml_logger.report_single_value(name=k, value=v, **kwargs)
+                continue
+            title, series = ClearMLTracker._get_title_series(k)
+            clearml_logger.report_scalar(title=title, series=series, value=v, iteration=step, **kwargs)
+
+    @on_main_process
+    def log_images(self, values: dict, step: Optional[int] = None, **kwargs):
+        """
+        Logs `images` to the current run.
+
+        Args:
+            values (`Dict[str, List[Union[np.ndarray, PIL.Image]]`):
+                Values to be logged as key-value pairs. The values need to have type `List` of `np.ndarray` or
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs:
+                Additional key word arguments passed along to the `clearml.Logger.report_image` method.
+        """
+        clearml_logger = self.task.get_logger()
+        for k, v in values.items():
+            title, series = ClearMLTracker._get_title_series(k)
+            clearml_logger.report_image(title=title, series=series, iteration=step, image=v, **kwargs)
+
+    @on_main_process
+    def log_table(
+        self,
+        table_name: str,
+        columns: Optional[list[str]] = None,
+        data: Optional[list[list[Any]]] = None,
+        dataframe: Any = None,
+        step: Optional[int] = None,
+        **kwargs,
+    ):
+        """
+        Log a Table to the task. Can be defined eitherwith `columns` and `data` or with `dataframe`.
+
+        Args:
+            table_name (`str`):
+                The name of the table
+            columns (list of `str`, *optional*):
+                The name of the columns on the table
+            data (List of List of Any data type, *optional*):
+                The data to be logged in the table. If `columns` is not specified, then the first entry in data will be
+                the name of the columns of the table
+            dataframe (Any data type, *optional*):
+                The data to be logged in the table
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs:
+                Additional key word arguments passed along to the `clearml.Logger.report_table` method.
+        """
+        to_report = dataframe
+        if dataframe is None:
+            if data is None:
+                raise ValueError(
+                    "`ClearMLTracker.log_table` requires that `data` to be supplied if `dataframe` is `None`"
+                )
+            to_report = [columns] + data if columns else data
+        title, series = ClearMLTracker._get_title_series(table_name)
+        self.task.get_logger().report_table(title=title, series=series, table_plot=to_report, iteration=step, **kwargs)
+
+    @on_main_process
+    def finish(self):
+        """
+        Close the ClearML task. If the task was initialized externally (e.g. by manually calling `Task.init`), this
+        function is a noop
+        """
+        if self.task and not self._initialized_externally:
+            self.task.close()
+
+    @staticmethod
+    def _get_title_series(name):
+        for prefix in ["eval", "test", "train"]:
+            if name.startswith(prefix + "_"):
+                return name[len(prefix) + 1 :], prefix
+        return name, "train"
+
+
+class DVCLiveTracker(GeneralTracker):
+    """
+    A `Tracker` class that supports `dvclive`. Should be initialized at the start of your script.
+
+    Args:
+        run_name (`str`, *optional*):
+            Ignored for dvclive. See `kwargs` instead.
+        kwargs:
+            Additional key word arguments passed along to [`dvclive.Live()`](https://dvc.org/doc/dvclive/live).
+
+    Example:
+
+    ```py
+    from accelerate import Accelerator
+
+    accelerator = Accelerator(log_with="dvclive")
+    accelerator.init_trackers(project_name="my_project", init_kwargs={"dvclive": {"dir": "my_directory"}})
+    ```
+    """
+
+    name = "dvclive"
+    requires_logging_directory = False
+
+    def __init__(self, run_name: Optional[str] = None, live: Optional[Any] = None, **kwargs):
+        super().__init__()
+        self.live = live
+        self.init_kwargs = kwargs
+
+    @on_main_process
+    def start(self):
+        from dvclive import Live
+
+        self.live = self.live if self.live is not None else Live(**self.init_kwargs)
+
+    @property
+    def tracker(self):
+        return self.live
+
+    @on_main_process
+    def store_init_configuration(self, values: dict):
+        """
+        Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment. Stores the
+        hyperparameters in a yaml file for future use.
+
+        Args:
+            values (Dictionary `str` to `bool`, `str`, `float`, `int`, or a List or Dict of those types):
+                Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`,
+                `str`, `float`, or `int`.
+        """
+        self.live.log_params(values)
+
+    @on_main_process
+    def log(self, values: dict, step: Optional[int] = None, **kwargs):
+        """
+        Logs `values` to the current run.
+
+        Args:
+            values (Dictionary `str` to `str`, `float`, or `int`):
+                Values to be logged as key-value pairs. The values need to have type `str`, `float`, or `int`.
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs:
+                Additional key word arguments passed along to `dvclive.Live.log_metric()`.
+        """
+        from dvclive.plots import Metric
+
+        if step is not None:
+            self.live.step = step
+        for k, v in values.items():
+            if Metric.could_log(v):
+                self.live.log_metric(k, v, **kwargs)
+            else:
+                logger.warning_once(
+                    "Accelerator attempted to log a value of "
+                    f'"{v}" of type {type(v)} for key "{k}" as a scalar. '
+                    "This invocation of DVCLive's Live.log_metric() "
+                    "is incorrect so we dropped this attribute."
+                )
+        self.live.next_step()
+
+    @on_main_process
+    def finish(self):
+        """
+        Closes `dvclive.Live()`.
+        """
+        self.live.end()
+
+
+class SwanLabTracker(GeneralTracker):
+    """
+    A `Tracker` class that supports `swanlab`. Should be initialized at the start of your script.
+
+    Args:
+        run_name (`str`):
+            The name of the experiment run.
+        **kwargs (additional keyword arguments, *optional*):
+            Additional key word arguments passed along to the `swanlab.init` method.
+    """
+
+    name = "swanlab"
+    requires_logging_directory = False
+    main_process_only = False
+
+    def __init__(self, run_name: str, **kwargs):
+        super().__init__()
+        self.run_name = run_name
+        self.init_kwargs = kwargs
+
+    @on_main_process
+    def start(self):
+        import swanlab
+
+        self.run = swanlab.init(project=self.run_name, **self.init_kwargs)
+        swanlab.config["FRAMEWORK"] = "🤗Accelerate"  # add accelerate logo in config
+        logger.debug(f"Initialized SwanLab project {self.run_name}")
+        logger.debug(
+            "Make sure to log any initial configurations with `self.store_init_configuration` before training!"
+        )
+
+    @property
+    def tracker(self):
+        return self.run
+
+    @on_main_process
+    def store_init_configuration(self, values: dict):
+        """
+        Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment.
+
+        Args:
+            values (Dictionary `str` to `bool`, `str`, `float` or `int`):
+                Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`,
+                `str`, `float`, `int`, or `None`.
+        """
+        import swanlab
+
+        swanlab.config.update(values, allow_val_change=True)
+        logger.debug("Stored initial configuration hyperparameters to SwanLab")
+
+    @on_main_process
+    def log(self, values: dict, step: Optional[int] = None, **kwargs):
+        """
+        Logs `values` to the current run.
+
+        Args:
+        data : Dict[str, DataType]
+            Data must be a dict. The key must be a string with 0-9, a-z, A-Z, " ", "_", "-", "/". The value must be a
+            `float`, `float convertible object`, `int` or `swanlab.data.BaseType`.
+        step : int, optional
+            The step number of the current data, if not provided, it will be automatically incremented.
+        If step is duplicated, the data will be ignored.
+            kwargs:
+                Additional key word arguments passed along to the `swanlab.log` method. Likes:
+                    print_to_console : bool, optional
+                        Whether to print the data to the console, the default is False.
+        """
+        self.run.log(values, step=step, **kwargs)
+        logger.debug("Successfully logged to SwanLab")
+
+    @on_main_process
+    def log_images(self, values: dict, step: Optional[int] = None, **kwargs):
+        """
+        Logs `images` to the current run.
+
+        Args:
+            values (Dictionary `str` to `List` of `np.ndarray` or `PIL.Image`):
+                Values to be logged as key-value pairs. The values need to have type `List` of `np.ndarray` or
+            step (`int`, *optional*):
+                The run step. If included, the log will be affiliated with this step.
+            kwargs:
+                Additional key word arguments passed along to the `swanlab.log` method. Likes:
+                    print_to_console : bool, optional
+                        Whether to print the data to the console, the default is False.
+        """
+        import swanlab
+
+        for k, v in values.items():
+            self.log({k: [swanlab.Image(image) for image in v]}, step=step, **kwargs)
+        logger.debug("Successfully logged images to SwanLab")
+
+    @on_main_process
+    def finish(self):
+        """
+        Closes `swanlab` writer
+        """
+        self.run.finish()
+        logger.debug("SwanLab run closed")
+
+
+LOGGER_TYPE_TO_CLASS = {
+    "aim": AimTracker,
+    "comet_ml": CometMLTracker,
+    "mlflow": MLflowTracker,
+    "tensorboard": TensorBoardTracker,
+    "wandb": WandBTracker,
+    "clearml": ClearMLTracker,
+    "dvclive": DVCLiveTracker,
+    "swanlab": SwanLabTracker,
+    "trackio": TrackioTracker,
+}
+
+
+def filter_trackers(
+    log_with: list[Union[str, LoggerType, GeneralTracker]],
+    logging_dir: Optional[Union[str, os.PathLike]] = None,
+):
+    """
+    Takes in a list of potential tracker types and checks that:
+        - The tracker wanted is available in that environment
+        - Filters out repeats of tracker types
+        - If `all` is in `log_with`, will return all trackers in the environment
+        - If a tracker requires a `logging_dir`, ensures that `logging_dir` is not `None`
+
+    Args:
+        log_with (list of `str`, [`~utils.LoggerType`] or [`~tracking.GeneralTracker`], *optional*):
+            A list of loggers to be setup for experiment tracking. Should be one or several of:
+
+            - `"all"`
+            - `"tensorboard"`
+            - `"wandb"`
+            - `"trackio"`
+            - `"aim"`
+            - `"comet_ml"`
+            - `"mlflow"`
+            - `"dvclive"`
+            - `"swanlab"`
+            If `"all"` is selected, will pick up all available trackers in the environment and initialize them. Can
+            also accept implementations of `GeneralTracker` for custom trackers, and can be combined with `"all"`.
+        logging_dir (`str`, `os.PathLike`, *optional*):
+            A path to a directory for storing logs of locally-compatible loggers.
+    """
+    loggers = []
+    if log_with is not None:
+        if not isinstance(log_with, (list, tuple)):
+            log_with = [log_with]
+        if "all" in log_with or LoggerType.ALL in log_with:
+            loggers = [o for o in log_with if issubclass(type(o), GeneralTracker)] + get_available_trackers()
+        else:
+            for log_type in log_with:
+                if log_type not in LoggerType and not issubclass(type(log_type), GeneralTracker):
+                    raise ValueError(f"Unsupported logging capability: {log_type}. Choose between {LoggerType.list()}")
+                if issubclass(type(log_type), GeneralTracker):
+                    loggers.append(log_type)
+                else:
+                    log_type = LoggerType(log_type)
+                    if log_type not in loggers:
+                        if log_type in get_available_trackers():
+                            tracker_init = LOGGER_TYPE_TO_CLASS[str(log_type)]
+                            if tracker_init.requires_logging_directory:
+                                if logging_dir is None:
+                                    raise ValueError(
+                                        f"Logging with `{log_type}` requires a `logging_dir` to be passed in."
+                                    )
+                            loggers.append(log_type)
+                        else:
+                            logger.debug(f"Tried adding logger {log_type}, but package is unavailable in the system.")
+
+    return loggers

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda_pathfinder-1.4.0.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda_pathfinder-1.4.0.dist-info/METADATA

@@ -0,0 +1,47 @@
+Metadata-Version: 2.4
+Name: cuda-pathfinder
+Version: 1.4.0
+Summary: Pathfinder for CUDA components
+Author-email: NVIDIA Corporation <cuda-python-conduct@nvidia.com>
+License-Expression: Apache-2.0
+Project-URL: Repository, https://github.com/NVIDIA/cuda-python
+Project-URL: Documentation, https://nvidia.github.io/cuda-python/
+Requires-Python: >=3.10
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Dynamic: license-file
+
+.. SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+.. SPDX-License-Identifier: Apache-2.0
+
+*******************************************************
+cuda-pathfinder: Utilities for locating CUDA components
+*******************************************************
+
+.. image:: https://img.shields.io/badge/NVIDIA-black?logo=nvidia
+   :target: https://www.nvidia.com/
+   :alt: NVIDIA
+
+`cuda.pathfinder <https://nvidia.github.io/cuda-python/cuda-pathfinder/>`_
+aims to be a one-stop solution for locating CUDA components. Currently
+it supports locating and loading dynamic libraries (``.so``, ``.dll``), and
+locating CTK header directories. Support for other artifacts is in progress.
+
+* `Documentation <https://nvidia.github.io/cuda-python/cuda-pathfinder/>`_
+* `Releases <https://nvidia.github.io/cuda-python/cuda-pathfinder/latest/release.html>`_
+* `Repository <https://github.com/NVIDIA/cuda-python/tree/main/cuda_pathfinder/>`_
+* `Issue tracker <https://github.com/NVIDIA/cuda-python/issues/>`_ (select component ``cuda.pathfinder``)
+
+``cuda.pathfinder`` is under active development. Feedback and suggestions are welcome.
+
+
+Installation
+============
+
+.. code-block:: bash
+
+   pip install cuda-pathfinder
+
+``cuda-pathfinder`` is `CUDA Toolkit (CTK) <https://developer.nvidia.com/cuda-toolkit>`_
+version-agnostic. It follows the general CUDA Toolkit support policy: the
+two most recent major versions are supported simultaneously.

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda_pathfinder-1.4.0.dist-info/RECORD

@@ -0,0 +1,47 @@
+cuda/pathfinder/README.md,sha256=JPbUfq3xYFMGa1ZlnpHkKsGXjJi4Zr1nNU3SzWJIviw,123
+cuda/pathfinder/__init__.py,sha256=rHJIVMBSxsxzk46Z7cwZpjTyJ_fsQy7TOn4VlF8bgqs,3238
+cuda/pathfinder/__pycache__/__init__.cpython-312.pyc,,
+cuda/pathfinder/__pycache__/_version.cpython-312.pyc,,
+cuda/pathfinder/_binaries/__pycache__/find_nvidia_binary_utility.cpython-312.pyc,,
+cuda/pathfinder/_binaries/__pycache__/supported_nvidia_binaries.cpython-312.pyc,,
+cuda/pathfinder/_binaries/find_nvidia_binary_utility.py,sha256=8wrArz138JmogO9_ZA968OqQAf0rPkLYo9tndVUt4ug,4296
+cuda/pathfinder/_binaries/supported_nvidia_binaries.py,sha256=dSdytgVLC8vZ2aF-iTQJ0DnbV2CrzKePLe4X9yWUu40,1285
+cuda/pathfinder/_dynamic_libs/__pycache__/canary_probe_subprocess.cpython-312.pyc,,
+cuda/pathfinder/_dynamic_libs/__pycache__/find_nvidia_dynamic_lib.cpython-312.pyc,,
+cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_common.cpython-312.pyc,,
+cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_linux.cpython-312.pyc,,
+cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_windows.cpython-312.pyc,,
+cuda/pathfinder/_dynamic_libs/__pycache__/load_nvidia_dynamic_lib.cpython-312.pyc,,
+cuda/pathfinder/_dynamic_libs/__pycache__/supported_nvidia_libs.cpython-312.pyc,,
+cuda/pathfinder/_dynamic_libs/canary_probe_subprocess.py,sha256=GdzJwYzbtdsOrUB9LsPhUeImapPG0YE4M9o4USBdQxc,994
+cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py,sha256=kNVNzDU3mwvXc9O444vdJRbGD09KlS6qIo7dmHSYkhA,10197
+cuda/pathfinder/_dynamic_libs/load_dl_common.py,sha256=vOuh20EKkNKwbRUZFFP621RcjCvgSoKTzaXVmRyW8-k,831
+cuda/pathfinder/_dynamic_libs/load_dl_linux.py,sha256=aSKBgYOm0CCjT-0TixLOgNOtD4kvHHe8gjCBH2aXhjk,8010
+cuda/pathfinder/_dynamic_libs/load_dl_windows.py,sha256=hznwPky7mh5GRYUw12VWcuqrrV-tDLQHCrR6uevbLyQ,6066
+cuda/pathfinder/_dynamic_libs/load_nvidia_dynamic_lib.py,sha256=Y8ZBeOw3_LfQKcPqTvd3yfE3OO8cIZXPfcg9aFLZwnU,11857
+cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py,sha256=G_Y4C4OdVu768Y8D018PiqhzC7i4Yhi5if3A_aH7oJ0,14330
+cuda/pathfinder/_headers/__pycache__/find_nvidia_headers.cpython-312.pyc,,
+cuda/pathfinder/_headers/__pycache__/supported_nvidia_headers.cpython-312.pyc,,
+cuda/pathfinder/_headers/find_nvidia_headers.py,sha256=FUxWFuHe2yZ91bAvt32vJOat0RSyUwNtAstofhCEOrs,7315
+cuda/pathfinder/_headers/supported_nvidia_headers.py,sha256=GIC9Bkh2ufPbOSXxa5vBpsdVnkXCI67F-UCvtdLcYkE,3374
+cuda/pathfinder/_static_libs/__pycache__/find_bitcode_lib.cpython-312.pyc,,
+cuda/pathfinder/_static_libs/find_bitcode_lib.py,sha256=W_AyfNCZ9zx8-2W4ZdYjH6aiH6ic2u9ZlDaY-PsvwiU,5460
+cuda/pathfinder/_utils/__pycache__/env_vars.cpython-312.pyc,,
+cuda/pathfinder/_utils/__pycache__/find_site_packages_dll.cpython-312.pyc,,
+cuda/pathfinder/_utils/__pycache__/find_site_packages_so.cpython-312.pyc,,
+cuda/pathfinder/_utils/__pycache__/find_sub_dirs.cpython-312.pyc,,
+cuda/pathfinder/_utils/__pycache__/platform_aware.cpython-312.pyc,,
+cuda/pathfinder/_utils/__pycache__/spawned_process_runner.cpython-312.pyc,,
+cuda/pathfinder/_utils/env_vars.py,sha256=wRpf6XnohhKIpiGz_HbvnGg9C9u0uY5uU3t9OyYCl1A,1756
+cuda/pathfinder/_utils/find_site_packages_dll.py,sha256=HixAy-Ubd5Gu9DZwCLAu2jvlEr1nfolOSZ_7GX3ZgbM,1143
+cuda/pathfinder/_utils/find_site_packages_so.py,sha256=_udOAI8RtTXRJWz-X6TUy-g9W_nW8PeAcG61k1AVSLw,1638
+cuda/pathfinder/_utils/find_sub_dirs.py,sha256=L5SIXikH8U-6BCIvTotR4xwAelWUvJohgZ1RAuxg2eA,2032
+cuda/pathfinder/_utils/platform_aware.py,sha256=oOCqy_VyNdEdISVM2ilde8Kl6rOeElItvsAnw0WD9b8,463
+cuda/pathfinder/_utils/spawned_process_runner.py,sha256=De1idhZq6v3JF2Ar2d5e08XBsoqXyJiuPZlXilYi7Cc,4355
+cuda/pathfinder/_version.py,sha256=cWq_fKDGb85xfXQfl5uEbSwbhongIFne__ApDDkK9B4,712
+cuda_pathfinder-1.4.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+cuda_pathfinder-1.4.0.dist-info/METADATA,sha256=XrzGu-7KW6TGCSXmdhPRWx9t_nxpF4xfnS32vcuqG60,1906
+cuda_pathfinder-1.4.0.dist-info/RECORD,,
+cuda_pathfinder-1.4.0.dist-info/WHEEL,sha256=YCfwYGOYMi5Jhw2fU4yNgwErybb2IX5PEwBKV4ZbdBo,91
+cuda_pathfinder-1.4.0.dist-info/licenses/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+cuda_pathfinder-1.4.0.dist-info/top_level.txt,sha256=U5vpnvwNpaJF8bl4KnoUluMDTRt0J972FipwXjgNQ3A,5

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda_pathfinder-1.4.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda_pathfinder-1.4.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+cuda

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/fsspec/caching.py

@@ -0,0 +1,950 @@
+from __future__ import annotations
+
+import collections
+import functools
+import logging
+import math
+import os
+import threading
+import warnings
+from concurrent.futures import Future, ThreadPoolExecutor
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    ClassVar,
+    Generic,
+    NamedTuple,
+    OrderedDict,
+    TypeVar,
+)
+
+if TYPE_CHECKING:
+    import mmap
+
+    from typing_extensions import ParamSpec
+
+    P = ParamSpec("P")
+else:
+    P = TypeVar("P")
+
+T = TypeVar("T")
+
+
+logger = logging.getLogger("fsspec")
+
+Fetcher = Callable[[int, int], bytes]  # Maps (start, end) to bytes
+
+
+class BaseCache:
+    """Pass-though cache: doesn't keep anything, calls every time
+
+    Acts as base class for other cachers
+
+    Parameters
+    ----------
+    blocksize: int
+        How far to read ahead in numbers of bytes
+    fetcher: func
+        Function of the form f(start, end) which gets bytes from remote as
+        specified
+    size: int
+        How big this file is
+    """
+
+    name: ClassVar[str] = "none"
+
+    def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None:
+        self.blocksize = blocksize
+        self.nblocks = 0
+        self.fetcher = fetcher
+        self.size = size
+        self.hit_count = 0
+        self.miss_count = 0
+        # the bytes that we actually requested
+        self.total_requested_bytes = 0
+
+    def _fetch(self, start: int | None, stop: int | None) -> bytes:
+        if start is None:
+            start = 0
+        if stop is None:
+            stop = self.size
+        if start >= self.size or start >= stop:
+            return b""
+        return self.fetcher(start, stop)
+
+    def _reset_stats(self) -> None:
+        """Reset hit and miss counts for a more ganular report e.g. by file."""
+        self.hit_count = 0
+        self.miss_count = 0
+        self.total_requested_bytes = 0
+
+    def _log_stats(self) -> str:
+        """Return a formatted string of the cache statistics."""
+        if self.hit_count == 0 and self.miss_count == 0:
+            # a cache that does nothing, this is for logs only
+            return ""
+        return " , %s: %d hits, %d misses, %d total requested bytes" % (
+            self.name,
+            self.hit_count,
+            self.miss_count,
+            self.total_requested_bytes,
+        )
+
+    def __repr__(self) -> str:
+        # TODO: use rich for better formatting
+        return f"""
+        <{self.__class__.__name__}:
+            block size  :   {self.blocksize}
+            block count :   {self.nblocks}
+            file size   :   {self.size}
+            cache hits  :   {self.hit_count}
+            cache misses:   {self.miss_count}
+            total requested bytes: {self.total_requested_bytes}>
+        """
+
+
+class MMapCache(BaseCache):
+    """memory-mapped sparse file cache
+
+    Opens temporary file, which is filled blocks-wise when data is requested.
+    Ensure there is enough disc space in the temporary location.
+
+    This cache method might only work on posix
+    """
+
+    name = "mmap"
+
+    def __init__(
+        self,
+        blocksize: int,
+        fetcher: Fetcher,
+        size: int,
+        location: str | None = None,
+        blocks: set[int] | None = None,
+    ) -> None:
+        super().__init__(blocksize, fetcher, size)
+        self.blocks = set() if blocks is None else blocks
+        self.location = location
+        self.cache = self._makefile()
+
+    def _makefile(self) -> mmap.mmap | bytearray:
+        import mmap
+        import tempfile
+
+        if self.size == 0:
+            return bytearray()
+
+        # posix version
+        if self.location is None or not os.path.exists(self.location):
+            if self.location is None:
+                fd = tempfile.TemporaryFile()
+                self.blocks = set()
+            else:
+                fd = open(self.location, "wb+")
+            fd.seek(self.size - 1)
+            fd.write(b"1")
+            fd.flush()
+        else:
+            fd = open(self.location, "r+b")
+
+        return mmap.mmap(fd.fileno(), self.size)
+
+    def _fetch(self, start: int | None, end: int | None) -> bytes:
+        logger.debug(f"MMap cache fetching {start}-{end}")
+        if start is None:
+            start = 0
+        if end is None:
+            end = self.size
+        if start >= self.size or start >= end:
+            return b""
+        start_block = start // self.blocksize
+        end_block = end // self.blocksize
+        need = [i for i in range(start_block, end_block + 1) if i not in self.blocks]
+        hits = [i for i in range(start_block, end_block + 1) if i in self.blocks]
+        self.miss_count += len(need)
+        self.hit_count += len(hits)
+        while need:
+            # TODO: not a for loop so we can consolidate blocks later to
+            # make fewer fetch calls; this could be parallel
+            i = need.pop(0)
+
+            sstart = i * self.blocksize
+            send = min(sstart + self.blocksize, self.size)
+            self.total_requested_bytes += send - sstart
+            logger.debug(f"MMap get block #{i} ({sstart}-{send})")
+            self.cache[sstart:send] = self.fetcher(sstart, send)
+            self.blocks.add(i)
+
+        return self.cache[start:end]
+
+    def __getstate__(self) -> dict[str, Any]:
+        state = self.__dict__.copy()
+        # Remove the unpicklable entries.
+        del state["cache"]
+        return state
+
+    def __setstate__(self, state: dict[str, Any]) -> None:
+        # Restore instance attributes
+        self.__dict__.update(state)
+        self.cache = self._makefile()
+
+
+class ReadAheadCache(BaseCache):
+    """Cache which reads only when we get beyond a block of data
+
+    This is a much simpler version of BytesCache, and does not attempt to
+    fill holes in the cache or keep fragments alive. It is best suited to
+    many small reads in a sequential order (e.g., reading lines from a file).
+    """
+
+    name = "readahead"
+
+    def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None:
+        super().__init__(blocksize, fetcher, size)
+        self.cache = b""
+        self.start = 0
+        self.end = 0
+
+    def _fetch(self, start: int | None, end: int | None) -> bytes:
+        if start is None:
+            start = 0
+        if end is None or end > self.size:
+            end = self.size
+        if start >= self.size or start >= end:
+            return b""
+        l = end - start
+        if start >= self.start and end <= self.end:
+            # cache hit
+            self.hit_count += 1
+            return self.cache[start - self.start : end - self.start]
+        elif self.start <= start < self.end:
+            # partial hit
+            self.miss_count += 1
+            part = self.cache[start - self.start :]
+            l -= len(part)
+            start = self.end
+        else:
+            # miss
+            self.miss_count += 1
+            part = b""
+        end = min(self.size, end + self.blocksize)
+        self.total_requested_bytes += end - start
+        self.cache = self.fetcher(start, end)  # new block replaces old
+        self.start = start
+        self.end = self.start + len(self.cache)
+        return part + self.cache[:l]
+
+
+class FirstChunkCache(BaseCache):
+    """Caches the first block of a file only
+
+    This may be useful for file types where the metadata is stored in the header,
+    but is randomly accessed.
+    """
+
+    name = "first"
+
+    def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None:
+        if blocksize > size:
+            # this will buffer the whole thing
+            blocksize = size
+        super().__init__(blocksize, fetcher, size)
+        self.cache: bytes | None = None
+
+    def _fetch(self, start: int | None, end: int | None) -> bytes:
+        start = start or 0
+        if start > self.size:
+            logger.debug("FirstChunkCache: requested start > file size")
+            return b""
+
+        end = min(end, self.size)
+
+        if start < self.blocksize:
+            if self.cache is None:
+                self.miss_count += 1
+                if end > self.blocksize:
+                    self.total_requested_bytes += end
+                    data = self.fetcher(0, end)
+                    self.cache = data[: self.blocksize]
+                    return data[start:]
+                self.cache = self.fetcher(0, self.blocksize)
+                self.total_requested_bytes += self.blocksize
+            part = self.cache[start:end]
+            if end > self.blocksize:
+                self.total_requested_bytes += end - self.blocksize
+                part += self.fetcher(self.blocksize, end)
+            self.hit_count += 1
+            return part
+        else:
+            self.miss_count += 1
+            self.total_requested_bytes += end - start
+            return self.fetcher(start, end)
+
+
+class BlockCache(BaseCache):
+    """
+    Cache holding memory as a set of blocks.
+
+    Requests are only ever made ``blocksize`` at a time, and are
+    stored in an LRU cache. The least recently accessed block is
+    discarded when more than ``maxblocks`` are stored.
+
+    Parameters
+    ----------
+    blocksize : int
+        The number of bytes to store in each block.
+        Requests are only ever made for ``blocksize``, so this
+        should balance the overhead of making a request against
+        the granularity of the blocks.
+    fetcher : Callable
+    size : int
+        The total size of the file being cached.
+    maxblocks : int
+        The maximum number of blocks to cache for. The maximum memory
+        use for this cache is then ``blocksize * maxblocks``.
+    """
+
+    name = "blockcache"
+
+    def __init__(
+        self, blocksize: int, fetcher: Fetcher, size: int, maxblocks: int = 32
+    ) -> None:
+        super().__init__(blocksize, fetcher, size)
+        self.nblocks = math.ceil(size / blocksize)
+        self.maxblocks = maxblocks
+        self._fetch_block_cached = functools.lru_cache(maxblocks)(self._fetch_block)
+
+    def cache_info(self):
+        """
+        The statistics on the block cache.
+
+        Returns
+        -------
+        NamedTuple
+            Returned directly from the LRU Cache used internally.
+        """
+        return self._fetch_block_cached.cache_info()
+
+    def __getstate__(self) -> dict[str, Any]:
+        state = self.__dict__
+        del state["_fetch_block_cached"]
+        return state
+
+    def __setstate__(self, state: dict[str, Any]) -> None:
+        self.__dict__.update(state)
+        self._fetch_block_cached = functools.lru_cache(state["maxblocks"])(
+            self._fetch_block
+        )
+
+    def _fetch(self, start: int | None, end: int | None) -> bytes:
+        if start is None:
+            start = 0
+        if end is None:
+            end = self.size
+        if start >= self.size or start >= end:
+            return b""
+
+        # byte position -> block numbers
+        start_block_number = start // self.blocksize
+        end_block_number = end // self.blocksize
+
+        # these are cached, so safe to do multiple calls for the same start and end.
+        for block_number in range(start_block_number, end_block_number + 1):
+            self._fetch_block_cached(block_number)
+
+        return self._read_cache(
+            start,
+            end,
+            start_block_number=start_block_number,
+            end_block_number=end_block_number,
+        )
+
+    def _fetch_block(self, block_number: int) -> bytes:
+        """
+        Fetch the block of data for `block_number`.
+        """
+        if block_number > self.nblocks:
+            raise ValueError(
+                f"'block_number={block_number}' is greater than "
+                f"the number of blocks ({self.nblocks})"
+            )
+
+        start = block_number * self.blocksize
+        end = start + self.blocksize
+        self.total_requested_bytes += end - start
+        self.miss_count += 1
+        logger.info("BlockCache fetching block %d", block_number)
+        block_contents = super()._fetch(start, end)
+        return block_contents
+
+    def _read_cache(
+        self, start: int, end: int, start_block_number: int, end_block_number: int
+    ) -> bytes:
+        """
+        Read from our block cache.
+
+        Parameters
+        ----------
+        start, end : int
+            The start and end byte positions.
+        start_block_number, end_block_number : int
+            The start and end block numbers.
+        """
+        start_pos = start % self.blocksize
+        end_pos = end % self.blocksize
+
+        self.hit_count += 1
+        if start_block_number == end_block_number:
+            block: bytes = self._fetch_block_cached(start_block_number)
+            return block[start_pos:end_pos]
+
+        else:
+            # read from the initial
+            out = [self._fetch_block_cached(start_block_number)[start_pos:]]
+
+            # intermediate blocks
+            # Note: it'd be nice to combine these into one big request. However
+            # that doesn't play nicely with our LRU cache.
+            out.extend(
+                map(
+                    self._fetch_block_cached,
+                    range(start_block_number + 1, end_block_number),
+                )
+            )
+
+            # final block
+            out.append(self._fetch_block_cached(end_block_number)[:end_pos])
+
+            return b"".join(out)
+
+
+class BytesCache(BaseCache):
+    """Cache which holds data in a in-memory bytes object
+
+    Implements read-ahead by the block size, for semi-random reads progressing
+    through the file.
+
+    Parameters
+    ----------
+    trim: bool
+        As we read more data, whether to discard the start of the buffer when
+        we are more than a blocksize ahead of it.
+    """
+
+    name: ClassVar[str] = "bytes"
+
+    def __init__(
+        self, blocksize: int, fetcher: Fetcher, size: int, trim: bool = True
+    ) -> None:
+        super().__init__(blocksize, fetcher, size)
+        self.cache = b""
+        self.start: int | None = None
+        self.end: int | None = None
+        self.trim = trim
+
+    def _fetch(self, start: int | None, end: int | None) -> bytes:
+        # TODO: only set start/end after fetch, in case it fails?
+        # is this where retry logic might go?
+        if start is None:
+            start = 0
+        if end is None:
+            end = self.size
+        if start >= self.size or start >= end:
+            return b""
+        if (
+            self.start is not None
+            and start >= self.start
+            and self.end is not None
+            and end < self.end
+        ):
+            # cache hit: we have all the required data
+            offset = start - self.start
+            self.hit_count += 1
+            return self.cache[offset : offset + end - start]
+
+        if self.blocksize:
+            bend = min(self.size, end + self.blocksize)
+        else:
+            bend = end
+
+        if bend == start or start > self.size:
+            return b""
+
+        if (self.start is None or start < self.start) and (
+            self.end is None or end > self.end
+        ):
+            # First read, or extending both before and after
+            self.total_requested_bytes += bend - start
+            self.miss_count += 1
+            self.cache = self.fetcher(start, bend)
+            self.start = start
+        else:
+            assert self.start is not None
+            assert self.end is not None
+            self.miss_count += 1
+
+            if start < self.start:
+                if self.end is None or self.end - end > self.blocksize:
+                    self.total_requested_bytes += bend - start
+                    self.cache = self.fetcher(start, bend)
+                    self.start = start
+                else:
+                    self.total_requested_bytes += self.start - start
+                    new = self.fetcher(start, self.start)
+                    self.start = start
+                    self.cache = new + self.cache
+            elif self.end is not None and bend > self.end:
+                if self.end > self.size:
+                    pass
+                elif end - self.end > self.blocksize:
+                    self.total_requested_bytes += bend - start
+                    self.cache = self.fetcher(start, bend)
+                    self.start = start
+                else:
+                    self.total_requested_bytes += bend - self.end
+                    new = self.fetcher(self.end, bend)
+                    self.cache = self.cache + new
+
+        self.end = self.start + len(self.cache)
+        offset = start - self.start
+        out = self.cache[offset : offset + end - start]
+        if self.trim:
+            num = (self.end - self.start) // (self.blocksize + 1)
+            if num > 1:
+                self.start += self.blocksize * num
+                self.cache = self.cache[self.blocksize * num :]
+        return out
+
+    def __len__(self) -> int:
+        return len(self.cache)
+
+
+class AllBytes(BaseCache):
+    """Cache entire contents of the file"""
+
+    name: ClassVar[str] = "all"
+
+    def __init__(
+        self,
+        blocksize: int | None = None,
+        fetcher: Fetcher | None = None,
+        size: int | None = None,
+        data: bytes | None = None,
+    ) -> None:
+        super().__init__(blocksize, fetcher, size)  # type: ignore[arg-type]
+        if data is None:
+            self.miss_count += 1
+            self.total_requested_bytes += self.size
+            data = self.fetcher(0, self.size)
+        self.data = data
+
+    def _fetch(self, start: int | None, stop: int | None) -> bytes:
+        self.hit_count += 1
+        return self.data[start:stop]
+
+
+class KnownPartsOfAFile(BaseCache):
+    """
+    Cache holding known file parts.
+
+    Parameters
+    ----------
+    blocksize: int
+        How far to read ahead in numbers of bytes
+    fetcher: func
+        Function of the form f(start, end) which gets bytes from remote as
+        specified
+    size: int
+        How big this file is
+    data: dict
+        A dictionary mapping explicit `(start, stop)` file-offset tuples
+        with known bytes.
+    strict: bool, default True
+        Whether to fetch reads that go beyond a known byte-range boundary.
+        If `False`, any read that ends outside a known part will be zero
+        padded. Note that zero padding will not be used for reads that
+        begin outside a known byte-range.
+    """
+
+    name: ClassVar[str] = "parts"
+
+    def __init__(
+        self,
+        blocksize: int,
+        fetcher: Fetcher,
+        size: int,
+        data: dict[tuple[int, int], bytes] = {},
+        strict: bool = True,
+        **_: Any,
+    ):
+        super().__init__(blocksize, fetcher, size)
+        self.strict = strict
+
+        # simple consolidation of contiguous blocks
+        if data:
+            old_offsets = sorted(data.keys())
+            offsets = [old_offsets[0]]
+            blocks = [data.pop(old_offsets[0])]
+            for start, stop in old_offsets[1:]:
+                start0, stop0 = offsets[-1]
+                if start == stop0:
+                    offsets[-1] = (start0, stop)
+                    blocks[-1] += data.pop((start, stop))
+                else:
+                    offsets.append((start, stop))
+                    blocks.append(data.pop((start, stop)))
+
+            self.data = dict(zip(offsets, blocks))
+        else:
+            self.data = data
+
+    def _fetch(self, start: int | None, stop: int | None) -> bytes:
+        if start is None:
+            start = 0
+        if stop is None:
+            stop = self.size
+
+        out = b""
+        for (loc0, loc1), data in self.data.items():
+            # If self.strict=False, use zero-padded data
+            # for reads beyond the end of a "known" buffer
+            if loc0 <= start < loc1:
+                off = start - loc0
+                out = data[off : off + stop - start]
+                if not self.strict or loc0 <= stop <= loc1:
+                    # The request is within a known range, or
+                    # it begins within a known range, and we
+                    # are allowed to pad reads beyond the
+                    # buffer with zero
+                    out += b"\x00" * (stop - start - len(out))
+                    self.hit_count += 1
+                    return out
+                else:
+                    # The request ends outside a known range,
+                    # and we are being "strict" about reads
+                    # beyond the buffer
+                    start = loc1
+                    break
+
+        # We only get here if there is a request outside the
+        # known parts of the file. In an ideal world, this
+        # should never happen
+        if self.fetcher is None:
+            # We cannot fetch the data, so raise an error
+            raise ValueError(f"Read is outside the known file parts: {(start, stop)}. ")
+        # We can fetch the data, but should warn the user
+        # that this may be slow
+        warnings.warn(
+            f"Read is outside the known file parts: {(start, stop)}. "
+            f"IO/caching performance may be poor!"
+        )
+        logger.debug(f"KnownPartsOfAFile cache fetching {start}-{stop}")
+        self.total_requested_bytes += stop - start
+        self.miss_count += 1
+        return out + super()._fetch(start, stop)
+
+
+class UpdatableLRU(Generic[P, T]):
+    """
+    Custom implementation of LRU cache that allows updating keys
+
+    Used by BackgroudBlockCache
+    """
+
+    class CacheInfo(NamedTuple):
+        hits: int
+        misses: int
+        maxsize: int
+        currsize: int
+
+    def __init__(self, func: Callable[P, T], max_size: int = 128) -> None:
+        self._cache: OrderedDict[Any, T] = collections.OrderedDict()
+        self._func = func
+        self._max_size = max_size
+        self._hits = 0
+        self._misses = 0
+        self._lock = threading.Lock()
+
+    def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T:
+        if kwargs:
+            raise TypeError(f"Got unexpected keyword argument {kwargs.keys()}")
+        with self._lock:
+            if args in self._cache:
+                self._cache.move_to_end(args)
+                self._hits += 1
+                return self._cache[args]
+
+        result = self._func(*args, **kwargs)
+
+        with self._lock:
+            self._cache[args] = result
+            self._misses += 1
+            if len(self._cache) > self._max_size:
+                self._cache.popitem(last=False)
+
+        return result
+
+    def is_key_cached(self, *args: Any) -> bool:
+        with self._lock:
+            return args in self._cache
+
+    def add_key(self, result: T, *args: Any) -> None:
+        with self._lock:
+            self._cache[args] = result
+            if len(self._cache) > self._max_size:
+                self._cache.popitem(last=False)
+
+    def cache_info(self) -> UpdatableLRU.CacheInfo:
+        with self._lock:
+            return self.CacheInfo(
+                maxsize=self._max_size,
+                currsize=len(self._cache),
+                hits=self._hits,
+                misses=self._misses,
+            )
+
+
+class BackgroundBlockCache(BaseCache):
+    """
+    Cache holding memory as a set of blocks with pre-loading of
+    the next block in the background.
+
+    Requests are only ever made ``blocksize`` at a time, and are
+    stored in an LRU cache. The least recently accessed block is
+    discarded when more than ``maxblocks`` are stored. If the
+    next block is not in cache, it is loaded in a separate thread
+    in non-blocking way.
+
+    Parameters
+    ----------
+    blocksize : int
+        The number of bytes to store in each block.
+        Requests are only ever made for ``blocksize``, so this
+        should balance the overhead of making a request against
+        the granularity of the blocks.
+    fetcher : Callable
+    size : int
+        The total size of the file being cached.
+    maxblocks : int
+        The maximum number of blocks to cache for. The maximum memory
+        use for this cache is then ``blocksize * maxblocks``.
+    """
+
+    name: ClassVar[str] = "background"
+
+    def __init__(
+        self, blocksize: int, fetcher: Fetcher, size: int, maxblocks: int = 32
+    ) -> None:
+        super().__init__(blocksize, fetcher, size)
+        self.nblocks = math.ceil(size / blocksize)
+        self.maxblocks = maxblocks
+        self._fetch_block_cached = UpdatableLRU(self._fetch_block, maxblocks)
+
+        self._thread_executor = ThreadPoolExecutor(max_workers=1)
+        self._fetch_future_block_number: int | None = None
+        self._fetch_future: Future[bytes] | None = None
+        self._fetch_future_lock = threading.Lock()
+
+    def cache_info(self) -> UpdatableLRU.CacheInfo:
+        """
+        The statistics on the block cache.
+
+        Returns
+        -------
+        NamedTuple
+            Returned directly from the LRU Cache used internally.
+        """
+        return self._fetch_block_cached.cache_info()
+
+    def __getstate__(self) -> dict[str, Any]:
+        state = self.__dict__
+        del state["_fetch_block_cached"]
+        del state["_thread_executor"]
+        del state["_fetch_future_block_number"]
+        del state["_fetch_future"]
+        del state["_fetch_future_lock"]
+        return state
+
+    def __setstate__(self, state) -> None:
+        self.__dict__.update(state)
+        self._fetch_block_cached = UpdatableLRU(self._fetch_block, state["maxblocks"])
+        self._thread_executor = ThreadPoolExecutor(max_workers=1)
+        self._fetch_future_block_number = None
+        self._fetch_future = None
+        self._fetch_future_lock = threading.Lock()
+
+    def _fetch(self, start: int | None, end: int | None) -> bytes:
+        if start is None:
+            start = 0
+        if end is None:
+            end = self.size
+        if start >= self.size or start >= end:
+            return b""
+
+        # byte position -> block numbers
+        start_block_number = start // self.blocksize
+        end_block_number = end // self.blocksize
+
+        fetch_future_block_number = None
+        fetch_future = None
+        with self._fetch_future_lock:
+            # Background thread is running. Check we we can or must join it.
+            if self._fetch_future is not None:
+                assert self._fetch_future_block_number is not None
+                if self._fetch_future.done():
+                    logger.info("BlockCache joined background fetch without waiting.")
+                    self._fetch_block_cached.add_key(
+                        self._fetch_future.result(), self._fetch_future_block_number
+                    )
+                    # Cleanup the fetch variables. Done with fetching the block.
+                    self._fetch_future_block_number = None
+                    self._fetch_future = None
+                else:
+                    # Must join if we need the block for the current fetch
+                    must_join = bool(
+                        start_block_number
+                        <= self._fetch_future_block_number
+                        <= end_block_number
+                    )
+                    if must_join:
+                        # Copy to the local variables to release lock
+                        # before waiting for result
+                        fetch_future_block_number = self._fetch_future_block_number
+                        fetch_future = self._fetch_future
+
+                        # Cleanup the fetch variables. Have a local copy.
+                        self._fetch_future_block_number = None
+                        self._fetch_future = None
+
+        # Need to wait for the future for the current read
+        if fetch_future is not None:
+            logger.info("BlockCache waiting for background fetch.")
+            # Wait until result and put it in cache
+            self._fetch_block_cached.add_key(
+                fetch_future.result(), fetch_future_block_number
+            )
+
+        # these are cached, so safe to do multiple calls for the same start and end.
+        for block_number in range(start_block_number, end_block_number + 1):
+            self._fetch_block_cached(block_number)
+
+        # fetch next block in the background if nothing is running in the background,
+        # the block is within file and it is not already cached
+        end_block_plus_1 = end_block_number + 1
+        with self._fetch_future_lock:
+            if (
+                self._fetch_future is None
+                and end_block_plus_1 <= self.nblocks
+                and not self._fetch_block_cached.is_key_cached(end_block_plus_1)
+            ):
+                self._fetch_future_block_number = end_block_plus_1
+                self._fetch_future = self._thread_executor.submit(
+                    self._fetch_block, end_block_plus_1, "async"
+                )
+
+        return self._read_cache(
+            start,
+            end,
+            start_block_number=start_block_number,
+            end_block_number=end_block_number,
+        )
+
+    def _fetch_block(self, block_number: int, log_info: str = "sync") -> bytes:
+        """
+        Fetch the block of data for `block_number`.
+        """
+        if block_number > self.nblocks:
+            raise ValueError(
+                f"'block_number={block_number}' is greater than "
+                f"the number of blocks ({self.nblocks})"
+            )
+
+        start = block_number * self.blocksize
+        end = start + self.blocksize
+        logger.info("BlockCache fetching block (%s) %d", log_info, block_number)
+        self.total_requested_bytes += end - start
+        self.miss_count += 1
+        block_contents = super()._fetch(start, end)
+        return block_contents
+
+    def _read_cache(
+        self, start: int, end: int, start_block_number: int, end_block_number: int
+    ) -> bytes:
+        """
+        Read from our block cache.
+
+        Parameters
+        ----------
+        start, end : int
+            The start and end byte positions.
+        start_block_number, end_block_number : int
+            The start and end block numbers.
+        """
+        start_pos = start % self.blocksize
+        end_pos = end % self.blocksize
+
+        # kind of pointless to count this as a hit, but it is
+        self.hit_count += 1
+
+        if start_block_number == end_block_number:
+            block = self._fetch_block_cached(start_block_number)
+            return block[start_pos:end_pos]
+
+        else:
+            # read from the initial
+            out = [self._fetch_block_cached(start_block_number)[start_pos:]]
+
+            # intermediate blocks
+            # Note: it'd be nice to combine these into one big request. However
+            # that doesn't play nicely with our LRU cache.
+            out.extend(
+                map(
+                    self._fetch_block_cached,
+                    range(start_block_number + 1, end_block_number),
+                )
+            )
+
+            # final block
+            out.append(self._fetch_block_cached(end_block_number)[:end_pos])
+
+            return b"".join(out)
+
+
+caches: dict[str | None, type[BaseCache]] = {
+    # one custom case
+    None: BaseCache,
+}
+
+
+def register_cache(cls: type[BaseCache], clobber: bool = False) -> None:
+    """'Register' cache implementation.
+
+    Parameters
+    ----------
+    clobber: bool, optional
+        If set to True (default is False) - allow to overwrite existing
+        entry.
+
+    Raises
+    ------
+    ValueError
+    """
+    name = cls.name
+    if not clobber and name in caches:
+        raise ValueError(f"Cache with name {name!r} is already known: {caches[name]}")
+    caches[name] = cls
+
+
+for c in (
+    BaseCache,
+    MMapCache,
+    BytesCache,
+    ReadAheadCache,
+    BlockCache,
+    FirstChunkCache,
+    AllBytes,
+    KnownPartsOfAFile,
+    BackgroundBlockCache,
+):
+    register_cache(c)

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/fsspec/dircache.py

@@ -0,0 +1,98 @@
+import time
+from collections.abc import MutableMapping
+from functools import lru_cache
+
+
+class DirCache(MutableMapping):
+    """
+    Caching of directory listings, in a structure like::
+
+        {"path0": [
+            {"name": "path0/file0",
+             "size": 123,
+             "type": "file",
+             ...
+            },
+            {"name": "path0/file1",
+            },
+            ...
+            ],
+         "path1": [...]
+        }
+
+    Parameters to this class control listing expiry or indeed turn
+    caching off
+    """
+
+    def __init__(
+        self,
+        use_listings_cache=True,
+        listings_expiry_time=None,
+        max_paths=None,
+        **kwargs,
+    ):
+        """
+
+        Parameters
+        ----------
+        use_listings_cache: bool
+            If False, this cache never returns items, but always reports KeyError,
+            and setting items has no effect
+        listings_expiry_time: int or float (optional)
+            Time in seconds that a listing is considered valid. If None,
+            listings do not expire.
+        max_paths: int (optional)
+            The number of most recent listings that are considered valid; 'recent'
+            refers to when the entry was set.
+        """
+        self._cache = {}
+        self._times = {}
+        if max_paths:
+            self._q = lru_cache(max_paths + 1)(lambda key: self._cache.pop(key, None))
+        self.use_listings_cache = use_listings_cache
+        self.listings_expiry_time = listings_expiry_time
+        self.max_paths = max_paths
+
+    def __getitem__(self, item):
+        if self.listings_expiry_time is not None:
+            if self._times.get(item, 0) - time.time() < -self.listings_expiry_time:
+                del self._cache[item]
+        if self.max_paths:
+            self._q(item)
+        return self._cache[item]  # maybe raises KeyError
+
+    def clear(self):
+        self._cache.clear()
+
+    def __len__(self):
+        return len(self._cache)
+
+    def __contains__(self, item):
+        try:
+            self[item]
+            return True
+        except KeyError:
+            return False
+
+    def __setitem__(self, key, value):
+        if not self.use_listings_cache:
+            return
+        if self.max_paths:
+            self._q(key)
+        self._cache[key] = value
+        if self.listings_expiry_time is not None:
+            self._times[key] = time.time()
+
+    def __delitem__(self, key):
+        del self._cache[key]
+
+    def __iter__(self):
+        entries = list(self._cache)
+
+        return (k for k in entries if k in self)
+
+    def __reduce__(self):
+        return (
+            DirCache,
+            (self.use_listings_cache, self.listings_expiry_time, self.max_paths),
+        )

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/fsspec/gui.py

@@ -0,0 +1,414 @@
+import ast
+import contextlib
+import logging
+import os
+import re
+from typing import ClassVar, Sequence
+
+import panel as pn
+
+from .core import OpenFile, get_filesystem_class, split_protocol
+from .registry import known_implementations
+
+pn.extension()
+logger = logging.getLogger("fsspec.gui")
+
+
+class SigSlot:
+    """Signal-slot mixin, for Panel event passing
+
+    Include this class in a widget manager's superclasses to be able to
+    register events and callbacks on Panel widgets managed by that class.
+
+    The method ``_register`` should be called as widgets are added, and external
+    code should call ``connect`` to associate callbacks.
+
+    By default, all signals emit a DEBUG logging statement.
+    """
+
+    # names of signals that this class may emit each of which must be
+    # set by _register for any new instance
+    signals: ClassVar[Sequence[str]] = []
+    # names of actions that this class may respond to
+    slots: ClassVar[Sequence[str]] = []
+
+    # each of which must be a method name
+
+    def __init__(self):
+        self._ignoring_events = False
+        self._sigs = {}
+        self._map = {}
+        self._setup()
+
+    def _setup(self):
+        """Create GUI elements and register signals"""
+        self.panel = pn.pane.PaneBase()
+        # no signals to set up in the base class
+
+    def _register(
+        self, widget, name, thing="value", log_level=logging.DEBUG, auto=False
+    ):
+        """Watch the given attribute of a widget and assign it a named event
+
+        This is normally called at the time a widget is instantiated, in the
+        class which owns it.
+
+        Parameters
+        ----------
+        widget : pn.layout.Panel or None
+            Widget to watch. If None, an anonymous signal not associated with
+            any widget.
+        name : str
+            Name of this event
+        thing : str
+            Attribute of the given widget to watch
+        log_level : int
+            When the signal is triggered, a logging event of the given level
+            will be fired in the dfviz logger.
+        auto : bool
+            If True, automatically connects with a method in this class of the
+            same name.
+        """
+        if name not in self.signals:
+            raise ValueError(f"Attempt to assign an undeclared signal: {name}")
+        self._sigs[name] = {
+            "widget": widget,
+            "callbacks": [],
+            "thing": thing,
+            "log": log_level,
+        }
+        wn = "-".join(
+            [
+                getattr(widget, "name", str(widget)) if widget is not None else "none",
+                thing,
+            ]
+        )
+        self._map[wn] = name
+        if widget is not None:
+            widget.param.watch(self._signal, thing, onlychanged=True)
+        if auto and hasattr(self, name):
+            self.connect(name, getattr(self, name))
+
+    def _repr_mimebundle_(self, *args, **kwargs):
+        """Display in a notebook or a server"""
+        try:
+            return self.panel._repr_mimebundle_(*args, **kwargs)
+        except (ValueError, AttributeError):
+            raise NotImplementedError("Panel does not seem to be set up properly")
+
+    def connect(self, signal, slot):
+        """Associate call back with given event
+
+        The callback must be a function which takes the "new" value of the
+        watched attribute as the only parameter. If the callback return False,
+        this cancels any further processing of the given event.
+
+        Alternatively, the callback can be a string, in which case it means
+        emitting the correspondingly-named event (i.e., connect to self)
+        """
+        self._sigs[signal]["callbacks"].append(slot)
+
+    def _signal(self, event):
+        """This is called by a an action on a widget
+
+        Within an self.ignore_events context, nothing happens.
+
+        Tests can execute this method by directly changing the values of
+        widget components.
+        """
+        if not self._ignoring_events:
+            wn = "-".join([event.obj.name, event.name])
+            if wn in self._map and self._map[wn] in self._sigs:
+                self._emit(self._map[wn], event.new)
+
+    @contextlib.contextmanager
+    def ignore_events(self):
+        """Temporarily turn off events processing in this instance
+
+        (does not propagate to children)
+        """
+        self._ignoring_events = True
+        try:
+            yield
+        finally:
+            self._ignoring_events = False
+
+    def _emit(self, sig, value=None):
+        """An event happened, call its callbacks
+
+        This method can be used in tests to simulate message passing without
+        directly changing visual elements.
+
+        Calling of callbacks will halt whenever one returns False.
+        """
+        logger.log(self._sigs[sig]["log"], f"{sig}: {value}")
+        for callback in self._sigs[sig]["callbacks"]:
+            if isinstance(callback, str):
+                self._emit(callback)
+            else:
+                try:
+                    # running callbacks should not break the interface
+                    ret = callback(value)
+                    if ret is False:
+                        break
+                except Exception as e:
+                    logger.exception(
+                        "Exception (%s) while executing callback for signal: %s",
+                        e,
+                        sig,
+                    )
+
+    def show(self, threads=False):
+        """Open a new browser tab and display this instance's interface"""
+        self.panel.show(threads=threads, verbose=False)
+        return self
+
+
+class SingleSelect(SigSlot):
+    """A multiselect which only allows you to select one item for an event"""
+
+    signals = ["_selected", "selected"]  # the first is internal
+    slots = ["set_options", "set_selection", "add", "clear", "select"]
+
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+        super().__init__()
+
+    def _setup(self):
+        self.panel = pn.widgets.MultiSelect(**self.kwargs)
+        self._register(self.panel, "_selected", "value")
+        self._register(None, "selected")
+        self.connect("_selected", self.select_one)
+
+    def _signal(self, *args, **kwargs):
+        super()._signal(*args, **kwargs)
+
+    def select_one(self, *_):
+        with self.ignore_events():
+            val = [self.panel.value[-1]] if self.panel.value else []
+            self.panel.value = val
+        self._emit("selected", self.panel.value)
+
+    def set_options(self, options):
+        self.panel.options = options
+
+    def clear(self):
+        self.panel.options = []
+
+    @property
+    def value(self):
+        return self.panel.value
+
+    def set_selection(self, selection):
+        self.panel.value = [selection]
+
+
+class FileSelector(SigSlot):
+    """Panel-based graphical file selector widget
+
+    Instances of this widget are interactive and can be displayed in jupyter by having
+    them as the output of a cell,  or in a separate browser tab using ``.show()``.
+    """
+
+    signals = [
+        "protocol_changed",
+        "selection_changed",
+        "directory_entered",
+        "home_clicked",
+        "up_clicked",
+        "go_clicked",
+        "filters_changed",
+    ]
+    slots = ["set_filters", "go_home"]
+
+    def __init__(self, url=None, filters=None, ignore=None, kwargs=None):
+        """
+
+        Parameters
+        ----------
+        url : str (optional)
+            Initial value of the URL to populate the dialog; should include protocol
+        filters : list(str) (optional)
+            File endings to include in the listings. If not included, all files are
+            allowed. Does not affect directories.
+            If given, the endings will appear as checkboxes in the interface
+        ignore : list(str) (optional)
+            Regex(s) of file basename patterns to ignore, e.g., "\\." for typical
+            hidden files on posix
+        kwargs : dict (optional)
+            To pass to file system instance
+        """
+        if url:
+            self.init_protocol, url = split_protocol(url)
+        else:
+            self.init_protocol, url = "file", os.getcwd()
+        self.init_url = url
+        self.init_kwargs = (kwargs if isinstance(kwargs, str) else str(kwargs)) or "{}"
+        self.filters = filters
+        self.ignore = [re.compile(i) for i in ignore or []]
+        self._fs = None
+        super().__init__()
+
+    def _setup(self):
+        self.url = pn.widgets.TextInput(
+            name="url",
+            value=self.init_url,
+            align="end",
+            sizing_mode="stretch_width",
+            width_policy="max",
+        )
+        self.protocol = pn.widgets.Select(
+            options=sorted(known_implementations),
+            value=self.init_protocol,
+            name="protocol",
+            align="center",
+        )
+        self.kwargs = pn.widgets.TextInput(
+            name="kwargs", value=self.init_kwargs, align="center"
+        )
+        self.go = pn.widgets.Button(name="⇨", align="end", width=45)
+        self.main = SingleSelect(size=10)
+        self.home = pn.widgets.Button(name="🏠", width=40, height=30, align="end")
+        self.up = pn.widgets.Button(name="‹", width=30, height=30, align="end")
+
+        self._register(self.protocol, "protocol_changed", auto=True)
+        self._register(self.go, "go_clicked", "clicks", auto=True)
+        self._register(self.up, "up_clicked", "clicks", auto=True)
+        self._register(self.home, "home_clicked", "clicks", auto=True)
+        self._register(None, "selection_changed")
+        self.main.connect("selected", self.selection_changed)
+        self._register(None, "directory_entered")
+        self.prev_protocol = self.protocol.value
+        self.prev_kwargs = self.storage_options
+
+        self.filter_sel = pn.widgets.CheckBoxGroup(
+            value=[], options=[], inline=False, align="end", width_policy="min"
+        )
+        self._register(self.filter_sel, "filters_changed", auto=True)
+
+        self.panel = pn.Column(
+            pn.Row(self.protocol, self.kwargs),
+            pn.Row(self.home, self.up, self.url, self.go, self.filter_sel),
+            self.main.panel,
+        )
+        self.set_filters(self.filters)
+        self.go_clicked()
+
+    def set_filters(self, filters=None):
+        self.filters = filters
+        if filters:
+            self.filter_sel.options = filters
+            self.filter_sel.value = filters
+        else:
+            self.filter_sel.options = []
+            self.filter_sel.value = []
+
+    @property
+    def storage_options(self):
+        """Value of the kwargs box as a dictionary"""
+        return ast.literal_eval(self.kwargs.value) or {}
+
+    @property
+    def fs(self):
+        """Current filesystem instance"""
+        if self._fs is None:
+            cls = get_filesystem_class(self.protocol.value)
+            self._fs = cls(**self.storage_options)
+        return self._fs
+
+    @property
+    def urlpath(self):
+        """URL of currently selected item"""
+        return (
+            (f"{self.protocol.value}://{self.main.value[0]}")
+            if self.main.value
+            else None
+        )
+
+    def open_file(self, mode="rb", compression=None, encoding=None):
+        """Create OpenFile instance for the currently selected item
+
+        For example, in a notebook you might do something like
+
+        .. code-block::
+
+            [ ]: sel = FileSelector(); sel
+
+            # user selects their file
+
+            [ ]: with sel.open_file('rb') as f:
+            ...      out = f.read()
+
+        Parameters
+        ----------
+        mode: str (optional)
+            Open mode for the file.
+        compression: str (optional)
+            The interact with the file as compressed. Set to 'infer' to guess
+            compression from the file ending
+        encoding: str (optional)
+            If using text mode, use this encoding; defaults to UTF8.
+        """
+        if self.urlpath is None:
+            raise ValueError("No file selected")
+        return OpenFile(self.fs, self.urlpath, mode, compression, encoding)
+
+    def filters_changed(self, values):
+        self.filters = values
+        self.go_clicked()
+
+    def selection_changed(self, *_):
+        if self.urlpath is None:
+            return
+        if self.fs.isdir(self.urlpath):
+            self.url.value = self.fs._strip_protocol(self.urlpath)
+        self.go_clicked()
+
+    def go_clicked(self, *_):
+        if (
+            self.prev_protocol != self.protocol.value
+            or self.prev_kwargs != self.storage_options
+        ):
+            self._fs = None  # causes fs to be recreated
+            self.prev_protocol = self.protocol.value
+            self.prev_kwargs = self.storage_options
+        listing = sorted(
+            self.fs.ls(self.url.value, detail=True), key=lambda x: x["name"]
+        )
+        listing = [
+            l
+            for l in listing
+            if not any(i.match(l["name"].rsplit("/", 1)[-1]) for i in self.ignore)
+        ]
+        folders = {
+            "📁 " + o["name"].rsplit("/", 1)[-1]: o["name"]
+            for o in listing
+            if o["type"] == "directory"
+        }
+        files = {
+            "📄 " + o["name"].rsplit("/", 1)[-1]: o["name"]
+            for o in listing
+            if o["type"] == "file"
+        }
+        if self.filters:
+            files = {
+                k: v
+                for k, v in files.items()
+                if any(v.endswith(ext) for ext in self.filters)
+            }
+        self.main.set_options(dict(**folders, **files))
+
+    def protocol_changed(self, *_):
+        self._fs = None
+        self.main.options = []
+        self.url.value = ""
+
+    def home_clicked(self, *_):
+        self.protocol.value = self.init_protocol
+        self.kwargs.value = self.init_kwargs
+        self.url.value = self.init_url
+        self.go_clicked()
+
+    def up_clicked(self, *_):
+        self.url.value = self.fs._parent(self.url.value)
+        self.go_clicked()

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/fsspec/spec.py

@@ -0,0 +1,1982 @@
+from __future__ import annotations
+
+import io
+import logging
+import os
+import threading
+import warnings
+import weakref
+from errno import ESPIPE
+from glob import has_magic
+from hashlib import sha256
+from typing import ClassVar
+
+from .callbacks import DEFAULT_CALLBACK
+from .config import apply_config, conf
+from .dircache import DirCache
+from .transaction import Transaction
+from .utils import (
+    _unstrip_protocol,
+    glob_translate,
+    isfilelike,
+    other_paths,
+    read_block,
+    stringify_path,
+    tokenize,
+)
+
+logger = logging.getLogger("fsspec")
+
+
+def make_instance(cls, args, kwargs):
+    return cls(*args, **kwargs)
+
+
+class _Cached(type):
+    """
+    Metaclass for caching file system instances.
+
+    Notes
+    -----
+    Instances are cached according to
+
+    * The values of the class attributes listed in `_extra_tokenize_attributes`
+    * The arguments passed to ``__init__``.
+
+    This creates an additional reference to the filesystem, which prevents the
+    filesystem from being garbage collected when all *user* references go away.
+    A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also*
+    be made for a filesystem instance to be garbage collected.
+    """
+
+    def __init__(cls, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # Note: we intentionally create a reference here, to avoid garbage
+        # collecting instances when all other references are gone. To really
+        # delete a FileSystem, the cache must be cleared.
+        if conf.get("weakref_instance_cache"):  # pragma: no cover
+            # debug option for analysing fork/spawn conditions
+            cls._cache = weakref.WeakValueDictionary()
+        else:
+            cls._cache = {}
+        cls._pid = os.getpid()
+
+    def __call__(cls, *args, **kwargs):
+        kwargs = apply_config(cls, kwargs)
+        extra_tokens = tuple(
+            getattr(cls, attr, None) for attr in cls._extra_tokenize_attributes
+        )
+        token = tokenize(
+            cls, cls._pid, threading.get_ident(), *args, *extra_tokens, **kwargs
+        )
+        skip = kwargs.pop("skip_instance_cache", False)
+        if os.getpid() != cls._pid:
+            cls._cache.clear()
+            cls._pid = os.getpid()
+        if not skip and cls.cachable and token in cls._cache:
+            cls._latest = token
+            return cls._cache[token]
+        else:
+            obj = super().__call__(*args, **kwargs)
+            # Setting _fs_token here causes some static linters to complain.
+            obj._fs_token_ = token
+            obj.storage_args = args
+            obj.storage_options = kwargs
+            if obj.async_impl and obj.mirror_sync_methods:
+                from .asyn import mirror_sync_methods
+
+                mirror_sync_methods(obj)
+
+            if cls.cachable and not skip:
+                cls._latest = token
+                cls._cache[token] = obj
+            return obj
+
+
+class AbstractFileSystem(metaclass=_Cached):
+    """
+    An abstract super-class for pythonic file-systems
+
+    Implementations are expected to be compatible with or, better, subclass
+    from here.
+    """
+
+    cachable = True  # this class can be cached, instances reused
+    _cached = False
+    blocksize = 2**22
+    sep = "/"
+    protocol: ClassVar[str | tuple[str, ...]] = "abstract"
+    _latest = None
+    async_impl = False
+    mirror_sync_methods = False
+    root_marker = ""  # For some FSs, may require leading '/' or other character
+    transaction_type = Transaction
+
+    #: Extra *class attributes* that should be considered when hashing.
+    _extra_tokenize_attributes = ()
+
+    def __init__(self, *args, **storage_options):
+        """Create and configure file-system instance
+
+        Instances may be cachable, so if similar enough arguments are seen
+        a new instance is not required. The token attribute exists to allow
+        implementations to cache instances if they wish.
+
+        A reasonable default should be provided if there are no arguments.
+
+        Subclasses should call this method.
+
+        Parameters
+        ----------
+        use_listings_cache, listings_expiry_time, max_paths:
+            passed to ``DirCache``, if the implementation supports
+            directory listing caching. Pass use_listings_cache=False
+            to disable such caching.
+        skip_instance_cache: bool
+            If this is a cachable implementation, pass True here to force
+            creating a new instance even if a matching instance exists, and prevent
+            storing this instance.
+        asynchronous: bool
+        loop: asyncio-compatible IOLoop or None
+        """
+        if self._cached:
+            # reusing instance, don't change
+            return
+        self._cached = True
+        self._intrans = False
+        self._transaction = None
+        self._invalidated_caches_in_transaction = []
+        self.dircache = DirCache(**storage_options)
+
+        if storage_options.pop("add_docs", None):
+            warnings.warn("add_docs is no longer supported.", FutureWarning)
+
+        if storage_options.pop("add_aliases", None):
+            warnings.warn("add_aliases has been removed.", FutureWarning)
+        # This is set in _Cached
+        self._fs_token_ = None
+
+    @property
+    def fsid(self):
+        """Persistent filesystem id that can be used to compare filesystems
+        across sessions.
+        """
+        raise NotImplementedError
+
+    @property
+    def _fs_token(self):
+        return self._fs_token_
+
+    def __dask_tokenize__(self):
+        return self._fs_token
+
+    def __hash__(self):
+        return int(self._fs_token, 16)
+
+    def __eq__(self, other):
+        return isinstance(other, type(self)) and self._fs_token == other._fs_token
+
+    def __reduce__(self):
+        return make_instance, (type(self), self.storage_args, self.storage_options)
+
+    @classmethod
+    def _strip_protocol(cls, path):
+        """Turn path from fully-qualified to file-system-specific
+
+        May require FS-specific handling, e.g., for relative paths or links.
+        """
+        if isinstance(path, list):
+            return [cls._strip_protocol(p) for p in path]
+        path = stringify_path(path)
+        protos = (cls.protocol,) if isinstance(cls.protocol, str) else cls.protocol
+        for protocol in protos:
+            if path.startswith(protocol + "://"):
+                path = path[len(protocol) + 3 :]
+            elif path.startswith(protocol + "::"):
+                path = path[len(protocol) + 2 :]
+        path = path.rstrip("/")
+        # use of root_marker to make minimum required path, e.g., "/"
+        return path or cls.root_marker
+
+    def unstrip_protocol(self, name: str) -> str:
+        """Format FS-specific path to generic, including protocol"""
+        protos = (self.protocol,) if isinstance(self.protocol, str) else self.protocol
+        for protocol in protos:
+            if name.startswith(f"{protocol}://"):
+                return name
+        return f"{protos[0]}://{name}"
+
+    @staticmethod
+    def _get_kwargs_from_urls(path):
+        """If kwargs can be encoded in the paths, extract them here
+
+        This should happen before instantiation of the class; incoming paths
+        then should be amended to strip the options in methods.
+
+        Examples may look like an sftp path "sftp://user@host:/my/path", where
+        the user and host should become kwargs and later get stripped.
+        """
+        # by default, nothing happens
+        return {}
+
+    @classmethod
+    def current(cls):
+        """Return the most recently instantiated FileSystem
+
+        If no instance has been created, then create one with defaults
+        """
+        if cls._latest in cls._cache:
+            return cls._cache[cls._latest]
+        return cls()
+
+    @property
+    def transaction(self):
+        """A context within which files are committed together upon exit
+
+        Requires the file class to implement `.commit()` and `.discard()`
+        for the normal and exception cases.
+        """
+        if self._transaction is None:
+            self._transaction = self.transaction_type(self)
+        return self._transaction
+
+    def start_transaction(self):
+        """Begin write transaction for deferring files, non-context version"""
+        self._intrans = True
+        self._transaction = self.transaction_type(self)
+        return self.transaction
+
+    def end_transaction(self):
+        """Finish write transaction, non-context version"""
+        self.transaction.complete()
+        self._transaction = None
+        # The invalid cache must be cleared after the transaction is completed.
+        for path in self._invalidated_caches_in_transaction:
+            self.invalidate_cache(path)
+        self._invalidated_caches_in_transaction.clear()
+
+    def invalidate_cache(self, path=None):
+        """
+        Discard any cached directory information
+
+        Parameters
+        ----------
+        path: string or None
+            If None, clear all listings cached else listings at or under given
+            path.
+        """
+        # Not necessary to implement invalidation mechanism, may have no cache.
+        # But if have, you should call this method of parent class from your
+        # subclass to ensure expiring caches after transacations correctly.
+        # See the implementation of FTPFileSystem in ftp.py
+        if self._intrans:
+            self._invalidated_caches_in_transaction.append(path)
+
+    def mkdir(self, path, create_parents=True, **kwargs):
+        """
+        Create directory entry at path
+
+        For systems that don't have true directories, may create an for
+        this instance only and not touch the real filesystem
+
+        Parameters
+        ----------
+        path: str
+            location
+        create_parents: bool
+            if True, this is equivalent to ``makedirs``
+        kwargs:
+            may be permissions, etc.
+        """
+        pass  # not necessary to implement, may not have directories
+
+    def makedirs(self, path, exist_ok=False):
+        """Recursively make directories
+
+        Creates directory at path and any intervening required directories.
+        Raises exception if, for instance, the path already exists but is a
+        file.
+
+        Parameters
+        ----------
+        path: str
+            leaf directory name
+        exist_ok: bool (False)
+            If False, will error if the target already exists
+        """
+        pass  # not necessary to implement, may not have directories
+
+    def rmdir(self, path):
+        """Remove a directory, if empty"""
+        pass  # not necessary to implement, may not have directories
+
+    def ls(self, path, detail=True, **kwargs):
+        """List objects at path.
+
+        This should include subdirectories and files at that location. The
+        difference between a file and a directory must be clear when details
+        are requested.
+
+        The specific keys, or perhaps a FileInfo class, or similar, is TBD,
+        but must be consistent across implementations.
+        Must include:
+
+        - full path to the entry (without protocol)
+        - size of the entry, in bytes. If the value cannot be determined, will
+          be ``None``.
+        - type of entry, "file", "directory" or other
+
+        Additional information
+        may be present, appropriate to the file-system, e.g., generation,
+        checksum, etc.
+
+        May use refresh=True|False to allow use of self._ls_from_cache to
+        check for a saved listing and avoid calling the backend. This would be
+        common where listing may be expensive.
+
+        Parameters
+        ----------
+        path: str
+        detail: bool
+            if True, gives a list of dictionaries, where each is the same as
+            the result of ``info(path)``. If False, gives a list of paths
+            (str).
+        kwargs: may have additional backend-specific options, such as version
+            information
+
+        Returns
+        -------
+        List of strings if detail is False, or list of directory information
+        dicts if detail is True.
+        """
+        raise NotImplementedError
+
+    def _ls_from_cache(self, path):
+        """Check cache for listing
+
+        Returns listing, if found (may be empty list for a directly that exists
+        but contains nothing), None if not in cache.
+        """
+        parent = self._parent(path)
+        try:
+            return self.dircache[path.rstrip("/")]
+        except KeyError:
+            pass
+        try:
+            files = [
+                f
+                for f in self.dircache[parent]
+                if f["name"] == path
+                or (f["name"] == path.rstrip("/") and f["type"] == "directory")
+            ]
+            if len(files) == 0:
+                # parent dir was listed but did not contain this file
+                raise FileNotFoundError(path)
+            return files
+        except KeyError:
+            pass
+
+    def walk(self, path, maxdepth=None, topdown=True, on_error="omit", **kwargs):
+        """Return all files belows path
+
+        List all files, recursing into subdirectories; output is iterator-style,
+        like ``os.walk()``. For a simple list of files, ``find()`` is available.
+
+        When topdown is True, the caller can modify the dirnames list in-place (perhaps
+        using del or slice assignment), and walk() will
+        only recurse into the subdirectories whose names remain in dirnames;
+        this can be used to prune the search, impose a specific order of visiting,
+        or even to inform walk() about directories the caller creates or renames before
+        it resumes walk() again.
+        Modifying dirnames when topdown is False has no effect. (see os.walk)
+
+        Note that the "files" outputted will include anything that is not
+        a directory, such as links.
+
+        Parameters
+        ----------
+        path: str
+            Root to recurse into
+        maxdepth: int
+            Maximum recursion depth. None means limitless, but not recommended
+            on link-based file-systems.
+        topdown: bool (True)
+            Whether to walk the directory tree from the top downwards or from
+            the bottom upwards.
+        on_error: "omit", "raise", a collable
+            if omit (default), path with exception will simply be empty;
+            If raise, an underlying exception will be raised;
+            if callable, it will be called with a single OSError instance as argument
+        kwargs: passed to ``ls``
+        """
+        if maxdepth is not None and maxdepth < 1:
+            raise ValueError("maxdepth must be at least 1")
+
+        path = self._strip_protocol(path)
+        full_dirs = {}
+        dirs = {}
+        files = {}
+
+        detail = kwargs.pop("detail", False)
+        try:
+            listing = self.ls(path, detail=True, **kwargs)
+        except (FileNotFoundError, OSError) as e:
+            if on_error == "raise":
+                raise
+            elif callable(on_error):
+                on_error(e)
+            if detail:
+                return path, {}, {}
+            return path, [], []
+
+        for info in listing:
+            # each info name must be at least [path]/part , but here
+            # we check also for names like [path]/part/
+            pathname = info["name"].rstrip("/")
+            name = pathname.rsplit("/", 1)[-1]
+            if info["type"] == "directory" and pathname != path:
+                # do not include "self" path
+                full_dirs[name] = pathname
+                dirs[name] = info
+            elif pathname == path:
+                # file-like with same name as give path
+                files[""] = info
+            else:
+                files[name] = info
+
+        if not detail:
+            dirs = list(dirs)
+            files = list(files)
+
+        if topdown:
+            # Yield before recursion if walking top down
+            yield path, dirs, files
+
+        if maxdepth is not None:
+            maxdepth -= 1
+            if maxdepth < 1:
+                if not topdown:
+                    yield path, dirs, files
+                return
+
+        for d in dirs:
+            yield from self.walk(
+                full_dirs[d],
+                maxdepth=maxdepth,
+                detail=detail,
+                topdown=topdown,
+                **kwargs,
+            )
+
+        if not topdown:
+            # Yield after recursion if walking bottom up
+            yield path, dirs, files
+
+    def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs):
+        """List all files below path.
+
+        Like posix ``find`` command without conditions
+
+        Parameters
+        ----------
+        path : str
+        maxdepth: int or None
+            If not None, the maximum number of levels to descend
+        withdirs: bool
+            Whether to include directory paths in the output. This is True
+            when used by glob, but users usually only want files.
+        kwargs are passed to ``ls``.
+        """
+        # TODO: allow equivalent of -name parameter
+        path = self._strip_protocol(path)
+        out = {}
+
+        # Add the root directory if withdirs is requested
+        # This is needed for posix glob compliance
+        if withdirs and path != "" and self.isdir(path):
+            out[path] = self.info(path)
+
+        for _, dirs, files in self.walk(path, maxdepth, detail=True, **kwargs):
+            if withdirs:
+                files.update(dirs)
+            out.update({info["name"]: info for name, info in files.items()})
+        if not out and self.isfile(path):
+            # walk works on directories, but find should also return [path]
+            # when path happens to be a file
+            out[path] = {}
+        names = sorted(out)
+        if not detail:
+            return names
+        else:
+            return {name: out[name] for name in names}
+
+    def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs):
+        """Space used by files and optionally directories within a path
+
+        Directory size does not include the size of its contents.
+
+        Parameters
+        ----------
+        path: str
+        total: bool
+            Whether to sum all the file sizes
+        maxdepth: int or None
+            Maximum number of directory levels to descend, None for unlimited.
+        withdirs: bool
+            Whether to include directory paths in the output.
+        kwargs: passed to ``find``
+
+        Returns
+        -------
+        Dict of {path: size} if total=False, or int otherwise, where numbers
+        refer to bytes used.
+        """
+        sizes = {}
+        if withdirs and self.isdir(path):
+            # Include top-level directory in output
+            info = self.info(path)
+            sizes[info["name"]] = info["size"]
+        for f in self.find(path, maxdepth=maxdepth, withdirs=withdirs, **kwargs):
+            info = self.info(f)
+            sizes[info["name"]] = info["size"]
+        if total:
+            return sum(sizes.values())
+        else:
+            return sizes
+
+    def glob(self, path, maxdepth=None, **kwargs):
+        """
+        Find files by glob-matching.
+
+        If the path ends with '/', only folders are returned.
+
+        We support ``"**"``,
+        ``"?"`` and ``"[..]"``. We do not support ^ for pattern negation.
+
+        The `maxdepth` option is applied on the first `**` found in the path.
+
+        kwargs are passed to ``ls``.
+        """
+        if maxdepth is not None and maxdepth < 1:
+            raise ValueError("maxdepth must be at least 1")
+
+        import re
+
+        seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,)
+        ends_with_sep = path.endswith(seps)  # _strip_protocol strips trailing slash
+        path = self._strip_protocol(path)
+        append_slash_to_dirname = ends_with_sep or path.endswith(
+            tuple(sep + "**" for sep in seps)
+        )
+        idx_star = path.find("*") if path.find("*") >= 0 else len(path)
+        idx_qmark = path.find("?") if path.find("?") >= 0 else len(path)
+        idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
+
+        min_idx = min(idx_star, idx_qmark, idx_brace)
+
+        detail = kwargs.pop("detail", False)
+
+        if not has_magic(path):
+            if self.exists(path, **kwargs):
+                if not detail:
+                    return [path]
+                else:
+                    return {path: self.info(path, **kwargs)}
+            else:
+                if not detail:
+                    return []  # glob of non-existent returns empty
+                else:
+                    return {}
+        elif "/" in path[:min_idx]:
+            min_idx = path[:min_idx].rindex("/")
+            root = path[: min_idx + 1]
+            depth = path[min_idx + 1 :].count("/") + 1
+        else:
+            root = ""
+            depth = path[min_idx + 1 :].count("/") + 1
+
+        if "**" in path:
+            if maxdepth is not None:
+                idx_double_stars = path.find("**")
+                depth_double_stars = path[idx_double_stars:].count("/") + 1
+                depth = depth - depth_double_stars + maxdepth
+            else:
+                depth = None
+
+        allpaths = self.find(root, maxdepth=depth, withdirs=True, detail=True, **kwargs)
+
+        pattern = glob_translate(path + ("/" if ends_with_sep else ""))
+        pattern = re.compile(pattern)
+
+        out = {
+            p: info
+            for p, info in sorted(allpaths.items())
+            if pattern.match(
+                (
+                    p + "/"
+                    if append_slash_to_dirname and info["type"] == "directory"
+                    else p
+                )
+            )
+        }
+
+        if detail:
+            return out
+        else:
+            return list(out)
+
+    def exists(self, path, **kwargs):
+        """Is there a file at the given path"""
+        try:
+            self.info(path, **kwargs)
+            return True
+        except:  # noqa: E722
+            # any exception allowed bar FileNotFoundError?
+            return False
+
+    def lexists(self, path, **kwargs):
+        """If there is a file at the given path (including
+        broken links)"""
+        return self.exists(path)
+
+    def info(self, path, **kwargs):
+        """Give details of entry at path
+
+        Returns a single dictionary, with exactly the same information as ``ls``
+        would with ``detail=True``.
+
+        The default implementation should calls ls and could be overridden by a
+        shortcut. kwargs are passed on to ```ls()``.
+
+        Some file systems might not be able to measure the file's size, in
+        which case, the returned dict will include ``'size': None``.
+
+        Returns
+        -------
+        dict with keys: name (full path in the FS), size (in bytes), type (file,
+        directory, or something else) and other FS-specific keys.
+        """
+        path = self._strip_protocol(path)
+        out = self.ls(self._parent(path), detail=True, **kwargs)
+        out = [o for o in out if o["name"].rstrip("/") == path]
+        if out:
+            return out[0]
+        out = self.ls(path, detail=True, **kwargs)
+        path = path.rstrip("/")
+        out1 = [o for o in out if o["name"].rstrip("/") == path]
+        if len(out1) == 1:
+            if "size" not in out1[0]:
+                out1[0]["size"] = None
+            return out1[0]
+        elif len(out1) > 1 or out:
+            return {"name": path, "size": 0, "type": "directory"}
+        else:
+            raise FileNotFoundError(path)
+
+    def checksum(self, path):
+        """Unique value for current version of file
+
+        If the checksum is the same from one moment to another, the contents
+        are guaranteed to be the same. If the checksum changes, the contents
+        *might* have changed.
+
+        This should normally be overridden; default will probably capture
+        creation/modification timestamp (which would be good) or maybe
+        access timestamp (which would be bad)
+        """
+        return int(tokenize(self.info(path)), 16)
+
+    def size(self, path):
+        """Size in bytes of file"""
+        return self.info(path).get("size", None)
+
+    def sizes(self, paths):
+        """Size in bytes of each file in a list of paths"""
+        return [self.size(p) for p in paths]
+
+    def isdir(self, path):
+        """Is this entry directory-like?"""
+        try:
+            return self.info(path)["type"] == "directory"
+        except OSError:
+            return False
+
+    def isfile(self, path):
+        """Is this entry file-like?"""
+        try:
+            return self.info(path)["type"] == "file"
+        except:  # noqa: E722
+            return False
+
+    def read_text(self, path, encoding=None, errors=None, newline=None, **kwargs):
+        """Get the contents of the file as a string.
+
+        Parameters
+        ----------
+        path: str
+            URL of file on this filesystems
+        encoding, errors, newline: same as `open`.
+        """
+        with self.open(
+            path,
+            mode="r",
+            encoding=encoding,
+            errors=errors,
+            newline=newline,
+            **kwargs,
+        ) as f:
+            return f.read()
+
+    def write_text(
+        self, path, value, encoding=None, errors=None, newline=None, **kwargs
+    ):
+        """Write the text to the given file.
+
+        An existing file will be overwritten.
+
+        Parameters
+        ----------
+        path: str
+            URL of file on this filesystems
+        value: str
+            Text to write.
+        encoding, errors, newline: same as `open`.
+        """
+        with self.open(
+            path,
+            mode="w",
+            encoding=encoding,
+            errors=errors,
+            newline=newline,
+            **kwargs,
+        ) as f:
+            return f.write(value)
+
+    def cat_file(self, path, start=None, end=None, **kwargs):
+        """Get the content of a file
+
+        Parameters
+        ----------
+        path: URL of file on this filesystems
+        start, end: int
+            Bytes limits of the read. If negative, backwards from end,
+            like usual python slices. Either can be None for start or
+            end of file, respectively
+        kwargs: passed to ``open()``.
+        """
+        # explicitly set buffering off?
+        with self.open(path, "rb", **kwargs) as f:
+            if start is not None:
+                if start >= 0:
+                    f.seek(start)
+                else:
+                    f.seek(max(0, f.size + start))
+            if end is not None:
+                if end < 0:
+                    end = f.size + end
+                return f.read(end - f.tell())
+            return f.read()
+
+    def pipe_file(self, path, value, **kwargs):
+        """Set the bytes of given file"""
+        with self.open(path, "wb", **kwargs) as f:
+            f.write(value)
+
+    def pipe(self, path, value=None, **kwargs):
+        """Put value into path
+
+        (counterpart to ``cat``)
+
+        Parameters
+        ----------
+        path: string or dict(str, bytes)
+            If a string, a single remote location to put ``value`` bytes; if a dict,
+            a mapping of {path: bytesvalue}.
+        value: bytes, optional
+            If using a single path, these are the bytes to put there. Ignored if
+            ``path`` is a dict
+        """
+        if isinstance(path, str):
+            self.pipe_file(self._strip_protocol(path), value, **kwargs)
+        elif isinstance(path, dict):
+            for k, v in path.items():
+                self.pipe_file(self._strip_protocol(k), v, **kwargs)
+        else:
+            raise ValueError("path must be str or dict")
+
+    def cat_ranges(
+        self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
+    ):
+        """Get the contents of byte ranges from one or more files
+
+        Parameters
+        ----------
+        paths: list
+            A list of of filepaths on this filesystems
+        starts, ends: int or list
+            Bytes limits of the read. If using a single int, the same value will be
+            used to read all the specified files.
+        """
+        if max_gap is not None:
+            raise NotImplementedError
+        if not isinstance(paths, list):
+            raise TypeError
+        if not isinstance(starts, list):
+            starts = [starts] * len(paths)
+        if not isinstance(ends, list):
+            ends = [ends] * len(paths)
+        if len(starts) != len(paths) or len(ends) != len(paths):
+            raise ValueError
+        out = []
+        for p, s, e in zip(paths, starts, ends):
+            try:
+                out.append(self.cat_file(p, s, e))
+            except Exception as e:
+                if on_error == "return":
+                    out.append(e)
+                else:
+                    raise
+        return out
+
+    def cat(self, path, recursive=False, on_error="raise", **kwargs):
+        """Fetch (potentially multiple) paths' contents
+
+        Parameters
+        ----------
+        recursive: bool
+            If True, assume the path(s) are directories, and get all the
+            contained files
+        on_error : "raise", "omit", "return"
+            If raise, an underlying exception will be raised (converted to KeyError
+            if the type is in self.missing_exceptions); if omit, keys with exception
+            will simply not be included in the output; if "return", all keys are
+            included in the output, but the value will be bytes or an exception
+            instance.
+        kwargs: passed to cat_file
+
+        Returns
+        -------
+        dict of {path: contents} if there are multiple paths
+        or the path has been otherwise expanded
+        """
+        paths = self.expand_path(path, recursive=recursive)
+        if (
+            len(paths) > 1
+            or isinstance(path, list)
+            or paths[0] != self._strip_protocol(path)
+        ):
+            out = {}
+            for path in paths:
+                try:
+                    out[path] = self.cat_file(path, **kwargs)
+                except Exception as e:
+                    if on_error == "raise":
+                        raise
+                    if on_error == "return":
+                        out[path] = e
+            return out
+        else:
+            return self.cat_file(paths[0], **kwargs)
+
+    def get_file(self, rpath, lpath, callback=DEFAULT_CALLBACK, outfile=None, **kwargs):
+        """Copy single remote file to local"""
+        from .implementations.local import LocalFileSystem
+
+        if isfilelike(lpath):
+            outfile = lpath
+        elif self.isdir(rpath):
+            os.makedirs(lpath, exist_ok=True)
+            return None
+
+        fs = LocalFileSystem(auto_mkdir=True)
+        fs.makedirs(fs._parent(lpath), exist_ok=True)
+
+        with self.open(rpath, "rb", **kwargs) as f1:
+            if outfile is None:
+                outfile = open(lpath, "wb")
+
+            try:
+                callback.set_size(getattr(f1, "size", None))
+                data = True
+                while data:
+                    data = f1.read(self.blocksize)
+                    segment_len = outfile.write(data)
+                    if segment_len is None:
+                        segment_len = len(data)
+                    callback.relative_update(segment_len)
+            finally:
+                if not isfilelike(lpath):
+                    outfile.close()
+
+    def get(
+        self,
+        rpath,
+        lpath,
+        recursive=False,
+        callback=DEFAULT_CALLBACK,
+        maxdepth=None,
+        **kwargs,
+    ):
+        """Copy file(s) to local.
+
+        Copies a specific file or tree of files (if recursive=True). If lpath
+        ends with a "/", it will be assumed to be a directory, and target files
+        will go within. Can submit a list of paths, which may be glob-patterns
+        and will be expanded.
+
+        Calls get_file for each source.
+        """
+        if isinstance(lpath, list) and isinstance(rpath, list):
+            # No need to expand paths when both source and destination
+            # are provided as lists
+            rpaths = rpath
+            lpaths = lpath
+        else:
+            from .implementations.local import (
+                LocalFileSystem,
+                make_path_posix,
+                trailing_sep,
+            )
+
+            source_is_str = isinstance(rpath, str)
+            rpaths = self.expand_path(rpath, recursive=recursive, maxdepth=maxdepth)
+            if source_is_str and (not recursive or maxdepth is not None):
+                # Non-recursive glob does not copy directories
+                rpaths = [p for p in rpaths if not (trailing_sep(p) or self.isdir(p))]
+                if not rpaths:
+                    return
+
+            if isinstance(lpath, str):
+                lpath = make_path_posix(lpath)
+
+            source_is_file = len(rpaths) == 1
+            dest_is_dir = isinstance(lpath, str) and (
+                trailing_sep(lpath) or LocalFileSystem().isdir(lpath)
+            )
+
+            exists = source_is_str and (
+                (has_magic(rpath) and source_is_file)
+                or (not has_magic(rpath) and dest_is_dir and not trailing_sep(rpath))
+            )
+            lpaths = other_paths(
+                rpaths,
+                lpath,
+                exists=exists,
+                flatten=not source_is_str,
+            )
+
+        callback.set_size(len(lpaths))
+        for lpath, rpath in callback.wrap(zip(lpaths, rpaths)):
+            with callback.branched(rpath, lpath) as child:
+                self.get_file(rpath, lpath, callback=child, **kwargs)
+
+    def put_file(self, lpath, rpath, callback=DEFAULT_CALLBACK, **kwargs):
+        """Copy single file to remote"""
+        if os.path.isdir(lpath):
+            self.makedirs(rpath, exist_ok=True)
+            return None
+
+        with open(lpath, "rb") as f1:
+            size = f1.seek(0, 2)
+            callback.set_size(size)
+            f1.seek(0)
+
+            self.mkdirs(self._parent(os.fspath(rpath)), exist_ok=True)
+            with self.open(rpath, "wb", **kwargs) as f2:
+                while f1.tell() < size:
+                    data = f1.read(self.blocksize)
+                    segment_len = f2.write(data)
+                    if segment_len is None:
+                        segment_len = len(data)
+                    callback.relative_update(segment_len)
+
+    def put(
+        self,
+        lpath,
+        rpath,
+        recursive=False,
+        callback=DEFAULT_CALLBACK,
+        maxdepth=None,
+        **kwargs,
+    ):
+        """Copy file(s) from local.
+
+        Copies a specific file or tree of files (if recursive=True). If rpath
+        ends with a "/", it will be assumed to be a directory, and target files
+        will go within.
+
+        Calls put_file for each source.
+        """
+        if isinstance(lpath, list) and isinstance(rpath, list):
+            # No need to expand paths when both source and destination
+            # are provided as lists
+            rpaths = rpath
+            lpaths = lpath
+        else:
+            from .implementations.local import (
+                LocalFileSystem,
+                make_path_posix,
+                trailing_sep,
+            )
+
+            source_is_str = isinstance(lpath, str)
+            if source_is_str:
+                lpath = make_path_posix(lpath)
+            fs = LocalFileSystem()
+            lpaths = fs.expand_path(lpath, recursive=recursive, maxdepth=maxdepth)
+            if source_is_str and (not recursive or maxdepth is not None):
+                # Non-recursive glob does not copy directories
+                lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))]
+                if not lpaths:
+                    return
+
+            source_is_file = len(lpaths) == 1
+            dest_is_dir = isinstance(rpath, str) and (
+                trailing_sep(rpath) or self.isdir(rpath)
+            )
+
+            rpath = (
+                self._strip_protocol(rpath)
+                if isinstance(rpath, str)
+                else [self._strip_protocol(p) for p in rpath]
+            )
+            exists = source_is_str and (
+                (has_magic(lpath) and source_is_file)
+                or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath))
+            )
+            rpaths = other_paths(
+                lpaths,
+                rpath,
+                exists=exists,
+                flatten=not source_is_str,
+            )
+
+        callback.set_size(len(rpaths))
+        for lpath, rpath in callback.wrap(zip(lpaths, rpaths)):
+            with callback.branched(lpath, rpath) as child:
+                self.put_file(lpath, rpath, callback=child, **kwargs)
+
+    def head(self, path, size=1024):
+        """Get the first ``size`` bytes from file"""
+        with self.open(path, "rb") as f:
+            return f.read(size)
+
+    def tail(self, path, size=1024):
+        """Get the last ``size`` bytes from file"""
+        with self.open(path, "rb") as f:
+            f.seek(max(-size, -f.size), 2)
+            return f.read()
+
+    def cp_file(self, path1, path2, **kwargs):
+        raise NotImplementedError
+
+    def copy(
+        self, path1, path2, recursive=False, maxdepth=None, on_error=None, **kwargs
+    ):
+        """Copy within two locations in the filesystem
+
+        on_error : "raise", "ignore"
+            If raise, any not-found exceptions will be raised; if ignore any
+            not-found exceptions will cause the path to be skipped; defaults to
+            raise unless recursive is true, where the default is ignore
+        """
+        if on_error is None and recursive:
+            on_error = "ignore"
+        elif on_error is None:
+            on_error = "raise"
+
+        if isinstance(path1, list) and isinstance(path2, list):
+            # No need to expand paths when both source and destination
+            # are provided as lists
+            paths1 = path1
+            paths2 = path2
+        else:
+            from .implementations.local import trailing_sep
+
+            source_is_str = isinstance(path1, str)
+            paths1 = self.expand_path(path1, recursive=recursive, maxdepth=maxdepth)
+            if source_is_str and (not recursive or maxdepth is not None):
+                # Non-recursive glob does not copy directories
+                paths1 = [p for p in paths1 if not (trailing_sep(p) or self.isdir(p))]
+                if not paths1:
+                    return
+
+            source_is_file = len(paths1) == 1
+            dest_is_dir = isinstance(path2, str) and (
+                trailing_sep(path2) or self.isdir(path2)
+            )
+
+            exists = source_is_str and (
+                (has_magic(path1) and source_is_file)
+                or (not has_magic(path1) and dest_is_dir and not trailing_sep(path1))
+            )
+            paths2 = other_paths(
+                paths1,
+                path2,
+                exists=exists,
+                flatten=not source_is_str,
+            )
+
+        for p1, p2 in zip(paths1, paths2):
+            try:
+                self.cp_file(p1, p2, **kwargs)
+            except FileNotFoundError:
+                if on_error == "raise":
+                    raise
+
+    def expand_path(self, path, recursive=False, maxdepth=None, **kwargs):
+        """Turn one or more globs or directories into a list of all matching paths
+        to files or directories.
+
+        kwargs are passed to ``glob`` or ``find``, which may in turn call ``ls``
+        """
+
+        if maxdepth is not None and maxdepth < 1:
+            raise ValueError("maxdepth must be at least 1")
+
+        if isinstance(path, (str, os.PathLike)):
+            out = self.expand_path([path], recursive, maxdepth)
+        else:
+            out = set()
+            path = [self._strip_protocol(p) for p in path]
+            for p in path:
+                if has_magic(p):
+                    bit = set(self.glob(p, maxdepth=maxdepth, **kwargs))
+                    out |= bit
+                    if recursive:
+                        # glob call above expanded one depth so if maxdepth is defined
+                        # then decrement it in expand_path call below. If it is zero
+                        # after decrementing then avoid expand_path call.
+                        if maxdepth is not None and maxdepth <= 1:
+                            continue
+                        out |= set(
+                            self.expand_path(
+                                list(bit),
+                                recursive=recursive,
+                                maxdepth=maxdepth - 1 if maxdepth is not None else None,
+                                **kwargs,
+                            )
+                        )
+                    continue
+                elif recursive:
+                    rec = set(
+                        self.find(
+                            p, maxdepth=maxdepth, withdirs=True, detail=False, **kwargs
+                        )
+                    )
+                    out |= rec
+                if p not in out and (recursive is False or self.exists(p)):
+                    # should only check once, for the root
+                    out.add(p)
+        if not out:
+            raise FileNotFoundError(path)
+        return sorted(out)
+
+    def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs):
+        """Move file(s) from one location to another"""
+        if path1 == path2:
+            logger.debug("%s mv: The paths are the same, so no files were moved.", self)
+        else:
+            # explicitly raise exception to prevent data corruption
+            self.copy(
+                path1, path2, recursive=recursive, maxdepth=maxdepth, onerror="raise"
+            )
+            self.rm(path1, recursive=recursive)
+
+    def rm_file(self, path):
+        """Delete a file"""
+        self._rm(path)
+
+    def _rm(self, path):
+        """Delete one file"""
+        # this is the old name for the method, prefer rm_file
+        raise NotImplementedError
+
+    def rm(self, path, recursive=False, maxdepth=None):
+        """Delete files.
+
+        Parameters
+        ----------
+        path: str or list of str
+            File(s) to delete.
+        recursive: bool
+            If file(s) are directories, recursively delete contents and then
+            also remove the directory
+        maxdepth: int or None
+            Depth to pass to walk for finding files to delete, if recursive.
+            If None, there will be no limit and infinite recursion may be
+            possible.
+        """
+        path = self.expand_path(path, recursive=recursive, maxdepth=maxdepth)
+        for p in reversed(path):
+            self.rm_file(p)
+
+    @classmethod
+    def _parent(cls, path):
+        path = cls._strip_protocol(path)
+        if "/" in path:
+            parent = path.rsplit("/", 1)[0].lstrip(cls.root_marker)
+            return cls.root_marker + parent
+        else:
+            return cls.root_marker
+
+    def _open(
+        self,
+        path,
+        mode="rb",
+        block_size=None,
+        autocommit=True,
+        cache_options=None,
+        **kwargs,
+    ):
+        """Return raw bytes-mode file-like from the file-system"""
+        return AbstractBufferedFile(
+            self,
+            path,
+            mode,
+            block_size,
+            autocommit,
+            cache_options=cache_options,
+            **kwargs,
+        )
+
+    def open(
+        self,
+        path,
+        mode="rb",
+        block_size=None,
+        cache_options=None,
+        compression=None,
+        **kwargs,
+    ):
+        """
+        Return a file-like object from the filesystem
+
+        The resultant instance must function correctly in a context ``with``
+        block.
+
+        Parameters
+        ----------
+        path: str
+            Target file
+        mode: str like 'rb', 'w'
+            See builtin ``open()``
+        block_size: int
+            Some indication of buffering - this is a value in bytes
+        cache_options : dict, optional
+            Extra arguments to pass through to the cache.
+        compression: string or None
+            If given, open file using compression codec. Can either be a compression
+            name (a key in ``fsspec.compression.compr``) or "infer" to guess the
+            compression from the filename suffix.
+        encoding, errors, newline: passed on to TextIOWrapper for text mode
+        """
+        import io
+
+        path = self._strip_protocol(path)
+        if "b" not in mode:
+            mode = mode.replace("t", "") + "b"
+
+            text_kwargs = {
+                k: kwargs.pop(k)
+                for k in ["encoding", "errors", "newline"]
+                if k in kwargs
+            }
+            return io.TextIOWrapper(
+                self.open(
+                    path,
+                    mode,
+                    block_size=block_size,
+                    cache_options=cache_options,
+                    compression=compression,
+                    **kwargs,
+                ),
+                **text_kwargs,
+            )
+        else:
+            ac = kwargs.pop("autocommit", not self._intrans)
+            f = self._open(
+                path,
+                mode=mode,
+                block_size=block_size,
+                autocommit=ac,
+                cache_options=cache_options,
+                **kwargs,
+            )
+            if compression is not None:
+                from fsspec.compression import compr
+                from fsspec.core import get_compression
+
+                compression = get_compression(path, compression)
+                compress = compr[compression]
+                f = compress(f, mode=mode[0])
+
+            if not ac and "r" not in mode:
+                self.transaction.files.append(f)
+            return f
+
+    def touch(self, path, truncate=True, **kwargs):
+        """Create empty file, or update timestamp
+
+        Parameters
+        ----------
+        path: str
+            file location
+        truncate: bool
+            If True, always set file size to 0; if False, update timestamp and
+            leave file unchanged, if backend allows this
+        """
+        if truncate or not self.exists(path):
+            with self.open(path, "wb", **kwargs):
+                pass
+        else:
+            raise NotImplementedError  # update timestamp, if possible
+
+    def ukey(self, path):
+        """Hash of file properties, to tell if it has changed"""
+        return sha256(str(self.info(path)).encode()).hexdigest()
+
+    def read_block(self, fn, offset, length, delimiter=None):
+        """Read a block of bytes from
+
+        Starting at ``offset`` of the file, read ``length`` bytes.  If
+        ``delimiter`` is set then we ensure that the read starts and stops at
+        delimiter boundaries that follow the locations ``offset`` and ``offset
+        + length``.  If ``offset`` is zero then we start at zero.  The
+        bytestring returned WILL include the end delimiter string.
+
+        If offset+length is beyond the eof, reads to eof.
+
+        Parameters
+        ----------
+        fn: string
+            Path to filename
+        offset: int
+            Byte offset to start read
+        length: int
+            Number of bytes to read. If None, read to end.
+        delimiter: bytes (optional)
+            Ensure reading starts and stops at delimiter bytestring
+
+        Examples
+        --------
+        >>> fs.read_block('data/file.csv', 0, 13)  # doctest: +SKIP
+        b'Alice, 100\\nBo'
+        >>> fs.read_block('data/file.csv', 0, 13, delimiter=b'\\n')  # doctest: +SKIP
+        b'Alice, 100\\nBob, 200\\n'
+
+        Use ``length=None`` to read to the end of the file.
+        >>> fs.read_block('data/file.csv', 0, None, delimiter=b'\\n')  # doctest: +SKIP
+        b'Alice, 100\\nBob, 200\\nCharlie, 300'
+
+        See Also
+        --------
+        :func:`fsspec.utils.read_block`
+        """
+        with self.open(fn, "rb") as f:
+            size = f.size
+            if length is None:
+                length = size
+            if size is not None and offset + length > size:
+                length = size - offset
+            return read_block(f, offset, length, delimiter)
+
+    def to_json(self):
+        """
+        JSON representation of this filesystem instance
+
+        Returns
+        -------
+        str: JSON structure with keys cls (the python location of this class),
+            protocol (text name of this class's protocol, first one in case of
+            multiple), args (positional args, usually empty), and all other
+            kwargs as their own keys.
+        """
+        import json
+
+        cls = type(self)
+        cls = ".".join((cls.__module__, cls.__name__))
+        proto = (
+            self.protocol[0]
+            if isinstance(self.protocol, (tuple, list))
+            else self.protocol
+        )
+        return json.dumps(
+            dict(
+                cls=cls,
+                protocol=proto,
+                args=self.storage_args,
+                **self.storage_options,
+            )
+        )
+
+    @staticmethod
+    def from_json(blob):
+        """
+        Recreate a filesystem instance from JSON representation
+
+        See ``.to_json()`` for the expected structure of the input
+
+        Parameters
+        ----------
+        blob: str
+
+        Returns
+        -------
+        file system instance, not necessarily of this particular class.
+        """
+        import json
+
+        from .registry import _import_class, get_filesystem_class
+
+        dic = json.loads(blob)
+        protocol = dic.pop("protocol")
+        try:
+            cls = _import_class(dic.pop("cls"))
+        except (ImportError, ValueError, RuntimeError, KeyError):
+            cls = get_filesystem_class(protocol)
+        return cls(*dic.pop("args", ()), **dic)
+
+    def _get_pyarrow_filesystem(self):
+        """
+        Make a version of the FS instance which will be acceptable to pyarrow
+        """
+        # all instances already also derive from pyarrow
+        return self
+
+    def get_mapper(self, root="", check=False, create=False, missing_exceptions=None):
+        """Create key/value store based on this file-system
+
+        Makes a MutableMapping interface to the FS at the given root path.
+        See ``fsspec.mapping.FSMap`` for further details.
+        """
+        from .mapping import FSMap
+
+        return FSMap(
+            root,
+            self,
+            check=check,
+            create=create,
+            missing_exceptions=missing_exceptions,
+        )
+
+    @classmethod
+    def clear_instance_cache(cls):
+        """
+        Clear the cache of filesystem instances.
+
+        Notes
+        -----
+        Unless overridden by setting the ``cachable`` class attribute to False,
+        the filesystem class stores a reference to newly created instances. This
+        prevents Python's normal rules around garbage collection from working,
+        since the instances refcount will not drop to zero until
+        ``clear_instance_cache`` is called.
+        """
+        cls._cache.clear()
+
+    def created(self, path):
+        """Return the created timestamp of a file as a datetime.datetime"""
+        raise NotImplementedError
+
+    def modified(self, path):
+        """Return the modified timestamp of a file as a datetime.datetime"""
+        raise NotImplementedError
+
+    # ------------------------------------------------------------------------
+    # Aliases
+
+    def read_bytes(self, path, start=None, end=None, **kwargs):
+        """Alias of `AbstractFileSystem.cat_file`."""
+        return self.cat_file(path, start=start, end=end, **kwargs)
+
+    def write_bytes(self, path, value, **kwargs):
+        """Alias of `AbstractFileSystem.pipe_file`."""
+        self.pipe_file(path, value, **kwargs)
+
+    def makedir(self, path, create_parents=True, **kwargs):
+        """Alias of `AbstractFileSystem.mkdir`."""
+        return self.mkdir(path, create_parents=create_parents, **kwargs)
+
+    def mkdirs(self, path, exist_ok=False):
+        """Alias of `AbstractFileSystem.makedirs`."""
+        return self.makedirs(path, exist_ok=exist_ok)
+
+    def listdir(self, path, detail=True, **kwargs):
+        """Alias of `AbstractFileSystem.ls`."""
+        return self.ls(path, detail=detail, **kwargs)
+
+    def cp(self, path1, path2, **kwargs):
+        """Alias of `AbstractFileSystem.copy`."""
+        return self.copy(path1, path2, **kwargs)
+
+    def move(self, path1, path2, **kwargs):
+        """Alias of `AbstractFileSystem.mv`."""
+        return self.mv(path1, path2, **kwargs)
+
+    def stat(self, path, **kwargs):
+        """Alias of `AbstractFileSystem.info`."""
+        return self.info(path, **kwargs)
+
+    def disk_usage(self, path, total=True, maxdepth=None, **kwargs):
+        """Alias of `AbstractFileSystem.du`."""
+        return self.du(path, total=total, maxdepth=maxdepth, **kwargs)
+
+    def rename(self, path1, path2, **kwargs):
+        """Alias of `AbstractFileSystem.mv`."""
+        return self.mv(path1, path2, **kwargs)
+
+    def delete(self, path, recursive=False, maxdepth=None):
+        """Alias of `AbstractFileSystem.rm`."""
+        return self.rm(path, recursive=recursive, maxdepth=maxdepth)
+
+    def upload(self, lpath, rpath, recursive=False, **kwargs):
+        """Alias of `AbstractFileSystem.put`."""
+        return self.put(lpath, rpath, recursive=recursive, **kwargs)
+
+    def download(self, rpath, lpath, recursive=False, **kwargs):
+        """Alias of `AbstractFileSystem.get`."""
+        return self.get(rpath, lpath, recursive=recursive, **kwargs)
+
+    def sign(self, path, expiration=100, **kwargs):
+        """Create a signed URL representing the given path
+
+        Some implementations allow temporary URLs to be generated, as a
+        way of delegating credentials.
+
+        Parameters
+        ----------
+        path : str
+             The path on the filesystem
+        expiration : int
+            Number of seconds to enable the URL for (if supported)
+
+        Returns
+        -------
+        URL : str
+            The signed URL
+
+        Raises
+        ------
+        NotImplementedError : if method is not implemented for a filesystem
+        """
+        raise NotImplementedError("Sign is not implemented for this filesystem")
+
+    def _isfilestore(self):
+        # Originally inherited from pyarrow DaskFileSystem. Keeping this
+        # here for backwards compatibility as long as pyarrow uses its
+        # legacy fsspec-compatible filesystems and thus accepts fsspec
+        # filesystems as well
+        return False
+
+
+class AbstractBufferedFile(io.IOBase):
+    """Convenient class to derive from to provide buffering
+
+    In the case that the backend does not provide a pythonic file-like object
+    already, this class contains much of the logic to build one. The only
+    methods that need to be overridden are ``_upload_chunk``,
+    ``_initiate_upload`` and ``_fetch_range``.
+    """
+
+    DEFAULT_BLOCK_SIZE = 5 * 2**20
+    _details = None
+
+    def __init__(
+        self,
+        fs,
+        path,
+        mode="rb",
+        block_size="default",
+        autocommit=True,
+        cache_type="readahead",
+        cache_options=None,
+        size=None,
+        **kwargs,
+    ):
+        """
+        Template for files with buffered reading and writing
+
+        Parameters
+        ----------
+        fs: instance of FileSystem
+        path: str
+            location in file-system
+        mode: str
+            Normal file modes. Currently only 'wb', 'ab' or 'rb'. Some file
+            systems may be read-only, and some may not support append.
+        block_size: int
+            Buffer size for reading or writing, 'default' for class default
+        autocommit: bool
+            Whether to write to final destination; may only impact what
+            happens when file is being closed.
+        cache_type: {"readahead", "none", "mmap", "bytes"}, default "readahead"
+            Caching policy in read mode. See the definitions in ``core``.
+        cache_options : dict
+            Additional options passed to the constructor for the cache specified
+            by `cache_type`.
+        size: int
+            If given and in read mode, suppressed having to look up the file size
+        kwargs:
+            Gets stored as self.kwargs
+        """
+        from .core import caches
+
+        self.path = path
+        self.fs = fs
+        self.mode = mode
+        self.blocksize = (
+            self.DEFAULT_BLOCK_SIZE if block_size in ["default", None] else block_size
+        )
+        self.loc = 0
+        self.autocommit = autocommit
+        self.end = None
+        self.start = None
+        self.closed = False
+
+        if cache_options is None:
+            cache_options = {}
+
+        if "trim" in kwargs:
+            warnings.warn(
+                "Passing 'trim' to control the cache behavior has been deprecated. "
+                "Specify it within the 'cache_options' argument instead.",
+                FutureWarning,
+            )
+            cache_options["trim"] = kwargs.pop("trim")
+
+        self.kwargs = kwargs
+
+        if mode not in {"ab", "rb", "wb"}:
+            raise NotImplementedError("File mode not supported")
+        if mode == "rb":
+            if size is not None:
+                self.size = size
+            else:
+                self.size = self.details["size"]
+            self.cache = caches[cache_type](
+                self.blocksize, self._fetch_range, self.size, **cache_options
+            )
+        else:
+            self.buffer = io.BytesIO()
+            self.offset = None
+            self.forced = False
+            self.location = None
+
+    @property
+    def details(self):
+        if self._details is None:
+            self._details = self.fs.info(self.path)
+        return self._details
+
+    @details.setter
+    def details(self, value):
+        self._details = value
+        self.size = value["size"]
+
+    @property
+    def full_name(self):
+        return _unstrip_protocol(self.path, self.fs)
+
+    @property
+    def closed(self):
+        # get around this attr being read-only in IOBase
+        # use getattr here, since this can be called during del
+        return getattr(self, "_closed", True)
+
+    @closed.setter
+    def closed(self, c):
+        self._closed = c
+
+    def __hash__(self):
+        if "w" in self.mode:
+            return id(self)
+        else:
+            return int(tokenize(self.details), 16)
+
+    def __eq__(self, other):
+        """Files are equal if they have the same checksum, only in read mode"""
+        if self is other:
+            return True
+        return (
+            isinstance(other, type(self))
+            and self.mode == "rb"
+            and other.mode == "rb"
+            and hash(self) == hash(other)
+        )
+
+    def commit(self):
+        """Move from temp to final destination"""
+
+    def discard(self):
+        """Throw away temporary file"""
+
+    def info(self):
+        """File information about this path"""
+        if "r" in self.mode:
+            return self.details
+        else:
+            raise ValueError("Info not available while writing")
+
+    def tell(self):
+        """Current file location"""
+        return self.loc
+
+    def seek(self, loc, whence=0):
+        """Set current file location
+
+        Parameters
+        ----------
+        loc: int
+            byte location
+        whence: {0, 1, 2}
+            from start of file, current location or end of file, resp.
+        """
+        loc = int(loc)
+        if not self.mode == "rb":
+            raise OSError(ESPIPE, "Seek only available in read mode")
+        if whence == 0:
+            nloc = loc
+        elif whence == 1:
+            nloc = self.loc + loc
+        elif whence == 2:
+            nloc = self.size + loc
+        else:
+            raise ValueError(f"invalid whence ({whence}, should be 0, 1 or 2)")
+        if nloc < 0:
+            raise ValueError("Seek before start of file")
+        self.loc = nloc
+        return self.loc
+
+    def write(self, data):
+        """
+        Write data to buffer.
+
+        Buffer only sent on flush() or if buffer is greater than
+        or equal to blocksize.
+
+        Parameters
+        ----------
+        data: bytes
+            Set of bytes to be written.
+        """
+        if self.mode not in {"wb", "ab"}:
+            raise ValueError("File not in write mode")
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        if self.forced:
+            raise ValueError("This file has been force-flushed, can only close")
+        out = self.buffer.write(data)
+        self.loc += out
+        if self.buffer.tell() >= self.blocksize:
+            self.flush()
+        return out
+
+    def flush(self, force=False):
+        """
+        Write buffered data to backend store.
+
+        Writes the current buffer, if it is larger than the block-size, or if
+        the file is being closed.
+
+        Parameters
+        ----------
+        force: bool
+            When closing, write the last block even if it is smaller than
+            blocks are allowed to be. Disallows further writing to this file.
+        """
+
+        if self.closed:
+            raise ValueError("Flush on closed file")
+        if force and self.forced:
+            raise ValueError("Force flush cannot be called more than once")
+        if force:
+            self.forced = True
+
+        if self.mode not in {"wb", "ab"}:
+            # no-op to flush on read-mode
+            return
+
+        if not force and self.buffer.tell() < self.blocksize:
+            # Defer write on small block
+            return
+
+        if self.offset is None:
+            # Initialize a multipart upload
+            self.offset = 0
+            try:
+                self._initiate_upload()
+            except:  # noqa: E722
+                self.closed = True
+                raise
+
+        if self._upload_chunk(final=force) is not False:
+            self.offset += self.buffer.seek(0, 2)
+            self.buffer = io.BytesIO()
+
+    def _upload_chunk(self, final=False):
+        """Write one part of a multi-block file upload
+
+        Parameters
+        ==========
+        final: bool
+            This is the last block, so should complete file, if
+            self.autocommit is True.
+        """
+        # may not yet have been initialized, may need to call _initialize_upload
+
+    def _initiate_upload(self):
+        """Create remote file/upload"""
+        pass
+
+    def _fetch_range(self, start, end):
+        """Get the specified set of bytes from remote"""
+        raise NotImplementedError
+
+    def read(self, length=-1):
+        """
+        Return data from cache, or fetch pieces as necessary
+
+        Parameters
+        ----------
+        length: int (-1)
+            Number of bytes to read; if <0, all remaining bytes.
+        """
+        length = -1 if length is None else int(length)
+        if self.mode != "rb":
+            raise ValueError("File not in read mode")
+        if length < 0:
+            length = self.size - self.loc
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        if length == 0:
+            # don't even bother calling fetch
+            return b""
+        out = self.cache._fetch(self.loc, self.loc + length)
+
+        logger.debug(
+            "%s read: %i - %i %s",
+            self,
+            self.loc,
+            self.loc + length,
+            self.cache._log_stats(),
+        )
+        self.loc += len(out)
+        return out
+
+    def readinto(self, b):
+        """mirrors builtin file's readinto method
+
+        https://docs.python.org/3/library/io.html#io.RawIOBase.readinto
+        """
+        out = memoryview(b).cast("B")
+        data = self.read(out.nbytes)
+        out[: len(data)] = data
+        return len(data)
+
+    def readuntil(self, char=b"\n", blocks=None):
+        """Return data between current position and first occurrence of char
+
+        char is included in the output, except if the end of the tile is
+        encountered first.
+
+        Parameters
+        ----------
+        char: bytes
+            Thing to find
+        blocks: None or int
+            How much to read in each go. Defaults to file blocksize - which may
+            mean a new read on every call.
+        """
+        out = []
+        while True:
+            start = self.tell()
+            part = self.read(blocks or self.blocksize)
+            if len(part) == 0:
+                break
+            found = part.find(char)
+            if found > -1:
+                out.append(part[: found + len(char)])
+                self.seek(start + found + len(char))
+                break
+            out.append(part)
+        return b"".join(out)
+
+    def readline(self):
+        """Read until first occurrence of newline character
+
+        Note that, because of character encoding, this is not necessarily a
+        true line ending.
+        """
+        return self.readuntil(b"\n")
+
+    def __next__(self):
+        out = self.readline()
+        if out:
+            return out
+        raise StopIteration
+
+    def __iter__(self):
+        return self
+
+    def readlines(self):
+        """Return all data, split by the newline character"""
+        data = self.read()
+        lines = data.split(b"\n")
+        out = [l + b"\n" for l in lines[:-1]]
+        if data.endswith(b"\n"):
+            return out
+        else:
+            return out + [lines[-1]]
+        # return list(self)  ???
+
+    def readinto1(self, b):
+        return self.readinto(b)
+
+    def close(self):
+        """Close file
+
+        Finalizes writes, discards cache
+        """
+        if getattr(self, "_unclosable", False):
+            return
+        if self.closed:
+            return
+        if self.mode == "rb":
+            self.cache = None
+        else:
+            if not self.forced:
+                self.flush(force=True)
+
+            if self.fs is not None:
+                self.fs.invalidate_cache(self.path)
+                self.fs.invalidate_cache(self.fs._parent(self.path))
+
+        self.closed = True
+
+    def readable(self):
+        """Whether opened for reading"""
+        return self.mode == "rb" and not self.closed
+
+    def seekable(self):
+        """Whether is seekable (only in read mode)"""
+        return self.readable()
+
+    def writable(self):
+        """Whether opened for writing"""
+        return self.mode in {"wb", "ab"} and not self.closed
+
+    def __del__(self):
+        if not self.closed:
+            self.close()
+
+    def __str__(self):
+        return f"<File-like object {type(self.fs).__name__}, {self.path}>"
+
+    __repr__ = __str__
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/hf_xet-1.3.2.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/hf_xet-1.3.2.dist-info/METADATA

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: hf-xet
+Version: 1.3.2
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: Free Threading
+Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Dist: pytest ; extra == 'tests'
+Provides-Extra: tests
+License-File: LICENSE
+Summary: Fast transfer of large files with the Hugging Face Hub.
+Maintainer-email: Rajat Arya <rajat@rajatarya.com>, Jared Sulzdorf <j.sulzdorf@gmail.com>, Di Xiao <di@huggingface.co>, Assaf Vayner <assaf@huggingface.co>, Hoyt Koepke <hoytak@gmail.com>
+License-Expression: Apache-2.0
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Documentation, https://huggingface.co/docs/hub/xet/index
+Project-URL: Homepage, https://github.com/huggingface/xet-core
+Project-URL: Issues, https://github.com/huggingface/xet-core/issues
+Project-URL: Repository, https://github.com/huggingface/xet-core.git
+
+<!---
+Copyright 2024 The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p align="center">
+    <a href="https://github.com/huggingface/xet-core/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/github/license/huggingface/xet-core.svg?color=blue"></a>
+    <a href="https://github.com/huggingface/xet-core/releases"><img alt="GitHub release" src="https://img.shields.io/github/release/huggingface/xet-core.svg"></a>
+    <a href="https://github.com/huggingface/xet-core/blob/main/CODE_OF_CONDUCT.md"><img alt="Contributor Covenant" src="https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg"></a>
+</p>
+
+<h3 align="center">
+  <p>🤗 hf-xet - xet client tech, used in <a target="_blank" href="https://github.com/huggingface/huggingface_hub/">huggingface_hub</a></p>
+</h3>
+
+## Welcome
+
+`hf-xet` enables `huggingface_hub` to utilize xet storage for uploading and downloading to HF Hub. Xet storage provides chunk-based deduplication, efficient storage/retrieval with local disk caching, and backwards compatibility with Git LFS. This library is not meant to be used directly, and is instead intended to be used from [huggingface_hub](https://pypi.org/project/huggingface-hub).
+
+## Key features
+
+♻ **chunk-based deduplication implementation**: avoid transferring and storing chunks that are shared across binary files (models, datasets, etc).
+
+🤗 **Python bindings**: bindings for [huggingface_hub](https://github.com/huggingface/huggingface_hub/) package.
+
+↔ **network communications**: concurrent communication to HF Hub Xet backend services (CAS).
+
+🔖 **local disk caching**: chunk-based cache that sits alongside the existing [huggingface_hub disk cache](https://huggingface.co/docs/huggingface_hub/guides/manage-cache).
+
+## Installation
+
+Install the `hf_xet` package with [pip](https://pypi.org/project/hf-xet/):
+
+```bash
+pip install hf_xet
+```
+
+## Quick Start
+
+`hf_xet` is not intended to be run independently as it is expected to be used from `huggingface_hub`, so to get started with `huggingface_hub` check out the documentation [here]("https://hf.co/docs/huggingface_hub").
+
+## Contributions (feature requests, bugs, etc.) are encouraged & appreciated 💙💚💛💜🧡❤️
+
+Please join us in making hf-xet better. We value everyone's contributions. Code is not the only way to help. Answering questions, helping each other, improving documentation, filing issues all help immensely. If you are interested in contributing (please do!), check out the [contribution guide](https://github.com/huggingface/xet-core/blob/main/CONTRIBUTING.md) for this repository.

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/hf_xet-1.3.2.dist-info/RECORD

@@ -0,0 +1,9 @@
+hf_xet-1.3.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+hf_xet-1.3.2.dist-info/METADATA,sha256=VdBkvkyMsWS6C51XipEai-iZGPIt9ydUqE6iReFQdVI,4882
+hf_xet-1.3.2.dist-info/RECORD,,
+hf_xet-1.3.2.dist-info/WHEEL,sha256=ycZRBBQNh-_vBSCR_X0ck7NlL8NDEI7G-tWH-Cjb8uw,143
+hf_xet-1.3.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+hf_xet-1.3.2.dist-info/sboms/hf_xet.cyclonedx.json,sha256=Z8lgnfjLuQ1OS7SIhJmZNn9qdETBotCqRMS1kgQOnxE,394826
+hf_xet/__init__.py,sha256=E8UDdyQ8glZ_nve9hHEf22bPang8-RKx4VuApXYeQUo,107
+hf_xet/__pycache__/__init__.cpython-312.pyc,,
+hf_xet/hf_xet.abi3.so,sha256=35NZSsDbLEwZbmB-1aRzdjHhZD_MwMLDqP-tmCeN2gE,10594520

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/hf_xet-1.3.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: maturin (1.12.4)
+Root-Is-Purelib: false
+Tag: cp37-abi3-manylinux_2_17_x86_64
+Tag: cp37-abi3-manylinux2014_x86_64

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/idna-3.10.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/idna-3.10.dist-info/LICENSE.md

@@ -0,0 +1,31 @@
+BSD 3-Clause License
+
+Copyright (c) 2013-2024, Kim Davies and contributors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/idna-3.10.dist-info/METADATA

@@ -0,0 +1,250 @@
+Metadata-Version: 2.1
+Name: idna
+Version: 3.10
+Summary: Internationalized Domain Names in Applications (IDNA)
+Author-email: Kim Davies <kim+pypi@gumleaf.org>
+Requires-Python: >=3.6
+Description-Content-Type: text/x-rst
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: Internet :: Name Service (DNS)
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Utilities
+Requires-Dist: ruff >= 0.6.2 ; extra == "all"
+Requires-Dist: mypy >= 1.11.2 ; extra == "all"
+Requires-Dist: pytest >= 8.3.2 ; extra == "all"
+Requires-Dist: flake8 >= 7.1.1 ; extra == "all"
+Project-URL: Changelog, https://github.com/kjd/idna/blob/master/HISTORY.rst
+Project-URL: Issue tracker, https://github.com/kjd/idna/issues
+Project-URL: Source, https://github.com/kjd/idna
+Provides-Extra: all
+
+Internationalized Domain Names in Applications (IDNA)
+=====================================================
+
+Support for the Internationalized Domain Names in
+Applications (IDNA) protocol as specified in `RFC 5891
+<https://tools.ietf.org/html/rfc5891>`_. This is the latest version of
+the protocol and is sometimes referred to as “IDNA 2008”.
+
+This library also provides support for Unicode Technical
+Standard 46, `Unicode IDNA Compatibility Processing
+<https://unicode.org/reports/tr46/>`_.
+
+This acts as a suitable replacement for the “encodings.idna”
+module that comes with the Python standard library, but which
+only supports the older superseded IDNA specification (`RFC 3490
+<https://tools.ietf.org/html/rfc3490>`_).
+
+Basic functions are simply executed:
+
+.. code-block:: pycon
+
+    >>> import idna
+    >>> idna.encode('ドメイン.テスト')
+    b'xn--eckwd4c7c.xn--zckzah'
+    >>> print(idna.decode('xn--eckwd4c7c.xn--zckzah'))
+    ドメイン.テスト
+
+
+Installation
+------------
+
+This package is available for installation from PyPI:
+
+.. code-block:: bash
+
+    $ python3 -m pip install idna
+
+
+Usage
+-----
+
+For typical usage, the ``encode`` and ``decode`` functions will take a
+domain name argument and perform a conversion to A-labels or U-labels
+respectively.
+
+.. code-block:: pycon
+
+    >>> import idna
+    >>> idna.encode('ドメイン.テスト')
+    b'xn--eckwd4c7c.xn--zckzah'
+    >>> print(idna.decode('xn--eckwd4c7c.xn--zckzah'))
+    ドメイン.テスト
+
+You may use the codec encoding and decoding methods using the
+``idna.codec`` module:
+
+.. code-block:: pycon
+
+    >>> import idna.codec
+    >>> print('домен.испытание'.encode('idna2008'))
+    b'xn--d1acufc.xn--80akhbyknj4f'
+    >>> print(b'xn--d1acufc.xn--80akhbyknj4f'.decode('idna2008'))
+    домен.испытание
+
+Conversions can be applied at a per-label basis using the ``ulabel`` or
+``alabel`` functions if necessary:
+
+.. code-block:: pycon
+
+    >>> idna.alabel('测试')
+    b'xn--0zwm56d'
+
+Compatibility Mapping (UTS #46)
++++++++++++++++++++++++++++++++
+
+As described in `RFC 5895 <https://tools.ietf.org/html/rfc5895>`_, the
+IDNA specification does not normalize input from different potential
+ways a user may input a domain name. This functionality, known as
+a “mapping”, is considered by the specification to be a local
+user-interface issue distinct from IDNA conversion functionality.
+
+This library provides one such mapping that was developed by the
+Unicode Consortium. Known as `Unicode IDNA Compatibility Processing
+<https://unicode.org/reports/tr46/>`_, it provides for both a regular
+mapping for typical applications, as well as a transitional mapping to
+help migrate from older IDNA 2003 applications. Strings are
+preprocessed according to Section 4.4 “Preprocessing for IDNA2008”
+prior to the IDNA operations.
+
+For example, “Königsgäßchen” is not a permissible label as *LATIN
+CAPITAL LETTER K* is not allowed (nor are capital letters in general).
+UTS 46 will convert this into lower case prior to applying the IDNA
+conversion.
+
+.. code-block:: pycon
+
+    >>> import idna
+    >>> idna.encode('Königsgäßchen')
+    ...
+    idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed
+    >>> idna.encode('Königsgäßchen', uts46=True)
+    b'xn--knigsgchen-b4a3dun'
+    >>> print(idna.decode('xn--knigsgchen-b4a3dun'))
+    königsgäßchen
+
+Transitional processing provides conversions to help transition from
+the older 2003 standard to the current standard. For example, in the
+original IDNA specification, the *LATIN SMALL LETTER SHARP S* (ß) was
+converted into two *LATIN SMALL LETTER S* (ss), whereas in the current
+IDNA specification this conversion is not performed.
+
+.. code-block:: pycon
+
+    >>> idna.encode('Königsgäßchen', uts46=True, transitional=True)
+    'xn--knigsgsschen-lcb0w'
+
+Implementers should use transitional processing with caution, only in
+rare cases where conversion from legacy labels to current labels must be
+performed (i.e. IDNA implementations that pre-date 2008). For typical
+applications that just need to convert labels, transitional processing
+is unlikely to be beneficial and could produce unexpected incompatible
+results.
+
+``encodings.idna`` Compatibility
+++++++++++++++++++++++++++++++++
+
+Function calls from the Python built-in ``encodings.idna`` module are
+mapped to their IDNA 2008 equivalents using the ``idna.compat`` module.
+Simply substitute the ``import`` clause in your code to refer to the new
+module name.
+
+Exceptions
+----------
+
+All errors raised during the conversion following the specification
+should raise an exception derived from the ``idna.IDNAError`` base
+class.
+
+More specific exceptions that may be generated as ``idna.IDNABidiError``
+when the error reflects an illegal combination of left-to-right and
+right-to-left characters in a label; ``idna.InvalidCodepoint`` when
+a specific codepoint is an illegal character in an IDN label (i.e.
+INVALID); and ``idna.InvalidCodepointContext`` when the codepoint is
+illegal based on its positional context (i.e. it is CONTEXTO or CONTEXTJ
+but the contextual requirements are not satisfied.)
+
+Building and Diagnostics
+------------------------
+
+The IDNA and UTS 46 functionality relies upon pre-calculated lookup
+tables for performance. These tables are derived from computing against
+eligibility criteria in the respective standards. These tables are
+computed using the command-line script ``tools/idna-data``.
+
+This tool will fetch relevant codepoint data from the Unicode repository
+and perform the required calculations to identify eligibility. There are
+three main modes:
+
+* ``idna-data make-libdata``. Generates ``idnadata.py`` and
+  ``uts46data.py``, the pre-calculated lookup tables used for IDNA and
+  UTS 46 conversions. Implementers who wish to track this library against
+  a different Unicode version may use this tool to manually generate a
+  different version of the ``idnadata.py`` and ``uts46data.py`` files.
+
+* ``idna-data make-table``. Generate a table of the IDNA disposition
+  (e.g. PVALID, CONTEXTJ, CONTEXTO) in the format found in Appendix
+  B.1 of RFC 5892 and the pre-computed tables published by `IANA
+  <https://www.iana.org/>`_.
+
+* ``idna-data U+0061``. Prints debugging output on the various
+  properties associated with an individual Unicode codepoint (in this
+  case, U+0061), that are used to assess the IDNA and UTS 46 status of a
+  codepoint. This is helpful in debugging or analysis.
+
+The tool accepts a number of arguments, described using ``idna-data
+-h``. Most notably, the ``--version`` argument allows the specification
+of the version of Unicode to be used in computing the table data. For
+example, ``idna-data --version 9.0.0 make-libdata`` will generate
+library data against Unicode 9.0.0.
+
+
+Additional Notes
+----------------
+
+* **Packages**. The latest tagged release version is published in the
+  `Python Package Index <https://pypi.org/project/idna/>`_.
+
+* **Version support**. This library supports Python 3.6 and higher.
+  As this library serves as a low-level toolkit for a variety of
+  applications, many of which strive for broad compatibility with older
+  Python versions, there is no rush to remove older interpreter support.
+  Removing support for older versions should be well justified in that the
+  maintenance burden has become too high.
+
+* **Python 2**. Python 2 is supported by version 2.x of this library.
+  Use "idna<3" in your requirements file if you need this library for
+  a Python 2 application. Be advised that these versions are no longer
+  actively developed.
+
+* **Testing**. The library has a test suite based on each rule of the
+  IDNA specification, as well as tests that are provided as part of the
+  Unicode Technical Standard 46, `Unicode IDNA Compatibility Processing
+  <https://unicode.org/reports/tr46/>`_.
+
+* **Emoji**. It is an occasional request to support emoji domains in
+  this library. Encoding of symbols like emoji is expressly prohibited by
+  the technical standard IDNA 2008 and emoji domains are broadly phased
+  out across the domain industry due to associated security risks. For
+  now, applications that need to support these non-compliant labels
+  may wish to consider trying the encode/decode operation in this library
+  first, and then falling back to using `encodings.idna`. See `the Github
+  project <https://github.com/kjd/idna/issues/18>`_ for more discussion.
+

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/idna-3.10.dist-info/RECORD

@@ -0,0 +1,22 @@
+idna-3.10.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+idna-3.10.dist-info/LICENSE.md,sha256=pZ8LDvNjWHQQmkRhykT_enDVBpboFHZ7-vch1Mmw2w8,1541
+idna-3.10.dist-info/METADATA,sha256=URR5ZyDfQ1PCEGhkYoojqfi2Ra0tau2--lhwG4XSfjI,10158
+idna-3.10.dist-info/RECORD,,
+idna-3.10.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+idna/__init__.py,sha256=MPqNDLZbXqGaNdXxAFhiqFPKEQXju2jNQhCey6-5eJM,868
+idna/__pycache__/__init__.cpython-312.pyc,,
+idna/__pycache__/codec.cpython-312.pyc,,
+idna/__pycache__/compat.cpython-312.pyc,,
+idna/__pycache__/core.cpython-312.pyc,,
+idna/__pycache__/idnadata.cpython-312.pyc,,
+idna/__pycache__/intranges.cpython-312.pyc,,
+idna/__pycache__/package_data.cpython-312.pyc,,
+idna/__pycache__/uts46data.cpython-312.pyc,,
+idna/codec.py,sha256=PEew3ItwzjW4hymbasnty2N2OXvNcgHB-JjrBuxHPYY,3422
+idna/compat.py,sha256=RzLy6QQCdl9784aFhb2EX9EKGCJjg0P3PilGdeXXcx8,316
+idna/core.py,sha256=YJYyAMnwiQEPjVC4-Fqu_p4CJ6yKKuDGmppBNQNQpFs,13239
+idna/idnadata.py,sha256=W30GcIGvtOWYwAjZj4ZjuouUutC6ffgNuyjJy7fZ-lo,78306
+idna/intranges.py,sha256=amUtkdhYcQG8Zr-CoMM_kVRacxkivC1WgxN1b63KKdU,1898
+idna/package_data.py,sha256=q59S3OXsc5VI8j6vSD0sGBMyk6zZ4vWFREE88yCJYKs,21
+idna/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+idna/uts46data.py,sha256=rt90K9J40gUSwppDPCrhjgi5AA6pWM65dEGRSf6rIhM,239289

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/idna-3.10.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: flit 3.9.0
+Root-Is-Purelib: true
+Tag: py3-none-any

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/nvidia_nccl_cu12-2.27.5.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/nvidia_nccl_cu12-2.27.5.dist-info/METADATA

@@ -0,0 +1,45 @@
+Metadata-Version: 2.4
+Name: nvidia-nccl-cu12
+Version: 2.27.5
+Summary: NVIDIA Collective Communication Library (NCCL) Runtime
+Home-page: https://developer.nvidia.com/cuda-zone
+Author: Nvidia CUDA Installer Team
+Author-email: compute_installer@nvidia.com
+License: BSD-3-Clause
+Keywords: cuda,nvidia,runtime,machine learning,deep learning
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: Other/Proprietary License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX :: Linux
+Requires-Python: >=3
+License-File: License.txt
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-python
+Dynamic: summary
+
+NCCL (pronounced "Nickel") is a stand-alone library of standard collective communication routines for GPUs, implementing all-reduce, all-gather, reduce, broadcast, and reduce-scatter. It has been optimized to achieve high bandwidth on any platform using PCIe, NVLink, NVswitch, as well as networking using InfiniBand Verbs or TCP/IP sockets.

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/nvidia_nccl_cu12-2.27.5.dist-info/RECORD

@@ -0,0 +1,8 @@
+nvidia/nccl/include/nccl.h,sha256=SyQKH6eYWMqM3_z1wvxArqyos-rsYXKCE3FfCQKro6g,22566
+nvidia/nccl/lib/libnccl.so.2,sha256=HZwQMi76zeaVz2QNsfKfwVhskp8Qh_DhIC0tXnD7cWg,429628328
+nvidia_nccl_cu12-2.27.5.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+nvidia_nccl_cu12-2.27.5.dist-info/METADATA,sha256=8QYX3uAOzTd6A-kQPrzyDDMwXQ30cZGlQEmKtF-Kxn0,2019
+nvidia_nccl_cu12-2.27.5.dist-info/RECORD,,
+nvidia_nccl_cu12-2.27.5.dist-info/WHEEL,sha256=gy6FWQgpujK_dnYc155G2NL32NQjpi5ebTEXjh8SGZQ,144
+nvidia_nccl_cu12-2.27.5.dist-info/licenses/License.txt,sha256=DwF0prTgszrCY3W_cpUzB1sy9MUaW2gCo9dC19zcmnY,1895
+nvidia_nccl_cu12-2.27.5.dist-info/top_level.txt,sha256=fTkAtiFuL16nUrB9ytDDtpytz2t0B4NvYTnRzwAhO14,7

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/nvidia_nccl_cu12-2.27.5.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-manylinux2014_x86_64
+Tag: py3-none-manylinux_2_17_x86_64
+

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/nvidia_nccl_cu12-2.27.5.dist-info/top_level.txt

@@ -0,0 +1 @@
+nvidia

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/psutil/__init__.py

@@ -0,0 +1,2485 @@
+# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""psutil is a cross-platform library for retrieving information on
+running processes and system utilization (CPU, memory, disks, network,
+sensors) in Python. Supported platforms:
+
+ - Linux
+ - Windows
+ - macOS
+ - FreeBSD
+ - OpenBSD
+ - NetBSD
+ - Sun Solaris
+ - AIX
+
+Supported Python versions are cPython 3.6+ and PyPy.
+"""
+
+import collections
+import contextlib
+import datetime
+import functools
+import os
+import signal
+import socket
+import subprocess
+import sys
+import threading
+import time
+
+try:
+    import pwd
+except ImportError:
+    pwd = None
+
+from . import _common
+from . import _ntuples as _ntp
+from ._common import AIX
+from ._common import BSD
+from ._common import CONN_CLOSE
+from ._common import CONN_CLOSE_WAIT
+from ._common import CONN_CLOSING
+from ._common import CONN_ESTABLISHED
+from ._common import CONN_FIN_WAIT1
+from ._common import CONN_FIN_WAIT2
+from ._common import CONN_LAST_ACK
+from ._common import CONN_LISTEN
+from ._common import CONN_NONE
+from ._common import CONN_SYN_RECV
+from ._common import CONN_SYN_SENT
+from ._common import CONN_TIME_WAIT
+from ._common import FREEBSD
+from ._common import LINUX
+from ._common import MACOS
+from ._common import NETBSD
+from ._common import NIC_DUPLEX_FULL
+from ._common import NIC_DUPLEX_HALF
+from ._common import NIC_DUPLEX_UNKNOWN
+from ._common import OPENBSD
+from ._common import OSX  # deprecated alias
+from ._common import POSIX
+from ._common import POWER_TIME_UNKNOWN
+from ._common import POWER_TIME_UNLIMITED
+from ._common import STATUS_DEAD
+from ._common import STATUS_DISK_SLEEP
+from ._common import STATUS_IDLE
+from ._common import STATUS_LOCKED
+from ._common import STATUS_PARKED
+from ._common import STATUS_RUNNING
+from ._common import STATUS_SLEEPING
+from ._common import STATUS_STOPPED
+from ._common import STATUS_TRACING_STOP
+from ._common import STATUS_WAITING
+from ._common import STATUS_WAKING
+from ._common import STATUS_ZOMBIE
+from ._common import SUNOS
+from ._common import WINDOWS
+from ._common import AccessDenied
+from ._common import Error
+from ._common import NoSuchProcess
+from ._common import TimeoutExpired
+from ._common import ZombieProcess
+from ._common import debug
+from ._common import memoize_when_activated
+from ._common import wrap_numbers as _wrap_numbers
+
+if LINUX:
+    # This is public API and it will be retrieved from _pslinux.py
+    # via sys.modules.
+    PROCFS_PATH = "/proc"
+
+    from . import _pslinux as _psplatform
+    from ._pslinux import IOPRIO_CLASS_BE  # noqa: F401
+    from ._pslinux import IOPRIO_CLASS_IDLE  # noqa: F401
+    from ._pslinux import IOPRIO_CLASS_NONE  # noqa: F401
+    from ._pslinux import IOPRIO_CLASS_RT  # noqa: F401
+
+elif WINDOWS:
+    from . import _pswindows as _psplatform
+    from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS  # noqa: F401
+    from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS  # noqa: F401
+    from ._psutil_windows import HIGH_PRIORITY_CLASS  # noqa: F401
+    from ._psutil_windows import IDLE_PRIORITY_CLASS  # noqa: F401
+    from ._psutil_windows import NORMAL_PRIORITY_CLASS  # noqa: F401
+    from ._psutil_windows import REALTIME_PRIORITY_CLASS  # noqa: F401
+    from ._pswindows import CONN_DELETE_TCB  # noqa: F401
+    from ._pswindows import IOPRIO_HIGH  # noqa: F401
+    from ._pswindows import IOPRIO_LOW  # noqa: F401
+    from ._pswindows import IOPRIO_NORMAL  # noqa: F401
+    from ._pswindows import IOPRIO_VERYLOW  # noqa: F401
+
+elif MACOS:
+    from . import _psosx as _psplatform
+
+elif BSD:
+    from . import _psbsd as _psplatform
+
+elif SUNOS:
+    from . import _pssunos as _psplatform
+    from ._pssunos import CONN_BOUND  # noqa: F401
+    from ._pssunos import CONN_IDLE  # noqa: F401
+
+    # This is public writable API which is read from _pslinux.py and
+    # _pssunos.py via sys.modules.
+    PROCFS_PATH = "/proc"
+
+elif AIX:
+    from . import _psaix as _psplatform
+
+    # This is public API and it will be retrieved from _pslinux.py
+    # via sys.modules.
+    PROCFS_PATH = "/proc"
+
+else:  # pragma: no cover
+    msg = f"platform {sys.platform} is not supported"
+    raise NotImplementedError(msg)
+
+
+# fmt: off
+__all__ = [
+    # exceptions
+    "Error", "NoSuchProcess", "ZombieProcess", "AccessDenied",
+    "TimeoutExpired",
+
+    # constants
+    "version_info", "__version__",
+
+    "STATUS_RUNNING", "STATUS_IDLE", "STATUS_SLEEPING", "STATUS_DISK_SLEEP",
+    "STATUS_STOPPED", "STATUS_TRACING_STOP", "STATUS_ZOMBIE", "STATUS_DEAD",
+    "STATUS_WAKING", "STATUS_LOCKED", "STATUS_WAITING", "STATUS_LOCKED",
+    "STATUS_PARKED",
+
+    "CONN_ESTABLISHED", "CONN_SYN_SENT", "CONN_SYN_RECV", "CONN_FIN_WAIT1",
+    "CONN_FIN_WAIT2", "CONN_TIME_WAIT", "CONN_CLOSE", "CONN_CLOSE_WAIT",
+    "CONN_LAST_ACK", "CONN_LISTEN", "CONN_CLOSING", "CONN_NONE",
+    # "CONN_IDLE", "CONN_BOUND",
+
+    "AF_LINK",
+
+    "NIC_DUPLEX_FULL", "NIC_DUPLEX_HALF", "NIC_DUPLEX_UNKNOWN",
+
+    "POWER_TIME_UNKNOWN", "POWER_TIME_UNLIMITED",
+
+    "BSD", "FREEBSD", "LINUX", "NETBSD", "OPENBSD", "MACOS", "OSX", "POSIX",
+    "SUNOS", "WINDOWS", "AIX",
+
+    # "RLIM_INFINITY", "RLIMIT_AS", "RLIMIT_CORE", "RLIMIT_CPU", "RLIMIT_DATA",
+    # "RLIMIT_FSIZE", "RLIMIT_LOCKS", "RLIMIT_MEMLOCK", "RLIMIT_NOFILE",
+    # "RLIMIT_NPROC", "RLIMIT_RSS", "RLIMIT_STACK", "RLIMIT_MSGQUEUE",
+    # "RLIMIT_NICE", "RLIMIT_RTPRIO", "RLIMIT_RTTIME", "RLIMIT_SIGPENDING",
+
+    # classes
+    "Process", "Popen",
+
+    # functions
+    "pid_exists", "pids", "process_iter", "wait_procs",             # proc
+    "virtual_memory", "swap_memory",                                # memory
+    "cpu_times", "cpu_percent", "cpu_times_percent", "cpu_count",   # cpu
+    "cpu_stats",  # "cpu_freq", "getloadavg"
+    "net_io_counters", "net_connections", "net_if_addrs",           # network
+    "net_if_stats",
+    "disk_io_counters", "disk_partitions", "disk_usage",            # disk
+    # "sensors_temperatures", "sensors_battery", "sensors_fans"     # sensors
+    "users", "boot_time",                                           # others
+]
+# fmt: on
+
+
+__all__.extend(_psplatform.__extra__all__)
+
+# Linux, FreeBSD
+if hasattr(_psplatform.Process, "rlimit"):
+    # Populate global namespace with RLIM* constants.
+    _globals = globals()
+    _name = None
+    for _name in dir(_psplatform.cext):
+        if _name.startswith('RLIM') and _name.isupper():
+            _globals[_name] = getattr(_psplatform.cext, _name)
+            __all__.append(_name)
+    del _globals, _name
+
+AF_LINK = _psplatform.AF_LINK
+
+__author__ = "Giampaolo Rodola'"
+__version__ = "7.2.2"
+version_info = tuple(int(num) for num in __version__.split('.'))
+
+_timer = getattr(time, 'monotonic', time.time)
+_TOTAL_PHYMEM = None
+_LOWEST_PID = None
+_SENTINEL = object()
+
+# Sanity check in case the user messed up with psutil installation
+# or did something weird with sys.path. In this case we might end
+# up importing a python module using a C extension module which
+# was compiled for a different version of psutil.
+# We want to prevent that by failing sooner rather than later.
+# See: https://github.com/giampaolo/psutil/issues/564
+if int(__version__.replace('.', '')) != getattr(
+    _psplatform.cext, 'version', None
+):
+    msg = f"version conflict: {_psplatform.cext.__file__!r} C extension "
+    msg += "module was built for another version of psutil"
+    if hasattr(_psplatform.cext, 'version'):
+        v = ".".join(list(str(_psplatform.cext.version)))
+        msg += f" ({v} instead of {__version__})"
+    else:
+        msg += f" (different than {__version__})"
+    what = getattr(
+        _psplatform.cext,
+        "__file__",
+        "the existing psutil install directory",
+    )
+    msg += f"; you may try to 'pip uninstall psutil', manually remove {what}"
+    msg += " or clean the virtual env somehow, then reinstall"
+    raise ImportError(msg)
+
+
+# =====================================================================
+# --- Utils
+# =====================================================================
+
+
+if hasattr(_psplatform, 'ppid_map'):
+    # Faster version (Windows and Linux).
+    _ppid_map = _psplatform.ppid_map
+else:  # pragma: no cover
+
+    def _ppid_map():
+        """Return a {pid: ppid, ...} dict for all running processes in
+        one shot. Used to speed up Process.children().
+        """
+        ret = {}
+        for pid in pids():
+            try:
+                ret[pid] = _psplatform.Process(pid).ppid()
+            except (NoSuchProcess, ZombieProcess):
+                pass
+        return ret
+
+
+def _pprint_secs(secs):
+    """Format seconds in a human readable form."""
+    now = time.time()
+    secs_ago = int(now - secs)
+    fmt = "%H:%M:%S" if secs_ago < 60 * 60 * 24 else "%Y-%m-%d %H:%M:%S"
+    return datetime.datetime.fromtimestamp(secs).strftime(fmt)
+
+
+def _check_conn_kind(kind):
+    """Check net_connections()'s `kind` parameter."""
+    kinds = tuple(_common.conn_tmap)
+    if kind not in kinds:
+        msg = f"invalid kind argument {kind!r}; valid ones are: {kinds}"
+        raise ValueError(msg)
+
+
+# =====================================================================
+# --- Process class
+# =====================================================================
+
+
+class Process:
+    """Represents an OS process with the given PID.
+    If PID is omitted current process PID (os.getpid()) is used.
+    Raise NoSuchProcess if PID does not exist.
+
+    Note that most of the methods of this class do not make sure that
+    the PID of the process being queried has been reused. That means
+    that you may end up retrieving information for another process.
+
+    The only exceptions for which process identity is pre-emptively
+    checked and guaranteed are:
+
+     - parent()
+     - children()
+     - nice() (set)
+     - ionice() (set)
+     - rlimit() (set)
+     - cpu_affinity (set)
+     - suspend()
+     - resume()
+     - send_signal()
+     - terminate()
+     - kill()
+
+    To prevent this problem for all other methods you can use
+    is_running() before querying the process.
+    """
+
+    def __init__(self, pid=None):
+        self._init(pid)
+
+    def _init(self, pid, _ignore_nsp=False):
+        if pid is None:
+            pid = os.getpid()
+        else:
+            if pid < 0:
+                msg = f"pid must be a positive integer (got {pid})"
+                raise ValueError(msg)
+            try:
+                _psplatform.cext.check_pid_range(pid)
+            except OverflowError as err:
+                msg = "process PID out of range"
+                raise NoSuchProcess(pid, msg=msg) from err
+
+        self._pid = pid
+        self._name = None
+        self._exe = None
+        self._create_time = None
+        self._gone = False
+        self._pid_reused = False
+        self._hash = None
+        self._lock = threading.RLock()
+        # used for caching on Windows only (on POSIX ppid may change)
+        self._ppid = None
+        # platform-specific modules define an _psplatform.Process
+        # implementation class
+        self._proc = _psplatform.Process(pid)
+        self._last_sys_cpu_times = None
+        self._last_proc_cpu_times = None
+        self._exitcode = _SENTINEL
+        self._ident = (self.pid, None)
+        try:
+            self._ident = self._get_ident()
+        except AccessDenied:
+            # This should happen on Windows only, since we use the fast
+            # create time method. AFAIK, on all other platforms we are
+            # able to get create time for all PIDs.
+            pass
+        except ZombieProcess:
+            # Zombies can still be queried by this class (although
+            # not always) and pids() return them so just go on.
+            pass
+        except NoSuchProcess:
+            if not _ignore_nsp:
+                msg = "process PID not found"
+                raise NoSuchProcess(pid, msg=msg) from None
+            self._gone = True
+
+    def _get_ident(self):
+        """Return a (pid, uid) tuple which is supposed to identify a
+        Process instance univocally over time. The PID alone is not
+        enough, as it can be assigned to a new process after this one
+        terminates, so we add process creation time to the mix. We need
+        this in order to prevent killing the wrong process later on.
+        This is also known as PID reuse or PID recycling problem.
+
+        The reliability of this strategy mostly depends on
+        create_time() precision, which is 0.01 secs on Linux. The
+        assumption is that, after a process terminates, the kernel
+        won't reuse the same PID after such a short period of time
+        (0.01 secs). Technically this is inherently racy, but
+        practically it should be good enough.
+
+        NOTE: unreliable on FreeBSD and OpenBSD as ctime is subject to
+        system clock updates.
+        """
+
+        if WINDOWS:
+            # Use create_time() fast method in order to speedup
+            # `process_iter()`. This means we'll get AccessDenied for
+            # most ADMIN processes, but that's fine since it means
+            # we'll also get AccessDenied on kill().
+            # https://github.com/giampaolo/psutil/issues/2366#issuecomment-2381646555
+            self._create_time = self._proc.create_time(fast_only=True)
+            return (self.pid, self._create_time)
+        elif LINUX or NETBSD or OSX:
+            # Use 'monotonic' process starttime since boot to form unique
+            # process identity, since it is stable over changes to system
+            # time.
+            return (self.pid, self._proc.create_time(monotonic=True))
+        else:
+            return (self.pid, self.create_time())
+
+    def __str__(self):
+        info = collections.OrderedDict()
+        info["pid"] = self.pid
+        if self._name:
+            info['name'] = self._name
+        with self.oneshot():
+            if self._pid_reused:
+                info["status"] = "terminated + PID reused"
+            else:
+                try:
+                    info["name"] = self.name()
+                    info["status"] = self.status()
+                except ZombieProcess:
+                    info["status"] = "zombie"
+                except NoSuchProcess:
+                    info["status"] = "terminated"
+                except AccessDenied:
+                    pass
+
+            if self._exitcode not in {_SENTINEL, None}:
+                info["exitcode"] = self._exitcode
+            if self._create_time is not None:
+                info['started'] = _pprint_secs(self._create_time)
+
+            return "{}.{}({})".format(
+                self.__class__.__module__,
+                self.__class__.__name__,
+                ", ".join([f"{k}={v!r}" for k, v in info.items()]),
+            )
+
+    __repr__ = __str__
+
+    def __eq__(self, other):
+        # Test for equality with another Process object based
+        # on PID and creation time.
+        if not isinstance(other, Process):
+            return NotImplemented
+        if OPENBSD or NETBSD or SUNOS:  # pragma: no cover
+            # Zombie processes on Open/NetBSD/illumos/Solaris have a
+            # creation time of 0.0.  This covers the case when a process
+            # started normally (so it has a ctime), then it turned into a
+            # zombie. It's important to do this because is_running()
+            # depends on __eq__.
+            pid1, ident1 = self._ident
+            pid2, ident2 = other._ident
+            if pid1 == pid2:
+                if ident1 and not ident2:
+                    try:
+                        return self.status() == STATUS_ZOMBIE
+                    except Error:
+                        pass
+        return self._ident == other._ident
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __hash__(self):
+        if self._hash is None:
+            self._hash = hash(self._ident)
+        return self._hash
+
+    def _raise_if_pid_reused(self):
+        """Raises NoSuchProcess in case process PID has been reused."""
+        if self._pid_reused or (not self.is_running() and self._pid_reused):
+            # We may directly raise NSP in here already if PID is just
+            # not running, but I prefer NSP to be raised naturally by
+            # the actual Process API call. This way unit tests will tell
+            # us if the API is broken (aka don't raise NSP when it
+            # should). We also remain consistent with all other "get"
+            # APIs which don't use _raise_if_pid_reused().
+            msg = "process no longer exists and its PID has been reused"
+            raise NoSuchProcess(self.pid, self._name, msg=msg)
+
+    @property
+    def pid(self):
+        """The process PID."""
+        return self._pid
+
+    # --- utility methods
+
+    @contextlib.contextmanager
+    def oneshot(self):
+        """Utility context manager which considerably speeds up the
+        retrieval of multiple process information at the same time.
+
+        Internally different process info (e.g. name, ppid, uids,
+        gids, ...) may be fetched by using the same routine, but
+        only one information is returned and the others are discarded.
+        When using this context manager the internal routine is
+        executed once (in the example below on name()) and the
+        other info are cached.
+
+        The cache is cleared when exiting the context manager block.
+        The advice is to use this every time you retrieve more than
+        one information about the process. If you're lucky, you'll
+        get a hell of a speedup.
+
+        >>> import psutil
+        >>> p = psutil.Process()
+        >>> with p.oneshot():
+        ...     p.name()  # collect multiple info
+        ...     p.cpu_times()  # return cached value
+        ...     p.cpu_percent()  # return cached value
+        ...     p.create_time()  # return cached value
+        ...
+        >>>
+        """
+        with self._lock:
+            if hasattr(self, "_cache"):
+                # NOOP: this covers the use case where the user enters the
+                # context twice:
+                #
+                # >>> with p.oneshot():
+                # ...    with p.oneshot():
+                # ...
+                #
+                # Also, since as_dict() internally uses oneshot()
+                # I expect that the code below will be a pretty common
+                # "mistake" that the user will make, so let's guard
+                # against that:
+                #
+                # >>> with p.oneshot():
+                # ...    p.as_dict()
+                # ...
+                yield
+            else:
+                try:
+                    # cached in case cpu_percent() is used
+                    self.cpu_times.cache_activate(self)
+                    # cached in case memory_percent() is used
+                    self.memory_info.cache_activate(self)
+                    # cached in case parent() is used
+                    self.ppid.cache_activate(self)
+                    # cached in case username() is used
+                    if POSIX:
+                        self.uids.cache_activate(self)
+                    # specific implementation cache
+                    self._proc.oneshot_enter()
+                    yield
+                finally:
+                    self.cpu_times.cache_deactivate(self)
+                    self.memory_info.cache_deactivate(self)
+                    self.ppid.cache_deactivate(self)
+                    if POSIX:
+                        self.uids.cache_deactivate(self)
+                    self._proc.oneshot_exit()
+
+    def as_dict(self, attrs=None, ad_value=None):
+        """Utility method returning process information as a
+        hashable dictionary.
+        If *attrs* is specified it must be a list of strings
+        reflecting available Process class' attribute names
+        (e.g. ['cpu_times', 'name']) else all public (read
+        only) attributes are assumed.
+        *ad_value* is the value which gets assigned in case
+        AccessDenied or ZombieProcess exception is raised when
+        retrieving that particular process information.
+        """
+        valid_names = _as_dict_attrnames
+        if attrs is not None:
+            if not isinstance(attrs, (list, tuple, set, frozenset)):
+                msg = f"invalid attrs type {type(attrs)}"
+                raise TypeError(msg)
+            attrs = set(attrs)
+            invalid_names = attrs - valid_names
+            if invalid_names:
+                msg = "invalid attr name{} {}".format(
+                    "s" if len(invalid_names) > 1 else "",
+                    ", ".join(map(repr, invalid_names)),
+                )
+                raise ValueError(msg)
+
+        retdict = {}
+        ls = attrs or valid_names
+        with self.oneshot():
+            for name in ls:
+                try:
+                    if name == 'pid':
+                        ret = self.pid
+                    else:
+                        meth = getattr(self, name)
+                        ret = meth()
+                except (AccessDenied, ZombieProcess):
+                    ret = ad_value
+                except NotImplementedError:
+                    # in case of not implemented functionality (may happen
+                    # on old or exotic systems) we want to crash only if
+                    # the user explicitly asked for that particular attr
+                    if attrs:
+                        raise
+                    continue
+                retdict[name] = ret
+        return retdict
+
+    def parent(self):
+        """Return the parent process as a Process object pre-emptively
+        checking whether PID has been reused.
+        If no parent is known return None.
+        """
+        lowest_pid = _LOWEST_PID if _LOWEST_PID is not None else pids()[0]
+        if self.pid == lowest_pid:
+            return None
+        ppid = self.ppid()
+        if ppid is not None:
+            # Get a fresh (non-cached) ctime in case the system clock
+            # was updated. TODO: use a monotonic ctime on platforms
+            # where it's supported.
+            proc_ctime = Process(self.pid).create_time()
+            try:
+                parent = Process(ppid)
+                if parent.create_time() <= proc_ctime:
+                    return parent
+                # ...else ppid has been reused by another process
+            except NoSuchProcess:
+                pass
+
+    def parents(self):
+        """Return the parents of this process as a list of Process
+        instances. If no parents are known return an empty list.
+        """
+        parents = []
+        proc = self.parent()
+        while proc is not None:
+            parents.append(proc)
+            proc = proc.parent()
+        return parents
+
+    def is_running(self):
+        """Return whether this process is running.
+
+        It also checks if PID has been reused by another process, in
+        which case it will remove the process from `process_iter()`
+        internal cache and return False.
+        """
+        if self._gone or self._pid_reused:
+            return False
+        try:
+            # Checking if PID is alive is not enough as the PID might
+            # have been reused by another process. Process identity /
+            # uniqueness over time is guaranteed by (PID + creation
+            # time) and that is verified in __eq__.
+            self._pid_reused = self != Process(self.pid)
+            if self._pid_reused:
+                _pids_reused.add(self.pid)
+                raise NoSuchProcess(self.pid)
+            return True
+        except ZombieProcess:
+            # We should never get here as it's already handled in
+            # Process.__init__; here just for extra safety.
+            return True
+        except NoSuchProcess:
+            self._gone = True
+            return False
+
+    # --- actual API
+
+    @memoize_when_activated
+    def ppid(self):
+        """The process parent PID.
+        On Windows the return value is cached after first call.
+        """
+        # On POSIX we don't want to cache the ppid as it may unexpectedly
+        # change to 1 (init) in case this process turns into a zombie:
+        # https://github.com/giampaolo/psutil/issues/321
+        # http://stackoverflow.com/questions/356722/
+
+        # XXX should we check creation time here rather than in
+        # Process.parent()?
+        self._raise_if_pid_reused()
+        if POSIX:
+            return self._proc.ppid()
+        else:  # pragma: no cover
+            self._ppid = self._ppid or self._proc.ppid()
+            return self._ppid
+
+    def name(self):
+        """The process name. The return value is cached after first call."""
+        # Process name is only cached on Windows as on POSIX it may
+        # change, see:
+        # https://github.com/giampaolo/psutil/issues/692
+        if WINDOWS and self._name is not None:
+            return self._name
+        name = self._proc.name()
+        if POSIX and len(name) >= 15:
+            # On UNIX the name gets truncated to the first 15 characters.
+            # If it matches the first part of the cmdline we return that
+            # one instead because it's usually more explicative.
+            # Examples are "gnome-keyring-d" vs. "gnome-keyring-daemon".
+            try:
+                cmdline = self.cmdline()
+            except (AccessDenied, ZombieProcess):
+                # Just pass and return the truncated name: it's better
+                # than nothing. Note: there are actual cases where a
+                # zombie process can return a name() but not a
+                # cmdline(), see:
+                # https://github.com/giampaolo/psutil/issues/2239
+                pass
+            else:
+                if cmdline:
+                    extended_name = os.path.basename(cmdline[0])
+                    if extended_name.startswith(name):
+                        name = extended_name
+        self._name = name
+        self._proc._name = name
+        return name
+
+    def exe(self):
+        """The process executable as an absolute path.
+        May also be an empty string.
+        The return value is cached after first call.
+        """
+
+        def guess_it(fallback):
+            # try to guess exe from cmdline[0] in absence of a native
+            # exe representation
+            cmdline = self.cmdline()
+            if cmdline and hasattr(os, 'access') and hasattr(os, 'X_OK'):
+                exe = cmdline[0]  # the possible exe
+                # Attempt to guess only in case of an absolute path.
+                # It is not safe otherwise as the process might have
+                # changed cwd.
+                if (
+                    os.path.isabs(exe)
+                    and os.path.isfile(exe)
+                    and os.access(exe, os.X_OK)
+                ):
+                    return exe
+            if isinstance(fallback, AccessDenied):
+                raise fallback
+            return fallback
+
+        if self._exe is None:
+            try:
+                exe = self._proc.exe()
+            except AccessDenied as err:
+                return guess_it(fallback=err)
+            else:
+                if not exe:
+                    # underlying implementation can legitimately return an
+                    # empty string; if that's the case we don't want to
+                    # raise AD while guessing from the cmdline
+                    try:
+                        exe = guess_it(fallback=exe)
+                    except AccessDenied:
+                        pass
+                self._exe = exe
+        return self._exe
+
+    def cmdline(self):
+        """The command line this process has been called with."""
+        return self._proc.cmdline()
+
+    def status(self):
+        """The process current status as a STATUS_* constant."""
+        try:
+            return self._proc.status()
+        except ZombieProcess:
+            return STATUS_ZOMBIE
+
+    def username(self):
+        """The name of the user that owns the process.
+        On UNIX this is calculated by using *real* process uid.
+        """
+        if POSIX:
+            if pwd is None:
+                # might happen if python was installed from sources
+                msg = "requires pwd module shipped with standard python"
+                raise ImportError(msg)
+            real_uid = self.uids().real
+            try:
+                return pwd.getpwuid(real_uid).pw_name
+            except KeyError:
+                # the uid can't be resolved by the system
+                return str(real_uid)
+        else:
+            return self._proc.username()
+
+    def create_time(self):
+        """The process creation time as a floating point number
+        expressed in seconds since the epoch (seconds since January 1,
+        1970, at midnight UTC). The return value, which is cached after
+        first call, is based on the system clock, which means it may be
+        affected by changes such as manual adjustments or time
+        synchronization (e.g. NTP).
+        """
+        if self._create_time is None:
+            self._create_time = self._proc.create_time()
+        return self._create_time
+
+    def cwd(self):
+        """Process current working directory as an absolute path."""
+        return self._proc.cwd()
+
+    def nice(self, value=None):
+        """Get or set process niceness (priority)."""
+        if value is None:
+            return self._proc.nice_get()
+        else:
+            self._raise_if_pid_reused()
+            self._proc.nice_set(value)
+
+    if POSIX:
+
+        @memoize_when_activated
+        def uids(self):
+            """Return process UIDs as a (real, effective, saved)
+            namedtuple.
+            """
+            return self._proc.uids()
+
+        def gids(self):
+            """Return process GIDs as a (real, effective, saved)
+            namedtuple.
+            """
+            return self._proc.gids()
+
+        def terminal(self):
+            """The terminal associated with this process, if any,
+            else None.
+            """
+            return self._proc.terminal()
+
+        def num_fds(self):
+            """Return the number of file descriptors opened by this
+            process (POSIX only).
+            """
+            return self._proc.num_fds()
+
+    # Linux, BSD, AIX and Windows only
+    if hasattr(_psplatform.Process, "io_counters"):
+
+        def io_counters(self):
+            """Return process I/O statistics as a
+            (read_count, write_count, read_bytes, write_bytes)
+            namedtuple.
+            Those are the number of read/write calls performed and the
+            amount of bytes read and written by the process.
+            """
+            return self._proc.io_counters()
+
+    # Linux and Windows
+    if hasattr(_psplatform.Process, "ionice_get"):
+
+        def ionice(self, ioclass=None, value=None):
+            """Get or set process I/O niceness (priority).
+
+            On Linux *ioclass* is one of the IOPRIO_CLASS_* constants.
+            *value* is a number which goes from 0 to 7. The higher the
+            value, the lower the I/O priority of the process.
+
+            On Windows only *ioclass* is used and it can be set to 2
+            (normal), 1 (low) or 0 (very low).
+
+            Available on Linux and Windows > Vista only.
+            """
+            if ioclass is None:
+                if value is not None:
+                    msg = "'ioclass' argument must be specified"
+                    raise ValueError(msg)
+                return self._proc.ionice_get()
+            else:
+                self._raise_if_pid_reused()
+                return self._proc.ionice_set(ioclass, value)
+
+    # Linux / FreeBSD only
+    if hasattr(_psplatform.Process, "rlimit"):
+
+        def rlimit(self, resource, limits=None):
+            """Get or set process resource limits as a (soft, hard)
+            tuple.
+
+            *resource* is one of the RLIMIT_* constants.
+            *limits* is supposed to be a (soft, hard) tuple.
+
+            See "man prlimit" for further info.
+            Available on Linux and FreeBSD only.
+            """
+            if limits is not None:
+                self._raise_if_pid_reused()
+            return self._proc.rlimit(resource, limits)
+
+    # Windows, Linux and FreeBSD only
+    if hasattr(_psplatform.Process, "cpu_affinity_get"):
+
+        def cpu_affinity(self, cpus=None):
+            """Get or set process CPU affinity.
+            If specified, *cpus* must be a list of CPUs for which you
+            want to set the affinity (e.g. [0, 1]).
+            If an empty list is passed, all egible CPUs are assumed
+            (and set).
+            (Windows, Linux and BSD only).
+            """
+            if cpus is None:
+                return sorted(set(self._proc.cpu_affinity_get()))
+            else:
+                self._raise_if_pid_reused()
+                if not cpus:
+                    if hasattr(self._proc, "_get_eligible_cpus"):
+                        cpus = self._proc._get_eligible_cpus()
+                    else:
+                        cpus = tuple(range(len(cpu_times(percpu=True))))
+                self._proc.cpu_affinity_set(list(set(cpus)))
+
+    # Linux, FreeBSD, SunOS
+    if hasattr(_psplatform.Process, "cpu_num"):
+
+        def cpu_num(self):
+            """Return what CPU this process is currently running on.
+            The returned number should be <= psutil.cpu_count()
+            and <= len(psutil.cpu_percent(percpu=True)).
+            It may be used in conjunction with
+            psutil.cpu_percent(percpu=True) to observe the system
+            workload distributed across CPUs.
+            """
+            return self._proc.cpu_num()
+
+    # All platforms has it, but maybe not in the future.
+    if hasattr(_psplatform.Process, "environ"):
+
+        def environ(self):
+            """The environment variables of the process as a dict.  Note: this
+            might not reflect changes made after the process started.
+            """
+            return self._proc.environ()
+
+    if WINDOWS:
+
+        def num_handles(self):
+            """Return the number of handles opened by this process
+            (Windows only).
+            """
+            return self._proc.num_handles()
+
+    def num_ctx_switches(self):
+        """Return the number of voluntary and involuntary context
+        switches performed by this process.
+        """
+        return self._proc.num_ctx_switches()
+
+    def num_threads(self):
+        """Return the number of threads used by this process."""
+        return self._proc.num_threads()
+
+    if hasattr(_psplatform.Process, "threads"):
+
+        def threads(self):
+            """Return threads opened by process as a list of
+            (id, user_time, system_time) namedtuples representing
+            thread id and thread CPU times (user/system).
+            On OpenBSD this method requires root access.
+            """
+            return self._proc.threads()
+
+    def children(self, recursive=False):
+        """Return the children of this process as a list of Process
+        instances, pre-emptively checking whether PID has been reused.
+        If *recursive* is True return all the parent descendants.
+
+        Example (A == this process):
+
+         A ─┐
+            │
+            ├─ B (child) ─┐
+            │             └─ X (grandchild) ─┐
+            │                                └─ Y (great grandchild)
+            ├─ C (child)
+            └─ D (child)
+
+        >>> import psutil
+        >>> p = psutil.Process()
+        >>> p.children()
+        B, C, D
+        >>> p.children(recursive=True)
+        B, X, Y, C, D
+
+        Note that in the example above if process X disappears
+        process Y won't be listed as the reference to process A
+        is lost.
+        """
+        self._raise_if_pid_reused()
+        ppid_map = _ppid_map()
+        # Get a fresh (non-cached) ctime in case the system clock was
+        # updated. TODO: use a monotonic ctime on platforms where it's
+        # supported.
+        proc_ctime = Process(self.pid).create_time()
+        ret = []
+        if not recursive:
+            for pid, ppid in ppid_map.items():
+                if ppid == self.pid:
+                    try:
+                        child = Process(pid)
+                        # if child happens to be older than its parent
+                        # (self) it means child's PID has been reused
+                        if proc_ctime <= child.create_time():
+                            ret.append(child)
+                    except (NoSuchProcess, ZombieProcess):
+                        pass
+        else:
+            # Construct a {pid: [child pids]} dict
+            reverse_ppid_map = collections.defaultdict(list)
+            for pid, ppid in ppid_map.items():
+                reverse_ppid_map[ppid].append(pid)
+            # Recursively traverse that dict, starting from self.pid,
+            # such that we only call Process() on actual children
+            seen = set()
+            stack = [self.pid]
+            while stack:
+                pid = stack.pop()
+                if pid in seen:
+                    # Since pids can be reused while the ppid_map is
+                    # constructed, there may be rare instances where
+                    # there's a cycle in the recorded process "tree".
+                    continue
+                seen.add(pid)
+                for child_pid in reverse_ppid_map[pid]:
+                    try:
+                        child = Process(child_pid)
+                        # if child happens to be older than its parent
+                        # (self) it means child's PID has been reused
+                        intime = proc_ctime <= child.create_time()
+                        if intime:
+                            ret.append(child)
+                            stack.append(child_pid)
+                    except (NoSuchProcess, ZombieProcess):
+                        pass
+        return ret
+
+    def cpu_percent(self, interval=None):
+        """Return a float representing the current process CPU
+        utilization as a percentage.
+
+        When *interval* is 0.0 or None (default) compares process times
+        to system CPU times elapsed since last call, returning
+        immediately (non-blocking). That means that the first time
+        this is called it will return a meaningful 0.0 value.
+
+        When *interval* is > 0.0 compares process times to system CPU
+        times elapsed before and after the interval (blocking).
+
+        In this case is recommended for accuracy that this function
+        be called with at least 0.1 seconds between calls.
+
+        A value > 100.0 can be returned in case of processes running
+        multiple threads on different CPU cores.
+
+        The returned value is explicitly NOT split evenly between
+        all available logical CPUs. This means that a busy loop process
+        running on a system with 2 logical CPUs will be reported as
+        having 100% CPU utilization instead of 50%.
+
+        Examples:
+
+          >>> import psutil
+          >>> p = psutil.Process(os.getpid())
+          >>> # blocking
+          >>> p.cpu_percent(interval=1)
+          2.0
+          >>> # non-blocking (percentage since last call)
+          >>> p.cpu_percent(interval=None)
+          2.9
+          >>>
+        """
+        blocking = interval is not None and interval > 0.0
+        if interval is not None and interval < 0:
+            msg = f"interval is not positive (got {interval!r})"
+            raise ValueError(msg)
+        num_cpus = cpu_count() or 1
+
+        def timer():
+            return _timer() * num_cpus
+
+        if blocking:
+            st1 = timer()
+            pt1 = self._proc.cpu_times()
+            time.sleep(interval)
+            st2 = timer()
+            pt2 = self._proc.cpu_times()
+        else:
+            st1 = self._last_sys_cpu_times
+            pt1 = self._last_proc_cpu_times
+            st2 = timer()
+            pt2 = self._proc.cpu_times()
+            if st1 is None or pt1 is None:
+                self._last_sys_cpu_times = st2
+                self._last_proc_cpu_times = pt2
+                return 0.0
+
+        delta_proc = (pt2.user - pt1.user) + (pt2.system - pt1.system)
+        delta_time = st2 - st1
+        # reset values for next call in case of interval == None
+        self._last_sys_cpu_times = st2
+        self._last_proc_cpu_times = pt2
+
+        try:
+            # This is the utilization split evenly between all CPUs.
+            # E.g. a busy loop process on a 2-CPU-cores system at this
+            # point is reported as 50% instead of 100%.
+            overall_cpus_percent = (delta_proc / delta_time) * 100
+        except ZeroDivisionError:
+            # interval was too low
+            return 0.0
+        else:
+            # Note 1:
+            # in order to emulate "top" we multiply the value for the num
+            # of CPU cores. This way the busy process will be reported as
+            # having 100% (or more) usage.
+            #
+            # Note 2:
+            # taskmgr.exe on Windows differs in that it will show 50%
+            # instead.
+            #
+            # Note 3:
+            # a percentage > 100 is legitimate as it can result from a
+            # process with multiple threads running on different CPU
+            # cores (top does the same), see:
+            # http://stackoverflow.com/questions/1032357
+            # https://github.com/giampaolo/psutil/issues/474
+            single_cpu_percent = overall_cpus_percent * num_cpus
+            return round(single_cpu_percent, 1)
+
+    @memoize_when_activated
+    def cpu_times(self):
+        """Return a (user, system, children_user, children_system)
+        namedtuple representing the accumulated process time, in
+        seconds.
+        This is similar to os.times() but per-process.
+        On macOS and Windows children_user and children_system are
+        always set to 0.
+        """
+        return self._proc.cpu_times()
+
+    @memoize_when_activated
+    def memory_info(self):
+        """Return a namedtuple with variable fields depending on the
+        platform, representing memory information about the process.
+
+        The "portable" fields available on all platforms are `rss` and `vms`.
+
+        All numbers are expressed in bytes.
+        """
+        return self._proc.memory_info()
+
+    def memory_full_info(self):
+        """This method returns the same information as memory_info(),
+        plus, on some platform (Linux, macOS, Windows), also provides
+        additional metrics (USS, PSS and swap).
+        The additional metrics provide a better representation of actual
+        process memory usage.
+
+        Namely USS is the memory which is unique to a process and which
+        would be freed if the process was terminated right now.
+
+        It does so by passing through the whole process address.
+        As such it usually requires higher user privileges than
+        memory_info() and is considerably slower.
+        """
+        return self._proc.memory_full_info()
+
+    def memory_percent(self, memtype="rss"):
+        """Compare process memory to total physical system memory and
+        calculate process memory utilization as a percentage.
+        *memtype* argument is a string that dictates what type of
+        process memory you want to compare against (defaults to "rss").
+        The list of available strings can be obtained like this:
+
+        >>> psutil.Process().memory_info()._fields
+        ('rss', 'vms', 'shared', 'text', 'lib', 'data', 'dirty', 'uss', 'pss')
+        """
+        valid_types = list(_ntp.pfullmem._fields)
+        if memtype not in valid_types:
+            msg = (
+                f"invalid memtype {memtype!r}; valid types are"
+                f" {tuple(valid_types)!r}"
+            )
+            raise ValueError(msg)
+        fun = (
+            self.memory_info
+            if memtype in _ntp.pmem._fields
+            else self.memory_full_info
+        )
+        metrics = fun()
+        value = getattr(metrics, memtype)
+
+        # use cached value if available
+        total_phymem = _TOTAL_PHYMEM or virtual_memory().total
+        if not total_phymem > 0:
+            # we should never get here
+            msg = (
+                "can't calculate process memory percent because total physical"
+                f" system memory is not positive ({total_phymem!r})"
+            )
+            raise ValueError(msg)
+        return (value / float(total_phymem)) * 100
+
+    if hasattr(_psplatform.Process, "memory_maps"):
+
+        def memory_maps(self, grouped=True):
+            """Return process' mapped memory regions as a list of namedtuples
+            whose fields are variable depending on the platform.
+
+            If *grouped* is True the mapped regions with the same 'path'
+            are grouped together and the different memory fields are summed.
+
+            If *grouped* is False every mapped region is shown as a single
+            entity and the namedtuple will also include the mapped region's
+            address space ('addr') and permission set ('perms').
+            """
+            it = self._proc.memory_maps()
+            if grouped:
+                d = {}
+                for tupl in it:
+                    path = tupl[2]
+                    nums = tupl[3:]
+                    try:
+                        d[path] = list(map(lambda x, y: x + y, d[path], nums))
+                    except KeyError:
+                        d[path] = nums
+                return [_ntp.pmmap_grouped(path, *d[path]) for path in d]
+            else:
+                return [_ntp.pmmap_ext(*x) for x in it]
+
+    def open_files(self):
+        """Return files opened by process as a list of
+        (path, fd) namedtuples including the absolute file name
+        and file descriptor number.
+        """
+        return self._proc.open_files()
+
+    def net_connections(self, kind='inet'):
+        """Return socket connections opened by process as a list of
+        (fd, family, type, laddr, raddr, status) namedtuples.
+        The *kind* parameter filters for connections that match the
+        following criteria:
+
+        +------------+----------------------------------------------------+
+        | Kind Value | Connections using                                  |
+        +------------+----------------------------------------------------+
+        | inet       | IPv4 and IPv6                                      |
+        | inet4      | IPv4                                               |
+        | inet6      | IPv6                                               |
+        | tcp        | TCP                                                |
+        | tcp4       | TCP over IPv4                                      |
+        | tcp6       | TCP over IPv6                                      |
+        | udp        | UDP                                                |
+        | udp4       | UDP over IPv4                                      |
+        | udp6       | UDP over IPv6                                      |
+        | unix       | UNIX socket (both UDP and TCP protocols)           |
+        | all        | the sum of all the possible families and protocols |
+        +------------+----------------------------------------------------+
+        """
+        _check_conn_kind(kind)
+        return self._proc.net_connections(kind)
+
+    @_common.deprecated_method(replacement="net_connections")
+    def connections(self, kind="inet"):
+        return self.net_connections(kind=kind)
+
+    # --- signals
+
+    if POSIX:
+
+        def _send_signal(self, sig):
+            assert not self.pid < 0, self.pid
+            self._raise_if_pid_reused()
+
+            pid, ppid, name = self.pid, self._ppid, self._name
+            if pid == 0:
+                # see "man 2 kill"
+                msg = (
+                    "preventing sending signal to process with PID 0 as it "
+                    "would affect every process in the process group of the "
+                    "calling process (os.getpid()) instead of PID 0"
+                )
+                raise ValueError(msg)
+            try:
+                os.kill(pid, sig)
+            except ProcessLookupError as err:
+                if OPENBSD and pid_exists(pid):
+                    # We do this because os.kill() lies in case of
+                    # zombie processes.
+                    raise ZombieProcess(pid, name, ppid) from err
+                self._gone = True
+                raise NoSuchProcess(pid, name) from err
+            except PermissionError as err:
+                raise AccessDenied(pid, name) from err
+
+    def send_signal(self, sig):
+        """Send a signal *sig* to process pre-emptively checking
+        whether PID has been reused (see signal module constants) .
+        On Windows only SIGTERM is valid and is treated as an alias
+        for kill().
+        """
+        if POSIX:
+            self._send_signal(sig)
+        else:  # pragma: no cover
+            self._raise_if_pid_reused()
+            if sig != signal.SIGTERM and not self.is_running():
+                msg = "process no longer exists"
+                raise NoSuchProcess(self.pid, self._name, msg=msg)
+            self._proc.send_signal(sig)
+
+    def suspend(self):
+        """Suspend process execution with SIGSTOP pre-emptively checking
+        whether PID has been reused.
+        On Windows this has the effect of suspending all process threads.
+        """
+        if POSIX:
+            self._send_signal(signal.SIGSTOP)
+        else:  # pragma: no cover
+            self._raise_if_pid_reused()
+            self._proc.suspend()
+
+    def resume(self):
+        """Resume process execution with SIGCONT pre-emptively checking
+        whether PID has been reused.
+        On Windows this has the effect of resuming all process threads.
+        """
+        if POSIX:
+            self._send_signal(signal.SIGCONT)
+        else:  # pragma: no cover
+            self._raise_if_pid_reused()
+            self._proc.resume()
+
+    def terminate(self):
+        """Terminate the process with SIGTERM pre-emptively checking
+        whether PID has been reused.
+        On Windows this is an alias for kill().
+        """
+        if POSIX:
+            self._send_signal(signal.SIGTERM)
+        else:  # pragma: no cover
+            self._raise_if_pid_reused()
+            self._proc.kill()
+
+    def kill(self):
+        """Kill the current process with SIGKILL pre-emptively checking
+        whether PID has been reused.
+        """
+        if POSIX:
+            self._send_signal(signal.SIGKILL)
+        else:  # pragma: no cover
+            self._raise_if_pid_reused()
+            self._proc.kill()
+
+    def wait(self, timeout=None):
+        """Wait for process to terminate, and if process is a children
+        of os.getpid(), also return its exit code, else None.
+        On Windows there's no such limitation (exit code is always
+        returned).
+
+        If the process is already terminated, immediately return None
+        instead of raising NoSuchProcess.
+
+        If *timeout* (in seconds) is specified and process is still
+        alive, raise TimeoutExpired.
+
+        If *timeout=0* either return immediately or raise
+        TimeoutExpired (non-blocking).
+
+        To wait for multiple Process objects use psutil.wait_procs().
+        """
+        if self.pid == 0:
+            msg = "can't wait for PID 0"
+            raise ValueError(msg)
+        if timeout is not None:
+            if not isinstance(timeout, (int, float)):
+                msg = f"timeout must be an int or float (got {type(timeout)})"
+                raise TypeError(msg)
+            if timeout < 0:
+                msg = f"timeout must be positive or zero (got {timeout})"
+                raise ValueError(msg)
+
+        if self._exitcode is not _SENTINEL:
+            return self._exitcode
+
+        try:
+            self._exitcode = self._proc.wait(timeout)
+        except TimeoutExpired as err:
+            exc = TimeoutExpired(timeout, pid=self.pid, name=self._name)
+            raise exc from err
+
+        return self._exitcode
+
+
+# The valid attr names which can be processed by Process.as_dict().
+# fmt: off
+_as_dict_attrnames = {
+    x for x in dir(Process) if not x.startswith("_") and x not in
+     {'send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait',
+      'is_running', 'as_dict', 'parent', 'parents', 'children', 'rlimit',
+      'connections', 'oneshot'}
+}
+# fmt: on
+
+
+# =====================================================================
+# --- Popen class
+# =====================================================================
+
+
+class Popen(Process):
+    """Same as subprocess.Popen, but in addition it provides all
+    psutil.Process methods in a single class.
+    For the following methods which are common to both classes, psutil
+    implementation takes precedence:
+
+    * send_signal()
+    * terminate()
+    * kill()
+
+    This is done in order to avoid killing another process in case its
+    PID has been reused, fixing BPO-6973.
+
+      >>> import psutil
+      >>> from subprocess import PIPE
+      >>> p = psutil.Popen(["python", "-c", "print 'hi'"], stdout=PIPE)
+      >>> p.name()
+      'python'
+      >>> p.uids()
+      user(real=1000, effective=1000, saved=1000)
+      >>> p.username()
+      'giampaolo'
+      >>> p.communicate()
+      ('hi', None)
+      >>> p.terminate()
+      >>> p.wait(timeout=2)
+      0
+      >>>
+    """
+
+    def __init__(self, *args, **kwargs):
+        # Explicitly avoid to raise NoSuchProcess in case the process
+        # spawned by subprocess.Popen terminates too quickly, see:
+        # https://github.com/giampaolo/psutil/issues/193
+        self.__subproc = subprocess.Popen(*args, **kwargs)
+        self._init(self.__subproc.pid, _ignore_nsp=True)
+
+    def __dir__(self):
+        return sorted(set(dir(Popen) + dir(subprocess.Popen)))
+
+    def __enter__(self):
+        if hasattr(self.__subproc, '__enter__'):
+            self.__subproc.__enter__()
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        if hasattr(self.__subproc, '__exit__'):
+            return self.__subproc.__exit__(*args, **kwargs)
+        else:
+            if self.stdout:
+                self.stdout.close()
+            if self.stderr:
+                self.stderr.close()
+            try:
+                # Flushing a BufferedWriter may raise an error.
+                if self.stdin:
+                    self.stdin.close()
+            finally:
+                # Wait for the process to terminate, to avoid zombies.
+                self.wait()
+
+    def __getattribute__(self, name):
+        try:
+            return object.__getattribute__(self, name)
+        except AttributeError:
+            try:
+                return object.__getattribute__(self.__subproc, name)
+            except AttributeError:
+                msg = f"{self.__class__!r} has no attribute {name!r}"
+                raise AttributeError(msg) from None
+
+    def wait(self, timeout=None):
+        if self.__subproc.returncode is not None:
+            return self.__subproc.returncode
+        ret = super().wait(timeout)
+        self.__subproc.returncode = ret
+        return ret
+
+
+# =====================================================================
+# --- system processes related functions
+# =====================================================================
+
+
+def pids():
+    """Return a list of current running PIDs."""
+    global _LOWEST_PID
+    ret = sorted(_psplatform.pids())
+    _LOWEST_PID = ret[0]
+    return ret
+
+
+def pid_exists(pid):
+    """Return True if given PID exists in the current process list.
+    This is faster than doing "pid in psutil.pids()" and
+    should be preferred.
+    """
+    if pid < 0:
+        return False
+    elif pid == 0 and POSIX:
+        # On POSIX we use os.kill() to determine PID existence.
+        # According to "man 2 kill" PID 0 has a special meaning
+        # though: it refers to <<every process in the process
+        # group of the calling process>> and that is not we want
+        # to do here.
+        return pid in pids()
+    else:
+        return _psplatform.pid_exists(pid)
+
+
+_pmap = {}
+_pids_reused = set()
+
+
+def process_iter(attrs=None, ad_value=None):
+    """Return a generator yielding a Process instance for all
+    running processes.
+
+    Every new Process instance is only created once and then cached
+    into an internal table which is updated every time this is used.
+    Cache can optionally be cleared via `process_iter.cache_clear()`.
+
+    The sorting order in which processes are yielded is based on
+    their PIDs.
+
+    *attrs* and *ad_value* have the same meaning as in
+    Process.as_dict(). If *attrs* is specified as_dict() is called
+    and the resulting dict is stored as a 'info' attribute attached
+    to returned Process instance.
+    If *attrs* is an empty list it will retrieve all process info
+    (slow).
+    """
+    global _pmap
+
+    def add(pid):
+        proc = Process(pid)
+        pmap[proc.pid] = proc
+        return proc
+
+    def remove(pid):
+        pmap.pop(pid, None)
+
+    pmap = _pmap.copy()
+    a = set(pids())
+    b = set(pmap.keys())
+    new_pids = a - b
+    gone_pids = b - a
+    for pid in gone_pids:
+        remove(pid)
+    while _pids_reused:
+        pid = _pids_reused.pop()
+        debug(f"refreshing Process instance for reused PID {pid}")
+        remove(pid)
+    try:
+        ls = sorted(list(pmap.items()) + list(dict.fromkeys(new_pids).items()))
+        for pid, proc in ls:
+            try:
+                if proc is None:  # new process
+                    proc = add(pid)
+                if attrs is not None:
+                    proc.info = proc.as_dict(attrs=attrs, ad_value=ad_value)
+                yield proc
+            except NoSuchProcess:
+                remove(pid)
+    finally:
+        _pmap = pmap
+
+
+process_iter.cache_clear = lambda: _pmap.clear()  # noqa: PLW0108
+process_iter.cache_clear.__doc__ = "Clear process_iter() internal cache."
+
+
+def wait_procs(procs, timeout=None, callback=None):
+    """Convenience function which waits for a list of processes to
+    terminate.
+
+    Return a (gone, alive) tuple indicating which processes
+    are gone and which ones are still alive.
+
+    The gone ones will have a new *returncode* attribute indicating
+    process exit status (may be None).
+
+    *callback* is a function which gets called every time a process
+    terminates (a Process instance is passed as callback argument).
+
+    Function will return as soon as all processes terminate or when
+    *timeout* occurs.
+    Differently from Process.wait() it will not raise TimeoutExpired if
+    *timeout* occurs.
+
+    Typical use case is:
+
+     - send SIGTERM to a list of processes
+     - give them some time to terminate
+     - send SIGKILL to those ones which are still alive
+
+    Example:
+
+    >>> def on_terminate(proc):
+    ...     print("process {} terminated".format(proc))
+    ...
+    >>> for p in procs:
+    ...    p.terminate()
+    ...
+    >>> gone, alive = wait_procs(procs, timeout=3, callback=on_terminate)
+    >>> for p in alive:
+    ...     p.kill()
+    """
+
+    def check_gone(proc, timeout):
+        try:
+            returncode = proc.wait(timeout=timeout)
+        except (TimeoutExpired, subprocess.TimeoutExpired):
+            pass
+        else:
+            if returncode is not None or not proc.is_running():
+                # Set new Process instance attribute.
+                proc.returncode = returncode
+                gone.add(proc)
+                if callback is not None:
+                    callback(proc)
+
+    if timeout is not None and not timeout >= 0:
+        msg = f"timeout must be a positive integer, got {timeout}"
+        raise ValueError(msg)
+    if callback is not None and not callable(callback):
+        msg = f"callback {callback!r} is not a callable"
+        raise TypeError(msg)
+
+    gone = set()
+    alive = set(procs)
+    if timeout is not None:
+        deadline = _timer() + timeout
+
+    while alive:
+        if timeout is not None and timeout <= 0:
+            break
+        for proc in alive:
+            # Make sure that every complete iteration (all processes)
+            # will last max 1 sec.
+            # We do this because we don't want to wait too long on a
+            # single process: in case it terminates too late other
+            # processes may disappear in the meantime and their PID
+            # reused.
+            max_timeout = 1.0 / len(alive)
+            if timeout is not None:
+                timeout = min((deadline - _timer()), max_timeout)
+                if timeout <= 0:
+                    break
+                check_gone(proc, timeout)
+            else:
+                check_gone(proc, max_timeout)
+        alive = alive - gone  # noqa: PLR6104
+
+    if alive:
+        # Last attempt over processes survived so far.
+        # timeout == 0 won't make this function wait any further.
+        for proc in alive:
+            check_gone(proc, 0)
+        alive = alive - gone  # noqa: PLR6104
+
+    return (list(gone), list(alive))
+
+
+# =====================================================================
+# --- CPU related functions
+# =====================================================================
+
+
+def cpu_count(logical=True):
+    """Return the number of logical CPUs in the system (same as
+    os.cpu_count()).
+
+    If *logical* is False return the number of physical cores only
+    (e.g. hyper thread CPUs are excluded).
+
+    Return None if undetermined.
+
+    The return value is cached after first call.
+    If desired cache can be cleared like this:
+
+    >>> psutil.cpu_count.cache_clear()
+    """
+    if logical:
+        ret = _psplatform.cpu_count_logical()
+    else:
+        ret = _psplatform.cpu_count_cores()
+    if ret is not None and ret < 1:
+        ret = None
+    return ret
+
+
+def cpu_times(percpu=False):
+    """Return system-wide CPU times as a namedtuple.
+    Every CPU time represents the seconds the CPU has spent in the
+    given mode. The namedtuple's fields availability varies depending on the
+    platform:
+
+     - user
+     - system
+     - idle
+     - nice (UNIX)
+     - iowait (Linux)
+     - irq (Linux, FreeBSD)
+     - softirq (Linux)
+     - steal (Linux >= 2.6.11)
+     - guest (Linux >= 2.6.24)
+     - guest_nice (Linux >= 3.2.0)
+
+    When *percpu* is True return a list of namedtuples for each CPU.
+    First element of the list refers to first CPU, second element
+    to second CPU and so on.
+    The order of the list is consistent across calls.
+    """
+    if not percpu:
+        return _psplatform.cpu_times()
+    else:
+        return _psplatform.per_cpu_times()
+
+
+try:
+    _last_cpu_times = {threading.current_thread().ident: cpu_times()}
+except Exception:  # noqa: BLE001
+    # Don't want to crash at import time.
+    _last_cpu_times = {}
+
+try:
+    _last_per_cpu_times = {
+        threading.current_thread().ident: cpu_times(percpu=True)
+    }
+except Exception:  # noqa: BLE001
+    # Don't want to crash at import time.
+    _last_per_cpu_times = {}
+
+
+def _cpu_tot_time(times):
+    """Given a cpu_time() ntuple calculates the total CPU time
+    (including idle time).
+    """
+    tot = sum(times)
+    if LINUX:
+        # On Linux guest times are already accounted in "user" or
+        # "nice" times, so we subtract them from total.
+        # Htop does the same. References:
+        # https://github.com/giampaolo/psutil/pull/940
+        # http://unix.stackexchange.com/questions/178045
+        # https://github.com/torvalds/linux/blob/
+        #     447976ef4fd09b1be88b316d1a81553f1aa7cd07/kernel/sched/
+        #     cputime.c#L158
+        tot -= getattr(times, "guest", 0)  # Linux 2.6.24+
+        tot -= getattr(times, "guest_nice", 0)  # Linux 3.2.0+
+    return tot
+
+
+def _cpu_busy_time(times):
+    """Given a cpu_time() ntuple calculates the busy CPU time.
+    We do so by subtracting all idle CPU times.
+    """
+    busy = _cpu_tot_time(times)
+    busy -= times.idle
+    # Linux: "iowait" is time during which the CPU does not do anything
+    # (waits for IO to complete). On Linux IO wait is *not* accounted
+    # in "idle" time so we subtract it. Htop does the same.
+    # References:
+    # https://github.com/torvalds/linux/blob/
+    #     447976ef4fd09b1be88b316d1a81553f1aa7cd07/kernel/sched/cputime.c#L244
+    busy -= getattr(times, "iowait", 0)
+    return busy
+
+
+def _cpu_times_deltas(t1, t2):
+    assert t1._fields == t2._fields, (t1, t2)
+    field_deltas = []
+    for field in _ntp.scputimes._fields:
+        field_delta = getattr(t2, field) - getattr(t1, field)
+        # CPU times are always supposed to increase over time
+        # or at least remain the same and that's because time
+        # cannot go backwards.
+        # Surprisingly sometimes this might not be the case (at
+        # least on Windows and Linux), see:
+        # https://github.com/giampaolo/psutil/issues/392
+        # https://github.com/giampaolo/psutil/issues/645
+        # https://github.com/giampaolo/psutil/issues/1210
+        # Trim negative deltas to zero to ignore decreasing fields.
+        # top does the same. Reference:
+        # https://gitlab.com/procps-ng/procps/blob/v3.3.12/top/top.c#L5063
+        field_delta = max(0, field_delta)
+        field_deltas.append(field_delta)
+    return _ntp.scputimes(*field_deltas)
+
+
+def cpu_percent(interval=None, percpu=False):
+    """Return a float representing the current system-wide CPU
+    utilization as a percentage.
+
+    When *interval* is > 0.0 compares system CPU times elapsed before
+    and after the interval (blocking).
+
+    When *interval* is 0.0 or None compares system CPU times elapsed
+    since last call or module import, returning immediately (non
+    blocking). That means the first time this is called it will
+    return a meaningless 0.0 value which you should ignore.
+    In this case is recommended for accuracy that this function be
+    called with at least 0.1 seconds between calls.
+
+    When *percpu* is True returns a list of floats representing the
+    utilization as a percentage for each CPU.
+    First element of the list refers to first CPU, second element
+    to second CPU and so on.
+    The order of the list is consistent across calls.
+
+    Examples:
+
+      >>> # blocking, system-wide
+      >>> psutil.cpu_percent(interval=1)
+      2.0
+      >>>
+      >>> # blocking, per-cpu
+      >>> psutil.cpu_percent(interval=1, percpu=True)
+      [2.0, 1.0]
+      >>>
+      >>> # non-blocking (percentage since last call)
+      >>> psutil.cpu_percent(interval=None)
+      2.9
+      >>>
+    """
+    tid = threading.current_thread().ident
+    blocking = interval is not None and interval > 0.0
+    if interval is not None and interval < 0:
+        msg = f"interval is not positive (got {interval})"
+        raise ValueError(msg)
+
+    def calculate(t1, t2):
+        times_delta = _cpu_times_deltas(t1, t2)
+        all_delta = _cpu_tot_time(times_delta)
+        busy_delta = _cpu_busy_time(times_delta)
+
+        try:
+            busy_perc = (busy_delta / all_delta) * 100
+        except ZeroDivisionError:
+            return 0.0
+        else:
+            return round(busy_perc, 1)
+
+    # system-wide usage
+    if not percpu:
+        if blocking:
+            t1 = cpu_times()
+            time.sleep(interval)
+        else:
+            t1 = _last_cpu_times.get(tid) or cpu_times()
+        _last_cpu_times[tid] = cpu_times()
+        return calculate(t1, _last_cpu_times[tid])
+    # per-cpu usage
+    else:
+        ret = []
+        if blocking:
+            tot1 = cpu_times(percpu=True)
+            time.sleep(interval)
+        else:
+            tot1 = _last_per_cpu_times.get(tid) or cpu_times(percpu=True)
+        _last_per_cpu_times[tid] = cpu_times(percpu=True)
+        for t1, t2 in zip(tot1, _last_per_cpu_times[tid]):
+            ret.append(calculate(t1, t2))
+        return ret
+
+
+# Use a separate dict for cpu_times_percent(), so it's independent from
+# cpu_percent() and they can both be used within the same program.
+_last_cpu_times_2 = _last_cpu_times.copy()
+_last_per_cpu_times_2 = _last_per_cpu_times.copy()
+
+
+def cpu_times_percent(interval=None, percpu=False):
+    """Same as cpu_percent() but provides utilization percentages
+    for each specific CPU time as is returned by cpu_times().
+    For instance, on Linux we'll get:
+
+      >>> cpu_times_percent()
+      cpupercent(user=4.8, nice=0.0, system=4.8, idle=90.5, iowait=0.0,
+                 irq=0.0, softirq=0.0, steal=0.0, guest=0.0, guest_nice=0.0)
+      >>>
+
+    *interval* and *percpu* arguments have the same meaning as in
+    cpu_percent().
+    """
+    tid = threading.current_thread().ident
+    blocking = interval is not None and interval > 0.0
+    if interval is not None and interval < 0:
+        msg = f"interval is not positive (got {interval!r})"
+        raise ValueError(msg)
+
+    def calculate(t1, t2):
+        nums = []
+        times_delta = _cpu_times_deltas(t1, t2)
+        all_delta = _cpu_tot_time(times_delta)
+        # "scale" is the value to multiply each delta with to get percentages.
+        # We use "max" to avoid division by zero (if all_delta is 0, then all
+        # fields are 0 so percentages will be 0 too. all_delta cannot be a
+        # fraction because cpu times are integers)
+        scale = 100.0 / max(1, all_delta)
+        for field_delta in times_delta:
+            field_perc = field_delta * scale
+            field_perc = round(field_perc, 1)
+            # make sure we don't return negative values or values over 100%
+            field_perc = min(max(0.0, field_perc), 100.0)
+            nums.append(field_perc)
+        return _ntp.scputimes(*nums)
+
+    # system-wide usage
+    if not percpu:
+        if blocking:
+            t1 = cpu_times()
+            time.sleep(interval)
+        else:
+            t1 = _last_cpu_times_2.get(tid) or cpu_times()
+        _last_cpu_times_2[tid] = cpu_times()
+        return calculate(t1, _last_cpu_times_2[tid])
+    # per-cpu usage
+    else:
+        ret = []
+        if blocking:
+            tot1 = cpu_times(percpu=True)
+            time.sleep(interval)
+        else:
+            tot1 = _last_per_cpu_times_2.get(tid) or cpu_times(percpu=True)
+        _last_per_cpu_times_2[tid] = cpu_times(percpu=True)
+        for t1, t2 in zip(tot1, _last_per_cpu_times_2[tid]):
+            ret.append(calculate(t1, t2))
+        return ret
+
+
+def cpu_stats():
+    """Return CPU statistics."""
+    return _psplatform.cpu_stats()
+
+
+if hasattr(_psplatform, "cpu_freq"):
+
+    def cpu_freq(percpu=False):
+        """Return CPU frequency as a namedtuple including current,
+        min and max frequency expressed in Mhz.
+
+        If *percpu* is True and the system supports per-cpu frequency
+        retrieval (Linux only) a list of frequencies is returned for
+        each CPU. If not a list with one element is returned.
+        """
+        ret = _psplatform.cpu_freq()
+        if percpu:
+            return ret
+        else:
+            num_cpus = float(len(ret))
+            if num_cpus == 0:
+                return None
+            elif num_cpus == 1:
+                return ret[0]
+            else:
+                currs, mins, maxs = 0.0, 0.0, 0.0
+                set_none = False
+                for cpu in ret:
+                    currs += cpu.current
+                    # On Linux if /proc/cpuinfo is used min/max are set
+                    # to None.
+                    if LINUX and cpu.min is None:
+                        set_none = True
+                        continue
+                    mins += cpu.min
+                    maxs += cpu.max
+
+                current = currs / num_cpus
+
+                if set_none:
+                    min_ = max_ = None
+                else:
+                    min_ = mins / num_cpus
+                    max_ = maxs / num_cpus
+
+                return _ntp.scpufreq(current, min_, max_)
+
+    __all__.append("cpu_freq")
+
+
+if hasattr(os, "getloadavg") or hasattr(_psplatform, "getloadavg"):
+    # Perform this hasattr check once on import time to either use the
+    # platform based code or proxy straight from the os module.
+    if hasattr(os, "getloadavg"):
+        getloadavg = os.getloadavg
+    else:
+        getloadavg = _psplatform.getloadavg
+
+    __all__.append("getloadavg")
+
+
+# =====================================================================
+# --- system memory related functions
+# =====================================================================
+
+
+def virtual_memory():
+    """Return statistics about system memory usage as a namedtuple
+    including the following fields, expressed in bytes:
+
+     - total:
+       total physical memory available.
+
+     - available:
+       the memory that can be given instantly to processes without the
+       system going into swap.
+       This is calculated by summing different memory values depending
+       on the platform and it is supposed to be used to monitor actual
+       memory usage in a cross platform fashion.
+
+     - percent:
+       the percentage usage calculated as (total - available) / total * 100
+
+     - used:
+        memory used, calculated differently depending on the platform and
+        designed for informational purposes only:
+        macOS: active + wired
+        BSD: active + wired + cached
+        Linux: total - free
+
+     - free:
+       memory not being used at all (zeroed) that is readily available;
+       note that this doesn't reflect the actual memory available
+       (use 'available' instead)
+
+    Platform-specific fields:
+
+     - active (UNIX):
+       memory currently in use or very recently used, and so it is in RAM.
+
+     - inactive (UNIX):
+       memory that is marked as not used.
+
+     - buffers (BSD, Linux):
+       cache for things like file system metadata.
+
+     - cached (BSD, macOS):
+       cache for various things.
+
+     - wired (macOS, BSD):
+       memory that is marked to always stay in RAM. It is never moved to disk.
+
+     - shared (BSD):
+       memory that may be simultaneously accessed by multiple processes.
+
+    The sum of 'used' and 'available' does not necessarily equal total.
+    On Windows 'available' and 'free' are the same.
+    """
+    global _TOTAL_PHYMEM
+    ret = _psplatform.virtual_memory()
+    # cached for later use in Process.memory_percent()
+    _TOTAL_PHYMEM = ret.total
+    return ret
+
+
+def swap_memory():
+    """Return system swap memory statistics as a namedtuple including
+    the following fields:
+
+     - total:   total swap memory in bytes
+     - used:    used swap memory in bytes
+     - free:    free swap memory in bytes
+     - percent: the percentage usage
+     - sin:     no. of bytes the system has swapped in from disk (cumulative)
+     - sout:    no. of bytes the system has swapped out from disk (cumulative)
+
+    'sin' and 'sout' on Windows are meaningless and always set to 0.
+    """
+    return _psplatform.swap_memory()
+
+
+# =====================================================================
+# --- disks/partitions related functions
+# =====================================================================
+
+
+def disk_usage(path):
+    """Return disk usage statistics about the given *path* as a
+    namedtuple including total, used and free space expressed in bytes
+    plus the percentage usage.
+    """
+    return _psplatform.disk_usage(path)
+
+
+def disk_partitions(all=False):
+    """Return mounted partitions as a list of
+    (device, mountpoint, fstype, opts) namedtuple.
+    'opts' field is a raw string separated by commas indicating mount
+    options which may vary depending on the platform.
+
+    If *all* parameter is False return physical devices only and ignore
+    all others.
+    """
+    return _psplatform.disk_partitions(all)
+
+
+def disk_io_counters(perdisk=False, nowrap=True):
+    """Return system disk I/O statistics as a namedtuple including
+    the following fields:
+
+     - read_count:  number of reads
+     - write_count: number of writes
+     - read_bytes:  number of bytes read
+     - write_bytes: number of bytes written
+     - read_time:   time spent reading from disk (in ms)
+     - write_time:  time spent writing to disk (in ms)
+
+    Platform specific:
+
+     - busy_time: (Linux, FreeBSD) time spent doing actual I/Os (in ms)
+     - read_merged_count (Linux): number of merged reads
+     - write_merged_count (Linux): number of merged writes
+
+    If *perdisk* is True return the same information for every
+    physical disk installed on the system as a dictionary
+    with partition names as the keys and the namedtuple
+    described above as the values.
+
+    If *nowrap* is True it detects and adjust the numbers which overflow
+    and wrap (restart from 0) and add "old value" to "new value" so that
+    the returned numbers will always be increasing or remain the same,
+    but never decrease.
+    "disk_io_counters.cache_clear()" can be used to invalidate the
+    cache.
+
+    On recent Windows versions 'diskperf -y' command may need to be
+    executed first otherwise this function won't find any disk.
+    """
+    kwargs = dict(perdisk=perdisk) if LINUX else {}
+    rawdict = _psplatform.disk_io_counters(**kwargs)
+    if not rawdict:
+        return {} if perdisk else None
+    if nowrap:
+        rawdict = _wrap_numbers(rawdict, 'psutil.disk_io_counters')
+    if perdisk:
+        for disk, fields in rawdict.items():
+            rawdict[disk] = _ntp.sdiskio(*fields)
+        return rawdict
+    else:
+        return _ntp.sdiskio(*(sum(x) for x in zip(*rawdict.values())))
+
+
+disk_io_counters.cache_clear = functools.partial(
+    _wrap_numbers.cache_clear, 'psutil.disk_io_counters'
+)
+disk_io_counters.cache_clear.__doc__ = "Clears nowrap argument cache"
+
+
+# =====================================================================
+# --- network related functions
+# =====================================================================
+
+
+def net_io_counters(pernic=False, nowrap=True):
+    """Return network I/O statistics as a namedtuple including
+    the following fields:
+
+     - bytes_sent:   number of bytes sent
+     - bytes_recv:   number of bytes received
+     - packets_sent: number of packets sent
+     - packets_recv: number of packets received
+     - errin:        total number of errors while receiving
+     - errout:       total number of errors while sending
+     - dropin:       total number of incoming packets which were dropped
+     - dropout:      total number of outgoing packets which were dropped
+                     (always 0 on macOS and BSD)
+
+    If *pernic* is True return the same information for every
+    network interface installed on the system as a dictionary
+    with network interface names as the keys and the namedtuple
+    described above as the values.
+
+    If *nowrap* is True it detects and adjust the numbers which overflow
+    and wrap (restart from 0) and add "old value" to "new value" so that
+    the returned numbers will always be increasing or remain the same,
+    but never decrease.
+    "net_io_counters.cache_clear()" can be used to invalidate the
+    cache.
+    """
+    rawdict = _psplatform.net_io_counters()
+    if not rawdict:
+        return {} if pernic else None
+    if nowrap:
+        rawdict = _wrap_numbers(rawdict, 'psutil.net_io_counters')
+    if pernic:
+        for nic, fields in rawdict.items():
+            rawdict[nic] = _ntp.snetio(*fields)
+        return rawdict
+    else:
+        return _ntp.snetio(*[sum(x) for x in zip(*rawdict.values())])
+
+
+net_io_counters.cache_clear = functools.partial(
+    _wrap_numbers.cache_clear, 'psutil.net_io_counters'
+)
+net_io_counters.cache_clear.__doc__ = "Clears nowrap argument cache"
+
+
+def net_connections(kind='inet'):
+    """Return system-wide socket connections as a list of
+    (fd, family, type, laddr, raddr, status, pid) namedtuples.
+    In case of limited privileges 'fd' and 'pid' may be set to -1
+    and None respectively.
+    The *kind* parameter filters for connections that fit the
+    following criteria:
+
+    +------------+----------------------------------------------------+
+    | Kind Value | Connections using                                  |
+    +------------+----------------------------------------------------+
+    | inet       | IPv4 and IPv6                                      |
+    | inet4      | IPv4                                               |
+    | inet6      | IPv6                                               |
+    | tcp        | TCP                                                |
+    | tcp4       | TCP over IPv4                                      |
+    | tcp6       | TCP over IPv6                                      |
+    | udp        | UDP                                                |
+    | udp4       | UDP over IPv4                                      |
+    | udp6       | UDP over IPv6                                      |
+    | unix       | UNIX socket (both UDP and TCP protocols)           |
+    | all        | the sum of all the possible families and protocols |
+    +------------+----------------------------------------------------+
+
+    On macOS this function requires root privileges.
+    """
+    _check_conn_kind(kind)
+    return _psplatform.net_connections(kind)
+
+
+def net_if_addrs():
+    """Return the addresses associated to each NIC (network interface
+    card) installed on the system as a dictionary whose keys are the
+    NIC names and value is a list of namedtuples for each address
+    assigned to the NIC. Each namedtuple includes 5 fields:
+
+     - family: can be either socket.AF_INET, socket.AF_INET6 or
+               psutil.AF_LINK, which refers to a MAC address.
+     - address: is the primary address and it is always set.
+     - netmask: and 'broadcast' and 'ptp' may be None.
+     - ptp: stands for "point to point" and references the
+            destination address on a point to point interface
+            (typically a VPN).
+     - broadcast: and *ptp* are mutually exclusive.
+
+    Note: you can have more than one address of the same family
+    associated with each interface.
+    """
+    rawlist = _psplatform.net_if_addrs()
+    rawlist.sort(key=lambda x: x[1])  # sort by family
+    ret = collections.defaultdict(list)
+    for name, fam, addr, mask, broadcast, ptp in rawlist:
+        try:
+            fam = socket.AddressFamily(fam)
+        except ValueError:
+            if WINDOWS and fam == -1:
+                fam = _psplatform.AF_LINK
+            elif (
+                hasattr(_psplatform, "AF_LINK") and fam == _psplatform.AF_LINK
+            ):
+                # Linux defines AF_LINK as an alias for AF_PACKET.
+                # We re-set the family here so that repr(family)
+                # will show AF_LINK rather than AF_PACKET
+                fam = _psplatform.AF_LINK
+
+        if fam == _psplatform.AF_LINK:
+            # The underlying C function may return an incomplete MAC
+            # address in which case we fill it with null bytes, see:
+            # https://github.com/giampaolo/psutil/issues/786
+            separator = ":" if POSIX else "-"
+            while addr.count(separator) < 5:
+                addr += f"{separator}00"
+
+        nt = _ntp.snicaddr(fam, addr, mask, broadcast, ptp)
+
+        # On Windows broadcast is None, so we determine it via
+        # ipaddress module.
+        if WINDOWS and fam in {socket.AF_INET, socket.AF_INET6}:
+            try:
+                broadcast = _common.broadcast_addr(nt)
+            except Exception as err:  # noqa: BLE001
+                debug(err)
+            else:
+                if broadcast is not None:
+                    nt._replace(broadcast=broadcast)
+
+        ret[name].append(nt)
+
+    return dict(ret)
+
+
+def net_if_stats():
+    """Return information about each NIC (network interface card)
+    installed on the system as a dictionary whose keys are the
+    NIC names and value is a namedtuple with the following fields:
+
+     - isup: whether the interface is up (bool)
+     - duplex: can be either NIC_DUPLEX_FULL, NIC_DUPLEX_HALF or
+               NIC_DUPLEX_UNKNOWN
+     - speed: the NIC speed expressed in mega bits (MB); if it can't
+              be determined (e.g. 'localhost') it will be set to 0.
+     - mtu: the maximum transmission unit expressed in bytes.
+    """
+    return _psplatform.net_if_stats()
+
+
+# =====================================================================
+# --- sensors
+# =====================================================================
+
+
+# Linux, macOS
+if hasattr(_psplatform, "sensors_temperatures"):
+
+    def sensors_temperatures(fahrenheit=False):
+        """Return hardware temperatures. Each entry is a namedtuple
+        representing a certain hardware sensor (it may be a CPU, an
+        hard disk or something else, depending on the OS and its
+        configuration).
+        All temperatures are expressed in celsius unless *fahrenheit*
+        is set to True.
+        """
+
+        def convert(n):
+            if n is not None:
+                return (float(n) * 9 / 5) + 32 if fahrenheit else n
+
+        ret = collections.defaultdict(list)
+        rawdict = _psplatform.sensors_temperatures()
+
+        for name, values in rawdict.items():
+            while values:
+                label, current, high, critical = values.pop(0)
+                current = convert(current)
+                high = convert(high)
+                critical = convert(critical)
+
+                if high and not critical:
+                    critical = high
+                elif critical and not high:
+                    high = critical
+
+                ret[name].append(_ntp.shwtemp(label, current, high, critical))
+
+        return dict(ret)
+
+    __all__.append("sensors_temperatures")
+
+
+# Linux
+if hasattr(_psplatform, "sensors_fans"):
+
+    def sensors_fans():
+        """Return fans speed. Each entry is a namedtuple
+        representing a certain hardware sensor.
+        All speed are expressed in RPM (rounds per minute).
+        """
+        return _psplatform.sensors_fans()
+
+    __all__.append("sensors_fans")
+
+
+# Linux, Windows, FreeBSD, macOS
+if hasattr(_psplatform, "sensors_battery"):
+
+    def sensors_battery():
+        """Return battery information. If no battery is installed
+        returns None.
+
+         - percent: battery power left as a percentage.
+         - secsleft: a rough approximation of how many seconds are left
+                     before the battery runs out of power. May be
+                     POWER_TIME_UNLIMITED or POWER_TIME_UNLIMITED.
+         - power_plugged: True if the AC power cable is connected.
+        """
+        return _psplatform.sensors_battery()
+
+    __all__.append("sensors_battery")
+
+
+# =====================================================================
+# --- other system related functions
+# =====================================================================
+
+
+def boot_time():
+    """Return the system boot time expressed in seconds since the epoch
+    (seconds since January 1, 1970, at midnight UTC). The returned
+    value is based on the system clock, which means it may be affected
+    by changes such as manual adjustments or time synchronization (e.g.
+    NTP).
+    """
+    return _psplatform.boot_time()
+
+
+def users():
+    """Return users currently connected on the system as a list of
+    namedtuples including the following fields.
+
+     - user: the name of the user
+     - terminal: the tty or pseudo-tty associated with the user, if any.
+     - host: the host name associated with the entry, if any.
+     - started: the creation time as a floating point number expressed in
+       seconds since the epoch.
+    """
+    return _psplatform.users()
+
+
+# =====================================================================
+# --- Windows services
+# =====================================================================
+
+
+if WINDOWS:
+
+    def win_service_iter():
+        """Return a generator yielding a WindowsService instance for all
+        Windows services installed.
+        """
+        return _psplatform.win_service_iter()
+
+    def win_service_get(name):
+        """Get a Windows service by *name*.
+        Raise NoSuchProcess if no service with such name exists.
+        """
+        return _psplatform.win_service_get(name)
+
+
+# =====================================================================
+# --- malloc / heap
+# =====================================================================
+
+
+# Linux + glibc, Windows, macOS, FreeBSD, NetBSD
+if hasattr(_psplatform, "heap_info"):
+
+    def heap_info():
+        """Return low-level heap statistics from the C heap allocator
+        (glibc).
+
+        - `heap_used`: the total number of bytes allocated via
+          malloc/free. These are typically allocations smaller than
+          MMAP_THRESHOLD.
+
+        - `mmap_used`: the total number of bytes allocated via `mmap()`
+          or via large ``malloc()`` allocations.
+
+        - `heap_count` (Windows only): number of private heaps created
+          via `HeapCreate()`.
+        """
+        return _ntp.pheap(*_psplatform.heap_info())
+
+    def heap_trim():
+        """Request that the underlying allocator free any unused memory
+        it's holding in the heap (typically small `malloc()`
+        allocations).
+
+        In practice, modern allocators rarely comply, so this is not a
+        general-purpose memory-reduction tool and won't meaningfully
+        shrink RSS in real programs. Its primary value is in **leak
+        detection tools**.
+
+        Calling `heap_trim()` before taking measurements helps reduce
+        allocator noise, giving you a cleaner baseline so that changes
+        in `heap_used` come from the code you're testing, not from
+        internal allocator caching or fragmentation. Its effectiveness
+        depends on allocator behavior and fragmentation patterns.
+        """
+        _psplatform.heap_trim()
+
+    __all__.append("heap_info")
+    __all__.append("heap_trim")
+
+
+# =====================================================================
+
+
+def _set_debug(value):
+    """Enable or disable PSUTIL_DEBUG option, which prints debugging
+    messages to stderr.
+    """
+    import psutil._common
+
+    psutil._common.PSUTIL_DEBUG = bool(value)
+    _psplatform.cext.set_debug(bool(value))
+
+
+del memoize_when_activated

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/psutil/_common.py

@@ -0,0 +1,861 @@
+# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Common objects shared by __init__.py and _ps*.py modules.
+
+Note: this module is imported by setup.py, so it should not import
+psutil or third-party modules.
+"""
+
+import collections
+import enum
+import functools
+import os
+import socket
+import stat
+import sys
+import threading
+import warnings
+from socket import AF_INET
+from socket import SOCK_DGRAM
+from socket import SOCK_STREAM
+
+try:
+    from socket import AF_INET6
+except ImportError:
+    AF_INET6 = None
+try:
+    from socket import AF_UNIX
+except ImportError:
+    AF_UNIX = None
+
+
+PSUTIL_DEBUG = bool(os.getenv('PSUTIL_DEBUG'))
+_DEFAULT = object()
+
+# fmt: off
+__all__ = [
+    # OS constants
+    'FREEBSD', 'BSD', 'LINUX', 'NETBSD', 'OPENBSD', 'MACOS', 'OSX', 'POSIX',
+    'SUNOS', 'WINDOWS',
+    # connection constants
+    'CONN_CLOSE', 'CONN_CLOSE_WAIT', 'CONN_CLOSING', 'CONN_ESTABLISHED',
+    'CONN_FIN_WAIT1', 'CONN_FIN_WAIT2', 'CONN_LAST_ACK', 'CONN_LISTEN',
+    'CONN_NONE', 'CONN_SYN_RECV', 'CONN_SYN_SENT', 'CONN_TIME_WAIT',
+    # net constants
+    'NIC_DUPLEX_FULL', 'NIC_DUPLEX_HALF', 'NIC_DUPLEX_UNKNOWN',  # noqa: F822
+    # process status constants
+    'STATUS_DEAD', 'STATUS_DISK_SLEEP', 'STATUS_IDLE', 'STATUS_LOCKED',
+    'STATUS_RUNNING', 'STATUS_SLEEPING', 'STATUS_STOPPED', 'STATUS_SUSPENDED',
+    'STATUS_TRACING_STOP', 'STATUS_WAITING', 'STATUS_WAKE_KILL',
+    'STATUS_WAKING', 'STATUS_ZOMBIE', 'STATUS_PARKED',
+    # other constants
+    'ENCODING', 'ENCODING_ERRS', 'AF_INET6',
+    # utility functions
+    'conn_tmap', 'deprecated_method', 'isfile_strict', 'memoize',
+    'parse_environ_block', 'path_exists_strict', 'usage_percent',
+    'supports_ipv6', 'sockfam_to_enum', 'socktype_to_enum', "wrap_numbers",
+    'open_text', 'open_binary', 'cat', 'bcat',
+    'bytes2human', 'conn_to_ntuple', 'debug',
+    # shell utils
+    'hilite', 'term_supports_colors', 'print_color',
+]
+# fmt: on
+
+
+# ===================================================================
+# --- OS constants
+# ===================================================================
+
+
+POSIX = os.name == "posix"
+WINDOWS = os.name == "nt"
+LINUX = sys.platform.startswith("linux")
+MACOS = sys.platform.startswith("darwin")
+OSX = MACOS  # deprecated alias
+FREEBSD = sys.platform.startswith(("freebsd", "midnightbsd"))
+OPENBSD = sys.platform.startswith("openbsd")
+NETBSD = sys.platform.startswith("netbsd")
+BSD = FREEBSD or OPENBSD or NETBSD
+SUNOS = sys.platform.startswith(("sunos", "solaris"))
+AIX = sys.platform.startswith("aix")
+
+
+# ===================================================================
+# --- API constants
+# ===================================================================
+
+
+# Process.status()
+STATUS_RUNNING = "running"
+STATUS_SLEEPING = "sleeping"
+STATUS_DISK_SLEEP = "disk-sleep"
+STATUS_STOPPED = "stopped"
+STATUS_TRACING_STOP = "tracing-stop"
+STATUS_ZOMBIE = "zombie"
+STATUS_DEAD = "dead"
+STATUS_WAKE_KILL = "wake-kill"
+STATUS_WAKING = "waking"
+STATUS_IDLE = "idle"  # Linux, macOS, FreeBSD
+STATUS_LOCKED = "locked"  # FreeBSD
+STATUS_WAITING = "waiting"  # FreeBSD
+STATUS_SUSPENDED = "suspended"  # NetBSD
+STATUS_PARKED = "parked"  # Linux
+
+# Process.net_connections() and psutil.net_connections()
+CONN_ESTABLISHED = "ESTABLISHED"
+CONN_SYN_SENT = "SYN_SENT"
+CONN_SYN_RECV = "SYN_RECV"
+CONN_FIN_WAIT1 = "FIN_WAIT1"
+CONN_FIN_WAIT2 = "FIN_WAIT2"
+CONN_TIME_WAIT = "TIME_WAIT"
+CONN_CLOSE = "CLOSE"
+CONN_CLOSE_WAIT = "CLOSE_WAIT"
+CONN_LAST_ACK = "LAST_ACK"
+CONN_LISTEN = "LISTEN"
+CONN_CLOSING = "CLOSING"
+CONN_NONE = "NONE"
+
+
+# net_if_stats()
+class NicDuplex(enum.IntEnum):
+    NIC_DUPLEX_FULL = 2
+    NIC_DUPLEX_HALF = 1
+    NIC_DUPLEX_UNKNOWN = 0
+
+
+globals().update(NicDuplex.__members__)
+
+
+# sensors_battery()
+class BatteryTime(enum.IntEnum):
+    POWER_TIME_UNKNOWN = -1
+    POWER_TIME_UNLIMITED = -2
+
+
+globals().update(BatteryTime.__members__)
+
+# --- others
+
+ENCODING = sys.getfilesystemencoding()
+ENCODING_ERRS = sys.getfilesystemencodeerrors()
+
+
+# ===================================================================
+# --- Process.net_connections() 'kind' parameter mapping
+# ===================================================================
+
+
+conn_tmap = {
+    "all": ([AF_INET, AF_INET6, AF_UNIX], [SOCK_STREAM, SOCK_DGRAM]),
+    "tcp": ([AF_INET, AF_INET6], [SOCK_STREAM]),
+    "tcp4": ([AF_INET], [SOCK_STREAM]),
+    "udp": ([AF_INET, AF_INET6], [SOCK_DGRAM]),
+    "udp4": ([AF_INET], [SOCK_DGRAM]),
+    "inet": ([AF_INET, AF_INET6], [SOCK_STREAM, SOCK_DGRAM]),
+    "inet4": ([AF_INET], [SOCK_STREAM, SOCK_DGRAM]),
+    "inet6": ([AF_INET6], [SOCK_STREAM, SOCK_DGRAM]),
+}
+
+if AF_INET6 is not None:
+    conn_tmap.update({
+        "tcp6": ([AF_INET6], [SOCK_STREAM]),
+        "udp6": ([AF_INET6], [SOCK_DGRAM]),
+    })
+
+if AF_UNIX is not None and not SUNOS:
+    conn_tmap.update({"unix": ([AF_UNIX], [SOCK_STREAM, SOCK_DGRAM])})
+
+
+# =====================================================================
+# --- Exceptions
+# =====================================================================
+
+
+class Error(Exception):
+    """Base exception class. All other psutil exceptions inherit
+    from this one.
+    """
+
+    __module__ = 'psutil'
+
+    def _infodict(self, attrs):
+        info = collections.OrderedDict()
+        for name in attrs:
+            value = getattr(self, name, None)
+            if value or (name == "pid" and value == 0):
+                info[name] = value
+        return info
+
+    def __str__(self):
+        # invoked on `raise Error`
+        info = self._infodict(("pid", "ppid", "name"))
+        if info:
+            details = "({})".format(
+                ", ".join([f"{k}={v!r}" for k, v in info.items()])
+            )
+        else:
+            details = None
+        return " ".join([x for x in (getattr(self, "msg", ""), details) if x])
+
+    def __repr__(self):
+        # invoked on `repr(Error)`
+        info = self._infodict(("pid", "ppid", "name", "seconds", "msg"))
+        details = ", ".join([f"{k}={v!r}" for k, v in info.items()])
+        return f"psutil.{self.__class__.__name__}({details})"
+
+
+class NoSuchProcess(Error):
+    """Exception raised when a process with a certain PID doesn't
+    or no longer exists.
+    """
+
+    __module__ = 'psutil'
+
+    def __init__(self, pid, name=None, msg=None):
+        Error.__init__(self)
+        self.pid = pid
+        self.name = name
+        self.msg = msg or "process no longer exists"
+
+    def __reduce__(self):
+        return (self.__class__, (self.pid, self.name, self.msg))
+
+
+class ZombieProcess(NoSuchProcess):
+    """Exception raised when querying a zombie process. This is
+    raised on macOS, BSD and Solaris only, and not always: depending
+    on the query the OS may be able to succeed anyway.
+    On Linux all zombie processes are querable (hence this is never
+    raised). Windows doesn't have zombie processes.
+    """
+
+    __module__ = 'psutil'
+
+    def __init__(self, pid, name=None, ppid=None, msg=None):
+        NoSuchProcess.__init__(self, pid, name, msg)
+        self.ppid = ppid
+        self.msg = msg or "PID still exists but it's a zombie"
+
+    def __reduce__(self):
+        return (self.__class__, (self.pid, self.name, self.ppid, self.msg))
+
+
+class AccessDenied(Error):
+    """Exception raised when permission to perform an action is denied."""
+
+    __module__ = 'psutil'
+
+    def __init__(self, pid=None, name=None, msg=None):
+        Error.__init__(self)
+        self.pid = pid
+        self.name = name
+        self.msg = msg or ""
+
+    def __reduce__(self):
+        return (self.__class__, (self.pid, self.name, self.msg))
+
+
+class TimeoutExpired(Error):
+    """Raised on Process.wait(timeout) if timeout expires and process
+    is still alive.
+    """
+
+    __module__ = 'psutil'
+
+    def __init__(self, seconds, pid=None, name=None):
+        Error.__init__(self)
+        self.seconds = seconds
+        self.pid = pid
+        self.name = name
+        self.msg = f"timeout after {seconds} seconds"
+
+    def __reduce__(self):
+        return (self.__class__, (self.seconds, self.pid, self.name))
+
+
+# ===================================================================
+# --- utils
+# ===================================================================
+
+
+def usage_percent(used, total, round_=None):
+    """Calculate percentage usage of 'used' against 'total'."""
+    try:
+        ret = (float(used) / total) * 100
+    except ZeroDivisionError:
+        return 0.0
+    else:
+        if round_ is not None:
+            ret = round(ret, round_)
+        return ret
+
+
+def memoize(fun):
+    """A simple memoize decorator for functions supporting (hashable)
+    positional arguments.
+    It also provides a cache_clear() function for clearing the cache:
+
+    >>> @memoize
+    ... def foo()
+    ...     return 1
+        ...
+    >>> foo()
+    1
+    >>> foo.cache_clear()
+    >>>
+
+    It supports:
+     - functions
+     - classes (acts as a @singleton)
+     - staticmethods
+     - classmethods
+
+    It does NOT support:
+     - methods
+    """
+
+    @functools.wraps(fun)
+    def wrapper(*args, **kwargs):
+        key = (args, frozenset(sorted(kwargs.items())))
+        try:
+            return cache[key]
+        except KeyError:
+            try:
+                ret = cache[key] = fun(*args, **kwargs)
+            except Exception as err:
+                raise err from None
+            return ret
+
+    def cache_clear():
+        """Clear cache."""
+        cache.clear()
+
+    cache = {}
+    wrapper.cache_clear = cache_clear
+    return wrapper
+
+
+def memoize_when_activated(fun):
+    """A memoize decorator which is disabled by default. It can be
+    activated and deactivated on request.
+    For efficiency reasons it can be used only against class methods
+    accepting no arguments.
+
+    >>> class Foo:
+    ...     @memoize
+    ...     def foo()
+    ...         print(1)
+    ...
+    >>> f = Foo()
+    >>> # deactivated (default)
+    >>> foo()
+    1
+    >>> foo()
+    1
+    >>>
+    >>> # activated
+    >>> foo.cache_activate(self)
+    >>> foo()
+    1
+    >>> foo()
+    >>> foo()
+    >>>
+    """
+
+    @functools.wraps(fun)
+    def wrapper(self):
+        try:
+            # case 1: we previously entered oneshot() ctx
+            ret = self._cache[fun]
+        except AttributeError:
+            # case 2: we never entered oneshot() ctx
+            try:
+                return fun(self)
+            except Exception as err:
+                raise err from None
+        except KeyError:
+            # case 3: we entered oneshot() ctx but there's no cache
+            # for this entry yet
+            try:
+                ret = fun(self)
+            except Exception as err:
+                raise err from None
+            try:
+                self._cache[fun] = ret
+            except AttributeError:
+                # multi-threading race condition, see:
+                # https://github.com/giampaolo/psutil/issues/1948
+                pass
+        return ret
+
+    def cache_activate(proc):
+        """Activate cache. Expects a Process instance. Cache will be
+        stored as a "_cache" instance attribute.
+        """
+        proc._cache = {}
+
+    def cache_deactivate(proc):
+        """Deactivate and clear cache."""
+        try:
+            del proc._cache
+        except AttributeError:
+            pass
+
+    wrapper.cache_activate = cache_activate
+    wrapper.cache_deactivate = cache_deactivate
+    return wrapper
+
+
+def isfile_strict(path):
+    """Same as os.path.isfile() but does not swallow EACCES / EPERM
+    exceptions, see:
+    http://mail.python.org/pipermail/python-dev/2012-June/120787.html.
+    """
+    try:
+        st = os.stat(path)
+    except PermissionError:
+        raise
+    except OSError:
+        return False
+    else:
+        return stat.S_ISREG(st.st_mode)
+
+
+def path_exists_strict(path):
+    """Same as os.path.exists() but does not swallow EACCES / EPERM
+    exceptions. See:
+    http://mail.python.org/pipermail/python-dev/2012-June/120787.html.
+    """
+    try:
+        os.stat(path)
+    except PermissionError:
+        raise
+    except OSError:
+        return False
+    else:
+        return True
+
+
+def supports_ipv6():
+    """Return True if IPv6 is supported on this platform."""
+    if not socket.has_ipv6 or AF_INET6 is None:
+        return False
+    try:
+        with socket.socket(AF_INET6, socket.SOCK_STREAM) as sock:
+            sock.bind(("::1", 0))
+        return True
+    except OSError:
+        return False
+
+
+def parse_environ_block(data):
+    """Parse a C environ block of environment variables into a dictionary."""
+    # The block is usually raw data from the target process.  It might contain
+    # trailing garbage and lines that do not look like assignments.
+    ret = {}
+    pos = 0
+
+    # localize global variable to speed up access.
+    WINDOWS_ = WINDOWS
+    while True:
+        next_pos = data.find("\0", pos)
+        # nul byte at the beginning or double nul byte means finish
+        if next_pos <= pos:
+            break
+        # there might not be an equals sign
+        equal_pos = data.find("=", pos, next_pos)
+        if equal_pos > pos:
+            key = data[pos:equal_pos]
+            value = data[equal_pos + 1 : next_pos]
+            # Windows expects environment variables to be uppercase only
+            if WINDOWS_:
+                key = key.upper()
+            ret[key] = value
+        pos = next_pos + 1
+
+    return ret
+
+
+def sockfam_to_enum(num):
+    """Convert a numeric socket family value to an IntEnum member.
+    If it's not a known member, return the numeric value itself.
+    """
+    try:
+        return socket.AddressFamily(num)
+    except ValueError:
+        return num
+
+
+def socktype_to_enum(num):
+    """Convert a numeric socket type value to an IntEnum member.
+    If it's not a known member, return the numeric value itself.
+    """
+    try:
+        return socket.SocketKind(num)
+    except ValueError:
+        return num
+
+
+def conn_to_ntuple(fd, fam, type_, laddr, raddr, status, status_map, pid=None):
+    """Convert a raw connection tuple to a proper ntuple."""
+    from . import _ntuples as ntp
+
+    if fam in {socket.AF_INET, AF_INET6}:
+        if laddr:
+            laddr = ntp.addr(*laddr)
+        if raddr:
+            raddr = ntp.addr(*raddr)
+    if type_ == socket.SOCK_STREAM and fam in {AF_INET, AF_INET6}:
+        status = status_map.get(status, CONN_NONE)
+    else:
+        status = CONN_NONE  # ignore whatever C returned to us
+    fam = sockfam_to_enum(fam)
+    type_ = socktype_to_enum(type_)
+    if pid is None:
+        return ntp.pconn(fd, fam, type_, laddr, raddr, status)
+    else:
+        return ntp.sconn(fd, fam, type_, laddr, raddr, status, pid)
+
+
+def broadcast_addr(addr):
+    """Given the address ntuple returned by ``net_if_addrs()``
+    calculates the broadcast address.
+    """
+    import ipaddress
+
+    if not addr.address or not addr.netmask:
+        return None
+    if addr.family == socket.AF_INET:
+        return str(
+            ipaddress.IPv4Network(
+                f"{addr.address}/{addr.netmask}", strict=False
+            ).broadcast_address
+        )
+    if addr.family == socket.AF_INET6:
+        return str(
+            ipaddress.IPv6Network(
+                f"{addr.address}/{addr.netmask}", strict=False
+            ).broadcast_address
+        )
+
+
+def deprecated_method(replacement):
+    """A decorator which can be used to mark a method as deprecated
+    'replcement' is the method name which will be called instead.
+    """
+
+    def outer(fun):
+        msg = (
+            f"{fun.__name__}() is deprecated and will be removed; use"
+            f" {replacement}() instead"
+        )
+        if fun.__doc__ is None:
+            fun.__doc__ = msg
+
+        @functools.wraps(fun)
+        def inner(self, *args, **kwargs):
+            warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+            return getattr(self, replacement)(*args, **kwargs)
+
+        return inner
+
+    return outer
+
+
+class _WrapNumbers:
+    """Watches numbers so that they don't overflow and wrap
+    (reset to zero).
+    """
+
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.cache = {}
+        self.reminders = {}
+        self.reminder_keys = {}
+
+    def _add_dict(self, input_dict, name):
+        assert name not in self.cache
+        assert name not in self.reminders
+        assert name not in self.reminder_keys
+        self.cache[name] = input_dict
+        self.reminders[name] = collections.defaultdict(int)
+        self.reminder_keys[name] = collections.defaultdict(set)
+
+    def _remove_dead_reminders(self, input_dict, name):
+        """In case the number of keys changed between calls (e.g. a
+        disk disappears) this removes the entry from self.reminders.
+        """
+        old_dict = self.cache[name]
+        gone_keys = set(old_dict.keys()) - set(input_dict.keys())
+        for gone_key in gone_keys:
+            for remkey in self.reminder_keys[name][gone_key]:
+                del self.reminders[name][remkey]
+            del self.reminder_keys[name][gone_key]
+
+    def run(self, input_dict, name):
+        """Cache dict and sum numbers which overflow and wrap.
+        Return an updated copy of `input_dict`.
+        """
+        if name not in self.cache:
+            # This was the first call.
+            self._add_dict(input_dict, name)
+            return input_dict
+
+        self._remove_dead_reminders(input_dict, name)
+
+        old_dict = self.cache[name]
+        new_dict = {}
+        for key in input_dict:
+            input_tuple = input_dict[key]
+            try:
+                old_tuple = old_dict[key]
+            except KeyError:
+                # The input dict has a new key (e.g. a new disk or NIC)
+                # which didn't exist in the previous call.
+                new_dict[key] = input_tuple
+                continue
+
+            bits = []
+            for i in range(len(input_tuple)):
+                input_value = input_tuple[i]
+                old_value = old_tuple[i]
+                remkey = (key, i)
+                if input_value < old_value:
+                    # it wrapped!
+                    self.reminders[name][remkey] += old_value
+                    self.reminder_keys[name][key].add(remkey)
+                bits.append(input_value + self.reminders[name][remkey])
+
+            new_dict[key] = tuple(bits)
+
+        self.cache[name] = input_dict
+        return new_dict
+
+    def cache_clear(self, name=None):
+        """Clear the internal cache, optionally only for function 'name'."""
+        with self.lock:
+            if name is None:
+                self.cache.clear()
+                self.reminders.clear()
+                self.reminder_keys.clear()
+            else:
+                self.cache.pop(name, None)
+                self.reminders.pop(name, None)
+                self.reminder_keys.pop(name, None)
+
+    def cache_info(self):
+        """Return internal cache dicts as a tuple of 3 elements."""
+        with self.lock:
+            return (self.cache, self.reminders, self.reminder_keys)
+
+
+def wrap_numbers(input_dict, name):
+    """Given an `input_dict` and a function `name`, adjust the numbers
+    which "wrap" (restart from zero) across different calls by adding
+    "old value" to "new value" and return an updated dict.
+    """
+    with _wn.lock:
+        return _wn.run(input_dict, name)
+
+
+_wn = _WrapNumbers()
+wrap_numbers.cache_clear = _wn.cache_clear
+wrap_numbers.cache_info = _wn.cache_info
+
+
+# The read buffer size for open() builtin. This (also) dictates how
+# much data we read(2) when iterating over file lines as in:
+#   >>> with open(file) as f:
+#   ...    for line in f:
+#   ...        ...
+# Default per-line buffer size for binary files is 1K. For text files
+# is 8K. We use a bigger buffer (32K) in order to have more consistent
+# results when reading /proc pseudo files on Linux, see:
+# https://github.com/giampaolo/psutil/issues/2050
+# https://github.com/giampaolo/psutil/issues/708
+FILE_READ_BUFFER_SIZE = 32 * 1024
+
+
+def open_binary(fname):
+    return open(fname, "rb", buffering=FILE_READ_BUFFER_SIZE)
+
+
+def open_text(fname):
+    """Open a file in text mode by using the proper FS encoding and
+    en/decoding error handlers.
+    """
+    # See:
+    # https://github.com/giampaolo/psutil/issues/675
+    # https://github.com/giampaolo/psutil/pull/733
+    fobj = open(  # noqa: SIM115
+        fname,
+        buffering=FILE_READ_BUFFER_SIZE,
+        encoding=ENCODING,
+        errors=ENCODING_ERRS,
+    )
+    try:
+        # Dictates per-line read(2) buffer size. Defaults is 8k. See:
+        # https://github.com/giampaolo/psutil/issues/2050#issuecomment-1013387546
+        fobj._CHUNK_SIZE = FILE_READ_BUFFER_SIZE
+    except AttributeError:
+        pass
+    except Exception:
+        fobj.close()
+        raise
+
+    return fobj
+
+
+def cat(fname, fallback=_DEFAULT, _open=open_text):
+    """Read entire file content and return it as a string. File is
+    opened in text mode. If specified, `fallback` is the value
+    returned in case of error, either if the file does not exist or
+    it can't be read().
+    """
+    if fallback is _DEFAULT:
+        with _open(fname) as f:
+            return f.read()
+    else:
+        try:
+            with _open(fname) as f:
+                return f.read()
+        except OSError:
+            return fallback
+
+
+def bcat(fname, fallback=_DEFAULT):
+    """Same as above but opens file in binary mode."""
+    return cat(fname, fallback=fallback, _open=open_binary)
+
+
+def bytes2human(n, format="%(value).1f%(symbol)s"):
+    """Used by various scripts. See: https://code.activestate.com/recipes/578019-bytes-to-human-human-to-bytes-converter/?in=user-4178764.
+
+    >>> bytes2human(10000)
+    '9.8K'
+    >>> bytes2human(100001221)
+    '95.4M'
+    """
+    symbols = ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
+    prefix = {}
+    for i, s in enumerate(symbols[1:]):
+        prefix[s] = 1 << (i + 1) * 10
+    for symbol in reversed(symbols[1:]):
+        if abs(n) >= prefix[symbol]:
+            value = float(n) / prefix[symbol]
+            return format % locals()
+    return format % dict(symbol=symbols[0], value=n)
+
+
+def get_procfs_path():
+    """Return updated psutil.PROCFS_PATH constant."""
+    return sys.modules['psutil'].PROCFS_PATH
+
+
+def decode(s):
+    return s.decode(encoding=ENCODING, errors=ENCODING_ERRS)
+
+
+# =====================================================================
+# --- shell utils
+# =====================================================================
+
+
+@memoize
+def term_supports_colors(file=sys.stdout):  # pragma: no cover
+    if not hasattr(file, "isatty") or not file.isatty():
+        return False
+    try:
+        file.fileno()
+    except Exception:  # noqa: BLE001
+        return False
+    return True
+
+
+def hilite(s, color=None, bold=False):  # pragma: no cover
+    """Return an highlighted version of 'string'."""
+    if not term_supports_colors():
+        return s
+    attr = []
+    colors = dict(
+        blue='34',
+        brown='33',
+        darkgrey='30',
+        green='32',
+        grey='37',
+        lightblue='36',
+        red='91',
+        violet='35',
+        yellow='93',
+    )
+    colors[None] = '29'
+    try:
+        color = colors[color]
+    except KeyError:
+        msg = f"invalid color {color!r}; choose amongst {list(colors.keys())}"
+        raise ValueError(msg) from None
+    attr.append(color)
+    if bold:
+        attr.append('1')
+    return f"\x1b[{';'.join(attr)}m{s}\x1b[0m"
+
+
+def print_color(
+    s, color=None, bold=False, file=sys.stdout
+):  # pragma: no cover
+    """Print a colorized version of string."""
+    if not term_supports_colors():
+        print(s, file=file)
+    elif POSIX:
+        print(hilite(s, color, bold), file=file)
+    else:
+        import ctypes
+
+        DEFAULT_COLOR = 7
+        GetStdHandle = ctypes.windll.Kernel32.GetStdHandle
+        SetConsoleTextAttribute = (
+            ctypes.windll.Kernel32.SetConsoleTextAttribute
+        )
+
+        colors = dict(green=2, red=4, brown=6, yellow=6)
+        colors[None] = DEFAULT_COLOR
+        try:
+            color = colors[color]
+        except KeyError:
+            msg = (
+                f"invalid color {color!r}; choose between"
+                f" {list(colors.keys())!r}"
+            )
+            raise ValueError(msg) from None
+        if bold and color <= 7:
+            color += 8
+
+        handle_id = -12 if file is sys.stderr else -11
+        GetStdHandle.restype = ctypes.c_ulong
+        handle = GetStdHandle(handle_id)
+        SetConsoleTextAttribute(handle, color)
+        try:
+            print(s, file=file)
+        finally:
+            SetConsoleTextAttribute(handle, DEFAULT_COLOR)
+
+
+def debug(msg):
+    """If PSUTIL_DEBUG env var is set, print a debug message to stderr."""
+    if PSUTIL_DEBUG:
+        import inspect
+
+        fname, lineno, _, _lines, _index = inspect.getframeinfo(
+            inspect.currentframe().f_back
+        )
+        if isinstance(msg, Exception):
+            if isinstance(msg, OSError):
+                # ...because str(exc) may contain info about the file name
+                msg = f"ignoring {msg}"
+            else:
+                msg = f"ignoring {msg!r}"
+        print(  # noqa: T201
+            f"psutil-debug [{fname}:{lineno}]> {msg}", file=sys.stderr
+        )

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/psutil/_ntuples.py

@@ -0,0 +1,429 @@
+# Copyright (c) 2009, Giampaolo Rodola". All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from collections import namedtuple as nt
+
+from ._common import AIX
+from ._common import BSD
+from ._common import FREEBSD
+from ._common import LINUX
+from ._common import MACOS
+from ._common import SUNOS
+from ._common import WINDOWS
+
+# ===================================================================
+# --- system functions
+# ===================================================================
+
+# psutil.swap_memory()
+sswap = nt("sswap", ("total", "used", "free", "percent", "sin", "sout"))
+
+# psutil.disk_usage()
+sdiskusage = nt("sdiskusage", ("total", "used", "free", "percent"))
+
+# psutil.disk_io_counters()
+sdiskio = nt(
+    "sdiskio",
+    (
+        "read_count",
+        "write_count",
+        "read_bytes",
+        "write_bytes",
+        "read_time",
+        "write_time",
+    ),
+)
+
+# psutil.disk_partitions()
+sdiskpart = nt("sdiskpart", ("device", "mountpoint", "fstype", "opts"))
+
+# psutil.net_io_counters()
+snetio = nt(
+    "snetio",
+    (
+        "bytes_sent",
+        "bytes_recv",
+        "packets_sent",
+        "packets_recv",
+        "errin",
+        "errout",
+        "dropin",
+        "dropout",
+    ),
+)
+
+# psutil.users()
+suser = nt("suser", ("name", "terminal", "host", "started", "pid"))
+
+# psutil.net_connections()
+sconn = nt(
+    "sconn", ("fd", "family", "type", "laddr", "raddr", "status", "pid")
+)
+
+# psutil.net_if_addrs()
+snicaddr = nt("snicaddr", ("family", "address", "netmask", "broadcast", "ptp"))
+
+# psutil.net_if_stats()
+snicstats = nt("snicstats", ("isup", "duplex", "speed", "mtu", "flags"))
+
+# psutil.cpu_stats()
+scpustats = nt(
+    "scpustats", ("ctx_switches", "interrupts", "soft_interrupts", "syscalls")
+)
+
+# psutil.cpu_freq()
+scpufreq = nt("scpufreq", ("current", "min", "max"))
+
+# psutil.sensors_temperatures()
+shwtemp = nt("shwtemp", ("label", "current", "high", "critical"))
+
+# psutil.sensors_battery()
+sbattery = nt("sbattery", ("percent", "secsleft", "power_plugged"))
+
+# psutil.sensors_fans()
+sfan = nt("sfan", ("label", "current"))
+
+# psutil.heap_info() (mallinfo2 Linux struct)
+if LINUX or WINDOWS or MACOS or BSD:
+    pheap = nt(
+        "pheap",
+        [
+            "heap_used",  # uordblks, memory allocated via malloc()
+            "mmap_used",  # hblkhd, memory allocated via mmap() (large blocks)
+        ],
+    )
+    if WINDOWS:
+        pheap = nt("pheap", pheap._fields + ("heap_count",))
+
+# ===================================================================
+# --- Process class
+# ===================================================================
+
+# psutil.Process.cpu_times()
+pcputimes = nt(
+    "pcputimes", ("user", "system", "children_user", "children_system")
+)
+
+# psutil.Process.open_files()
+popenfile = nt("popenfile", ("path", "fd"))
+
+# psutil.Process.threads()
+pthread = nt("pthread", ("id", "user_time", "system_time"))
+
+# psutil.Process.uids()
+puids = nt("puids", ("real", "effective", "saved"))
+
+# psutil.Process.gids()
+pgids = nt("pgids", ("real", "effective", "saved"))
+
+# psutil.Process.io_counters()
+pio = nt("pio", ("read_count", "write_count", "read_bytes", "write_bytes"))
+
+# psutil.Process.ionice()
+pionice = nt("pionice", ("ioclass", "value"))
+
+# psutil.Process.ctx_switches()
+pctxsw = nt("pctxsw", ("voluntary", "involuntary"))
+
+# psutil.Process.net_connections()
+pconn = nt("pconn", ("fd", "family", "type", "laddr", "raddr", "status"))
+
+# psutil.net_connections() and psutil.Process.net_connections()
+addr = nt("addr", ("ip", "port"))
+
+# ===================================================================
+# --- Linux
+# ===================================================================
+
+if LINUX:
+
+    # This gets set from _pslinux.py
+    scputimes = None
+
+    # psutil.virtual_memory()
+    svmem = nt(
+        "svmem",
+        (
+            "total",
+            "available",
+            "percent",
+            "used",
+            "free",
+            "active",
+            "inactive",
+            "buffers",
+            "cached",
+            "shared",
+            "slab",
+        ),
+    )
+
+    # psutil.disk_io_counters()
+    sdiskio = nt(
+        "sdiskio",
+        (
+            "read_count",
+            "write_count",
+            "read_bytes",
+            "write_bytes",
+            "read_time",
+            "write_time",
+            "read_merged_count",
+            "write_merged_count",
+            "busy_time",
+        ),
+    )
+
+    # psutil.Process().open_files()
+    popenfile = nt("popenfile", ("path", "fd", "position", "mode", "flags"))
+
+    # psutil.Process().memory_info()
+    pmem = nt("pmem", ("rss", "vms", "shared", "text", "lib", "data", "dirty"))
+
+    # psutil.Process().memory_full_info()
+    pfullmem = nt("pfullmem", pmem._fields + ("uss", "pss", "swap"))
+
+    # psutil.Process().memory_maps(grouped=True)
+    pmmap_grouped = nt(
+        "pmmap_grouped",
+        (
+            "path",
+            "rss",
+            "size",
+            "pss",
+            "shared_clean",
+            "shared_dirty",
+            "private_clean",
+            "private_dirty",
+            "referenced",
+            "anonymous",
+            "swap",
+        ),
+    )
+
+    # psutil.Process().memory_maps(grouped=False)
+    pmmap_ext = nt(
+        "pmmap_ext", "addr perms " + " ".join(pmmap_grouped._fields)
+    )
+
+    # psutil.Process.io_counters()
+    pio = nt(
+        "pio",
+        (
+            "read_count",
+            "write_count",
+            "read_bytes",
+            "write_bytes",
+            "read_chars",
+            "write_chars",
+        ),
+    )
+
+    # psutil.Process.cpu_times()
+    pcputimes = nt(
+        "pcputimes",
+        ("user", "system", "children_user", "children_system", "iowait"),
+    )
+
+# ===================================================================
+# --- Windows
+# ===================================================================
+
+elif WINDOWS:
+
+    # psutil.cpu_times()
+    scputimes = nt("scputimes", ("user", "system", "idle", "interrupt", "dpc"))
+
+    # psutil.virtual_memory()
+    svmem = nt("svmem", ("total", "available", "percent", "used", "free"))
+
+    # psutil.Process.memory_info()
+    pmem = nt(
+        "pmem",
+        (
+            "rss",
+            "vms",
+            "num_page_faults",
+            "peak_wset",
+            "wset",
+            "peak_paged_pool",
+            "paged_pool",
+            "peak_nonpaged_pool",
+            "nonpaged_pool",
+            "pagefile",
+            "peak_pagefile",
+            "private",
+        ),
+    )
+
+    # psutil.Process.memory_full_info()
+    pfullmem = nt("pfullmem", pmem._fields + ("uss",))
+
+    # psutil.Process.memory_maps(grouped=True)
+    pmmap_grouped = nt("pmmap_grouped", ("path", "rss"))
+
+    # psutil.Process.memory_maps(grouped=False)
+    pmmap_ext = nt(
+        "pmmap_ext", "addr perms " + " ".join(pmmap_grouped._fields)
+    )
+
+    # psutil.Process.io_counters()
+    pio = nt(
+        "pio",
+        (
+            "read_count",
+            "write_count",
+            "read_bytes",
+            "write_bytes",
+            "other_count",
+            "other_bytes",
+        ),
+    )
+
+# ===================================================================
+# --- macOS
+# ===================================================================
+
+elif MACOS:
+
+    # psutil.cpu_times()
+    scputimes = nt("scputimes", ("user", "nice", "system", "idle"))
+
+    # psutil.virtual_memory()
+    svmem = nt(
+        "svmem",
+        (
+            "total",
+            "available",
+            "percent",
+            "used",
+            "free",
+            "active",
+            "inactive",
+            "wired",
+        ),
+    )
+
+    # psutil.Process.memory_info()
+    pmem = nt("pmem", ("rss", "vms", "pfaults", "pageins"))
+
+    # psutil.Process.memory_full_info()
+    pfullmem = nt("pfullmem", pmem._fields + ("uss",))
+
+# ===================================================================
+# --- BSD
+# ===================================================================
+
+elif BSD:
+
+    # psutil.virtual_memory()
+    svmem = nt(
+        "svmem",
+        (
+            "total",
+            "available",
+            "percent",
+            "used",
+            "free",
+            "active",
+            "inactive",
+            "buffers",
+            "cached",
+            "shared",
+            "wired",
+        ),
+    )
+
+    # psutil.cpu_times()
+    scputimes = nt("scputimes", ("user", "nice", "system", "idle", "irq"))
+
+    # psutil.Process.memory_info()
+    pmem = nt("pmem", ("rss", "vms", "text", "data", "stack"))
+
+    # psutil.Process.memory_full_info()
+    pfullmem = pmem
+
+    # psutil.Process.cpu_times()
+    pcputimes = nt(
+        "pcputimes", ("user", "system", "children_user", "children_system")
+    )
+
+    # psutil.Process.memory_maps(grouped=True)
+    pmmap_grouped = nt(
+        "pmmap_grouped", "path rss, private, ref_count, shadow_count"
+    )
+
+    # psutil.Process.memory_maps(grouped=False)
+    pmmap_ext = nt(
+        "pmmap_ext", "addr, perms path rss, private, ref_count, shadow_count"
+    )
+
+    # psutil.disk_io_counters()
+    if FREEBSD:
+        sdiskio = nt(
+            "sdiskio",
+            (
+                "read_count",
+                "write_count",
+                "read_bytes",
+                "write_bytes",
+                "read_time",
+                "write_time",
+                "busy_time",
+            ),
+        )
+    else:
+        sdiskio = nt(
+            "sdiskio",
+            ("read_count", "write_count", "read_bytes", "write_bytes"),
+        )
+
+# ===================================================================
+# --- SunOS
+# ===================================================================
+
+elif SUNOS:
+
+    # psutil.cpu_times()
+    scputimes = nt("scputimes", ("user", "system", "idle", "iowait"))
+
+    # psutil.cpu_times(percpu=True)
+    pcputimes = nt(
+        "pcputimes", ("user", "system", "children_user", "children_system")
+    )
+
+    # psutil.virtual_memory()
+    svmem = nt("svmem", ("total", "available", "percent", "used", "free"))
+
+    # psutil.Process.memory_info()
+    pmem = nt("pmem", ("rss", "vms"))
+
+    # psutil.Process.memory_full_info()
+    pfullmem = pmem
+
+    # psutil.Process.memory_maps(grouped=True)
+    pmmap_grouped = nt("pmmap_grouped", ("path", "rss", "anonymous", "locked"))
+
+    # psutil.Process.memory_maps(grouped=False)
+    pmmap_ext = nt(
+        "pmmap_ext", "addr perms " + " ".join(pmmap_grouped._fields)
+    )
+
+# ===================================================================
+# --- AIX
+# ===================================================================
+
+elif AIX:
+
+    # psutil.Process.memory_info()
+    pmem = nt("pmem", ("rss", "vms"))
+
+    # psutil.Process.memory_full_info()
+    pfullmem = pmem
+
+    # psutil.Process.cpu_times()
+    scputimes = nt("scputimes", ("user", "system", "idle", "iowait"))
+
+    # psutil.virtual_memory()
+    svmem = nt("svmem", ("total", "available", "percent", "used", "free"))

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/psutil/_psaix.py

@@ -0,0 +1,546 @@
+# Copyright (c) 2009, Giampaolo Rodola'
+# Copyright (c) 2017, Arnon Yaari
+# All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""AIX platform implementation."""
+
+import functools
+import glob
+import os
+import re
+import subprocess
+import sys
+
+from . import _common
+from . import _ntuples as ntp
+from . import _psposix
+from . import _psutil_aix as cext
+from ._common import NIC_DUPLEX_FULL
+from ._common import NIC_DUPLEX_HALF
+from ._common import NIC_DUPLEX_UNKNOWN
+from ._common import AccessDenied
+from ._common import NoSuchProcess
+from ._common import ZombieProcess
+from ._common import conn_to_ntuple
+from ._common import get_procfs_path
+from ._common import memoize_when_activated
+from ._common import usage_percent
+
+__extra__all__ = ["PROCFS_PATH"]
+
+
+# =====================================================================
+# --- globals
+# =====================================================================
+
+
+HAS_THREADS = hasattr(cext, "proc_threads")
+HAS_NET_IO_COUNTERS = hasattr(cext, "net_io_counters")
+HAS_PROC_IO_COUNTERS = hasattr(cext, "proc_io_counters")
+
+PAGE_SIZE = cext.getpagesize()
+AF_LINK = cext.AF_LINK
+
+PROC_STATUSES = {
+    cext.SIDL: _common.STATUS_IDLE,
+    cext.SZOMB: _common.STATUS_ZOMBIE,
+    cext.SACTIVE: _common.STATUS_RUNNING,
+    cext.SSWAP: _common.STATUS_RUNNING,  # TODO what status is this?
+    cext.SSTOP: _common.STATUS_STOPPED,
+}
+
+TCP_STATUSES = {
+    cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED,
+    cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT,
+    cext.TCPS_SYN_RCVD: _common.CONN_SYN_RECV,
+    cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1,
+    cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2,
+    cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT,
+    cext.TCPS_CLOSED: _common.CONN_CLOSE,
+    cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT,
+    cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK,
+    cext.TCPS_LISTEN: _common.CONN_LISTEN,
+    cext.TCPS_CLOSING: _common.CONN_CLOSING,
+    cext.PSUTIL_CONN_NONE: _common.CONN_NONE,
+}
+
+proc_info_map = dict(
+    ppid=0,
+    rss=1,
+    vms=2,
+    create_time=3,
+    nice=4,
+    num_threads=5,
+    status=6,
+    ttynr=7,
+)
+
+
+# =====================================================================
+# --- memory
+# =====================================================================
+
+
+def virtual_memory():
+    total, avail, free, _pinned, inuse = cext.virtual_mem()
+    percent = usage_percent((total - avail), total, round_=1)
+    return ntp.svmem(total, avail, percent, inuse, free)
+
+
+def swap_memory():
+    """Swap system memory as a (total, used, free, sin, sout) tuple."""
+    total, free, sin, sout = cext.swap_mem()
+    used = total - free
+    percent = usage_percent(used, total, round_=1)
+    return ntp.sswap(total, used, free, percent, sin, sout)
+
+
+# =====================================================================
+# --- CPU
+# =====================================================================
+
+
+def cpu_times():
+    """Return system-wide CPU times as a named tuple."""
+    ret = cext.per_cpu_times()
+    return ntp.scputimes(*[sum(x) for x in zip(*ret)])
+
+
+def per_cpu_times():
+    """Return system per-CPU times as a list of named tuples."""
+    ret = cext.per_cpu_times()
+    return [ntp.scputimes(*x) for x in ret]
+
+
+def cpu_count_logical():
+    """Return the number of logical CPUs in the system."""
+    try:
+        return os.sysconf("SC_NPROCESSORS_ONLN")
+    except ValueError:
+        # mimic os.cpu_count() behavior
+        return None
+
+
+def cpu_count_cores():
+    cmd = ["lsdev", "-Cc", "processor"]
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    stdout, stderr = (x.decode(sys.stdout.encoding) for x in (stdout, stderr))
+    if p.returncode != 0:
+        msg = f"{cmd!r} command error\n{stderr}"
+        raise RuntimeError(msg)
+    processors = stdout.strip().splitlines()
+    return len(processors) or None
+
+
+def cpu_stats():
+    """Return various CPU stats as a named tuple."""
+    ctx_switches, interrupts, soft_interrupts, syscalls = cext.cpu_stats()
+    return ntp.scpustats(ctx_switches, interrupts, soft_interrupts, syscalls)
+
+
+# =====================================================================
+# --- disks
+# =====================================================================
+
+
+disk_io_counters = cext.disk_io_counters
+disk_usage = _psposix.disk_usage
+
+
+def disk_partitions(all=False):
+    """Return system disk partitions."""
+    # TODO - the filtering logic should be better checked so that
+    # it tries to reflect 'df' as much as possible
+    retlist = []
+    partitions = cext.disk_partitions()
+    for partition in partitions:
+        device, mountpoint, fstype, opts = partition
+        if device == 'none':
+            device = ''
+        if not all:
+            # Differently from, say, Linux, we don't have a list of
+            # common fs types so the best we can do, AFAIK, is to
+            # filter by filesystem having a total size > 0.
+            if not disk_usage(mountpoint).total:
+                continue
+        ntuple = ntp.sdiskpart(device, mountpoint, fstype, opts)
+        retlist.append(ntuple)
+    return retlist
+
+
+# =====================================================================
+# --- network
+# =====================================================================
+
+
+net_if_addrs = cext.net_if_addrs
+
+if HAS_NET_IO_COUNTERS:
+    net_io_counters = cext.net_io_counters
+
+
+def net_connections(kind, _pid=-1):
+    """Return socket connections.  If pid == -1 return system-wide
+    connections (as opposed to connections opened by one process only).
+    """
+    families, types = _common.conn_tmap[kind]
+    rawlist = cext.net_connections(_pid)
+    ret = []
+    for item in rawlist:
+        fd, fam, type_, laddr, raddr, status, pid = item
+        if fam not in families:
+            continue
+        if type_ not in types:
+            continue
+        nt = conn_to_ntuple(
+            fd,
+            fam,
+            type_,
+            laddr,
+            raddr,
+            status,
+            TCP_STATUSES,
+            pid=pid if _pid == -1 else None,
+        )
+        ret.append(nt)
+    return ret
+
+
+def net_if_stats():
+    """Get NIC stats (isup, duplex, speed, mtu)."""
+    duplex_map = {"Full": NIC_DUPLEX_FULL, "Half": NIC_DUPLEX_HALF}
+    names = {x[0] for x in net_if_addrs()}
+    ret = {}
+    for name in names:
+        mtu = cext.net_if_mtu(name)
+        flags = cext.net_if_flags(name)
+
+        # try to get speed and duplex
+        # TODO: rewrite this in C (entstat forks, so use truss -f to follow.
+        # looks like it is using an undocumented ioctl?)
+        duplex = ""
+        speed = 0
+        p = subprocess.Popen(
+            ["/usr/bin/entstat", "-d", name],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        stdout, stderr = p.communicate()
+        stdout, stderr = (
+            x.decode(sys.stdout.encoding) for x in (stdout, stderr)
+        )
+        if p.returncode == 0:
+            re_result = re.search(
+                r"Running: (\d+) Mbps.*?(\w+) Duplex", stdout
+            )
+            if re_result is not None:
+                speed = int(re_result.group(1))
+                duplex = re_result.group(2)
+
+        output_flags = ','.join(flags)
+        isup = 'running' in flags
+        duplex = duplex_map.get(duplex, NIC_DUPLEX_UNKNOWN)
+        ret[name] = ntp.snicstats(isup, duplex, speed, mtu, output_flags)
+    return ret
+
+
+# =====================================================================
+# --- other system functions
+# =====================================================================
+
+
+def boot_time():
+    """The system boot time expressed in seconds since the epoch."""
+    return cext.boot_time()
+
+
+def users():
+    """Return currently connected users as a list of namedtuples."""
+    retlist = []
+    rawlist = cext.users()
+    localhost = (':0.0', ':0')
+    for item in rawlist:
+        user, tty, hostname, tstamp, user_process, pid = item
+        # note: the underlying C function includes entries about
+        # system boot, run level and others.  We might want
+        # to use them in the future.
+        if not user_process:
+            continue
+        if hostname in localhost:
+            hostname = 'localhost'
+        nt = ntp.suser(user, tty, hostname, tstamp, pid)
+        retlist.append(nt)
+    return retlist
+
+
+# =====================================================================
+# --- processes
+# =====================================================================
+
+
+def pids():
+    """Returns a list of PIDs currently running on the system."""
+    return [int(x) for x in os.listdir(get_procfs_path()) if x.isdigit()]
+
+
+def pid_exists(pid):
+    """Check for the existence of a unix pid."""
+    return os.path.exists(os.path.join(get_procfs_path(), str(pid), "psinfo"))
+
+
+def wrap_exceptions(fun):
+    """Call callable into a try/except clause and translate ENOENT,
+    EACCES and EPERM in NoSuchProcess or AccessDenied exceptions.
+    """
+
+    @functools.wraps(fun)
+    def wrapper(self, *args, **kwargs):
+        pid, ppid, name = self.pid, self._ppid, self._name
+        try:
+            return fun(self, *args, **kwargs)
+        except (FileNotFoundError, ProcessLookupError) as err:
+            # ENOENT (no such file or directory) gets raised on open().
+            # ESRCH (no such process) can get raised on read() if
+            # process is gone in meantime.
+            if not pid_exists(pid):
+                raise NoSuchProcess(pid, name) from err
+            raise ZombieProcess(pid, name, ppid) from err
+        except PermissionError as err:
+            raise AccessDenied(pid, name) from err
+
+    return wrapper
+
+
+class Process:
+    """Wrapper class around underlying C implementation."""
+
+    __slots__ = ["_cache", "_name", "_ppid", "_procfs_path", "pid"]
+
+    def __init__(self, pid):
+        self.pid = pid
+        self._name = None
+        self._ppid = None
+        self._procfs_path = get_procfs_path()
+
+    def oneshot_enter(self):
+        self._proc_basic_info.cache_activate(self)
+        self._proc_cred.cache_activate(self)
+
+    def oneshot_exit(self):
+        self._proc_basic_info.cache_deactivate(self)
+        self._proc_cred.cache_deactivate(self)
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def _proc_basic_info(self):
+        return cext.proc_basic_info(self.pid, self._procfs_path)
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def _proc_cred(self):
+        return cext.proc_cred(self.pid, self._procfs_path)
+
+    @wrap_exceptions
+    def name(self):
+        if self.pid == 0:
+            return "swapper"
+        # note: max 16 characters
+        return cext.proc_name(self.pid, self._procfs_path).rstrip("\x00")
+
+    @wrap_exceptions
+    def exe(self):
+        # there is no way to get executable path in AIX other than to guess,
+        # and guessing is more complex than what's in the wrapping class
+        cmdline = self.cmdline()
+        if not cmdline:
+            return ''
+        exe = cmdline[0]
+        if os.path.sep in exe:
+            # relative or absolute path
+            if not os.path.isabs(exe):
+                # if cwd has changed, we're out of luck - this may be wrong!
+                exe = os.path.abspath(os.path.join(self.cwd(), exe))
+            if (
+                os.path.isabs(exe)
+                and os.path.isfile(exe)
+                and os.access(exe, os.X_OK)
+            ):
+                return exe
+            # not found, move to search in PATH using basename only
+            exe = os.path.basename(exe)
+        # search for exe name PATH
+        for path in os.environ["PATH"].split(":"):
+            possible_exe = os.path.abspath(os.path.join(path, exe))
+            if os.path.isfile(possible_exe) and os.access(
+                possible_exe, os.X_OK
+            ):
+                return possible_exe
+        return ''
+
+    @wrap_exceptions
+    def cmdline(self):
+        return cext.proc_args(self.pid)
+
+    @wrap_exceptions
+    def environ(self):
+        return cext.proc_environ(self.pid)
+
+    @wrap_exceptions
+    def create_time(self):
+        return self._proc_basic_info()[proc_info_map['create_time']]
+
+    @wrap_exceptions
+    def num_threads(self):
+        return self._proc_basic_info()[proc_info_map['num_threads']]
+
+    if HAS_THREADS:
+
+        @wrap_exceptions
+        def threads(self):
+            rawlist = cext.proc_threads(self.pid)
+            retlist = []
+            for thread_id, utime, stime in rawlist:
+                ntuple = ntp.pthread(thread_id, utime, stime)
+                retlist.append(ntuple)
+            # The underlying C implementation retrieves all OS threads
+            # and filters them by PID.  At this point we can't tell whether
+            # an empty list means there were no connections for process or
+            # process is no longer active so we force NSP in case the PID
+            # is no longer there.
+            if not retlist:
+                # will raise NSP if process is gone
+                os.stat(f"{self._procfs_path}/{self.pid}")
+            return retlist
+
+    @wrap_exceptions
+    def net_connections(self, kind='inet'):
+        ret = net_connections(kind, _pid=self.pid)
+        # The underlying C implementation retrieves all OS connections
+        # and filters them by PID.  At this point we can't tell whether
+        # an empty list means there were no connections for process or
+        # process is no longer active so we force NSP in case the PID
+        # is no longer there.
+        if not ret:
+            # will raise NSP if process is gone
+            os.stat(f"{self._procfs_path}/{self.pid}")
+        return ret
+
+    @wrap_exceptions
+    def nice_get(self):
+        return cext.proc_priority_get(self.pid)
+
+    @wrap_exceptions
+    def nice_set(self, value):
+        return cext.proc_priority_set(self.pid, value)
+
+    @wrap_exceptions
+    def ppid(self):
+        self._ppid = self._proc_basic_info()[proc_info_map['ppid']]
+        return self._ppid
+
+    @wrap_exceptions
+    def uids(self):
+        real, effective, saved, _, _, _ = self._proc_cred()
+        return ntp.puids(real, effective, saved)
+
+    @wrap_exceptions
+    def gids(self):
+        _, _, _, real, effective, saved = self._proc_cred()
+        return ntp.puids(real, effective, saved)
+
+    @wrap_exceptions
+    def cpu_times(self):
+        t = cext.proc_cpu_times(self.pid, self._procfs_path)
+        return ntp.pcputimes(*t)
+
+    @wrap_exceptions
+    def terminal(self):
+        ttydev = self._proc_basic_info()[proc_info_map['ttynr']]
+        # convert from 64-bit dev_t to 32-bit dev_t and then map the device
+        ttydev = ((ttydev & 0x0000FFFF00000000) >> 16) | (ttydev & 0xFFFF)
+        # try to match rdev of /dev/pts/* files ttydev
+        for dev in glob.glob("/dev/**/*"):
+            if os.stat(dev).st_rdev == ttydev:
+                return dev
+        return None
+
+    @wrap_exceptions
+    def cwd(self):
+        procfs_path = self._procfs_path
+        try:
+            result = os.readlink(f"{procfs_path}/{self.pid}/cwd")
+            return result.rstrip('/')
+        except FileNotFoundError:
+            os.stat(f"{procfs_path}/{self.pid}")  # raise NSP or AD
+            return ""
+
+    @wrap_exceptions
+    def memory_info(self):
+        ret = self._proc_basic_info()
+        rss = ret[proc_info_map['rss']] * 1024
+        vms = ret[proc_info_map['vms']] * 1024
+        return ntp.pmem(rss, vms)
+
+    memory_full_info = memory_info
+
+    @wrap_exceptions
+    def status(self):
+        code = self._proc_basic_info()[proc_info_map['status']]
+        # XXX is '?' legit? (we're not supposed to return it anyway)
+        return PROC_STATUSES.get(code, '?')
+
+    def open_files(self):
+        # TODO rewrite without using procfiles (stat /proc/pid/fd/* and then
+        # find matching name of the inode)
+        p = subprocess.Popen(
+            ["/usr/bin/procfiles", "-n", str(self.pid)],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        stdout, stderr = p.communicate()
+        stdout, stderr = (
+            x.decode(sys.stdout.encoding) for x in (stdout, stderr)
+        )
+        if "no such process" in stderr.lower():
+            raise NoSuchProcess(self.pid, self._name)
+        procfiles = re.findall(r"(\d+): S_IFREG.*name:(.*)\n", stdout)
+        retlist = []
+        for fd, path in procfiles:
+            path = path.strip()
+            if path.startswith("//"):
+                path = path[1:]
+            if path.lower() == "cannot be retrieved":
+                continue
+            retlist.append(ntp.popenfile(path, int(fd)))
+        return retlist
+
+    @wrap_exceptions
+    def num_fds(self):
+        if self.pid == 0:  # no /proc/0/fd
+            return 0
+        return len(os.listdir(f"{self._procfs_path}/{self.pid}/fd"))
+
+    @wrap_exceptions
+    def num_ctx_switches(self):
+        return ntp.pctxsw(*cext.proc_num_ctx_switches(self.pid))
+
+    @wrap_exceptions
+    def wait(self, timeout=None):
+        return _psposix.wait_pid(self.pid, timeout)
+
+    if HAS_PROC_IO_COUNTERS:
+
+        @wrap_exceptions
+        def io_counters(self):
+            try:
+                rc, wc, rb, wb = cext.proc_io_counters(self.pid)
+            except OSError as err:
+                # if process is terminated, proc_io_counters returns OSError
+                # instead of NSP
+                if not pid_exists(self.pid):
+                    raise NoSuchProcess(self.pid, self._name) from err
+                raise
+            return ntp.pio(rc, wc, rb, wb)

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/psutil/_psbsd.py

@@ -0,0 +1,904 @@
+# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""FreeBSD, OpenBSD and NetBSD platforms implementation."""
+
+import contextlib
+import errno
+import functools
+import os
+from collections import defaultdict
+from collections import namedtuple
+from xml.etree import ElementTree  # noqa: ICN001
+
+from . import _common
+from . import _ntuples as ntp
+from . import _psposix
+from . import _psutil_bsd as cext
+from ._common import FREEBSD
+from ._common import NETBSD
+from ._common import OPENBSD
+from ._common import AccessDenied
+from ._common import NoSuchProcess
+from ._common import ZombieProcess
+from ._common import conn_tmap
+from ._common import conn_to_ntuple
+from ._common import debug
+from ._common import memoize
+from ._common import memoize_when_activated
+from ._common import usage_percent
+
+__extra__all__ = []
+
+
+# =====================================================================
+# --- globals
+# =====================================================================
+
+
+if FREEBSD:
+    PROC_STATUSES = {
+        cext.SIDL: _common.STATUS_IDLE,
+        cext.SRUN: _common.STATUS_RUNNING,
+        cext.SSLEEP: _common.STATUS_SLEEPING,
+        cext.SSTOP: _common.STATUS_STOPPED,
+        cext.SZOMB: _common.STATUS_ZOMBIE,
+        cext.SWAIT: _common.STATUS_WAITING,
+        cext.SLOCK: _common.STATUS_LOCKED,
+    }
+elif OPENBSD:
+    PROC_STATUSES = {
+        cext.SIDL: _common.STATUS_IDLE,
+        cext.SSLEEP: _common.STATUS_SLEEPING,
+        cext.SSTOP: _common.STATUS_STOPPED,
+        # According to /usr/include/sys/proc.h SZOMB is unused.
+        # test_zombie_process() shows that SDEAD is the right
+        # equivalent. Also it appears there's no equivalent of
+        # psutil.STATUS_DEAD. SDEAD really means STATUS_ZOMBIE.
+        # cext.SZOMB: _common.STATUS_ZOMBIE,
+        cext.SDEAD: _common.STATUS_ZOMBIE,
+        cext.SZOMB: _common.STATUS_ZOMBIE,
+        # From http://www.eecs.harvard.edu/~margo/cs161/videos/proc.h.txt
+        # OpenBSD has SRUN and SONPROC: SRUN indicates that a process
+        # is runnable but *not* yet running, i.e. is on a run queue.
+        # SONPROC indicates that the process is actually executing on
+        # a CPU, i.e. it is no longer on a run queue.
+        # As such we'll map SRUN to STATUS_WAKING and SONPROC to
+        # STATUS_RUNNING
+        cext.SRUN: _common.STATUS_WAKING,
+        cext.SONPROC: _common.STATUS_RUNNING,
+    }
+elif NETBSD:
+    PROC_STATUSES = {
+        cext.SIDL: _common.STATUS_IDLE,
+        cext.SSLEEP: _common.STATUS_SLEEPING,
+        cext.SSTOP: _common.STATUS_STOPPED,
+        cext.SZOMB: _common.STATUS_ZOMBIE,
+        cext.SRUN: _common.STATUS_WAKING,
+        cext.SONPROC: _common.STATUS_RUNNING,
+    }
+
+TCP_STATUSES = {
+    cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED,
+    cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT,
+    cext.TCPS_SYN_RECEIVED: _common.CONN_SYN_RECV,
+    cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1,
+    cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2,
+    cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT,
+    cext.TCPS_CLOSED: _common.CONN_CLOSE,
+    cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT,
+    cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK,
+    cext.TCPS_LISTEN: _common.CONN_LISTEN,
+    cext.TCPS_CLOSING: _common.CONN_CLOSING,
+    cext.PSUTIL_CONN_NONE: _common.CONN_NONE,
+}
+
+PAGESIZE = cext.getpagesize()
+AF_LINK = cext.AF_LINK
+
+HAS_PROC_NUM_THREADS = hasattr(cext, "proc_num_threads")
+
+kinfo_proc_map = dict(
+    ppid=0,
+    status=1,
+    real_uid=2,
+    effective_uid=3,
+    saved_uid=4,
+    real_gid=5,
+    effective_gid=6,
+    saved_gid=7,
+    ttynr=8,
+    create_time=9,
+    ctx_switches_vol=10,
+    ctx_switches_unvol=11,
+    read_io_count=12,
+    write_io_count=13,
+    user_time=14,
+    sys_time=15,
+    ch_user_time=16,
+    ch_sys_time=17,
+    rss=18,
+    vms=19,
+    memtext=20,
+    memdata=21,
+    memstack=22,
+    cpunum=23,
+    name=24,
+)
+
+
+# =====================================================================
+# --- memory
+# =====================================================================
+
+
+def virtual_memory():
+    mem = cext.virtual_mem()
+    if NETBSD:
+        total, free, active, inactive, wired, cached = mem
+        # On NetBSD buffers and shared mem is determined via /proc.
+        # The C ext set them to 0.
+        with open('/proc/meminfo', 'rb') as f:
+            for line in f:
+                if line.startswith(b'Buffers:'):
+                    buffers = int(line.split()[1]) * 1024
+                elif line.startswith(b'MemShared:'):
+                    shared = int(line.split()[1]) * 1024
+        # Before avail was calculated as (inactive + cached + free),
+        # same as zabbix, but it turned out it could exceed total (see
+        # #2233), so zabbix seems to be wrong. Htop calculates it
+        # differently, and the used value seem more realistic, so let's
+        # match htop.
+        # https://github.com/htop-dev/htop/blob/e7f447b/netbsd/NetBSDProcessList.c#L162
+        # https://github.com/zabbix/zabbix/blob/af5e0f8/src/libs/zbxsysinfo/netbsd/memory.c#L135
+        used = active + wired
+        avail = total - used
+    else:
+        total, free, active, inactive, wired, cached, buffers, shared = mem
+        # matches freebsd-memory CLI:
+        # * https://people.freebsd.org/~rse/dist/freebsd-memory
+        # * https://www.cyberciti.biz/files/scripts/freebsd-memory.pl.txt
+        # matches zabbix:
+        # * https://github.com/zabbix/zabbix/blob/af5e0f8/src/libs/zbxsysinfo/freebsd/memory.c#L143
+        avail = inactive + cached + free
+        used = active + wired + cached
+
+    percent = usage_percent((total - avail), total, round_=1)
+    return ntp.svmem(
+        total,
+        avail,
+        percent,
+        used,
+        free,
+        active,
+        inactive,
+        buffers,
+        cached,
+        shared,
+        wired,
+    )
+
+
+def swap_memory():
+    """System swap memory as (total, used, free, sin, sout) namedtuple."""
+    total, used, free, sin, sout = cext.swap_mem()
+    percent = usage_percent(used, total, round_=1)
+    return ntp.sswap(total, used, free, percent, sin, sout)
+
+
+# malloc / heap functions (FreeBSD / NetBSD)
+if hasattr(cext, "heap_info"):
+    heap_info = cext.heap_info
+    heap_trim = cext.heap_trim
+
+
+# =====================================================================
+# --- CPU
+# =====================================================================
+
+
+def cpu_times():
+    """Return system per-CPU times as a namedtuple."""
+    user, nice, system, idle, irq = cext.cpu_times()
+    return ntp.scputimes(user, nice, system, idle, irq)
+
+
+def per_cpu_times():
+    """Return system CPU times as a namedtuple."""
+    ret = []
+    for cpu_t in cext.per_cpu_times():
+        user, nice, system, idle, irq = cpu_t
+        item = ntp.scputimes(user, nice, system, idle, irq)
+        ret.append(item)
+    return ret
+
+
+def cpu_count_logical():
+    """Return the number of logical CPUs in the system."""
+    return cext.cpu_count_logical()
+
+
+if OPENBSD or NETBSD:
+
+    def cpu_count_cores():
+        # OpenBSD and NetBSD do not implement this.
+        return 1 if cpu_count_logical() == 1 else None
+
+else:
+
+    def cpu_count_cores():
+        """Return the number of CPU cores in the system."""
+        # From the C module we'll get an XML string similar to this:
+        # http://manpages.ubuntu.com/manpages/precise/man4/smp.4freebsd.html
+        # We may get None in case "sysctl kern.sched.topology_spec"
+        # is not supported on this BSD version, in which case we'll mimic
+        # os.cpu_count() and return None.
+        ret = None
+        s = cext.cpu_topology()
+        if s is not None:
+            # get rid of padding chars appended at the end of the string
+            index = s.rfind("</groups>")
+            if index != -1:
+                s = s[: index + 9]
+                root = ElementTree.fromstring(s)
+                try:
+                    ret = len(root.findall('group/children/group/cpu')) or None
+                finally:
+                    # needed otherwise it will memleak
+                    root.clear()
+        if not ret:
+            # If logical CPUs == 1 it's obvious we' have only 1 core.
+            if cpu_count_logical() == 1:
+                return 1
+        return ret
+
+
+def cpu_stats():
+    """Return various CPU stats as a named tuple."""
+    if FREEBSD:
+        # Note: the C ext is returning some metrics we are not exposing:
+        # traps.
+        ctxsw, intrs, soft_intrs, syscalls, _traps = cext.cpu_stats()
+    elif NETBSD:
+        # XXX
+        # Note about intrs: the C extension returns 0. intrs
+        # can be determined via /proc/stat; it has the same value as
+        # soft_intrs thought so the kernel is faking it (?).
+        #
+        # Note about syscalls: the C extension always sets it to 0 (?).
+        #
+        # Note: the C ext is returning some metrics we are not exposing:
+        # traps, faults and forks.
+        ctxsw, intrs, soft_intrs, syscalls, _traps, _faults, _forks = (
+            cext.cpu_stats()
+        )
+        with open('/proc/stat', 'rb') as f:
+            for line in f:
+                if line.startswith(b'intr'):
+                    intrs = int(line.split()[1])
+    elif OPENBSD:
+        # Note: the C ext is returning some metrics we are not exposing:
+        # traps, faults and forks.
+        ctxsw, intrs, soft_intrs, syscalls, _traps, _faults, _forks = (
+            cext.cpu_stats()
+        )
+    return ntp.scpustats(ctxsw, intrs, soft_intrs, syscalls)
+
+
+if FREEBSD:
+
+    def cpu_freq():
+        """Return frequency metrics for CPUs. As of Dec 2018 only
+        CPU 0 appears to be supported by FreeBSD and all other cores
+        match the frequency of CPU 0.
+        """
+        ret = []
+        num_cpus = cpu_count_logical()
+        for cpu in range(num_cpus):
+            try:
+                current, available_freq = cext.cpu_freq(cpu)
+            except NotImplementedError:
+                continue
+            if available_freq:
+                try:
+                    min_freq = int(available_freq.split(" ")[-1].split("/")[0])
+                except (IndexError, ValueError):
+                    min_freq = None
+                try:
+                    max_freq = int(available_freq.split(" ")[0].split("/")[0])
+                except (IndexError, ValueError):
+                    max_freq = None
+            ret.append(ntp.scpufreq(current, min_freq, max_freq))
+        return ret
+
+elif OPENBSD:
+
+    def cpu_freq():
+        curr = float(cext.cpu_freq())
+        return [ntp.scpufreq(curr, 0.0, 0.0)]
+
+
+# =====================================================================
+# --- disks
+# =====================================================================
+
+
+def disk_partitions(all=False):
+    """Return mounted disk partitions as a list of namedtuples.
+    'all' argument is ignored, see:
+    https://github.com/giampaolo/psutil/issues/906.
+    """
+    retlist = []
+    partitions = cext.disk_partitions()
+    for partition in partitions:
+        device, mountpoint, fstype, opts = partition
+        ntuple = ntp.sdiskpart(device, mountpoint, fstype, opts)
+        retlist.append(ntuple)
+    return retlist
+
+
+disk_usage = _psposix.disk_usage
+disk_io_counters = cext.disk_io_counters
+
+
+# =====================================================================
+# --- network
+# =====================================================================
+
+
+net_io_counters = cext.net_io_counters
+net_if_addrs = cext.net_if_addrs
+
+
+def net_if_stats():
+    """Get NIC stats (isup, duplex, speed, mtu)."""
+    names = net_io_counters().keys()
+    ret = {}
+    for name in names:
+        try:
+            mtu = cext.net_if_mtu(name)
+            flags = cext.net_if_flags(name)
+            duplex, speed = cext.net_if_duplex_speed(name)
+        except OSError as err:
+            # https://github.com/giampaolo/psutil/issues/1279
+            if err.errno != errno.ENODEV:
+                raise
+        else:
+            if hasattr(_common, 'NicDuplex'):
+                duplex = _common.NicDuplex(duplex)
+            output_flags = ','.join(flags)
+            isup = 'running' in flags
+            ret[name] = ntp.snicstats(isup, duplex, speed, mtu, output_flags)
+    return ret
+
+
+def net_connections(kind):
+    """System-wide network connections."""
+    families, types = conn_tmap[kind]
+    ret = set()
+    if OPENBSD:
+        rawlist = cext.net_connections(-1, families, types)
+    elif NETBSD:
+        rawlist = cext.net_connections(-1, kind)
+    else:  # FreeBSD
+        rawlist = cext.net_connections(families, types)
+
+    for item in rawlist:
+        fd, fam, type, laddr, raddr, status, pid = item
+        nt = conn_to_ntuple(
+            fd, fam, type, laddr, raddr, status, TCP_STATUSES, pid
+        )
+        ret.add(nt)
+    return list(ret)
+
+
+# =====================================================================
+#  --- sensors
+# =====================================================================
+
+
+if FREEBSD:
+
+    def sensors_battery():
+        """Return battery info."""
+        try:
+            percent, minsleft, power_plugged = cext.sensors_battery()
+        except NotImplementedError:
+            # See: https://github.com/giampaolo/psutil/issues/1074
+            return None
+        power_plugged = power_plugged == 1
+        if power_plugged:
+            secsleft = _common.POWER_TIME_UNLIMITED
+        elif minsleft == -1:
+            secsleft = _common.POWER_TIME_UNKNOWN
+        else:
+            secsleft = minsleft * 60
+        return ntp.sbattery(percent, secsleft, power_plugged)
+
+    def sensors_temperatures():
+        """Return CPU cores temperatures if available, else an empty dict."""
+        ret = defaultdict(list)
+        num_cpus = cpu_count_logical()
+        for cpu in range(num_cpus):
+            try:
+                current, high = cext.sensors_cpu_temperature(cpu)
+                if high <= 0:
+                    high = None
+                name = f"Core {cpu}"
+                ret["coretemp"].append(ntp.shwtemp(name, current, high, high))
+            except NotImplementedError:
+                pass
+
+        return ret
+
+
+# =====================================================================
+#  --- other system functions
+# =====================================================================
+
+
+def boot_time():
+    """The system boot time expressed in seconds since the epoch."""
+    return cext.boot_time()
+
+
+if NETBSD:
+
+    try:
+        INIT_BOOT_TIME = boot_time()
+    except Exception as err:  # noqa: BLE001
+        # Don't want to crash at import time.
+        debug(f"ignoring exception on import: {err!r}")
+        INIT_BOOT_TIME = 0
+
+    def adjust_proc_create_time(ctime):
+        """Account for system clock updates."""
+        if INIT_BOOT_TIME == 0:
+            return ctime
+
+        diff = INIT_BOOT_TIME - boot_time()
+        if diff == 0 or abs(diff) < 1:
+            return ctime
+
+        debug("system clock was updated; adjusting process create_time()")
+        if diff < 0:
+            return ctime - diff
+        return ctime + diff
+
+
+def users():
+    """Return currently connected users as a list of namedtuples."""
+    retlist = []
+    rawlist = cext.users()
+    for item in rawlist:
+        user, tty, hostname, tstamp, pid = item
+        if tty == '~':
+            continue  # reboot or shutdown
+        nt = ntp.suser(user, tty or None, hostname, tstamp, pid)
+        retlist.append(nt)
+    return retlist
+
+
+# =====================================================================
+# --- processes
+# =====================================================================
+
+
+@memoize
+def _pid_0_exists():
+    try:
+        Process(0).name()
+    except NoSuchProcess:
+        return False
+    except AccessDenied:
+        return True
+    else:
+        return True
+
+
+def pids():
+    """Returns a list of PIDs currently running on the system."""
+    ret = cext.pids()
+    if OPENBSD and (0 not in ret) and _pid_0_exists():
+        # On OpenBSD the kernel does not return PID 0 (neither does
+        # ps) but it's actually querable (Process(0) will succeed).
+        ret.insert(0, 0)
+    return ret
+
+
+if NETBSD:
+
+    def pid_exists(pid):
+        exists = _psposix.pid_exists(pid)
+        if not exists:
+            # We do this because _psposix.pid_exists() lies in case of
+            # zombie processes.
+            return pid in pids()
+        else:
+            return True
+
+elif OPENBSD:
+
+    def pid_exists(pid):
+        exists = _psposix.pid_exists(pid)
+        if not exists:
+            return False
+        else:
+            # OpenBSD seems to be the only BSD platform where
+            # _psposix.pid_exists() returns True for thread IDs (tids),
+            # so we can't use it.
+            return pid in pids()
+
+else:  # FreeBSD
+    pid_exists = _psposix.pid_exists
+
+
+def wrap_exceptions(fun):
+    """Decorator which translates bare OSError exceptions into
+    NoSuchProcess and AccessDenied.
+    """
+
+    @functools.wraps(fun)
+    def wrapper(self, *args, **kwargs):
+        pid, ppid, name = self.pid, self._ppid, self._name
+        try:
+            return fun(self, *args, **kwargs)
+        except ProcessLookupError as err:
+            if cext.proc_is_zombie(pid):
+                raise ZombieProcess(pid, name, ppid) from err
+            raise NoSuchProcess(pid, name) from err
+        except PermissionError as err:
+            raise AccessDenied(pid, name) from err
+        except cext.ZombieProcessError as err:
+            raise ZombieProcess(pid, name, ppid) from err
+        except OSError as err:
+            if pid == 0 and 0 in pids():
+                raise AccessDenied(pid, name) from err
+            raise err from None
+
+    return wrapper
+
+
+@contextlib.contextmanager
+def wrap_exceptions_procfs(inst):
+    """Same as above, for routines relying on reading /proc fs."""
+    pid, name, ppid = inst.pid, inst._name, inst._ppid
+    try:
+        yield
+    except (ProcessLookupError, FileNotFoundError) as err:
+        # ENOENT (no such file or directory) gets raised on open().
+        # ESRCH (no such process) can get raised on read() if
+        # process is gone in meantime.
+        if cext.proc_is_zombie(inst.pid):
+            raise ZombieProcess(pid, name, ppid) from err
+        else:
+            raise NoSuchProcess(pid, name) from err
+    except PermissionError as err:
+        raise AccessDenied(pid, name) from err
+
+
+class Process:
+    """Wrapper class around underlying C implementation."""
+
+    __slots__ = ["_cache", "_name", "_ppid", "pid"]
+
+    def __init__(self, pid):
+        self.pid = pid
+        self._name = None
+        self._ppid = None
+
+    def _assert_alive(self):
+        """Raise NSP if the process disappeared on us."""
+        # For those C function who do not raise NSP, possibly returning
+        # incorrect or incomplete result.
+        cext.proc_name(self.pid)
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def oneshot(self):
+        """Retrieves multiple process info in one shot as a raw tuple."""
+        ret = cext.proc_oneshot_info(self.pid)
+        assert len(ret) == len(kinfo_proc_map)
+        return ret
+
+    def oneshot_enter(self):
+        self.oneshot.cache_activate(self)
+
+    def oneshot_exit(self):
+        self.oneshot.cache_deactivate(self)
+
+    @wrap_exceptions
+    def name(self):
+        name = self.oneshot()[kinfo_proc_map['name']]
+        return name if name is not None else cext.proc_name(self.pid)
+
+    @wrap_exceptions
+    def exe(self):
+        if FREEBSD:
+            if self.pid == 0:
+                return ''  # else NSP
+            return cext.proc_exe(self.pid)
+        elif NETBSD:
+            if self.pid == 0:
+                # /proc/0 dir exists but /proc/0/exe doesn't
+                return ""
+            with wrap_exceptions_procfs(self):
+                return os.readlink(f"/proc/{self.pid}/exe")
+        else:
+            # OpenBSD: exe cannot be determined; references:
+            # https://chromium.googlesource.com/chromium/src/base/+/
+            #     master/base_paths_posix.cc
+            # We try our best guess by using which against the first
+            # cmdline arg (may return None).
+            import shutil
+
+            cmdline = self.cmdline()
+            if cmdline:
+                return shutil.which(cmdline[0]) or ""
+            else:
+                return ""
+
+    @wrap_exceptions
+    def cmdline(self):
+        if OPENBSD and self.pid == 0:
+            return []  # ...else it crashes
+        elif NETBSD:
+            # XXX - most of the times the underlying sysctl() call on
+            # NetBSD and OpenBSD returns a truncated string. Also
+            # /proc/pid/cmdline behaves the same so it looks like this
+            # is a kernel bug.
+            try:
+                return cext.proc_cmdline(self.pid)
+            except OSError as err:
+                if err.errno == errno.EINVAL:
+                    pid, name, ppid = self.pid, self._name, self._ppid
+                    if cext.proc_is_zombie(self.pid):
+                        raise ZombieProcess(pid, name, ppid) from err
+                    if not pid_exists(self.pid):
+                        raise NoSuchProcess(pid, name, ppid) from err
+                    # XXX: this happens with unicode tests. It means the C
+                    # routine is unable to decode invalid unicode chars.
+                    debug(f"ignoring {err!r} and returning an empty list")
+                    return []
+                else:
+                    raise
+        else:
+            return cext.proc_cmdline(self.pid)
+
+    @wrap_exceptions
+    def environ(self):
+        return cext.proc_environ(self.pid)
+
+    @wrap_exceptions
+    def terminal(self):
+        tty_nr = self.oneshot()[kinfo_proc_map['ttynr']]
+        tmap = _psposix.get_terminal_map()
+        try:
+            return tmap[tty_nr]
+        except KeyError:
+            return None
+
+    @wrap_exceptions
+    def ppid(self):
+        self._ppid = self.oneshot()[kinfo_proc_map['ppid']]
+        return self._ppid
+
+    @wrap_exceptions
+    def uids(self):
+        rawtuple = self.oneshot()
+        return ntp.puids(
+            rawtuple[kinfo_proc_map['real_uid']],
+            rawtuple[kinfo_proc_map['effective_uid']],
+            rawtuple[kinfo_proc_map['saved_uid']],
+        )
+
+    @wrap_exceptions
+    def gids(self):
+        rawtuple = self.oneshot()
+        return ntp.pgids(
+            rawtuple[kinfo_proc_map['real_gid']],
+            rawtuple[kinfo_proc_map['effective_gid']],
+            rawtuple[kinfo_proc_map['saved_gid']],
+        )
+
+    @wrap_exceptions
+    def cpu_times(self):
+        rawtuple = self.oneshot()
+        return ntp.pcputimes(
+            rawtuple[kinfo_proc_map['user_time']],
+            rawtuple[kinfo_proc_map['sys_time']],
+            rawtuple[kinfo_proc_map['ch_user_time']],
+            rawtuple[kinfo_proc_map['ch_sys_time']],
+        )
+
+    if FREEBSD:
+
+        @wrap_exceptions
+        def cpu_num(self):
+            return self.oneshot()[kinfo_proc_map['cpunum']]
+
+    @wrap_exceptions
+    def memory_info(self):
+        rawtuple = self.oneshot()
+        return ntp.pmem(
+            rawtuple[kinfo_proc_map['rss']],
+            rawtuple[kinfo_proc_map['vms']],
+            rawtuple[kinfo_proc_map['memtext']],
+            rawtuple[kinfo_proc_map['memdata']],
+            rawtuple[kinfo_proc_map['memstack']],
+        )
+
+    memory_full_info = memory_info
+
+    @wrap_exceptions
+    def create_time(self, monotonic=False):
+        ctime = self.oneshot()[kinfo_proc_map['create_time']]
+        if NETBSD and not monotonic:
+            # NetBSD: ctime subject to system clock updates.
+            ctime = adjust_proc_create_time(ctime)
+        return ctime
+
+    @wrap_exceptions
+    def num_threads(self):
+        if HAS_PROC_NUM_THREADS:
+            # FreeBSD / NetBSD
+            return cext.proc_num_threads(self.pid)
+        else:
+            return len(self.threads())
+
+    @wrap_exceptions
+    def num_ctx_switches(self):
+        rawtuple = self.oneshot()
+        return ntp.pctxsw(
+            rawtuple[kinfo_proc_map['ctx_switches_vol']],
+            rawtuple[kinfo_proc_map['ctx_switches_unvol']],
+        )
+
+    @wrap_exceptions
+    def threads(self):
+        # Note: on OpenSBD this (/dev/mem) requires root access.
+        rawlist = cext.proc_threads(self.pid)
+        retlist = []
+        for thread_id, utime, stime in rawlist:
+            ntuple = ntp.pthread(thread_id, utime, stime)
+            retlist.append(ntuple)
+        if OPENBSD:
+            self._assert_alive()
+        return retlist
+
+    @wrap_exceptions
+    def net_connections(self, kind='inet'):
+        families, types = conn_tmap[kind]
+        ret = []
+
+        if NETBSD:
+            rawlist = cext.net_connections(self.pid, kind)
+        elif OPENBSD:
+            rawlist = cext.net_connections(self.pid, families, types)
+        else:
+            rawlist = cext.proc_net_connections(self.pid, families, types)
+
+        for item in rawlist:
+            fd, fam, type, laddr, raddr, status = item[:6]
+            if FREEBSD:
+                if (fam not in families) or (type not in types):
+                    continue
+            nt = conn_to_ntuple(
+                fd, fam, type, laddr, raddr, status, TCP_STATUSES
+            )
+            ret.append(nt)
+
+        self._assert_alive()
+        return ret
+
+    @wrap_exceptions
+    def wait(self, timeout=None):
+        return _psposix.wait_pid(self.pid, timeout)
+
+    @wrap_exceptions
+    def nice_get(self):
+        return cext.proc_priority_get(self.pid)
+
+    @wrap_exceptions
+    def nice_set(self, value):
+        return cext.proc_priority_set(self.pid, value)
+
+    @wrap_exceptions
+    def status(self):
+        code = self.oneshot()[kinfo_proc_map['status']]
+        # XXX is '?' legit? (we're not supposed to return it anyway)
+        return PROC_STATUSES.get(code, '?')
+
+    @wrap_exceptions
+    def io_counters(self):
+        rawtuple = self.oneshot()
+        return ntp.pio(
+            rawtuple[kinfo_proc_map['read_io_count']],
+            rawtuple[kinfo_proc_map['write_io_count']],
+            -1,
+            -1,
+        )
+
+    @wrap_exceptions
+    def cwd(self):
+        """Return process current working directory."""
+        # sometimes we get an empty string, in which case we turn
+        # it into None
+        if OPENBSD and self.pid == 0:
+            return ""  # ...else it would raise EINVAL
+        return cext.proc_cwd(self.pid)
+
+    nt_mmap_grouped = namedtuple(
+        'mmap', 'path rss, private, ref_count, shadow_count'
+    )
+    nt_mmap_ext = namedtuple(
+        'mmap', 'addr, perms path rss, private, ref_count, shadow_count'
+    )
+
+    @wrap_exceptions
+    def open_files(self):
+        """Return files opened by process as a list of namedtuples."""
+        rawlist = cext.proc_open_files(self.pid)
+        return [ntp.popenfile(path, fd) for path, fd in rawlist]
+
+    @wrap_exceptions
+    def num_fds(self):
+        """Return the number of file descriptors opened by this process."""
+        ret = cext.proc_num_fds(self.pid)
+        if NETBSD:
+            self._assert_alive()
+        return ret
+
+    # --- FreeBSD only APIs
+
+    if FREEBSD:
+
+        @wrap_exceptions
+        def cpu_affinity_get(self):
+            return cext.proc_cpu_affinity_get(self.pid)
+
+        @wrap_exceptions
+        def cpu_affinity_set(self, cpus):
+            # Pre-emptively check if CPUs are valid because the C
+            # function has a weird behavior in case of invalid CPUs,
+            # see: https://github.com/giampaolo/psutil/issues/586
+            allcpus = set(range(len(per_cpu_times())))
+            for cpu in cpus:
+                if cpu not in allcpus:
+                    msg = f"invalid CPU {cpu!r} (choose between {allcpus})"
+                    raise ValueError(msg)
+            try:
+                cext.proc_cpu_affinity_set(self.pid, cpus)
+            except OSError as err:
+                # 'man cpuset_setaffinity' about EDEADLK:
+                # <<the call would leave a thread without a valid CPU to run
+                # on because the set does not overlap with the thread's
+                # anonymous mask>>
+                if err.errno in {errno.EINVAL, errno.EDEADLK}:
+                    for cpu in cpus:
+                        if cpu not in allcpus:
+                            msg = (
+                                f"invalid CPU {cpu!r} (choose between"
+                                f" {allcpus})"
+                            )
+                            raise ValueError(msg) from err
+                raise
+
+        @wrap_exceptions
+        def memory_maps(self):
+            return cext.proc_memory_maps(self.pid)
+
+        @wrap_exceptions
+        def rlimit(self, resource, limits=None):
+            if limits is None:
+                return cext.proc_getrlimit(self.pid, resource)
+            else:
+                if len(limits) != 2:
+                    msg = (
+                        "second argument must be a (soft, hard) tuple, got"
+                        f" {limits!r}"
+                    )
+                    raise ValueError(msg)
+                soft, hard = limits
+                return cext.proc_setrlimit(self.pid, resource, soft, hard)

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/psutil/_pslinux.py

@@ -0,0 +1,2269 @@
+# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Linux platform implementation."""
+
+import base64
+import collections
+import enum
+import errno
+import functools
+import glob
+import os
+import re
+import resource
+import socket
+import struct
+import sys
+import warnings
+from collections import defaultdict
+from collections import namedtuple
+
+from . import _common
+from . import _ntuples as ntp
+from . import _psposix
+from . import _psutil_linux as cext
+from ._common import ENCODING
+from ._common import NIC_DUPLEX_FULL
+from ._common import NIC_DUPLEX_HALF
+from ._common import NIC_DUPLEX_UNKNOWN
+from ._common import AccessDenied
+from ._common import NoSuchProcess
+from ._common import ZombieProcess
+from ._common import bcat
+from ._common import cat
+from ._common import debug
+from ._common import decode
+from ._common import get_procfs_path
+from ._common import isfile_strict
+from ._common import memoize
+from ._common import memoize_when_activated
+from ._common import open_binary
+from ._common import open_text
+from ._common import parse_environ_block
+from ._common import path_exists_strict
+from ._common import supports_ipv6
+from ._common import usage_percent
+
+# fmt: off
+__extra__all__ = [
+    'PROCFS_PATH',
+    # io prio constants
+    "IOPRIO_CLASS_NONE", "IOPRIO_CLASS_RT", "IOPRIO_CLASS_BE",
+    "IOPRIO_CLASS_IDLE",
+    # connection status constants
+    "CONN_ESTABLISHED", "CONN_SYN_SENT", "CONN_SYN_RECV", "CONN_FIN_WAIT1",
+    "CONN_FIN_WAIT2", "CONN_TIME_WAIT", "CONN_CLOSE", "CONN_CLOSE_WAIT",
+    "CONN_LAST_ACK", "CONN_LISTEN", "CONN_CLOSING",
+]
+# fmt: on
+
+
+# =====================================================================
+# --- globals
+# =====================================================================
+
+
+POWER_SUPPLY_PATH = "/sys/class/power_supply"
+HAS_PROC_SMAPS = os.path.exists(f"/proc/{os.getpid()}/smaps")
+HAS_PROC_SMAPS_ROLLUP = os.path.exists(f"/proc/{os.getpid()}/smaps_rollup")
+HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get")
+HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get")
+
+# Number of clock ticks per second
+CLOCK_TICKS = os.sysconf("SC_CLK_TCK")
+PAGESIZE = cext.getpagesize()
+LITTLE_ENDIAN = sys.byteorder == 'little'
+UNSET = object()
+
+# "man iostat" states that sectors are equivalent with blocks and have
+# a size of 512 bytes. Despite this value can be queried at runtime
+# via /sys/block/{DISK}/queue/hw_sector_size and results may vary
+# between 1k, 2k, or 4k... 512 appears to be a magic constant used
+# throughout Linux source code:
+# * https://stackoverflow.com/a/38136179/376587
+# * https://lists.gt.net/linux/kernel/2241060
+# * https://github.com/giampaolo/psutil/issues/1305
+# * https://github.com/torvalds/linux/blob/
+#     4f671fe2f9523a1ea206f63fe60a7c7b3a56d5c7/include/linux/bio.h#L99
+# * https://lkml.org/lkml/2015/8/17/234
+DISK_SECTOR_SIZE = 512
+
+AddressFamily = enum.IntEnum(
+    'AddressFamily', {'AF_LINK': int(socket.AF_PACKET)}
+)
+AF_LINK = AddressFamily.AF_LINK
+
+
+# ioprio_* constants http://linux.die.net/man/2/ioprio_get
+class IOPriority(enum.IntEnum):
+    IOPRIO_CLASS_NONE = 0
+    IOPRIO_CLASS_RT = 1
+    IOPRIO_CLASS_BE = 2
+    IOPRIO_CLASS_IDLE = 3
+
+
+globals().update(IOPriority.__members__)
+
+# See:
+# https://github.com/torvalds/linux/blame/master/fs/proc/array.c
+# ...and (TASK_* constants):
+# https://github.com/torvalds/linux/blob/master/include/linux/sched.h
+PROC_STATUSES = {
+    "R": _common.STATUS_RUNNING,
+    "S": _common.STATUS_SLEEPING,
+    "D": _common.STATUS_DISK_SLEEP,
+    "T": _common.STATUS_STOPPED,
+    "t": _common.STATUS_TRACING_STOP,
+    "Z": _common.STATUS_ZOMBIE,
+    "X": _common.STATUS_DEAD,
+    "x": _common.STATUS_DEAD,
+    "K": _common.STATUS_WAKE_KILL,
+    "W": _common.STATUS_WAKING,
+    "I": _common.STATUS_IDLE,
+    "P": _common.STATUS_PARKED,
+}
+
+# https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h
+TCP_STATUSES = {
+    "01": _common.CONN_ESTABLISHED,
+    "02": _common.CONN_SYN_SENT,
+    "03": _common.CONN_SYN_RECV,
+    "04": _common.CONN_FIN_WAIT1,
+    "05": _common.CONN_FIN_WAIT2,
+    "06": _common.CONN_TIME_WAIT,
+    "07": _common.CONN_CLOSE,
+    "08": _common.CONN_CLOSE_WAIT,
+    "09": _common.CONN_LAST_ACK,
+    "0A": _common.CONN_LISTEN,
+    "0B": _common.CONN_CLOSING,
+}
+
+
+# =====================================================================
+# --- utils
+# =====================================================================
+
+
+def readlink(path):
+    """Wrapper around os.readlink()."""
+    assert isinstance(path, str), path
+    path = os.readlink(path)
+    # readlink() might return paths containing null bytes ('\x00')
+    # resulting in "TypeError: must be encoded string without NULL
+    # bytes, not str" errors when the string is passed to other
+    # fs-related functions (os.*, open(), ...).
+    # Apparently everything after '\x00' is garbage (we can have
+    # ' (deleted)', 'new' and possibly others), see:
+    # https://github.com/giampaolo/psutil/issues/717
+    path = path.split('\x00')[0]
+    # Certain paths have ' (deleted)' appended. Usually this is
+    # bogus as the file actually exists. Even if it doesn't we
+    # don't care.
+    if path.endswith(' (deleted)') and not path_exists_strict(path):
+        path = path[:-10]
+    return path
+
+
+def file_flags_to_mode(flags):
+    """Convert file's open() flags into a readable string.
+    Used by Process.open_files().
+    """
+    modes_map = {os.O_RDONLY: 'r', os.O_WRONLY: 'w', os.O_RDWR: 'w+'}
+    mode = modes_map[flags & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR)]
+    if flags & os.O_APPEND:
+        mode = mode.replace('w', 'a', 1)
+    mode = mode.replace('w+', 'r+')
+    # possible values: r, w, a, r+, a+
+    return mode
+
+
+def is_storage_device(name):
+    """Return True if the given name refers to a root device (e.g.
+    "sda", "nvme0n1") as opposed to a logical partition (e.g.  "sda1",
+    "nvme0n1p1"). If name is a virtual device (e.g. "loop1", "ram")
+    return True.
+    """
+    # Re-adapted from iostat source code, see:
+    # https://github.com/sysstat/sysstat/blob/
+    #     97912938cd476645b267280069e83b1c8dc0e1c7/common.c#L208
+    # Some devices may have a slash in their name (e.g. cciss/c0d0...).
+    name = name.replace('/', '!')
+    including_virtual = True
+    if including_virtual:
+        path = f"/sys/block/{name}"
+    else:
+        path = f"/sys/block/{name}/device"
+    return os.access(path, os.F_OK)
+
+
+@memoize
+def _scputimes_ntuple(procfs_path):
+    """Return a namedtuple of variable fields depending on the CPU times
+    available on this Linux kernel version which may be:
+    (user, nice, system, idle, iowait, irq, softirq, [steal, [guest,
+     [guest_nice]]])
+    Used by cpu_times() function.
+    """
+    with open_binary(f"{procfs_path}/stat") as f:
+        values = f.readline().split()[1:]
+    fields = ['user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq']
+    vlen = len(values)
+    if vlen >= 8:
+        # Linux >= 2.6.11
+        fields.append('steal')
+    if vlen >= 9:
+        # Linux >= 2.6.24
+        fields.append('guest')
+    if vlen >= 10:
+        # Linux >= 3.2.0
+        fields.append('guest_nice')
+    return namedtuple('scputimes', fields)
+
+
+# Set it into _ntuples.py namespace.
+try:
+    ntp.scputimes = _scputimes_ntuple("/proc")
+except Exception as err:  # noqa: BLE001
+    # Don't want to crash at import time.
+    debug(f"ignoring exception on import: {err!r}")
+    ntp.scputimes = namedtuple('scputimes', 'user system idle')(0.0, 0.0, 0.0)
+
+# XXX: must be available also at this module level in order to be
+# serialized (tests/test_misc.py::TestMisc::test_serialization).
+scputimes = ntp.scputimes
+
+
+# =====================================================================
+# --- system memory
+# =====================================================================
+
+
+def calculate_avail_vmem(mems):
+    """Fallback for kernels < 3.14 where /proc/meminfo does not provide
+    "MemAvailable", see:
+    https://blog.famzah.net/2014/09/24/.
+
+    This code reimplements the algorithm outlined here:
+    https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/
+        commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773
+
+    We use this function also when "MemAvailable" returns 0 (possibly a
+    kernel bug, see: https://github.com/giampaolo/psutil/issues/1915).
+    In that case this routine matches "free" CLI tool result ("available"
+    column).
+
+    XXX: on recent kernels this calculation may differ by ~1.5% compared
+    to "MemAvailable:", as it's calculated slightly differently.
+    It is still way more realistic than doing (free + cached) though.
+    See:
+    * https://gitlab.com/procps-ng/procps/issues/42
+    * https://github.com/famzah/linux-memavailable-procfs/issues/2
+    """
+    # Note about "fallback" value. According to:
+    # https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/
+    #     commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773
+    # ...long ago "available" memory was calculated as (free + cached),
+    # We use fallback when one of these is missing from /proc/meminfo:
+    # "Active(file)": introduced in 2.6.28 / Dec 2008
+    # "Inactive(file)": introduced in 2.6.28 / Dec 2008
+    # "SReclaimable": introduced in 2.6.19 / Nov 2006
+    # /proc/zoneinfo: introduced in 2.6.13 / Aug 2005
+    free = mems[b'MemFree:']
+    fallback = free + mems.get(b"Cached:", 0)
+    try:
+        lru_active_file = mems[b'Active(file):']
+        lru_inactive_file = mems[b'Inactive(file):']
+        slab_reclaimable = mems[b'SReclaimable:']
+    except KeyError as err:
+        debug(
+            f"{err.args[0]} is missing from /proc/meminfo; using an"
+            " approximation for calculating available memory"
+        )
+        return fallback
+    try:
+        f = open_binary(f"{get_procfs_path()}/zoneinfo")
+    except OSError:
+        return fallback  # kernel 2.6.13
+
+    watermark_low = 0
+    with f:
+        for line in f:
+            line = line.strip()
+            if line.startswith(b'low'):
+                watermark_low += int(line.split()[1])
+    watermark_low *= PAGESIZE
+
+    avail = free - watermark_low
+    pagecache = lru_active_file + lru_inactive_file
+    pagecache -= min(pagecache / 2, watermark_low)
+    avail += pagecache
+    avail += slab_reclaimable - min(slab_reclaimable / 2.0, watermark_low)
+    return int(avail)
+
+
+def virtual_memory():
+    """Report virtual memory stats.
+    This implementation mimics procps-ng-3.3.12, aka "free" CLI tool:
+    https://gitlab.com/procps-ng/procps/blob/
+        24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L778-791
+    The returned values are supposed to match both "free" and "vmstat -s"
+    CLI tools.
+    """
+    missing_fields = []
+    mems = {}
+    with open_binary(f"{get_procfs_path()}/meminfo") as f:
+        for line in f:
+            fields = line.split()
+            mems[fields[0]] = int(fields[1]) * 1024
+
+    # /proc doc states that the available fields in /proc/meminfo vary
+    # by architecture and compile options, but these 3 values are also
+    # returned by sysinfo(2); as such we assume they are always there.
+    total = mems[b'MemTotal:']
+    free = mems[b'MemFree:']
+    try:
+        buffers = mems[b'Buffers:']
+    except KeyError:
+        # https://github.com/giampaolo/psutil/issues/1010
+        buffers = 0
+        missing_fields.append('buffers')
+    try:
+        cached = mems[b"Cached:"]
+    except KeyError:
+        cached = 0
+        missing_fields.append('cached')
+    else:
+        # "free" cmdline utility sums reclaimable to cached.
+        # Older versions of procps used to add slab memory instead.
+        # This got changed in:
+        # https://gitlab.com/procps-ng/procps/commit/
+        #     05d751c4f076a2f0118b914c5e51cfbb4762ad8e
+        cached += mems.get(b"SReclaimable:", 0)  # since kernel 2.6.19
+
+    try:
+        shared = mems[b'Shmem:']  # since kernel 2.6.32
+    except KeyError:
+        try:
+            shared = mems[b'MemShared:']  # kernels 2.4
+        except KeyError:
+            shared = 0
+            missing_fields.append('shared')
+
+    try:
+        active = mems[b"Active:"]
+    except KeyError:
+        active = 0
+        missing_fields.append('active')
+
+    try:
+        inactive = mems[b"Inactive:"]
+    except KeyError:
+        try:
+            inactive = (
+                mems[b"Inact_dirty:"]
+                + mems[b"Inact_clean:"]
+                + mems[b"Inact_laundry:"]
+            )
+        except KeyError:
+            inactive = 0
+            missing_fields.append('inactive')
+
+    try:
+        slab = mems[b"Slab:"]
+    except KeyError:
+        slab = 0
+
+    # - starting from 4.4.0 we match free's "available" column.
+    #   Before 4.4.0 we calculated it as (free + buffers + cached)
+    #   which matched htop.
+    # - free and htop available memory differs as per:
+    #   http://askubuntu.com/a/369589
+    #   http://unix.stackexchange.com/a/65852/168884
+    # - MemAvailable has been introduced in kernel 3.14
+    try:
+        avail = mems[b'MemAvailable:']
+    except KeyError:
+        avail = calculate_avail_vmem(mems)
+    else:
+        if avail == 0:
+            # Yes, it can happen (probably a kernel bug):
+            # https://github.com/giampaolo/psutil/issues/1915
+            # In this case "free" CLI tool makes an estimate. We do the same,
+            # and it matches "free" CLI tool.
+            avail = calculate_avail_vmem(mems)
+
+    if avail < 0:
+        avail = 0
+        missing_fields.append('available')
+    elif avail > total:
+        # If avail is greater than total or our calculation overflows,
+        # that's symptomatic of running within a LCX container where such
+        # values will be dramatically distorted over those of the host.
+        # https://gitlab.com/procps-ng/procps/blob/
+        #     24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L764
+        avail = free
+
+    used = total - avail
+
+    percent = usage_percent((total - avail), total, round_=1)
+
+    # Warn about missing metrics which are set to 0.
+    if missing_fields:
+        msg = "{} memory stats couldn't be determined and {} set to 0".format(
+            ", ".join(missing_fields),
+            "was" if len(missing_fields) == 1 else "were",
+        )
+        warnings.warn(msg, RuntimeWarning, stacklevel=2)
+
+    return ntp.svmem(
+        total,
+        avail,
+        percent,
+        used,
+        free,
+        active,
+        inactive,
+        buffers,
+        cached,
+        shared,
+        slab,
+    )
+
+
+def swap_memory():
+    """Return swap memory metrics."""
+    mems = {}
+    with open_binary(f"{get_procfs_path()}/meminfo") as f:
+        for line in f:
+            fields = line.split()
+            mems[fields[0]] = int(fields[1]) * 1024
+    # We prefer /proc/meminfo over sysinfo() syscall so that
+    # psutil.PROCFS_PATH can be used in order to allow retrieval
+    # for linux containers, see:
+    # https://github.com/giampaolo/psutil/issues/1015
+    try:
+        total = mems[b'SwapTotal:']
+        free = mems[b'SwapFree:']
+    except KeyError:
+        _, _, _, _, total, free, unit_multiplier = cext.linux_sysinfo()
+        total *= unit_multiplier
+        free *= unit_multiplier
+
+    used = total - free
+    percent = usage_percent(used, total, round_=1)
+    # get pgin/pgouts
+    try:
+        f = open_binary(f"{get_procfs_path()}/vmstat")
+    except OSError as err:
+        # see https://github.com/giampaolo/psutil/issues/722
+        msg = (
+            "'sin' and 'sout' swap memory stats couldn't "
+            f"be determined and were set to 0 ({err})"
+        )
+        warnings.warn(msg, RuntimeWarning, stacklevel=2)
+        sin = sout = 0
+    else:
+        with f:
+            sin = sout = None
+            for line in f:
+                # values are expressed in 4 kilo bytes, we want
+                # bytes instead
+                if line.startswith(b'pswpin'):
+                    sin = int(line.split(b' ')[1]) * 4 * 1024
+                elif line.startswith(b'pswpout'):
+                    sout = int(line.split(b' ')[1]) * 4 * 1024
+                if sin is not None and sout is not None:
+                    break
+            else:
+                # we might get here when dealing with exotic Linux
+                # flavors, see:
+                # https://github.com/giampaolo/psutil/issues/313
+                msg = "'sin' and 'sout' swap memory stats couldn't "
+                msg += "be determined and were set to 0"
+                warnings.warn(msg, RuntimeWarning, stacklevel=2)
+                sin = sout = 0
+    return ntp.sswap(total, used, free, percent, sin, sout)
+
+
+# malloc / heap functions; require glibc
+if hasattr(cext, "heap_info"):
+    heap_info = cext.heap_info
+    heap_trim = cext.heap_trim
+
+
+# =====================================================================
+# --- CPU
+# =====================================================================
+
+
+def cpu_times():
+    """Return a named tuple representing the following system-wide
+    CPU times:
+    (user, nice, system, idle, iowait, irq, softirq [steal, [guest,
+     [guest_nice]]])
+    Last 3 fields may not be available on all Linux kernel versions.
+    """
+    procfs_path = get_procfs_path()
+    with open_binary(f"{procfs_path}/stat") as f:
+        values = f.readline().split()
+    fields = values[1 : len(ntp.scputimes._fields) + 1]
+    fields = [float(x) / CLOCK_TICKS for x in fields]
+    return ntp.scputimes(*fields)
+
+
+def per_cpu_times():
+    """Return a list of namedtuple representing the CPU times
+    for every CPU available on the system.
+    """
+    procfs_path = get_procfs_path()
+    cpus = []
+    with open_binary(f"{procfs_path}/stat") as f:
+        # get rid of the first line which refers to system wide CPU stats
+        f.readline()
+        for line in f:
+            if line.startswith(b'cpu'):
+                values = line.split()
+                fields = values[1 : len(ntp.scputimes._fields) + 1]
+                fields = [float(x) / CLOCK_TICKS for x in fields]
+                entry = ntp.scputimes(*fields)
+                cpus.append(entry)
+        return cpus
+
+
+def cpu_count_logical():
+    """Return the number of logical CPUs in the system."""
+    try:
+        return os.sysconf("SC_NPROCESSORS_ONLN")
+    except ValueError:
+        # as a second fallback we try to parse /proc/cpuinfo
+        num = 0
+        with open_binary(f"{get_procfs_path()}/cpuinfo") as f:
+            for line in f:
+                if line.lower().startswith(b'processor'):
+                    num += 1
+
+        # unknown format (e.g. amrel/sparc architectures), see:
+        # https://github.com/giampaolo/psutil/issues/200
+        # try to parse /proc/stat as a last resort
+        if num == 0:
+            search = re.compile(r'cpu\d')
+            with open_text(f"{get_procfs_path()}/stat") as f:
+                for line in f:
+                    line = line.split(' ')[0]
+                    if search.match(line):
+                        num += 1
+
+        if num == 0:
+            # mimic os.cpu_count()
+            return None
+        return num
+
+
+def cpu_count_cores():
+    """Return the number of CPU cores in the system."""
+    # Method #1
+    ls = set()
+    # These 2 files are the same but */core_cpus_list is newer while
+    # */thread_siblings_list is deprecated and may disappear in the future.
+    # https://www.kernel.org/doc/Documentation/admin-guide/cputopology.rst
+    # https://github.com/giampaolo/psutil/pull/1727#issuecomment-707624964
+    # https://lkml.org/lkml/2019/2/26/41
+    p1 = "/sys/devices/system/cpu/cpu[0-9]*/topology/core_cpus_list"
+    p2 = "/sys/devices/system/cpu/cpu[0-9]*/topology/thread_siblings_list"
+    for path in glob.glob(p1) or glob.glob(p2):
+        with open_binary(path) as f:
+            ls.add(f.read().strip())
+    result = len(ls)
+    if result != 0:
+        return result
+
+    # Method #2
+    mapping = {}
+    current_info = {}
+    with open_binary(f"{get_procfs_path()}/cpuinfo") as f:
+        for line in f:
+            line = line.strip().lower()
+            if not line:
+                # new section
+                try:
+                    mapping[current_info[b'physical id']] = current_info[
+                        b'cpu cores'
+                    ]
+                except KeyError:
+                    pass
+                current_info = {}
+            elif line.startswith((b'physical id', b'cpu cores')):
+                # ongoing section
+                key, value = line.split(b'\t:', 1)
+                current_info[key] = int(value)
+
+    result = sum(mapping.values())
+    return result or None  # mimic os.cpu_count()
+
+
+def cpu_stats():
+    """Return various CPU stats as a named tuple."""
+    with open_binary(f"{get_procfs_path()}/stat") as f:
+        ctx_switches = None
+        interrupts = None
+        soft_interrupts = None
+        for line in f:
+            if line.startswith(b'ctxt'):
+                ctx_switches = int(line.split()[1])
+            elif line.startswith(b'intr'):
+                interrupts = int(line.split()[1])
+            elif line.startswith(b'softirq'):
+                soft_interrupts = int(line.split()[1])
+            if (
+                ctx_switches is not None
+                and soft_interrupts is not None
+                and interrupts is not None
+            ):
+                break
+    syscalls = 0
+    return ntp.scpustats(ctx_switches, interrupts, soft_interrupts, syscalls)
+
+
+def _cpu_get_cpuinfo_freq():
+    """Return current CPU frequency from cpuinfo if available."""
+    with open_binary(f"{get_procfs_path()}/cpuinfo") as f:
+        return [
+            float(line.split(b':', 1)[1])
+            for line in f
+            if line.lower().startswith(b'cpu mhz')
+        ]
+
+
+if os.path.exists("/sys/devices/system/cpu/cpufreq/policy0") or os.path.exists(
+    "/sys/devices/system/cpu/cpu0/cpufreq"
+):
+
+    def cpu_freq():
+        """Return frequency metrics for all CPUs.
+        Contrarily to other OSes, Linux updates these values in
+        real-time.
+        """
+        cpuinfo_freqs = _cpu_get_cpuinfo_freq()
+        paths = glob.glob(
+            "/sys/devices/system/cpu/cpufreq/policy[0-9]*"
+        ) or glob.glob("/sys/devices/system/cpu/cpu[0-9]*/cpufreq")
+        paths.sort(key=lambda x: int(re.search(r"[0-9]+", x).group()))
+        ret = []
+        pjoin = os.path.join
+        for i, path in enumerate(paths):
+            if len(paths) == len(cpuinfo_freqs):
+                # take cached value from cpuinfo if available, see:
+                # https://github.com/giampaolo/psutil/issues/1851
+                curr = cpuinfo_freqs[i] * 1000
+            else:
+                curr = bcat(pjoin(path, "scaling_cur_freq"), fallback=None)
+            if curr is None:
+                # Likely an old RedHat, see:
+                # https://github.com/giampaolo/psutil/issues/1071
+                curr = bcat(pjoin(path, "cpuinfo_cur_freq"), fallback=None)
+                if curr is None:
+                    online_path = f"/sys/devices/system/cpu/cpu{i}/online"
+                    # if cpu core is offline, set to all zeroes
+                    if cat(online_path, fallback=None) == "0\n":
+                        ret.append(ntp.scpufreq(0.0, 0.0, 0.0))
+                        continue
+                    msg = "can't find current frequency file"
+                    raise NotImplementedError(msg)
+            curr = int(curr) / 1000
+            max_ = int(bcat(pjoin(path, "scaling_max_freq"))) / 1000
+            min_ = int(bcat(pjoin(path, "scaling_min_freq"))) / 1000
+            ret.append(ntp.scpufreq(curr, min_, max_))
+        return ret
+
+else:
+
+    def cpu_freq():
+        """Alternate implementation using /proc/cpuinfo.
+        min and max frequencies are not available and are set to None.
+        """
+        return [ntp.scpufreq(x, 0.0, 0.0) for x in _cpu_get_cpuinfo_freq()]
+
+
+# =====================================================================
+# --- network
+# =====================================================================
+
+
+net_if_addrs = cext.net_if_addrs
+
+
+class _Ipv6UnsupportedError(Exception):
+    pass
+
+
+class NetConnections:
+    """A wrapper on top of /proc/net/* files, retrieving per-process
+    and system-wide open connections (TCP, UDP, UNIX) similarly to
+    "netstat -an".
+
+    Note: in case of UNIX sockets we're only able to determine the
+    local endpoint/path, not the one it's connected to.
+    According to [1] it would be possible but not easily.
+
+    [1] http://serverfault.com/a/417946
+    """
+
+    def __init__(self):
+        # The string represents the basename of the corresponding
+        # /proc/net/{proto_name} file.
+        tcp4 = ("tcp", socket.AF_INET, socket.SOCK_STREAM)
+        tcp6 = ("tcp6", socket.AF_INET6, socket.SOCK_STREAM)
+        udp4 = ("udp", socket.AF_INET, socket.SOCK_DGRAM)
+        udp6 = ("udp6", socket.AF_INET6, socket.SOCK_DGRAM)
+        unix = ("unix", socket.AF_UNIX, None)
+        self.tmap = {
+            "all": (tcp4, tcp6, udp4, udp6, unix),
+            "tcp": (tcp4, tcp6),
+            "tcp4": (tcp4,),
+            "tcp6": (tcp6,),
+            "udp": (udp4, udp6),
+            "udp4": (udp4,),
+            "udp6": (udp6,),
+            "unix": (unix,),
+            "inet": (tcp4, tcp6, udp4, udp6),
+            "inet4": (tcp4, udp4),
+            "inet6": (tcp6, udp6),
+        }
+        self._procfs_path = None
+
+    def get_proc_inodes(self, pid):
+        inodes = defaultdict(list)
+        for fd in os.listdir(f"{self._procfs_path}/{pid}/fd"):
+            try:
+                inode = readlink(f"{self._procfs_path}/{pid}/fd/{fd}")
+            except (FileNotFoundError, ProcessLookupError):
+                # ENOENT == file which is gone in the meantime;
+                # os.stat(f"/proc/{self.pid}") will be done later
+                # to force NSP (if it's the case)
+                continue
+            except OSError as err:
+                if err.errno == errno.EINVAL:
+                    # not a link
+                    continue
+                if err.errno == errno.ENAMETOOLONG:
+                    # file name too long
+                    debug(err)
+                    continue
+                raise
+            else:
+                if inode.startswith('socket:['):
+                    # the process is using a socket
+                    inode = inode[8:][:-1]
+                    inodes[inode].append((pid, int(fd)))
+        return inodes
+
+    def get_all_inodes(self):
+        inodes = {}
+        for pid in pids():
+            try:
+                inodes.update(self.get_proc_inodes(pid))
+            except (FileNotFoundError, ProcessLookupError, PermissionError):
+                # os.listdir() is gonna raise a lot of access denied
+                # exceptions in case of unprivileged user; that's fine
+                # as we'll just end up returning a connection with PID
+                # and fd set to None anyway.
+                # Both netstat -an and lsof does the same so it's
+                # unlikely we can do any better.
+                # ENOENT just means a PID disappeared on us.
+                continue
+        return inodes
+
+    @staticmethod
+    def decode_address(addr, family):
+        """Accept an "ip:port" address as displayed in /proc/net/*
+        and convert it into a human readable form, like:
+
+        "0500000A:0016" -> ("10.0.0.5", 22)
+        "0000000000000000FFFF00000100007F:9E49" -> ("::ffff:127.0.0.1", 40521)
+
+        The IP address portion is a little or big endian four-byte
+        hexadecimal number; that is, the least significant byte is listed
+        first, so we need to reverse the order of the bytes to convert it
+        to an IP address.
+        The port is represented as a two-byte hexadecimal number.
+
+        Reference:
+        http://linuxdevcenter.com/pub/a/linux/2000/11/16/LinuxAdmin.html
+        """
+        ip, port = addr.split(':')
+        port = int(port, 16)
+        # this usually refers to a local socket in listen mode with
+        # no end-points connected
+        if not port:
+            return ()
+        ip = ip.encode('ascii')
+        if family == socket.AF_INET:
+            # see: https://github.com/giampaolo/psutil/issues/201
+            if LITTLE_ENDIAN:
+                ip = socket.inet_ntop(family, base64.b16decode(ip)[::-1])
+            else:
+                ip = socket.inet_ntop(family, base64.b16decode(ip))
+        else:  # IPv6
+            ip = base64.b16decode(ip)
+            try:
+                # see: https://github.com/giampaolo/psutil/issues/201
+                if LITTLE_ENDIAN:
+                    ip = socket.inet_ntop(
+                        socket.AF_INET6,
+                        struct.pack('>4I', *struct.unpack('<4I', ip)),
+                    )
+                else:
+                    ip = socket.inet_ntop(
+                        socket.AF_INET6,
+                        struct.pack('<4I', *struct.unpack('<4I', ip)),
+                    )
+            except ValueError:
+                # see: https://github.com/giampaolo/psutil/issues/623
+                if not supports_ipv6():
+                    raise _Ipv6UnsupportedError from None
+                raise
+        return ntp.addr(ip, port)
+
+    @staticmethod
+    def process_inet(file, family, type_, inodes, filter_pid=None):
+        """Parse /proc/net/tcp* and /proc/net/udp* files."""
+        if file.endswith('6') and not os.path.exists(file):
+            # IPv6 not supported
+            return
+        with open_text(file) as f:
+            f.readline()  # skip the first line
+            for lineno, line in enumerate(f, 1):
+                try:
+                    _, laddr, raddr, status, _, _, _, _, _, inode = (
+                        line.split()[:10]
+                    )
+                except ValueError:
+                    msg = (
+                        f"error while parsing {file}; malformed line"
+                        f" {lineno} {line!r}"
+                    )
+                    raise RuntimeError(msg) from None
+                if inode in inodes:
+                    # # We assume inet sockets are unique, so we error
+                    # # out if there are multiple references to the
+                    # # same inode. We won't do this for UNIX sockets.
+                    # if len(inodes[inode]) > 1 and family != socket.AF_UNIX:
+                    #     raise ValueError("ambiguous inode with multiple "
+                    #                      "PIDs references")
+                    pid, fd = inodes[inode][0]
+                else:
+                    pid, fd = None, -1
+                if filter_pid is not None and filter_pid != pid:
+                    continue
+                else:
+                    if type_ == socket.SOCK_STREAM:
+                        status = TCP_STATUSES[status]
+                    else:
+                        status = _common.CONN_NONE
+                    try:
+                        laddr = NetConnections.decode_address(laddr, family)
+                        raddr = NetConnections.decode_address(raddr, family)
+                    except _Ipv6UnsupportedError:
+                        continue
+                    yield (fd, family, type_, laddr, raddr, status, pid)
+
+    @staticmethod
+    def process_unix(file, family, inodes, filter_pid=None):
+        """Parse /proc/net/unix files."""
+        with open_text(file) as f:
+            f.readline()  # skip the first line
+            for line in f:
+                tokens = line.split()
+                try:
+                    _, _, _, _, type_, _, inode = tokens[0:7]
+                except ValueError:
+                    if ' ' not in line:
+                        # see: https://github.com/giampaolo/psutil/issues/766
+                        continue
+                    msg = (
+                        f"error while parsing {file}; malformed line {line!r}"
+                    )
+                    raise RuntimeError(msg)  # noqa: B904
+                if inode in inodes:  # noqa: SIM108
+                    # With UNIX sockets we can have a single inode
+                    # referencing many file descriptors.
+                    pairs = inodes[inode]
+                else:
+                    pairs = [(None, -1)]
+                for pid, fd in pairs:
+                    if filter_pid is not None and filter_pid != pid:
+                        continue
+                    else:
+                        path = tokens[-1] if len(tokens) == 8 else ''
+                        type_ = _common.socktype_to_enum(int(type_))
+                        # XXX: determining the remote endpoint of a
+                        # UNIX socket on Linux is not possible, see:
+                        # https://serverfault.com/questions/252723/
+                        raddr = ""
+                        status = _common.CONN_NONE
+                        yield (fd, family, type_, path, raddr, status, pid)
+
+    def retrieve(self, kind, pid=None):
+        self._procfs_path = get_procfs_path()
+        if pid is not None:
+            inodes = self.get_proc_inodes(pid)
+            if not inodes:
+                # no connections for this process
+                return []
+        else:
+            inodes = self.get_all_inodes()
+        ret = set()
+        for proto_name, family, type_ in self.tmap[kind]:
+            path = f"{self._procfs_path}/net/{proto_name}"
+            if family in {socket.AF_INET, socket.AF_INET6}:
+                ls = self.process_inet(
+                    path, family, type_, inodes, filter_pid=pid
+                )
+            else:
+                ls = self.process_unix(path, family, inodes, filter_pid=pid)
+            for fd, family, type_, laddr, raddr, status, bound_pid in ls:
+                if pid:
+                    conn = ntp.pconn(fd, family, type_, laddr, raddr, status)
+                else:
+                    conn = ntp.sconn(
+                        fd, family, type_, laddr, raddr, status, bound_pid
+                    )
+                ret.add(conn)
+        return list(ret)
+
+
+_net_connections = NetConnections()
+
+
+def net_connections(kind='inet'):
+    """Return system-wide open connections."""
+    return _net_connections.retrieve(kind)
+
+
+def net_io_counters():
+    """Return network I/O statistics for every network interface
+    installed on the system as a dict of raw tuples.
+    """
+    with open_text(f"{get_procfs_path()}/net/dev") as f:
+        lines = f.readlines()
+    retdict = {}
+    for line in lines[2:]:
+        colon = line.rfind(':')
+        assert colon > 0, repr(line)
+        name = line[:colon].strip()
+        fields = line[colon + 1 :].strip().split()
+
+        (
+            # in
+            bytes_recv,
+            packets_recv,
+            errin,
+            dropin,
+            _fifoin,  # unused
+            _framein,  # unused
+            _compressedin,  # unused
+            _multicastin,  # unused
+            # out
+            bytes_sent,
+            packets_sent,
+            errout,
+            dropout,
+            _fifoout,  # unused
+            _collisionsout,  # unused
+            _carrierout,  # unused
+            _compressedout,  # unused
+        ) = map(int, fields)
+
+        retdict[name] = (
+            bytes_sent,
+            bytes_recv,
+            packets_sent,
+            packets_recv,
+            errin,
+            errout,
+            dropin,
+            dropout,
+        )
+    return retdict
+
+
+def net_if_stats():
+    """Get NIC stats (isup, duplex, speed, mtu)."""
+    duplex_map = {
+        cext.DUPLEX_FULL: NIC_DUPLEX_FULL,
+        cext.DUPLEX_HALF: NIC_DUPLEX_HALF,
+        cext.DUPLEX_UNKNOWN: NIC_DUPLEX_UNKNOWN,
+    }
+    names = net_io_counters().keys()
+    ret = {}
+    for name in names:
+        try:
+            mtu = cext.net_if_mtu(name)
+            flags = cext.net_if_flags(name)
+            duplex, speed = cext.net_if_duplex_speed(name)
+        except OSError as err:
+            # https://github.com/giampaolo/psutil/issues/1279
+            if err.errno != errno.ENODEV:
+                raise
+            debug(err)
+        else:
+            output_flags = ','.join(flags)
+            isup = 'running' in flags
+            ret[name] = ntp.snicstats(
+                isup, duplex_map[duplex], speed, mtu, output_flags
+            )
+    return ret
+
+
+# =====================================================================
+# --- disks
+# =====================================================================
+
+
+disk_usage = _psposix.disk_usage
+
+
+def disk_io_counters(perdisk=False):
+    """Return disk I/O statistics for every disk installed on the
+    system as a dict of raw tuples.
+    """
+
+    def read_procfs():
+        # OK, this is a bit confusing. The format of /proc/diskstats can
+        # have 3 variations.
+        # On Linux 2.4 each line has always 15 fields, e.g.:
+        # "3     0   8 hda 8 8 8 8 8 8 8 8 8 8 8"
+        # On Linux 2.6+ each line *usually* has 14 fields, and the disk
+        # name is in another position, like this:
+        # "3    0   hda 8 8 8 8 8 8 8 8 8 8 8"
+        # ...unless (Linux 2.6) the line refers to a partition instead
+        # of a disk, in which case the line has less fields (7):
+        # "3    1   hda1 8 8 8 8"
+        # 4.18+ has 4 fields added:
+        # "3    0   hda 8 8 8 8 8 8 8 8 8 8 8 0 0 0 0"
+        # 5.5 has 2 more fields.
+        # See:
+        # https://www.kernel.org/doc/Documentation/iostats.txt
+        # https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats
+        with open_text(f"{get_procfs_path()}/diskstats") as f:
+            lines = f.readlines()
+        for line in lines:
+            fields = line.split()
+            flen = len(fields)
+            # fmt: off
+            if flen == 15:
+                # Linux 2.4
+                name = fields[3]
+                reads = int(fields[2])
+                (reads_merged, rbytes, rtime, writes, writes_merged,
+                    wbytes, wtime, _, busy_time, _) = map(int, fields[4:14])
+            elif flen == 14 or flen >= 18:
+                # Linux 2.6+, line referring to a disk
+                name = fields[2]
+                (reads, reads_merged, rbytes, rtime, writes, writes_merged,
+                    wbytes, wtime, _, busy_time, _) = map(int, fields[3:14])
+            elif flen == 7:
+                # Linux 2.6+, line referring to a partition
+                name = fields[2]
+                reads, rbytes, writes, wbytes = map(int, fields[3:])
+                rtime = wtime = reads_merged = writes_merged = busy_time = 0
+            else:
+                msg = f"not sure how to interpret line {line!r}"
+                raise ValueError(msg)
+            yield (name, reads, writes, rbytes, wbytes, rtime, wtime,
+                   reads_merged, writes_merged, busy_time)
+            # fmt: on
+
+    def read_sysfs():
+        for block in os.listdir('/sys/block'):
+            for root, _, files in os.walk(os.path.join('/sys/block', block)):
+                if 'stat' not in files:
+                    continue
+                with open_text(os.path.join(root, 'stat')) as f:
+                    fields = f.read().strip().split()
+                name = os.path.basename(root)
+                # fmt: off
+                (reads, reads_merged, rbytes, rtime, writes, writes_merged,
+                    wbytes, wtime, _, busy_time) = map(int, fields[:10])
+                yield (name, reads, writes, rbytes, wbytes, rtime,
+                       wtime, reads_merged, writes_merged, busy_time)
+                # fmt: on
+
+    if os.path.exists(f"{get_procfs_path()}/diskstats"):
+        gen = read_procfs()
+    elif os.path.exists('/sys/block'):
+        gen = read_sysfs()
+    else:
+        msg = (
+            f"{get_procfs_path()}/diskstats nor /sys/block are available on"
+            " this system"
+        )
+        raise NotImplementedError(msg)
+
+    retdict = {}
+    for entry in gen:
+        # fmt: off
+        (name, reads, writes, rbytes, wbytes, rtime, wtime, reads_merged,
+            writes_merged, busy_time) = entry
+        if not perdisk and not is_storage_device(name):
+            # perdisk=False means we want to calculate totals so we skip
+            # partitions (e.g. 'sda1', 'nvme0n1p1') and only include
+            # base disk devices (e.g. 'sda', 'nvme0n1'). Base disks
+            # include a total of all their partitions + some extra size
+            # of their own:
+            #     $ cat /proc/diskstats
+            #     259       0 sda 10485760 ...
+            #     259       1 sda1 5186039 ...
+            #     259       1 sda2 5082039 ...
+            # See:
+            # https://github.com/giampaolo/psutil/pull/1313
+            continue
+
+        rbytes *= DISK_SECTOR_SIZE
+        wbytes *= DISK_SECTOR_SIZE
+        retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime,
+                         reads_merged, writes_merged, busy_time)
+        # fmt: on
+
+    return retdict
+
+
+class RootFsDeviceFinder:
+    """disk_partitions() may return partitions with device == "/dev/root"
+    or "rootfs". This container class uses different strategies to try to
+    obtain the real device path. Resources:
+    https://bootlin.com/blog/find-root-device/
+    https://www.systutorials.com/how-to-find-the-disk-where-root-is-on-in-bash-on-linux/.
+    """
+
+    __slots__ = ['major', 'minor']
+
+    def __init__(self):
+        dev = os.stat("/").st_dev
+        self.major = os.major(dev)
+        self.minor = os.minor(dev)
+
+    def ask_proc_partitions(self):
+        with open_text(f"{get_procfs_path()}/partitions") as f:
+            for line in f.readlines()[2:]:
+                fields = line.split()
+                if len(fields) < 4:  # just for extra safety
+                    continue
+                major = int(fields[0]) if fields[0].isdigit() else None
+                minor = int(fields[1]) if fields[1].isdigit() else None
+                name = fields[3]
+                if major == self.major and minor == self.minor:
+                    if name:  # just for extra safety
+                        return f"/dev/{name}"
+
+    def ask_sys_dev_block(self):
+        path = f"/sys/dev/block/{self.major}:{self.minor}/uevent"
+        with open_text(path) as f:
+            for line in f:
+                if line.startswith("DEVNAME="):
+                    name = line.strip().rpartition("DEVNAME=")[2]
+                    if name:  # just for extra safety
+                        return f"/dev/{name}"
+
+    def ask_sys_class_block(self):
+        needle = f"{self.major}:{self.minor}"
+        files = glob.iglob("/sys/class/block/*/dev")
+        for file in files:
+            try:
+                f = open_text(file)
+            except FileNotFoundError:  # race condition
+                continue
+            else:
+                with f:
+                    data = f.read().strip()
+                    if data == needle:
+                        name = os.path.basename(os.path.dirname(file))
+                        return f"/dev/{name}"
+
+    def find(self):
+        path = None
+        if path is None:
+            try:
+                path = self.ask_proc_partitions()
+            except OSError as err:
+                debug(err)
+        if path is None:
+            try:
+                path = self.ask_sys_dev_block()
+            except OSError as err:
+                debug(err)
+        if path is None:
+            try:
+                path = self.ask_sys_class_block()
+            except OSError as err:
+                debug(err)
+        # We use exists() because the "/dev/*" part of the path is hard
+        # coded, so we want to be sure.
+        if path is not None and os.path.exists(path):
+            return path
+
+
+def disk_partitions(all=False):
+    """Return mounted disk partitions as a list of namedtuples."""
+    fstypes = set()
+    procfs_path = get_procfs_path()
+    if not all:
+        with open_text(f"{procfs_path}/filesystems") as f:
+            for line in f:
+                line = line.strip()
+                if not line.startswith("nodev"):
+                    fstypes.add(line.strip())
+                else:
+                    # ignore all lines starting with "nodev" except "nodev zfs"
+                    fstype = line.split("\t")[1]
+                    if fstype == "zfs":
+                        fstypes.add("zfs")
+
+    # See: https://github.com/giampaolo/psutil/issues/1307
+    if procfs_path == "/proc" and os.path.isfile('/etc/mtab'):
+        mounts_path = os.path.realpath("/etc/mtab")
+    else:
+        mounts_path = os.path.realpath(f"{procfs_path}/self/mounts")
+
+    retlist = []
+    partitions = cext.disk_partitions(mounts_path)
+    for partition in partitions:
+        device, mountpoint, fstype, opts = partition
+        if device == 'none':
+            device = ''
+        if device in {"/dev/root", "rootfs"}:
+            device = RootFsDeviceFinder().find() or device
+        if not all:
+            if not device or fstype not in fstypes:
+                continue
+        ntuple = ntp.sdiskpart(device, mountpoint, fstype, opts)
+        retlist.append(ntuple)
+
+    return retlist
+
+
+# =====================================================================
+# --- sensors
+# =====================================================================
+
+
+def sensors_temperatures():
+    """Return hardware (CPU and others) temperatures as a dict
+    including hardware name, label, current, max and critical
+    temperatures.
+
+    Implementation notes:
+    - /sys/class/hwmon looks like the most recent interface to
+      retrieve this info, and this implementation relies on it
+      only (old distros will probably use something else)
+    - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon
+    - /sys/class/thermal/thermal_zone* is another one but it's more
+      difficult to parse
+    """
+    ret = collections.defaultdict(list)
+    basenames = glob.glob('/sys/class/hwmon/hwmon*/temp*_*')
+    # CentOS has an intermediate /device directory:
+    # https://github.com/giampaolo/psutil/issues/971
+    # https://github.com/nicolargo/glances/issues/1060
+    basenames.extend(glob.glob('/sys/class/hwmon/hwmon*/device/temp*_*'))
+    basenames = sorted({x.split('_')[0] for x in basenames})
+
+    # Only add the coretemp hwmon entries if they're not already in
+    # /sys/class/hwmon/
+    # https://github.com/giampaolo/psutil/issues/1708
+    # https://github.com/giampaolo/psutil/pull/1648
+    basenames2 = glob.glob(
+        '/sys/devices/platform/coretemp.*/hwmon/hwmon*/temp*_*'
+    )
+    repl = re.compile(r"/sys/devices/platform/coretemp.*/hwmon/")
+    for name in basenames2:
+        altname = repl.sub('/sys/class/hwmon/', name)
+        if altname not in basenames:
+            basenames.append(name)
+
+    for base in basenames:
+        try:
+            path = base + '_input'
+            current = float(bcat(path)) / 1000.0
+            path = os.path.join(os.path.dirname(base), 'name')
+            unit_name = cat(path).strip()
+        except (OSError, ValueError):
+            # A lot of things can go wrong here, so let's just skip the
+            # whole entry. Sure thing is Linux's /sys/class/hwmon really
+            # is a stinky broken mess.
+            # https://github.com/giampaolo/psutil/issues/1009
+            # https://github.com/giampaolo/psutil/issues/1101
+            # https://github.com/giampaolo/psutil/issues/1129
+            # https://github.com/giampaolo/psutil/issues/1245
+            # https://github.com/giampaolo/psutil/issues/1323
+            continue
+
+        high = bcat(base + '_max', fallback=None)
+        critical = bcat(base + '_crit', fallback=None)
+        label = cat(base + '_label', fallback='').strip()
+
+        if high is not None:
+            try:
+                high = float(high) / 1000.0
+            except ValueError:
+                high = None
+        if critical is not None:
+            try:
+                critical = float(critical) / 1000.0
+            except ValueError:
+                critical = None
+
+        ret[unit_name].append((label, current, high, critical))
+
+    # Indication that no sensors were detected in /sys/class/hwmon/
+    if not basenames:
+        basenames = glob.glob('/sys/class/thermal/thermal_zone*')
+        basenames = sorted(set(basenames))
+
+        for base in basenames:
+            try:
+                path = os.path.join(base, 'temp')
+                current = float(bcat(path)) / 1000.0
+                path = os.path.join(base, 'type')
+                unit_name = cat(path).strip()
+            except (OSError, ValueError) as err:
+                debug(err)
+                continue
+
+            trip_paths = glob.glob(base + '/trip_point*')
+            trip_points = {
+                '_'.join(os.path.basename(p).split('_')[0:3])
+                for p in trip_paths
+            }
+            critical = None
+            high = None
+            for trip_point in trip_points:
+                path = os.path.join(base, trip_point + "_type")
+                trip_type = cat(path, fallback='').strip()
+                if trip_type == 'critical':
+                    critical = bcat(
+                        os.path.join(base, trip_point + "_temp"), fallback=None
+                    )
+                elif trip_type == 'high':
+                    high = bcat(
+                        os.path.join(base, trip_point + "_temp"), fallback=None
+                    )
+
+                if high is not None:
+                    try:
+                        high = float(high) / 1000.0
+                    except ValueError:
+                        high = None
+                if critical is not None:
+                    try:
+                        critical = float(critical) / 1000.0
+                    except ValueError:
+                        critical = None
+
+            ret[unit_name].append(('', current, high, critical))
+
+    return dict(ret)
+
+
+def sensors_fans():
+    """Return hardware fans info (for CPU and other peripherals) as a
+    dict including hardware label and current speed.
+
+    Implementation notes:
+    - /sys/class/hwmon looks like the most recent interface to
+      retrieve this info, and this implementation relies on it
+      only (old distros will probably use something else)
+    - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon
+    """
+    ret = collections.defaultdict(list)
+    basenames = glob.glob('/sys/class/hwmon/hwmon*/fan*_*')
+    if not basenames:
+        # CentOS has an intermediate /device directory:
+        # https://github.com/giampaolo/psutil/issues/971
+        basenames = glob.glob('/sys/class/hwmon/hwmon*/device/fan*_*')
+
+    basenames = sorted({x.split("_")[0] for x in basenames})
+    for base in basenames:
+        try:
+            current = int(bcat(base + '_input'))
+        except OSError as err:
+            debug(err)
+            continue
+        unit_name = cat(os.path.join(os.path.dirname(base), 'name')).strip()
+        label = cat(base + '_label', fallback='').strip()
+        ret[unit_name].append(ntp.sfan(label, current))
+
+    return dict(ret)
+
+
+def sensors_battery():
+    """Return battery information.
+    Implementation note: it appears /sys/class/power_supply/BAT0/
+    directory structure may vary and provide files with the same
+    meaning but under different names, see:
+    https://github.com/giampaolo/psutil/issues/966.
+    """
+    null = object()
+
+    def multi_bcat(*paths):
+        """Attempt to read the content of multiple files which may
+        not exist. If none of them exist return None.
+        """
+        for path in paths:
+            ret = bcat(path, fallback=null)
+            if ret != null:
+                try:
+                    return int(ret)
+                except ValueError:
+                    return ret.strip()
+        return None
+
+    bats = [
+        x
+        for x in os.listdir(POWER_SUPPLY_PATH)
+        if x.startswith('BAT') or 'battery' in x.lower()
+    ]
+    if not bats:
+        return None
+    # Get the first available battery. Usually this is "BAT0", except
+    # some rare exceptions:
+    # https://github.com/giampaolo/psutil/issues/1238
+    root = os.path.join(POWER_SUPPLY_PATH, min(bats))
+
+    # Base metrics.
+    energy_now = multi_bcat(root + "/energy_now", root + "/charge_now")
+    power_now = multi_bcat(root + "/power_now", root + "/current_now")
+    energy_full = multi_bcat(root + "/energy_full", root + "/charge_full")
+    time_to_empty = multi_bcat(root + "/time_to_empty_now")
+
+    # Percent. If we have energy_full the percentage will be more
+    # accurate compared to reading /capacity file (float vs. int).
+    if energy_full is not None and energy_now is not None:
+        try:
+            percent = 100.0 * energy_now / energy_full
+        except ZeroDivisionError:
+            percent = 0.0
+    else:
+        percent = int(cat(root + "/capacity", fallback=-1))
+        if percent == -1:
+            return None
+
+    # Is AC power cable plugged in?
+    # Note: AC0 is not always available and sometimes (e.g. CentOS7)
+    # it's called "AC".
+    power_plugged = None
+    online = multi_bcat(
+        os.path.join(POWER_SUPPLY_PATH, "AC0/online"),
+        os.path.join(POWER_SUPPLY_PATH, "AC/online"),
+    )
+    if online is not None:
+        power_plugged = online == 1
+    else:
+        status = cat(root + "/status", fallback="").strip().lower()
+        if status == "discharging":
+            power_plugged = False
+        elif status in {"charging", "full"}:
+            power_plugged = True
+
+    # Seconds left.
+    # Note to self: we may also calculate the charging ETA as per:
+    # https://github.com/thialfihar/dotfiles/blob/
+    #     013937745fd9050c30146290e8f963d65c0179e6/bin/battery.py#L55
+    if power_plugged:
+        secsleft = _common.POWER_TIME_UNLIMITED
+    elif energy_now is not None and power_now is not None:
+        try:
+            secsleft = int(energy_now / abs(power_now) * 3600)
+        except ZeroDivisionError:
+            secsleft = _common.POWER_TIME_UNKNOWN
+    elif time_to_empty is not None:
+        secsleft = int(time_to_empty * 60)
+        if secsleft < 0:
+            secsleft = _common.POWER_TIME_UNKNOWN
+    else:
+        secsleft = _common.POWER_TIME_UNKNOWN
+
+    return ntp.sbattery(percent, secsleft, power_plugged)
+
+
+# =====================================================================
+# --- other system functions
+# =====================================================================
+
+
+def users():
+    """Return currently connected users as a list of namedtuples."""
+    retlist = []
+    rawlist = cext.users()
+    for item in rawlist:
+        user, tty, hostname, tstamp, pid = item
+        nt = ntp.suser(user, tty or None, hostname, tstamp, pid)
+        retlist.append(nt)
+    return retlist
+
+
+def boot_time():
+    """Return the system boot time expressed in seconds since the epoch."""
+    path = f"{get_procfs_path()}/stat"
+    with open_binary(path) as f:
+        for line in f:
+            if line.startswith(b'btime'):
+                return float(line.strip().split()[1])
+        msg = f"line 'btime' not found in {path}"
+        raise RuntimeError(msg)
+
+
+# =====================================================================
+# --- processes
+# =====================================================================
+
+
+def pids():
+    """Returns a list of PIDs currently running on the system."""
+    path = get_procfs_path().encode(ENCODING)
+    return [int(x) for x in os.listdir(path) if x.isdigit()]
+
+
+def pid_exists(pid):
+    """Check for the existence of a unix PID. Linux TIDs are not
+    supported (always return False).
+    """
+    if not _psposix.pid_exists(pid):
+        return False
+    else:
+        # Linux's apparently does not distinguish between PIDs and TIDs
+        # (thread IDs).
+        # listdir("/proc") won't show any TID (only PIDs) but
+        # os.stat("/proc/{tid}") will succeed if {tid} exists.
+        # os.kill() can also be passed a TID. This is quite confusing.
+        # In here we want to enforce this distinction and support PIDs
+        # only, see:
+        # https://github.com/giampaolo/psutil/issues/687
+        try:
+            # Note: already checked that this is faster than using a
+            # regular expr. Also (a lot) faster than doing
+            # 'return pid in pids()'
+            path = f"{get_procfs_path()}/{pid}/status"
+            with open_binary(path) as f:
+                for line in f:
+                    if line.startswith(b"Tgid:"):
+                        tgid = int(line.split()[1])
+                        # If tgid and pid are the same then we're
+                        # dealing with a process PID.
+                        return tgid == pid
+                msg = f"'Tgid' line not found in {path}"
+                raise ValueError(msg)
+        except (OSError, ValueError):
+            return pid in pids()
+
+
+def ppid_map():
+    """Obtain a {pid: ppid, ...} dict for all running processes in
+    one shot. Used to speed up Process.children().
+    """
+    ret = {}
+    procfs_path = get_procfs_path()
+    for pid in pids():
+        try:
+            with open_binary(f"{procfs_path}/{pid}/stat") as f:
+                data = f.read()
+        except (FileNotFoundError, ProcessLookupError):
+            pass
+        except PermissionError as err:
+            raise AccessDenied(pid) from err
+        else:
+            rpar = data.rfind(b')')
+            dset = data[rpar + 2 :].split()
+            ppid = int(dset[1])
+            ret[pid] = ppid
+    return ret
+
+
+def wrap_exceptions(fun):
+    """Decorator which translates bare OSError exceptions into
+    NoSuchProcess and AccessDenied.
+    """
+
+    @functools.wraps(fun)
+    def wrapper(self, *args, **kwargs):
+        pid, name = self.pid, self._name
+        try:
+            return fun(self, *args, **kwargs)
+        except PermissionError as err:
+            raise AccessDenied(pid, name) from err
+        except ProcessLookupError as err:
+            self._raise_if_zombie()
+            raise NoSuchProcess(pid, name) from err
+        except FileNotFoundError as err:
+            self._raise_if_zombie()
+            # /proc/PID directory may still exist, but the files within
+            # it may not, indicating the process is gone, see:
+            # https://github.com/giampaolo/psutil/issues/2418
+            if not os.path.exists(f"{self._procfs_path}/{pid}/stat"):
+                raise NoSuchProcess(pid, name) from err
+            raise
+
+    return wrapper
+
+
+class Process:
+    """Linux process implementation."""
+
+    __slots__ = [
+        "_cache",
+        "_ctime",
+        "_name",
+        "_ppid",
+        "_procfs_path",
+        "pid",
+    ]
+
+    def __init__(self, pid):
+        self.pid = pid
+        self._name = None
+        self._ppid = None
+        self._ctime = None
+        self._procfs_path = get_procfs_path()
+
+    def _is_zombie(self):
+        # Note: most of the times Linux is able to return info about the
+        # process even if it's a zombie, and /proc/{pid} will exist.
+        # There are some exceptions though, like exe(), cmdline() and
+        # memory_maps(). In these cases /proc/{pid}/{file} exists but
+        # it's empty. Instead of returning a "null" value we'll raise an
+        # exception.
+        try:
+            data = bcat(f"{self._procfs_path}/{self.pid}/stat")
+        except OSError:
+            return False
+        else:
+            rpar = data.rfind(b')')
+            status = data[rpar + 2 : rpar + 3]
+            return status == b"Z"
+
+    def _raise_if_zombie(self):
+        if self._is_zombie():
+            raise ZombieProcess(self.pid, self._name, self._ppid)
+
+    def _raise_if_not_alive(self):
+        """Raise NSP if the process disappeared on us."""
+        # For those C function who do not raise NSP, possibly returning
+        # incorrect or incomplete result.
+        os.stat(f"{self._procfs_path}/{self.pid}")
+
+    def _readlink(self, path, fallback=UNSET):
+        # * https://github.com/giampaolo/psutil/issues/503
+        #   os.readlink('/proc/pid/exe') may raise ESRCH (ProcessLookupError)
+        #   instead of ENOENT (FileNotFoundError) when it races.
+        # * ENOENT may occur also if the path actually exists if PID is
+        #   a low PID (~0-20 range).
+        # * https://github.com/giampaolo/psutil/issues/2514
+        try:
+            return readlink(path)
+        except (FileNotFoundError, ProcessLookupError):
+            if os.path.lexists(f"{self._procfs_path}/{self.pid}"):
+                self._raise_if_zombie()
+                if fallback is not UNSET:
+                    return fallback
+            raise
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def _parse_stat_file(self):
+        """Parse /proc/{pid}/stat file and return a dict with various
+        process info.
+        Using "man proc" as a reference: where "man proc" refers to
+        position N always subtract 3 (e.g ppid position 4 in
+        'man proc' == position 1 in here).
+        The return value is cached in case oneshot() ctx manager is
+        in use.
+        """
+        data = bcat(f"{self._procfs_path}/{self.pid}/stat")
+        # Process name is between parentheses. It can contain spaces and
+        # other parentheses. This is taken into account by looking for
+        # the first occurrence of "(" and the last occurrence of ")".
+        rpar = data.rfind(b')')
+        name = data[data.find(b'(') + 1 : rpar]
+        fields = data[rpar + 2 :].split()
+
+        ret = {}
+        ret['name'] = name
+        ret['status'] = fields[0]
+        ret['ppid'] = fields[1]
+        ret['ttynr'] = fields[4]
+        ret['utime'] = fields[11]
+        ret['stime'] = fields[12]
+        ret['children_utime'] = fields[13]
+        ret['children_stime'] = fields[14]
+        ret['create_time'] = fields[19]
+        ret['cpu_num'] = fields[36]
+        try:
+            ret['blkio_ticks'] = fields[39]  # aka 'delayacct_blkio_ticks'
+        except IndexError:
+            # https://github.com/giampaolo/psutil/issues/2455
+            debug("can't get blkio_ticks, set iowait to 0")
+            ret['blkio_ticks'] = 0
+
+        return ret
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def _read_status_file(self):
+        """Read /proc/{pid}/stat file and return its content.
+        The return value is cached in case oneshot() ctx manager is
+        in use.
+        """
+        with open_binary(f"{self._procfs_path}/{self.pid}/status") as f:
+            return f.read()
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def _read_smaps_file(self):
+        with open_binary(f"{self._procfs_path}/{self.pid}/smaps") as f:
+            return f.read().strip()
+
+    def oneshot_enter(self):
+        self._parse_stat_file.cache_activate(self)
+        self._read_status_file.cache_activate(self)
+        self._read_smaps_file.cache_activate(self)
+
+    def oneshot_exit(self):
+        self._parse_stat_file.cache_deactivate(self)
+        self._read_status_file.cache_deactivate(self)
+        self._read_smaps_file.cache_deactivate(self)
+
+    @wrap_exceptions
+    def name(self):
+        # XXX - gets changed later and probably needs refactoring
+        return decode(self._parse_stat_file()['name'])
+
+    @wrap_exceptions
+    def exe(self):
+        return self._readlink(
+            f"{self._procfs_path}/{self.pid}/exe", fallback=""
+        )
+
+    @wrap_exceptions
+    def cmdline(self):
+        with open_text(f"{self._procfs_path}/{self.pid}/cmdline") as f:
+            data = f.read()
+        if not data:
+            # may happen in case of zombie process
+            self._raise_if_zombie()
+            return []
+        # 'man proc' states that args are separated by null bytes '\0'
+        # and last char is supposed to be a null byte. Nevertheless
+        # some processes may change their cmdline after being started
+        # (via setproctitle() or similar), they are usually not
+        # compliant with this rule and use spaces instead. Google
+        # Chrome process is an example. See:
+        # https://github.com/giampaolo/psutil/issues/1179
+        sep = '\x00' if data.endswith('\x00') else ' '
+        if data.endswith(sep):
+            data = data[:-1]
+        cmdline = data.split(sep)
+        # Sometimes last char is a null byte '\0' but the args are
+        # separated by spaces, see: https://github.com/giampaolo/psutil/
+        # issues/1179#issuecomment-552984549
+        if sep == '\x00' and len(cmdline) == 1 and ' ' in data:
+            cmdline = data.split(' ')
+        return cmdline
+
+    @wrap_exceptions
+    def environ(self):
+        with open_text(f"{self._procfs_path}/{self.pid}/environ") as f:
+            data = f.read()
+        return parse_environ_block(data)
+
+    @wrap_exceptions
+    def terminal(self):
+        tty_nr = int(self._parse_stat_file()['ttynr'])
+        tmap = _psposix.get_terminal_map()
+        try:
+            return tmap[tty_nr]
+        except KeyError:
+            return None
+
+    # May not be available on old kernels.
+    if os.path.exists(f"/proc/{os.getpid()}/io"):
+
+        @wrap_exceptions
+        def io_counters(self):
+            fname = f"{self._procfs_path}/{self.pid}/io"
+            fields = {}
+            with open_binary(fname) as f:
+                for line in f:
+                    # https://github.com/giampaolo/psutil/issues/1004
+                    line = line.strip()
+                    if line:
+                        try:
+                            name, value = line.split(b': ')
+                        except ValueError:
+                            # https://github.com/giampaolo/psutil/issues/1004
+                            continue
+                        else:
+                            fields[name] = int(value)
+            if not fields:
+                msg = f"{fname} file was empty"
+                raise RuntimeError(msg)
+            try:
+                return ntp.pio(
+                    fields[b'syscr'],  # read syscalls
+                    fields[b'syscw'],  # write syscalls
+                    fields[b'read_bytes'],  # read bytes
+                    fields[b'write_bytes'],  # write bytes
+                    fields[b'rchar'],  # read chars
+                    fields[b'wchar'],  # write chars
+                )
+            except KeyError as err:
+                msg = (
+                    f"{err.args[0]!r} field was not found in {fname}; found"
+                    f" fields are {fields!r}"
+                )
+                raise ValueError(msg) from None
+
+    @wrap_exceptions
+    def cpu_times(self):
+        values = self._parse_stat_file()
+        utime = float(values['utime']) / CLOCK_TICKS
+        stime = float(values['stime']) / CLOCK_TICKS
+        children_utime = float(values['children_utime']) / CLOCK_TICKS
+        children_stime = float(values['children_stime']) / CLOCK_TICKS
+        iowait = float(values['blkio_ticks']) / CLOCK_TICKS
+        return ntp.pcputimes(
+            utime, stime, children_utime, children_stime, iowait
+        )
+
+    @wrap_exceptions
+    def cpu_num(self):
+        """What CPU the process is on."""
+        return int(self._parse_stat_file()['cpu_num'])
+
+    @wrap_exceptions
+    def wait(self, timeout=None):
+        return _psposix.wait_pid(self.pid, timeout)
+
+    @wrap_exceptions
+    def create_time(self, monotonic=False):
+        # The 'starttime' field in /proc/[pid]/stat is expressed in
+        # jiffies (clock ticks per second), a relative value which
+        # represents the number of clock ticks that have passed since
+        # the system booted until the process was created. It never
+        # changes and is unaffected by system clock updates.
+        if self._ctime is None:
+            self._ctime = (
+                float(self._parse_stat_file()['create_time']) / CLOCK_TICKS
+            )
+        if monotonic:
+            return self._ctime
+        # Add the boot time, returning time expressed in seconds since
+        # the epoch. This is subject to system clock updates.
+        return self._ctime + boot_time()
+
+    @wrap_exceptions
+    def memory_info(self):
+        #  ============================================================
+        # | FIELD  | DESCRIPTION                         | AKA  | TOP  |
+        #  ============================================================
+        # | rss    | resident set size                   |      | RES  |
+        # | vms    | total program size                  | size | VIRT |
+        # | shared | shared pages (from shared mappings) |      | SHR  |
+        # | text   | text ('code')                       | trs  | CODE |
+        # | lib    | library (unused in Linux 2.6)       | lrs  |      |
+        # | data   | data + stack                        | drs  | DATA |
+        # | dirty  | dirty pages (unused in Linux 2.6)   | dt   |      |
+        #  ============================================================
+        with open_binary(f"{self._procfs_path}/{self.pid}/statm") as f:
+            vms, rss, shared, text, lib, data, dirty = (
+                int(x) * PAGESIZE for x in f.readline().split()[:7]
+            )
+        return ntp.pmem(rss, vms, shared, text, lib, data, dirty)
+
+    if HAS_PROC_SMAPS_ROLLUP or HAS_PROC_SMAPS:
+
+        def _parse_smaps_rollup(self):
+            # /proc/pid/smaps_rollup was added to Linux in 2017. Faster
+            # than /proc/pid/smaps. It reports higher PSS than */smaps
+            # (from 1k up to 200k higher; tested against all processes).
+            # IMPORTANT: /proc/pid/smaps_rollup is weird, because it
+            # raises ESRCH / ENOENT for many PIDs, even if they're alive
+            # (also as root). In that case we'll use /proc/pid/smaps as
+            # fallback, which is slower but has a +50% success rate
+            # compared to /proc/pid/smaps_rollup.
+            uss = pss = swap = 0
+            with open_binary(
+                f"{self._procfs_path}/{self.pid}/smaps_rollup"
+            ) as f:
+                for line in f:
+                    if line.startswith(b"Private_"):
+                        # Private_Clean, Private_Dirty, Private_Hugetlb
+                        uss += int(line.split()[1]) * 1024
+                    elif line.startswith(b"Pss:"):
+                        pss = int(line.split()[1]) * 1024
+                    elif line.startswith(b"Swap:"):
+                        swap = int(line.split()[1]) * 1024
+            return (uss, pss, swap)
+
+        @wrap_exceptions
+        def _parse_smaps(
+            self,
+            # Gets Private_Clean, Private_Dirty, Private_Hugetlb.
+            _private_re=re.compile(br"\nPrivate.*:\s+(\d+)"),
+            _pss_re=re.compile(br"\nPss\:\s+(\d+)"),
+            _swap_re=re.compile(br"\nSwap\:\s+(\d+)"),
+        ):
+            # /proc/pid/smaps does not exist on kernels < 2.6.14 or if
+            # CONFIG_MMU kernel configuration option is not enabled.
+
+            # Note: using 3 regexes is faster than reading the file
+            # line by line.
+            #
+            # You might be tempted to calculate USS by subtracting
+            # the "shared" value from the "resident" value in
+            # /proc/<pid>/statm. But at least on Linux, statm's "shared"
+            # value actually counts pages backed by files, which has
+            # little to do with whether the pages are actually shared.
+            # /proc/self/smaps on the other hand appears to give us the
+            # correct information.
+            smaps_data = self._read_smaps_file()
+            # Note: smaps file can be empty for certain processes.
+            # The code below will not crash though and will result to 0.
+            uss = sum(map(int, _private_re.findall(smaps_data))) * 1024
+            pss = sum(map(int, _pss_re.findall(smaps_data))) * 1024
+            swap = sum(map(int, _swap_re.findall(smaps_data))) * 1024
+            return (uss, pss, swap)
+
+        @wrap_exceptions
+        def memory_full_info(self):
+            if HAS_PROC_SMAPS_ROLLUP:  # faster
+                try:
+                    uss, pss, swap = self._parse_smaps_rollup()
+                except (ProcessLookupError, FileNotFoundError):
+                    uss, pss, swap = self._parse_smaps()
+            else:
+                uss, pss, swap = self._parse_smaps()
+            basic_mem = self.memory_info()
+            return ntp.pfullmem(*basic_mem + (uss, pss, swap))
+
+    else:
+        memory_full_info = memory_info
+
+    if HAS_PROC_SMAPS:
+
+        @wrap_exceptions
+        def memory_maps(self):
+            """Return process's mapped memory regions as a list of named
+            tuples. Fields are explained in 'man proc'; here is an updated
+            (Apr 2012) version: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/proc.txt?id=b76437579d1344b612cf1851ae610c636cec7db0.
+
+            /proc/{PID}/smaps does not exist on kernels < 2.6.14 or if
+            CONFIG_MMU kernel configuration option is not enabled.
+            """
+
+            def get_blocks(lines, current_block):
+                data = {}
+                for line in lines:
+                    fields = line.split(None, 5)
+                    if not fields[0].endswith(b':'):
+                        # new block section
+                        yield (current_block.pop(), data)
+                        current_block.append(line)
+                    else:
+                        try:
+                            data[fields[0]] = int(fields[1]) * 1024
+                        except (ValueError, IndexError):
+                            if fields[0].startswith(b'VmFlags:'):
+                                # see issue #369
+                                continue
+                            msg = f"don't know how to interpret line {line!r}"
+                            raise ValueError(msg) from None
+                yield (current_block.pop(), data)
+
+            data = self._read_smaps_file()
+            # Note: smaps file can be empty for certain processes or for
+            # zombies.
+            if not data:
+                self._raise_if_zombie()
+                return []
+            lines = data.split(b'\n')
+            ls = []
+            first_line = lines.pop(0)
+            current_block = [first_line]
+            for header, data in get_blocks(lines, current_block):
+                hfields = header.split(None, 5)
+                try:
+                    addr, perms, _offset, _dev, _inode, path = hfields
+                except ValueError:
+                    addr, perms, _offset, _dev, _inode, path = hfields + ['']
+                if not path:
+                    path = '[anon]'
+                else:
+                    path = decode(path)
+                    path = path.strip()
+                    if path.endswith(' (deleted)') and not path_exists_strict(
+                        path
+                    ):
+                        path = path[:-10]
+                item = (
+                    decode(addr),
+                    decode(perms),
+                    path,
+                    data.get(b'Rss:', 0),
+                    data.get(b'Size:', 0),
+                    data.get(b'Pss:', 0),
+                    data.get(b'Shared_Clean:', 0),
+                    data.get(b'Shared_Dirty:', 0),
+                    data.get(b'Private_Clean:', 0),
+                    data.get(b'Private_Dirty:', 0),
+                    data.get(b'Referenced:', 0),
+                    data.get(b'Anonymous:', 0),
+                    data.get(b'Swap:', 0),
+                )
+                ls.append(item)
+            return ls
+
+    @wrap_exceptions
+    def cwd(self):
+        return self._readlink(
+            f"{self._procfs_path}/{self.pid}/cwd", fallback=""
+        )
+
+    @wrap_exceptions
+    def num_ctx_switches(
+        self, _ctxsw_re=re.compile(br'ctxt_switches:\t(\d+)')
+    ):
+        data = self._read_status_file()
+        ctxsw = _ctxsw_re.findall(data)
+        if not ctxsw:
+            msg = (
+                "'voluntary_ctxt_switches' and"
+                " 'nonvoluntary_ctxt_switches'lines were not found in"
+                f" {self._procfs_path}/{self.pid}/status; the kernel is"
+                " probably older than 2.6.23"
+            )
+            raise NotImplementedError(msg)
+        return ntp.pctxsw(int(ctxsw[0]), int(ctxsw[1]))
+
+    @wrap_exceptions
+    def num_threads(self, _num_threads_re=re.compile(br'Threads:\t(\d+)')):
+        # Using a re is faster than iterating over file line by line.
+        data = self._read_status_file()
+        return int(_num_threads_re.findall(data)[0])
+
+    @wrap_exceptions
+    def threads(self):
+        thread_ids = os.listdir(f"{self._procfs_path}/{self.pid}/task")
+        thread_ids.sort()
+        retlist = []
+        hit_enoent = False
+        for thread_id in thread_ids:
+            fname = f"{self._procfs_path}/{self.pid}/task/{thread_id}/stat"
+            try:
+                with open_binary(fname) as f:
+                    st = f.read().strip()
+            except (FileNotFoundError, ProcessLookupError):
+                # no such file or directory or no such process;
+                # it means thread disappeared on us
+                hit_enoent = True
+                continue
+            # ignore the first two values ("pid (exe)")
+            st = st[st.find(b')') + 2 :]
+            values = st.split(b' ')
+            utime = float(values[11]) / CLOCK_TICKS
+            stime = float(values[12]) / CLOCK_TICKS
+            ntuple = ntp.pthread(int(thread_id), utime, stime)
+            retlist.append(ntuple)
+        if hit_enoent:
+            self._raise_if_not_alive()
+        return retlist
+
+    @wrap_exceptions
+    def nice_get(self):
+        # with open_text(f"{self._procfs_path}/{self.pid}/stat") as f:
+        #   data = f.read()
+        #   return int(data.split()[18])
+
+        # Use C implementation
+        return cext.proc_priority_get(self.pid)
+
+    @wrap_exceptions
+    def nice_set(self, value):
+        return cext.proc_priority_set(self.pid, value)
+
+    # starting from CentOS 6.
+    if HAS_CPU_AFFINITY:
+
+        @wrap_exceptions
+        def cpu_affinity_get(self):
+            return cext.proc_cpu_affinity_get(self.pid)
+
+        def _get_eligible_cpus(
+            self, _re=re.compile(br"Cpus_allowed_list:\t(\d+)-(\d+)")
+        ):
+            # See: https://github.com/giampaolo/psutil/issues/956
+            data = self._read_status_file()
+            match = _re.findall(data)
+            if match:
+                return list(range(int(match[0][0]), int(match[0][1]) + 1))
+            else:
+                return list(range(len(per_cpu_times())))
+
+        @wrap_exceptions
+        def cpu_affinity_set(self, cpus):
+            try:
+                cext.proc_cpu_affinity_set(self.pid, cpus)
+            except (OSError, ValueError) as err:
+                if isinstance(err, ValueError) or err.errno == errno.EINVAL:
+                    eligible_cpus = self._get_eligible_cpus()
+                    all_cpus = tuple(range(len(per_cpu_times())))
+                    for cpu in cpus:
+                        if cpu not in all_cpus:
+                            msg = (
+                                f"invalid CPU {cpu!r}; choose between"
+                                f" {eligible_cpus!r}"
+                            )
+                            raise ValueError(msg) from None
+                        if cpu not in eligible_cpus:
+                            msg = (
+                                f"CPU number {cpu} is not eligible; choose"
+                                f" between {eligible_cpus}"
+                            )
+                            raise ValueError(msg) from err
+                raise
+
+    # only starting from kernel 2.6.13
+    if HAS_PROC_IO_PRIORITY:
+
+        @wrap_exceptions
+        def ionice_get(self):
+            ioclass, value = cext.proc_ioprio_get(self.pid)
+            ioclass = IOPriority(ioclass)
+            return ntp.pionice(ioclass, value)
+
+        @wrap_exceptions
+        def ionice_set(self, ioclass, value):
+            if value is None:
+                value = 0
+            if value and ioclass in {
+                IOPriority.IOPRIO_CLASS_IDLE,
+                IOPriority.IOPRIO_CLASS_NONE,
+            }:
+                msg = f"{ioclass!r} ioclass accepts no value"
+                raise ValueError(msg)
+            if value < 0 or value > 7:
+                msg = "value not in 0-7 range"
+                raise ValueError(msg)
+            return cext.proc_ioprio_set(self.pid, ioclass, value)
+
+    if hasattr(resource, "prlimit"):
+
+        @wrap_exceptions
+        def rlimit(self, resource_, limits=None):
+            # If pid is 0 prlimit() applies to the calling process and
+            # we don't want that. We should never get here though as
+            # PID 0 is not supported on Linux.
+            if self.pid == 0:
+                msg = "can't use prlimit() against PID 0 process"
+                raise ValueError(msg)
+            try:
+                if limits is None:
+                    # get
+                    return resource.prlimit(self.pid, resource_)
+                else:
+                    # set
+                    if len(limits) != 2:
+                        msg = (
+                            "second argument must be a (soft, hard) "
+                            f"tuple, got {limits!r}"
+                        )
+                        raise ValueError(msg)
+                    resource.prlimit(self.pid, resource_, limits)
+            except OSError as err:
+                if err.errno == errno.ENOSYS:
+                    # I saw this happening on Travis:
+                    # https://travis-ci.org/giampaolo/psutil/jobs/51368273
+                    self._raise_if_zombie()
+                raise
+
+    @wrap_exceptions
+    def status(self):
+        letter = self._parse_stat_file()['status']
+        letter = letter.decode()
+        # XXX is '?' legit? (we're not supposed to return it anyway)
+        return PROC_STATUSES.get(letter, '?')
+
+    @wrap_exceptions
+    def open_files(self):
+        retlist = []
+        files = os.listdir(f"{self._procfs_path}/{self.pid}/fd")
+        hit_enoent = False
+        for fd in files:
+            file = f"{self._procfs_path}/{self.pid}/fd/{fd}"
+            try:
+                path = readlink(file)
+            except (FileNotFoundError, ProcessLookupError):
+                # ENOENT == file which is gone in the meantime
+                hit_enoent = True
+                continue
+            except OSError as err:
+                if err.errno == errno.EINVAL:
+                    # not a link
+                    continue
+                if err.errno == errno.ENAMETOOLONG:
+                    # file name too long
+                    debug(err)
+                    continue
+                raise
+            else:
+                # If path is not an absolute there's no way to tell
+                # whether it's a regular file or not, so we skip it.
+                # A regular file is always supposed to be have an
+                # absolute path though.
+                if path.startswith('/') and isfile_strict(path):
+                    # Get file position and flags.
+                    file = f"{self._procfs_path}/{self.pid}/fdinfo/{fd}"
+                    try:
+                        with open_binary(file) as f:
+                            pos = int(f.readline().split()[1])
+                            flags = int(f.readline().split()[1], 8)
+                    except (FileNotFoundError, ProcessLookupError):
+                        # fd gone in the meantime; process may
+                        # still be alive
+                        hit_enoent = True
+                    else:
+                        mode = file_flags_to_mode(flags)
+                        ntuple = ntp.popenfile(
+                            path, int(fd), int(pos), mode, flags
+                        )
+                        retlist.append(ntuple)
+        if hit_enoent:
+            self._raise_if_not_alive()
+        return retlist
+
+    @wrap_exceptions
+    def net_connections(self, kind='inet'):
+        ret = _net_connections.retrieve(kind, self.pid)
+        self._raise_if_not_alive()
+        return ret
+
+    @wrap_exceptions
+    def num_fds(self):
+        return len(os.listdir(f"{self._procfs_path}/{self.pid}/fd"))
+
+    @wrap_exceptions
+    def ppid(self):
+        return int(self._parse_stat_file()['ppid'])
+
+    @wrap_exceptions
+    def uids(self, _uids_re=re.compile(br'Uid:\t(\d+)\t(\d+)\t(\d+)')):
+        data = self._read_status_file()
+        real, effective, saved = _uids_re.findall(data)[0]
+        return ntp.puids(int(real), int(effective), int(saved))
+
+    @wrap_exceptions
+    def gids(self, _gids_re=re.compile(br'Gid:\t(\d+)\t(\d+)\t(\d+)')):
+        data = self._read_status_file()
+        real, effective, saved = _gids_re.findall(data)[0]
+        return ntp.pgids(int(real), int(effective), int(saved))

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/psutil/_psosx.py

@@ -0,0 +1,549 @@
+# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""macOS platform implementation."""
+
+import errno
+import functools
+import os
+
+from . import _common
+from . import _ntuples as ntp
+from . import _psposix
+from . import _psutil_osx as cext
+from ._common import AccessDenied
+from ._common import NoSuchProcess
+from ._common import ZombieProcess
+from ._common import conn_tmap
+from ._common import conn_to_ntuple
+from ._common import debug
+from ._common import isfile_strict
+from ._common import memoize_when_activated
+from ._common import parse_environ_block
+from ._common import usage_percent
+
+__extra__all__ = []
+
+
+# =====================================================================
+# --- globals
+# =====================================================================
+
+
+PAGESIZE = cext.getpagesize()
+AF_LINK = cext.AF_LINK
+
+TCP_STATUSES = {
+    cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED,
+    cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT,
+    cext.TCPS_SYN_RECEIVED: _common.CONN_SYN_RECV,
+    cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1,
+    cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2,
+    cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT,
+    cext.TCPS_CLOSED: _common.CONN_CLOSE,
+    cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT,
+    cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK,
+    cext.TCPS_LISTEN: _common.CONN_LISTEN,
+    cext.TCPS_CLOSING: _common.CONN_CLOSING,
+    cext.PSUTIL_CONN_NONE: _common.CONN_NONE,
+}
+
+PROC_STATUSES = {
+    cext.SIDL: _common.STATUS_IDLE,
+    cext.SRUN: _common.STATUS_RUNNING,
+    cext.SSLEEP: _common.STATUS_SLEEPING,
+    cext.SSTOP: _common.STATUS_STOPPED,
+    cext.SZOMB: _common.STATUS_ZOMBIE,
+}
+
+kinfo_proc_map = dict(
+    ppid=0,
+    ruid=1,
+    euid=2,
+    suid=3,
+    rgid=4,
+    egid=5,
+    sgid=6,
+    ttynr=7,
+    ctime=8,
+    status=9,
+    name=10,
+)
+
+pidtaskinfo_map = dict(
+    cpuutime=0,
+    cpustime=1,
+    rss=2,
+    vms=3,
+    pfaults=4,
+    pageins=5,
+    numthreads=6,
+    volctxsw=7,
+)
+
+
+# =====================================================================
+# --- memory
+# =====================================================================
+
+
+def virtual_memory():
+    """System virtual memory as a namedtuple."""
+    total, active, inactive, wired, free, speculative = cext.virtual_mem()
+    # This is how Zabbix calculate avail and used mem:
+    # https://github.com/zabbix/zabbix/blob/master/src/libs/zbxsysinfo/osx/memory.c
+    # Also see: https://github.com/giampaolo/psutil/issues/1277
+    avail = inactive + free
+    used = active + wired
+    # This is NOT how Zabbix calculates free mem but it matches "free"
+    # cmdline utility.
+    free -= speculative
+    percent = usage_percent((total - avail), total, round_=1)
+    return ntp.svmem(
+        total, avail, percent, used, free, active, inactive, wired
+    )
+
+
+def swap_memory():
+    """Swap system memory as a (total, used, free, sin, sout) tuple."""
+    total, used, free, sin, sout = cext.swap_mem()
+    percent = usage_percent(used, total, round_=1)
+    return ntp.sswap(total, used, free, percent, sin, sout)
+
+
+# malloc / heap functions
+heap_info = cext.heap_info
+heap_trim = cext.heap_trim
+
+
+# =====================================================================
+# --- CPU
+# =====================================================================
+
+
+def cpu_times():
+    """Return system CPU times as a namedtuple."""
+    user, nice, system, idle = cext.cpu_times()
+    return ntp.scputimes(user, nice, system, idle)
+
+
+def per_cpu_times():
+    """Return system CPU times as a named tuple."""
+    ret = []
+    for cpu_t in cext.per_cpu_times():
+        user, nice, system, idle = cpu_t
+        item = ntp.scputimes(user, nice, system, idle)
+        ret.append(item)
+    return ret
+
+
+def cpu_count_logical():
+    """Return the number of logical CPUs in the system."""
+    return cext.cpu_count_logical()
+
+
+def cpu_count_cores():
+    """Return the number of CPU cores in the system."""
+    return cext.cpu_count_cores()
+
+
+def cpu_stats():
+    ctx_switches, interrupts, soft_interrupts, syscalls, _traps = (
+        cext.cpu_stats()
+    )
+    return ntp.scpustats(ctx_switches, interrupts, soft_interrupts, syscalls)
+
+
+if cext.has_cpu_freq():  # not always available on ARM64
+
+    def cpu_freq():
+        """Return CPU frequency.
+        On macOS per-cpu frequency is not supported.
+        Also, the returned frequency never changes, see:
+        https://arstechnica.com/civis/viewtopic.php?f=19&t=465002.
+        """
+        curr, min_, max_ = cext.cpu_freq()
+        return [ntp.scpufreq(curr, min_, max_)]
+
+
+# =====================================================================
+# --- disks
+# =====================================================================
+
+
+disk_usage = _psposix.disk_usage
+disk_io_counters = cext.disk_io_counters
+
+
+def disk_partitions(all=False):
+    """Return mounted disk partitions as a list of namedtuples."""
+    retlist = []
+    partitions = cext.disk_partitions()
+    for partition in partitions:
+        device, mountpoint, fstype, opts = partition
+        if device == 'none':
+            device = ''
+        if not all:
+            if not os.path.isabs(device) or not os.path.exists(device):
+                continue
+        ntuple = ntp.sdiskpart(device, mountpoint, fstype, opts)
+        retlist.append(ntuple)
+    return retlist
+
+
+# =====================================================================
+# --- sensors
+# =====================================================================
+
+
+def sensors_battery():
+    """Return battery information."""
+    try:
+        percent, minsleft, power_plugged = cext.sensors_battery()
+    except NotImplementedError:
+        # no power source - return None according to interface
+        return None
+    power_plugged = power_plugged == 1
+    if power_plugged:
+        secsleft = _common.POWER_TIME_UNLIMITED
+    elif minsleft == -1:
+        secsleft = _common.POWER_TIME_UNKNOWN
+    else:
+        secsleft = minsleft * 60
+    return ntp.sbattery(percent, secsleft, power_plugged)
+
+
+# =====================================================================
+# --- network
+# =====================================================================
+
+
+net_io_counters = cext.net_io_counters
+net_if_addrs = cext.net_if_addrs
+
+
+def net_connections(kind='inet'):
+    """System-wide network connections."""
+    # Note: on macOS this will fail with AccessDenied unless
+    # the process is owned by root.
+    ret = []
+    for pid in pids():
+        try:
+            cons = Process(pid).net_connections(kind)
+        except NoSuchProcess:
+            continue
+        else:
+            if cons:
+                for c in cons:
+                    c = list(c) + [pid]
+                    ret.append(ntp.sconn(*c))
+    return ret
+
+
+def net_if_stats():
+    """Get NIC stats (isup, duplex, speed, mtu)."""
+    names = net_io_counters().keys()
+    ret = {}
+    for name in names:
+        try:
+            mtu = cext.net_if_mtu(name)
+            flags = cext.net_if_flags(name)
+            duplex, speed = cext.net_if_duplex_speed(name)
+        except OSError as err:
+            # https://github.com/giampaolo/psutil/issues/1279
+            if err.errno != errno.ENODEV:
+                raise
+        else:
+            if hasattr(_common, 'NicDuplex'):
+                duplex = _common.NicDuplex(duplex)
+            output_flags = ','.join(flags)
+            isup = 'running' in flags
+            ret[name] = ntp.snicstats(isup, duplex, speed, mtu, output_flags)
+    return ret
+
+
+# =====================================================================
+# --- other system functions
+# =====================================================================
+
+
+def boot_time():
+    """The system boot time expressed in seconds since the epoch."""
+    return cext.boot_time()
+
+
+try:
+    INIT_BOOT_TIME = boot_time()
+except Exception as err:  # noqa: BLE001
+    # Don't want to crash at import time.
+    debug(f"ignoring exception on import: {err!r}")
+    INIT_BOOT_TIME = 0
+
+
+def adjust_proc_create_time(ctime):
+    """Account for system clock updates."""
+    if INIT_BOOT_TIME == 0:
+        return ctime
+
+    diff = INIT_BOOT_TIME - boot_time()
+    if diff == 0 or abs(diff) < 1:
+        return ctime
+
+    debug("system clock was updated; adjusting process create_time()")
+    if diff < 0:
+        return ctime - diff
+    return ctime + diff
+
+
+def users():
+    """Return currently connected users as a list of namedtuples."""
+    retlist = []
+    rawlist = cext.users()
+    for item in rawlist:
+        user, tty, hostname, tstamp, pid = item
+        if tty == '~':
+            continue  # reboot or shutdown
+        if not tstamp:
+            continue
+        nt = ntp.suser(user, tty or None, hostname or None, tstamp, pid)
+        retlist.append(nt)
+    return retlist
+
+
+# =====================================================================
+# --- processes
+# =====================================================================
+
+
+def pids():
+    ls = cext.pids()
+    if 0 not in ls:
+        # On certain macOS versions pids() C doesn't return PID 0 but
+        # "ps" does and the process is querable via sysctl():
+        # https://travis-ci.org/giampaolo/psutil/jobs/309619941
+        try:
+            Process(0).create_time()
+            ls.insert(0, 0)
+        except NoSuchProcess:
+            pass
+        except AccessDenied:
+            ls.insert(0, 0)
+    return ls
+
+
+pid_exists = _psposix.pid_exists
+
+
+def wrap_exceptions(fun):
+    """Decorator which translates bare OSError exceptions into
+    NoSuchProcess and AccessDenied.
+    """
+
+    @functools.wraps(fun)
+    def wrapper(self, *args, **kwargs):
+        pid, ppid, name = self.pid, self._ppid, self._name
+        try:
+            return fun(self, *args, **kwargs)
+        except ProcessLookupError as err:
+            if cext.proc_is_zombie(pid):
+                raise ZombieProcess(pid, name, ppid) from err
+            raise NoSuchProcess(pid, name) from err
+        except PermissionError as err:
+            raise AccessDenied(pid, name) from err
+        except cext.ZombieProcessError as err:
+            raise ZombieProcess(pid, name, ppid) from err
+
+    return wrapper
+
+
+class Process:
+    """Wrapper class around underlying C implementation."""
+
+    __slots__ = ["_cache", "_name", "_ppid", "pid"]
+
+    def __init__(self, pid):
+        self.pid = pid
+        self._name = None
+        self._ppid = None
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def _get_kinfo_proc(self):
+        # Note: should work with all PIDs without permission issues.
+        ret = cext.proc_kinfo_oneshot(self.pid)
+        assert len(ret) == len(kinfo_proc_map)
+        return ret
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def _get_pidtaskinfo(self):
+        # Note: should work for PIDs owned by user only.
+        ret = cext.proc_pidtaskinfo_oneshot(self.pid)
+        assert len(ret) == len(pidtaskinfo_map)
+        return ret
+
+    def oneshot_enter(self):
+        self._get_kinfo_proc.cache_activate(self)
+        self._get_pidtaskinfo.cache_activate(self)
+
+    def oneshot_exit(self):
+        self._get_kinfo_proc.cache_deactivate(self)
+        self._get_pidtaskinfo.cache_deactivate(self)
+
+    @wrap_exceptions
+    def name(self):
+        name = self._get_kinfo_proc()[kinfo_proc_map['name']]
+        return name if name is not None else cext.proc_name(self.pid)
+
+    @wrap_exceptions
+    def exe(self):
+        return cext.proc_exe(self.pid)
+
+    @wrap_exceptions
+    def cmdline(self):
+        return cext.proc_cmdline(self.pid)
+
+    @wrap_exceptions
+    def environ(self):
+        return parse_environ_block(cext.proc_environ(self.pid))
+
+    @wrap_exceptions
+    def ppid(self):
+        self._ppid = self._get_kinfo_proc()[kinfo_proc_map['ppid']]
+        return self._ppid
+
+    @wrap_exceptions
+    def cwd(self):
+        return cext.proc_cwd(self.pid)
+
+    @wrap_exceptions
+    def uids(self):
+        rawtuple = self._get_kinfo_proc()
+        return ntp.puids(
+            rawtuple[kinfo_proc_map['ruid']],
+            rawtuple[kinfo_proc_map['euid']],
+            rawtuple[kinfo_proc_map['suid']],
+        )
+
+    @wrap_exceptions
+    def gids(self):
+        rawtuple = self._get_kinfo_proc()
+        return ntp.puids(
+            rawtuple[kinfo_proc_map['rgid']],
+            rawtuple[kinfo_proc_map['egid']],
+            rawtuple[kinfo_proc_map['sgid']],
+        )
+
+    @wrap_exceptions
+    def terminal(self):
+        tty_nr = self._get_kinfo_proc()[kinfo_proc_map['ttynr']]
+        tmap = _psposix.get_terminal_map()
+        try:
+            return tmap[tty_nr]
+        except KeyError:
+            return None
+
+    @wrap_exceptions
+    def memory_info(self):
+        rawtuple = self._get_pidtaskinfo()
+        return ntp.pmem(
+            rawtuple[pidtaskinfo_map['rss']],
+            rawtuple[pidtaskinfo_map['vms']],
+            rawtuple[pidtaskinfo_map['pfaults']],
+            rawtuple[pidtaskinfo_map['pageins']],
+        )
+
+    @wrap_exceptions
+    def memory_full_info(self):
+        basic_mem = self.memory_info()
+        uss = cext.proc_memory_uss(self.pid)
+        return ntp.pfullmem(*basic_mem + (uss,))
+
+    @wrap_exceptions
+    def cpu_times(self):
+        rawtuple = self._get_pidtaskinfo()
+        return ntp.pcputimes(
+            rawtuple[pidtaskinfo_map['cpuutime']],
+            rawtuple[pidtaskinfo_map['cpustime']],
+            # children user / system times are not retrievable (set to 0)
+            0.0,
+            0.0,
+        )
+
+    @wrap_exceptions
+    def create_time(self, monotonic=False):
+        ctime = self._get_kinfo_proc()[kinfo_proc_map['ctime']]
+        if not monotonic:
+            ctime = adjust_proc_create_time(ctime)
+        return ctime
+
+    @wrap_exceptions
+    def num_ctx_switches(self):
+        # Unvoluntary value seems not to be available;
+        # getrusage() numbers seems to confirm this theory.
+        # We set it to 0.
+        vol = self._get_pidtaskinfo()[pidtaskinfo_map['volctxsw']]
+        return ntp.pctxsw(vol, 0)
+
+    @wrap_exceptions
+    def num_threads(self):
+        return self._get_pidtaskinfo()[pidtaskinfo_map['numthreads']]
+
+    @wrap_exceptions
+    def open_files(self):
+        if self.pid == 0:
+            return []
+        files = []
+        rawlist = cext.proc_open_files(self.pid)
+        for path, fd in rawlist:
+            if isfile_strict(path):
+                ntuple = ntp.popenfile(path, fd)
+                files.append(ntuple)
+        return files
+
+    @wrap_exceptions
+    def net_connections(self, kind='inet'):
+        families, types = conn_tmap[kind]
+        rawlist = cext.proc_net_connections(self.pid, families, types)
+        ret = []
+        for item in rawlist:
+            fd, fam, type, laddr, raddr, status = item
+            nt = conn_to_ntuple(
+                fd, fam, type, laddr, raddr, status, TCP_STATUSES
+            )
+            ret.append(nt)
+        return ret
+
+    @wrap_exceptions
+    def num_fds(self):
+        if self.pid == 0:
+            return 0
+        return cext.proc_num_fds(self.pid)
+
+    @wrap_exceptions
+    def wait(self, timeout=None):
+        return _psposix.wait_pid(self.pid, timeout)
+
+    @wrap_exceptions
+    def nice_get(self):
+        return cext.proc_priority_get(self.pid)
+
+    @wrap_exceptions
+    def nice_set(self, value):
+        return cext.proc_priority_set(self.pid, value)
+
+    @wrap_exceptions
+    def status(self):
+        code = self._get_kinfo_proc()[kinfo_proc_map['status']]
+        # XXX is '?' legit? (we're not supposed to return it anyway)
+        return PROC_STATUSES.get(code, '?')
+
+    @wrap_exceptions
+    def threads(self):
+        rawlist = cext.proc_threads(self.pid)
+        retlist = []
+        for thread_id, utime, stime in rawlist:
+            ntuple = ntp.pthread(thread_id, utime, stime)
+            retlist.append(ntuple)
+        return retlist

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/psutil/_psposix.py

@@ -0,0 +1,363 @@
+# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Routines common to all posix systems."""
+
+import enum
+import errno
+import glob
+import os
+import select
+import signal
+import time
+
+from . import _ntuples as ntp
+from ._common import MACOS
+from ._common import TimeoutExpired
+from ._common import debug
+from ._common import memoize
+from ._common import usage_percent
+
+if MACOS:
+    from . import _psutil_osx
+
+
+__all__ = ['pid_exists', 'wait_pid', 'disk_usage', 'get_terminal_map']
+
+
+def pid_exists(pid):
+    """Check whether pid exists in the current process table."""
+    if pid == 0:
+        # According to "man 2 kill" PID 0 has a special meaning:
+        # it refers to <<every process in the process group of the
+        # calling process>> so we don't want to go any further.
+        # If we get here it means this UNIX platform *does* have
+        # a process with id 0.
+        return True
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        # EPERM clearly means there's a process to deny access to
+        return True
+    # According to "man 2 kill" possible error values are
+    # (EINVAL, EPERM, ESRCH)
+    else:
+        return True
+
+
+Negsignal = enum.IntEnum(
+    'Negsignal', {x.name: -x.value for x in signal.Signals}
+)
+
+
+def negsig_to_enum(num):
+    """Convert a negative signal value to an enum."""
+    try:
+        return Negsignal(num)
+    except ValueError:
+        return num
+
+
+def convert_exit_code(status):
+    """Convert a os.waitpid() status to an exit code."""
+    if os.WIFEXITED(status):
+        # Process terminated normally by calling exit(3) or _exit(2),
+        # or by returning from main(). The return value is the
+        # positive integer passed to *exit().
+        return os.WEXITSTATUS(status)
+    if os.WIFSIGNALED(status):
+        # Process exited due to a signal. Return the negative value
+        # of that signal.
+        return negsig_to_enum(-os.WTERMSIG(status))
+    # if os.WIFSTOPPED(status):
+    #     # Process was stopped via SIGSTOP or is being traced, and
+    #     # waitpid() was called with WUNTRACED flag. PID is still
+    #     # alive. From now on waitpid() will keep returning (0, 0)
+    #     # until the process state doesn't change.
+    #     # It may make sense to catch/enable this since stopped PIDs
+    #     # ignore SIGTERM.
+    #     interval = sleep(interval)
+    #     continue
+    # if os.WIFCONTINUED(status):
+    #     # Process was resumed via SIGCONT and waitpid() was called
+    #     # with WCONTINUED flag.
+    #     interval = sleep(interval)
+    #     continue
+
+    # Should never happen.
+    msg = f"unknown process exit status {status!r}"
+    raise ValueError(msg)
+
+
+def wait_pid_posix(
+    pid,
+    timeout=None,
+    _waitpid=os.waitpid,
+    _timer=getattr(time, 'monotonic', time.time),  # noqa: B008
+    _min=min,
+    _sleep=time.sleep,
+    _pid_exists=pid_exists,
+):
+    """Wait for a process PID to terminate.
+
+    If the process terminated normally by calling exit(3) or _exit(2),
+    or by returning from main(), the return value is the positive integer
+    passed to *exit().
+
+    If it was terminated by a signal it returns the negated value of the
+    signal which caused the termination (e.g. -SIGTERM).
+
+    If PID is not a children of os.getpid() (current process) just
+    wait until the process disappears and return None.
+
+    If PID does not exist at all return None immediately.
+
+    If timeout is specified and process is still alive raise
+    TimeoutExpired.
+
+    If timeout=0 either return immediately or raise TimeoutExpired
+    (non-blocking).
+    """
+    interval = 0.0001
+    max_interval = 0.04
+    flags = 0
+    stop_at = None
+
+    if timeout is not None:
+        flags |= os.WNOHANG
+        if timeout != 0:
+            stop_at = _timer() + timeout
+
+    def sleep_or_timeout(interval):
+        # Sleep for some time and return a new increased interval.
+        if timeout == 0 or (stop_at is not None and _timer() >= stop_at):
+            raise TimeoutExpired(timeout)
+        _sleep(interval)
+        return _min(interval * 2, max_interval)
+
+    # See: https://linux.die.net/man/2/waitpid
+    while True:
+        try:
+            retpid, status = os.waitpid(pid, flags)
+        except ChildProcessError:
+            # This has two meanings:
+            # - PID is not a child of os.getpid() in which case
+            #   we keep polling until it's gone
+            # - PID never existed in the first place
+            # In both cases we'll eventually return None as we
+            # can't determine its exit status code.
+            while _pid_exists(pid):
+                interval = sleep_or_timeout(interval)
+            return None
+        else:
+            if retpid == 0:
+                # WNOHANG flag was used and PID is still running.
+                interval = sleep_or_timeout(interval)
+            else:
+                return convert_exit_code(status)
+
+
+def _waitpid(pid, timeout):
+    """Wrapper around os.waitpid(). PID is supposed to be gone already,
+    it just returns the exit code.
+    """
+    try:
+        retpid, status = os.waitpid(pid, 0)
+    except ChildProcessError:
+        # PID is not a child of os.getpid().
+        return wait_pid_posix(pid, timeout)
+    else:
+        assert retpid != 0
+        return convert_exit_code(status)
+
+
+def wait_pid_pidfd_open(pid, timeout=None):
+    """Wait for PID to terminate using pidfd_open() + poll(). Linux >=
+    5.3 + Python >= 3.9 only.
+    """
+    try:
+        pidfd = os.pidfd_open(pid, 0)
+    except OSError as err:
+        if err.errno == errno.ESRCH:
+            # No such process. os.waitpid() may still be able to return
+            # the status code.
+            return wait_pid_posix(pid, timeout)
+        if err.errno in {errno.EMFILE, errno.ENFILE, errno.ENODEV}:
+            # EMFILE, ENFILE: too many open files
+            # ENODEV: anonymous inode filesystem not supported
+            debug(f"pidfd_open() failed ({err!r}); use fallback")
+            return wait_pid_posix(pid, timeout)
+        raise
+
+    try:
+        # poll() / select() have the advantage of not requiring any
+        # extra file descriptor, contrary to epoll() / kqueue().
+        # select() crashes if process opens > 1024 FDs, so we use
+        # poll().
+        poller = select.poll()
+        poller.register(pidfd, select.POLLIN)
+        timeout_ms = None if timeout is None else int(timeout * 1000)
+        events = poller.poll(timeout_ms)  # wait
+
+        if not events:
+            raise TimeoutExpired(timeout)
+        return _waitpid(pid, timeout)
+    finally:
+        os.close(pidfd)
+
+
+def wait_pid_kqueue(pid, timeout=None):
+    """Wait for PID to terminate using kqueue(). macOS and BSD only."""
+    try:
+        kq = select.kqueue()
+    except OSError as err:
+        if err.errno in {errno.EMFILE, errno.ENFILE}:  # too many open files
+            debug(f"kqueue() failed ({err!r}); use fallback")
+            return wait_pid_posix(pid, timeout)
+        raise
+
+    try:
+        kev = select.kevent(
+            pid,
+            filter=select.KQ_FILTER_PROC,
+            flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT,
+            fflags=select.KQ_NOTE_EXIT,
+        )
+        try:
+            events = kq.control([kev], 1, timeout)  # wait
+        except OSError as err:
+            if err.errno in {errno.EACCES, errno.EPERM, errno.ESRCH}:
+                debug(f"kqueue.control() failed ({err!r}); use fallback")
+                return wait_pid_posix(pid, timeout)
+            raise
+        else:
+            if not events:
+                raise TimeoutExpired(timeout)
+            return _waitpid(pid, timeout)
+    finally:
+        kq.close()
+
+
+@memoize
+def can_use_pidfd_open():
+    # Availability: Linux >= 5.3, Python >= 3.9
+    if not hasattr(os, "pidfd_open"):
+        return False
+    try:
+        pidfd = os.pidfd_open(os.getpid(), 0)
+    except OSError as err:
+        if err.errno in {errno.EMFILE, errno.ENFILE}:  # noqa: SIM103
+            # transitory 'too many open files'
+            return True
+        # likely blocked by security policy like SECCOMP (EPERM,
+        # EACCES, ENOSYS)
+        return False
+    else:
+        os.close(pidfd)
+        return True
+
+
+@memoize
+def can_use_kqueue():
+    # Availability: macOS, BSD
+    names = (
+        "kqueue",
+        "KQ_EV_ADD",
+        "KQ_EV_ONESHOT",
+        "KQ_FILTER_PROC",
+        "KQ_NOTE_EXIT",
+    )
+    if not all(hasattr(select, x) for x in names):
+        return False
+    kq = None
+    try:
+        kq = select.kqueue()
+        kev = select.kevent(
+            os.getpid(),
+            filter=select.KQ_FILTER_PROC,
+            flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT,
+            fflags=select.KQ_NOTE_EXIT,
+        )
+        kq.control([kev], 1, 0)
+        return True
+    except OSError as err:
+        if err.errno in {errno.EMFILE, errno.ENFILE}:  # noqa: SIM103
+            # transitory 'too many open files'
+            return True
+        return False
+    finally:
+        if kq is not None:
+            kq.close()
+
+
+def wait_pid(pid, timeout=None):
+    # PID 0 passed to waitpid() waits for any child of the current
+    # process to change state.
+    assert pid > 0
+    if timeout is not None:
+        assert timeout >= 0
+
+    if can_use_pidfd_open():
+        return wait_pid_pidfd_open(pid, timeout)
+    elif can_use_kqueue():
+        return wait_pid_kqueue(pid, timeout)
+    else:
+        return wait_pid_posix(pid, timeout)
+
+
+wait_pid.__doc__ = wait_pid_posix.__doc__
+
+
+def disk_usage(path):
+    """Return disk usage associated with path.
+    Note: UNIX usually reserves 5% disk space which is not accessible
+    by user. In this function "total" and "used" values reflect the
+    total and used disk space whereas "free" and "percent" represent
+    the "free" and "used percent" user disk space.
+    """
+    st = os.statvfs(path)
+    # Total space which is only available to root (unless changed
+    # at system level).
+    total = st.f_blocks * st.f_frsize
+    # Remaining free space usable by root.
+    avail_to_root = st.f_bfree * st.f_frsize
+    # Remaining free space usable by user.
+    avail_to_user = st.f_bavail * st.f_frsize
+    # Total space being used in general.
+    used = total - avail_to_root
+    if MACOS:
+        # see: https://github.com/giampaolo/psutil/pull/2152
+        used = _psutil_osx.disk_usage_used(path, used)
+    # Total space which is available to user (same as 'total' but
+    # for the user).
+    total_user = used + avail_to_user
+    # User usage percent compared to the total amount of space
+    # the user can use. This number would be higher if compared
+    # to root's because the user has less space (usually -5%).
+    usage_percent_user = usage_percent(used, total_user, round_=1)
+
+    # NB: the percentage is -5% than what shown by df due to
+    # reserved blocks that we are currently not considering:
+    # https://github.com/giampaolo/psutil/issues/829#issuecomment-223750462
+    return ntp.sdiskusage(
+        total=total, used=used, free=avail_to_user, percent=usage_percent_user
+    )
+
+
+@memoize
+def get_terminal_map():
+    """Get a map of device-id -> path as a dict.
+    Used by Process.terminal().
+    """
+    ret = {}
+    ls = glob.glob('/dev/tty*') + glob.glob('/dev/pts/*')
+    for name in ls:
+        assert name not in ret, name
+        try:
+            ret[os.stat(name).st_rdev] = name
+        except FileNotFoundError:
+            pass
+    return ret

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/psutil/_pssunos.py

@@ -0,0 +1,704 @@
+# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Sun OS Solaris platform implementation."""
+
+import errno
+import functools
+import os
+import socket
+import subprocess
+import sys
+from collections import namedtuple
+from socket import AF_INET
+
+from . import _common
+from . import _ntuples as ntp
+from . import _psposix
+from . import _psutil_sunos as cext
+from ._common import AF_INET6
+from ._common import ENCODING
+from ._common import AccessDenied
+from ._common import NoSuchProcess
+from ._common import ZombieProcess
+from ._common import debug
+from ._common import get_procfs_path
+from ._common import isfile_strict
+from ._common import memoize_when_activated
+from ._common import sockfam_to_enum
+from ._common import socktype_to_enum
+from ._common import usage_percent
+
+__extra__all__ = ["CONN_IDLE", "CONN_BOUND", "PROCFS_PATH"]
+
+
+# =====================================================================
+# --- globals
+# =====================================================================
+
+
+PAGE_SIZE = cext.getpagesize()
+AF_LINK = cext.AF_LINK
+IS_64_BIT = sys.maxsize > 2**32
+
+CONN_IDLE = "IDLE"
+CONN_BOUND = "BOUND"
+
+PROC_STATUSES = {
+    cext.SSLEEP: _common.STATUS_SLEEPING,
+    cext.SRUN: _common.STATUS_RUNNING,
+    cext.SZOMB: _common.STATUS_ZOMBIE,
+    cext.SSTOP: _common.STATUS_STOPPED,
+    cext.SIDL: _common.STATUS_IDLE,
+    cext.SONPROC: _common.STATUS_RUNNING,  # same as run
+    cext.SWAIT: _common.STATUS_WAITING,
+}
+
+TCP_STATUSES = {
+    cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED,
+    cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT,
+    cext.TCPS_SYN_RCVD: _common.CONN_SYN_RECV,
+    cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1,
+    cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2,
+    cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT,
+    cext.TCPS_CLOSED: _common.CONN_CLOSE,
+    cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT,
+    cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK,
+    cext.TCPS_LISTEN: _common.CONN_LISTEN,
+    cext.TCPS_CLOSING: _common.CONN_CLOSING,
+    cext.PSUTIL_CONN_NONE: _common.CONN_NONE,
+    cext.TCPS_IDLE: CONN_IDLE,  # sunos specific
+    cext.TCPS_BOUND: CONN_BOUND,  # sunos specific
+}
+
+proc_info_map = dict(
+    ppid=0,
+    rss=1,
+    vms=2,
+    create_time=3,
+    nice=4,
+    num_threads=5,
+    status=6,
+    ttynr=7,
+    uid=8,
+    euid=9,
+    gid=10,
+    egid=11,
+)
+
+
+# =====================================================================
+# --- memory
+# =====================================================================
+
+
+def virtual_memory():
+    """Report virtual memory metrics."""
+    # we could have done this with kstat, but IMHO this is good enough
+    total = os.sysconf('SC_PHYS_PAGES') * PAGE_SIZE
+    # note: there's no difference on Solaris
+    free = avail = os.sysconf('SC_AVPHYS_PAGES') * PAGE_SIZE
+    used = total - free
+    percent = usage_percent(used, total, round_=1)
+    return ntp.svmem(total, avail, percent, used, free)
+
+
+def swap_memory():
+    """Report swap memory metrics."""
+    sin, sout = cext.swap_mem()
+    # XXX
+    # we are supposed to get total/free by doing so:
+    # http://cvs.opensolaris.org/source/xref/onnv/onnv-gate/
+    #     usr/src/cmd/swap/swap.c
+    # ...nevertheless I can't manage to obtain the same numbers as 'swap'
+    # cmdline utility, so let's parse its output (sigh!)
+    p = subprocess.Popen(
+        [
+            '/usr/bin/env',
+            f"PATH=/usr/sbin:/sbin:{os.environ['PATH']}",
+            'swap',
+            '-l',
+        ],
+        stdout=subprocess.PIPE,
+    )
+    stdout, _ = p.communicate()
+    stdout = stdout.decode(sys.stdout.encoding)
+    if p.returncode != 0:
+        msg = f"'swap -l' failed (retcode={p.returncode})"
+        raise RuntimeError(msg)
+
+    lines = stdout.strip().split('\n')[1:]
+    if not lines:
+        msg = 'no swap device(s) configured'
+        raise RuntimeError(msg)
+    total = free = 0
+    for line in lines:
+        line = line.split()
+        t, f = line[3:5]
+        total += int(int(t) * 512)
+        free += int(int(f) * 512)
+    used = total - free
+    percent = usage_percent(used, total, round_=1)
+    return ntp.sswap(
+        total, used, free, percent, sin * PAGE_SIZE, sout * PAGE_SIZE
+    )
+
+
+# =====================================================================
+# --- CPU
+# =====================================================================
+
+
+def cpu_times():
+    """Return system-wide CPU times as a named tuple."""
+    ret = cext.per_cpu_times()
+    return ntp.scputimes(*[sum(x) for x in zip(*ret)])
+
+
+def per_cpu_times():
+    """Return system per-CPU times as a list of named tuples."""
+    ret = cext.per_cpu_times()
+    return [ntp.scputimes(*x) for x in ret]
+
+
+def cpu_count_logical():
+    """Return the number of logical CPUs in the system."""
+    try:
+        return os.sysconf("SC_NPROCESSORS_ONLN")
+    except ValueError:
+        # mimic os.cpu_count() behavior
+        return None
+
+
+def cpu_count_cores():
+    """Return the number of CPU cores in the system."""
+    return cext.cpu_count_cores()
+
+
+def cpu_stats():
+    """Return various CPU stats as a named tuple."""
+    ctx_switches, interrupts, syscalls, _traps = cext.cpu_stats()
+    soft_interrupts = 0
+    return ntp.scpustats(ctx_switches, interrupts, soft_interrupts, syscalls)
+
+
+# =====================================================================
+# --- disks
+# =====================================================================
+
+
+disk_io_counters = cext.disk_io_counters
+disk_usage = _psposix.disk_usage
+
+
+def disk_partitions(all=False):
+    """Return system disk partitions."""
+    # TODO - the filtering logic should be better checked so that
+    # it tries to reflect 'df' as much as possible
+    retlist = []
+    partitions = cext.disk_partitions()
+    for partition in partitions:
+        device, mountpoint, fstype, opts = partition
+        if device == 'none':
+            device = ''
+        if not all:
+            # Differently from, say, Linux, we don't have a list of
+            # common fs types so the best we can do, AFAIK, is to
+            # filter by filesystem having a total size > 0.
+            try:
+                if not disk_usage(mountpoint).total:
+                    continue
+            except OSError as err:
+                # https://github.com/giampaolo/psutil/issues/1674
+                debug(f"skipping {mountpoint!r}: {err}")
+                continue
+        ntuple = ntp.sdiskpart(device, mountpoint, fstype, opts)
+        retlist.append(ntuple)
+    return retlist
+
+
+# =====================================================================
+# --- network
+# =====================================================================
+
+
+net_io_counters = cext.net_io_counters
+net_if_addrs = cext.net_if_addrs
+
+
+def net_connections(kind, _pid=-1):
+    """Return socket connections.  If pid == -1 return system-wide
+    connections (as opposed to connections opened by one process only).
+    Only INET sockets are returned (UNIX are not).
+    """
+    families, types = _common.conn_tmap[kind]
+    rawlist = cext.net_connections(_pid)
+    ret = set()
+    for item in rawlist:
+        fd, fam, type_, laddr, raddr, status, pid = item
+        if fam not in families:
+            continue
+        if type_ not in types:
+            continue
+        # TODO: refactor and use _common.conn_to_ntuple.
+        if fam in {AF_INET, AF_INET6}:
+            if laddr:
+                laddr = ntp.addr(*laddr)
+            if raddr:
+                raddr = ntp.addr(*raddr)
+        status = TCP_STATUSES[status]
+        fam = sockfam_to_enum(fam)
+        type_ = socktype_to_enum(type_)
+        if _pid == -1:
+            nt = ntp.sconn(fd, fam, type_, laddr, raddr, status, pid)
+        else:
+            nt = ntp.pconn(fd, fam, type_, laddr, raddr, status)
+        ret.add(nt)
+    return list(ret)
+
+
+def net_if_stats():
+    """Get NIC stats (isup, duplex, speed, mtu)."""
+    ret = cext.net_if_stats()
+    for name, items in ret.items():
+        isup, duplex, speed, mtu = items
+        if hasattr(_common, 'NicDuplex'):
+            duplex = _common.NicDuplex(duplex)
+        ret[name] = ntp.snicstats(isup, duplex, speed, mtu, '')
+    return ret
+
+
+# =====================================================================
+# --- other system functions
+# =====================================================================
+
+
+def boot_time():
+    """The system boot time expressed in seconds since the epoch."""
+    return cext.boot_time()
+
+
+def users():
+    """Return currently connected users as a list of namedtuples."""
+    retlist = []
+    rawlist = cext.users()
+    localhost = (':0.0', ':0')
+    for item in rawlist:
+        user, tty, hostname, tstamp, user_process, pid = item
+        # note: the underlying C function includes entries about
+        # system boot, run level and others.  We might want
+        # to use them in the future.
+        if not user_process:
+            continue
+        if hostname in localhost:
+            hostname = 'localhost'
+        nt = ntp.suser(user, tty, hostname, tstamp, pid)
+        retlist.append(nt)
+    return retlist
+
+
+# =====================================================================
+# --- processes
+# =====================================================================
+
+
+def pids():
+    """Returns a list of PIDs currently running on the system."""
+    path = get_procfs_path().encode(ENCODING)
+    return [int(x) for x in os.listdir(path) if x.isdigit()]
+
+
+def pid_exists(pid):
+    """Check for the existence of a unix pid."""
+    return _psposix.pid_exists(pid)
+
+
+def wrap_exceptions(fun):
+    """Call callable into a try/except clause and translate ENOENT,
+    EACCES and EPERM in NoSuchProcess or AccessDenied exceptions.
+    """
+
+    @functools.wraps(fun)
+    def wrapper(self, *args, **kwargs):
+        pid, ppid, name = self.pid, self._ppid, self._name
+        try:
+            return fun(self, *args, **kwargs)
+        except (FileNotFoundError, ProcessLookupError) as err:
+            # ENOENT (no such file or directory) gets raised on open().
+            # ESRCH (no such process) can get raised on read() if
+            # process is gone in meantime.
+            if not pid_exists(pid):
+                raise NoSuchProcess(pid, name) from err
+            raise ZombieProcess(pid, name, ppid) from err
+        except PermissionError as err:
+            raise AccessDenied(pid, name) from err
+        except OSError as err:
+            if pid == 0:
+                if 0 in pids():
+                    raise AccessDenied(pid, name) from err
+                raise
+            raise
+
+    return wrapper
+
+
+class Process:
+    """Wrapper class around underlying C implementation."""
+
+    __slots__ = ["_cache", "_name", "_ppid", "_procfs_path", "pid"]
+
+    def __init__(self, pid):
+        self.pid = pid
+        self._name = None
+        self._ppid = None
+        self._procfs_path = get_procfs_path()
+
+    def _assert_alive(self):
+        """Raise NSP if the process disappeared on us."""
+        # For those C function who do not raise NSP, possibly returning
+        # incorrect or incomplete result.
+        os.stat(f"{self._procfs_path}/{self.pid}")
+
+    def oneshot_enter(self):
+        self._proc_name_and_args.cache_activate(self)
+        self._proc_basic_info.cache_activate(self)
+        self._proc_cred.cache_activate(self)
+
+    def oneshot_exit(self):
+        self._proc_name_and_args.cache_deactivate(self)
+        self._proc_basic_info.cache_deactivate(self)
+        self._proc_cred.cache_deactivate(self)
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def _proc_name_and_args(self):
+        return cext.proc_name_and_args(self.pid, self._procfs_path)
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def _proc_basic_info(self):
+        if self.pid == 0 and not os.path.exists(
+            f"{self._procfs_path}/{self.pid}/psinfo"
+        ):
+            raise AccessDenied(self.pid)
+        ret = cext.proc_basic_info(self.pid, self._procfs_path)
+        assert len(ret) == len(proc_info_map)
+        return ret
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def _proc_cred(self):
+        return cext.proc_cred(self.pid, self._procfs_path)
+
+    @wrap_exceptions
+    def name(self):
+        # note: max len == 15
+        return self._proc_name_and_args()[0]
+
+    @wrap_exceptions
+    def exe(self):
+        try:
+            return os.readlink(f"{self._procfs_path}/{self.pid}/path/a.out")
+        except OSError:
+            pass  # continue and guess the exe name from the cmdline
+        # Will be guessed later from cmdline but we want to explicitly
+        # invoke cmdline here in order to get an AccessDenied
+        # exception if the user has not enough privileges.
+        self.cmdline()
+        return ""
+
+    @wrap_exceptions
+    def cmdline(self):
+        return self._proc_name_and_args()[1]
+
+    @wrap_exceptions
+    def environ(self):
+        return cext.proc_environ(self.pid, self._procfs_path)
+
+    @wrap_exceptions
+    def create_time(self):
+        return self._proc_basic_info()[proc_info_map['create_time']]
+
+    @wrap_exceptions
+    def num_threads(self):
+        return self._proc_basic_info()[proc_info_map['num_threads']]
+
+    @wrap_exceptions
+    def nice_get(self):
+        # Note #1: getpriority(3) doesn't work for realtime processes.
+        # Psinfo is what ps uses, see:
+        # https://github.com/giampaolo/psutil/issues/1194
+        return self._proc_basic_info()[proc_info_map['nice']]
+
+    @wrap_exceptions
+    def nice_set(self, value):
+        if self.pid in {2, 3}:
+            # Special case PIDs: internally setpriority(3) return ESRCH
+            # (no such process), no matter what.
+            # The process actually exists though, as it has a name,
+            # creation time, etc.
+            raise AccessDenied(self.pid, self._name)
+        return cext.proc_priority_set(self.pid, value)
+
+    @wrap_exceptions
+    def ppid(self):
+        self._ppid = self._proc_basic_info()[proc_info_map['ppid']]
+        return self._ppid
+
+    @wrap_exceptions
+    def uids(self):
+        try:
+            real, effective, saved, _, _, _ = self._proc_cred()
+        except AccessDenied:
+            real = self._proc_basic_info()[proc_info_map['uid']]
+            effective = self._proc_basic_info()[proc_info_map['euid']]
+            saved = None
+        return ntp.puids(real, effective, saved)
+
+    @wrap_exceptions
+    def gids(self):
+        try:
+            _, _, _, real, effective, saved = self._proc_cred()
+        except AccessDenied:
+            real = self._proc_basic_info()[proc_info_map['gid']]
+            effective = self._proc_basic_info()[proc_info_map['egid']]
+            saved = None
+        return ntp.puids(real, effective, saved)
+
+    @wrap_exceptions
+    def cpu_times(self):
+        try:
+            times = cext.proc_cpu_times(self.pid, self._procfs_path)
+        except OSError as err:
+            if err.errno == errno.EOVERFLOW and not IS_64_BIT:
+                # We may get here if we attempt to query a 64bit process
+                # with a 32bit python.
+                # Error originates from read() and also tools like "cat"
+                # fail in the same way (!).
+                # Since there simply is no way to determine CPU times we
+                # return 0.0 as a fallback. See:
+                # https://github.com/giampaolo/psutil/issues/857
+                times = (0.0, 0.0, 0.0, 0.0)
+            else:
+                raise
+        return ntp.pcputimes(*times)
+
+    @wrap_exceptions
+    def cpu_num(self):
+        return cext.proc_cpu_num(self.pid, self._procfs_path)
+
+    @wrap_exceptions
+    def terminal(self):
+        procfs_path = self._procfs_path
+        hit_enoent = False
+        tty = wrap_exceptions(self._proc_basic_info()[proc_info_map['ttynr']])
+        if tty != cext.PRNODEV:
+            for x in (0, 1, 2, 255):
+                try:
+                    return os.readlink(f"{procfs_path}/{self.pid}/path/{x}")
+                except FileNotFoundError:
+                    hit_enoent = True
+                    continue
+        if hit_enoent:
+            self._assert_alive()
+
+    @wrap_exceptions
+    def cwd(self):
+        # /proc/PID/path/cwd may not be resolved by readlink() even if
+        # it exists (ls shows it). If that's the case and the process
+        # is still alive return None (we can return None also on BSD).
+        # Reference: https://groups.google.com/g/comp.unix.solaris/c/tcqvhTNFCAs
+        procfs_path = self._procfs_path
+        try:
+            return os.readlink(f"{procfs_path}/{self.pid}/path/cwd")
+        except FileNotFoundError:
+            os.stat(f"{procfs_path}/{self.pid}")  # raise NSP or AD
+            return ""
+
+    @wrap_exceptions
+    def memory_info(self):
+        ret = self._proc_basic_info()
+        rss = ret[proc_info_map['rss']] * 1024
+        vms = ret[proc_info_map['vms']] * 1024
+        return ntp.pmem(rss, vms)
+
+    memory_full_info = memory_info
+
+    @wrap_exceptions
+    def status(self):
+        code = self._proc_basic_info()[proc_info_map['status']]
+        # XXX is '?' legit? (we're not supposed to return it anyway)
+        return PROC_STATUSES.get(code, '?')
+
+    @wrap_exceptions
+    def threads(self):
+        procfs_path = self._procfs_path
+        ret = []
+        tids = os.listdir(f"{procfs_path}/{self.pid}/lwp")
+        hit_enoent = False
+        for tid in tids:
+            tid = int(tid)
+            try:
+                utime, stime = cext.query_process_thread(
+                    self.pid, tid, procfs_path
+                )
+            except OSError as err:
+                if err.errno == errno.EOVERFLOW and not IS_64_BIT:
+                    # We may get here if we attempt to query a 64bit process
+                    # with a 32bit python.
+                    # Error originates from read() and also tools like "cat"
+                    # fail in the same way (!).
+                    # Since there simply is no way to determine CPU times we
+                    # return 0.0 as a fallback. See:
+                    # https://github.com/giampaolo/psutil/issues/857
+                    continue
+                # ENOENT == thread gone in meantime
+                if err.errno == errno.ENOENT:
+                    hit_enoent = True
+                    continue
+                raise
+            else:
+                nt = ntp.pthread(tid, utime, stime)
+                ret.append(nt)
+        if hit_enoent:
+            self._assert_alive()
+        return ret
+
+    @wrap_exceptions
+    def open_files(self):
+        retlist = []
+        hit_enoent = False
+        procfs_path = self._procfs_path
+        pathdir = f"{procfs_path}/{self.pid}/path"
+        for fd in os.listdir(f"{procfs_path}/{self.pid}/fd"):
+            path = os.path.join(pathdir, fd)
+            if os.path.islink(path):
+                try:
+                    file = os.readlink(path)
+                except FileNotFoundError:
+                    hit_enoent = True
+                    continue
+                else:
+                    if isfile_strict(file):
+                        retlist.append(ntp.popenfile(file, int(fd)))
+        if hit_enoent:
+            self._assert_alive()
+        return retlist
+
+    def _get_unix_sockets(self, pid):
+        """Get UNIX sockets used by process by parsing 'pfiles' output."""
+        # TODO: rewrite this in C (...but the damn netstat source code
+        # does not include this part! Argh!!)
+        cmd = ["pfiles", str(pid)]
+        p = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+        stdout, stderr = p.communicate()
+        stdout, stderr = (
+            x.decode(sys.stdout.encoding) for x in (stdout, stderr)
+        )
+        if p.returncode != 0:
+            if 'permission denied' in stderr.lower():
+                raise AccessDenied(self.pid, self._name)
+            if 'no such process' in stderr.lower():
+                raise NoSuchProcess(self.pid, self._name)
+            msg = f"{cmd!r} command error\n{stderr}"
+            raise RuntimeError(msg)
+
+        lines = stdout.split('\n')[2:]
+        for i, line in enumerate(lines):
+            line = line.lstrip()
+            if line.startswith('sockname: AF_UNIX'):
+                path = line.split(' ', 2)[2]
+                type = lines[i - 2].strip()
+                if type == 'SOCK_STREAM':
+                    type = socket.SOCK_STREAM
+                elif type == 'SOCK_DGRAM':
+                    type = socket.SOCK_DGRAM
+                else:
+                    type = -1
+                yield (-1, socket.AF_UNIX, type, path, "", _common.CONN_NONE)
+
+    @wrap_exceptions
+    def net_connections(self, kind='inet'):
+        ret = net_connections(kind, _pid=self.pid)
+        # The underlying C implementation retrieves all OS connections
+        # and filters them by PID.  At this point we can't tell whether
+        # an empty list means there were no connections for process or
+        # process is no longer active so we force NSP in case the PID
+        # is no longer there.
+        if not ret:
+            # will raise NSP if process is gone
+            os.stat(f"{self._procfs_path}/{self.pid}")
+
+        # UNIX sockets
+        if kind in {'all', 'unix'}:
+            ret.extend(
+                [ntp.pconn(*conn) for conn in self._get_unix_sockets(self.pid)]
+            )
+        return ret
+
+    nt_mmap_grouped = namedtuple('mmap', 'path rss anon locked')
+    nt_mmap_ext = namedtuple('mmap', 'addr perms path rss anon locked')
+
+    @wrap_exceptions
+    def memory_maps(self):
+        def toaddr(start, end):
+            return "{}-{}".format(
+                hex(start)[2:].strip('L'), hex(end)[2:].strip('L')
+            )
+
+        procfs_path = self._procfs_path
+        retlist = []
+        try:
+            rawlist = cext.proc_memory_maps(self.pid, procfs_path)
+        except OSError as err:
+            if err.errno == errno.EOVERFLOW and not IS_64_BIT:
+                # We may get here if we attempt to query a 64bit process
+                # with a 32bit python.
+                # Error originates from read() and also tools like "cat"
+                # fail in the same way (!).
+                # Since there simply is no way to determine CPU times we
+                # return 0.0 as a fallback. See:
+                # https://github.com/giampaolo/psutil/issues/857
+                return []
+            else:
+                raise
+        hit_enoent = False
+        for item in rawlist:
+            addr, addrsize, perm, name, rss, anon, locked = item
+            addr = toaddr(addr, addrsize)
+            if not name.startswith('['):
+                try:
+                    name = os.readlink(f"{procfs_path}/{self.pid}/path/{name}")
+                except OSError as err:
+                    if err.errno == errno.ENOENT:
+                        # sometimes the link may not be resolved by
+                        # readlink() even if it exists (ls shows it).
+                        # If that's the case we just return the
+                        # unresolved link path.
+                        # This seems an inconsistency with /proc similar
+                        # to: http://goo.gl/55XgO
+                        name = f"{procfs_path}/{self.pid}/path/{name}"
+                        hit_enoent = True
+                    else:
+                        raise
+            retlist.append((addr, perm, name, rss, anon, locked))
+        if hit_enoent:
+            self._assert_alive()
+        return retlist
+
+    @wrap_exceptions
+    def num_fds(self):
+        return len(os.listdir(f"{self._procfs_path}/{self.pid}/fd"))
+
+    @wrap_exceptions
+    def num_ctx_switches(self):
+        return ntp.pctxsw(
+            *cext.proc_num_ctx_switches(self.pid, self._procfs_path)
+        )
+
+    @wrap_exceptions
+    def wait(self, timeout=None):
+        return _psposix.wait_pid(self.pid, timeout)

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/psutil/_pswindows.py

@@ -0,0 +1,1096 @@
+# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Windows platform implementation."""
+
+import contextlib
+import enum
+import functools
+import os
+import signal
+import sys
+import threading
+import time
+
+from . import _common
+from . import _ntuples as ntp
+from ._common import ENCODING
+from ._common import AccessDenied
+from ._common import NoSuchProcess
+from ._common import TimeoutExpired
+from ._common import conn_tmap
+from ._common import conn_to_ntuple
+from ._common import debug
+from ._common import isfile_strict
+from ._common import memoize
+from ._common import memoize_when_activated
+from ._common import parse_environ_block
+from ._common import usage_percent
+from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS
+from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS
+from ._psutil_windows import HIGH_PRIORITY_CLASS
+from ._psutil_windows import IDLE_PRIORITY_CLASS
+from ._psutil_windows import NORMAL_PRIORITY_CLASS
+from ._psutil_windows import REALTIME_PRIORITY_CLASS
+
+try:
+    from . import _psutil_windows as cext
+except ImportError as err:
+    if (
+        str(err).lower().startswith("dll load failed")
+        and sys.getwindowsversion()[0] < 6
+    ):
+        # We may get here if:
+        # 1) we are on an old Windows version
+        # 2) psutil was installed via pip + wheel
+        # See: https://github.com/giampaolo/psutil/issues/811
+        msg = "this Windows version is too old (< Windows Vista); "
+        msg += "psutil 3.4.2 is the latest version which supports Windows "
+        msg += "2000, XP and 2003 server"
+        raise RuntimeError(msg) from err
+    else:
+        raise
+
+
+# process priority constants, import from __init__.py:
+# http://msdn.microsoft.com/en-us/library/ms686219(v=vs.85).aspx
+# fmt: off
+__extra__all__ = [
+    "win_service_iter", "win_service_get",
+    # Process priority
+    "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS",
+    "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", "NORMAL_PRIORITY_CLASS",
+    "REALTIME_PRIORITY_CLASS",
+    # IO priority
+    "IOPRIO_VERYLOW", "IOPRIO_LOW", "IOPRIO_NORMAL", "IOPRIO_HIGH",
+    # others
+    "CONN_DELETE_TCB", "AF_LINK",
+]
+# fmt: on
+
+
+# =====================================================================
+# --- globals
+# =====================================================================
+
+CONN_DELETE_TCB = "DELETE_TCB"
+ERROR_PARTIAL_COPY = 299
+PYPY = '__pypy__' in sys.builtin_module_names
+
+AddressFamily = enum.IntEnum('AddressFamily', {'AF_LINK': -1})
+AF_LINK = AddressFamily.AF_LINK
+
+TCP_STATUSES = {
+    cext.MIB_TCP_STATE_ESTAB: _common.CONN_ESTABLISHED,
+    cext.MIB_TCP_STATE_SYN_SENT: _common.CONN_SYN_SENT,
+    cext.MIB_TCP_STATE_SYN_RCVD: _common.CONN_SYN_RECV,
+    cext.MIB_TCP_STATE_FIN_WAIT1: _common.CONN_FIN_WAIT1,
+    cext.MIB_TCP_STATE_FIN_WAIT2: _common.CONN_FIN_WAIT2,
+    cext.MIB_TCP_STATE_TIME_WAIT: _common.CONN_TIME_WAIT,
+    cext.MIB_TCP_STATE_CLOSED: _common.CONN_CLOSE,
+    cext.MIB_TCP_STATE_CLOSE_WAIT: _common.CONN_CLOSE_WAIT,
+    cext.MIB_TCP_STATE_LAST_ACK: _common.CONN_LAST_ACK,
+    cext.MIB_TCP_STATE_LISTEN: _common.CONN_LISTEN,
+    cext.MIB_TCP_STATE_CLOSING: _common.CONN_CLOSING,
+    cext.MIB_TCP_STATE_DELETE_TCB: CONN_DELETE_TCB,
+    cext.PSUTIL_CONN_NONE: _common.CONN_NONE,
+}
+
+
+class Priority(enum.IntEnum):
+    ABOVE_NORMAL_PRIORITY_CLASS = ABOVE_NORMAL_PRIORITY_CLASS
+    BELOW_NORMAL_PRIORITY_CLASS = BELOW_NORMAL_PRIORITY_CLASS
+    HIGH_PRIORITY_CLASS = HIGH_PRIORITY_CLASS
+    IDLE_PRIORITY_CLASS = IDLE_PRIORITY_CLASS
+    NORMAL_PRIORITY_CLASS = NORMAL_PRIORITY_CLASS
+    REALTIME_PRIORITY_CLASS = REALTIME_PRIORITY_CLASS
+
+
+globals().update(Priority.__members__)
+
+
+class IOPriority(enum.IntEnum):
+    IOPRIO_VERYLOW = 0
+    IOPRIO_LOW = 1
+    IOPRIO_NORMAL = 2
+    IOPRIO_HIGH = 3
+
+
+globals().update(IOPriority.__members__)
+
+pinfo_map = dict(
+    num_handles=0,
+    ctx_switches=1,
+    user_time=2,
+    kernel_time=3,
+    create_time=4,
+    num_threads=5,
+    io_rcount=6,
+    io_wcount=7,
+    io_rbytes=8,
+    io_wbytes=9,
+    io_count_others=10,
+    io_bytes_others=11,
+    num_page_faults=12,
+    peak_wset=13,
+    wset=14,
+    peak_paged_pool=15,
+    paged_pool=16,
+    peak_non_paged_pool=17,
+    non_paged_pool=18,
+    pagefile=19,
+    peak_pagefile=20,
+    mem_private=21,
+)
+
+
+# =====================================================================
+# --- utils
+# =====================================================================
+
+
+@functools.lru_cache(maxsize=512)
+def convert_dos_path(s):
+    r"""Convert paths using native DOS format like:
+        "\Device\HarddiskVolume1\Windows\systemew\file.txt" or
+        "\??\C:\Windows\systemew\file.txt"
+    into:
+        "C:\Windows\systemew\file.txt".
+    """
+    if s.startswith('\\\\'):
+        return s
+    rawdrive = '\\'.join(s.split('\\')[:3])
+    if rawdrive in {"\\??\\UNC", "\\Device\\Mup"}:
+        rawdrive = '\\'.join(s.split('\\')[:5])
+        driveletter = '\\\\' + '\\'.join(s.split('\\')[3:5])
+    elif rawdrive.startswith('\\??\\'):
+        driveletter = s.split('\\')[2]
+    else:
+        driveletter = cext.QueryDosDevice(rawdrive)
+    remainder = s[len(rawdrive) :]
+    return os.path.join(driveletter, remainder)
+
+
+@memoize
+def getpagesize():
+    return cext.getpagesize()
+
+
+# =====================================================================
+# --- memory
+# =====================================================================
+
+
+def virtual_memory():
+    """System virtual memory as a namedtuple."""
+    mem = cext.virtual_mem()
+    totphys, availphys, _totsys, _availsys = mem
+    total = totphys
+    avail = availphys
+    free = availphys
+    used = total - avail
+    percent = usage_percent((total - avail), total, round_=1)
+    return ntp.svmem(total, avail, percent, used, free)
+
+
+def swap_memory():
+    """Swap system memory as a (total, used, free, sin, sout) tuple."""
+    mem = cext.virtual_mem()
+
+    total_phys = mem[0]
+    total_system = mem[2]
+
+    # system memory (commit total/limit) is the sum of physical and swap
+    # thus physical memory values need to be subtracted to get swap values
+    total = total_system - total_phys
+    # commit total is incremented immediately (decrementing free_system)
+    # while the corresponding free physical value is not decremented until
+    # pages are accessed, so we can't use free system memory for swap.
+    # instead, we calculate page file usage based on performance counter
+    if total > 0:
+        percentswap = cext.swap_percent()
+        used = int(0.01 * percentswap * total)
+    else:
+        percentswap = 0.0
+        used = 0
+
+    free = total - used
+    percent = round(percentswap, 1)
+    return ntp.sswap(total, used, free, percent, 0, 0)
+
+
+# malloc / heap functions
+heap_info = cext.heap_info
+heap_trim = cext.heap_trim
+
+
+# =====================================================================
+# --- disk
+# =====================================================================
+
+
+disk_io_counters = cext.disk_io_counters
+
+
+def disk_usage(path):
+    """Return disk usage associated with path."""
+    if isinstance(path, bytes):
+        # XXX: do we want to use "strict"? Probably yes, in order
+        # to fail immediately. After all we are accepting input here...
+        path = path.decode(ENCODING, errors="strict")
+    total, used, free = cext.disk_usage(path)
+    percent = usage_percent(used, total, round_=1)
+    return ntp.sdiskusage(total, used, free, percent)
+
+
+def disk_partitions(all):
+    """Return disk partitions."""
+    rawlist = cext.disk_partitions(all)
+    return [ntp.sdiskpart(*x) for x in rawlist]
+
+
+# =====================================================================
+# --- CPU
+# =====================================================================
+
+
+def cpu_times():
+    """Return system CPU times as a named tuple."""
+    user, system, idle = cext.cpu_times()
+    # Internally, GetSystemTimes() is used, and it doesn't return
+    # interrupt and dpc times. cext.per_cpu_times() does, so we
+    # rely on it to get those only.
+    percpu_summed = ntp.scputimes(
+        *[sum(n) for n in zip(*cext.per_cpu_times())]
+    )
+    return ntp.scputimes(
+        user, system, idle, percpu_summed.interrupt, percpu_summed.dpc
+    )
+
+
+def per_cpu_times():
+    """Return system per-CPU times as a list of named tuples."""
+    ret = []
+    for user, system, idle, interrupt, dpc in cext.per_cpu_times():
+        item = ntp.scputimes(user, system, idle, interrupt, dpc)
+        ret.append(item)
+    return ret
+
+
+def cpu_count_logical():
+    """Return the number of logical CPUs in the system."""
+    return cext.cpu_count_logical()
+
+
+def cpu_count_cores():
+    """Return the number of CPU cores in the system."""
+    return cext.cpu_count_cores()
+
+
+def cpu_stats():
+    """Return CPU statistics."""
+    ctx_switches, interrupts, _dpcs, syscalls = cext.cpu_stats()
+    soft_interrupts = 0
+    return ntp.scpustats(ctx_switches, interrupts, soft_interrupts, syscalls)
+
+
+def cpu_freq():
+    """Return CPU frequency.
+    On Windows per-cpu frequency is not supported.
+    """
+    curr, max_ = cext.cpu_freq()
+    min_ = 0.0
+    return [ntp.scpufreq(float(curr), min_, float(max_))]
+
+
+_loadavg_initialized = False
+_lock = threading.Lock()
+
+
+def _getloadavg_impl():
+    # Drop to 2 decimal points which is what Linux does
+    raw_loads = cext.getloadavg()
+    return tuple(round(load, 2) for load in raw_loads)
+
+
+def getloadavg():
+    """Return the number of processes in the system run queue averaged
+    over the last 1, 5, and 15 minutes respectively as a tuple.
+    """
+    global _loadavg_initialized
+
+    if _loadavg_initialized:
+        return _getloadavg_impl()
+
+    with _lock:
+        if not _loadavg_initialized:
+            cext.init_loadavg_counter()
+            _loadavg_initialized = True
+
+    return _getloadavg_impl()
+
+
+# =====================================================================
+# --- network
+# =====================================================================
+
+
+def net_connections(kind, _pid=-1):
+    """Return socket connections.  If pid == -1 return system-wide
+    connections (as opposed to connections opened by one process only).
+    """
+    families, types = conn_tmap[kind]
+    rawlist = cext.net_connections(_pid, families, types)
+    ret = set()
+    for item in rawlist:
+        fd, fam, type, laddr, raddr, status, pid = item
+        nt = conn_to_ntuple(
+            fd,
+            fam,
+            type,
+            laddr,
+            raddr,
+            status,
+            TCP_STATUSES,
+            pid=pid if _pid == -1 else None,
+        )
+        ret.add(nt)
+    return list(ret)
+
+
+def net_if_stats():
+    """Get NIC stats (isup, duplex, speed, mtu)."""
+    ret = {}
+    rawdict = cext.net_if_stats()
+    for name, items in rawdict.items():
+        isup, duplex, speed, mtu = items
+        if hasattr(_common, 'NicDuplex'):
+            duplex = _common.NicDuplex(duplex)
+        ret[name] = ntp.snicstats(isup, duplex, speed, mtu, '')
+    return ret
+
+
+def net_io_counters():
+    """Return network I/O statistics for every network interface
+    installed on the system as a dict of raw tuples.
+    """
+    return cext.net_io_counters()
+
+
+def net_if_addrs():
+    """Return the addresses associated to each NIC."""
+    return cext.net_if_addrs()
+
+
+# =====================================================================
+# --- sensors
+# =====================================================================
+
+
+def sensors_battery():
+    """Return battery information."""
+    # For constants meaning see:
+    # https://msdn.microsoft.com/en-us/library/windows/desktop/
+    #     aa373232(v=vs.85).aspx
+    acline_status, flags, percent, secsleft = cext.sensors_battery()
+    power_plugged = acline_status == 1
+    no_battery = bool(flags & 128)
+    charging = bool(flags & 8)
+
+    if no_battery:
+        return None
+    if power_plugged or charging:
+        secsleft = _common.POWER_TIME_UNLIMITED
+    elif secsleft == -1:
+        secsleft = _common.POWER_TIME_UNKNOWN
+
+    return ntp.sbattery(percent, secsleft, power_plugged)
+
+
+# =====================================================================
+# --- other system functions
+# =====================================================================
+
+
+_last_btime = 0
+
+
+def boot_time():
+    """The system boot time expressed in seconds since the epoch. This
+    also includes the time spent during hybernate / suspend.
+    """
+    # This dirty hack is to adjust the precision of the returned
+    # value which may have a 1 second fluctuation, see:
+    # https://github.com/giampaolo/psutil/issues/1007
+    global _last_btime
+    ret = time.time() - cext.uptime()
+    if abs(ret - _last_btime) <= 1:
+        return _last_btime
+    else:
+        _last_btime = ret
+        return ret
+
+
+def users():
+    """Return currently connected users as a list of namedtuples."""
+    retlist = []
+    rawlist = cext.users()
+    for item in rawlist:
+        user, hostname, tstamp = item
+        nt = ntp.suser(user, None, hostname, tstamp, None)
+        retlist.append(nt)
+    return retlist
+
+
+# =====================================================================
+# --- Windows services
+# =====================================================================
+
+
+def win_service_iter():
+    """Yields a list of WindowsService instances."""
+    for name, display_name in cext.winservice_enumerate():
+        yield WindowsService(name, display_name)
+
+
+def win_service_get(name):
+    """Open a Windows service and return it as a WindowsService instance."""
+    service = WindowsService(name, None)
+    service._display_name = service._query_config()['display_name']
+    return service
+
+
+class WindowsService:  # noqa: PLW1641
+    """Represents an installed Windows service."""
+
+    def __init__(self, name, display_name):
+        self._name = name
+        self._display_name = display_name
+
+    def __str__(self):
+        details = f"(name={self._name!r}, display_name={self._display_name!r})"
+        return f"{self.__class__.__name__}{details}"
+
+    def __repr__(self):
+        return f"<{self.__str__()} at {id(self)}>"
+
+    def __eq__(self, other):
+        # Test for equality with another WindosService object based
+        # on name.
+        if not isinstance(other, WindowsService):
+            return NotImplemented
+        return self._name == other._name
+
+    def __ne__(self, other):
+        return not self == other
+
+    def _query_config(self):
+        with self._wrap_exceptions():
+            display_name, binpath, username, start_type = (
+                cext.winservice_query_config(self._name)
+            )
+        # XXX - update _self.display_name?
+        return dict(
+            display_name=display_name,
+            binpath=binpath,
+            username=username,
+            start_type=start_type,
+        )
+
+    def _query_status(self):
+        with self._wrap_exceptions():
+            status, pid = cext.winservice_query_status(self._name)
+        if pid == 0:
+            pid = None
+        return dict(status=status, pid=pid)
+
+    @contextlib.contextmanager
+    def _wrap_exceptions(self):
+        """Ctx manager which translates bare OSError and WindowsError
+        exceptions into NoSuchProcess and AccessDenied.
+        """
+        try:
+            yield
+        except OSError as err:
+            name = self._name
+            if is_permission_err(err):
+                msg = (
+                    f"service {name!r} is not querable (not enough privileges)"
+                )
+                raise AccessDenied(pid=None, name=name, msg=msg) from err
+            elif err.winerror in {
+                cext.ERROR_INVALID_NAME,
+                cext.ERROR_SERVICE_DOES_NOT_EXIST,
+            }:
+                msg = f"service {name!r} does not exist"
+                raise NoSuchProcess(pid=None, name=name, msg=msg) from err
+            else:
+                raise
+
+    # config query
+
+    def name(self):
+        """The service name. This string is how a service is referenced
+        and can be passed to win_service_get() to get a new
+        WindowsService instance.
+        """
+        return self._name
+
+    def display_name(self):
+        """The service display name. The value is cached when this class
+        is instantiated.
+        """
+        return self._display_name
+
+    def binpath(self):
+        """The fully qualified path to the service binary/exe file as
+        a string, including command line arguments.
+        """
+        return self._query_config()['binpath']
+
+    def username(self):
+        """The name of the user that owns this service."""
+        return self._query_config()['username']
+
+    def start_type(self):
+        """A string which can either be "automatic", "manual" or
+        "disabled".
+        """
+        return self._query_config()['start_type']
+
+    # status query
+
+    def pid(self):
+        """The process PID, if any, else None. This can be passed
+        to Process class to control the service's process.
+        """
+        return self._query_status()['pid']
+
+    def status(self):
+        """Service status as a string."""
+        return self._query_status()['status']
+
+    def description(self):
+        """Service long description."""
+        return cext.winservice_query_descr(self.name())
+
+    # utils
+
+    def as_dict(self):
+        """Utility method retrieving all the information above as a
+        dictionary.
+        """
+        d = self._query_config()
+        d.update(self._query_status())
+        d['name'] = self.name()
+        d['display_name'] = self.display_name()
+        d['description'] = self.description()
+        return d
+
+    # actions
+    # XXX: the necessary C bindings for start() and stop() are
+    # implemented but for now I prefer not to expose them.
+    # I may change my mind in the future. Reasons:
+    # - they require Administrator privileges
+    # - can't implement a timeout for stop() (unless by using a thread,
+    #   which sucks)
+    # - would require adding ServiceAlreadyStarted and
+    #   ServiceAlreadyStopped exceptions, adding two new APIs.
+    # - we might also want to have modify(), which would basically mean
+    #   rewriting win32serviceutil.ChangeServiceConfig, which involves a
+    #   lot of stuff (and API constants which would pollute the API), see:
+    #   http://pyxr.sourceforge.net/PyXR/c/python24/lib/site-packages/
+    #       win32/lib/win32serviceutil.py.html#0175
+    # - psutil is typically about "read only" monitoring stuff;
+    #   win_service_* APIs should only be used to retrieve a service and
+    #   check whether it's running
+
+    # def start(self, timeout=None):
+    #     with self._wrap_exceptions():
+    #         cext.winservice_start(self.name())
+    #         if timeout:
+    #             giveup_at = time.time() + timeout
+    #             while True:
+    #                 if self.status() == "running":
+    #                     return
+    #                 else:
+    #                     if time.time() > giveup_at:
+    #                         raise TimeoutExpired(timeout)
+    #                     else:
+    #                         time.sleep(.1)
+
+    # def stop(self):
+    #     # Note: timeout is not implemented because it's just not
+    #     # possible, see:
+    #     # http://stackoverflow.com/questions/11973228/
+    #     with self._wrap_exceptions():
+    #         return cext.winservice_stop(self.name())
+
+
+# =====================================================================
+# --- processes
+# =====================================================================
+
+
+pids = cext.pids
+pid_exists = cext.pid_exists
+ppid_map = cext.ppid_map  # used internally by Process.children()
+
+
+def is_permission_err(exc):
+    """Return True if this is a permission error."""
+    assert isinstance(exc, OSError), exc
+    return isinstance(exc, PermissionError) or exc.winerror in {
+        cext.ERROR_ACCESS_DENIED,
+        cext.ERROR_PRIVILEGE_NOT_HELD,
+    }
+
+
+def convert_oserror(exc, pid=None, name=None):
+    """Convert OSError into NoSuchProcess or AccessDenied."""
+    assert isinstance(exc, OSError), exc
+    if is_permission_err(exc):
+        return AccessDenied(pid=pid, name=name)
+    if isinstance(exc, ProcessLookupError):
+        return NoSuchProcess(pid=pid, name=name)
+    raise exc
+
+
+def wrap_exceptions(fun):
+    """Decorator which converts OSError into NoSuchProcess or AccessDenied."""
+
+    @functools.wraps(fun)
+    def wrapper(self, *args, **kwargs):
+        try:
+            return fun(self, *args, **kwargs)
+        except OSError as err:
+            raise convert_oserror(err, pid=self.pid, name=self._name) from err
+
+    return wrapper
+
+
+def retry_error_partial_copy(fun):
+    """Workaround for https://github.com/giampaolo/psutil/issues/875.
+    See: https://stackoverflow.com/questions/4457745#4457745.
+    """
+
+    @functools.wraps(fun)
+    def wrapper(self, *args, **kwargs):
+        delay = 0.0001
+        times = 33
+        for _ in range(times):  # retries for roughly 1 second
+            try:
+                return fun(self, *args, **kwargs)
+            except OSError as _:
+                err = _
+                if err.winerror == ERROR_PARTIAL_COPY:
+                    time.sleep(delay)
+                    delay = min(delay * 2, 0.04)
+                    continue
+                raise
+        msg = (
+            f"{fun} retried {times} times, converted to AccessDenied as it's "
+            f"still returning {err}"
+        )
+        raise AccessDenied(pid=self.pid, name=self._name, msg=msg)
+
+    return wrapper
+
+
+class Process:
+    """Wrapper class around underlying C implementation."""
+
+    __slots__ = ["_cache", "_name", "_ppid", "pid"]
+
+    def __init__(self, pid):
+        self.pid = pid
+        self._name = None
+        self._ppid = None
+
+    # --- oneshot() stuff
+
+    def oneshot_enter(self):
+        self._proc_info.cache_activate(self)
+        self.exe.cache_activate(self)
+
+    def oneshot_exit(self):
+        self._proc_info.cache_deactivate(self)
+        self.exe.cache_deactivate(self)
+
+    @memoize_when_activated
+    def _proc_info(self):
+        """Return multiple information about this process as a
+        raw tuple.
+        """
+        ret = cext.proc_info(self.pid)
+        assert len(ret) == len(pinfo_map)
+        return ret
+
+    def name(self):
+        """Return process name, which on Windows is always the final
+        part of the executable.
+        """
+        # This is how PIDs 0 and 4 are always represented in taskmgr
+        # and process-hacker.
+        if self.pid == 0:
+            return "System Idle Process"
+        if self.pid == 4:
+            return "System"
+        return os.path.basename(self.exe())
+
+    @wrap_exceptions
+    @memoize_when_activated
+    def exe(self):
+        if PYPY:
+            try:
+                exe = cext.proc_exe(self.pid)
+            except OSError as err:
+                # 24 = ERROR_TOO_MANY_OPEN_FILES. Not sure why this happens
+                # (perhaps PyPy's JIT delaying garbage collection of files?).
+                if err.errno == 24:
+                    debug(f"{err!r} translated into AccessDenied")
+                    raise AccessDenied(self.pid, self._name) from err
+                raise
+        else:
+            exe = cext.proc_exe(self.pid)
+        if exe.startswith('\\'):
+            return convert_dos_path(exe)
+        return exe  # May be "Registry", "MemCompression", ...
+
+    @wrap_exceptions
+    @retry_error_partial_copy
+    def cmdline(self):
+        if cext.WINVER >= cext.WINDOWS_8_1:
+            # PEB method detects cmdline changes but requires more
+            # privileges: https://github.com/giampaolo/psutil/pull/1398
+            try:
+                return cext.proc_cmdline(self.pid, use_peb=True)
+            except OSError as err:
+                if is_permission_err(err):
+                    return cext.proc_cmdline(self.pid, use_peb=False)
+                else:
+                    raise
+        else:
+            return cext.proc_cmdline(self.pid, use_peb=True)
+
+    @wrap_exceptions
+    @retry_error_partial_copy
+    def environ(self):
+        s = cext.proc_environ(self.pid)
+        return parse_environ_block(s)
+
+    def ppid(self):
+        try:
+            return ppid_map()[self.pid]
+        except KeyError:
+            raise NoSuchProcess(self.pid, self._name) from None
+
+    def _get_raw_meminfo(self):
+        try:
+            return cext.proc_memory_info(self.pid)
+        except OSError as err:
+            if is_permission_err(err):
+                # TODO: the C ext can probably be refactored in order
+                # to get this from cext.proc_info()
+                debug("attempting memory_info() fallback (slower)")
+                info = self._proc_info()
+                return (
+                    info[pinfo_map['num_page_faults']],
+                    info[pinfo_map['peak_wset']],
+                    info[pinfo_map['wset']],
+                    info[pinfo_map['peak_paged_pool']],
+                    info[pinfo_map['paged_pool']],
+                    info[pinfo_map['peak_non_paged_pool']],
+                    info[pinfo_map['non_paged_pool']],
+                    info[pinfo_map['pagefile']],
+                    info[pinfo_map['peak_pagefile']],
+                    info[pinfo_map['mem_private']],
+                )
+            raise
+
+    @wrap_exceptions
+    def memory_info(self):
+        # on Windows RSS == WorkingSetSize and VSM == PagefileUsage.
+        # Underlying C function returns fields of PROCESS_MEMORY_COUNTERS
+        # struct.
+        t = self._get_raw_meminfo()
+        rss = t[2]  # wset
+        vms = t[7]  # pagefile
+        return ntp.pmem(*(rss, vms) + t)
+
+    @wrap_exceptions
+    def memory_full_info(self):
+        basic_mem = self.memory_info()
+        uss = cext.proc_memory_uss(self.pid)
+        uss *= getpagesize()
+        return ntp.pfullmem(*basic_mem + (uss,))
+
+    def memory_maps(self):
+        try:
+            raw = cext.proc_memory_maps(self.pid)
+        except OSError as err:
+            # XXX - can't use wrap_exceptions decorator as we're
+            # returning a generator; probably needs refactoring.
+            raise convert_oserror(err, self.pid, self._name) from err
+        else:
+            for addr, perm, path, rss in raw:
+                path = convert_dos_path(path)
+                addr = hex(addr)
+                yield (addr, perm, path, rss)
+
+    @wrap_exceptions
+    def kill(self):
+        return cext.proc_kill(self.pid)
+
+    @wrap_exceptions
+    def send_signal(self, sig):
+        if sig == signal.SIGTERM:
+            cext.proc_kill(self.pid)
+        elif sig in {signal.CTRL_C_EVENT, signal.CTRL_BREAK_EVENT}:
+            os.kill(self.pid, sig)
+        else:
+            msg = (
+                "only SIGTERM, CTRL_C_EVENT and CTRL_BREAK_EVENT signals "
+                "are supported on Windows"
+            )
+            raise ValueError(msg)
+
+    @wrap_exceptions
+    def wait(self, timeout=None):
+        if timeout is None:
+            cext_timeout = cext.INFINITE
+        else:
+            # WaitForSingleObject() expects time in milliseconds.
+            cext_timeout = int(timeout * 1000)
+
+        timer = getattr(time, 'monotonic', time.time)
+        stop_at = timer() + timeout if timeout is not None else None
+
+        try:
+            # Exit code is supposed to come from GetExitCodeProcess().
+            # May also be None if OpenProcess() failed with
+            # ERROR_INVALID_PARAMETER, meaning PID is already gone.
+            exit_code = cext.proc_wait(self.pid, cext_timeout)
+        except cext.TimeoutExpired as err:
+            # WaitForSingleObject() returned WAIT_TIMEOUT. Just raise.
+            raise TimeoutExpired(timeout, self.pid, self._name) from err
+        except cext.TimeoutAbandoned:
+            # WaitForSingleObject() returned WAIT_ABANDONED, see:
+            # https://github.com/giampaolo/psutil/issues/1224
+            # We'll just rely on the internal polling and return None
+            # when the PID disappears. Subprocess module does the same
+            # (return None):
+            # https://github.com/python/cpython/blob/
+            #     be50a7b627d0aa37e08fa8e2d5568891f19903ce/
+            #     Lib/subprocess.py#L1193-L1194
+            exit_code = None
+
+        # At this point WaitForSingleObject() returned WAIT_OBJECT_0,
+        # meaning the process is gone. Stupidly there are cases where
+        # its PID may still stick around so we do a further internal
+        # polling.
+        delay = 0.0001
+        while True:
+            if not pid_exists(self.pid):
+                return exit_code
+            if stop_at and timer() >= stop_at:
+                raise TimeoutExpired(timeout, pid=self.pid, name=self._name)
+            time.sleep(delay)
+            delay = min(delay * 2, 0.04)  # incremental delay
+
+    @wrap_exceptions
+    def username(self):
+        if self.pid in {0, 4}:
+            return 'NT AUTHORITY\\SYSTEM'
+        domain, user = cext.proc_username(self.pid)
+        return f"{domain}\\{user}"
+
+    @wrap_exceptions
+    def create_time(self, fast_only=False):
+        # Note: proc_times() not put under oneshot() 'cause create_time()
+        # is already cached by the main Process class.
+        try:
+            _user, _system, created = cext.proc_times(self.pid)
+            return created
+        except OSError as err:
+            if is_permission_err(err):
+                if fast_only:
+                    raise
+                debug("attempting create_time() fallback (slower)")
+                return self._proc_info()[pinfo_map['create_time']]
+            raise
+
+    @wrap_exceptions
+    def num_threads(self):
+        return self._proc_info()[pinfo_map['num_threads']]
+
+    @wrap_exceptions
+    def threads(self):
+        rawlist = cext.proc_threads(self.pid)
+        retlist = []
+        for thread_id, utime, stime in rawlist:
+            ntuple = ntp.pthread(thread_id, utime, stime)
+            retlist.append(ntuple)
+        return retlist
+
+    @wrap_exceptions
+    def cpu_times(self):
+        try:
+            user, system, _created = cext.proc_times(self.pid)
+        except OSError as err:
+            if not is_permission_err(err):
+                raise
+            debug("attempting cpu_times() fallback (slower)")
+            info = self._proc_info()
+            user = info[pinfo_map['user_time']]
+            system = info[pinfo_map['kernel_time']]
+        # Children user/system times are not retrievable (set to 0).
+        return ntp.pcputimes(user, system, 0.0, 0.0)
+
+    @wrap_exceptions
+    def suspend(self):
+        cext.proc_suspend_or_resume(self.pid, True)
+
+    @wrap_exceptions
+    def resume(self):
+        cext.proc_suspend_or_resume(self.pid, False)
+
+    @wrap_exceptions
+    @retry_error_partial_copy
+    def cwd(self):
+        if self.pid in {0, 4}:
+            raise AccessDenied(self.pid, self._name)
+        # return a normalized pathname since the native C function appends
+        # "\\" at the and of the path
+        path = cext.proc_cwd(self.pid)
+        return os.path.normpath(path)
+
+    @wrap_exceptions
+    def open_files(self):
+        if self.pid in {0, 4}:
+            return []
+        ret = set()
+        # Filenames come in in native format like:
+        # "\Device\HarddiskVolume1\Windows\systemew\file.txt"
+        # Convert the first part in the corresponding drive letter
+        # (e.g. "C:\") by using Windows's QueryDosDevice()
+        raw_file_names = cext.proc_open_files(self.pid)
+        for file in raw_file_names:
+            file = convert_dos_path(file)
+            if isfile_strict(file):
+                ntuple = ntp.popenfile(file, -1)
+                ret.add(ntuple)
+        return list(ret)
+
+    @wrap_exceptions
+    def net_connections(self, kind='inet'):
+        return net_connections(kind, _pid=self.pid)
+
+    @wrap_exceptions
+    def nice_get(self):
+        value = cext.proc_priority_get(self.pid)
+        value = Priority(value)
+        return value
+
+    @wrap_exceptions
+    def nice_set(self, value):
+        return cext.proc_priority_set(self.pid, value)
+
+    @wrap_exceptions
+    def ionice_get(self):
+        ret = cext.proc_io_priority_get(self.pid)
+        ret = IOPriority(ret)
+        return ret
+
+    @wrap_exceptions
+    def ionice_set(self, ioclass, value):
+        if value:
+            msg = "value argument not accepted on Windows"
+            raise TypeError(msg)
+        if ioclass not in {
+            IOPriority.IOPRIO_VERYLOW,
+            IOPriority.IOPRIO_LOW,
+            IOPriority.IOPRIO_NORMAL,
+            IOPriority.IOPRIO_HIGH,
+        }:
+            msg = f"{ioclass} is not a valid priority"
+            raise ValueError(msg)
+        cext.proc_io_priority_set(self.pid, ioclass)
+
+    @wrap_exceptions
+    def io_counters(self):
+        try:
+            ret = cext.proc_io_counters(self.pid)
+        except OSError as err:
+            if not is_permission_err(err):
+                raise
+            debug("attempting io_counters() fallback (slower)")
+            info = self._proc_info()
+            ret = (
+                info[pinfo_map['io_rcount']],
+                info[pinfo_map['io_wcount']],
+                info[pinfo_map['io_rbytes']],
+                info[pinfo_map['io_wbytes']],
+                info[pinfo_map['io_count_others']],
+                info[pinfo_map['io_bytes_others']],
+            )
+        return ntp.pio(*ret)
+
+    @wrap_exceptions
+    def status(self):
+        suspended = cext.proc_is_suspended(self.pid)
+        if suspended:
+            return _common.STATUS_STOPPED
+        else:
+            return _common.STATUS_RUNNING
+
+    @wrap_exceptions
+    def cpu_affinity_get(self):
+        def from_bitmask(x):
+            return [i for i in range(64) if (1 << i) & x]
+
+        bitmask = cext.proc_cpu_affinity_get(self.pid)
+        return from_bitmask(bitmask)
+
+    @wrap_exceptions
+    def cpu_affinity_set(self, value):
+        def to_bitmask(ls):
+            if not ls:
+                msg = f"invalid argument {ls!r}"
+                raise ValueError(msg)
+            out = 0
+            for b in ls:
+                out |= 2**b
+            return out
+
+        # SetProcessAffinityMask() states that ERROR_INVALID_PARAMETER
+        # is returned for an invalid CPU but this seems not to be true,
+        # therefore we check CPUs validy beforehand.
+        allcpus = list(range(len(per_cpu_times())))
+        for cpu in value:
+            if cpu not in allcpus:
+                if not isinstance(cpu, int):
+                    msg = f"invalid CPU {cpu!r}; an integer is required"
+                    raise TypeError(msg)
+                msg = f"invalid CPU {cpu!r}"
+                raise ValueError(msg)
+
+        bitmask = to_bitmask(value)
+        cext.proc_cpu_affinity_set(self.pid, bitmask)
+
+    @wrap_exceptions
+    def num_handles(self):
+        try:
+            return cext.proc_num_handles(self.pid)
+        except OSError as err:
+            if is_permission_err(err):
+                debug("attempting num_handles() fallback (slower)")
+                return self._proc_info()[pinfo_map['num_handles']]
+            raise
+
+    @wrap_exceptions
+    def num_ctx_switches(self):
+        ctx_switches = self._proc_info()[pinfo_map['ctx_switches']]
+        # only voluntary ctx switches are supported
+        return ntp.pctxsw(ctx_switches, 0)

Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/pytablewriter-1.2.1.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip